Implement YAML Based Shuffle Test (#815)

beacon-chain/chaintest/BUILD.bazel

@@ -22,10 +22,7 @@ go_binary(
 go_test(
     name = "go_default_test",
     srcs = ["yaml_test.go"],
-    data = glob(["sampletests/**"]),
+    data = glob(["tests/**"]),
     embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/chaintest/backend:go_default_library",
-        "@com_github_go_yaml_yaml//:go_default_library",
-    ],
+    deps = ["//beacon-chain/chaintest/backend:go_default_library"],
 )

beacon-chain/chaintest/backend/BUILD.bazel

@@ -3,18 +3,21 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "chain_test_format.go",
         "setup_db.go",
+        "shuffle_test_format.go",
         "simulated_backend.go",
-        "test_format.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend",
     visibility = ["//beacon-chain/chaintest:__subpackages__"],
     deps = [
         "//beacon-chain/blockchain:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/utils:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/chaintest/backend/shuffle_test_format.go

@@ -0,0 +1,18 @@
+package backend
+
+// ShuffleTest --
+type ShuffleTest struct {
+	Title     string             `yaml:"title"`
+	Summary   string             `yaml:"summary"`
+	TestSuite string             `yaml:"test_suite"`
+	Fork      string             `yaml:"fork"`
+	Version   string             `yaml:"version"`
+	TestCases []*ShuffleTestCase `yaml:"test_cases"`
+}
+
+// ShuffleTestCase --
+type ShuffleTestCase struct {
+	Input  []uint32 `yaml:"input,flow"`
+	Output []uint32 `yaml:"output,flow"`
+	Seed   string
+}

beacon-chain/chaintest/backend/simulated_backend.go

@@ -6,9 +6,13 @@ package backend
 import (
 	"context"
 	"fmt"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/common"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/utils"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -80,3 +84,19 @@ func (sb *SimulatedBackend) RunChainTest(testCase *ChainTestCase) error {
 	// chain's head is the expected result from the test case.
 	return nil
 }
+
+// RunShuffleTest uses validator set specified from a YAML file, runs the validator shuffle
+// algorithm, then compare the output with the expected output from the YAML file.
+func (sb *SimulatedBackend) RunShuffleTest(testCase *ShuffleTestCase) error {
+	defer teardownDB(sb.db)
+
+	seed := common.BytesToHash([]byte(testCase.Seed))
+	output, err := utils.ShuffleIndices(seed, testCase.Input)
+	if err != nil {
+		return err
+	}
+	if !reflect.DeepEqual(output, testCase.Output) {
+		return fmt.Errorf("shuffle result error: expected %v, actual %v", testCase.Output, output)
+	}
+	return nil
+}

beacon-chain/chaintest/backend/simulated_backend_test.go

@@ -19,3 +19,18 @@ func TestRunChainTest(t *testing.T) {
 		t.Errorf("Could not run chaintest: %v", err)
 	}
 }
+
+func TestRunShuffleTest(t *testing.T) {
+	sb, err := NewSimulatedBackend()
+	if err != nil {
+		t.Fatal(err)
+	}
+	testCase := &ShuffleTestCase{
+		Input:  []uint32{1, 2, 3, 4, 5},
+		Output: []uint32{2, 4, 1, 3, 5},
+		Seed:   "abcde",
+	}
+	if err := sb.RunShuffleTest(testCase); err != nil {
+		t.Errorf("Could not run chaintest: %v", err)
+	}
+}

beacon-chain/chaintest/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"path"
 	"time"
@@ -12,6 +13,76 @@ import (
 	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 )
 
+func readTestsFromYaml(yamlDir string) ([]interface{}, error) {
+	const chainTestsFolderName = "chain-tests"
+	const shuffleTestsFolderName = "shuffle-tests"
+
+	var tests []interface{}
+
+	dirs, err := ioutil.ReadDir(yamlDir)
+	if err != nil {
+		return nil, fmt.Errorf("could not read yaml tests directory: %v", err)
+	}
+	for _, dir := range dirs {
+		files, err := ioutil.ReadDir(path.Join(yamlDir, dir.Name()))
+		if err != nil {
+			return nil, fmt.Errorf("could not read yaml tests directory: %v", err)
+		}
+		for _, file := range files {
+			filePath := path.Join(yamlDir, dir.Name(), file.Name())
+			data, err := ioutil.ReadFile(filePath) // #nosec
+			if err != nil {
+				return nil, fmt.Errorf("could not read yaml file: %v", err)
+			}
+			switch dir.Name() {
+			case chainTestsFolderName:
+				decoded := &backend.ChainTest{}
+				if err := yaml.Unmarshal(data, decoded); err != nil {
+					return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
+				}
+				tests = append(tests, decoded)
+			case shuffleTestsFolderName:
+				decoded := &backend.ShuffleTest{}
+				if err := yaml.Unmarshal(data, decoded); err != nil {
+					return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
+				}
+				tests = append(tests, decoded)
+			}
+		}
+	}
+	return tests, nil
+}
+
+func runTests(tests []interface{}, sb *backend.SimulatedBackend) error {
+	for _, tt := range tests {
+		switch typedTest := tt.(type) {
+		case *backend.ChainTest:
+			log.Infof("Title: %v", typedTest.Title)
+			log.Infof("Summary: %v", typedTest.Summary)
+			log.Infof("Test Suite: %v", typedTest.TestSuite)
+			for _, testCase := range typedTest.TestCases {
+				if err := sb.RunChainTest(testCase); err != nil {
+					return fmt.Errorf("chain test failed: %v", err)
+				}
+			}
+		case *backend.ShuffleTest:
+			log.Infof("Title: %v", typedTest.Title)
+			log.Infof("Summary: %v", typedTest.Summary)
+			log.Infof("Test Suite: %v", typedTest.TestSuite)
+			log.Infof("Fork: %v", typedTest.Fork)
+			log.Infof("Version: %v", typedTest.Version)
+			for _, testCase := range typedTest.TestCases {
+				if err := sb.RunShuffleTest(testCase); err != nil {
+					return fmt.Errorf("chain test failed: %v", err)
+				}
+			}
+		default:
+			return fmt.Errorf("receive unknown test type: %T", typedTest)
+		}
+	}
+	return nil
+}
+
 func main() {
 	var yamlDir = flag.String("tests-dir", "", "path to directory of yaml tests")
 	flag.Parse()
@@ -21,23 +92,9 @@ func main() {
 	customFormatter.FullTimestamp = true
 	log.SetFormatter(customFormatter)
 
-	var chainTests []*backend.ChainTest
-
-	files, err := ioutil.ReadDir(*yamlDir)
+	tests, err := readTestsFromYaml(*yamlDir)
 	if err != nil {
-		log.Fatalf("Could not read yaml tests directory: %v", err)
-	}
-
-	for _, file := range files {
-		data, err := ioutil.ReadFile(path.Join(*yamlDir, file.Name()))
-		if err != nil {
-			log.Fatalf("Could not read yaml file: %v", err)
-		}
-		decoded := &backend.ChainTest{}
-		if err := yaml.Unmarshal(data, decoded); err != nil {
-			log.Fatalf("Could not unmarshal YAML file into test struct: %v", err)
-		}
-		chainTests = append(chainTests, decoded)
+		log.Fatalf("Fail to load tests from yaml: %v", err)
 	}
 
 	sb, err := backend.NewSimulatedBackend()
@@ -45,18 +102,12 @@ func main() {
 		log.Fatalf("Could not create backend: %v", err)
 	}
 
-	log.Info("----Running Chain Tests----")
+	log.Info("----Running Tests----")
 	startTime := time.Now()
 
-	for _, tt := range chainTests {
-		log.Infof("Title: %v", tt.Title)
-		log.Infof("Summary: %v", tt.Summary)
-		log.Infof("Test Suite: %v", tt.TestSuite)
-		for _, testCase := range tt.TestCases {
-			if err := sb.RunChainTest(testCase); err != nil {
-				log.Fatalf("Could not run chain test: %v", err)
-			}
-		}
+	err = runTests(tests, sb)
+	if err != nil {
+		log.Fatalf("Test failed %v", err)
 	}
 
 	endTime := time.Now()

beacon-chain/chaintest/tests/shuffle-tests/shuffle.yaml

@@ -0,0 +1,44 @@
+# Credits to Danny Ryan (Ethereum Foundation)
+---
+
+title: Shuffling Algorithm Tests
+summary: Test vectors for shuffling a list based upon a seed using `shuffle`
+test_suite: shuffle
+fork: tchaikovsky
+version: 1.0
+
+test_cases:
+- config:
+    validator_count: 100
+    cycle_length: 8
+    shard_count: 32
+    min_committee_size: 8
+- input: []
+  output: []
+  seed: !!binary ""
+- name: boring_list
+  description: List with a single element, 0
+  input: [0]
+  output: [0]
+  seed: !!binary ""
+- input: [255]
+  output: [255]
+  seed: !!binary ""
+- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [1, 1, 2, 6, 4, 4, 5, 6, 2, 6]
+  seed: !!binary ""
+- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+  output: [3, 8, 11, 7, 9, 5, 13, 4, 10, 6, 1, 2, 12]
+  seed: !!binary ""
+- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5]
+  output: [65, 6, 6, 6, 2, 4, 1, 2, 1, 5]
+  seed: !!binary |
+    JlAYJ5H2j8g7PLiPHZI/rTS1uAvKiieOrifPN6Moso0=
+- input: [35, 6, 2, 6, 1, 4, 6, 2, 1, 5, 7, 98, 3, 2, 11]
+  output: [98, 2, 3, 6, 1, 6, 7, 2, 2, 35, 5, 6, 4, 1, 11]
+  seed: !!binary |
+    VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=
+- input: [35, 6, 2, 6, 1, 4, 6, 2, 1, 5, 7, 98, 3, 2, 11]
+  output: [6, 6, 1, 98, 6, 2, 5, 7, 2, 11, 35, 1, 2, 4, 3]
+  seed: !!binary |
+    rDTbe23J4UA0yLIurjbJqk49VcavAC0Nysas+l5MlwvLc0B/JqQ=

beacon-chain/chaintest/yaml_test.go

@@ -1,44 +1,38 @@
 package main
 
 import (
-	"io/ioutil"
-	"path"
 	"testing"
 
-	"github.com/go-yaml/yaml"
 	"github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend"
 )
 
-func TestYamls(t *testing.T) {
+// TestReadTestsFromYaml tests constructing test cases from yaml file.
+func TestReadTestsFromYaml(t *testing.T) {
+	if _, err := readTestsFromYaml(string("./tests")); err != nil {
+		t.Fatalf("Failed to read yaml files: %v", err)
+	}
+}
+
+// TestReadTestsFromYaml tests the running of provided tests structs.
+func TestRunTests(t *testing.T) {
 	sb, err := backend.NewSimulatedBackend()
 	if err != nil {
 		t.Fatalf("Could not create backend: %v", err)
 	}
 
-	var chainTests []*backend.ChainTest
-
-	files, err := ioutil.ReadDir("./sampletests")
-	if err != nil {
-		t.Fatalf("Could not read yaml tests directory: %v", err)
+	chainTestCase := &backend.ChainTestCase{
+		Config: &backend.ChainTestConfig{
+			ShardCount:       3,
+			CycleLength:      10,
+			MinCommitteeSize: 3,
+			ValidatorCount:   100,
+		},
 	}
+	shuffleTestCase := &backend.ShuffleTestCase{}
 
-	for _, file := range files {
-		data, err := ioutil.ReadFile(path.Join("./sampletests", file.Name()))
-		if err != nil {
-			t.Fatalf("Could not read yaml file: %v", err)
-		}
-		decoded := &backend.ChainTest{}
-		if err := yaml.Unmarshal(data, decoded); err != nil {
-			t.Fatalf("Could not unmarshal YAML file into test struct: %v", err)
-		}
-		chainTests = append(chainTests, decoded)
-	}
-
-	for _, tt := range chainTests {
-		for _, testCase := range tt.TestCases {
-			if err := sb.RunChainTest(testCase); err != nil {
-				t.Errorf("Beacon Chain test failed: %v", err)
-			}
-		}
+	chainTest := &backend.ChainTest{TestCases: []*backend.ChainTestCase{chainTestCase}}
+	shuffleTest := &backend.ShuffleTest{TestCases: []*backend.ShuffleTestCase{shuffleTestCase}}
+	if err = runTests([]interface{}{chainTest, shuffleTest}, sb); err != nil {
+		t.Fatalf("Failed to run test cases: %v", err)
 	}
 }