From bd400707c24ef97830fa89b87cb1f0e1ddb43ebc Mon Sep 17 00:00:00 2001 From: Jie Hou Date: Sun, 18 Nov 2018 22:54:04 -0800 Subject: [PATCH] Implement YAML Based Shuffle Test (#815) --- beacon-chain/chaintest/BUILD.bazel | 7 +- beacon-chain/chaintest/backend/BUILD.bazel | 5 +- .../{test_format.go => chain_test_format.go} | 0 .../chaintest/backend/shuffle_test_format.go | 18 +++ .../chaintest/backend/simulated_backend.go | 20 ++++ .../backend/simulated_backend_test.go | 15 +++ beacon-chain/chaintest/main.go | 103 +++++++++++++----- .../chain-tests}/basic_fork_choice.yaml | 0 .../tests/shuffle-tests/shuffle.yaml | 44 ++++++++ beacon-chain/chaintest/yaml_test.go | 48 ++++---- 10 files changed, 201 insertions(+), 59 deletions(-) rename beacon-chain/chaintest/backend/{test_format.go => chain_test_format.go} (100%) create mode 100644 beacon-chain/chaintest/backend/shuffle_test_format.go rename beacon-chain/chaintest/{sampletests => tests/chain-tests}/basic_fork_choice.yaml (100%) create mode 100644 beacon-chain/chaintest/tests/shuffle-tests/shuffle.yaml diff --git a/beacon-chain/chaintest/BUILD.bazel b/beacon-chain/chaintest/BUILD.bazel index a3fc7f49e..3ea1902f5 100644 --- a/beacon-chain/chaintest/BUILD.bazel +++ b/beacon-chain/chaintest/BUILD.bazel @@ -22,10 +22,7 @@ go_binary( go_test( name = "go_default_test", srcs = ["yaml_test.go"], - data = glob(["sampletests/**"]), + data = glob(["tests/**"]), embed = [":go_default_library"], - deps = [ - "//beacon-chain/chaintest/backend:go_default_library", - "@com_github_go_yaml_yaml//:go_default_library", - ], + deps = ["//beacon-chain/chaintest/backend:go_default_library"], ) diff --git a/beacon-chain/chaintest/backend/BUILD.bazel b/beacon-chain/chaintest/backend/BUILD.bazel index 55ba37bd3..bf634293f 100644 --- a/beacon-chain/chaintest/backend/BUILD.bazel +++ b/beacon-chain/chaintest/backend/BUILD.bazel @@ -3,18 +3,21 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "chain_test_format.go", "setup_db.go", + "shuffle_test_format.go", "simulated_backend.go", - "test_format.go", ], importpath = "github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend", visibility = ["//beacon-chain/chaintest:__subpackages__"], deps = [ "//beacon-chain/blockchain:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/utils:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/hashutil:go_default_library", "//shared/params:go_default_library", + "@com_github_ethereum_go_ethereum//common:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", ], ) diff --git a/beacon-chain/chaintest/backend/test_format.go b/beacon-chain/chaintest/backend/chain_test_format.go similarity index 100% rename from beacon-chain/chaintest/backend/test_format.go rename to beacon-chain/chaintest/backend/chain_test_format.go diff --git a/beacon-chain/chaintest/backend/shuffle_test_format.go b/beacon-chain/chaintest/backend/shuffle_test_format.go new file mode 100644 index 000000000..a2e55498e --- /dev/null +++ b/beacon-chain/chaintest/backend/shuffle_test_format.go @@ -0,0 +1,18 @@ +package backend + +// ShuffleTest -- +type ShuffleTest struct { + Title string `yaml:"title"` + Summary string `yaml:"summary"` + TestSuite string `yaml:"test_suite"` + Fork string `yaml:"fork"` + Version string `yaml:"version"` + TestCases []*ShuffleTestCase `yaml:"test_cases"` +} + +// ShuffleTestCase -- +type ShuffleTestCase struct { + Input []uint32 `yaml:"input,flow"` + Output []uint32 `yaml:"output,flow"` + Seed string +} diff --git a/beacon-chain/chaintest/backend/simulated_backend.go b/beacon-chain/chaintest/backend/simulated_backend.go index 636770885..36a27468b 100644 --- a/beacon-chain/chaintest/backend/simulated_backend.go +++ b/beacon-chain/chaintest/backend/simulated_backend.go @@ -6,9 +6,13 @@ package backend import ( "context" "fmt" + "reflect" + + "github.com/ethereum/go-ethereum/common" "github.com/prysmaticlabs/prysm/beacon-chain/blockchain" "github.com/prysmaticlabs/prysm/beacon-chain/db" + "github.com/prysmaticlabs/prysm/beacon-chain/utils" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" @@ -80,3 +84,19 @@ func (sb *SimulatedBackend) RunChainTest(testCase *ChainTestCase) error { // chain's head is the expected result from the test case. return nil } + +// RunShuffleTest uses validator set specified from a YAML file, runs the validator shuffle +// algorithm, then compare the output with the expected output from the YAML file. +func (sb *SimulatedBackend) RunShuffleTest(testCase *ShuffleTestCase) error { + defer teardownDB(sb.db) + + seed := common.BytesToHash([]byte(testCase.Seed)) + output, err := utils.ShuffleIndices(seed, testCase.Input) + if err != nil { + return err + } + if !reflect.DeepEqual(output, testCase.Output) { + return fmt.Errorf("shuffle result error: expected %v, actual %v", testCase.Output, output) + } + return nil +} diff --git a/beacon-chain/chaintest/backend/simulated_backend_test.go b/beacon-chain/chaintest/backend/simulated_backend_test.go index ad6d4904b..955cd6a76 100644 --- a/beacon-chain/chaintest/backend/simulated_backend_test.go +++ b/beacon-chain/chaintest/backend/simulated_backend_test.go @@ -19,3 +19,18 @@ func TestRunChainTest(t *testing.T) { t.Errorf("Could not run chaintest: %v", err) } } + +func TestRunShuffleTest(t *testing.T) { + sb, err := NewSimulatedBackend() + if err != nil { + t.Fatal(err) + } + testCase := &ShuffleTestCase{ + Input: []uint32{1, 2, 3, 4, 5}, + Output: []uint32{2, 4, 1, 3, 5}, + Seed: "abcde", + } + if err := sb.RunShuffleTest(testCase); err != nil { + t.Errorf("Could not run chaintest: %v", err) + } +} diff --git a/beacon-chain/chaintest/main.go b/beacon-chain/chaintest/main.go index 62f646e5a..4fff59f13 100644 --- a/beacon-chain/chaintest/main.go +++ b/beacon-chain/chaintest/main.go @@ -2,6 +2,7 @@ package main import ( "flag" + "fmt" "io/ioutil" "path" "time" @@ -12,6 +13,76 @@ import ( prefixed "github.com/x-cray/logrus-prefixed-formatter" ) +func readTestsFromYaml(yamlDir string) ([]interface{}, error) { + const chainTestsFolderName = "chain-tests" + const shuffleTestsFolderName = "shuffle-tests" + + var tests []interface{} + + dirs, err := ioutil.ReadDir(yamlDir) + if err != nil { + return nil, fmt.Errorf("could not read yaml tests directory: %v", err) + } + for _, dir := range dirs { + files, err := ioutil.ReadDir(path.Join(yamlDir, dir.Name())) + if err != nil { + return nil, fmt.Errorf("could not read yaml tests directory: %v", err) + } + for _, file := range files { + filePath := path.Join(yamlDir, dir.Name(), file.Name()) + data, err := ioutil.ReadFile(filePath) // #nosec + if err != nil { + return nil, fmt.Errorf("could not read yaml file: %v", err) + } + switch dir.Name() { + case chainTestsFolderName: + decoded := &backend.ChainTest{} + if err := yaml.Unmarshal(data, decoded); err != nil { + return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err) + } + tests = append(tests, decoded) + case shuffleTestsFolderName: + decoded := &backend.ShuffleTest{} + if err := yaml.Unmarshal(data, decoded); err != nil { + return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err) + } + tests = append(tests, decoded) + } + } + } + return tests, nil +} + +func runTests(tests []interface{}, sb *backend.SimulatedBackend) error { + for _, tt := range tests { + switch typedTest := tt.(type) { + case *backend.ChainTest: + log.Infof("Title: %v", typedTest.Title) + log.Infof("Summary: %v", typedTest.Summary) + log.Infof("Test Suite: %v", typedTest.TestSuite) + for _, testCase := range typedTest.TestCases { + if err := sb.RunChainTest(testCase); err != nil { + return fmt.Errorf("chain test failed: %v", err) + } + } + case *backend.ShuffleTest: + log.Infof("Title: %v", typedTest.Title) + log.Infof("Summary: %v", typedTest.Summary) + log.Infof("Test Suite: %v", typedTest.TestSuite) + log.Infof("Fork: %v", typedTest.Fork) + log.Infof("Version: %v", typedTest.Version) + for _, testCase := range typedTest.TestCases { + if err := sb.RunShuffleTest(testCase); err != nil { + return fmt.Errorf("chain test failed: %v", err) + } + } + default: + return fmt.Errorf("receive unknown test type: %T", typedTest) + } + } + return nil +} + func main() { var yamlDir = flag.String("tests-dir", "", "path to directory of yaml tests") flag.Parse() @@ -21,23 +92,9 @@ func main() { customFormatter.FullTimestamp = true log.SetFormatter(customFormatter) - var chainTests []*backend.ChainTest - - files, err := ioutil.ReadDir(*yamlDir) + tests, err := readTestsFromYaml(*yamlDir) if err != nil { - log.Fatalf("Could not read yaml tests directory: %v", err) - } - - for _, file := range files { - data, err := ioutil.ReadFile(path.Join(*yamlDir, file.Name())) - if err != nil { - log.Fatalf("Could not read yaml file: %v", err) - } - decoded := &backend.ChainTest{} - if err := yaml.Unmarshal(data, decoded); err != nil { - log.Fatalf("Could not unmarshal YAML file into test struct: %v", err) - } - chainTests = append(chainTests, decoded) + log.Fatalf("Fail to load tests from yaml: %v", err) } sb, err := backend.NewSimulatedBackend() @@ -45,18 +102,12 @@ func main() { log.Fatalf("Could not create backend: %v", err) } - log.Info("----Running Chain Tests----") + log.Info("----Running Tests----") startTime := time.Now() - for _, tt := range chainTests { - log.Infof("Title: %v", tt.Title) - log.Infof("Summary: %v", tt.Summary) - log.Infof("Test Suite: %v", tt.TestSuite) - for _, testCase := range tt.TestCases { - if err := sb.RunChainTest(testCase); err != nil { - log.Fatalf("Could not run chain test: %v", err) - } - } + err = runTests(tests, sb) + if err != nil { + log.Fatalf("Test failed %v", err) } endTime := time.Now() diff --git a/beacon-chain/chaintest/sampletests/basic_fork_choice.yaml b/beacon-chain/chaintest/tests/chain-tests/basic_fork_choice.yaml similarity index 100% rename from beacon-chain/chaintest/sampletests/basic_fork_choice.yaml rename to beacon-chain/chaintest/tests/chain-tests/basic_fork_choice.yaml diff --git a/beacon-chain/chaintest/tests/shuffle-tests/shuffle.yaml b/beacon-chain/chaintest/tests/shuffle-tests/shuffle.yaml new file mode 100644 index 000000000..e4a6300d1 --- /dev/null +++ b/beacon-chain/chaintest/tests/shuffle-tests/shuffle.yaml @@ -0,0 +1,44 @@ +# Credits to Danny Ryan (Ethereum Foundation) +--- + +title: Shuffling Algorithm Tests +summary: Test vectors for shuffling a list based upon a seed using `shuffle` +test_suite: shuffle +fork: tchaikovsky +version: 1.0 + +test_cases: +- config: + validator_count: 100 + cycle_length: 8 + shard_count: 32 + min_committee_size: 8 +- input: [] + output: [] + seed: !!binary "" +- name: boring_list + description: List with a single element, 0 + input: [0] + output: [0] + seed: !!binary "" +- input: [255] + output: [255] + seed: !!binary "" +- input: [4, 6, 2, 6, 1, 4, 6, 2, 1, 5] + output: [1, 1, 2, 6, 4, 4, 5, 6, 2, 6] + seed: !!binary "" +- input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + output: [3, 8, 11, 7, 9, 5, 13, 4, 10, 6, 1, 2, 12] + seed: !!binary "" +- input: [65, 6, 2, 6, 1, 4, 6, 2, 1, 5] + output: [65, 6, 6, 6, 2, 4, 1, 2, 1, 5] + seed: !!binary | + JlAYJ5H2j8g7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= +- input: [35, 6, 2, 6, 1, 4, 6, 2, 1, 5, 7, 98, 3, 2, 11] + output: [98, 2, 3, 6, 1, 6, 7, 2, 2, 35, 5, 6, 4, 1, 11] + seed: !!binary | + VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4= +- input: [35, 6, 2, 6, 1, 4, 6, 2, 1, 5, 7, 98, 3, 2, 11] + output: [6, 6, 1, 98, 6, 2, 5, 7, 2, 11, 35, 1, 2, 4, 3] + seed: !!binary | + rDTbe23J4UA0yLIurjbJqk49VcavAC0Nysas+l5MlwvLc0B/JqQ= diff --git a/beacon-chain/chaintest/yaml_test.go b/beacon-chain/chaintest/yaml_test.go index b2156afc6..617565dd7 100644 --- a/beacon-chain/chaintest/yaml_test.go +++ b/beacon-chain/chaintest/yaml_test.go @@ -1,44 +1,38 @@ package main import ( - "io/ioutil" - "path" "testing" - "github.com/go-yaml/yaml" "github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend" ) -func TestYamls(t *testing.T) { +// TestReadTestsFromYaml tests constructing test cases from yaml file. +func TestReadTestsFromYaml(t *testing.T) { + if _, err := readTestsFromYaml(string("./tests")); err != nil { + t.Fatalf("Failed to read yaml files: %v", err) + } +} + +// TestReadTestsFromYaml tests the running of provided tests structs. +func TestRunTests(t *testing.T) { sb, err := backend.NewSimulatedBackend() if err != nil { t.Fatalf("Could not create backend: %v", err) } - var chainTests []*backend.ChainTest - - files, err := ioutil.ReadDir("./sampletests") - if err != nil { - t.Fatalf("Could not read yaml tests directory: %v", err) + chainTestCase := &backend.ChainTestCase{ + Config: &backend.ChainTestConfig{ + ShardCount: 3, + CycleLength: 10, + MinCommitteeSize: 3, + ValidatorCount: 100, + }, } + shuffleTestCase := &backend.ShuffleTestCase{} - for _, file := range files { - data, err := ioutil.ReadFile(path.Join("./sampletests", file.Name())) - if err != nil { - t.Fatalf("Could not read yaml file: %v", err) - } - decoded := &backend.ChainTest{} - if err := yaml.Unmarshal(data, decoded); err != nil { - t.Fatalf("Could not unmarshal YAML file into test struct: %v", err) - } - chainTests = append(chainTests, decoded) - } - - for _, tt := range chainTests { - for _, testCase := range tt.TestCases { - if err := sb.RunChainTest(testCase); err != nil { - t.Errorf("Beacon Chain test failed: %v", err) - } - } + chainTest := &backend.ChainTest{TestCases: []*backend.ChainTestCase{chainTestCase}} + shuffleTest := &backend.ShuffleTest{TestCases: []*backend.ShuffleTestCase{shuffleTestCase}} + if err = runTests([]interface{}{chainTest, shuffleTest}, sb); err != nil { + t.Fatalf("Failed to run test cases: %v", err) } }