From 68edad13bcd0346eaf8a67ce5d3c6fe6eb9c152e Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 15 Nov 2019 13:56:26 -0500 Subject: [PATCH] End To End Tests for Demo and Minimal config (#3932) * Begin working on end to end tests using geth dev chain * Start on beacon node set up * More progress on bnode setup * Complete flow until chainstart, begin work on evaluators * More progress on evaluators * Start changing bazel run to direct binary * Move endtoend to inside beacon-chain * use bazel provided geth, use bazel test * tempdir * use fork rules_go * Change to use UUID dir and bazel binaries * Truncate UUID a bit * Get full run from chainstart to evaluating * Rewrite to react to logs rather than arbitrarily wait * Fix export * Move evaluators to evaluators.go * Add peer check test * Add more comments * Remove unneeded exports * Check all nodes have the correct amount of peers * Change name to onGenesisEpoch * Remove extra wait times where not needed * Cleanup * Add log for beacon start * Fix deposit amount * Make room for eth1follow distnce * Cleanup and fix minimal test * Goimports * Fix imports * gazelle and minimal * manual * Fix for comments * Make timing rely on reading logs, and cleanup * Fix for comments * Fix workspace * Cleanup * Fix visibility * Cleanup and some comments * Address comments * Fix for v0.9 * Modify for v0.9 * Move to own package outside of beacon-chain * Gazelle * Polishing, logging * Fix filenames * Add more logs * Add flag logging * Cover for page not having libp2p info * Improve multiAddr detection * Add more logs * Add missing flags * Add log printing to defer * Get multiAddr from logs * Fix logging and detection * Change evaluators to rely on EpochTimer * Add evaluator for ValidatorParticipation * Fix validator participation evaluator * Cleanup, comments and fix participation calculation * Cleanup * Let the file searcher search for longer * Change participation to check for full * Log out file contents if text isnt found * Split into different files * Disable IPC and use RPC instead, change tmp dir to bazel dir * Change visibility * Gazelle * Add e2e tag * new line --- WORKSPACE | 16 ++- beacon-chain/BUILD.bazel | 5 +- beacon-chain/node/node.go | 2 +- endtoend/BUILD.bazel | 61 ++++++++++++ endtoend/beacon_node.go | 163 +++++++++++++++++++++++++++++++ endtoend/demo_e2e_test.go | 27 +++++ endtoend/endtoend_test.go | 153 +++++++++++++++++++++++++++++ endtoend/epochTimer.go | 79 +++++++++++++++ endtoend/eth1.go | 146 +++++++++++++++++++++++++++ endtoend/evaluators/BUILD.bazel | 18 ++++ endtoend/evaluators/finality.go | 54 ++++++++++ endtoend/evaluators/validator.go | 94 ++++++++++++++++++ endtoend/minimal_e2e_test.go | 27 +++++ endtoend/validator.go | 116 ++++++++++++++++++++++ shared/params/config.go | 2 +- validator/BUILD.bazel | 5 +- 16 files changed, 955 insertions(+), 13 deletions(-) create mode 100644 endtoend/BUILD.bazel create mode 100644 endtoend/beacon_node.go create mode 100644 endtoend/demo_e2e_test.go create mode 100644 endtoend/endtoend_test.go create mode 100644 endtoend/epochTimer.go create mode 100644 endtoend/eth1.go create mode 100644 endtoend/evaluators/BUILD.bazel create mode 100644 endtoend/evaluators/finality.go create mode 100644 endtoend/evaluators/validator.go create mode 100644 endtoend/minimal_e2e_test.go create mode 100644 endtoend/validator.go diff --git a/WORKSPACE b/WORKSPACE index 4159598cd..286b53f9b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -8,15 +8,6 @@ http_archive( url = "https://github.com/bazelbuild/bazel-skylib/archive/0.8.0.tar.gz", ) -http_archive( - name = "io_bazel_rules_go", - sha256 = "513c12397db1bc9aa46dd62f02dd94b49a9b5d17444d49b5a04c5a89f3053c1c", - urls = [ - "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.19.5/rules_go-v0.19.5.tar.gz", - "https://github.com/bazelbuild/rules_go/releases/download/v0.19.5/rules_go-v0.19.5.tar.gz", - ], -) - http_archive( name = "bazel_gazelle", sha256 = "7fc87f4170011201b1690326e8c16c5d802836e3a0d617d8f75c3af2b23180c4", @@ -40,6 +31,13 @@ http_archive( url = "https://github.com/bazelbuild/rules_docker/archive/v0.10.1.tar.gz", ) +http_archive( + name = "io_bazel_rules_go", + sha256 = "886db2f8d620fcb5791c8e2a402a575bc70728e17ec116841d78f3837a09f69e", + strip_prefix = "rules_go-9bb1562710f7077cd109b66cd4b45900e6d7ae73", + urls = ["https://github.com/bazelbuild/rules_go/archive/9bb1562710f7077cd109b66cd4b45900e6d7ae73.tar.gz"], +) + http_archive( name = "build_bazel_rules_nodejs", sha256 = "0942d188f4d0de6ddb743b9f6642a26ce1ad89f09c0035a9a5ca5ba9615c96aa", diff --git a/beacon-chain/BUILD.bazel b/beacon-chain/BUILD.bazel index 33f8a7695..3e86b6f6e 100644 --- a/beacon-chain/BUILD.bazel +++ b/beacon-chain/BUILD.bazel @@ -79,7 +79,10 @@ docker_push( go_binary( name = "beacon-chain", embed = [":go_default_library"], - visibility = ["//beacon-chain:__subpackages__"], + visibility = [ + "//beacon-chain:__subpackages__", + "//endtoend:__pkg__", + ], ) go_test( diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 6aac02530..1caf7bd6a 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -80,7 +80,7 @@ func NewBeaconNode(ctx *cli.Context) (*BeaconNode, error) { stop: make(chan struct{}), } - // Use custom config values if the --no-custom-config flag is set. + // Use custom config values if the --no-custom-config flag is not set. if !ctx.GlobalBool(flags.NoCustomConfigFlag.Name) { if featureconfig.Get().MinimalConfig { log.WithField( diff --git a/endtoend/BUILD.bazel b/endtoend/BUILD.bazel new file mode 100644 index 000000000..63cf08aed --- /dev/null +++ b/endtoend/BUILD.bazel @@ -0,0 +1,61 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_test( + name = "go_default_test", + size = "enormous", + srcs = [ + "demo_e2e_test.go", + "endtoend_test.go", + "minimal_e2e_test.go", + ], + data = [ + "//beacon-chain", + "//validator", + "@com_github_ethereum_go_ethereum//cmd/geth", + ], + embed = [":go_default_library"], + shard_count = 2, + tags = [ + "block-network", + "exclusive", + "manual", + "minimal", + "e2e", + ], + deps = [ + "//endtoend/evaluators:go_default_library", + "//proto/eth/v1alpha1:go_default_library", + "//shared/params:go_default_library", + "//shared/testutil:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@io_bazel_rules_go//go/tools/bazel:go_default_library", + "@org_golang_google_grpc//:go_default_library", + ], +) + +go_library( + name = "go_default_library", + testonly = True, + srcs = [ + "beacon_node.go", + "epochTimer.go", + "eth1.go", + "validator.go", + ], + importpath = "github.com/prysmaticlabs/prysm/endtoend", + visibility = ["//endtoend:__subpackages__"], + deps = [ + "//contracts/deposit-contract:go_default_library", + "//endtoend/evaluators:go_default_library", + "//shared/params:go_default_library", + "//shared/roughtime:go_default_library", + "//shared/testutil:go_default_library", + "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library", + "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library", + "@com_github_ethereum_go_ethereum//common:go_default_library", + "@com_github_ethereum_go_ethereum//core/types:go_default_library", + "@com_github_ethereum_go_ethereum//ethclient:go_default_library", + "@com_github_ethereum_go_ethereum//rpc:go_default_library", + "@io_bazel_rules_go//go/tools/bazel:go_default_library", + ], +) diff --git a/endtoend/beacon_node.go b/endtoend/beacon_node.go new file mode 100644 index 000000000..6a3eb1f38 --- /dev/null +++ b/endtoend/beacon_node.go @@ -0,0 +1,163 @@ +package endtoend + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/ethereum/go-ethereum/common" + ev "github.com/prysmaticlabs/prysm/endtoend/evaluators" +) + +type beaconNodeInfo struct { + processID int + datadir string + rpcPort uint64 + monitorPort uint64 + grpcPort uint64 + multiAddr string +} + +type end2EndConfig struct { + minimalConfig bool + tmpPath string + epochsToRun uint64 + numValidators uint64 + numBeaconNodes uint64 + contractAddr common.Address + evaluators []ev.Evaluator +} + +// startBeaconNodes starts the requested amount of beacon nodes, passing in the deposit contract given. +func startBeaconNodes(t *testing.T, config *end2EndConfig) []*beaconNodeInfo { + numNodes := config.numBeaconNodes + + nodeInfo := []*beaconNodeInfo{} + for i := uint64(0); i < numNodes; i++ { + newNode := startNewBeaconNode(t, config, nodeInfo) + nodeInfo = append(nodeInfo, newNode) + } + + return nodeInfo +} + +func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beaconNodeInfo) *beaconNodeInfo { + tmpPath := config.tmpPath + index := len(beaconNodes) + binaryPath, found := bazel.FindBinary("beacon-chain", "beacon-chain") + if !found { + t.Log(binaryPath) + t.Fatal("beacon chain binary not found") + } + file, err := os.Create(path.Join(tmpPath, fmt.Sprintf("beacon-%d.log", index))) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--no-discovery", + "--http-web3provider=http://127.0.0.1:8545", + "--web3provider=ws://127.0.0.1:8546", + fmt.Sprintf("--datadir=%s/eth2-beacon-node-%d", tmpPath, index), + fmt.Sprintf("--deposit-contract=%s", config.contractAddr.Hex()), + fmt.Sprintf("--rpc-port=%d", 4000+index), + fmt.Sprintf("--p2p-udp-port=%d", 12000+index), + fmt.Sprintf("--p2p-tcp-port=%d", 13000+index), + fmt.Sprintf("--monitoring-port=%d", 8080+index), + fmt.Sprintf("--grpc-gateway-port=%d", 3200+index), + } + + if config.minimalConfig { + args = append(args, "--minimal-config") + } + // After the first node is made, have all following nodes connect to all previously made nodes. + if index >= 1 { + for p := 0; p < index; p++ { + args = append(args, fmt.Sprintf("--peer=%s", beaconNodes[p].multiAddr)) + } + } + + t.Logf("Starting beacon chain with flags %s", strings.Join(args, " ")) + cmd := exec.Command(binaryPath, args...) + cmd.Stderr = file + cmd.Stdout = file + if err := cmd.Start(); err != nil { + t.Fatalf("failed to start beacon node: %v", err) + } + + if err = waitForTextInFile(file, "Node started p2p server"); err != nil { + t.Fatal(err) + } + + multiAddr, err := getMultiAddrFromLogFile(file.Name()) + if err != nil { + t.Fatal(err) + } + + return &beaconNodeInfo{ + processID: cmd.Process.Pid, + datadir: fmt.Sprintf("%s/eth2-beacon-node-%d", tmpPath, index), + rpcPort: 4000 + uint64(index), + monitorPort: 8080 + uint64(index), + grpcPort: 3200 + uint64(index), + multiAddr: multiAddr, + } +} + +func getMultiAddrFromLogFile(name string) (string, error) { + byteContent, err := ioutil.ReadFile(name) + if err != nil { + return "", err + } + contents := string(byteContent) + + searchText := "\"Node started p2p server\" multiAddr=\"" + startIdx := strings.Index(contents, searchText) + if startIdx == -1 { + return "", fmt.Errorf("did not find peer text in %s", contents) + } + startIdx += len(searchText) + endIdx := strings.Index(contents[startIdx:], "\"") + if endIdx == -1 { + return "", fmt.Errorf("did not find peer text in %s", contents) + } + return contents[startIdx : startIdx+endIdx], nil +} + +func waitForTextInFile(file *os.File, text string) error { + wait := 0 + // Putting the wait cap at 24 seconds. + totalWait := 24 + for wait < totalWait { + time.Sleep(2 * time.Second) + // Rewind the file pointer to the start of the file so we can read it again. + _, err := file.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("could not rewind file to start: %v", err) + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), text) { + return nil + } + } + if err := scanner.Err(); err != nil { + return err + } + wait += 2 + } + contents, err := ioutil.ReadFile(file.Name()) + if err != nil { + return err + } + return fmt.Errorf("could not find requested text %s in logs:\n%s", text, string(contents)) +} diff --git a/endtoend/demo_e2e_test.go b/endtoend/demo_e2e_test.go new file mode 100644 index 000000000..0b31730b6 --- /dev/null +++ b/endtoend/demo_e2e_test.go @@ -0,0 +1,27 @@ +package endtoend + +import ( + "testing" + + ev "github.com/prysmaticlabs/prysm/endtoend/evaluators" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" +) + +func TestEndToEnd_DemoConfig(t *testing.T) { + testutil.ResetCache() + params.UseDemoBeaconConfig() + + demoConfig := &end2EndConfig{ + minimalConfig: false, + epochsToRun: 5, + numBeaconNodes: 4, + numValidators: params.BeaconConfig().MinGenesisActiveValidatorCount, + evaluators: []ev.Evaluator{ + ev.ValidatorsAreActive, + ev.ValidatorsParticipating, + ev.FinalizationOccurs, + }, + } + runEndToEndTest(t, demoConfig) +} diff --git a/endtoend/endtoend_test.go b/endtoend/endtoend_test.go new file mode 100644 index 000000000..d7757aad1 --- /dev/null +++ b/endtoend/endtoend_test.go @@ -0,0 +1,153 @@ +package endtoend + +import ( + "bufio" + "context" + "fmt" + "github.com/bazelbuild/rules_go/go/tools/bazel" + "io/ioutil" + "net/http" + "os" + "path" + "strconv" + "strings" + "testing" + "time" + + ptypes "github.com/gogo/protobuf/types" + "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/params" + "google.golang.org/grpc" +) + +func runEndToEndTest(t *testing.T, config *end2EndConfig) { + tmpPath := bazel.TestTmpDir() + config.tmpPath = tmpPath + t.Logf("Test Path: %s\n", tmpPath) + + contractAddr, keystorePath, eth1PID := startEth1(t, tmpPath) + config.contractAddr = contractAddr + beaconNodes := startBeaconNodes(t, config) + valClients := initializeValidators(t, config, keystorePath, beaconNodes) + processIDs := []int{eth1PID} + for _, vv := range valClients { + processIDs = append(processIDs, vv.processID) + } + for _, bb := range beaconNodes { + processIDs = append(processIDs, bb.processID) + } + defer logOutput(t, tmpPath) + defer killProcesses(t, processIDs) + + if config.numBeaconNodes > 1 { + t.Run("all_peers_connect", func(t *testing.T) { + for _, bNode := range beaconNodes { + if err := peersConnect(bNode.monitorPort, config.numBeaconNodes-1); err != nil { + t.Fatalf("failed to connect to peers: %v", err) + } + } + }) + } + + beaconLogFile, err := os.Open(path.Join(tmpPath, "beacon-0.log")) + if err != nil { + t.Fatal(err) + } + if err := waitForTextInFile(beaconLogFile, "Sending genesis time notification"); err != nil { + t.Fatal(err) + } + conn, err := grpc.Dial("127.0.0.1:4000", grpc.WithInsecure()) + if err != nil { + t.Fatalf("fail to dial: %v", err) + } + beaconClient := eth.NewBeaconChainClient(conn) + nodeClient := eth.NewNodeClient(conn) + + genesis, err := nodeClient.GetGenesis(context.Background(), &ptypes.Empty{}) + if err != nil { + t.Fatal(err) + } + // Small offset so evaluators perform in the middle of an epoch. + epochSeconds := params.BeaconConfig().SecondsPerSlot * params.BeaconConfig().SlotsPerEpoch + genesisTime := time.Unix(genesis.GenesisTime.Seconds+int64(epochSeconds/2), 0) + currentEpoch := uint64(0) + ticker := GetEpochTicker(genesisTime, epochSeconds) + for c := range ticker.C() { + if c >= config.epochsToRun { + ticker.Done() + break + } + for _, evaluator := range config.evaluators { + // Only run if the policy says so. + if !evaluator.Policy(currentEpoch) { + continue + } + t.Run(fmt.Sprintf(evaluator.Name, currentEpoch), func(t *testing.T) { + if err := evaluator.Evaluation(beaconClient); err != nil { + t.Fatal(err) + } + }) + } + currentEpoch++ + } + + if currentEpoch < config.epochsToRun { + t.Fatalf("test ended prematurely, only reached epoch %d", currentEpoch) + } +} + +func peersConnect(port uint64, expectedPeers uint64) error { + response, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/p2p", port)) + if err != nil { + return err + } + dataInBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + return err + } + pageContent := string(dataInBytes) + if err := response.Body.Close(); err != nil { + return err + } + // Subtracting by 2 here since the libp2p page has "3 peers" as text. + // With a starting index before the "p", going two characters back should give us + // the number we need. + startIdx := strings.Index(pageContent, "peers") - 2 + if startIdx == -3 { + return fmt.Errorf("could not find needed text in %s", pageContent) + } + peerCount, err := strconv.Atoi(pageContent[startIdx : startIdx+1]) + if err != nil { + return err + } + if expectedPeers != uint64(peerCount) { + return fmt.Errorf("unexpected amount of peers, expected %d, received %d", expectedPeers, peerCount) + } + return nil +} + +func killProcesses(t *testing.T, pIDs []int) { + for _, id := range pIDs { + process, err := os.FindProcess(id) + if err != nil { + t.Fatalf("could not find process %d: %v", id, err) + } + if err := process.Kill(); err != nil { + t.Fatal(err) + } + } +} + +func logOutput(t *testing.T, tmpPath string) { + if t.Failed() { + beacon1LogFile, err := os.Open(path.Join(tmpPath, "beacon-1.log")) + if err != nil { + t.Fatal(err) + } + scanner := bufio.NewScanner(beacon1LogFile) + for scanner.Scan() { + currentLine := scanner.Text() + t.Log(currentLine) + } + } +} diff --git a/endtoend/epochTimer.go b/endtoend/epochTimer.go new file mode 100644 index 000000000..0048a7b72 --- /dev/null +++ b/endtoend/epochTimer.go @@ -0,0 +1,79 @@ +package endtoend + +import ( + "time" + + "github.com/prysmaticlabs/prysm/shared/roughtime" +) + +// EpochTicker is a special ticker for timing epoch changes. +// The channel emits over the epoch interval, and ensures that +// the ticks are in line with the genesis time. This means that +// the duration between the ticks and the genesis time are always a +// multiple of the epoch duration. +// In addition, the channel returns the new epoch number. +type EpochTicker struct { + c chan uint64 + done chan struct{} +} + +// C returns the ticker channel. Call Cancel afterwards to ensure +// that the goroutine exits cleanly. +func (s *EpochTicker) C() <-chan uint64 { + return s.c +} + +// Done should be called to clean up the ticker. +func (s *EpochTicker) Done() { + go func() { + s.done <- struct{}{} + }() +} + +// GetEpochTicker is the constructor for EpochTicker. +func GetEpochTicker(genesisTime time.Time, secondsPerEpoch uint64) *EpochTicker { + ticker := &EpochTicker{ + c: make(chan uint64), + done: make(chan struct{}), + } + ticker.start(genesisTime, secondsPerEpoch, roughtime.Since, roughtime.Until, time.After) + return ticker +} + +func (s *EpochTicker) start( + genesisTime time.Time, + secondsPerEpoch uint64, + since func(time.Time) time.Duration, + until func(time.Time) time.Duration, + after func(time.Duration) <-chan time.Time) { + + d := time.Duration(secondsPerEpoch) * time.Second + + go func() { + sinceGenesis := since(genesisTime) + + var nextTickTime time.Time + var epoch uint64 + if sinceGenesis < 0 { + // Handle when the current time is before the genesis time. + nextTickTime = genesisTime + epoch = 0 + } else { + nextTick := sinceGenesis.Truncate(d) + d + nextTickTime = genesisTime.Add(nextTick) + epoch = uint64(nextTick / d) + } + + for { + waitTime := until(nextTickTime) + select { + case <-after(waitTime): + s.c <- epoch + epoch++ + nextTickTime = nextTickTime.Add(d) + case <-s.done: + return + } + } + }() +} diff --git a/endtoend/eth1.go b/endtoend/eth1.go new file mode 100644 index 000000000..b713f42d0 --- /dev/null +++ b/endtoend/eth1.go @@ -0,0 +1,146 @@ +package endtoend + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "math/big" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + contracts "github.com/prysmaticlabs/prysm/contracts/deposit-contract" + "github.com/prysmaticlabs/prysm/shared/params" +) + +// startEth1 starts an eth1 local dev chain and deploys a deposit contract. +func startEth1(t *testing.T, tmpPath string) (common.Address, string, int) { + binaryPath, found := bazel.FindBinary("cmd/geth", "geth") + if !found { + t.Fatal("go-ethereum binary not found") + } + + args := []string{ + fmt.Sprintf("--datadir=%s", path.Join(tmpPath, "eth1data/")), + "--rpc", + "--rpcaddr=0.0.0.0", + "--rpccorsdomain=\"*\"", + "--rpcvhosts=\"*\"", + "--ws", + "--wsaddr=0.0.0.0", + "--wsorigins=\"*\"", + "--dev", + "--dev.period=0", + "--ipcdisable", + } + cmd := exec.Command(binaryPath, args...) + file, err := os.Create(path.Join(tmpPath, "eth1.log")) + if err != nil { + t.Fatal(err) + } + cmd.Stdout = file + cmd.Stderr = file + if err := cmd.Start(); err != nil { + t.Fatalf("failed to start eth1 chain: %v", err) + } + + if err = waitForTextInFile(file, "Commit new mining work"); err != nil { + t.Fatal(err) + } + + // Connect to the started geth dev chain. + client, err := rpc.DialHTTP("http://127.0.0.1:8545") + if err != nil { + t.Fatalf("failed to connect to ipc: %v", err) + } + web3 := ethclient.NewClient(client) + + // Access the dev account keystore to deploy the contract. + fileName, err := exec.Command("ls", path.Join(tmpPath, "eth1data/keystore")).Output() + if err != nil { + t.Fatal(err) + } + keystorePath := path.Join(tmpPath, fmt.Sprintf("eth1data/keystore/%s", strings.TrimSpace(string(fileName)))) + jsonBytes, err := ioutil.ReadFile(keystorePath) + if err != nil { + t.Fatal(err) + } + keystore, err := keystore.DecryptKey(jsonBytes, "" /*password*/) + if err != nil { + t.Fatal(err) + } + + // Advancing the blocks eth1follow distance to prevent issues reading the chain. + if err := mineBlocks(web3, keystore, params.BeaconConfig().Eth1FollowDistance); err != nil { + t.Fatalf("unable to advance chain: %v", err) + } + + txOpts, err := bind.NewTransactor(bytes.NewReader(jsonBytes), "" /*password*/) + if err != nil { + t.Fatal(err) + } + nonce, err := web3.PendingNonceAt(context.Background(), keystore.Address) + if err != nil { + t.Fatal(err) + } + txOpts.Nonce = big.NewInt(int64(nonce)) + contractAddr, tx, _, err := contracts.DeployDepositContract(txOpts, web3, txOpts.From) + if err != nil { + t.Fatalf("failed to deploy deposit contract: %v", err) + } + + // Wait for contract to mine. + for pending := true; pending; _, pending, err = web3.TransactionByHash(context.Background(), tx.Hash()) { + if err != nil { + t.Fatal(err) + } + time.Sleep(100 * time.Millisecond) + } + + return contractAddr, keystorePath, cmd.Process.Pid +} + +func mineBlocks(web3 *ethclient.Client, keystore *keystore.Key, blocksToMake uint64) error { + nonce, err := web3.PendingNonceAt(context.Background(), keystore.Address) + if err != nil { + return err + } + chainID, err := web3.NetworkID(context.Background()) + if err != nil { + return err + } + block, err := web3.BlockByNumber(context.Background(), nil) + if err != nil { + return err + } + finishBlock := block.NumberU64() + blocksToMake + + for block.NumberU64() <= finishBlock { + spamTX := types.NewTransaction(nonce, keystore.Address, big.NewInt(0), 21000, big.NewInt(1e6), []byte{}) + signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), keystore.PrivateKey) + if err != nil { + return err + } + if err := web3.SendTransaction(context.Background(), signed); err != nil { + return err + } + nonce++ + time.Sleep(250 * time.Microsecond) + block, err = web3.BlockByNumber(context.Background(), nil) + if err != nil { + return err + } + } + return nil +} diff --git a/endtoend/evaluators/BUILD.bazel b/endtoend/evaluators/BUILD.bazel new file mode 100644 index 000000000..e8587e993 --- /dev/null +++ b/endtoend/evaluators/BUILD.bazel @@ -0,0 +1,18 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + testonly = True, + srcs = [ + "finality.go", + "validator.go", + ], + importpath = "github.com/prysmaticlabs/prysm/endtoend/evaluators", + visibility = ["//endtoend:__subpackages__"], + deps = [ + "//proto/eth/v1alpha1:go_default_library", + "//shared/params:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], +) diff --git a/endtoend/evaluators/finality.go b/endtoend/evaluators/finality.go new file mode 100644 index 000000000..519a1a7ec --- /dev/null +++ b/endtoend/evaluators/finality.go @@ -0,0 +1,54 @@ +package evaluators + +import ( + "context" + "fmt" + + ptypes "github.com/gogo/protobuf/types" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/params" +) + +// FinalizationOccurs is an evaluator to make sure finalization is performing as it should. +// Requires to be run after at least 4 epochs have passed. +var FinalizationOccurs = Evaluator{ + Name: "finalizes_at_epoch_%d", + Policy: afterNthEpoch(3), + Evaluation: finalizationOccurs, +} + +func finalizationOccurs(client eth.BeaconChainClient) error { + chainHead, err := client.GetChainHead(context.Background(), &ptypes.Empty{}) + if err != nil { + return errors.Wrap(err, "failed to get chain head") + } + currentEpoch := chainHead.BlockSlot / params.BeaconConfig().SlotsPerEpoch + finalizedEpoch := chainHead.FinalizedSlot / params.BeaconConfig().SlotsPerEpoch + + expectedFinalizedEpoch := currentEpoch - 2 + if expectedFinalizedEpoch != finalizedEpoch { + return fmt.Errorf( + "expected finalized epoch to be %d, received: %d", + expectedFinalizedEpoch, + finalizedEpoch, + ) + } + previousJustifiedEpoch := chainHead.PreviousJustifiedSlot / params.BeaconConfig().SlotsPerEpoch + currentJustifiedEpoch := chainHead.JustifiedSlot / params.BeaconConfig().SlotsPerEpoch + if previousJustifiedEpoch+1 != currentJustifiedEpoch { + return fmt.Errorf( + "there should be no gaps between current and previous justified epochs, received current %d and previous %d", + currentJustifiedEpoch, + previousJustifiedEpoch, + ) + } + if currentJustifiedEpoch+1 != currentEpoch { + return fmt.Errorf( + "there should be no gaps between current epoch and current justified epoch, received current %d and justified %d", + currentEpoch, + currentJustifiedEpoch, + ) + } + return nil +} diff --git a/endtoend/evaluators/validator.go b/endtoend/evaluators/validator.go new file mode 100644 index 000000000..5cb46daf9 --- /dev/null +++ b/endtoend/evaluators/validator.go @@ -0,0 +1,94 @@ +package evaluators + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/params" +) + +// Evaluator defines the structure of the evaluators used to +// conduct the current beacon state during the E2E. +type Evaluator struct { + Name string + Policy func(currentEpoch uint64) bool + Evaluation func(client eth.BeaconChainClient) error +} + +// ValidatorsAreActive ensures the expected amount of validators are active. +var ValidatorsAreActive = Evaluator{ + Name: "validators_active_epoch_%d", + Policy: onGenesisEpoch, + Evaluation: validatorsAreActive, +} + +// ValidatorsParticipating ensures the expected amount of validators are active. +var ValidatorsParticipating = Evaluator{ + Name: "validators_participating_epoch_%d", + Policy: afterNthEpoch(1), + Evaluation: validatorsParticipating, +} + +func onGenesisEpoch(currentEpoch uint64) bool { + return currentEpoch < 2 +} + +// Not including first epoch because of issues with genesis. +func afterNthEpoch(afterEpoch uint64) func(uint64) bool { + return func(currentEpoch uint64) bool { + return currentEpoch > afterEpoch + } +} + +func validatorsAreActive(client eth.BeaconChainClient) error { + // Balances actually fluctuate but we just want to check initial balance. + validatorRequest := ð.GetValidatorsRequest{} + validators, err := client.GetValidators(context.Background(), validatorRequest) + if err != nil { + return errors.Wrap(err, "failed to get validators") + } + + expectedCount := params.BeaconConfig().MinGenesisActiveValidatorCount + receivedCount := uint64(len(validators.Validators)) + if expectedCount != receivedCount { + return fmt.Errorf("expected validator count to be %d, recevied %d", expectedCount, receivedCount) + } + + for _, val := range validators.Validators { + if val.ActivationEpoch != 0 { + return fmt.Errorf("expected genesis validator epoch to be 0, received %d", val.ActivationEpoch) + } + if val.ExitEpoch != params.BeaconConfig().FarFutureEpoch { + return fmt.Errorf("expected genesis validator exit epoch to be far future, received %d", val.ExitEpoch) + } + if val.WithdrawableEpoch != params.BeaconConfig().FarFutureEpoch { + return fmt.Errorf("expected genesis validator withdrawable epoch to be far future, received %d", val.WithdrawableEpoch) + } + if val.EffectiveBalance != params.BeaconConfig().MaxEffectiveBalance { + return fmt.Errorf( + "expected genesis validator effective balance to be %d, received %d", + params.BeaconConfig().MaxEffectiveBalance, + val.EffectiveBalance, + ) + } + } + return nil +} + +// validatorsParticipating ensures the validators have an acceptable participation rate. +func validatorsParticipating(client eth.BeaconChainClient) error { + validatorRequest := ð.GetValidatorParticipationRequest{} + participation, err := client.GetValidatorParticipation(context.Background(), validatorRequest) + if err != nil { + return errors.Wrap(err, "failed to get validator participation") + } + + partRate := participation.Participation.GlobalParticipationRate + expected := float32(1) + if partRate < expected { + return fmt.Errorf("validator participation was below expected %f, received: %f", expected, partRate) + } + return nil +} diff --git a/endtoend/minimal_e2e_test.go b/endtoend/minimal_e2e_test.go new file mode 100644 index 000000000..4b46bea3d --- /dev/null +++ b/endtoend/minimal_e2e_test.go @@ -0,0 +1,27 @@ +package endtoend + +import ( + "testing" + + ev "github.com/prysmaticlabs/prysm/endtoend/evaluators" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" +) + +func TestEndToEnd_MinimalConfig(t *testing.T) { + testutil.ResetCache() + params.UseMinimalConfig() + + minimalConfig := &end2EndConfig{ + minimalConfig: true, + epochsToRun: 5, + numBeaconNodes: 4, + numValidators: params.BeaconConfig().MinGenesisActiveValidatorCount, + evaluators: []ev.Evaluator{ + ev.ValidatorsAreActive, + ev.ValidatorsParticipating, + ev.FinalizationOccurs, + }, + } + runEndToEndTest(t, minimalConfig) +} diff --git a/endtoend/validator.go b/endtoend/validator.go new file mode 100644 index 000000000..fe65cf199 --- /dev/null +++ b/endtoend/validator.go @@ -0,0 +1,116 @@ +package endtoend + +import ( + "bytes" + "fmt" + "io/ioutil" + "math/big" + "os" + "os/exec" + "path" + "strings" + "testing" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + contracts "github.com/prysmaticlabs/prysm/contracts/deposit-contract" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" +) + +type validatorClientInfo struct { + processID int + monitorPort uint64 +} + +// initializeValidators sends the deposits to the eth1 chain and starts the validator clients. +func initializeValidators( + t *testing.T, + config *end2EndConfig, + keystorePath string, + beaconNodes []*beaconNodeInfo, +) []*validatorClientInfo { + binaryPath, found := bazel.FindBinary("validator", "validator") + if !found { + t.Fatal("validator binary not found") + } + + tmpPath := config.tmpPath + validatorNum := config.numValidators + beaconNodeNum := config.numBeaconNodes + if validatorNum%beaconNodeNum != 0 { + t.Fatal("Validator count is not easily divisible by beacon node count.") + } + + valClients := make([]*validatorClientInfo, beaconNodeNum) + validatorsPerNode := validatorNum / beaconNodeNum + for n := uint64(0); n < beaconNodeNum; n++ { + file, err := os.Create(path.Join(tmpPath, fmt.Sprintf("vals-%d.log", n))) + if err != nil { + t.Fatal(err) + } + args := []string{ + fmt.Sprintf("--interop-num-validators=%d", validatorsPerNode), + fmt.Sprintf("--interop-start-index=%d", validatorsPerNode*n), + fmt.Sprintf("--monitoring-port=%d", 9080+n), + fmt.Sprintf("--beacon-rpc-provider=localhost:%d", 4000+n), + } + cmd := exec.Command(binaryPath, args...) + cmd.Stdout = file + cmd.Stderr = file + t.Logf("Starting validator client with flags %s", strings.Join(args, " ")) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + valClients[n] = &validatorClientInfo{ + processID: cmd.Process.Pid, + monitorPort: 9080 + n, + } + } + + client, err := rpc.DialHTTP("http://127.0.0.1:8545") + if err != nil { + t.Fatal(err) + } + web3 := ethclient.NewClient(client) + + jsonBytes, err := ioutil.ReadFile(keystorePath) + if err != nil { + t.Fatal(err) + } + txOps, err := bind.NewTransactor(bytes.NewReader(jsonBytes), "" /*password*/) + if err != nil { + t.Fatal(err) + } + depositInGwei := big.NewInt(int64(params.BeaconConfig().MaxEffectiveBalance)) + txOps.Value = depositInGwei.Mul(depositInGwei, big.NewInt(int64(params.BeaconConfig().GweiPerEth))) + txOps.GasLimit = 4000000 + + contract, err := contracts.NewDepositContract(config.contractAddr, web3) + if err != nil { + t.Fatal(err) + } + + deposits, roots, _ := testutil.SetupInitialDeposits(t, validatorNum) + for index, dd := range deposits { + _, err = contract.Deposit(txOps, dd.Data.PublicKey, dd.Data.WithdrawalCredentials, dd.Data.Signature, roots[index]) + if err != nil { + t.Error("unable to send transaction to contract") + } + } + + keystore, err := keystore.DecryptKey(jsonBytes, "" /*password*/) + if err != nil { + t.Fatal(err) + } + // Picked 20 for this as a "safe" number of blocks to mine so the deposits + // are detected. + if err := mineBlocks(web3, keystore, 20); err != nil { + t.Fatal(err) + } + + return valClients +} diff --git a/shared/params/config.go b/shared/params/config.go index 7b99f75a0..a8476e7ac 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -29,7 +29,7 @@ type BeaconChainConfig struct { // Gwei value constants. MinDepositAmount uint64 `yaml:"MIN_DEPOSIT_AMOUNT"` // MinDepositAmount is the maximal amount of Gwei a validator can send to the deposit contract at once. - MaxEffectiveBalance uint64 `yaml:"MAX_EFFECTIVE_BALANCE"` // MaxEffectiveBalance is the maximal amount of Gwie that is effective for staking. + MaxEffectiveBalance uint64 `yaml:"MAX_EFFECTIVE_BALANCE"` // MaxEffectiveBalance is the maximal amount of Gwei that is effective for staking. EjectionBalance uint64 `yaml:"EJECTION_BALANCE"` // EjectionBalance is the minimal GWei a validator needs to have before ejected. EffectiveBalanceIncrement uint64 `yaml:"EFFECTIVE_BALANCE_INCREMENT"` // EffectiveBalanceIncrement is used for converting the high balance into the low balance for validators. diff --git a/validator/BUILD.bazel b/validator/BUILD.bazel index 25f81d88c..b8a514dc8 100644 --- a/validator/BUILD.bazel +++ b/validator/BUILD.bazel @@ -79,7 +79,10 @@ go_binary( name = "validator", embed = [":go_default_library"], pure = "off", # Enabled unless there is a valid reason to include cgo dep. - visibility = ["//validator:__subpackages__"], + visibility = [ + "//validator:__subpackages__", + "//endtoend:__pkg__", + ], ) [go_binary(