From 2b0f74904e291539a27c98093f7d716f45b2e8fb Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Thu, 3 Feb 2022 03:13:52 +0800 Subject: [PATCH] Multiclient E2E With Lighthouse (#10020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * save work * current progress * fix it more * save progress * fixes so far * add signature test * fix up changes so far * change to latest * update lighthouse version * fix build * fix again * do one * clean up * fix build * fix it * fix test * change tag * remove e2e * Update config/params/testnet_e2e_config.go Co-authored-by: RadosÅ‚aw Kapka * update * Update sha Co-authored-by: RadosÅ‚aw Kapka Co-authored-by: Raul Jordan Co-authored-by: prestonvanloon --- beacon-chain/p2p/BUILD.bazel | 1 + beacon-chain/p2p/options.go | 7 + beacon-chain/p2p/subnets.go | 7 +- cmd/BUILD.bazel | 1 + cmd/config.go | 11 +- config/fieldparams/mainnet.go | 1 + config/fieldparams/mainnet_test.go | 3 + config/fieldparams/minimal.go | 1 + config/fieldparams/minimal_test.go | 3 + config/params/testnet_config_test.go | 17 ++ config/params/testnet_e2e_config.go | 76 ++++++ testing/endtoend/BUILD.bazel | 55 +++- testing/endtoend/components/BUILD.bazel | 7 + testing/endtoend/components/beacon_node.go | 18 ++ .../endtoend/components/lighthouse_beacon.go | 187 +++++++++++++ .../components/lighthouse_validator.go | 255 ++++++++++++++++++ testing/endtoend/components/validator.go | 7 +- testing/endtoend/deps.bzl | 10 + testing/endtoend/endtoend_test.go | 43 ++- .../evaluators/api_gateway_v1alpha1.go | 36 ++- testing/endtoend/evaluators/node.go | 2 +- testing/endtoend/evaluators/operations.go | 4 +- testing/endtoend/helpers/helpers.go | 47 ++++ testing/endtoend/lighthouse.BUILD | 7 + testing/endtoend/mainnet_e2e_test.go | 75 ++++++ testing/endtoend/params/params.go | 59 +++- testing/endtoend/types/types.go | 2 + validator/keymanager/BUILD.bazel | 1 + 28 files changed, 909 insertions(+), 34 deletions(-) create mode 100644 testing/endtoend/components/lighthouse_beacon.go create mode 100644 testing/endtoend/components/lighthouse_validator.go create mode 100644 testing/endtoend/lighthouse.BUILD create mode 100644 testing/endtoend/mainnet_e2e_test.go diff --git a/beacon-chain/p2p/BUILD.bazel b/beacon-chain/p2p/BUILD.bazel index 9ecddb939..f3c196b5d 100644 --- a/beacon-chain/p2p/BUILD.bazel +++ b/beacon-chain/p2p/BUILD.bazel @@ -58,6 +58,7 @@ go_library( "//crypto/hash:go_default_library", "//encoding/bytesutil:go_default_library", "//io/file:go_default_library", + "//math:go_default_library", "//monitoring/tracing:go_default_library", "//network:go_default_library", "//network/forks:go_default_library", diff --git a/beacon-chain/p2p/options.go b/beacon-chain/p2p/options.go index 967c84650..9b07a8439 100644 --- a/beacon-chain/p2p/options.go +++ b/beacon-chain/p2p/options.go @@ -30,6 +30,13 @@ func (s *Service) buildOptions(ip net.IP, priKey *ecdsa.PrivateKey) []libp2p.Opt log.Fatalf("Failed to p2p listen: %v", err) } } + ifaceKey := convertToInterfacePrivkey(priKey) + id, err := peer.IDFromPublicKey(ifaceKey.GetPublic()) + if err != nil { + log.Fatalf("Failed to retrieve peer id: %v", err) + } + log.Infof("Running node with peer id of %s ", id.String()) + options := []libp2p.Option{ privKeyOption(priKey), libp2p.ListenAddrs(listen), diff --git a/beacon-chain/p2p/subnets.go b/beacon-chain/p2p/subnets.go index 8c12e55f9..5d367e328 100644 --- a/beacon-chain/p2p/subnets.go +++ b/beacon-chain/p2p/subnets.go @@ -9,6 +9,8 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/pkg/errors" "github.com/prysmaticlabs/go-bitfield" + "github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags" + mathutil "github.com/prysmaticlabs/prysm/math" "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper" "go.opencensus.io/trace" @@ -132,7 +134,10 @@ func (s *Service) filterPeerForSyncSubnet(index uint64) func(node *enode.Node) b // for a subnet. So that even in the event of poor peer // connectivity, we can still broadcast an attestation. func (s *Service) hasPeerWithSubnet(topic string) bool { - return len(s.pubsub.ListPeers(topic+s.Encoding().ProtocolSuffix())) >= 1 + // In the event peer threshold is lower, we will choose the lower + // threshold. + minPeers := mathutil.Min(1, uint64(flags.Get().MinimumPeersPerSubnet)) + return len(s.pubsub.ListPeers(topic+s.Encoding().ProtocolSuffix())) >= int(minPeers) } // Updates the service's discv5 listener record's attestation subnet diff --git a/cmd/BUILD.bazel b/cmd/BUILD.bazel index a6c8986ec..0930a46cc 100644 --- a/cmd/BUILD.bazel +++ b/cmd/BUILD.bazel @@ -13,6 +13,7 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/cmd", visibility = ["//visibility:public"], deps = [ + "//config/fieldparams:go_default_library", "//config/params:go_default_library", "//io/file:go_default_library", "@com_github_pkg_errors//:go_default_library", diff --git a/cmd/config.go b/cmd/config.go index ebedc7577..0642bbb65 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -1,6 +1,7 @@ package cmd import ( + fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams" "github.com/prysmaticlabs/prysm/config/params" "github.com/urfave/cli/v2" ) @@ -66,8 +67,14 @@ func newConfig(ctx *cli.Context) *Flags { } if ctx.Bool(E2EConfigFlag.Name) { log.Warn("Using end-to-end testing config") - cfg.MinimalConfig = true - params.UseE2EConfig() + switch fieldparams.Preset { + case "mainnet": + params.UseE2EMainnetConfig() + case "minimal": + params.UseE2EConfig() + default: + log.Fatalf("Unrecognized preset being used: %s", fieldparams.Preset) + } } return cfg } diff --git a/config/fieldparams/mainnet.go b/config/fieldparams/mainnet.go index f52b15c85..0119827d5 100644 --- a/config/fieldparams/mainnet.go +++ b/config/fieldparams/mainnet.go @@ -3,6 +3,7 @@ package field_params const ( + Preset = "mainnet" BlockRootsLength = 8192 // SLOTS_PER_HISTORICAL_ROOT StateRootsLength = 8192 // SLOTS_PER_HISTORICAL_ROOT RandaoMixesLength = 65536 // EPOCHS_PER_HISTORICAL_VECTOR diff --git a/config/fieldparams/mainnet_test.go b/config/fieldparams/mainnet_test.go index 1b2487710..dce9d0378 100644 --- a/config/fieldparams/mainnet_test.go +++ b/config/fieldparams/mainnet_test.go @@ -5,10 +5,13 @@ package field_params_test import ( "testing" + fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams" "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/testing/assert" ) func TestFieldParametersValues(t *testing.T) { params.UseMainnetConfig() + assert.Equal(t, "mainnet", fieldparams.Preset) testFieldParametersMatchConfig(t) } diff --git a/config/fieldparams/minimal.go b/config/fieldparams/minimal.go index 26832f7f4..12d78abff 100644 --- a/config/fieldparams/minimal.go +++ b/config/fieldparams/minimal.go @@ -3,6 +3,7 @@ package field_params const ( + Preset = "minimal" BlockRootsLength = 64 // SLOTS_PER_HISTORICAL_ROOT StateRootsLength = 64 // SLOTS_PER_HISTORICAL_ROOT RandaoMixesLength = 64 // EPOCHS_PER_HISTORICAL_VECTOR diff --git a/config/fieldparams/minimal_test.go b/config/fieldparams/minimal_test.go index b59426eea..0b46bab1f 100644 --- a/config/fieldparams/minimal_test.go +++ b/config/fieldparams/minimal_test.go @@ -5,10 +5,13 @@ package field_params_test import ( "testing" + fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams" "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/testing/assert" ) func TestFieldParametersValues(t *testing.T) { params.UseMinimalConfig() + assert.Equal(t, "minimal", fieldparams.Preset) testFieldParametersMatchConfig(t) } diff --git a/config/params/testnet_config_test.go b/config/params/testnet_config_test.go index 89e234d98..34555239e 100644 --- a/config/params/testnet_config_test.go +++ b/config/params/testnet_config_test.go @@ -2,9 +2,13 @@ package params_test import ( "path" + "path/filepath" "testing" "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/io/file" + "github.com/prysmaticlabs/prysm/testing/assert" "github.com/prysmaticlabs/prysm/testing/require" ) @@ -14,3 +18,16 @@ func testnetConfigFilePath(t *testing.T, network string) string { configFilePath := path.Join(filepath, "shared", network, "config.yaml") return configFilePath } + +func TestE2EConfigParity(t *testing.T) { + params.SetupTestConfigCleanup(t) + testDir := bazel.TestTmpDir() + yamlDir := filepath.Join(testDir, "config.yaml") + + testCfg := params.E2EMainnetTestConfig() + yamlObj := params.E2EMainnetConfigYaml() + assert.NoError(t, file.WriteFile(yamlDir, yamlObj)) + + params.LoadChainConfigFile(yamlDir) + assert.DeepEqual(t, params.BeaconConfig(), testCfg) +} diff --git a/config/params/testnet_e2e_config.go b/config/params/testnet_e2e_config.go index 049199cd3..41cb3346a 100644 --- a/config/params/testnet_e2e_config.go +++ b/config/params/testnet_e2e_config.go @@ -1,5 +1,10 @@ package params +import ( + "fmt" + "strings" +) + const altairE2EForkEpoch = 6 // UseE2EConfig for beacon chain services. @@ -10,6 +15,14 @@ func UseE2EConfig() { OverrideBeaconNetworkConfig(cfg) } +// UseE2EMainnetConfig for beacon chain services. +func UseE2EMainnetConfig() { + beaconConfig = E2EMainnetTestConfig() + + cfg := BeaconNetworkConfig().Copy() + OverrideBeaconNetworkConfig(cfg) +} + // E2ETestConfig retrieves the configurations made specifically for E2E testing. // Warning: This config is only for testing, it is not meant for use outside of E2E. func E2ETestConfig() *BeaconChainConfig { @@ -42,3 +55,66 @@ func E2ETestConfig() *BeaconChainConfig { return e2eConfig } + +func E2EMainnetTestConfig() *BeaconChainConfig { + e2eConfig := MainnetConfig().Copy() + + // Misc. + e2eConfig.MinGenesisActiveValidatorCount = 256 + e2eConfig.GenesisDelay = 25 // 25 seconds so E2E has enough time to process deposits and get started. + e2eConfig.ChurnLimitQuotient = 65536 + + // Time parameters. + e2eConfig.SecondsPerSlot = 4 + e2eConfig.SqrRootSlotsPerEpoch = 5 + e2eConfig.SecondsPerETH1Block = 2 + e2eConfig.Eth1FollowDistance = 4 + e2eConfig.ShardCommitteePeriod = 4 + + // PoW parameters. + e2eConfig.DepositChainID = 1337 // Chain ID of eth1 dev net. + e2eConfig.DepositNetworkID = 1337 // Network ID of eth1 dev net. + + // Altair Fork Parameters. + e2eConfig.AltairForkEpoch = altairE2EForkEpoch + + // Prysm constants. + e2eConfig.ConfigName = ConfigNames[EndToEnd] + + return e2eConfig +} + +// E2EMainnetConfigYaml returns the e2e config in yaml format. +func E2EMainnetConfigYaml() []byte { + cfg := E2EMainnetTestConfig() + lines := []string{} + lines = append(lines, fmt.Sprintf("PRESET_BASE: '%s'", cfg.PresetBase)) + lines = append(lines, fmt.Sprintf("CONFIG_NAME: '%s'", cfg.ConfigName)) + lines = append(lines, fmt.Sprintf("MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: %d", cfg.MinGenesisActiveValidatorCount)) + lines = append(lines, fmt.Sprintf("GENESIS_DELAY: %d", cfg.GenesisDelay)) + lines = append(lines, fmt.Sprintf("MIN_GENESIS_TIME: %d", cfg.MinGenesisTime)) + lines = append(lines, fmt.Sprintf("GENESIS_FORK_VERSION: %#x", cfg.GenesisForkVersion)) + lines = append(lines, fmt.Sprintf("CHURN_LIMIT_QUOTIENT: %d", cfg.ChurnLimitQuotient)) + lines = append(lines, fmt.Sprintf("SECONDS_PER_SLOT: %d", cfg.SecondsPerSlot)) + lines = append(lines, fmt.Sprintf("SLOTS_PER_EPOCH: %d", cfg.SlotsPerEpoch)) + lines = append(lines, fmt.Sprintf("SECONDS_PER_ETH1_BLOCK: %d", cfg.SecondsPerETH1Block)) + lines = append(lines, fmt.Sprintf("ETH1_FOLLOW_DISTANCE: %d", cfg.Eth1FollowDistance)) + lines = append(lines, fmt.Sprintf("EPOCHS_PER_ETH1_VOTING_PERIOD: %d", cfg.EpochsPerEth1VotingPeriod)) + lines = append(lines, fmt.Sprintf("SHARD_COMMITTEE_PERIOD: %d", cfg.ShardCommitteePeriod)) + lines = append(lines, fmt.Sprintf("MIN_VALIDATOR_WITHDRAWABILITY_DELAY: %d", cfg.MinValidatorWithdrawabilityDelay)) + lines = append(lines, fmt.Sprintf("MAX_SEED_LOOKAHEAD: %d", cfg.MaxSeedLookahead)) + lines = append(lines, fmt.Sprintf("EJECTION_BALANCE: %d", cfg.EjectionBalance)) + lines = append(lines, fmt.Sprintf("MIN_PER_EPOCH_CHURN_LIMIT: %d", cfg.MinPerEpochChurnLimit)) + lines = append(lines, fmt.Sprintf("DEPOSIT_CHAIN_ID: %d", cfg.DepositChainID)) + lines = append(lines, fmt.Sprintf("DEPOSIT_NETWORK_ID: %d", cfg.DepositNetworkID)) + lines = append(lines, fmt.Sprintf("ALTAIR_FORK_EPOCH: %d", cfg.AltairForkEpoch)) + lines = append(lines, fmt.Sprintf("ALTAIR_FORK_VERSION: %#x", cfg.AltairForkVersion)) + lines = append(lines, fmt.Sprintf("INACTIVITY_SCORE_BIAS: %d", cfg.InactivityScoreBias)) + lines = append(lines, fmt.Sprintf("INACTIVITY_SCORE_RECOVERY_RATE: %d", cfg.InactivityScoreRecoveryRate)) + lines = append(lines, fmt.Sprintf("TERMINAL_TOTAL_DIFFICULTY: %d", cfg.TerminalTotalDifficulty)) + lines = append(lines, fmt.Sprintf("TERMINAL_BLOCK_HASH: %#x", cfg.TerminalBlockHash)) + lines = append(lines, fmt.Sprintf("TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: %d", cfg.TerminalBlockHashActivationEpoch)) + + yamlFile := []byte(strings.Join(lines, "\n")) + return yamlFile +} diff --git a/testing/endtoend/BUILD.bazel b/testing/endtoend/BUILD.bazel index 90c2f40b9..0427cbb41 100644 --- a/testing/endtoend/BUILD.bazel +++ b/testing/endtoend/BUILD.bazel @@ -1,7 +1,7 @@ load("@prysm//tools/go:def.bzl", "go_test") # gazelle:exclude geth_deps.go - +# gazelle:exclude mainnet_e2e_test.go go_test( name = "go_default_test", size = "large", @@ -56,3 +56,56 @@ go_test( "@org_golang_x_sync//errgroup:go_default_library", ], ) + +go_test( + name = "go_mainnet_test", + size = "large", + testonly = True, + srcs = [ + "endtoend_test.go", + "mainnet_e2e_test.go", + ], + args = ["-test.v"], + data = [ + "//:prysm_sh", + "//cmd/beacon-chain", + "//cmd/validator", + "//tools/bootnode", + "@com_github_ethereum_go_ethereum//cmd/geth", + "@web3signer", + ], + eth_network = "mainnet", + shard_count = 2, + tags = [ + "mainnet", + "manual", + "multiclient", + "requires-network", + ], + deps = [ + "//beacon-chain/blockchain/testing:go_default_library", + "//beacon-chain/core/transition:go_default_library", + "//beacon-chain/db/testing:go_default_library", + "//beacon-chain/operations/slashings:go_default_library", + "//beacon-chain/state/stategen:go_default_library", + "//config/params:go_default_library", + "//crypto/bls:go_default_library", + "//proto/prysm/v1alpha1:go_default_library", + "//testing/assert:go_default_library", + "//testing/endtoend/components:go_default_library", + "//testing/endtoend/evaluators:go_default_library", + "//testing/endtoend/helpers:go_default_library", + "//testing/endtoend/params:go_default_library", + "//testing/endtoend/types:go_default_library", + "//testing/require:go_default_library", + "//testing/slasher/simulator:go_default_library", + "//testing/util:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@com_github_prysmaticlabs_eth2_types//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_sirupsen_logrus//hooks/test:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_protobuf//types/known/emptypb:go_default_library", + "@org_golang_x_sync//errgroup:go_default_library", + ], +) diff --git a/testing/endtoend/components/BUILD.bazel b/testing/endtoend/components/BUILD.bazel index e5950a78e..3b099c4aa 100644 --- a/testing/endtoend/components/BUILD.bazel +++ b/testing/endtoend/components/BUILD.bazel @@ -7,11 +7,14 @@ go_library( "beacon_node.go", "boot_node.go", "eth1.go", + "lighthouse_beacon.go", + "lighthouse_validator.go", "log.go", "tracing_sink.go", "validator.go", "web3remotesigner.go", ], + data = ["@lighthouse//:lighthouse_bin"], importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components", visibility = ["//testing/endtoend:__subpackages__"], deps = [ @@ -24,19 +27,23 @@ go_library( "//contracts/deposit/mock:go_default_library", "//crypto/bls:go_default_library", "//encoding/bytesutil:go_default_library", + "//io/file:go_default_library", "//runtime/interop:go_default_library", "//testing/endtoend/helpers:go_default_library", "//testing/endtoend/params:go_default_library", "//testing/endtoend/types:go_default_library", "//testing/util:go_default_library", + "//validator/keymanager:go_default_library", "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library", "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library", "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library", "@com_github_ethereum_go_ethereum//core/types:go_default_library", "@com_github_ethereum_go_ethereum//ethclient:go_default_library", "@com_github_ethereum_go_ethereum//rpc:go_default_library", + "@com_github_google_uuid//:go_default_library", "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4//:go_default_library", "@in_gopkg_yaml_v2//:go_default_library", "@io_bazel_rules_go//go/tools/bazel:go_default_library", ], diff --git a/testing/endtoend/components/beacon_node.go b/testing/endtoend/components/beacon_node.go index 498ce0868..c29d5fa29 100644 --- a/testing/endtoend/components/beacon_node.go +++ b/testing/endtoend/components/beacon_node.go @@ -29,6 +29,7 @@ type BeaconNodeSet struct { e2etypes.ComponentRunner config *e2etypes.E2EConfig enr string + ids []string started chan struct{} } @@ -60,6 +61,12 @@ func (s *BeaconNodeSet) Start(ctx context.Context) error { // Wait for all nodes to finish their job (blocking). // Once nodes are ready passed in handler function will be called. return helpers.WaitOnNodes(ctx, nodes, func() { + if s.config.UseFixedPeerIDs { + for i := 0; i < len(nodes); i++ { + s.ids = append(s.ids, nodes[i].(*BeaconNode).peerID) + } + s.config.PeerIDs = s.ids + } // All nodes stated, close channel, so that all services waiting on a set, can proceed. close(s.started) }) @@ -77,6 +84,7 @@ type BeaconNode struct { started chan struct{} index int enr string + peerID string } // NewBeaconNode creates and returns a beacon node. @@ -102,6 +110,7 @@ func (node *BeaconNode) Start(ctx context.Context) error { if err != nil { return err } + expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount - 1 args := []string{ fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index), @@ -112,6 +121,7 @@ func (node *BeaconNode) Start(ctx context.Context) error { fmt.Sprintf("--%s=%d", flags.MinSyncPeers.Name, e2e.TestParams.BeaconNodeCount-1), fmt.Sprintf("--%s=%d", cmdshared.P2PUDPPort.Name, e2e.TestParams.BeaconNodeRPCPort+index+10), fmt.Sprintf("--%s=%d", cmdshared.P2PTCPPort.Name, e2e.TestParams.BeaconNodeRPCPort+index+20), + fmt.Sprintf("--%s=%d", cmdshared.P2PMaxPeers.Name, expectedNumOfPeers), fmt.Sprintf("--%s=%d", flags.MonitoringPortFlag.Name, e2e.TestParams.BeaconNodeMetricsPort+index), fmt.Sprintf("--%s=%d", flags.GRPCGatewayPort.Name, e2e.TestParams.BeaconNodeRPCPort+index+40), fmt.Sprintf("--%s=%d", flags.ContractDeploymentBlock.Name, 0), @@ -158,6 +168,14 @@ func (node *BeaconNode) Start(ctx context.Context) error { return fmt.Errorf("could not find multiaddr for node %d, this means the node had issues starting: %w", index, err) } + if config.UseFixedPeerIDs { + peerId, err := helpers.FindFollowingTextInFile(stdOutFile, "Running node with peer id of ") + if err != nil { + return fmt.Errorf("could not find peer id: %w", err) + } + node.peerID = peerId + } + // Mark node as ready. close(node.started) diff --git a/testing/endtoend/components/lighthouse_beacon.go b/testing/endtoend/components/lighthouse_beacon.go new file mode 100644 index 000000000..60ffe31d4 --- /dev/null +++ b/testing/endtoend/components/lighthouse_beacon.go @@ -0,0 +1,187 @@ +package components + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/io/file" + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" +) + +var _ e2etypes.ComponentRunner = (*LighthouseBeaconNode)(nil) + +// LighthouseBeaconNodeSet represents set of lighthouse beacon nodes. +type LighthouseBeaconNodeSet struct { + e2etypes.ComponentRunner + config *e2etypes.E2EConfig + enr string + started chan struct{} +} + +// SetENR assigns ENR to the set of beacon nodes. +func (s *LighthouseBeaconNodeSet) SetENR(enr string) { + s.enr = enr +} + +// NewLighthouseBeaconNodes creates and returns a set of lighthouse beacon nodes. +func NewLighthouseBeaconNodes(config *e2etypes.E2EConfig) *LighthouseBeaconNodeSet { + return &LighthouseBeaconNodeSet{ + config: config, + started: make(chan struct{}, 1), + } +} + +// Start starts all the beacon nodes in set. +func (s *LighthouseBeaconNodeSet) Start(ctx context.Context) error { + if s.enr == "" { + return errors.New("empty ENR") + } + + // Create beacon nodes. + nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.LighthouseBeaconNodeCount) + for i := 0; i < e2e.TestParams.LighthouseBeaconNodeCount; i++ { + nodes[i] = NewLighthouseBeaconNode(s.config, i, s.enr) + } + + // Wait for all nodes to finish their job (blocking). + // Once nodes are ready passed in handler function will be called. + return helpers.WaitOnNodes(ctx, nodes, func() { + // All nodes stated, close channel, so that all services waiting on a set, can proceed. + close(s.started) + }) +} + +// Started checks whether beacon node set is started and all nodes are ready to be queried. +func (s *LighthouseBeaconNodeSet) Started() <-chan struct{} { + return s.started +} + +// LighthouseBeaconNode represents a lighthouse beacon node. +type LighthouseBeaconNode struct { + e2etypes.ComponentRunner + config *e2etypes.E2EConfig + started chan struct{} + index int + enr string +} + +// NewBeaconNode creates and returns a beacon node. +func NewLighthouseBeaconNode(config *e2etypes.E2EConfig, index int, enr string) *LighthouseBeaconNode { + return &LighthouseBeaconNode{ + config: config, + index: index, + enr: enr, + started: make(chan struct{}, 1), + } +} + +// Start starts a fresh beacon node, connecting to all passed in beacon nodes. +func (node *LighthouseBeaconNode) Start(ctx context.Context) error { + binaryPath, found := bazel.FindBinary("external/lighthouse", "lighthouse") + if !found { + log.Info(binaryPath) + log.Error("beacon chain binary not found") + } + + _, index, _ := node.config, node.index, node.enr + testDir, err := node.createTestnetDir(index) + if err != nil { + return err + } + + args := []string{ + "beacon_node", + fmt.Sprintf("--datadir=%s/lighthouse-beacon-node-%d", e2e.TestParams.TestPath, index), + fmt.Sprintf("--testnet-dir=%s", testDir), + "--staking", + "--enr-address=127.0.0.1", + fmt.Sprintf("--enr-udp-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+200), + fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+200), + fmt.Sprintf("--port=%d", e2e.TestParams.BeaconNodeRPCPort+index+200), + fmt.Sprintf("--http-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+250), + fmt.Sprintf("--target-peers=%d", 10), + fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Eth1RPCPort), + fmt.Sprintf("--boot-nodes=%s", node.enr), + fmt.Sprintf("--metrics-port=%d", e2e.TestParams.BeaconNodeMetricsPort+index+300), + "--metrics", + "--http", + "--debug-level=debug", + } + if node.config.UseFixedPeerIDs { + flagVal := strings.Join(node.config.PeerIDs, ",") + args = append(args, + fmt.Sprintf("--trusted-peers=%s", flagVal)) + } + cmd := exec.CommandContext(ctx, binaryPath, args...) /* #nosec G204 */ + // Write stdout and stderr to log files. + stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stdout.log", index))) + if err != nil { + return err + } + stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stderr.log", index))) + if err != nil { + return err + } + defer func() { + if err := stdout.Close(); err != nil { + log.WithError(err).Error("Failed to close stdout file") + } + if err := stderr.Close(); err != nil { + log.WithError(err).Error("Failed to close stderr file") + } + }() + cmd.Stdout = stdout + cmd.Stderr = stderr + log.Infof("Starting lighthouse beacon chain %d with flags: %s", index, strings.Join(args[2:], " ")) + if err = cmd.Start(); err != nil { + return fmt.Errorf("failed to start beacon node: %w", err) + } + + if err = helpers.WaitForTextInFile(stderr, "Configured for network"); err != nil { + return fmt.Errorf("could not find initialization for node %d, this means the node had issues starting: %w", index, err) + } + + // Mark node as ready. + close(node.started) + + return cmd.Wait() +} + +// Started checks whether beacon node is started and ready to be queried. +func (node *LighthouseBeaconNode) Started() <-chan struct{} { + return node.started +} + +func (node *LighthouseBeaconNode) createTestnetDir(index int) (string, error) { + testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/lighthouse-testnet-%d", index) + configPath := filepath.Join(testNetDir, "config.yaml") + rawYaml := params.E2EMainnetConfigYaml() + // Add in deposit contract in yaml + depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", e2e.TestParams.ContractAddress) + rawYaml = append(rawYaml, []byte(depContractStr)...) + + if err := file.MkdirAll(testNetDir); err != nil { + return "", err + } + if err := file.WriteFile(configPath, rawYaml); err != nil { + return "", err + } + bootPath := filepath.Join(testNetDir, "boot_enr.yaml") + enrYaml := []byte(fmt.Sprintf("[%s]", node.enr)) + if err := file.WriteFile(bootPath, enrYaml); err != nil { + return "", err + } + deployPath := filepath.Join(testNetDir, "deploy_block.txt") + deployYaml := []byte("0") + return testNetDir, file.WriteFile(deployPath, deployYaml) +} diff --git a/testing/endtoend/components/lighthouse_validator.go b/testing/endtoend/components/lighthouse_validator.go new file mode 100644 index 000000000..a78a9bd74 --- /dev/null +++ b/testing/endtoend/components/lighthouse_validator.go @@ -0,0 +1,255 @@ +package components + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/io/file" + "github.com/prysmaticlabs/prysm/runtime/interop" + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" + "github.com/prysmaticlabs/prysm/validator/keymanager" + keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4" +) + +var _ e2etypes.ComponentRunner = (*LighthouseValidatorNode)(nil) +var _ e2etypes.ComponentRunner = (*LighthouseValidatorNodeSet)(nil) + +// LighthouseValidatorNodeSet represents set of lighthouse validator nodes. +type LighthouseValidatorNodeSet struct { + e2etypes.ComponentRunner + config *e2etypes.E2EConfig + started chan struct{} +} + +// NewLighthouseValidatorNodeSet creates and returns a set of lighthouse validator nodes. +func NewLighthouseValidatorNodeSet(config *e2etypes.E2EConfig) *LighthouseValidatorNodeSet { + return &LighthouseValidatorNodeSet{ + config: config, + started: make(chan struct{}, 1), + } +} + +// Start starts the configured amount of validators, also sending and mining their deposits. +func (s *LighthouseValidatorNodeSet) Start(ctx context.Context) error { + // Always using genesis count since using anything else would be difficult to test for. + validatorNum := int(params.BeaconConfig().MinGenesisActiveValidatorCount) + lighthouseBeaconNum := e2e.TestParams.LighthouseBeaconNodeCount + prysmBeaconNum := e2e.TestParams.BeaconNodeCount + beaconNodeNum := lighthouseBeaconNum + prysmBeaconNum + if validatorNum%beaconNodeNum != 0 { + return errors.New("validator count is not easily divisible by beacon node count") + } + validatorsPerNode := validatorNum / beaconNodeNum + + // Create validator nodes. + nodes := make([]e2etypes.ComponentRunner, lighthouseBeaconNum) + for i := 0; i < lighthouseBeaconNum; i++ { + offsetIdx := i + prysmBeaconNum + nodes[i] = NewLighthouseValidatorNode(s.config, validatorsPerNode, i, validatorsPerNode*offsetIdx) + } + + // Wait for all nodes to finish their job (blocking). + // Once nodes are ready passed in handler function will be called. + return helpers.WaitOnNodes(ctx, nodes, func() { + // All nodes stated, close channel, so that all services waiting on a set, can proceed. + close(s.started) + }) +} + +// Started checks whether validator node set is started and all nodes are ready to be queried. +func (s *LighthouseValidatorNodeSet) Started() <-chan struct{} { + return s.started +} + +// LighthouseValidatorNode represents a lighthouse validator node. +type LighthouseValidatorNode struct { + e2etypes.ComponentRunner + config *e2etypes.E2EConfig + started chan struct{} + validatorNum int + index int + offset int +} + +// NewLighthouseValidatorNode creates and returns a lighthouse validator node. +func NewLighthouseValidatorNode(config *e2etypes.E2EConfig, validatorNum, index, offset int) *LighthouseValidatorNode { + return &LighthouseValidatorNode{ + config: config, + validatorNum: validatorNum, + index: index, + offset: offset, + started: make(chan struct{}, 1), + } +} + +// Start starts a validator client. +func (v *LighthouseValidatorNode) Start(ctx context.Context) error { + binaryPath, found := bazel.FindBinary("external/lighthouse", "lighthouse") + if !found { + log.Info(binaryPath) + log.Error("validator binary not found") + } + + _, _, index, _ := v.config, v.validatorNum, v.index, v.offset + beaconRPCPort := e2e.TestParams.BeaconNodeRPCPort + index + if beaconRPCPort >= e2e.TestParams.BeaconNodeRPCPort+e2e.TestParams.BeaconNodeCount { + // Point any extra validator clients to a node we know is running. + beaconRPCPort = e2e.TestParams.BeaconNodeRPCPort + } + kPath := e2e.TestParams.TestPath + fmt.Sprintf("/lighthouse-validator-%d", index) + testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/lighthouse-testnet-%d", index) + args := []string{ + "validator_client", + "--debug-level=debug", + "--init-slashing-protection", + fmt.Sprintf("--datadir=%s", kPath), + fmt.Sprintf("--testnet-dir=%s", testNetDir), + fmt.Sprintf("--beacon-nodes=http://localhost:%d", e2e.TestParams.BeaconNodeRPCPort+index+250), + } + + cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe + + // Write stdout and stderr to log files. + stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_validator_%d_stdout.log", index))) + if err != nil { + return err + } + stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_validator_%d_stderr.log", index))) + if err != nil { + return err + } + defer func() { + if err := stdout.Close(); err != nil { + log.WithError(err).Error("Failed to close stdout file") + } + if err := stderr.Close(); err != nil { + log.WithError(err).Error("Failed to close stderr file") + } + }() + cmd.Stdout = stdout + cmd.Stderr = stderr + + log.Infof("Starting lighthouse validator client %d with flags: %s %s", index, binaryPath, strings.Join(args, " ")) + if err = cmd.Start(); err != nil { + return err + } + + // Mark node as ready. + close(v.started) + + return cmd.Wait() +} + +// Started checks whether validator node is started and ready to be queried. +func (v *LighthouseValidatorNode) Started() <-chan struct{} { + return v.started +} + +type KeystoreGenerator struct { + started chan struct{} +} + +func NewKeystoreGenerator() *KeystoreGenerator { + return &KeystoreGenerator{started: make(chan struct{})} +} + +func (k *KeystoreGenerator) Start(ctx context.Context) error { + validatorNum := int(params.BeaconConfig().MinGenesisActiveValidatorCount) + lighthouseBeaconNum := e2e.TestParams.LighthouseBeaconNodeCount + prysmBeaconNum := e2e.TestParams.BeaconNodeCount + beaconNodeNum := lighthouseBeaconNum + prysmBeaconNum + if validatorNum%beaconNodeNum != 0 { + return errors.New("validator count is not easily divisible by beacon node count") + } + validatorsPerNode := validatorNum / beaconNodeNum + + for i := 0; i < lighthouseBeaconNum; i++ { + offsetIdx := i + prysmBeaconNum + _, err := setupKeystores(i, validatorsPerNode*offsetIdx, validatorsPerNode) + if err != nil { + return err + } + log.Infof("Generated lighthouse keystores from %d onwards with %d keys", validatorsPerNode*offsetIdx, validatorsPerNode) + } + // Mark component as ready. + close(k.started) + return nil +} + +func (k *KeystoreGenerator) Started() <-chan struct{} { + return k.started +} + +func setupKeystores(valClientIdx, startIdx, numOfKeys int) (string, error) { + testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/lighthouse-validator-%d", valClientIdx) + if err := file.MkdirAll(testNetDir); err != nil { + return "", err + } + secretsPath := filepath.Join(testNetDir, "secrets") + validatorKeystorePath := filepath.Join(testNetDir, "validators") + if err := file.MkdirAll(secretsPath); err != nil { + return "", err + } + if err := file.MkdirAll(validatorKeystorePath); err != nil { + return "", err + } + privKeys, pubKeys, err := interop.DeterministicallyGenerateKeys(uint64(startIdx), uint64(numOfKeys)) + if err != nil { + return "", err + } + encryptor := keystorev4.New() + // Use lighthouse's default password for their insecure keystores. + password := "222222222222222222222222222222222222222222222222222" + for i, pk := range pubKeys { + pubKeyBytes := pk.Marshal() + cryptoFields, err := encryptor.Encrypt(privKeys[i].Marshal(), password) + if err != nil { + return "", errors.Wrapf( + err, + "could not encrypt secret key for public key %#x", + pubKeyBytes, + ) + } + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + kStore := &keymanager.Keystore{ + Crypto: cryptoFields, + ID: id.String(), + Pubkey: fmt.Sprintf("%x", pubKeyBytes), + Version: encryptor.Version(), + Name: encryptor.Name(), + } + + fPath := filepath.Join(secretsPath, "0x"+kStore.Pubkey) + if err := file.WriteFile(fPath, []byte(password)); err != nil { + return "", err + } + keystorePath := filepath.Join(validatorKeystorePath, "0x"+kStore.Pubkey) + if err := file.MkdirAll(keystorePath); err != nil { + return "", err + } + fPath = filepath.Join(keystorePath, "voting-keystore.json") + encodedFile, err := json.MarshalIndent(kStore, "", "\t") + if err != nil { + return "", errors.Wrap(err, "could not marshal keystore to JSON file") + } + if err := file.WriteFile(fPath, encodedFile); err != nil { + return "", err + } + } + return testNetDir, nil +} diff --git a/testing/endtoend/components/validator.go b/testing/endtoend/components/validator.go index f2b6f1026..70a247d6d 100644 --- a/testing/endtoend/components/validator.go +++ b/testing/endtoend/components/validator.go @@ -55,15 +55,16 @@ func NewValidatorNodeSet(config *e2etypes.E2EConfig) *ValidatorNodeSet { func (s *ValidatorNodeSet) Start(ctx context.Context) error { // Always using genesis count since using anything else would be difficult to test for. validatorNum := int(params.BeaconConfig().MinGenesisActiveValidatorCount) - beaconNodeNum := e2e.TestParams.BeaconNodeCount + prysmBeaconNodeNum := e2e.TestParams.BeaconNodeCount + beaconNodeNum := prysmBeaconNodeNum + e2e.TestParams.LighthouseBeaconNodeCount if validatorNum%beaconNodeNum != 0 { return errors.New("validator count is not easily divisible by beacon node count") } validatorsPerNode := validatorNum / beaconNodeNum // Create validator nodes. - nodes := make([]e2etypes.ComponentRunner, beaconNodeNum) - for i := 0; i < beaconNodeNum; i++ { + nodes := make([]e2etypes.ComponentRunner, prysmBeaconNodeNum) + for i := 0; i < prysmBeaconNodeNum; i++ { nodes[i] = NewValidatorNode(s.config, validatorsPerNode, i, validatorsPerNode*i) } diff --git a/testing/endtoend/deps.bzl b/testing/endtoend/deps.bzl index fffac14d6..053e6beb8 100644 --- a/testing/endtoend/deps.bzl +++ b/testing/endtoend/deps.bzl @@ -1,5 +1,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # gazelle:keep +lighthouse_version = "v2.1.2" +lighthouse_archive_name = "lighthouse-%s-x86_64-unknown-linux-gnu-portable.tar.gz" % lighthouse_version + def e2e_deps(): http_archive( name = "web3signer", @@ -8,3 +11,10 @@ def e2e_deps(): build_file = "@prysm//testing/endtoend:web3signer.BUILD", strip_prefix = "web3signer-21.10.5", ) + + http_archive( + name = "lighthouse", + sha256 = "8a83ba0f7c24cc4e5b588e7a09bb4e5a1f919346ccf7000d3409a3690a85b221", + build_file = "@prysm//testing/endtoend:lighthouse.BUILD", + url = ("https://github.com/sigp/lighthouse/releases/download/%s/" + lighthouse_archive_name) % lighthouse_version, + ) diff --git a/testing/endtoend/endtoend_test.go b/testing/endtoend/endtoend_test.go index b381704ab..70cb78254 100644 --- a/testing/endtoend/endtoend_test.go +++ b/testing/endtoend/endtoend_test.go @@ -67,6 +67,9 @@ func (r *testRunner) run() { t.Logf("Log Path: %s\n", e2e.TestParams.LogPath) minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount) + multiClientActive := e2e.TestParams.LighthouseBeaconNodeCount > 0 + var keyGen, lighthouseValidatorNodes e2etypes.ComponentRunner + var lighthouseNodes *components.LighthouseBeaconNodeSet ctx, done := context.WithCancel(context.Background()) g, ctx := errgroup.WithContext(ctx) @@ -76,6 +79,15 @@ func (r *testRunner) run() { return tracingSink.Start(ctx) }) + if multiClientActive { + keyGen = components.NewKeystoreGenerator() + + // Generate lighthouse keystores. + g.Go(func() error { + return keyGen.Start(ctx) + }) + } + // ETH1 node. eth1Node := components.NewEth1Node() g.Go(func() error { @@ -102,7 +114,6 @@ func (r *testRunner) run() { } return nil }) - // Beacon nodes. beaconNodes := components.NewBeaconNodes(config) g.Go(func() error { @@ -128,6 +139,19 @@ func (r *testRunner) run() { }) } + if multiClientActive { + lighthouseNodes = components.NewLighthouseBeaconNodes(config) + g.Go(func() error { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node, bootNode, beaconNodes}); err != nil { + return errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run") + } + lighthouseNodes.SetENR(bootNode.ENR()) + if err := lighthouseNodes.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start lighthouse beacon nodes") + } + return nil + }) + } // Validator nodes. validatorNodes := components.NewValidatorNodeSet(config) g.Go(func() error { @@ -144,6 +168,20 @@ func (r *testRunner) run() { return nil }) + if multiClientActive { + // Lighthouse Validator nodes. + lighthouseValidatorNodes = components.NewLighthouseValidatorNodeSet(config) + g.Go(func() error { + if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{keyGen, lighthouseNodes}); err != nil { + return errors.Wrap(err, "validator nodes require beacon nodes to run") + } + if err := lighthouseValidatorNodes.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start validator nodes") + } + return nil + }) + } + // Run E2E evaluators and tests. g.Go(func() error { // When everything is done, cancel parent context (will stop all spawned nodes). @@ -156,6 +194,9 @@ func (r *testRunner) run() { requiredComponents := []e2etypes.ComponentRunner{ tracingSink, eth1Node, bootNode, beaconNodes, validatorNodes, } + if multiClientActive { + requiredComponents = append(requiredComponents, []e2etypes.ComponentRunner{keyGen, lighthouseNodes, lighthouseValidatorNodes}...) + } ctxAllNodesReady, cancel := context.WithTimeout(ctx, allNodesStartTimeout) defer cancel() if err := helpers.ComponentsStarted(ctxAllNodesReady, requiredComponents); err != nil { diff --git a/testing/endtoend/evaluators/api_gateway_v1alpha1.go b/testing/endtoend/evaluators/api_gateway_v1alpha1.go index 3ab7ebce8..a802d1eb4 100644 --- a/testing/endtoend/evaluators/api_gateway_v1alpha1.go +++ b/testing/endtoend/evaluators/api_gateway_v1alpha1.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1" e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" "github.com/prysmaticlabs/prysm/testing/endtoend/policies" @@ -73,6 +74,7 @@ func withComparePeers(beaconNodeIdx int, conn *grpc.ClientConn) error { ); err != nil { return err } + if len(respJSON.Peers) != len(resp.Peers) { return fmt.Errorf( "HTTP gateway number of peers %d does not match gRPC %d", @@ -80,44 +82,54 @@ func withComparePeers(beaconNodeIdx int, conn *grpc.ClientConn) error { len(resp.Peers), ) } - for i, peer := range respJSON.Peers { - grpcPeer := resp.Peers[i] + grpcPeerMap := make(map[string]*ethpb.Peer) + jsonPeerMap := make(map[string]*peerJSON) + for i := 0; i < len(respJSON.Peers); i++ { + grpcPeerMap[resp.Peers[i].PeerId] = resp.Peers[i] + jsonPeerMap[respJSON.Peers[i].PeerId] = respJSON.Peers[i] + } + + for id, peer := range jsonPeerMap { + grpcPeer, ok := grpcPeerMap[id] + if !ok { + return errors.Errorf("grpc peer %s doesn't exist", id) + } if peer.Address != grpcPeer.Address { return fmt.Errorf( - "HTTP gateway peer %d address %s does not match gRPC %s", - i, + "HTTP gateway peer %s with address %s does not match gRPC %s", + id, peer.Address, grpcPeer.Address, ) } if peer.Direction != grpcPeer.Direction.String() { return fmt.Errorf( - "HTTP gateway peer %d direction %s does not match gRPC %s", - i, + "HTTP gateway peer %s with direction %s does not match gRPC %s", + id, peer.Direction, grpcPeer.Direction, ) } if peer.ConnectionState != grpcPeer.ConnectionState.String() { return fmt.Errorf( - "HTTP gateway peer %d connection state %s does not match gRPC %s", - i, + "HTTP gateway peer %s with connection state %s does not match gRPC %s", + id, peer.ConnectionState, grpcPeer.ConnectionState, ) } if peer.PeerId != grpcPeer.PeerId { return fmt.Errorf( - "HTTP gateway peer %d peer id %s does not match gRPC %s", - i, + "HTTP gateway peer %s with peer id %s does not match gRPC %s", + id, peer.PeerId, grpcPeer.PeerId, ) } if peer.Enr != grpcPeer.Enr { return fmt.Errorf( - "HTTP gateway peer %d enr %s does not match gRPC %s", - i, + "HTTP gateway peer %s with enr %s does not match gRPC %s", + id, peer.Enr, grpcPeer.Enr, ) diff --git a/testing/endtoend/evaluators/node.go b/testing/endtoend/evaluators/node.go index 42bcc1720..30b38283c 100644 --- a/testing/endtoend/evaluators/node.go +++ b/testing/endtoend/evaluators/node.go @@ -105,7 +105,7 @@ func peersConnect(conns ...*grpc.ClientConn) error { if err != nil { return err } - expectedPeers := len(conns) - 1 + expectedPeers := len(conns) - 1 + e2e.TestParams.LighthouseBeaconNodeCount if expectedPeers != len(peersResp.Peers) { return fmt.Errorf("unexpected amount of peers, expected %d, received %d", expectedPeers, len(peersResp.Peers)) } diff --git a/testing/endtoend/evaluators/operations.go b/testing/endtoend/evaluators/operations.go index 4c21babc6..5a2eb1191 100644 --- a/testing/endtoend/evaluators/operations.go +++ b/testing/endtoend/evaluators/operations.go @@ -381,12 +381,12 @@ func validatorsVoteWithTheMajority(conns ...*grpc.ClientConn) error { // - this evaluator is not executed for epoch 0 so we have to calculate the first slot differently // - for some reason the vote for the first slot in epoch 1 is 0x000... so we skip this slot var isFirstSlotInVotingPeriod bool - if chainHead.HeadEpoch == 1 && slot%params.E2ETestConfig().SlotsPerEpoch == 0 { + if chainHead.HeadEpoch == 1 && slot%params.BeaconConfig().SlotsPerEpoch == 0 { continue } // We skipped the first slot so we treat the second slot as the starting slot of epoch 1. if chainHead.HeadEpoch == 1 { - isFirstSlotInVotingPeriod = slot%params.E2ETestConfig().SlotsPerEpoch == 1 + isFirstSlotInVotingPeriod = slot%params.BeaconConfig().SlotsPerEpoch == 1 } else { isFirstSlotInVotingPeriod = slot%slotsPerVotingPeriod == 0 } diff --git a/testing/endtoend/helpers/helpers.go b/testing/endtoend/helpers/helpers.go index 9ac0c07f6..85e514acd 100644 --- a/testing/endtoend/helpers/helpers.go +++ b/testing/endtoend/helpers/helpers.go @@ -95,6 +95,53 @@ func WaitForTextInFile(file *os.File, text string) error { } } +// FindFollowingTextInFile checks a file every polling interval for the following text requested. +func FindFollowingTextInFile(file *os.File, text string) (string, error) { + d := time.Now().Add(maxPollingWaitTime) + ctx, cancel := context.WithDeadline(context.Background(), d) + defer cancel() + + // Use a ticker with a deadline to poll a given file. + ticker := time.NewTicker(filePollingInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + contents, err := ioutil.ReadAll(file) + if err != nil { + return "", err + } + return "", fmt.Errorf("could not find requested text \"%s\" in logs:\n%s", text, contents) + case <-ticker.C: + fileScanner := bufio.NewScanner(file) + buf := make([]byte, 0, fileBufferSize) + fileScanner.Buffer(buf, maxFileBufferSize) + for fileScanner.Scan() { + scanned := fileScanner.Text() + if strings.Contains(scanned, text) { + lastIdx := strings.LastIndex(scanned, text) + truncatedIdx := lastIdx + len(text) + if len(scanned) <= truncatedIdx { + return "", fmt.Errorf("truncated index is larger than the size of whole scanned line") + } + splitObjs := strings.Split(scanned[truncatedIdx:], " ") + if len(splitObjs) == 0 { + return "", fmt.Errorf("0 split substrings retrieved") + } + return splitObjs[0], nil + } + } + if err := fileScanner.Err(); err != nil { + return "", err + } + _, err := file.Seek(0, io.SeekStart) + if err != nil { + return "", err + } + } + } +} + // GraffitiYamlFile outputs graffiti YAML file into a testing directory. func GraffitiYamlFile(testDir string) (string, error) { b := []byte(`default: "Rice" diff --git a/testing/endtoend/lighthouse.BUILD b/testing/endtoend/lighthouse.BUILD new file mode 100644 index 000000000..6119a46d0 --- /dev/null +++ b/testing/endtoend/lighthouse.BUILD @@ -0,0 +1,7 @@ +sh_binary( + name = "lighthouse_bin", + srcs = [ + "lighthouse", + ], + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/testing/endtoend/mainnet_e2e_test.go b/testing/endtoend/mainnet_e2e_test.go new file mode 100644 index 000000000..c814c134d --- /dev/null +++ b/testing/endtoend/mainnet_e2e_test.go @@ -0,0 +1,75 @@ +package endtoend + +import ( + "fmt" + "os" + "strconv" + "testing" + + "github.com/prysmaticlabs/prysm/config/params" + ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators" + "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" + e2eParams "github.com/prysmaticlabs/prysm/testing/endtoend/params" + "github.com/prysmaticlabs/prysm/testing/endtoend/types" + "github.com/prysmaticlabs/prysm/testing/require" +) + +func TestEndToEnd_MainnetConfig(t *testing.T) { + e2eMainnet(t, false /*usePrysmSh*/) +} + +func e2eMainnet(t *testing.T, usePrysmSh bool) { + params.UseE2EMainnetConfig() + require.NoError(t, e2eParams.InitMultiClient(e2eParams.StandardBeaconCount, e2eParams.StandardLighthouseNodeCount)) + + // Run for 10 epochs if not in long-running to confirm long-running has no issues. + var err error + epochsToRun := 10 + epochStr, longRunning := os.LookupEnv("E2E_EPOCHS") + if longRunning { + epochsToRun, err = strconv.Atoi(epochStr) + require.NoError(t, err) + } + if usePrysmSh { + // If using prysm.sh, run for only 6 epochs. + // TODO(#9166): remove this block once v2 changes are live. + epochsToRun = helpers.AltairE2EForkEpoch - 1 + } + tracingPort := 9411 + e2eParams.TestParams.TestShardIndex + tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort) + evals := []types.Evaluator{ + ev.PeersConnect, + ev.HealthzCheck, + ev.MetricsCheck, + ev.ValidatorsAreActive, + ev.ValidatorsParticipatingAtEpoch(2), + ev.FinalizationOccurs(3), + ev.ProposeVoluntaryExit, + ev.ValidatorHasExited, + ev.ColdStateCheckpoint, + ev.ForkTransition, + ev.APIMiddlewareVerifyIntegrity, + ev.APIGatewayV1Alpha1VerifyIntegrity, + ev.FinishedSyncing, + ev.AllNodesHaveSameHead, + } + testConfig := &types.E2EConfig{ + BeaconFlags: []string{ + fmt.Sprintf("--slots-per-archive-point=%d", params.BeaconConfig().SlotsPerEpoch*16), + fmt.Sprintf("--tracing-endpoint=http://%s", tracingEndpoint), + "--enable-tracing", + "--trace-sample-fraction=1.0", + }, + ValidatorFlags: []string{}, + EpochsToRun: uint64(epochsToRun), + TestSync: true, + TestDeposits: true, + UseFixedPeerIDs: true, + UsePrysmShValidator: usePrysmSh, + UsePprof: !longRunning, + TracingSinkEndpoint: tracingEndpoint, + Evaluators: evals, + } + + newTestRunner(t, testConfig).run() +} diff --git a/testing/endtoend/params/params.go b/testing/endtoend/params/params.go index 96b77541f..08f6f9249 100644 --- a/testing/endtoend/params/params.go +++ b/testing/endtoend/params/params.go @@ -15,17 +15,18 @@ import ( // params struct defines the parameters needed for running E2E tests to properly handle test sharding. type params struct { - TestPath string - LogPath string - TestShardIndex int - BeaconNodeCount int - Eth1RPCPort int - ContractAddress common.Address - BootNodePort int - BeaconNodeRPCPort int - BeaconNodeMetricsPort int - ValidatorMetricsPort int - ValidatorGatewayPort int + TestPath string + LogPath string + TestShardIndex int + BeaconNodeCount int + LighthouseBeaconNodeCount int + Eth1RPCPort int + ContractAddress common.Address + BootNodePort int + BeaconNodeRPCPort int + BeaconNodeMetricsPort int + ValidatorMetricsPort int + ValidatorGatewayPort int } // TestParams is the globally accessible var for getting config elements. @@ -46,6 +47,9 @@ var ValidatorLogFileName = "vals-%d.log" // StandardBeaconCount is a global constant for the count of beacon nodes of standard E2E tests. var StandardBeaconCount = 2 +// StandardLighthouseNodeCount is a global constant for the count of lighthouse beacon nodes of standard E2E tests. +var StandardLighthouseNodeCount = 2 + // DepositCount is the amount of deposits E2E makes on a separate validator client. var DepositCount = uint64(64) @@ -80,3 +84,36 @@ func Init(beaconNodeCount int) error { } return nil } + +// InitMultiClient initializes the multiclient E2E config, properly handling test sharding. +func InitMultiClient(beaconNodeCount int, lighthouseNodeCount int) error { + testPath := bazel.TestTmpDir() + logPath, ok := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR") + if !ok { + return errors.New("expected TEST_UNDECLARED_OUTPUTS_DIR to be defined") + } + testIndexStr, ok := os.LookupEnv("TEST_SHARD_INDEX") + if !ok { + testIndexStr = "0" + } + testIndex, err := strconv.Atoi(testIndexStr) + if err != nil { + return err + } + testPath = filepath.Join(testPath, fmt.Sprintf("shard-%d", testIndex)) + + TestParams = ¶ms{ + TestPath: testPath, + LogPath: logPath, + TestShardIndex: testIndex, + BeaconNodeCount: beaconNodeCount, + LighthouseBeaconNodeCount: lighthouseNodeCount, + Eth1RPCPort: 3100 + testIndex*100, // Multiplying 100 here so the test index doesn't conflict with the other node ports. + BootNodePort: 4100 + testIndex*100, + BeaconNodeRPCPort: 4150 + testIndex*100, + BeaconNodeMetricsPort: 5100 + testIndex*100, + ValidatorMetricsPort: 6100 + testIndex*100, + ValidatorGatewayPort: 7150 + testIndex*100, + } + return nil +} diff --git a/testing/endtoend/types/types.go b/testing/endtoend/types/types.go index d85d5f3d3..821b90d59 100644 --- a/testing/endtoend/types/types.go +++ b/testing/endtoend/types/types.go @@ -16,11 +16,13 @@ type E2EConfig struct { UsePprof bool UseWeb3RemoteSigner bool TestDeposits bool + UseFixedPeerIDs bool EpochsToRun uint64 TracingSinkEndpoint string Evaluators []Evaluator BeaconFlags []string ValidatorFlags []string + PeerIDs []string } // Evaluator defines the structure of the evaluators used to diff --git a/validator/keymanager/BUILD.bazel b/validator/keymanager/BUILD.bazel index f54fdf890..d0cffb9e5 100644 --- a/validator/keymanager/BUILD.bazel +++ b/validator/keymanager/BUILD.bazel @@ -8,6 +8,7 @@ go_library( ], importpath = "github.com/prysmaticlabs/prysm/validator/keymanager", visibility = [ + "//testing/endtoend/components:__subpackages__", "//tools:__subpackages__", "//validator:__pkg__", "//validator:__subpackages__",