diff --git a/WORKSPACE b/WORKSPACE index 925eb605b..477f54f96 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -377,6 +377,10 @@ load("@prysm//third_party/herumi:herumi.bzl", "bls_dependencies") bls_dependencies() +load("@prysm//testing/endtoend:deps.bzl", "e2e_deps") + +e2e_deps() + load( "@io_bazel_rules_docker//go:image.bzl", _go_image_repos = "repositories", diff --git a/testing/endtoend/BUILD.bazel b/testing/endtoend/BUILD.bazel index 65ef0b552..811ee886c 100644 --- a/testing/endtoend/BUILD.bazel +++ b/testing/endtoend/BUILD.bazel @@ -19,6 +19,7 @@ go_test( "//cmd/validator", "//tools/bootnode", "@com_github_ethereum_go_ethereum//cmd/geth", + "@web3signer", ], eth_network = "minimal", shard_count = 2, diff --git a/testing/endtoend/README.md b/testing/endtoend/README.md index df92fef8e..4532176fa 100644 --- a/testing/endtoend/README.md +++ b/testing/endtoend/README.md @@ -18,8 +18,10 @@ Evaluators have 3 parts, the name for it's test name, a `policy` which declares ## Instructions +Note: Java 11 or greater is required to run web3signer. + If you wish to run all the minimal spec E2E tests, you can run them through bazel with: ``` -bazel test //testing/endtoend:go_default_test --define=ssz=minimal --test_output=streamed +bazel test //testing/endtoend:go_default_test --test_output=streamed ``` diff --git a/testing/endtoend/components/BUILD.bazel b/testing/endtoend/components/BUILD.bazel index 21df5fcd9..ba32575cd 100644 --- a/testing/endtoend/components/BUILD.bazel +++ b/testing/endtoend/components/BUILD.bazel @@ -1,4 +1,4 @@ -load("@prysm//tools/go:def.bzl", "go_library") +load("@prysm//tools/go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -10,6 +10,7 @@ go_library( "log.go", "tracing_sink.go", "validator.go", + "web3remotesigner.go", ], importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components", visibility = ["//testing/endtoend:__subpackages__"], @@ -20,18 +21,35 @@ go_library( "//config/features:go_default_library", "//config/params:go_default_library", "//contracts/deposit:go_default_library", + "//crypto/bls:go_default_library", "//encoding/bytesutil:go_default_library", + "//runtime/interop:go_default_library", "//testing/endtoend/helpers:go_default_library", "//testing/endtoend/params:go_default_library", "//testing/endtoend/types:go_default_library", "//testing/util:go_default_library", "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library", "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library", + "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library", "@com_github_ethereum_go_ethereum//core/types:go_default_library", "@com_github_ethereum_go_ethereum//ethclient:go_default_library", "@com_github_ethereum_go_ethereum//rpc:go_default_library", "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@in_gopkg_yaml_v2//:go_default_library", "@io_bazel_rules_go//go/tools/bazel:go_default_library", ], ) + +go_test( + name = "go_default_test", + size = "small", + srcs = ["web3remotesigner_test.go"], + data = ["@web3signer"], + deps = [ + ":go_default_library", + "//config/params:go_default_library", + "//testing/endtoend/params:go_default_library", + "//testing/require:go_default_library", + ], +) diff --git a/testing/endtoend/components/validator.go b/testing/endtoend/components/validator.go index d33fa4f37..431163db4 100644 --- a/testing/endtoend/components/validator.go +++ b/testing/endtoend/components/validator.go @@ -3,6 +3,7 @@ package components import ( "bytes" "context" + "encoding/hex" "fmt" "io/ioutil" "math/big" @@ -23,6 +24,7 @@ import ( "github.com/prysmaticlabs/prysm/config/params" contracts "github.com/prysmaticlabs/prysm/contracts/deposit" "github.com/prysmaticlabs/prysm/encoding/bytesutil" + "github.com/prysmaticlabs/prysm/runtime/interop" "github.com/prysmaticlabs/prysm/testing/endtoend/helpers" e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" @@ -133,8 +135,6 @@ func (v *ValidatorNode) Start(ctx context.Context) error { fmt.Sprintf("--%s=%s/eth2-val-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index), fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, file.Name()), fmt.Sprintf("--%s=%s", flags.GraffitiFileFlag.Name, gFile), - fmt.Sprintf("--%s=%d", flags.InteropNumValidators.Name, validatorNum), - fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset), fmt.Sprintf("--%s=%d", flags.MonitoringPortFlag.Name, e2e.TestParams.ValidatorMetricsPort+index), fmt.Sprintf("--%s=%d", flags.GRPCGatewayPort.Name, e2e.TestParams.ValidatorGatewayPort+index), fmt.Sprintf("--%s=localhost:%d", flags.BeaconRPCProviderFlag.Name, beaconRPCPort), @@ -148,6 +148,27 @@ func (v *ValidatorNode) Start(ctx context.Context) error { if !v.config.UsePrysmShValidator { args = append(args, features.E2EValidatorFlags...) } + if v.config.UseWeb3RemoteSigner { + // TODO(9994): Replace "validators-external-signer-url" with flags.Web3RemoteSignerURLFlag.Name + args = append(args, fmt.Sprintf("--%s=localhost:%d", "validators-external-signer-url", Web3RemoteSignerPort)) + // Write the pubkeys as comma seperated hex strings with 0x prefix. + // See: https://docs.teku.consensys.net/en/latest/HowTo/External-Signer/Use-External-Signer/ + _, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum)) + if err != nil { + return err + } + var hexPubs []string + for _, pub := range pubs { + hexPubs = append(hexPubs, "0x"+hex.EncodeToString(pub.Marshal())) + } + // TODO(9994): Replace "validators-external-signer-public-keys" with flags.Web3RemoteSignerPubkeysFlag.Name + args = append(args, fmt.Sprintf("--validators-external-signer-public-keys=%s", strings.Join(hexPubs, ","))) + } else { + // When not using remote key signer, use interop keys. + args = append(args, + fmt.Sprintf("--%s=%d", flags.InteropNumValidators.Name, validatorNum), + fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset)) + } args = append(args, config.ValidatorFlags...) if v.config.UsePrysmShValidator { diff --git a/testing/endtoend/components/web3remotesigner.go b/testing/endtoend/components/web3remotesigner.go new file mode 100644 index 000000000..d3ca6ea80 --- /dev/null +++ b/testing/endtoend/components/web3remotesigner.go @@ -0,0 +1,223 @@ +package components + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "strings" + "time" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/crypto/bls" + "github.com/prysmaticlabs/prysm/runtime/interop" + e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params" + e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types" + "gopkg.in/yaml.v2" +) + +const Web3RemoteSignerPort = 9000 + +var _ e2etypes.ComponentRunner = (*Web3RemoteSigner)(nil) + +// rawKeyFile used for consensys's web3signer config files. +// See: https://docs.web3signer.consensys.net/en/latest/Reference/Key-Configuration-Files/#raw-unencrypted-files +type rawKeyFile struct { + Type string `yaml:"type"` // always "file-raw" for this test. + KeyType string `yaml:"keyType"` // always "BLS" for this test. + PrivateKey string `yaml:"privateKey"` // hex encoded private key with 0x prefix. +} + +type Web3RemoteSigner struct { + ctx context.Context + started chan struct{} +} + +func NewWeb3RemoteSigner() *Web3RemoteSigner { + return &Web3RemoteSigner{ + started: make(chan struct{}, 1), + } +} + +// Start the web3remotesigner component with a keystore populated with the deterministic validator +// keys. +func (w *Web3RemoteSigner) Start(ctx context.Context) error { + w.ctx = ctx + + binaryPath, found := bazel.FindBinary("", "web3signer") + if !found { + return errors.New("web3signer binary not found") + } + + keystorePath := path.Join(bazel.TestTmpDir(), "web3signerkeystore") + if err := writeKeystoreKeys(ctx, keystorePath, params.BeaconConfig().MinGenesisActiveValidatorCount); err != nil { + return err + } + websignerDataDir := path.Join(bazel.TestTmpDir(), "web3signerdata") + if err := os.MkdirAll(websignerDataDir, 0750); err != nil { + return err + } + + args := []string{ + // Global flags + fmt.Sprintf("--key-store-path=%s", keystorePath), + fmt.Sprintf("--data-path=%s", websignerDataDir), + fmt.Sprintf("--http-listen-port=%d", Web3RemoteSignerPort), + // Command + "eth2", + // Command flags + "--network=minimal", + "--slashing-protection-enabled=false", // Otherwise, a postgres DB is required. + "--enable-key-manager-api=true", + } + + cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Test code is safe to do this. + + // Write stdout and stderr to log files. + stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, "web3signer.stdout.log")) + if err != nil { + return err + } + stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, "web3signer.stderr.log")) + if err != nil { + return err + } + defer func() { + if err := stdout.Close(); err != nil { + log.WithError(err).Error("Failed to close stdout file") + } + if err := stderr.Close(); err != nil { + log.WithError(err).Error("Failed to close stderr file") + } + }() + cmd.Stdout = stdout + cmd.Stderr = stderr + + log.Infof("Starting web3signer with flags: %s %s", binaryPath, strings.Join(args, " ")) + if err = cmd.Start(); err != nil { + return err + } + + go w.monitorStart() + + return cmd.Wait() +} + +func (w *Web3RemoteSigner) Started() <-chan struct{} { + return w.started +} + +// monitorStart by polling server until it returns a 200 at /upcheck. +func (w *Web3RemoteSigner) monitorStart() { + client := &http.Client{} + for { + req, err := http.NewRequestWithContext(w.ctx, "GET", fmt.Sprintf("http://localhost:%d/upcheck", Web3RemoteSignerPort), nil) + if err != nil { + panic(err) + } + res, err := client.Do(req) + _ = err + if res != nil && res.StatusCode == 200 { + close(w.started) + return + } + time.Sleep(time.Second) + } +} + +func (w *Web3RemoteSigner) wait(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-w.ctx.Done(): + return + case <-w.started: + return + } +} + +// PublicKeys queries the web3signer and returns the response keys. +func (w *Web3RemoteSigner) PublicKeys(ctx context.Context) ([]bls.PublicKey, error) { + w.wait(ctx) + + client := &http.Client{} + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://localhost:%d/api/v1/eth2/publicKeys", Web3RemoteSignerPort), nil) + if err != nil { + return nil, err + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("returned status code %d", res.StatusCode) + } + b, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } else if len(b) == 0 { + return nil, errors.New("no response body") + } + var keys []string + if err := json.Unmarshal(b, &keys); err != nil { + return nil, err + } + if len(keys) == 0 { + return nil, errors.New("no keys returned") + } + + var pks []bls.PublicKey + for _, key := range keys { + if ctx.Err() != nil { + return nil, ctx.Err() + } + raw, err := hexutil.Decode(key) + if err != nil { + return nil, err + } + pk, err := bls.PublicKeyFromBytes(raw) + if err != nil { + return nil, err + } + pks = append(pks, pk) + } + return pks, nil +} + +func writeKeystoreKeys(ctx context.Context, keystorePath string, numKeys uint64) error { + if err := os.MkdirAll(keystorePath, 0750); err != nil { + return err + } + + priv, pub, err := interop.DeterministicallyGenerateKeys(0, numKeys) + if err != nil { + return err + } + for i, pk := range priv { + if ctx.Err() != nil { + return ctx.Err() + } + rkf := &rawKeyFile{ + Type: "file-raw", + KeyType: "BLS", + PrivateKey: hexutil.Encode(pk.Marshal()), + } + b, err := yaml.Marshal(rkf) + if err != nil { + return err + } + if err := os.WriteFile(path.Join(keystorePath, fmt.Sprintf("key-0x%s.yaml", hex.EncodeToString(pub[i].Marshal()))), b, 0600); err != nil { + return err + } + } + + return nil +} diff --git a/testing/endtoend/components/web3remotesigner_test.go b/testing/endtoend/components/web3remotesigner_test.go new file mode 100644 index 000000000..7491843a7 --- /dev/null +++ b/testing/endtoend/components/web3remotesigner_test.go @@ -0,0 +1,44 @@ +package components_test + +import ( + "context" + "testing" + "time" + + "github.com/prysmaticlabs/prysm/config/params" + "github.com/prysmaticlabs/prysm/testing/endtoend/components" + e2eparams "github.com/prysmaticlabs/prysm/testing/endtoend/params" + "github.com/prysmaticlabs/prysm/testing/require" +) + +func TestWeb3RemoteSigner_StartsAndReturnsPublicKeys(t *testing.T) { + require.NoError(t, e2eparams.Init(0)) + wsc := components.NewWeb3RemoteSigner() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + go func() { + if err := wsc.Start(ctx); err != nil { + t.Error(err) + panic(err) + } + }() + + select { + case <-ctx.Done(): + t.Fatal("Web3RemoteSigner did not start within timeout") + case <-wsc.Started(): + t.Log("Web3RemoteSigner started") + break + } + + time.Sleep(10 * time.Second) + + keys, err := wsc.PublicKeys(ctx) + require.NoError(t, err) + + if uint64(len(keys)) != params.BeaconConfig().MinGenesisActiveValidatorCount { + t.Fatalf("Expected %d keys, got %d", params.BeaconConfig().MinGenesisActiveValidatorCount, len(keys)) + } +} diff --git a/testing/endtoend/deps.bzl b/testing/endtoend/deps.bzl new file mode 100644 index 000000000..fffac14d6 --- /dev/null +++ b/testing/endtoend/deps.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # gazelle:keep + +def e2e_deps(): + http_archive( + name = "web3signer", + urls = ["https://artifacts.consensys.net/public/web3signer/raw/names/web3signer.tar.gz/versions/21.10.5/web3signer-21.10.5.tar.gz"], + sha256 = "d122429f6a310bc555d1281e0b3f4e3ac43a7beec5e5dcf0a0d2416a5984f461", + build_file = "@prysm//testing/endtoend:web3signer.BUILD", + strip_prefix = "web3signer-21.10.5", + ) diff --git a/testing/endtoend/endtoend_test.go b/testing/endtoend/endtoend_test.go index 189930339..b381704ab 100644 --- a/testing/endtoend/endtoend_test.go +++ b/testing/endtoend/endtoend_test.go @@ -116,10 +116,26 @@ func (r *testRunner) run() { return nil }) + // Web3 remote signer. + var web3RemoteSigner *components.Web3RemoteSigner + if config.UseWeb3RemoteSigner { + web3RemoteSigner = components.NewWeb3RemoteSigner() + g.Go(func() error { + if err := web3RemoteSigner.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start web3 remote signer") + } + return nil + }) + } + // Validator nodes. validatorNodes := components.NewValidatorNodeSet(config) g.Go(func() error { - if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{beaconNodes}); err != nil { + comps := []e2etypes.ComponentRunner{beaconNodes} + if config.UseWeb3RemoteSigner { + comps = append(comps, web3RemoteSigner) + } + if err := helpers.ComponentsStarted(ctx, comps); err != nil { return errors.Wrap(err, "validator nodes require beacon nodes to run") } if err := validatorNodes.Start(ctx); err != nil { diff --git a/testing/endtoend/minimal_e2e_test.go b/testing/endtoend/minimal_e2e_test.go index 62e00256e..ccc0ff9d5 100644 --- a/testing/endtoend/minimal_e2e_test.go +++ b/testing/endtoend/minimal_e2e_test.go @@ -14,16 +14,35 @@ import ( "github.com/prysmaticlabs/prysm/testing/require" ) +type testArgs struct { + usePrysmSh bool + useWeb3RemoteSigner bool +} + func TestEndToEnd_MinimalConfig(t *testing.T) { - e2eMinimal(t, false /*usePrysmSh*/) + e2eMinimal(t, &testArgs{ + usePrysmSh: false, + useWeb3RemoteSigner: false, + }) +} + +func TestEndToEnd_MinimalConfig_Web3Signer(t *testing.T) { + t.Skip("TODO(9994): Complete web3signer client implementation") + e2eMinimal(t, &testArgs{ + usePrysmSh: false, + useWeb3RemoteSigner: true, + }) } // Run minimal e2e config with the current release validator against latest beacon node. func TestEndToEnd_MinimalConfig_ValidatorAtCurrentRelease(t *testing.T) { - e2eMinimal(t, true /*usePrysmSh*/) + e2eMinimal(t, &testArgs{ + usePrysmSh: true, + useWeb3RemoteSigner: false, + }) } -func e2eMinimal(t *testing.T, usePrysmSh bool) { +func e2eMinimal(t *testing.T, args *testArgs) { params.UseE2EConfig() require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount)) @@ -35,7 +54,7 @@ func e2eMinimal(t *testing.T, usePrysmSh bool) { epochsToRun, err = strconv.Atoi(epochStr) require.NoError(t, err) } - if usePrysmSh { + if args.usePrysmSh { // If using prysm.sh, run for only 6 epochs. // TODO(#9166): remove this block once v2 changes are live. epochsToRun = helpers.AltairE2EForkEpoch - 1 @@ -75,8 +94,9 @@ func e2eMinimal(t *testing.T, usePrysmSh bool) { EpochsToRun: uint64(epochsToRun), TestSync: true, TestDeposits: true, - UsePrysmShValidator: usePrysmSh, + UsePrysmShValidator: args.usePrysmSh, UsePprof: !longRunning, + UseWeb3RemoteSigner: args.useWeb3RemoteSigner, TracingSinkEndpoint: tracingEndpoint, Evaluators: evals, } diff --git a/testing/endtoend/types/types.go b/testing/endtoend/types/types.go index 70e083ab2..d85d5f3d3 100644 --- a/testing/endtoend/types/types.go +++ b/testing/endtoend/types/types.go @@ -14,6 +14,7 @@ type E2EConfig struct { TestSync bool UsePrysmShValidator bool UsePprof bool + UseWeb3RemoteSigner bool TestDeposits bool EpochsToRun uint64 TracingSinkEndpoint string diff --git a/testing/endtoend/web3signer.BUILD b/testing/endtoend/web3signer.BUILD new file mode 100644 index 000000000..cd688bbc9 --- /dev/null +++ b/testing/endtoend/web3signer.BUILD @@ -0,0 +1,7 @@ +sh_binary( + name = "web3signer", + srcs = [ + "bin/web3signer", + ], + visibility = ["//visibility:public"], +)