e2e: Add web3signer component (#10088)

* Add initial web3signer binary

* Add support for web3signer component in e2e

* revert some changes

* Reference https://github.com/ConsenSys/web3signer/issues/485 in commentary

* gofmt

* Add notice about java 11 requirement

* Sec issue, revert to 0750 permissions

* remove unused param

* remove unused import

* use hexutil

* Fix yaml struct tags. See https://github.com/ConsenSys/web3signer/issues/485\#issuecomment-1015994840

* fmt

Co-authored-by: james-prysm <90280386+james-prysm@users.noreply.github.com>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

WORKSPACE

@@ -377,6 +377,10 @@ load("@prysm//third_party/herumi:herumi.bzl", "bls_dependencies")
 
 bls_dependencies()
 
+load("@prysm//testing/endtoend:deps.bzl", "e2e_deps")
+
+e2e_deps()
+
 load(
     "@io_bazel_rules_docker//go:image.bzl",
     _go_image_repos = "repositories",

testing/endtoend/BUILD.bazel

@@ -19,6 +19,7 @@ go_test(
         "//cmd/validator",
         "//tools/bootnode",
         "@com_github_ethereum_go_ethereum//cmd/geth",
+        "@web3signer",
     ],
     eth_network = "minimal",
     shard_count = 2,

testing/endtoend/README.md

@@ -18,8 +18,10 @@ Evaluators have 3 parts, the name for it's test name, a `policy` which declares
 
 ## Instructions
 
+Note: Java 11 or greater is required to run web3signer.
+
 If you wish to run all the minimal spec E2E tests, you can run them through bazel with:
 
 ```
-bazel test //testing/endtoend:go_default_test --define=ssz=minimal --test_output=streamed
+bazel test //testing/endtoend:go_default_test --test_output=streamed
 ```

testing/endtoend/components/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@prysm//tools/go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -10,6 +10,7 @@ go_library(
         "log.go",
         "tracing_sink.go",
         "validator.go",
+        "web3remotesigner.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components",
     visibility = ["//testing/endtoend:__subpackages__"],
@@ -20,18 +21,35 @@ go_library(
         "//config/features:go_default_library",
         "//config/params:go_default_library",
         "//contracts/deposit:go_default_library",
+        "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//runtime/interop:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",
         "//testing/util:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@in_gopkg_yaml_v2//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["web3remotesigner_test.go"],
+    data = ["@web3signer"],
+    deps = [
+        ":go_default_library",
+        "//config/params:go_default_library",
+        "//testing/endtoend/params:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

testing/endtoend/components/validator.go

@@ -3,6 +3,7 @@ package components
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"math/big"
@@ -23,6 +24,7 @@ import (
 	"github.com/prysmaticlabs/prysm/config/params"
 	contracts "github.com/prysmaticlabs/prysm/contracts/deposit"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/runtime/interop"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
@@ -133,8 +135,6 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--%s=%s/eth2-val-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
 		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, file.Name()),
 		fmt.Sprintf("--%s=%s", flags.GraffitiFileFlag.Name, gFile),
-		fmt.Sprintf("--%s=%d", flags.InteropNumValidators.Name, validatorNum),
-		fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset),
 		fmt.Sprintf("--%s=%d", flags.MonitoringPortFlag.Name, e2e.TestParams.ValidatorMetricsPort+index),
 		fmt.Sprintf("--%s=%d", flags.GRPCGatewayPort.Name, e2e.TestParams.ValidatorGatewayPort+index),
 		fmt.Sprintf("--%s=localhost:%d", flags.BeaconRPCProviderFlag.Name, beaconRPCPort),
@@ -148,6 +148,27 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 	if !v.config.UsePrysmShValidator {
 		args = append(args, features.E2EValidatorFlags...)
 	}
+	if v.config.UseWeb3RemoteSigner {
+		// TODO(9994): Replace "validators-external-signer-url" with flags.Web3RemoteSignerURLFlag.Name
+		args = append(args, fmt.Sprintf("--%s=localhost:%d", "validators-external-signer-url", Web3RemoteSignerPort))
+		// Write the pubkeys as comma seperated hex strings with 0x prefix.
+		// See: https://docs.teku.consensys.net/en/latest/HowTo/External-Signer/Use-External-Signer/
+		_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum))
+		if err != nil {
+			return err
+		}
+		var hexPubs []string
+		for _, pub := range pubs {
+			hexPubs = append(hexPubs, "0x"+hex.EncodeToString(pub.Marshal()))
+		}
+		// TODO(9994): Replace "validators-external-signer-public-keys" with flags.Web3RemoteSignerPubkeysFlag.Name
+		args = append(args, fmt.Sprintf("--validators-external-signer-public-keys=%s", strings.Join(hexPubs, ",")))
+	} else {
+		// When not using remote key signer, use interop keys.
+		args = append(args,
+			fmt.Sprintf("--%s=%d", flags.InteropNumValidators.Name, validatorNum),
+			fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset))
+	}
 	args = append(args, config.ValidatorFlags...)
 
 	if v.config.UsePrysmShValidator {

testing/endtoend/components/web3remotesigner.go

@@ -0,0 +1,223 @@
+package components
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/crypto/bls"
+	"github.com/prysmaticlabs/prysm/runtime/interop"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
+	"gopkg.in/yaml.v2"
+)
+
+const Web3RemoteSignerPort = 9000
+
+var _ e2etypes.ComponentRunner = (*Web3RemoteSigner)(nil)
+
+// rawKeyFile used for consensys's web3signer config files.
+// See: https://docs.web3signer.consensys.net/en/latest/Reference/Key-Configuration-Files/#raw-unencrypted-files
+type rawKeyFile struct {
+	Type       string `yaml:"type"`       // always "file-raw" for this test.
+	KeyType    string `yaml:"keyType"`    // always "BLS" for this test.
+	PrivateKey string `yaml:"privateKey"` // hex encoded private key with 0x prefix.
+}
+
+type Web3RemoteSigner struct {
+	ctx     context.Context
+	started chan struct{}
+}
+
+func NewWeb3RemoteSigner() *Web3RemoteSigner {
+	return &Web3RemoteSigner{
+		started: make(chan struct{}, 1),
+	}
+}
+
+// Start the web3remotesigner component with a keystore populated with the deterministic validator
+// keys.
+func (w *Web3RemoteSigner) Start(ctx context.Context) error {
+	w.ctx = ctx
+
+	binaryPath, found := bazel.FindBinary("", "web3signer")
+	if !found {
+		return errors.New("web3signer binary not found")
+	}
+
+	keystorePath := path.Join(bazel.TestTmpDir(), "web3signerkeystore")
+	if err := writeKeystoreKeys(ctx, keystorePath, params.BeaconConfig().MinGenesisActiveValidatorCount); err != nil {
+		return err
+	}
+	websignerDataDir := path.Join(bazel.TestTmpDir(), "web3signerdata")
+	if err := os.MkdirAll(websignerDataDir, 0750); err != nil {
+		return err
+	}
+
+	args := []string{
+		// Global flags
+		fmt.Sprintf("--key-store-path=%s", keystorePath),
+		fmt.Sprintf("--data-path=%s", websignerDataDir),
+		fmt.Sprintf("--http-listen-port=%d", Web3RemoteSignerPort),
+		// Command
+		"eth2",
+		// Command flags
+		"--network=minimal",
+		"--slashing-protection-enabled=false", // Otherwise, a postgres DB is required.
+		"--enable-key-manager-api=true",
+	}
+
+	cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Test code is safe to do this.
+
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, "web3signer.stdout.log"))
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, "web3signer.stderr.log"))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := stdout.Close(); err != nil {
+			log.WithError(err).Error("Failed to close stdout file")
+		}
+		if err := stderr.Close(); err != nil {
+			log.WithError(err).Error("Failed to close stderr file")
+		}
+	}()
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+
+	log.Infof("Starting web3signer with flags: %s %s", binaryPath, strings.Join(args, " "))
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	go w.monitorStart()
+
+	return cmd.Wait()
+}
+
+func (w *Web3RemoteSigner) Started() <-chan struct{} {
+	return w.started
+}
+
+// monitorStart by polling server until it returns a 200 at /upcheck.
+func (w *Web3RemoteSigner) monitorStart() {
+	client := &http.Client{}
+	for {
+		req, err := http.NewRequestWithContext(w.ctx, "GET", fmt.Sprintf("http://localhost:%d/upcheck", Web3RemoteSignerPort), nil)
+		if err != nil {
+			panic(err)
+		}
+		res, err := client.Do(req)
+		_ = err
+		if res != nil && res.StatusCode == 200 {
+			close(w.started)
+			return
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func (w *Web3RemoteSigner) wait(ctx context.Context) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-w.ctx.Done():
+		return
+	case <-w.started:
+		return
+	}
+}
+
+// PublicKeys queries the web3signer and returns the response keys.
+func (w *Web3RemoteSigner) PublicKeys(ctx context.Context) ([]bls.PublicKey, error) {
+	w.wait(ctx)
+
+	client := &http.Client{}
+	req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://localhost:%d/api/v1/eth2/publicKeys", Web3RemoteSignerPort), nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("returned status code %d", res.StatusCode)
+	}
+	b, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	} else if len(b) == 0 {
+		return nil, errors.New("no response body")
+	}
+	var keys []string
+	if err := json.Unmarshal(b, &keys); err != nil {
+		return nil, err
+	}
+	if len(keys) == 0 {
+		return nil, errors.New("no keys returned")
+	}
+
+	var pks []bls.PublicKey
+	for _, key := range keys {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		raw, err := hexutil.Decode(key)
+		if err != nil {
+			return nil, err
+		}
+		pk, err := bls.PublicKeyFromBytes(raw)
+		if err != nil {
+			return nil, err
+		}
+		pks = append(pks, pk)
+	}
+	return pks, nil
+}
+
+func writeKeystoreKeys(ctx context.Context, keystorePath string, numKeys uint64) error {
+	if err := os.MkdirAll(keystorePath, 0750); err != nil {
+		return err
+	}
+
+	priv, pub, err := interop.DeterministicallyGenerateKeys(0, numKeys)
+	if err != nil {
+		return err
+	}
+	for i, pk := range priv {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		rkf := &rawKeyFile{
+			Type:       "file-raw",
+			KeyType:    "BLS",
+			PrivateKey: hexutil.Encode(pk.Marshal()),
+		}
+		b, err := yaml.Marshal(rkf)
+		if err != nil {
+			return err
+		}
+		if err := os.WriteFile(path.Join(keystorePath, fmt.Sprintf("key-0x%s.yaml", hex.EncodeToString(pub[i].Marshal()))), b, 0600); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

testing/endtoend/components/web3remotesigner_test.go

@@ -0,0 +1,44 @@
+package components_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/components"
+	e2eparams "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestWeb3RemoteSigner_StartsAndReturnsPublicKeys(t *testing.T) {
+	require.NoError(t, e2eparams.Init(0))
+	wsc := components.NewWeb3RemoteSigner()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	go func() {
+		if err := wsc.Start(ctx); err != nil {
+			t.Error(err)
+			panic(err)
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Web3RemoteSigner did not start within timeout")
+	case <-wsc.Started():
+		t.Log("Web3RemoteSigner started")
+		break
+	}
+
+	time.Sleep(10 * time.Second)
+
+	keys, err := wsc.PublicKeys(ctx)
+	require.NoError(t, err)
+
+	if uint64(len(keys)) != params.BeaconConfig().MinGenesisActiveValidatorCount {
+		t.Fatalf("Expected %d keys, got %d", params.BeaconConfig().MinGenesisActiveValidatorCount, len(keys))
+	}
+}

testing/endtoend/deps.bzl

@@ -0,0 +1,10 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")  # gazelle:keep
+
+def e2e_deps():
+    http_archive(
+        name = "web3signer",
+        urls = ["https://artifacts.consensys.net/public/web3signer/raw/names/web3signer.tar.gz/versions/21.10.5/web3signer-21.10.5.tar.gz"],
+        sha256 = "d122429f6a310bc555d1281e0b3f4e3ac43a7beec5e5dcf0a0d2416a5984f461",
+        build_file = "@prysm//testing/endtoend:web3signer.BUILD",
+        strip_prefix = "web3signer-21.10.5",
+    )

testing/endtoend/endtoend_test.go

@@ -116,10 +116,26 @@ func (r *testRunner) run() {
 		return nil
 	})
 
+	// Web3 remote signer.
+	var web3RemoteSigner *components.Web3RemoteSigner
+	if config.UseWeb3RemoteSigner {
+		web3RemoteSigner = components.NewWeb3RemoteSigner()
+		g.Go(func() error {
+			if err := web3RemoteSigner.Start(ctx); err != nil {
+				return errors.Wrap(err, "failed to start web3 remote signer")
+			}
+			return nil
+		})
+	}
+
 	// Validator nodes.
 	validatorNodes := components.NewValidatorNodeSet(config)
 	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{beaconNodes}); err != nil {
+		comps := []e2etypes.ComponentRunner{beaconNodes}
+		if config.UseWeb3RemoteSigner {
+			comps = append(comps, web3RemoteSigner)
+		}
+		if err := helpers.ComponentsStarted(ctx, comps); err != nil {
 			return errors.Wrap(err, "validator nodes require beacon nodes to run")
 		}
 		if err := validatorNodes.Start(ctx); err != nil {

testing/endtoend/minimal_e2e_test.go

@@ -14,16 +14,35 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
+type testArgs struct {
+	usePrysmSh          bool
+	useWeb3RemoteSigner bool
+}
+
 func TestEndToEnd_MinimalConfig(t *testing.T) {
-	e2eMinimal(t, false /*usePrysmSh*/)
+	e2eMinimal(t, &testArgs{
+		usePrysmSh:          false,
+		useWeb3RemoteSigner: false,
+	})
+}
+
+func TestEndToEnd_MinimalConfig_Web3Signer(t *testing.T) {
+	t.Skip("TODO(9994): Complete web3signer client implementation")
+	e2eMinimal(t, &testArgs{
+		usePrysmSh:          false,
+		useWeb3RemoteSigner: true,
+	})
 }
 
 // Run minimal e2e config with the current release validator against latest beacon node.
 func TestEndToEnd_MinimalConfig_ValidatorAtCurrentRelease(t *testing.T) {
-	e2eMinimal(t, true /*usePrysmSh*/)
+	e2eMinimal(t, &testArgs{
+		usePrysmSh:          true,
+		useWeb3RemoteSigner: false,
+	})
 }
 
-func e2eMinimal(t *testing.T, usePrysmSh bool) {
+func e2eMinimal(t *testing.T, args *testArgs) {
 	params.UseE2EConfig()
 	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
 
@@ -35,7 +54,7 @@ func e2eMinimal(t *testing.T, usePrysmSh bool) {
 		epochsToRun, err = strconv.Atoi(epochStr)
 		require.NoError(t, err)
 	}
-	if usePrysmSh {
+	if args.usePrysmSh {
 		// If using prysm.sh, run for only 6 epochs.
 		// TODO(#9166): remove this block once v2 changes are live.
 		epochsToRun = helpers.AltairE2EForkEpoch - 1
@@ -75,8 +94,9 @@ func e2eMinimal(t *testing.T, usePrysmSh bool) {
 		EpochsToRun:         uint64(epochsToRun),
 		TestSync:            true,
 		TestDeposits:        true,
-		UsePrysmShValidator: usePrysmSh,
+		UsePrysmShValidator: args.usePrysmSh,
 		UsePprof:            !longRunning,
+		UseWeb3RemoteSigner: args.useWeb3RemoteSigner,
 		TracingSinkEndpoint: tracingEndpoint,
 		Evaluators:          evals,
 	}

testing/endtoend/types/types.go

@@ -14,6 +14,7 @@ type E2EConfig struct {
 	TestSync            bool
 	UsePrysmShValidator bool
 	UsePprof            bool
+	UseWeb3RemoteSigner bool
 	TestDeposits        bool
 	EpochsToRun         uint64
 	TracingSinkEndpoint string

testing/endtoend/web3signer.BUILD

@@ -0,0 +1,7 @@
+sh_binary(
+    name = "web3signer",
+    srcs = [
+        "bin/web3signer",
+    ],
+    visibility = ["//visibility:public"],
+)