Load in JSON Validators for Genesis State Generation (#7643)

* load in genesis state from JSON validators

* completed feature with tests

* bazel change

* fix docker image

* deep source

shared/interop/generate_genesis_state.go

@@ -36,11 +36,19 @@ func GenerateGenesisState(genesisTime, numValidators uint64) (*pb.BeaconState, [
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not generate deposit data from keys")
 	}
+	return GenerateGenesisStateFromDepositData(genesisTime, depositDataItems, depositDataRoots)
+}
+
+// GenerateGenesisStateFromDepositData creates a genesis state given a list of
+// deposit data items and their corresponding roots.
+func GenerateGenesisStateFromDepositData(
+	genesisTime uint64, depositData []*ethpb.Deposit_Data, depositDataRoots [][]byte,
+) (*pb.BeaconState, []*ethpb.Deposit, error) {
 	trie, err := trieutil.GenerateTrieFromItems(depositDataRoots, params.BeaconConfig().DepositContractTreeDepth)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not generate Merkle trie for deposit proofs")
 	}
-	deposits, err := GenerateDepositsFromData(depositDataItems, trie)
+	deposits, err := GenerateDepositsFromData(depositData, trie)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "could not generate deposits from the deposit data provided")
 	}

tools/genesis-state-gen/BUILD.bazel

@@ -1,5 +1,5 @@
 load("@prysm//tools/go:def.bzl", "go_library")
-load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
@@ -10,9 +10,12 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/tools/genesis-state-gen",
     visibility = ["//visibility:private"],
     deps = [
+        "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/interop:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_ghodss_yaml//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )
 
@@ -39,10 +42,12 @@ go_image(
     tags = ["manual"],
     visibility = ["//visibility:private"],
     deps = [
+        "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/interop:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_ghodss_yaml//:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )
 
@@ -60,3 +65,16 @@ docker_push(
     bundle = ":image_bundle",
     tags = ["manual"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["main_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//shared/bls:go_default_library",
+        "//shared/interop:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+    ],
+)

tools/genesis-state-gen/main.go

@@ -1,18 +1,38 @@
 package main
 
 import (
+	"encoding/hex"
 	"encoding/json"
 	"flag"
+	"io"
 	"io/ioutil"
 	"log"
+	"os"
+	"strings"
 
 	"github.com/ghodss/yaml"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 	"github.com/prysmaticlabs/prysm/shared/interop"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
+// GenesisValidator struct representing JSON input we can accept to generate
+// a genesis state from, containing a validator's full deposit data as a hex string.
+type GenesisValidator struct {
+	DepositData string `json:"deposit_data"`
+}
+
 var (
-	numValidators    = flag.Int("num-validators", 0, "Number of validators to deterministically include in the generated genesis state")
+	validatorJSONInput = flag.String(
+		"validator-json-file",
+		"",
+		"Path to JSON file formatted as a list of hex public keys and their corresponding deposit data as hex"+
+			" such as [ { public_key: '0x1', deposit_data: '0x2' }, ... ]"+
+			" this file will be used for generating a genesis state from a list of specified validator public keys",
+	)
+	numValidators    = flag.Int("num-validators", 0, "Number of validators to deterministically generate in the generated genesis state")
 	useMainnetConfig = flag.Bool("mainnet-config", false, "Select whether genesis state should be generated with mainnet or minimal (default) params")
 	genesisTime      = flag.Uint64("genesis-time", 0, "Unix timestamp used as the genesis time in the generated genesis state (defaults to now)")
 	sszOutputFile    = flag.String("output-ssz", "", "Output filename of the SSZ marshaling of the generated genesis state")
@@ -22,51 +42,124 @@ var (
 
 func main() {
 	flag.Parse()
-	if *numValidators == 0 {
-		log.Fatal("Expected --num-validators to have been provided, received 0")
-	}
 	if *genesisTime == 0 {
 		log.Print("No --genesis-time specified, defaulting to now")
 	}
 	if *sszOutputFile == "" && *yamlOutputFile == "" && *jsonOutputFile == "" {
-		log.Fatal("Expected --output-ssz, --output-yaml, or --output-json to have been provided, received nil")
+		log.Println("Expected --output-ssz, --output-yaml, or --output-json to have been provided, received nil")
+		return
 	}
 	if !*useMainnetConfig {
 		params.OverrideBeaconConfig(params.MinimalSpecConfig())
 	}
-
-	genesisState, _, err := interop.GenerateGenesisState(*genesisTime, uint64(*numValidators))
-	if err != nil {
-		log.Fatalf("Could not generate genesis beacon state: %v", err)
+	var genesisState *pb.BeaconState
+	var err error
+	if *validatorJSONInput != "" {
+		inputFile := *validatorJSONInput
+		expanded, err := fileutil.ExpandPath(inputFile)
+		if err != nil {
+			log.Printf("Could not expand file path %s: %v", inputFile, err)
+			return
+		}
+		inputJSON, err := os.Open(expanded)
+		if err != nil {
+			log.Printf("Could not open JSON file for reading: %v", err)
+			return
+		}
+		defer func() {
+			if err := inputJSON.Close(); err != nil {
+				log.Printf("Could not close file %s: %v", inputFile, err)
+			}
+		}()
+		log.Printf("Generating genesis state from input JSON deposit data %s", inputFile)
+		genesisState, err = genesisStateFromJSONValidators(inputJSON, *genesisTime)
+		if err != nil {
+			log.Printf("Could not generate genesis beacon state: %v", err)
+			return
+		}
+	} else {
+		if *numValidators == 0 {
+			log.Println("Expected --num-validators to have been provided, received 0")
+			return
+		}
+		// If no JSON input is specified, we create the state deterministically from interop keys.
+		genesisState, _, err = interop.GenerateGenesisState(*genesisTime, uint64(*numValidators))
+		if err != nil {
+			log.Printf("Could not generate genesis beacon state: %v", err)
+			return
+		}
 	}
+
 	if *sszOutputFile != "" {
 		encodedState, err := genesisState.MarshalSSZ()
 		if err != nil {
-			log.Fatalf("Could not ssz marshal the genesis beacon state: %v", err)
+			log.Printf("Could not ssz marshal the genesis beacon state: %v", err)
+			return
 		}
 		if err := ioutil.WriteFile(*sszOutputFile, encodedState, 0644); err != nil {
-			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
+			return
 		}
 		log.Printf("Done writing to %s", *sszOutputFile)
 	}
 	if *yamlOutputFile != "" {
 		encodedState, err := yaml.Marshal(genesisState)
 		if err != nil {
-			log.Fatalf("Could not yaml marshal the genesis beacon state: %v", err)
+			log.Printf("Could not yaml marshal the genesis beacon state: %v", err)
+			return
 		}
 		if err := ioutil.WriteFile(*yamlOutputFile, encodedState, 0644); err != nil {
-			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
+			return
 		}
 		log.Printf("Done writing to %s", *yamlOutputFile)
 	}
 	if *jsonOutputFile != "" {
 		encodedState, err := json.Marshal(genesisState)
 		if err != nil {
-			log.Fatalf("Could not json marshal the genesis beacon state: %v", err)
+			log.Printf("Could not json marshal the genesis beacon state: %v", err)
+			return
 		}
 		if err := ioutil.WriteFile(*jsonOutputFile, encodedState, 0644); err != nil {
-			log.Fatalf("Could not write encoded genesis beacon state to file: %v", err)
+			log.Printf("Could not write encoded genesis beacon state to file: %v", err)
+			return
 		}
 		log.Printf("Done writing to %s", *jsonOutputFile)
 	}
 }
+
+func genesisStateFromJSONValidators(r io.Reader, genesisTime uint64) (*pb.BeaconState, error) {
+	enc, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	var validatorsJSON []*GenesisValidator
+	if err := json.Unmarshal(enc, &validatorsJSON); err != nil {
+		return nil, err
+	}
+	depositDataList := make([]*ethpb.Deposit_Data, len(validatorsJSON))
+	depositDataRoots := make([][]byte, len(validatorsJSON))
+	for i, val := range validatorsJSON {
+		depositDataString := val.DepositData
+		depositDataString = strings.TrimPrefix(depositDataString, "0x")
+		depositDataHex, err := hex.DecodeString(depositDataString)
+		if err != nil {
+			return nil, err
+		}
+		data := &ethpb.Deposit_Data{}
+		if err := data.UnmarshalSSZ(depositDataHex); err != nil {
+			return nil, err
+		}
+		depositDataList[i] = data
+		root, err := data.HashTreeRoot()
+		if err != nil {
+			return nil, err
+		}
+		depositDataRoots[i] = root[:]
+	}
+	beaconState, _, err := interop.GenerateGenesisStateFromDepositData(genesisTime, depositDataList, depositDataRoots)
+	if err != nil {
+		return nil, err
+	}
+	return beaconState, nil
+}

tools/genesis-state-gen/main_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/interop"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func Test_genesisStateFromJSONValidators(t *testing.T) {
+	jsonData, depositDataList := createGenesisDepositData(t)
+	jsonInput, err := json.Marshal(jsonData)
+	require.NoError(t, err)
+	genesisState, err := genesisStateFromJSONValidators(
+		bytes.NewReader(jsonInput), 0, /* genesis time defaults to time.Now() */
+	)
+	require.NoError(t, err)
+	for i, val := range genesisState.Validators {
+		assert.DeepEqual(t, val.PublicKey, depositDataList[i].PublicKey)
+	}
+}
+
+func createGenesisDepositData(t *testing.T) ([]*GenesisValidator, []*ethpb.Deposit_Data) {
+	numKeys := 5
+	pubKeys := make([]bls.PublicKey, numKeys)
+	privKeys := make([]bls.SecretKey, numKeys)
+	for i := 0; i < numKeys; i++ {
+		randKey := bls.RandKey()
+		privKeys[i] = randKey
+		pubKeys[i] = randKey.PublicKey()
+	}
+	dataList, _, err := interop.DepositDataFromKeys(privKeys, pubKeys)
+	require.NoError(t, err)
+	jsonData := make([]*GenesisValidator, numKeys)
+	for i := 0; i < numKeys; i++ {
+		data := dataList[i]
+		enc, err := data.MarshalSSZ()
+		require.NoError(t, err)
+		jsonData[i] = &GenesisValidator{
+			DepositData: fmt.Sprintf("%#x", enc),
+		}
+	}
+	return jsonData, dataList
+}