Allow test_harness to load validators from file.

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -11,6 +11,7 @@ use rayon::prelude::*;
 use slot_clock::TestingSlotClock;
 use ssz::TreeHash;
 use std::collections::HashSet;
+use std::fs::File;
 use std::iter::FromIterator;
 use std::path::Path;
 use std::sync::Arc;
@@ -54,22 +55,44 @@ impl BeaconChainHarness {
             block_hash: Hash256::zero(),
         };
 
-        let (keypairs, initial_validator_deposits) = if let Some(path) = validators_dir {
+        let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec);
-            let keypairs_path = path.join("keypairs.yaml");
+
-            let deposits_path = path.join("deposits.yaml");
+        // If a `validators_dir` is specified, load the keypairs and validators from YAML files.
-            load_deposits_from_file(
+        //
-                validator_count,
+        // Otherwise, build all the keypairs and initial validator deposits manually.
-                &keypairs_path.as_path(),
+        //
-                &deposits_path.as_path(),
+        // It is _much_ faster to load from YAML, however it does skip all the initial processing
-            )
+        // and verification of `Deposits`, so it is a slightly less comprehensive test.
+        let keypairs = if let Some(path) = validators_dir {
+            debug!("Loading validator keypairs from file...");
+            let keypairs_file = File::open(path.join("keypairs.yaml")).unwrap();
+            let mut keypairs: Vec<Keypair> = serde_yaml::from_reader(&keypairs_file).unwrap();
+            keypairs.truncate(validator_count);
+
+            debug!("Loading validators from file...");
+            let validators_file = File::open(path.join("validators.yaml")).unwrap();
+            let mut validators: Vec<Validator> = serde_yaml::from_reader(&validators_file).unwrap();
+            validators.truncate(validator_count);
+
+            let balances = vec![32_000_000_000; validator_count];
+
+            state_builder.import_existing_validators(
+                validators,
+                balances,
+                validator_count as u64,
+                &spec,
+            );
+
+            keypairs
         } else {
+            debug!("Generating validator keypairs...");
             let keypairs = generate_deterministic_keypairs(validator_count);
+            debug!("Generating initial validator deposits...");
             let deposits = generate_deposits_from_keypairs(&keypairs, genesis_time, &spec);
-            (keypairs, deposits)
+            state_builder.process_initial_deposits(&deposits, &spec);
+            keypairs
         };
 
-        let mut state_builder = BeaconStateBuilder::new(genesis_time, latest_eth1_data, &spec);
-        state_builder.process_initial_deposits(&initial_validator_deposits, &spec);
         let genesis_state = state_builder.build(&spec).unwrap();
         let state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
         let genesis_block = BeaconBlock::genesis(state_root, &spec);

beacon_node/beacon_chain/test_harness/src/prepare.rs

@@ -1,6 +1,5 @@
-use crate::beacon_chain_harness::{
+use crate::beacon_chain_harness::generate_deterministic_keypairs;
-    generate_deposits_from_keypairs, generate_deterministic_keypairs,
+use bls::get_withdrawal_credentials;
-};
 use clap::{value_t, ArgMatches};
 use log::debug;
 use serde_yaml;
@@ -9,33 +8,52 @@ use std::{fs, fs::File};
 use types::*;
 
 const KEYPAIRS_FILE: &str = "keypairs.yaml";
-const DEPOSITS_FILE: &str = "deposits.yaml";
+const VALIDATORS_FILE: &str = "validators.yaml";
 
 pub fn prepare(matches: &ArgMatches, spec: &ChainSpec) {
     let validator_count = value_t!(matches.value_of("validator_count"), usize)
         .expect("Validator count is required argument");
-    let genesis_time =
-        value_t!(matches.value_of("genesis_time"), u64).expect("Genesis time is required argument");
     let output_dir = matches
         .value_of("output_dir")
         .expect("Output dir has a default value.");
 
-    debug!("Created keypairs and deposits, writing to file...");
+    debug!("Created keypairs and validators, writing to file...");
 
     fs::create_dir_all(Path::new(output_dir)).unwrap();
 
-    // Ensure that keypairs is dropped before writing deposits, this provides a big memory saving
+    // Ensure that keypairs is dropped before writing validators, this provides a big memory saving
     // for large validator_counts.
-    let deposits = {
+    let validators: Vec<Validator> = {
         debug!("Creating {} keypairs...", validator_count);
         let keypairs = generate_deterministic_keypairs(validator_count);
         debug!("Writing {} keypairs to file...", validator_count);
         write_keypairs(output_dir, &keypairs);
-        debug!("Creating {} deposits to file...", validator_count);
+        debug!("Creating {} validators...", validator_count);
-        generate_deposits_from_keypairs(&keypairs, genesis_time, &spec)
+        keypairs
+            .iter()
+            .map(|keypair| generate_validator(&keypair, spec))
+            .collect()
     };
-    debug!("Writing {} deposits to file...", validator_count);
+
-    write_deposits(output_dir, &deposits);
+    debug!("Writing {} validators to file...", validator_count);
+    write_validators(output_dir, &validators);
+}
+
+fn generate_validator(keypair: &Keypair, spec: &ChainSpec) -> Validator {
+    let withdrawal_credentials = Hash256::from_slice(&get_withdrawal_credentials(
+        &keypair.pk,
+        spec.bls_withdrawal_prefix_byte,
+    ));
+
+    Validator {
+        pubkey: keypair.pk.clone(),
+        withdrawal_credentials,
+        activation_epoch: spec.far_future_epoch,
+        exit_epoch: spec.far_future_epoch,
+        withdrawable_epoch: spec.far_future_epoch,
+        initiated_exit: false,
+        slashed: false,
+    }
 }
 
 fn write_keypairs(output_dir: &str, keypairs: &[Keypair]) {
@@ -44,8 +62,8 @@ fn write_keypairs(output_dir: &str, keypairs: &[Keypair]) {
     serde_yaml::to_writer(keypairs_file, &keypairs).unwrap();
 }
 
-fn write_deposits(output_dir: &str, deposits: &[Deposit]) {
+fn write_validators(output_dir: &str, validators: &[Validator]) {
-    let deposits_path = Path::new(output_dir).join(DEPOSITS_FILE);
+    let validators_path = Path::new(output_dir).join(VALIDATORS_FILE);
-    let deposits_file = File::create(deposits_path).unwrap();
+    let validators_file = File::create(validators_path).unwrap();
-    serde_yaml::to_writer(deposits_file, &deposits).unwrap();
+    serde_yaml::to_writer(validators_file, &validators).unwrap();
 }

eth2/types/Cargo.toml

@@ -17,6 +17,7 @@ rand = "0.5.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
+serde_yaml = "0.8"
 slog = "^2.2.3"
 ssz = { path = "../utils/ssz" }
 ssz_derive = { path = "../utils/ssz_derive" }

eth2/types/src/beacon_state/builder.rs

@@ -33,7 +33,7 @@ impl BeaconStateBuilder {
         }
     }
 
-    /// Produce the first state of the Beacon Chain.
+    /// Process deposit objects.
     ///
     /// Spec v0.4.0
     pub fn process_initial_deposits(
@@ -48,19 +48,47 @@ impl BeaconStateBuilder {
 
         self.state.process_deposits(deposit_data, spec);
 
+        self.activate_genesis_validators(spec);
+
+        self.state.deposit_index = initial_validator_deposits.len() as u64;
+    }
+
+    fn activate_genesis_validators(&mut self, spec: &ChainSpec) {
         for validator_index in 0..self.state.validator_registry.len() {
             if self.state.get_effective_balance(validator_index, spec) >= spec.max_deposit_amount {
                 self.state.activate_validator(validator_index, true, spec);
             }
         }
-
-        self.state.deposit_index = initial_validator_deposits.len() as u64;
     }
 
-    /// Builds a `BeaconState` using the `BeaconState::genesis(..)` function.
+    /// Instantiate the validator registry from a YAML file.
     ///
-    /// Each validator is assigned a unique, randomly-generated keypair and all
+    /// This skips a lot of signing and verification, useful for fast test setups.
-    /// proof-of-possessions are verified during genesis.
+    ///
+    /// Spec v0.4.0
+    pub fn import_existing_validators(
+        &mut self,
+        validators: Vec<Validator>,
+        initial_balances: Vec<u64>,
+        deposit_index: u64,
+        spec: &ChainSpec,
+    ) {
+        self.state.validator_registry = validators;
+
+        assert_eq!(
+            self.state.validator_registry.len(),
+            initial_balances.len(),
+            "Not enough balances for validators"
+        );
+
+        self.state.validator_balances = initial_balances;
+
+        self.activate_genesis_validators(spec);
+
+        self.state.deposit_index = deposit_index;
+    }
+
+    /// Updates the final state variables and returns a fully built genesis state.
     ///
     /// Spec v0.4.0
     pub fn build(mut self, spec: &ChainSpec) -> Result<BeaconState, BeaconStateError> {

eth2/types/src/slot_epoch.rs

@@ -12,7 +12,7 @@ use crate::slot_height::SlotHeight;
 /// may lead to programming errors which are not detected by the compiler.
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use slog;
 use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash};
 use std::cmp::{Ord, Ordering};
@@ -21,10 +21,10 @@ use std::hash::{Hash, Hasher};
 use std::iter::Iterator;
 use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign};
 
-#[derive(Eq, Debug, Clone, Copy, Default, Serialize)]
+#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)]
 pub struct Slot(u64);
 
-#[derive(Eq, Debug, Clone, Copy, Default, Serialize)]
+#[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)]
 pub struct Epoch(u64);
 
 impl_common!(Slot);

eth2/types/src/validator.rs

@@ -1,13 +1,13 @@
 use crate::{test_utils::TestRandom, Epoch, Hash256, PublicKey};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Information about a `BeaconChain` validator.
 ///
 /// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TestRandom, TreeHash)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)]
 pub struct Validator {
     pub pubkey: PublicKey,
     pub withdrawal_credentials: Hash256,