Add basic YAML test_harness tests

Works, however ignores a lot of fields in the YAML.

beacon_node/beacon_chain/test_harness/Cargo.toml

@@ -4,6 +4,14 @@ version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
+[[bin]]
+name = "test_harness"
+path = "src/bin.rs"
+
+[lib]
+name = "test_harness"
+path = "src/lib.rs"
+
 [[bench]]
 name = "state_transition"
 harness = false
@@ -18,6 +26,7 @@ beacon_chain = { path = "../../beacon_chain" }
 block_proposer = { path = "../../../eth2/block_proposer" }
 bls = { path = "../../../eth2/utils/bls" }
 boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" }
+clap = "2.32.0"
 db = { path = "../../db" }
 parking_lot = "0.7"
 failure = "0.1"
@@ -33,3 +42,4 @@ serde_json = "1.0"
 slot_clock = { path = "../../../eth2/utils/slot_clock" }
 ssz = { path = "../../../eth2/utils/ssz" }
 types = { path = "../../../eth2/types" }
+yaml-rust = "0.4.2"

beacon_node/beacon_chain/test_harness/examples/chain.yaml

@@ -0,0 +1,77 @@
+title: Sample Ethereum Serenity State Transition Tests
+summary: Testing full state transition block processing
+test_suite: prysm
+fork: sapphire
+version: 1.0
+test_cases:
+  - config:
+      epoch_length: 64
+      deposits_for_chain_start: 1000
+      num_slots: 32 # Testing advancing state to slot < SlotsPerEpoch
+    results:
+      slot: 32
+      num_validators: 1000
+  - config:
+      epoch_length: 64
+      deposits_for_chain_start: 16384
+      num_slots: 64
+      deposits:
+        - slot: 1
+          amount: 32
+          merkle_index: 0
+          pubkey: !!binary |
+            SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0=
+        - slot: 15
+          amount: 32
+          merkle_index: 1
+          pubkey: !!binary |
+            Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd
+        - slot: 55
+          amount: 32
+          merkle_index: 2
+          pubkey: !!binary |
+            LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd
+      proposer_slashings:
+        - slot: 16 # At slot 16, we trigger a proposal slashing occurring
+          proposer_index: 16385 # We penalize the proposer that was just added from slot 15
+          proposal_1_shard: 0
+          proposal_1_slot: 15
+          proposal_1_root: !!binary |
+            LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd
+          proposal_2_shard: 0
+          proposal_2_slot: 15
+          proposal_2_root: !!binary |
+            LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd
+      attester_slashings:
+        - slot: 59 # At slot 59, we trigger a attester slashing
+          slashable_vote_data_1_slot: 55
+          slashable_vote_data_2_slot: 55
+          slashable_vote_data_1_justified_slot: 0
+          slashable_vote_data_2_justified_slot: 1
+          slashable_vote_data_1_custody_0_indices: [16386]
+          slashable_vote_data_1_custody_1_indices: []
+          slashable_vote_data_2_custody_0_indices: []
+          slashable_vote_data_2_custody_1_indices: [16386]
+    results:
+      slot: 64
+      num_validators: 16387
+      penalized_validators: [16385, 16386] # We test that the validators at indices 16385, 16386 were indeed penalized
+  - config:
+      skip_slots: [10, 20]
+      epoch_length: 64
+      deposits_for_chain_start: 1000
+      num_slots: 128 # Testing advancing state's slot == 2*SlotsPerEpoch
+      deposits:
+        - slot: 10
+          amount: 32
+          merkle_index: 0
+          pubkey: !!binary |
+            SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0=
+        - slot: 20
+          amount: 32
+          merkle_index: 1
+          pubkey: !!binary |
+            Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd
+    results:
+      slot: 128
+      num_validators: 1000 # Validator registry should not have grown if slots 10 and 20 were skipped

beacon_node/beacon_chain/test_harness/src/bin.rs

@@ -0,0 +1,148 @@
+use self::beacon_chain_harness::BeaconChainHarness;
+use self::validator_harness::ValidatorHarness;
+use clap::{App, Arg};
+use env_logger::{Builder, Env};
+use log::info;
+use std::{fs::File, io::prelude::*};
+use types::*;
+use yaml_rust::{Yaml, YamlLoader};
+
+mod beacon_chain_harness;
+mod validator_harness;
+
+fn main() {
+    let matches = App::new("Lighthouse Test Harness Runner")
+        .version("0.0.1")
+        .author("Sigma Prime <contact@sigmaprime.io>")
+        .about("Runs `test_harness` using a YAML manifest.")
+        .arg(
+            Arg::with_name("yaml")
+                .long("yaml")
+                .value_name("FILE")
+                .help("YAML file manifest.")
+                .required(true),
+        )
+        .get_matches();
+
+    Builder::from_env(Env::default().default_filter_or("debug")).init();
+
+    if let Some(yaml_file) = matches.value_of("yaml") {
+        let docs = {
+            let mut file = File::open(yaml_file).unwrap();
+
+            let mut yaml_str = String::new();
+            file.read_to_string(&mut yaml_str).unwrap();
+
+            YamlLoader::load_from_str(&yaml_str).unwrap()
+        };
+
+        for doc in &docs {
+            for test_case in doc["test_cases"].as_vec().unwrap() {
+                let manifest = Manifest::from_yaml(test_case);
+                manifest.execute();
+            }
+        }
+    }
+}
+
+struct Manifest {
+    pub results: Results,
+    pub config: Config,
+}
+
+impl Manifest {
+    pub fn from_yaml(test_case: &Yaml) -> Self {
+        Self {
+            results: Results::from_yaml(&test_case["results"]),
+            config: Config::from_yaml(&test_case["config"]),
+        }
+    }
+
+    fn spec(&self) -> ChainSpec {
+        let mut spec = ChainSpec::foundation();
+
+        if let Some(n) = self.config.epoch_length {
+            spec.epoch_length = n;
+        }
+
+        spec
+    }
+
+    pub fn execute(&self) {
+        let spec = self.spec();
+        let validator_count = self.config.deposits_for_chain_start;
+        let slots = self.results.slot;
+
+        info!(
+            "Building BeaconChainHarness with {} validators...",
+            validator_count
+        );
+
+        let mut harness = BeaconChainHarness::new(spec, validator_count);
+
+        info!("Starting simulation across {} slots...", slots);
+
+        for _ in 0..self.results.slot {
+            harness.advance_chain_with_block();
+        }
+
+        harness.run_fork_choice();
+
+        let dump = harness.chain_dump().expect("Chain dump failed.");
+
+        assert_eq!(dump.len() as u64, slots + 1); // + 1 for genesis block.
+
+        // harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump);
+    }
+}
+
+struct Results {
+    pub slot: u64,
+    pub num_validators: Option<usize>,
+    pub slashed_validators: Option<Vec<u64>>,
+    pub exited_validators: Option<Vec<u64>>,
+}
+
+impl Results {
+    pub fn from_yaml(yaml: &Yaml) -> Self {
+        Self {
+            slot: as_u64(&yaml, "slot").expect("Must have end slot"),
+            num_validators: as_usize(&yaml, "num_validators"),
+            slashed_validators: as_vec_u64(&yaml, "slashed_validators"),
+            exited_validators: as_vec_u64(&yaml, "exited_validators"),
+        }
+    }
+}
+
+struct Config {
+    pub deposits_for_chain_start: usize,
+    pub epoch_length: Option<u64>,
+}
+
+impl Config {
+    pub fn from_yaml(yaml: &Yaml) -> Self {
+        Self {
+            deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start")
+                .expect("Must specify validator count"),
+            epoch_length: as_u64(&yaml, "epoch_length"),
+        }
+    }
+}
+
+fn as_usize(yaml: &Yaml, key: &str) -> Option<usize> {
+    yaml[key].as_i64().and_then(|n| Some(n as usize))
+}
+
+fn as_u64(yaml: &Yaml, key: &str) -> Option<u64> {
+    yaml[key].as_i64().and_then(|n| Some(n as u64))
+}
+
+fn as_vec_u64(yaml: &Yaml, key: &str) -> Option<Vec<u64>> {
+    yaml[key].clone().into_vec().and_then(|vec| {
+        Some(
+            vec.iter()
+                .map(|item| item.as_i64().unwrap() as u64)
+                .collect(),
+        )
+    })
+}