From 8b06fa31da5e44c4a438d297d842695ab4dd315b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Feb 2019 23:13:00 +1100 Subject: [PATCH] Add basic YAML test_harness tests Works, however ignores a lot of fields in the YAML. --- .../beacon_chain/test_harness/Cargo.toml | 10 ++ .../test_harness/examples/chain.yaml | 77 +++++++++ .../beacon_chain/test_harness/src/bin.rs | 148 ++++++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 beacon_node/beacon_chain/test_harness/examples/chain.yaml create mode 100644 beacon_node/beacon_chain/test_harness/src/bin.rs diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 657cc7955..bd7a58270 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -4,6 +4,14 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bin]] +name = "test_harness" +path = "src/bin.rs" + +[lib] +name = "test_harness" +path = "src/lib.rs" + [[bench]] name = "state_transition" harness = false @@ -18,6 +26,7 @@ beacon_chain = { path = "../../beacon_chain" } block_proposer = { path = "../../../eth2/block_proposer" } bls = { path = "../../../eth2/utils/bls" } boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" } +clap = "2.32.0" db = { path = "../../db" } parking_lot = "0.7" failure = "0.1" @@ -33,3 +42,4 @@ serde_json = "1.0" slot_clock = { path = "../../../eth2/utils/slot_clock" } ssz = { path = "../../../eth2/utils/ssz" } types = { path = "../../../eth2/types" } +yaml-rust = "0.4.2" diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml new file mode 100644 index 000000000..5d8e34795 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -0,0 +1,77 @@ +title: Sample Ethereum Serenity State Transition Tests +summary: Testing full state transition block processing +test_suite: prysm +fork: sapphire +version: 1.0 +test_cases: + - config: + epoch_length: 64 + deposits_for_chain_start: 1000 + num_slots: 32 # Testing advancing state to slot < SlotsPerEpoch + results: + slot: 32 + num_validators: 1000 + - config: + epoch_length: 64 + deposits_for_chain_start: 16384 + num_slots: 64 + deposits: + - slot: 1 + amount: 32 + merkle_index: 0 + pubkey: !!binary | + SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= + - slot: 15 + amount: 32 + merkle_index: 1 + pubkey: !!binary | + Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd + - slot: 55 + amount: 32 + merkle_index: 2 + pubkey: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + proposer_slashings: + - slot: 16 # At slot 16, we trigger a proposal slashing occurring + proposer_index: 16385 # We penalize the proposer that was just added from slot 15 + proposal_1_shard: 0 + proposal_1_slot: 15 + proposal_1_root: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + proposal_2_shard: 0 + proposal_2_slot: 15 + proposal_2_root: !!binary | + LkmqmqoodLKAslkjdkajsdljasdkajlksjdasldjasdd + attester_slashings: + - slot: 59 # At slot 59, we trigger a attester slashing + slashable_vote_data_1_slot: 55 + slashable_vote_data_2_slot: 55 + slashable_vote_data_1_justified_slot: 0 + slashable_vote_data_2_justified_slot: 1 + slashable_vote_data_1_custody_0_indices: [16386] + slashable_vote_data_1_custody_1_indices: [] + slashable_vote_data_2_custody_0_indices: [] + slashable_vote_data_2_custody_1_indices: [16386] + results: + slot: 64 + num_validators: 16387 + penalized_validators: [16385, 16386] # We test that the validators at indices 16385, 16386 were indeed penalized + - config: + skip_slots: [10, 20] + epoch_length: 64 + deposits_for_chain_start: 1000 + num_slots: 128 # Testing advancing state's slot == 2*SlotsPerEpoch + deposits: + - slot: 10 + amount: 32 + merkle_index: 0 + pubkey: !!binary | + SlAAbShSkUg7PLiPHZI/rTS1uAvKiieOrifPN6Moso0= + - slot: 20 + amount: 32 + merkle_index: 1 + pubkey: !!binary | + Oklajsjdkaklsdlkajsdjlajslkdjlkasjlkdjlajdsd + results: + slot: 128 + num_validators: 1000 # Validator registry should not have grown if slots 10 and 20 were skipped diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs new file mode 100644 index 000000000..007ec3f60 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -0,0 +1,148 @@ +use self::beacon_chain_harness::BeaconChainHarness; +use self::validator_harness::ValidatorHarness; +use clap::{App, Arg}; +use env_logger::{Builder, Env}; +use log::info; +use std::{fs::File, io::prelude::*}; +use types::*; +use yaml_rust::{Yaml, YamlLoader}; + +mod beacon_chain_harness; +mod validator_harness; + +fn main() { + let matches = App::new("Lighthouse Test Harness Runner") + .version("0.0.1") + .author("Sigma Prime ") + .about("Runs `test_harness` using a YAML manifest.") + .arg( + Arg::with_name("yaml") + .long("yaml") + .value_name("FILE") + .help("YAML file manifest.") + .required(true), + ) + .get_matches(); + + Builder::from_env(Env::default().default_filter_or("debug")).init(); + + if let Some(yaml_file) = matches.value_of("yaml") { + let docs = { + let mut file = File::open(yaml_file).unwrap(); + + let mut yaml_str = String::new(); + file.read_to_string(&mut yaml_str).unwrap(); + + YamlLoader::load_from_str(&yaml_str).unwrap() + }; + + for doc in &docs { + for test_case in doc["test_cases"].as_vec().unwrap() { + let manifest = Manifest::from_yaml(test_case); + manifest.execute(); + } + } + } +} + +struct Manifest { + pub results: Results, + pub config: Config, +} + +impl Manifest { + pub fn from_yaml(test_case: &Yaml) -> Self { + Self { + results: Results::from_yaml(&test_case["results"]), + config: Config::from_yaml(&test_case["config"]), + } + } + + fn spec(&self) -> ChainSpec { + let mut spec = ChainSpec::foundation(); + + if let Some(n) = self.config.epoch_length { + spec.epoch_length = n; + } + + spec + } + + pub fn execute(&self) { + let spec = self.spec(); + let validator_count = self.config.deposits_for_chain_start; + let slots = self.results.slot; + + info!( + "Building BeaconChainHarness with {} validators...", + validator_count + ); + + let mut harness = BeaconChainHarness::new(spec, validator_count); + + info!("Starting simulation across {} slots...", slots); + + for _ in 0..self.results.slot { + harness.advance_chain_with_block(); + } + + harness.run_fork_choice(); + + let dump = harness.chain_dump().expect("Chain dump failed."); + + assert_eq!(dump.len() as u64, slots + 1); // + 1 for genesis block. + + // harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump); + } +} + +struct Results { + pub slot: u64, + pub num_validators: Option, + pub slashed_validators: Option>, + pub exited_validators: Option>, +} + +impl Results { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + slot: as_u64(&yaml, "slot").expect("Must have end slot"), + num_validators: as_usize(&yaml, "num_validators"), + slashed_validators: as_vec_u64(&yaml, "slashed_validators"), + exited_validators: as_vec_u64(&yaml, "exited_validators"), + } + } +} + +struct Config { + pub deposits_for_chain_start: usize, + pub epoch_length: Option, +} + +impl Config { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") + .expect("Must specify validator count"), + epoch_length: as_u64(&yaml, "epoch_length"), + } + } +} + +fn as_usize(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as usize)) +} + +fn as_u64(yaml: &Yaml, key: &str) -> Option { + yaml[key].as_i64().and_then(|n| Some(n as u64)) +} + +fn as_vec_u64(yaml: &Yaml, key: &str) -> Option> { + yaml[key].clone().into_vec().and_then(|vec| { + Some( + vec.iter() + .map(|item| item.as_i64().unwrap() as u64) + .collect(), + ) + }) +}