From 4db2f082e133697493794194bf29750a371d429d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 2 Mar 2019 20:17:14 +1100 Subject: [PATCH] Add state-checks to test_harness YAML Runs tests against a state at some slot --- .../test_harness/examples/chain.yaml | 11 ++- .../test_harness/src/manifest/config.rs | 1 + .../test_harness/src/manifest/mod.rs | 66 +++++++-------- .../test_harness/src/manifest/results.rs | 24 ++++-- .../test_harness/src/manifest/state_check.rs | 84 +++++++++++++++++++ eth2/types/src/validator.rs | 12 ++- 6 files changed, 151 insertions(+), 47 deletions(-) create mode 100644 beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs diff --git a/beacon_node/beacon_chain/test_harness/examples/chain.yaml b/beacon_node/beacon_chain/test_harness/examples/chain.yaml index 9e00f4569..f22d0874e 100644 --- a/beacon_node/beacon_chain/test_harness/examples/chain.yaml +++ b/beacon_node/beacon_chain/test_harness/examples/chain.yaml @@ -7,7 +7,7 @@ test_cases: - config: epoch_length: 64 deposits_for_chain_start: 1000 - num_slots: 65 + num_slots: 64 skip_slots: [2, 3] deposits: - slot: 1 @@ -34,4 +34,11 @@ test_cases: - slot: 5 validator_indices: [14] results: - num_validators: 1000 + num_skipped_slots: 2 + states: + - slot: 63 + num_validators: 1003 + # slashed_validators: [11, 12, 13, 14, 42] + slashed_validators: [13, 42] # This line is incorrect, our implementation isn't processing attester_slashings. + exited_validators: [] + diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs index f960a0bb8..0e66a120b 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -7,6 +7,7 @@ pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, u64); pub type AttesterSlashingTuple = (u64, Vec); +#[derive(Debug)] pub struct Config { pub deposits_for_chain_start: usize, pub epoch_length: Option, diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs index 3149a7d16..d16912205 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -11,8 +11,10 @@ use yaml_rust::Yaml; mod config; mod results; +mod state_check; mod yaml_helpers; +#[derive(Debug)] pub struct Manifest { pub results: Results, pub config: Config, @@ -20,6 +22,7 @@ pub struct Manifest { pub struct ExecutionResult { pub chain: Vec, + pub spec: ChainSpec, } impl Manifest { @@ -54,7 +57,8 @@ impl Manifest { info!("Starting simulation across {} slots...", slots); - for slot_height in 0..slots { + // -1 slots because genesis counts as a slot. + for slot_height in 0..slots - 1 { // Feed deposits to the BeaconChain. if let Some(ref deposits) = self.config.deposits { for (slot, deposit, keypair) in deposits { @@ -113,51 +117,41 @@ impl Manifest { ExecutionResult { chain: harness.chain_dump().expect("Chain dump failed."), + spec: (*harness.spec).clone(), } } - pub fn assert_result_valid(&self, result: ExecutionResult) { + pub fn assert_result_valid(&self, execution_result: ExecutionResult) { info!("Verifying test results..."); + let spec = &execution_result.spec; - let skipped_slots = self - .config - .skip_slots - .clone() - .and_then(|slots| Some(slots.len())) - .unwrap_or_else(|| 0); - let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots; - - assert_eq!(result.chain.len(), expected_blocks); - - info!( - "OK: Chain length is {} ({} skipped slots).", - result.chain.len(), - skipped_slots - ); - - if let Some(ref skip_slots) = self.config.skip_slots { - for checkpoint in &result.chain { - let block_slot = checkpoint.beacon_block.slot.as_u64(); - assert!( - !skip_slots.contains(&block_slot), - "Slot {} was not skipped.", - block_slot - ); - } - info!("OK: Skipped slots not present in chain."); - } - - if let Some(ref deposits) = self.config.deposits { - let latest_state = &result.chain.last().expect("Empty chain.").beacon_state; + if let Some(num_skipped_slots) = self.results.num_skipped_slots { assert_eq!( - latest_state.validator_registry.len(), - self.config.deposits_for_chain_start + deposits.len() + execution_result.chain.len(), + self.config.num_slots as usize - num_skipped_slots, + "actual skipped slots != expected." ); info!( - "OK: Validator registry has {} more validators.", - deposits.len() + "OK: Chain length is {} ({} skipped slots).", + execution_result.chain.len(), + num_skipped_slots ); } + + if let Some(ref state_checks) = self.results.state_checks { + for checkpoint in &execution_result.chain { + let state = &checkpoint.beacon_state; + + for state_check in state_checks { + let adjusted_state_slot = + state.slot - spec.genesis_epoch.start_slot(spec.epoch_length); + + if state_check.slot == adjusted_state_slot { + state_check.assert_valid(state, spec); + } + } + } + } } } diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs index 2d84e12dc..844695910 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs @@ -1,18 +1,28 @@ -use super::yaml_helpers::{as_usize, as_vec_u64}; +use super::state_check::StateCheck; +use super::yaml_helpers::as_usize; use yaml_rust::Yaml; +#[derive(Debug)] pub struct Results { - pub num_validators: Option, - pub slashed_validators: Option>, - pub exited_validators: Option>, + pub num_skipped_slots: Option, + pub state_checks: Option>, } impl Results { pub fn from_yaml(yaml: &Yaml) -> Self { Self { - num_validators: as_usize(&yaml, "num_validators"), - slashed_validators: as_vec_u64(&yaml, "slashed_validators"), - exited_validators: as_vec_u64(&yaml, "exited_validators"), + num_skipped_slots: as_usize(yaml, "num_skipped_slots"), + state_checks: parse_state_checks(yaml), } } } + +fn parse_state_checks(yaml: &Yaml) -> Option> { + let mut states = vec![]; + + for state_yaml in yaml["states"].as_vec()? { + states.push(StateCheck::from_yaml(state_yaml)); + } + + Some(states) +} diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs new file mode 100644 index 000000000..0415d4896 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs @@ -0,0 +1,84 @@ +use super::yaml_helpers::{as_u64, as_usize, as_vec_u64}; +use log::info; +use types::*; +use yaml_rust::Yaml; + +#[derive(Debug)] +pub struct StateCheck { + pub slot: Slot, + pub num_validators: Option, + pub slashed_validators: Option>, + pub exited_validators: Option>, +} + +impl StateCheck { + pub fn from_yaml(yaml: &Yaml) -> Self { + Self { + slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")), + num_validators: as_usize(&yaml, "num_validators"), + slashed_validators: as_vec_u64(&yaml, "slashed_validators"), + exited_validators: as_vec_u64(&yaml, "exited_validators"), + } + } + + pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) { + let state_epoch = state.slot.epoch(spec.epoch_length); + + info!("Running state check for slot height {}.", self.slot); + + assert_eq!( + self.slot, + state.slot - spec.genesis_epoch.start_slot(spec.epoch_length), + "State slot is invalid." + ); + + if let Some(num_validators) = self.num_validators { + assert_eq!( + state.validator_registry.len(), + num_validators, + "State validator count != expected." + ); + info!("OK: num_validators = {}.", num_validators); + } + + if let Some(ref slashed_validators) = self.slashed_validators { + let actually_slashed_validators: Vec = state + .validator_registry + .iter() + .enumerate() + .filter_map(|(i, validator)| { + if validator.is_penalized_at(state_epoch) { + Some(i as u64) + } else { + None + } + }) + .collect(); + assert_eq!( + actually_slashed_validators, *slashed_validators, + "Slashed validators != expected." + ); + info!("OK: slashed_validators = {:?}.", slashed_validators); + } + + if let Some(ref exited_validators) = self.exited_validators { + let actually_exited_validators: Vec = state + .validator_registry + .iter() + .enumerate() + .filter_map(|(i, validator)| { + if validator.is_exited_at(state_epoch) { + Some(i as u64) + } else { + None + } + }) + .collect(); + assert_eq!( + actually_exited_validators, *exited_validators, + "Exited validators != expected." + ); + info!("OK: exited_validators = {:?}.", exited_validators); + } + } +} diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index b832283a0..bc8d467ec 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -55,8 +55,16 @@ pub struct Validator { impl Validator { /// This predicate indicates if the validator represented by this record is considered "active" at `slot`. - pub fn is_active_at(&self, slot: Epoch) -> bool { - self.activation_epoch <= slot && slot < self.exit_epoch + pub fn is_active_at(&self, epoch: Epoch) -> bool { + self.activation_epoch <= epoch && epoch < self.exit_epoch + } + + pub fn is_exited_at(&self, epoch: Epoch) -> bool { + self.exit_epoch <= epoch + } + + pub fn is_penalized_at(&self, epoch: Epoch) -> bool { + self.penalized_epoch <= epoch } }