Add state-checks to test_harness YAML

Runs tests against a state at some slot

beacon_node/beacon_chain/test_harness/examples/chain.yaml

@@ -7,7 +7,7 @@ test_cases:
   - config:
       epoch_length: 64
       deposits_for_chain_start: 1000
-      num_slots: 65
+      num_slots: 64
       skip_slots: [2, 3]
       deposits:
         - slot: 1
@@ -34,4 +34,11 @@ test_cases:
         - slot: 5
           validator_indices: [14]
     results:
-      num_validators: 1000
+      num_skipped_slots: 2
+      states:
+        - slot: 63
+          num_validators: 1003
+          # slashed_validators: [11, 12, 13, 14, 42]
+          slashed_validators: [13, 42] # This line is incorrect, our implementation isn't processing attester_slashings.
+          exited_validators: []
+

beacon_node/beacon_chain/test_harness/src/manifest/config.rs

@@ -7,6 +7,7 @@ pub type DepositTuple = (u64, Deposit, Keypair);
 pub type ProposerSlashingTuple = (u64, u64);
 pub type AttesterSlashingTuple = (u64, Vec<u64>);
 
+#[derive(Debug)]
 pub struct Config {
     pub deposits_for_chain_start: usize,
     pub epoch_length: Option<u64>,

beacon_node/beacon_chain/test_harness/src/manifest/mod.rs

@@ -11,8 +11,10 @@ use yaml_rust::Yaml;
 
 mod config;
 mod results;
+mod state_check;
 mod yaml_helpers;
 
+#[derive(Debug)]
 pub struct Manifest {
     pub results: Results,
     pub config: Config,
@@ -20,6 +22,7 @@ pub struct Manifest {
 
 pub struct ExecutionResult {
     pub chain: Vec<CheckPoint>,
+    pub spec: ChainSpec,
 }
 
 impl Manifest {
@@ -54,7 +57,8 @@ impl Manifest {
 
         info!("Starting simulation across {} slots...", slots);
 
-        for slot_height in 0..slots {
+        // -1 slots because genesis counts as a slot.
+        for slot_height in 0..slots - 1 {
             // Feed deposits to the BeaconChain.
             if let Some(ref deposits) = self.config.deposits {
                 for (slot, deposit, keypair) in deposits {
@@ -113,51 +117,41 @@ impl Manifest {
 
         ExecutionResult {
             chain: harness.chain_dump().expect("Chain dump failed."),
+            spec: (*harness.spec).clone(),
         }
     }
 
-    pub fn assert_result_valid(&self, result: ExecutionResult) {
+    pub fn assert_result_valid(&self, execution_result: ExecutionResult) {
         info!("Verifying test results...");
+        let spec = &execution_result.spec;
 
-        let skipped_slots = self
-            .config
-            .skip_slots
-            .clone()
-            .and_then(|slots| Some(slots.len()))
-            .unwrap_or_else(|| 0);
-        let expected_blocks = self.config.num_slots as usize + 1 - skipped_slots;
-
-        assert_eq!(result.chain.len(), expected_blocks);
-
-        info!(
-            "OK: Chain length is {} ({} skipped slots).",
-            result.chain.len(),
-            skipped_slots
-        );
-
-        if let Some(ref skip_slots) = self.config.skip_slots {
-            for checkpoint in &result.chain {
-                let block_slot = checkpoint.beacon_block.slot.as_u64();
-                assert!(
-                    !skip_slots.contains(&block_slot),
-                    "Slot {} was not skipped.",
-                    block_slot
-                );
-            }
-            info!("OK: Skipped slots not present in chain.");
-        }
-
-        if let Some(ref deposits) = self.config.deposits {
-            let latest_state = &result.chain.last().expect("Empty chain.").beacon_state;
+        if let Some(num_skipped_slots) = self.results.num_skipped_slots {
             assert_eq!(
-                latest_state.validator_registry.len(),
-                self.config.deposits_for_chain_start + deposits.len()
+                execution_result.chain.len(),
+                self.config.num_slots as usize - num_skipped_slots,
+                "actual skipped slots != expected."
             );
             info!(
-                "OK: Validator registry has {} more validators.",
-                deposits.len()
+                "OK: Chain length is {} ({} skipped slots).",
+                execution_result.chain.len(),
+                num_skipped_slots
             );
         }
+
+        if let Some(ref state_checks) = self.results.state_checks {
+            for checkpoint in &execution_result.chain {
+                let state = &checkpoint.beacon_state;
+
+                for state_check in state_checks {
+                    let adjusted_state_slot =
+                        state.slot - spec.genesis_epoch.start_slot(spec.epoch_length);
+
+                    if state_check.slot == adjusted_state_slot {
+                        state_check.assert_valid(state, spec);
+                    }
+                }
+            }
+        }
     }
 }
 

beacon_node/beacon_chain/test_harness/src/manifest/results.rs

@@ -1,18 +1,28 @@
-use super::yaml_helpers::{as_usize, as_vec_u64};
+use super::state_check::StateCheck;
+use super::yaml_helpers::as_usize;
 use yaml_rust::Yaml;
 
+#[derive(Debug)]
 pub struct Results {
-    pub num_validators: Option<usize>,
-    pub slashed_validators: Option<Vec<u64>>,
-    pub exited_validators: Option<Vec<u64>>,
+    pub num_skipped_slots: Option<usize>,
+    pub state_checks: Option<Vec<StateCheck>>,
 }
 
 impl Results {
     pub fn from_yaml(yaml: &Yaml) -> Self {
         Self {
-            num_validators: as_usize(&yaml, "num_validators"),
-            slashed_validators: as_vec_u64(&yaml, "slashed_validators"),
-            exited_validators: as_vec_u64(&yaml, "exited_validators"),
+            num_skipped_slots: as_usize(yaml, "num_skipped_slots"),
+            state_checks: parse_state_checks(yaml),
         }
     }
 }
+
+fn parse_state_checks(yaml: &Yaml) -> Option<Vec<StateCheck>> {
+    let mut states = vec![];
+
+    for state_yaml in yaml["states"].as_vec()? {
+        states.push(StateCheck::from_yaml(state_yaml));
+    }
+
+    Some(states)
+}

beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs

@@ -0,0 +1,84 @@
+use super::yaml_helpers::{as_u64, as_usize, as_vec_u64};
+use log::info;
+use types::*;
+use yaml_rust::Yaml;
+
+#[derive(Debug)]
+pub struct StateCheck {
+    pub slot: Slot,
+    pub num_validators: Option<usize>,
+    pub slashed_validators: Option<Vec<u64>>,
+    pub exited_validators: Option<Vec<u64>>,
+}
+
+impl StateCheck {
+    pub fn from_yaml(yaml: &Yaml) -> Self {
+        Self {
+            slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")),
+            num_validators: as_usize(&yaml, "num_validators"),
+            slashed_validators: as_vec_u64(&yaml, "slashed_validators"),
+            exited_validators: as_vec_u64(&yaml, "exited_validators"),
+        }
+    }
+
+    pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) {
+        let state_epoch = state.slot.epoch(spec.epoch_length);
+
+        info!("Running state check for slot height {}.", self.slot);
+
+        assert_eq!(
+            self.slot,
+            state.slot - spec.genesis_epoch.start_slot(spec.epoch_length),
+            "State slot is invalid."
+        );
+
+        if let Some(num_validators) = self.num_validators {
+            assert_eq!(
+                state.validator_registry.len(),
+                num_validators,
+                "State validator count != expected."
+            );
+            info!("OK: num_validators = {}.", num_validators);
+        }
+
+        if let Some(ref slashed_validators) = self.slashed_validators {
+            let actually_slashed_validators: Vec<u64> = state
+                .validator_registry
+                .iter()
+                .enumerate()
+                .filter_map(|(i, validator)| {
+                    if validator.is_penalized_at(state_epoch) {
+                        Some(i as u64)
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+            assert_eq!(
+                actually_slashed_validators, *slashed_validators,
+                "Slashed validators != expected."
+            );
+            info!("OK: slashed_validators = {:?}.", slashed_validators);
+        }
+
+        if let Some(ref exited_validators) = self.exited_validators {
+            let actually_exited_validators: Vec<u64> = state
+                .validator_registry
+                .iter()
+                .enumerate()
+                .filter_map(|(i, validator)| {
+                    if validator.is_exited_at(state_epoch) {
+                        Some(i as u64)
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+            assert_eq!(
+                actually_exited_validators, *exited_validators,
+                "Exited validators != expected."
+            );
+            info!("OK: exited_validators = {:?}.", exited_validators);
+        }
+    }
+}

eth2/types/src/validator.rs

@@ -55,8 +55,16 @@ pub struct Validator {
 
 impl Validator {
     /// This predicate indicates if the validator represented by this record is considered "active" at `slot`.
-    pub fn is_active_at(&self, slot: Epoch) -> bool {
-        self.activation_epoch <= slot && slot < self.exit_epoch
+    pub fn is_active_at(&self, epoch: Epoch) -> bool {
+        self.activation_epoch <= epoch && epoch < self.exit_epoch
+    }
+
+    pub fn is_exited_at(&self, epoch: Epoch) -> bool {
+        self.exit_epoch <= epoch
+    }
+
+    pub fn is_penalized_at(&self, epoch: Epoch) -> bool {
+        self.penalized_epoch <= epoch
     }
 }