Merge changes from master into spec update

beacon_chain/spec/src/foundation.rs

@@ -114,7 +114,7 @@ fn initial_validators_for_testing() -> Vec<ValidatorRecord> {
             withdrawal_slot: u64::max_value(),
             penalized_slot: u64::max_value(),
             exit_count: 0,
-            status: From::from(0),
+            status_flags: None,
             custody_commitment: Hash256::zero(),
             latest_custody_reseed_slot: 0,
             penultimate_custody_reseed_slot: 0,

beacon_chain/types/src/attestation_data_and_custody_bit.rs

@@ -0,0 +1,60 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use rand::RngCore;
+use crate::test_utils::TestRandom;
+use super::AttestationData;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct AttestationDataAndCustodyBit {
+    pub data: AttestationData,
+    pub custody_bit: bool,
+}
+
+impl Encodable for AttestationDataAndCustodyBit {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.data);
+        s.append(&self.custody_bit);
+    }
+}
+
+impl Decodable for AttestationDataAndCustodyBit {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (data, i) = <_>::ssz_decode(bytes, i)?;
+        let (custody_bit, i) = <_>::ssz_decode(bytes, i)?;
+
+        let attestation_data_and_custody_bit = AttestationDataAndCustodyBit {
+            data,
+            custody_bit,
+        };
+
+        Ok((attestation_data_and_custody_bit, i))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for AttestationDataAndCustodyBit {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            data: <_>::random_for_test(rng),
+            custody_bit: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+    use super::*;
+    use super::super::ssz::ssz_encode;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+
+        let original = AttestationDataAndCustodyBit::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/lib.rs

@@ -26,6 +26,7 @@ pub mod shard_reassignment_record;
 pub mod slashable_vote_data;
 pub mod special_record;
 pub mod validator_record;
+pub mod validator_registry;
 
 pub mod readers;
 
@@ -50,7 +51,7 @@ pub use crate::proposer_slashing::ProposerSlashing;
 pub use crate::shard_committee::ShardCommittee;
 pub use crate::slashable_vote_data::SlashableVoteData;
 pub use crate::special_record::{SpecialRecord, SpecialRecordKind};
-pub use crate::validator_record::{ValidatorRecord, ValidatorStatus};
+pub use crate::validator_record::{StatusFlags as ValidatorStatusFlags, ValidatorRecord};
 
 pub type Hash256 = H256;
 pub type Address = H160;

beacon_chain/types/src/validator_record.rs

@@ -3,29 +3,43 @@ use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
 use ssz::{Decodable, DecodeError, Encodable, SszStream};
-use std::convert;
+
+const STATUS_FLAG_INITIATED_EXIT: u8 = 1;
+const STATUS_FLAG_WITHDRAWABLE: u8 = 2;
 
 #[derive(Debug, PartialEq, Clone, Copy)]
-pub enum ValidatorStatus {
+pub enum StatusFlags {
-    PendingActivation,
+    InitiatedExit,
-    Active,
+    Withdrawable,
-    PendingExit,
-    PendingWithdraw,
-    Withdrawn,
-    Penalized,
 }
 
-impl convert::From<u8> for ValidatorStatus {
+struct StatusFlagsDecodeError;
-    fn from(status: u8) -> Self {
+
-        match status {
+impl From<StatusFlagsDecodeError> for DecodeError {
-            0 => ValidatorStatus::PendingActivation,
+    fn from(_: StatusFlagsDecodeError) -> DecodeError {
-            1 => ValidatorStatus::Active,
+        DecodeError::Invalid
-            2 => ValidatorStatus::PendingExit,
+    }
-            3 => ValidatorStatus::PendingWithdraw,
+}
-            5 => ValidatorStatus::Withdrawn,
+
-            127 => ValidatorStatus::Penalized,
+/// Handles the serialization logic for the `status_flags` field of the `ValidatorRecord`.
-            _ => unreachable!(),
+fn status_flag_to_byte(flag: Option<StatusFlags>) -> u8 {
+    if let Some(flag) = flag {
+        match flag {
+            StatusFlags::InitiatedExit => STATUS_FLAG_INITIATED_EXIT,
+            StatusFlags::Withdrawable => STATUS_FLAG_WITHDRAWABLE,
         }
+    } else {
+        0
+    }
+}
+
+/// Handles the deserialization logic for the `status_flags` field of the `ValidatorRecord`.
+fn status_flag_from_byte(flag: u8) -> Result<Option<StatusFlags>, StatusFlagsDecodeError> {
+    match flag {
+        0 => Ok(None),
+        1 => Ok(Some(StatusFlags::InitiatedExit)),
+        2 => Ok(Some(StatusFlags::Withdrawable)),
+        _ => Err(StatusFlagsDecodeError),
     }
 }
 
@@ -40,59 +54,44 @@ pub struct ValidatorRecord {
     pub withdrawal_slot: u64,
     pub penalized_slot: u64,
     pub exit_count: u64,
-    pub status: ValidatorStatus,
+    pub status_flags: Option<StatusFlags>,
     pub custody_commitment: Hash256,
     pub latest_custody_reseed_slot: u64,
     pub penultimate_custody_reseed_slot: u64,
 }
 
 impl ValidatorRecord {
-    pub fn status_is(&self, status: ValidatorStatus) -> bool {
+    /// This predicate indicates if the validator represented by this record is considered "active" at `slot`.
-        self.status == status
+    pub fn is_active_at(&self, slot: u64) -> bool {
+        self.activation_slot <= slot && slot < self.exit_slot
     }
 }
 
-impl Encodable for ValidatorStatus {
+impl Default for ValidatorRecord {
-    fn ssz_append(&self, s: &mut SszStream) {
+    /// Yields a "default" `ValidatorRecord`. Primarily used for testing.
-        let byte: u8 = match self {
+    fn default() -> Self {
-            ValidatorStatus::PendingActivation => 0,
+        Self {
-            ValidatorStatus::Active => 1,
+            pubkey: PublicKey::default(),
-            ValidatorStatus::PendingExit => 2,
+            withdrawal_credentials: Hash256::default(),
-            ValidatorStatus::PendingWithdraw => 3,
+            randao_commitment: Hash256::default(),
-            ValidatorStatus::Withdrawn => 5,
+            randao_layers: 0,
-            ValidatorStatus::Penalized => 127,
+            activation_slot: std::u64::MAX,
-        };
+            exit_slot: std::u64::MAX,
-        s.append(&byte);
+            withdrawal_slot: std::u64::MAX,
+            penalized_slot: std::u64::MAX,
+            exit_count: 0,
+            status_flags: None,
+            custody_commitment: Hash256::default(),
+            latest_custody_reseed_slot: 0, // NOTE: is `GENESIS_SLOT`
+            penultimate_custody_reseed_slot: 0, // NOTE: is `GENESIS_SLOT`
+        }
     }
 }
 
-impl Decodable for ValidatorStatus {
+impl<T: RngCore> TestRandom<T> for StatusFlags {
-    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
-        let (byte, i) = u8::ssz_decode(bytes, i)?;
-        let status = match byte {
-            0 => ValidatorStatus::PendingActivation,
-            1 => ValidatorStatus::Active,
-            2 => ValidatorStatus::PendingExit,
-            3 => ValidatorStatus::PendingWithdraw,
-            5 => ValidatorStatus::Withdrawn,
-            127 => ValidatorStatus::Penalized,
-            _ => return Err(DecodeError::Invalid),
-        };
-        Ok((status, i))
-    }
-}
-
-impl<T: RngCore> TestRandom<T> for ValidatorStatus {
     fn random_for_test(rng: &mut T) -> Self {
-        let options = vec![
+        let options = vec![StatusFlags::InitiatedExit, StatusFlags::Withdrawable];
-            ValidatorStatus::PendingActivation,
+        options[(rng.next_u32() as usize) % options.len()].clone()
-            ValidatorStatus::Active,
-            ValidatorStatus::PendingExit,
-            ValidatorStatus::PendingWithdraw,
-            ValidatorStatus::Withdrawn,
-            ValidatorStatus::Penalized,
-        ];
-        options[(rng.next_u32() as usize) % options.len()]
     }
 }
 
@@ -107,7 +106,7 @@ impl Encodable for ValidatorRecord {
         s.append(&self.withdrawal_slot);
         s.append(&self.penalized_slot);
         s.append(&self.exit_count);
-        s.append(&self.status);
+        s.append(&status_flag_to_byte(self.status_flags));
         s.append(&self.custody_commitment);
         s.append(&self.latest_custody_reseed_slot);
         s.append(&self.penultimate_custody_reseed_slot);
@@ -125,11 +124,13 @@ impl Decodable for ValidatorRecord {
         let (withdrawal_slot, i) = <_>::ssz_decode(bytes, i)?;
         let (penalized_slot, i) = <_>::ssz_decode(bytes, i)?;
         let (exit_count, i) = <_>::ssz_decode(bytes, i)?;
-        let (status, i) = <_>::ssz_decode(bytes, i)?;
+        let (status_flags_byte, i): (u8, usize) = <_>::ssz_decode(bytes, i)?;
         let (custody_commitment, i) = <_>::ssz_decode(bytes, i)?;
         let (latest_custody_reseed_slot, i) = <_>::ssz_decode(bytes, i)?;
         let (penultimate_custody_reseed_slot, i) = <_>::ssz_decode(bytes, i)?;
 
+        let status_flags = status_flag_from_byte(status_flags_byte)?;
+
         Ok((
             Self {
                 pubkey,
@@ -141,7 +142,7 @@ impl Decodable for ValidatorRecord {
                 withdrawal_slot,
                 penalized_slot,
                 exit_count,
-                status,
+                status_flags,
                 custody_commitment,
                 latest_custody_reseed_slot,
                 penultimate_custody_reseed_slot,
@@ -163,7 +164,7 @@ impl<T: RngCore> TestRandom<T> for ValidatorRecord {
             withdrawal_slot: <_>::random_for_test(rng),
             penalized_slot: <_>::random_for_test(rng),
             exit_count: <_>::random_for_test(rng),
-            status: <_>::random_for_test(rng),
+            status_flags: Some(<_>::random_for_test(rng)),
             custody_commitment: <_>::random_for_test(rng),
             latest_custody_reseed_slot: <_>::random_for_test(rng),
             penultimate_custody_reseed_slot: <_>::random_for_test(rng),
@@ -189,13 +190,24 @@ mod tests {
     }
 
     #[test]
-    pub fn test_validator_status_ssz_round_trip() {
+    fn test_validator_can_be_active() {
         let mut rng = XorShiftRng::from_seed([42; 16]);
-        let original = ValidatorStatus::random_for_test(&mut rng);
+        let mut validator = ValidatorRecord::random_for_test(&mut rng);
 
-        let bytes = ssz_encode(&original);
+        let activation_slot = u64::random_for_test(&mut rng);
-        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+        let exit_slot = activation_slot + 234;
 
-        assert_eq!(original, decoded);
+        validator.activation_slot = activation_slot;
+        validator.exit_slot = exit_slot;
+
+        for slot in (activation_slot - 100)..(exit_slot + 100) {
+            if slot < activation_slot {
+                assert!(!validator.is_active_at(slot));
+            } else if slot >= exit_slot {
+                assert!(!validator.is_active_at(slot));
+            } else {
+                assert!(validator.is_active_at(slot));
+            }
+        }
     }
 }

beacon_chain/types/src/validator_registry.rs

@@ -0,0 +1,171 @@
+/// Contains logic to manipulate a `&[ValidatorRecord]`.
+/// For now, we avoid defining a newtype and just have flat functions here.
+use super::validator_record::*;
+
+/// Given an indexed sequence of `validators`, return the indices corresponding to validators that are active at `slot`.
+pub fn get_active_validator_indices(validators: &[ValidatorRecord], slot: u64) -> Vec<usize> {
+    validators
+        .iter()
+        .enumerate()
+        .filter_map(|(index, validator)| {
+            if validator.is_active_at(slot) {
+                Some(index)
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng};
+
+    #[test]
+    fn can_get_empty_active_validator_indices() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+
+        let validators = vec![];
+        let some_slot = u64::random_for_test(&mut rng);
+        let indices = get_active_validator_indices(&validators, some_slot);
+        assert_eq!(indices, vec![]);
+    }
+
+    #[test]
+    fn can_get_no_active_validator_indices() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let mut validators = vec![];
+        let count_validators = 10;
+        for _ in 0..count_validators {
+            validators.push(ValidatorRecord::default())
+        }
+
+        let some_slot = u64::random_for_test(&mut rng);
+        let indices = get_active_validator_indices(&validators, some_slot);
+        assert_eq!(indices, vec![]);
+    }
+
+    #[test]
+    fn can_get_all_active_validator_indices() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let count_validators = 10;
+        let some_slot = u64::random_for_test(&mut rng);
+
+        let mut validators = (0..count_validators)
+            .into_iter()
+            .map(|_| {
+                let mut validator = ValidatorRecord::default();
+
+                let activation_offset = u64::random_for_test(&mut rng);
+                let exit_offset = u64::random_for_test(&mut rng);
+
+                validator.activation_slot = some_slot.checked_sub(activation_offset).unwrap_or(0);
+                validator.exit_slot = some_slot.checked_add(exit_offset).unwrap_or(std::u64::MAX);
+
+                validator
+            })
+            .collect::<Vec<_>>();
+
+        // test boundary condition by ensuring that at least one validator in the list just activated
+        if let Some(validator) = validators.get_mut(0) {
+            validator.activation_slot = some_slot;
+        }
+
+        let indices = get_active_validator_indices(&validators, some_slot);
+        assert_eq!(
+            indices,
+            (0..count_validators).into_iter().collect::<Vec<_>>()
+        );
+    }
+
+    fn set_validators_to_default_entry_exit(validators: &mut [ValidatorRecord]) {
+        for validator in validators.iter_mut() {
+            validator.activation_slot = std::u64::MAX;
+            validator.exit_slot = std::u64::MAX;
+        }
+    }
+
+    // sets all `validators` to be active as of some slot prior to `slot`. returns the activation slot.
+    fn set_validators_to_activated(validators: &mut [ValidatorRecord], slot: u64) -> u64 {
+        let activation_slot = slot - 10;
+        for validator in validators.iter_mut() {
+            validator.activation_slot = activation_slot;
+        }
+        activation_slot
+    }
+
+    // sets all `validators` to be exited as of some slot before `slot`.
+    fn set_validators_to_exited(
+        validators: &mut [ValidatorRecord],
+        slot: u64,
+        activation_slot: u64,
+    ) {
+        assert!(activation_slot < slot);
+        let mut exit_slot = activation_slot + 10;
+        while exit_slot >= slot {
+            exit_slot -= 1;
+        }
+        assert!(activation_slot < exit_slot && exit_slot < slot);
+
+        for validator in validators.iter_mut() {
+            validator.exit_slot = exit_slot;
+        }
+    }
+
+    #[test]
+    fn can_get_some_active_validator_indices() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        const COUNT_PARTITIONS: usize = 3;
+        const COUNT_VALIDATORS: usize = 3 * COUNT_PARTITIONS;
+        let some_slot: u64 = u64::random_for_test(&mut rng);
+
+        let mut validators = (0..COUNT_VALIDATORS)
+            .into_iter()
+            .map(|_| {
+                let mut validator = ValidatorRecord::default();
+
+                let activation_offset = u64::random_for_test(&mut rng);
+                let exit_offset = u64::random_for_test(&mut rng);
+
+                validator.activation_slot = some_slot.checked_sub(activation_offset).unwrap_or(0);
+                validator.exit_slot = some_slot.checked_add(exit_offset).unwrap_or(std::u64::MAX);
+
+                validator
+            })
+            .collect::<Vec<_>>();
+
+        // we partition the set into partitions based on lifecycle:
+        for (i, chunk) in validators.chunks_exact_mut(COUNT_PARTITIONS).enumerate() {
+            match i {
+                0 => {
+                    // 1. not activated (Default::default())
+                    set_validators_to_default_entry_exit(chunk);
+                }
+                1 => {
+                    // 2. activated, but not exited
+                    set_validators_to_activated(chunk, some_slot);
+                    // test boundary condition by ensuring that at least one validator in the list just activated
+                    if let Some(validator) = chunk.get_mut(0) {
+                        validator.activation_slot = some_slot;
+                    }
+                }
+                2 => {
+                    // 3. exited
+                    let activation_slot = set_validators_to_activated(chunk, some_slot);
+                    set_validators_to_exited(chunk, some_slot, activation_slot);
+                    // test boundary condition by ensuring that at least one validator in the list just exited
+                    if let Some(validator) = chunk.get_mut(0) {
+                        validator.exit_slot = some_slot;
+                    }
+                }
+                _ => unreachable!(
+                    "constants local to this test not in sync with generation of test case"
+                ),
+            }
+        }
+
+        let indices = get_active_validator_indices(&validators, some_slot);
+        assert_eq!(indices, vec![3, 4, 5]);
+    }
+}

beacon_chain/utils/bls/src/public_key.rs

@@ -1,6 +1,7 @@
 use super::SecretKey;
 use bls_aggregates::PublicKey as RawPublicKey;
 use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+use std::default;
 
 /// A single BLS signature.
 ///
@@ -20,6 +21,13 @@ impl PublicKey {
     }
 }
 
+impl default::Default for PublicKey {
+    fn default() -> Self {
+        let secret_key = SecretKey::random();
+        PublicKey::from_secret_key(&secret_key)
+    }
+}
+
 impl Encodable for PublicKey {
     fn ssz_append(&self, s: &mut SszStream) {
         s.append_vec(&self.0.as_bytes());

beacon_chain/validator_induction/src/inductor.rs

@@ -1,6 +1,6 @@
 use bls::verify_proof_of_possession;
 use spec::ChainSpec;
-use types::{BeaconState, Deposit, ValidatorRecord, ValidatorStatus};
+use types::{BeaconState, Deposit, ValidatorRecord};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum ValidatorInductionError {
@@ -49,7 +49,7 @@ pub fn process_deposit(
                 withdrawal_slot: spec.far_future_slot,
                 penalized_slot: spec.far_future_slot,
                 exit_count: 0,
-                status: ValidatorStatus::PendingActivation,
+                status_flags: None,
                 custody_commitment: deposit_input.custody_commitment,
                 latest_custody_reseed_slot: 0,
                 penultimate_custody_reseed_slot: 0,

beacon_chain/validator_shuffling/src/shuffle.rs

@@ -2,7 +2,8 @@ use std::cmp::min;
 
 use honey_badger_split::SplitExt;
 use spec::ChainSpec;
-use types::{ShardCommittee, ValidatorRecord, ValidatorStatus};
+use types::validator_registry::get_active_validator_indices;
+use types::{ShardCommittee, ValidatorRecord};
 use vec_shuffle::{shuffle, ShuffleErr};
 
 type DelegatedCycle = Vec<Vec<ShardCommittee>>;
@@ -24,17 +25,7 @@ pub fn shard_and_committees_for_cycle(
     spec: &ChainSpec,
 ) -> Result<DelegatedCycle, ValidatorAssignmentError> {
     let shuffled_validator_indices = {
-        let validator_indices = validators
+        let validator_indices = get_active_validator_indices(validators, 0);
-            .iter()
-            .enumerate()
-            .filter_map(|(i, validator)| {
-                if validator.status_is(ValidatorStatus::Active) {
-                    Some(i)
-                } else {
-                    None
-                }
-            })
-            .collect();
         shuffle(seed, validator_indices)?
     };
     let shard_indices: Vec<usize> = (0_usize..spec.shard_count as usize).into_iter().collect();