From 9dcec214af0f3208115e1bf5075e21c2074f8b15 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 13:00:25 +1000 Subject: [PATCH 01/29] Add block validation code This was created in another branch along with a whole bunch of other commits building out databases and other common elements. This full history of these works is available on master, however it should not be necessary. --- lighthouse/state/attestation_record/mod.rs | 7 + .../validation/attestation_validation.rs | 203 ++++++++++++++++ .../validation/message_generation.rs | 32 +++ .../attestation_record/validation/mod.rs | 16 ++ .../validation/signature_verification.rs | 183 ++++++++++++++ lighthouse/state/block/mod.rs | 5 +- lighthouse/state/block/validation/mod.rs | 19 ++ lighthouse/state/block/validation/tests.rs | 191 +++++++++++++++ .../block/validation/validate_ssz_block.rs | 227 ++++++++++++++++++ lighthouse/state/mod.rs | 1 + 10 files changed, 883 insertions(+), 1 deletion(-) create mode 100644 lighthouse/state/attestation_record/validation/attestation_validation.rs create mode 100644 lighthouse/state/attestation_record/validation/message_generation.rs create mode 100644 lighthouse/state/attestation_record/validation/mod.rs create mode 100644 lighthouse/state/attestation_record/validation/signature_verification.rs create mode 100644 lighthouse/state/block/validation/mod.rs create mode 100644 lighthouse/state/block/validation/tests.rs create mode 100644 lighthouse/state/block/validation/validate_ssz_block.rs diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index 5751bcd24..8f09f9f36 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -1,10 +1,13 @@ use super::bls; +use super::common; +use super::db; use super::ssz; use super::utils; mod structs; mod ssz_splitter; +mod validation; pub use self::structs::{ AttestationRecord, @@ -15,3 +18,7 @@ pub use self::ssz_splitter::{ split_one_attestation, AttestationSplitError, }; +pub use self::validation::{ + validate_attestation, + AttestationValidationError, +}; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs new file mode 100644 index 000000000..389b9f3c3 --- /dev/null +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -0,0 +1,203 @@ +use std::collections::HashSet; +use std::sync::Arc; +use super::structs::AttestationRecord; +use super::AttesterMap; +use super::attestation_parent_hashes::{ + attestation_parent_hashes, + ParentHashesError, +}; +use super::db::{ + ClientDB, + DBError +}; +use super::db::stores::{ + BlockStore, + ValidatorStore, +}; +use super::utils::types::{ + Hash256, +}; +use super::message_generation::generate_signed_message; +use super::signature_verification::{ + verify_aggregate_signature_for_indices, + SignatureVerificationError, +}; + +#[derive(Debug,PartialEq)] +pub enum AttestationValidationError { + SlotTooHigh, + SlotTooLow, + JustifiedSlotTooHigh, + UnknownJustifiedBlock, + TooManyObliqueHashes, + BadCurrentHashes, + BadObliqueHashes, + BadAttesterMap, + IntWrapping, + PublicKeyCorrupt, + NoPublicKeyForValidator, + BadBitfieldLength, + InvalidBitfield, + InvalidBitfieldEndBits, + NoSignatures, + NonZeroTrailingBits, + AggregateSignatureFail, + DBError(String), +} + +fn bytes_for_bits(bits: usize) -> usize { + (bits.saturating_sub(1) / 8) + 1 +} + +fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool { + let shift = 8_u16.saturating_sub(n as u16); + ((!0 << shift) & u16::from(byte)) > 0 +} + +pub fn validate_attestation(a: &AttestationRecord, + block_slot: u64, + cycle_length: u8, + known_last_justified_slot: u64, + known_parent_hashes: &Arc>, + block_store: &Arc>, + validator_store: &Arc>, + attester_map: &Arc) + -> Result>, AttestationValidationError> + where T: ClientDB + Sized +{ + /* + * The attesation slot must not be higher than the block that contained it. + */ + if a.slot > block_slot { + return Err(AttestationValidationError::SlotTooHigh); + } + + /* + * The slot of this attestation must not be more than cycle_length + 1 distance + * from the block that contained it. + * + * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. + */ + if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) { + return Err(AttestationValidationError::SlotTooLow); + } + + /* + * The attestation must indicate that its last justified slot is the same as the last justified + * slot known to us. + */ + if a.justified_slot > known_last_justified_slot { + return Err(AttestationValidationError::JustifiedSlotTooHigh); + } + + /* + * There is no need to include more oblique parents hashes than there are blocks + * in a cycle. + */ + if a.oblique_parent_hashes.len() > usize::from(cycle_length) { + return Err(AttestationValidationError::TooManyObliqueHashes); + } + + /* + * Retrieve the set of attestation indices for this slot and shard id. + * + * This is an array mapping the order that validators will appear in the bitfield to the + * canonincal index of a validator. + */ + let attestation_indices = attester_map.get(&(a.slot, a.shard_id)) + .ok_or(AttestationValidationError::BadAttesterMap)?; + + /* + * The bitfield must be no longer than the minimum required to represent each validator in the + * attestation indicies for this slot and shard id. + */ + if a.attester_bitfield.num_bytes() != + bytes_for_bits(attestation_indices.len()) + { + return Err(AttestationValidationError::BadBitfieldLength); + } + + /* + * If there are excess bits in the bitfield because the number of a validators in not a + * multiple of 8, reject this attestation record. + * + * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to + * refer to the same AttesationRecord. + */ + let last_byte = + a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) + .ok_or(AttestationValidationError::InvalidBitfield)?; + if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) { + return Err(AttestationValidationError::InvalidBitfieldEndBits) + } + + /* + * The specified justified block hash must be known to us + */ + if !block_store.block_exists(&a.justified_block_hash)? { + return Err(AttestationValidationError::UnknownJustifiedBlock) + } + + let signed_message = { + let parent_hashes = attestation_parent_hashes( + cycle_length, + block_slot, + a.slot, + &known_parent_hashes, + &a.oblique_parent_hashes)?; + generate_signed_message( + a.slot, + &parent_hashes, + a.shard_id, + &a.shard_block_hash, + a.justified_slot) + }; + + let voted_hashmap = + verify_aggregate_signature_for_indices( + &signed_message, + &a.aggregate_sig, + &attestation_indices, + &a.attester_bitfield, + &validator_store)?; + + Ok(voted_hashmap) +} + +impl From for AttestationValidationError { + fn from(e: ParentHashesError) -> Self { + match e { + ParentHashesError::BadCurrentHashes + => AttestationValidationError::BadCurrentHashes, + ParentHashesError::BadObliqueHashes + => AttestationValidationError::BadObliqueHashes, + ParentHashesError::SlotTooLow + => AttestationValidationError::SlotTooLow, + ParentHashesError::SlotTooHigh + => AttestationValidationError::SlotTooHigh, + ParentHashesError::IntWrapping + => AttestationValidationError::IntWrapping + } + } +} + +impl From for AttestationValidationError { + fn from(e: DBError) -> Self { + AttestationValidationError::DBError(e.message) + } +} + +impl From for AttestationValidationError { + fn from(e: SignatureVerificationError) -> Self { + match e { + SignatureVerificationError::BadValidatorIndex + => AttestationValidationError::BadAttesterMap, + SignatureVerificationError::PublicKeyCorrupt + => AttestationValidationError::PublicKeyCorrupt, + SignatureVerificationError::NoPublicKeyForValidator + => AttestationValidationError::NoPublicKeyForValidator, + SignatureVerificationError::DBError(s) + => AttestationValidationError::DBError(s), + } + } +} diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs new file mode 100644 index 000000000..3bcc4c16a --- /dev/null +++ b/lighthouse/state/attestation_record/validation/message_generation.rs @@ -0,0 +1,32 @@ +use super::ssz::SszStream; +use super::utils::hash::canonical_hash; +use super::utils::types::Hash256; + +/// Generates the message used to validate the signature provided with an AttestationRecord. +/// +/// Ensures that the signer of the message has a view of the chain that is compatible with ours. +pub fn generate_signed_message(slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) + -> Vec +{ + /* + * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ + * (for example, SSZ might change whilst this doesn't). + * + * I have suggested switching this to ssz here: + * https://github.com/ethereum/eth2.0-specs/issues/5 + * + * If this doesn't happen, it would be safer to not use SSZ at all. + */ + let mut ssz_stream = SszStream::new(); + ssz_stream.append(&slot); + ssz_stream.append_vec(&parent_hashes.to_vec()); + ssz_stream.append(&shard_id); + ssz_stream.append(shard_block_hash); + ssz_stream.append(&justified_slot); + let bytes = ssz_stream.drain(); + canonical_hash(&bytes) +} diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs new file mode 100644 index 000000000..43ae6fc0c --- /dev/null +++ b/lighthouse/state/attestation_record/validation/mod.rs @@ -0,0 +1,16 @@ +use super::common::maps::AttesterMap; +use super::db; +use super::bls; +use super::structs; +use super::ssz; +use super::common::attestation_parent_hashes; +use super::utils; + +mod attestation_validation; +mod signature_verification; +mod message_generation; + +pub use self::attestation_validation::{ + validate_attestation, + AttestationValidationError, +}; diff --git a/lighthouse/state/attestation_record/validation/signature_verification.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs new file mode 100644 index 000000000..c33dc2806 --- /dev/null +++ b/lighthouse/state/attestation_record/validation/signature_verification.rs @@ -0,0 +1,183 @@ +use std::collections::HashSet; +use super::bls::{ + AggregateSignature, + AggregatePublicKey, +}; +use super::db::ClientDB; +use super::db::stores::{ + ValidatorStore, + ValidatorStoreError, +}; +use super::utils::types::Bitfield; + +#[derive(Debug, PartialEq)] +pub enum SignatureVerificationError { + BadValidatorIndex, + PublicKeyCorrupt, + NoPublicKeyForValidator, + DBError(String), +} + +/// Verify an aggregate signature across the supplied message. +/// +/// The public keys used for verification are collected by mapping +/// each true bitfield bit to canonical ValidatorRecord index through +/// the attestation_indicies map. +/// +/// Each public key is loaded from the store on-demand. +pub fn verify_aggregate_signature_for_indices( + message: &[u8], + agg_sig: &AggregateSignature, + attestation_indices: &[usize], + bitfield: &Bitfield, + validator_store: &ValidatorStore) + -> Result>, SignatureVerificationError> + where T: ClientDB + Sized +{ + let mut voters = HashSet::new(); + let mut agg_pub_key = AggregatePublicKey::new(); + + for i in 0..attestation_indices.len() { + let voted = bitfield.get_bit(i); + if voted { + /* + * De-reference the attestation index into a canonical ValidatorRecord index. + */ + let validator = *attestation_indices.get(i) + .ok_or(SignatureVerificationError::BadValidatorIndex)?; + /* + * Load the validators public key from our store. + */ + let pub_key = validator_store + .get_public_key_by_index(validator)? + .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; + /* + * Add the validators public key to the aggregate public key. + */ + agg_pub_key.add(&pub_key); + /* + * Add to the validator to the set of voters for this attestation record. + */ + voters.insert(validator); + } + } + /* + * Verify the aggregate public key against the aggregate signature. + * + * This verification will only succeed if the exact set of public keys + * were added to the aggregate public key as those that signed the aggregate signature. + */ + if agg_sig.verify(&message, &agg_pub_key) { + Ok(Some(voters)) + } else { + Ok(None) + } +} + +impl From for SignatureVerificationError { + fn from(error: ValidatorStoreError) -> Self { + match error { + ValidatorStoreError::DBError(s) => + SignatureVerificationError::DBError(s), + ValidatorStoreError::DecodeError => + SignatureVerificationError::PublicKeyCorrupt, + } + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use super::super::bls::{ + Keypair, + Signature, + }; + use super::super::db::MemoryDB; + use std::sync::Arc; + + /* + * Cases that still need testing: + * + * - No signatures. + * - Database failure. + * - Unknown validator index. + * - Extra validator on signature. + */ + + #[test] + fn test_signature_verification() { + let message = "cats".as_bytes(); + let signing_keypairs = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + let non_signing_keypairs = vec![ + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + Keypair::random(), + ]; + /* + * Signing keypairs first, then non-signing + */ + let mut all_keypairs = signing_keypairs.clone(); + all_keypairs.append(&mut non_signing_keypairs.clone()); + + let attestation_indices: Vec = (0..all_keypairs.len()) + .collect(); + let mut bitfield = Bitfield::new(); + for i in 0..signing_keypairs.len() { + bitfield.set_bit(i, true); + } + + let db = Arc::new(MemoryDB::open()); + let store = ValidatorStore::new(db); + + for (i, keypair) in all_keypairs.iter().enumerate() { + store.put_public_key_by_index(i, &keypair.pk).unwrap(); + } + + let mut agg_sig = AggregateSignature::new(); + for keypair in &signing_keypairs { + let sig = Signature::new(&message, &keypair.sk); + agg_sig.add(&sig); + } + + /* + * Test using all valid parameters. + */ + let voters = verify_aggregate_signature_for_indices( + &message, + &agg_sig, + &attestation_indices, + &bitfield, + &store).unwrap(); + + let voters = voters.unwrap(); + (0..signing_keypairs.len()) + .for_each(|i| assert!(voters.contains(&i))); + (signing_keypairs.len()..non_signing_keypairs.len()) + .for_each(|i| assert!(!voters.contains(&i))); + + /* + * Add another validator to the bitfield, run validation will all other + * parameters the same and assert that it fails. + */ + bitfield.set_bit(signing_keypairs.len() + 1, true); + let voters = verify_aggregate_signature_for_indices( + &message, + &agg_sig, + &attestation_indices, + &bitfield, + &store).unwrap(); + + assert_eq!(voters, None); + } +} diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 04b70e6dd..1b8a19439 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -1,11 +1,14 @@ extern crate blake2_rfc; +use super::attestation_record; +use super::common; +use super::db; use super::ssz; use super::utils; -use super::attestation_record; mod structs; mod ssz_block; +mod validation; pub use self::structs::Block; pub use self::ssz_block::SszBlock; diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs new file mode 100644 index 000000000..59aa3fc15 --- /dev/null +++ b/lighthouse/state/block/validation/mod.rs @@ -0,0 +1,19 @@ +mod validate_ssz_block; +#[cfg(test)] +mod tests; + +use super::attestation_record; +use super::SszBlock; +use super::db; +use super::ssz; +use super::utils; + +use super::common::maps::{ + AttesterMap, + ProposerMap, +}; +pub use self::validate_ssz_block::{ + validate_ssz_block, + SszBlockValidationError, + BlockStatus, +}; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs new file mode 100644 index 000000000..3bb5c9c18 --- /dev/null +++ b/lighthouse/state/block/validation/tests.rs @@ -0,0 +1,191 @@ +extern crate ssz; + +use self::ssz::{ + SszStream, +}; +use std::sync::Arc; +use super::{ + validate_ssz_block, + BlockStatus, + AttesterMap, + ProposerMap, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::db::{ + MemoryDB, +}; +use super::utils::hash::canonical_hash; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use super::SszBlock; +use super::super::Block; +use super::super::attestation_record::AttestationRecord; +use super::super::super::bls::{ + Keypair, + Signature, + AggregateSignature, +}; + +struct TestStore { + db: Arc, + block: Arc>, + pow_chain: Arc>, + validator: Arc>, +} + +impl TestStore { + pub fn new() -> Self { + let db = Arc::new(MemoryDB::open()); + let block = Arc::new(BlockStore::new(db.clone())); + let pow_chain = Arc::new(PoWChainStore::new(db.clone())); + let validator = Arc::new(ValidatorStore::new(db.clone())); + Self { + db, + block, + pow_chain, + validator, + } + } +} + +#[test] +fn test_block_validation() { + let stores = TestStore::new(); + + let cycle_length: u8 = 2; + let shard_count: u16 = 2; + let present_slot = u64::from(cycle_length) * 10000; + let justified_slot = present_slot - u64::from(cycle_length); + let justified_block_hash = Hash256::from("justified_hash".as_bytes()); + let shard_block_hash = Hash256::from("shard_hash".as_bytes()); + let parent_hashes: Vec = (0..(cycle_length * 2)) + .map(|i| Hash256::from(i as u64)) + .collect(); + let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); + let active_state_root = Hash256::from("active_state".as_bytes()); + let crystallized_state_root = Hash256::from("cry_state".as_bytes()); + + stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); + stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); + + let validators_per_shard = 10; + + let block_slot = present_slot; + let validator_index: usize = 0; + let proposer_map = { + let mut proposer_map = ProposerMap::new(); + proposer_map.insert(present_slot, validator_index); + proposer_map + }; + let (attester_map, attestations, _keypairs) = { + let mut i = 0; + let mut attester_map = AttesterMap::new(); + let mut attestations = vec![]; + let mut keypairs = vec![]; + for shard in 0..shard_count { + let mut attesters = vec![]; + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + let attestation_slot = block_slot - 1; + + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + + let attestation_message = { + let mut stream = SszStream::new(); + stream.append(&attestation_slot); + stream.append_vec(&parent_hashes_slice.to_vec()); + stream.append(&shard); + stream.append(&shard_block_hash); + stream.append(&justified_slot); + let bytes = stream.drain(); + canonical_hash(&bytes) + }; + + + + for attestation_index in 0..validators_per_shard { + /* + * Add the attester to the attestation indices for this shard. + */ + attesters.push(i); + /* + * Set the voters bit on the bitfield to true. + */ + attester_bitfield.set_bit(attestation_index, true); + /* + * Generate a random keypair for this validatior and clone it into the + * list of keypairs. + */ + let keypair = Keypair::random(); + keypairs.push(keypair.clone()); + /* + * Store the validators public key in the database. + */ + stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); + /* + * Generate a new signature and aggregate it on the rolling signature. + */ + let sig = Signature::new(&attestation_message, &keypair.sk); + aggregate_sig.add(&sig); + /* + * Increment the validator counter to monotonically assign validators. + */ + i += 1; + } + + attester_map.insert((attestation_slot, shard), attesters); + let attestation = AttestationRecord { + slot: attestation_slot, + shard_id: shard, + oblique_parent_hashes: vec![], + shard_block_hash, + attester_bitfield, + justified_slot, + justified_block_hash, + aggregate_sig, + }; + attestations.push(attestation); + } + (attester_map, attestations, keypairs) + }; + + let block = Block { + parent_hash: Hash256::from("parent".as_bytes()), + slot_number: block_slot, + randao_reveal: Hash256::from("randao".as_bytes()), + attestations, + pow_chain_ref, + active_state_root, + crystallized_state_root, + }; + + let mut stream = SszStream::new(); + stream.append(&block); + let serialized_block = stream.drain(); + let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); + + let status = validate_ssz_block( + &ssz_block, + present_slot, + cycle_length, + justified_slot, + &Arc::new(parent_hashes), + &Arc::new(proposer_map), + &Arc::new(attester_map), + &stores.block.clone(), + &stores.validator.clone(), + &stores.pow_chain.clone()).unwrap(); + + assert_eq!(status, BlockStatus::NewBlock); +} diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs new file mode 100644 index 000000000..39b0551af --- /dev/null +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -0,0 +1,227 @@ +use std::sync::Arc; +use super::attestation_record::{ + validate_attestation, + AttestationValidationError, +}; +use super::attestation_record::{ + AttestationRecord, + split_one_attestation, + split_all_attestations, + AttestationSplitError, +}; +use super::{ + AttesterMap, + ProposerMap, +}; +use super::SszBlock; +use super::db::{ + ClientDB, + DBError, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::ssz::{ + Decodable, + DecodeError, +}; +use super::utils::types::Hash256; + +#[derive(Debug, PartialEq)] +pub enum BlockStatus { + NewBlock, + KnownBlock, +} + +#[derive(Debug, PartialEq)] +pub enum SszBlockValidationError { + FutureSlot, + UnknownPoWChainRef, + BadAttestationSsz, + AttestationValidationError(AttestationValidationError), + AttestationSignatureFailed, + FirstAttestationSignatureFailed, + NoProposerSignature, + BadProposerMap, + DatabaseError(String), +} + +/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure +/// that reads directly from encoded SSZ. +/// +/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is +/// a suspicion that the block might be invalid. Such a suspicion should be applied to +/// all blocks coming from the network. +/// +/// Of course, this function will only be more efficient if a block is already serialized. +/// Serializing a complete block and then validating with this function will be less +/// efficient than just validating the original block. +/// +/// This function will determine if the block is new, already known or invalid (either +/// intrinsically or due to some application error.) +#[allow(dead_code)] +pub fn validate_ssz_block(b: &SszBlock, + expected_slot: u64, + cycle_length: u8, + last_justified_slot: u64, + parent_hashes: &Arc>, + proposer_map: &Arc, + attester_map: &Arc, + block_store: &Arc>, + validator_store: &Arc>, + pow_store: &Arc>) + -> Result + where T: ClientDB + Sized +{ + /* + * If this block is already known, return immediately. + */ + if block_store.block_exists(&b.block_hash())? { + return Ok(BlockStatus::KnownBlock); + } + + /* + * Copy the block slot (will be used multiple times) + */ + let block_slot = b.slot_number(); + + /* + * If the block slot corresponds to a slot in the future (according to the local time), + * drop it. + */ + if block_slot > expected_slot { + return Err(SszBlockValidationError::FutureSlot); + } + + /* + * If the PoW chain hash is not known to us, drop it. + * + * We only accept blocks that reference a known PoW hash. + * + * Note: it is not clear what a "known" PoW chain ref is. Likely, + * it means "sufficienty deep in the canonical PoW chain". + */ + if !pow_store.block_hash_exists(b.pow_chain_ref())? { + return Err(SszBlockValidationError::UnknownPoWChainRef); + } + + /* + * Store a reference to the serialized attestations from the block. + */ + let attestations_ssz = &b.attestations(); + + /* + * Get a slice of the first serialized attestation (the 0th) and decode it into + * a full AttestationRecord object. + */ + let (first_attestation_ssz, next_index) = split_one_attestation( + &attestations_ssz, + 0)?; + let (first_attestation, _) = AttestationRecord::ssz_decode( + &first_attestation_ssz, 0)?; + + /* + * Validate this first attestation. + * + * It is a requirement that the block proposer for this slot + * must have signed the 0th attestation record. + */ + let attestation_voters = validate_attestation( + &first_attestation, + block_slot, + cycle_length, + last_justified_slot, + &parent_hashes, + &block_store, + &validator_store, + &attester_map)?; + + /* + * If the set of voters is None, the attestation was invalid. + */ + let attestation_voters = attestation_voters + .ok_or(SszBlockValidationError:: + FirstAttestationSignatureFailed)?; + + /* + * Read the proposer from the map of slot -> validator index. + */ + let proposer = proposer_map.get(&block_slot) + .ok_or(SszBlockValidationError::BadProposerMap)?; + + /* + * If the proposer for this slot was not a voter, reject the block. + */ + if !attestation_voters.contains(&proposer) { + return Err(SszBlockValidationError::NoProposerSignature); + } + + /* + * Split the remaining attestations into a vector of slices, each containing + * a single serialized attestation record. + */ + let other_attestations = split_all_attestations(attestations_ssz, + next_index)?; + + /* + * Verify each other AttestationRecord. + * + * TODO: make this parallelized. + */ + for attestation in other_attestations { + let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?; + let attestation_voters = validate_attestation( + &a, + block_slot, + cycle_length, + last_justified_slot, + &parent_hashes, + &block_store, + &validator_store, + &attester_map)?; + if attestation_voters.is_none() { + return Err(SszBlockValidationError:: + AttestationSignatureFailed); + } + } + + /* + * If we have reached this point, the block is a new valid block that is worthy of + * processing. + */ + Ok(BlockStatus::NewBlock) +} + +impl From for SszBlockValidationError { + fn from(e: DBError) -> Self { + SszBlockValidationError::DatabaseError(e.message) + } +} + +impl From for SszBlockValidationError { + fn from(e: AttestationSplitError) -> Self { + match e { + AttestationSplitError::TooShort => + SszBlockValidationError::BadAttestationSsz + } + } +} + +impl From for SszBlockValidationError { + fn from(e: DecodeError) -> Self { + match e { + DecodeError::TooShort => + SszBlockValidationError::BadAttestationSsz, + DecodeError::TooLong => + SszBlockValidationError::BadAttestationSsz, + } + } +} + +impl From for SszBlockValidationError { + fn from(e: AttestationValidationError) -> Self { + SszBlockValidationError::AttestationValidationError(e) + } +} diff --git a/lighthouse/state/mod.rs b/lighthouse/state/mod.rs index 1a8d4a2c4..b49ab52cf 100644 --- a/lighthouse/state/mod.rs +++ b/lighthouse/state/mod.rs @@ -16,4 +16,5 @@ pub mod shard_and_committee; pub mod validator_record; use super::bls; +use super::db; use super::utils; From 84bb40855f89a3191d6a436d28c72611d7a1332d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 13:31:58 +1000 Subject: [PATCH 02/29] Split block validation tests into functions --- lighthouse/state/block/validation/tests.rs | 176 ++++++++++++--------- 1 file changed, 100 insertions(+), 76 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 3bb5c9c18..ff9a01ede 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -54,6 +54,93 @@ impl TestStore { } } +fn generate_attestations_for_slot(attestation_slot: u64, + block_slot: u64, + shard_count: u16, + validators_per_shard: usize, + cycle_length: u8, + parent_hashes: &[Hash256], + shard_block_hash: &Hash256, + justified_block_hash: &Hash256, + justified_slot: u64, + stores: &TestStore) + -> (AttesterMap, Vec, Vec) +{ + let mut i = 0; + let mut attester_map = AttesterMap::new(); + let mut attestations = vec![]; + let mut keypairs = vec![]; + for shard in 0..shard_count { + let mut attesters = vec![]; + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + + let attestation_message = { + let mut stream = SszStream::new(); + stream.append(&attestation_slot); + stream.append_vec(&parent_hashes_slice.to_vec()); + stream.append(&shard); + stream.append(shard_block_hash); + stream.append(&justified_slot); + let bytes = stream.drain(); + canonical_hash(&bytes) + }; + + + + for attestation_index in 0..validators_per_shard { + /* + * Add the attester to the attestation indices for this shard. + */ + attesters.push(i); + /* + * Set the voters bit on the bitfield to true. + */ + attester_bitfield.set_bit(attestation_index, true); + /* + * Generate a random keypair for this validatior and clone it into the + * list of keypairs. + */ + let keypair = Keypair::random(); + keypairs.push(keypair.clone()); + /* + * Store the validators public key in the database. + */ + stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); + /* + * Generate a new signature and aggregate it on the rolling signature. + */ + let sig = Signature::new(&attestation_message, &keypair.sk); + aggregate_sig.add(&sig); + /* + * Increment the validator counter to monotonically assign validators. + */ + i += 1; + } + + attester_map.insert((attestation_slot, shard), attesters); + let attestation = AttestationRecord { + slot: attestation_slot, + shard_id: shard, + oblique_parent_hashes: vec![], + shard_block_hash: *shard_block_hash, + attester_bitfield, + justified_slot, + justified_block_hash: *justified_block_hash, + aggregate_sig, + }; + attestations.push(attestation); + } + (attester_map, attestations, keypairs) +} + #[test] fn test_block_validation() { let stores = TestStore::new(); @@ -83,82 +170,19 @@ fn test_block_validation() { proposer_map.insert(present_slot, validator_index); proposer_map }; - let (attester_map, attestations, _keypairs) = { - let mut i = 0; - let mut attester_map = AttesterMap::new(); - let mut attestations = vec![]; - let mut keypairs = vec![]; - for shard in 0..shard_count { - let mut attesters = vec![]; - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - let attestation_slot = block_slot - 1; - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - let attestation_message = { - let mut stream = SszStream::new(); - stream.append(&attestation_slot); - stream.append_vec(&parent_hashes_slice.to_vec()); - stream.append(&shard); - stream.append(&shard_block_hash); - stream.append(&justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) - }; - - - - for attestation_index in 0..validators_per_shard { - /* - * Add the attester to the attestation indices for this shard. - */ - attesters.push(i); - /* - * Set the voters bit on the bitfield to true. - */ - attester_bitfield.set_bit(attestation_index, true); - /* - * Generate a random keypair for this validatior and clone it into the - * list of keypairs. - */ - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - /* - * Store the validators public key in the database. - */ - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - /* - * Generate a new signature and aggregate it on the rolling signature. - */ - let sig = Signature::new(&attestation_message, &keypair.sk); - aggregate_sig.add(&sig); - /* - * Increment the validator counter to monotonically assign validators. - */ - i += 1; - } - - attester_map.insert((attestation_slot, shard), attesters); - let attestation = AttestationRecord { - slot: attestation_slot, - shard_id: shard, - oblique_parent_hashes: vec![], - shard_block_hash, - attester_bitfield, - justified_slot, - justified_block_hash, - aggregate_sig, - }; - attestations.push(attestation); - } - (attester_map, attestations, keypairs) - }; + let attestation_slot = block_slot - 1; + let (attester_map, attestations, _keypairs) = + generate_attestations_for_slot( + attestation_slot, + block_slot, + shard_count, + validators_per_shard, + cycle_length, + &parent_hashes, + &shard_block_hash, + &justified_block_hash, + justified_slot, + &stores); let block = Block { parent_hash: Hash256::from("parent".as_bytes()), From a8b08fb300a2be0c032f3d06ebedb74323b12bb6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 14:06:16 +1000 Subject: [PATCH 03/29] Add first block validation benches --- lighthouse/main.rs | 2 + lighthouse/state/block/validation/benches.rs | 154 +++++++++++++++++++ lighthouse/state/block/validation/mod.rs | 2 + lighthouse/state/block/validation/tests.rs | 12 +- 4 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 lighthouse/state/block/validation/benches.rs diff --git a/lighthouse/main.rs b/lighthouse/main.rs index c90bcdaea..33edddd55 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -1,3 +1,5 @@ +#![feature(test)] + #[macro_use] extern crate slog; extern crate slog_term; diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs new file mode 100644 index 000000000..7c9ed1d4f --- /dev/null +++ b/lighthouse/state/block/validation/benches.rs @@ -0,0 +1,154 @@ +/* + * Note: this is a copy of the ./tests.rs file that is rigged to give some fast and easy + * benchmarking. + * + * This file should be moved into a bench/ dir in the root and structured correctly. + */ + +extern crate ssz; +extern crate test; + +use self::test::Bencher; +use self::ssz::{ + SszStream, +}; +use std::sync::Arc; +use super::{ + validate_ssz_block, + BlockStatus, + ProposerMap, +}; +use super::utils::types::{ + Hash256, +}; +use super::SszBlock; +use super::super::Block; + +use super::tests::{ + TestStore, + generate_attestations_for_slot, +}; + +#[derive(Debug)] +struct BenchmarkParams { + total_validators: usize, + cycle_length: u8, + shard_count: u16, + shards_per_slot: u16, + validators_per_shard: usize, +} + +impl BenchmarkParams { + pub fn danny_wants() -> Self { + /* + * 10M Eth where each validator is 32 ETH + */ + let total_validators: usize = 10_000_000 / 32; + /* + * 64 slots per cycle + */ + let cycle_length: u8 = 64; + /* + * 1024 shards + */ + let shard_count: u16 = 1024; + /* + * Number of shards per slot + */ + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + /* + * Number of validators in each shard + */ + let validators_per_shard: usize = total_validators / usize::from(shard_count); + + Self { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + } + } +} + +#[bench] +fn bench_block_validation(b: &mut Bencher) { + let stores = TestStore::new(); + + let params = BenchmarkParams::danny_wants(); + + println!("{:?}", params); + + let cycle_length = params.cycle_length; + let shards_per_slot = params.shards_per_slot; + let validators_per_shard = params.validators_per_shard; + + let present_slot = u64::from(cycle_length) * 10000; + let justified_slot = present_slot - u64::from(cycle_length); + let justified_block_hash = Hash256::from("justified_hash".as_bytes()); + let shard_block_hash = Hash256::from("shard_hash".as_bytes()); + let parent_hashes: Vec = (0..(cycle_length * 2)) + .map(|i| Hash256::from(i as u64)) + .collect(); + let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); + let active_state_root = Hash256::from("active_state".as_bytes()); + let crystallized_state_root = Hash256::from("cry_state".as_bytes()); + + stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); + stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); + + + let block_slot = present_slot; + let validator_index: usize = 0; + let proposer_map = { + let mut proposer_map = ProposerMap::new(); + proposer_map.insert(present_slot, validator_index); + proposer_map + }; + let attestation_slot = block_slot - 1; + let (attester_map, attestations, _keypairs) = + generate_attestations_for_slot( + attestation_slot, + block_slot, + shards_per_slot, + validators_per_shard, + cycle_length, + &parent_hashes, + &shard_block_hash, + &justified_block_hash, + justified_slot, + &stores); + + let block = Block { + parent_hash: Hash256::from("parent".as_bytes()), + slot_number: block_slot, + randao_reveal: Hash256::from("randao".as_bytes()), + attestations, + pow_chain_ref, + active_state_root, + crystallized_state_root, + }; + + let mut stream = SszStream::new(); + stream.append(&block); + let serialized_block = stream.drain(); + let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); + + let parent_hashes = Arc::new(parent_hashes); + let proposer_map = Arc::new(proposer_map); + let attester_map = Arc::new(attester_map); + b.iter(|| { + let status = validate_ssz_block( + &ssz_block, + present_slot, + cycle_length, + justified_slot, + &parent_hashes, + &proposer_map, + &attester_map, + &stores.block.clone(), + &stores.validator.clone(), + &stores.pow_chain.clone()).unwrap(); + assert_eq!(status, BlockStatus::NewBlock); + }); +} diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 59aa3fc15..6f2b1cfb0 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,6 +1,8 @@ mod validate_ssz_block; #[cfg(test)] mod tests; +#[cfg(test)] +mod benches; use super::attestation_record; use super::SszBlock; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index ff9a01ede..b2584b677 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -32,11 +32,11 @@ use super::super::super::bls::{ AggregateSignature, }; -struct TestStore { - db: Arc, - block: Arc>, - pow_chain: Arc>, - validator: Arc>, +pub struct TestStore { + pub db: Arc, + pub block: Arc>, + pub pow_chain: Arc>, + pub validator: Arc>, } impl TestStore { @@ -54,7 +54,7 @@ impl TestStore { } } -fn generate_attestations_for_slot(attestation_slot: u64, +pub fn generate_attestations_for_slot(attestation_slot: u64, block_slot: u64, shard_count: u16, validators_per_shard: usize, From debc642b50a4a5121d650dec15a4b73007bece86 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 26 Sep 2018 23:28:47 +1000 Subject: [PATCH 04/29] Move attestation validation into iter.for_each --- .../block/validation/validate_ssz_block.rs | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 39b0551af..ada0be5a4 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::sync::{ + Arc, + RwLock, +}; use super::attestation_record::{ validate_attestation, AttestationValidationError, @@ -170,22 +173,44 @@ pub fn validate_ssz_block(b: &SszBlock, * * TODO: make this parallelized. */ - for attestation in other_attestations { - let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?; - let attestation_voters = validate_attestation( - &a, - block_slot, - cycle_length, - last_justified_slot, - &parent_hashes, - &block_store, - &validator_store, - &attester_map)?; - if attestation_voters.is_none() { - return Err(SszBlockValidationError:: - AttestationSignatureFailed); - } - } + let failure: Option = None; + let failure = RwLock::new(failure); + other_attestations.iter() + .for_each(|attestation| { + if let Some(_) = *failure.read().unwrap() { + () + }; + match AttestationRecord::ssz_decode(&attestation, 0) { + Ok((a, _)) => { + let result = validate_attestation( + &a, + block_slot, + cycle_length, + last_justified_slot, + &parent_hashes, + &block_store, + &validator_store, + &attester_map); + match result { + Err(e) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::from(e)); + } + Ok(None) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::AttestationSignatureFailed); + } + _ => () + } + } + Err(e) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::from(e)); + } + }; + }); + + // TODO: handle validation failure. Presently, it will just pass everything /* * If we have reached this point, the block is a new valid block that is worthy of From 05fe231e413a9f049f7326241d077d8b117b50be Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Sep 2018 15:37:53 +0930 Subject: [PATCH 05/29] Fix bug with bitfield validation --- .../validation/attestation_validation.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index 389b9f3c3..db2a781de 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -45,15 +45,6 @@ pub enum AttestationValidationError { DBError(String), } -fn bytes_for_bits(bits: usize) -> usize { - (bits.saturating_sub(1) / 8) + 1 -} - -fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool { - let shift = 8_u16.saturating_sub(n as u16); - ((!0 << shift) & u16::from(byte)) > 0 -} - pub fn validate_attestation(a: &AttestationRecord, block_slot: u64, cycle_length: u8, @@ -164,6 +155,20 @@ pub fn validate_attestation(a: &AttestationRecord, Ok(voted_hashmap) } +fn bytes_for_bits(bits: usize) -> usize { + (bits.saturating_sub(1) / 8) + 1 +} + +fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool { + for i in 0..n { + let mask = 0_u8 >> 8_usize - i as usize; + if byte & mask > 0 { + return true + } + } + false +} + impl From for AttestationValidationError { fn from(e: ParentHashesError) -> Self { match e { From b92d88d42bb5f8d99d0977cf745f4184f810cf8e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Sep 2018 15:38:51 +0930 Subject: [PATCH 06/29] Add (untested) parallelization for att validation --- Cargo.toml | 1 + .../state/block/validation/validate_ssz_block.rs | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index eaca0fd61..c076461c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ ethereum-types = "0.4.0" futures = "0.1.23" network-libp2p = { path = "network-libp2p" } rand = "0.3" +rayon = "1.0.2" rocksdb = "0.10.1" rlp = { git = "https://github.com/paritytech/parity-common" } slog = "^2.2.3" diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index ada0be5a4..1067798db 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -1,3 +1,6 @@ +extern crate rayon; +use self::rayon::prelude::*; + use std::sync::{ Arc, RwLock, @@ -48,6 +51,7 @@ pub enum SszBlockValidationError { FirstAttestationSignatureFailed, NoProposerSignature, BadProposerMap, + RwLockPoisoned, DatabaseError(String), } @@ -175,7 +179,7 @@ pub fn validate_ssz_block(b: &SszBlock, */ let failure: Option = None; let failure = RwLock::new(failure); - other_attestations.iter() + other_attestations.par_iter() .for_each(|attestation| { if let Some(_) = *failure.read().unwrap() { () @@ -210,6 +214,16 @@ pub fn validate_ssz_block(b: &SszBlock, }; }); + match failure.into_inner() { + Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), + Ok(failure) => { + match failure { + Some(error) => return Err(error), + _ => () + } + + } + } // TODO: handle validation failure. Presently, it will just pass everything /* From 19ddbdefa8d52fe11193f22b61d313ea1f90553f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Sep 2018 17:15:29 +0930 Subject: [PATCH 07/29] Refact block val. tests to be more modular --- lighthouse/state/block/validation/tests.rs | 332 ++++++++++++++------- 1 file changed, 219 insertions(+), 113 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index b2584b677..e5522e37f 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -6,6 +6,7 @@ use self::ssz::{ use std::sync::Arc; use super::{ validate_ssz_block, + SszBlockValidationError, BlockStatus, AttesterMap, ProposerMap, @@ -54,162 +55,267 @@ impl TestStore { } } -pub fn generate_attestations_for_slot(attestation_slot: u64, - block_slot: u64, - shard_count: u16, - validators_per_shard: usize, - cycle_length: u8, - parent_hashes: &[Hash256], - shard_block_hash: &Hash256, - justified_block_hash: &Hash256, - justified_slot: u64, - stores: &TestStore) - -> (AttesterMap, Vec, Vec) -{ - let mut i = 0; - let mut attester_map = AttesterMap::new(); - let mut attestations = vec![]; - let mut keypairs = vec![]; - for shard in 0..shard_count { - let mut attesters = vec![]; - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - let attestation_message = { - let mut stream = SszStream::new(); - stream.append(&attestation_slot); - stream.append_vec(&parent_hashes_slice.to_vec()); - stream.append(&shard); - stream.append(shard_block_hash); - stream.append(&justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) - }; - - - - for attestation_index in 0..validators_per_shard { - /* - * Add the attester to the attestation indices for this shard. - */ - attesters.push(i); - /* - * Set the voters bit on the bitfield to true. - */ - attester_bitfield.set_bit(attestation_index, true); - /* - * Generate a random keypair for this validatior and clone it into the - * list of keypairs. - */ - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - /* - * Store the validators public key in the database. - */ - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - /* - * Generate a new signature and aggregate it on the rolling signature. - */ - let sig = Signature::new(&attestation_message, &keypair.sk); - aggregate_sig.add(&sig); - /* - * Increment the validator counter to monotonically assign validators. - */ - i += 1; - } - - attester_map.insert((attestation_slot, shard), attesters); - let attestation = AttestationRecord { - slot: attestation_slot, - shard_id: shard, - oblique_parent_hashes: vec![], - shard_block_hash: *shard_block_hash, - attester_bitfield, - justified_slot, - justified_block_hash: *justified_block_hash, - aggregate_sig, - }; - attestations.push(attestation); - } - (attester_map, attestations, keypairs) +#[derive(Debug)] +pub struct TestParams { + pub total_validators: usize, + pub cycle_length: u8, + pub shard_count: u16, + pub shards_per_slot: u16, + pub validators_per_shard: usize, + pub block_slot: u64, + pub attestations_justified_slot: u64, } +type ParentHashes = Vec; -#[test] -fn test_block_validation() { +/// Setup for a block validation function, without actually executing the +/// block validation function. +pub fn setup_block_validation_scenario(params: &TestParams) + -> (Block, ParentHashes, AttesterMap, ProposerMap, TestStore) +{ let stores = TestStore::new(); - let cycle_length: u8 = 2; - let shard_count: u16 = 2; - let present_slot = u64::from(cycle_length) * 10000; - let justified_slot = present_slot - u64::from(cycle_length); - let justified_block_hash = Hash256::from("justified_hash".as_bytes()); - let shard_block_hash = Hash256::from("shard_hash".as_bytes()); + let cycle_length = params.cycle_length; + let shards_per_slot = params.shards_per_slot; + let validators_per_shard = params.validators_per_shard; + let block_slot = params.block_slot; + let attestations_justified_slot = params.attestations_justified_slot; + let parent_hashes: Vec = (0..(cycle_length * 2)) .map(|i| Hash256::from(i as u64)) .collect(); + let parent_hash = Hash256::from("parent_hash".as_bytes()); + let randao_reveal = Hash256::from("randao_reveal".as_bytes()); + let justified_block_hash = Hash256::from("justified_hash".as_bytes()); let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); let active_state_root = Hash256::from("active_state".as_bytes()); let crystallized_state_root = Hash256::from("cry_state".as_bytes()); + let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - let validators_per_shard = 10; - - let block_slot = present_slot; let validator_index: usize = 0; let proposer_map = { let mut proposer_map = ProposerMap::new(); - proposer_map.insert(present_slot, validator_index); + proposer_map.insert(block_slot, validator_index); proposer_map }; + /* let attestation_slot = block_slot - 1; let (attester_map, attestations, _keypairs) = generate_attestations_for_slot( attestation_slot, block_slot, - shard_count, + shards_per_slot, validators_per_shard, cycle_length, &parent_hashes, - &shard_block_hash, + &Hash256::from("shard_hash".as_bytes()), &justified_block_hash, - justified_slot, + attestations_justified_slot, &stores); + */ + let (attester_map, attestations, _keypairs) = { + let mut i = 0; + let attestation_slot = block_slot - 1; + let mut attester_map = AttesterMap::new(); + let mut attestations = vec![]; + let mut keypairs = vec![]; + for shard in 0..shards_per_slot { + let mut attesters = vec![]; + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + + let attestation_message = { + let mut stream = SszStream::new(); + stream.append(&attestation_slot); + stream.append_vec(&parent_hashes_slice.to_vec()); + stream.append(&shard); + stream.append(&shard_block_hash); + stream.append(&attestations_justified_slot); + let bytes = stream.drain(); + canonical_hash(&bytes) + }; + + + + for attestation_index in 0..validators_per_shard { + /* + * Add the attester to the attestation indices for this shard. + */ + attesters.push(i); + /* + * Set the voters bit on the bitfield to true. + */ + attester_bitfield.set_bit(attestation_index, true); + /* + * Generate a random keypair for this validatior and clone it into the + * list of keypairs. + */ + let keypair = Keypair::random(); + keypairs.push(keypair.clone()); + /* + * Store the validators public key in the database. + */ + stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); + /* + * Generate a new signature and aggregate it on the rolling signature. + */ + let sig = Signature::new(&attestation_message, &keypair.sk); + aggregate_sig.add(&sig); + /* + * Increment the validator counter to monotonically assign validators. + */ + i += 1; + } + + attester_map.insert((attestation_slot, shard), attesters); + let attestation = AttestationRecord { + slot: attestation_slot, + shard_id: shard, + oblique_parent_hashes: vec![], + shard_block_hash, + attester_bitfield, + justified_slot: attestations_justified_slot, + justified_block_hash, + aggregate_sig, + }; + attestations.push(attestation); + } + (attester_map, attestations, keypairs) + }; let block = Block { - parent_hash: Hash256::from("parent".as_bytes()), + parent_hash, slot_number: block_slot, - randao_reveal: Hash256::from("randao".as_bytes()), + randao_reveal, attestations, pow_chain_ref, active_state_root, crystallized_state_root, }; - let mut stream = SszStream::new(); - stream.append(&block); - let serialized_block = stream.drain(); - let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); + (block, + parent_hashes, + attester_map, + proposer_map, + stores) +} - let status = validate_ssz_block( +/// Helper function to take some Block and SSZ serialize it. +pub fn serialize_block(b: &Block) -> Vec { + let mut stream = SszStream::new(); + stream.append(b); + stream.drain() +} + +/// Setup and run a block validation scenario, given some parameters. +/// +/// Returns the Result returned from the block validation function. +pub fn run_block_validation_scenario( + validation_slot: u64, + validation_last_justified_slot: u64, + params: &TestParams, + mutator_func: F) + -> Result + where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) + -> (Block, AttesterMap, ProposerMap, TestStore) +{ + let (block, + parent_hashes, + attester_map, + proposer_map, + stores) = setup_block_validation_scenario(¶ms); + + let (block, + attester_map, + proposer_map, + stores) = mutator_func(block, attester_map, proposer_map, stores); + + let ssz_bytes = serialize_block(&block); + let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) + .unwrap(); + + validate_ssz_block( &ssz_block, - present_slot, - cycle_length, - justified_slot, + validation_slot, + params.cycle_length, + validation_last_justified_slot, &Arc::new(parent_hashes), &Arc::new(proposer_map), &Arc::new(attester_map), &stores.block.clone(), &stores.validator.clone(), - &stores.pow_chain.clone()).unwrap(); - - assert_eq!(status, BlockStatus::NewBlock); + &stores.pow_chain.clone()) +} + +fn get_simple_params() -> TestParams { + let validators_per_shard: usize = 5; + let cycle_length: u8 = 2; + let shard_count: u16 = 4; + let shards_per_slot: u16 = shard_count / u16::from(cycle_length); + let total_validators: usize = validators_per_shard * shard_count as usize; + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + + TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + block_slot, + attestations_justified_slot, + } +} + +#[test] +fn test_block_validation_simple_scenario_valid() { + let params = get_simple_params(); + + let validation_slot = params.block_slot; + let validation_last_justified_slot = params.attestations_justified_slot; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + validation_slot, + validation_last_justified_slot, + ¶ms, + no_mutate); + + assert_eq!(status.unwrap(), BlockStatus::NewBlock); +} + +#[test] +fn test_block_validation_simple_scenario_invalid_2nd_attestation() { + let params = get_simple_params(); + + let validation_slot = params.block_slot; + let validation_last_justified_slot = params.attestations_justified_slot; + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + /* + * Set the second attestaion record to have an invalid signature. + */ + block.attestations[1].aggregate_sig = AggregateSignature::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + validation_slot, + validation_last_justified_slot, + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::AttestationSignatureFailed)); } From 8020b897c6d26f3cf49ac858ffdf4c4fac1caa7a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Sep 2018 17:15:42 +0930 Subject: [PATCH 08/29] Refactor block val. benches to use test funtions --- lighthouse/state/block/validation/benches.rs | 232 +++++++++---------- 1 file changed, 110 insertions(+), 122 deletions(-) diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 7c9ed1d4f..481ee224e 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -4,151 +4,139 @@ * * This file should be moved into a bench/ dir in the root and structured correctly. */ - -extern crate ssz; extern crate test; use self::test::Bencher; -use self::ssz::{ - SszStream, -}; + use std::sync::Arc; + use super::{ validate_ssz_block, - BlockStatus, + AttesterMap, ProposerMap, }; -use super::utils::types::{ - Hash256, -}; -use super::SszBlock; -use super::super::Block; use super::tests::{ TestStore, - generate_attestations_for_slot, + TestParams, + setup_block_validation_scenario, + serialize_block, }; -#[derive(Debug)] -struct BenchmarkParams { - total_validators: usize, - cycle_length: u8, - shard_count: u16, - shards_per_slot: u16, - validators_per_shard: usize, -} +use super::super::{ + Block, + SszBlock, +}; -impl BenchmarkParams { - pub fn danny_wants() -> Self { - /* - * 10M Eth where each validator is 32 ETH - */ - let total_validators: usize = 10_000_000 / 32; - /* - * 64 slots per cycle - */ - let cycle_length: u8 = 64; - /* - * 1024 shards - */ - let shard_count: u16 = 1024; - /* - * Number of shards per slot - */ - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - /* - * Number of validators in each shard - */ - let validators_per_shard: usize = total_validators / usize::from(shard_count); +fn bench_block_validation_scenario( + b: &mut Bencher, + validation_slot: u64, + validation_last_justified_slot: u64, + params: &TestParams, + mutator_func: F) + where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) + -> (Block, AttesterMap, ProposerMap, TestStore) +{ + let (block, + parent_hashes, + attester_map, + proposer_map, + stores) = setup_block_validation_scenario(¶ms); - Self { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - } - } -} + let (block, + attester_map, + proposer_map, + stores) = mutator_func(block, attester_map, proposer_map, stores); -#[bench] -fn bench_block_validation(b: &mut Bencher) { - let stores = TestStore::new(); - - let params = BenchmarkParams::danny_wants(); - - println!("{:?}", params); - - let cycle_length = params.cycle_length; - let shards_per_slot = params.shards_per_slot; - let validators_per_shard = params.validators_per_shard; - - let present_slot = u64::from(cycle_length) * 10000; - let justified_slot = present_slot - u64::from(cycle_length); - let justified_block_hash = Hash256::from("justified_hash".as_bytes()); - let shard_block_hash = Hash256::from("shard_hash".as_bytes()); - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); - let active_state_root = Hash256::from("active_state".as_bytes()); - let crystallized_state_root = Hash256::from("cry_state".as_bytes()); - - stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); - stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - - - let block_slot = present_slot; - let validator_index: usize = 0; - let proposer_map = { - let mut proposer_map = ProposerMap::new(); - proposer_map.insert(present_slot, validator_index); - proposer_map - }; - let attestation_slot = block_slot - 1; - let (attester_map, attestations, _keypairs) = - generate_attestations_for_slot( - attestation_slot, - block_slot, - shards_per_slot, - validators_per_shard, - cycle_length, - &parent_hashes, - &shard_block_hash, - &justified_block_hash, - justified_slot, - &stores); - - let block = Block { - parent_hash: Hash256::from("parent".as_bytes()), - slot_number: block_slot, - randao_reveal: Hash256::from("randao".as_bytes()), - attestations, - pow_chain_ref, - active_state_root, - crystallized_state_root, - }; - - let mut stream = SszStream::new(); - stream.append(&block); - let serialized_block = stream.drain(); - let ssz_block = SszBlock::from_slice(&serialized_block[..]).unwrap(); + let ssz_bytes = serialize_block(&block); + let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) + .unwrap(); let parent_hashes = Arc::new(parent_hashes); let proposer_map = Arc::new(proposer_map); let attester_map = Arc::new(attester_map); b.iter(|| { - let status = validate_ssz_block( + validate_ssz_block( &ssz_block, - present_slot, - cycle_length, - justified_slot, - &parent_hashes, - &proposer_map, - &attester_map, + validation_slot, + params.cycle_length, + validation_last_justified_slot, + &parent_hashes.clone(), + &proposer_map.clone(), + &attester_map.clone(), &stores.block.clone(), &stores.validator.clone(), - &stores.pow_chain.clone()).unwrap(); - assert_eq!(status, BlockStatus::NewBlock); + &stores.pow_chain.clone()) }); } + +#[bench] +#[ignore] +fn bench_block_validation_10m_eth(b: &mut Bencher) { + let total_validators: usize = 10_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + block_slot, + attestations_justified_slot, + }; + let validation_slot = params.block_slot; + let validation_last_justified_slot = params.attestations_justified_slot; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + validation_slot, + validation_last_justified_slot, + ¶ms, + no_mutate); +} + +#[bench] +#[ignore] +fn bench_block_validation_100m_eth(b: &mut Bencher) { + let total_validators: usize = 100_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + block_slot, + attestations_justified_slot, + }; + + let validation_slot = params.block_slot; + let validation_last_justified_slot = params.attestations_justified_slot; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + validation_slot, + validation_last_justified_slot, + ¶ms, + no_mutate); +} From 13467abd7f7c288d8f2177106da95460f781af6f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Sep 2018 17:30:49 +0930 Subject: [PATCH 09/29] Tidy benches file --- lighthouse/state/block/validation/benches.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 481ee224e..8da101589 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -1,9 +1,3 @@ -/* - * Note: this is a copy of the ./tests.rs file that is rigged to give some fast and easy - * benchmarking. - * - * This file should be moved into a bench/ dir in the root and structured correctly. - */ extern crate test; use self::test::Bencher; From 7c88f94377f9cc3949651281287972bf6b27cb92 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 29 Sep 2018 12:32:33 +0930 Subject: [PATCH 10/29] Tidy comments in block validation --- lighthouse/state/block/validation/validate_ssz_block.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 1067798db..524ce07ba 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -175,7 +175,9 @@ pub fn validate_ssz_block(b: &SszBlock, /* * Verify each other AttestationRecord. * - * TODO: make this parallelized. + * Note: this uses the `rayon` library to do "sometimes" parallelization. Put simply, + * if there's some spare threads the verification of attestation records will happen + * concurrently. */ let failure: Option = None; let failure = RwLock::new(failure); @@ -224,7 +226,6 @@ pub fn validate_ssz_block(b: &SszBlock, } } - // TODO: handle validation failure. Presently, it will just pass everything /* * If we have reached this point, the block is a new valid block that is worthy of From e0e8aa98f4102c739180e2c41afb65b48a51d9a6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 29 Sep 2018 13:01:33 +0930 Subject: [PATCH 11/29] Introduce "benches" feature So that benchmarks don't run each time tests run --- Cargo.toml | 3 +++ lighthouse/state/block/validation/benches.rs | 2 -- lighthouse/state/block/validation/mod.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c076461c3..682adbe63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,3 +36,6 @@ ring = { git = "https://github.com/paritytech/ring" } [[bin]] path = "lighthouse/main.rs" name = "lighthouse" + +[features] +benches = [] diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 8da101589..0aca5a03a 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -65,7 +65,6 @@ fn bench_block_validation_scenario( } #[bench] -#[ignore] fn bench_block_validation_10m_eth(b: &mut Bencher) { let total_validators: usize = 10_000_000 / 32; let cycle_length: u8 = 64; @@ -100,7 +99,6 @@ fn bench_block_validation_10m_eth(b: &mut Bencher) { } #[bench] -#[ignore] fn bench_block_validation_100m_eth(b: &mut Bencher) { let total_validators: usize = 100_000_000 / 32; let cycle_length: u8 = 64; diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 6f2b1cfb0..1b7ebec59 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,7 +1,7 @@ mod validate_ssz_block; #[cfg(test)] mod tests; -#[cfg(test)] +#[cfg(all(feature = "benches", test))] mod benches; use super::attestation_record; From bc27be147fb14d49505b4df8641f7d9ecd7223cb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 29 Sep 2018 15:38:54 +0930 Subject: [PATCH 12/29] Further development on block validation - Return a fully deserialized block from validate_ssz_block - Ensure the parent_hash field is handled --- lighthouse/db/stores/block_store.rs | 7 + lighthouse/state/block/structs.rs | 1 + lighthouse/state/block/validation/mod.rs | 5 +- lighthouse/state/block/validation/tests.rs | 28 +++- .../block/validation/validate_ssz_block.rs | 125 +++++++++++++----- 5 files changed, 129 insertions(+), 37 deletions(-) diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/stores/block_store.rs index 1836923c9..2166bea5a 100644 --- a/lighthouse/db/stores/block_store.rs +++ b/lighthouse/db/stores/block_store.rs @@ -35,6 +35,13 @@ impl BlockStore { { self.db.exists(DB_COLUMN, hash) } + + pub fn block_exists_in_canonical_chain(&self, hash: &[u8]) + -> Result + { + // TODO: implement logic for canonical chain + self.db.exists(DB_COLUMN, hash) + } } #[cfg(test)] diff --git a/lighthouse/state/block/structs.rs b/lighthouse/state/block/structs.rs index e3a75b1c5..f22bc370d 100644 --- a/lighthouse/state/block/structs.rs +++ b/lighthouse/state/block/structs.rs @@ -13,6 +13,7 @@ pub const MIN_SSZ_BLOCK_LENGTH: usize = { }; pub const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); +#[derive(Debug, PartialEq)] pub struct Block { pub parent_hash: Hash256, pub slot_number: u64, diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 1b7ebec59..23a485aad 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -5,7 +5,10 @@ mod tests; mod benches; use super::attestation_record; -use super::SszBlock; +use super::{ + SszBlock, + Block, +}; use super::db; use super::ssz; use super::utils; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index e5522e37f..52975959f 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -93,6 +93,7 @@ pub fn setup_block_validation_scenario(params: &TestParams) stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); + stores.block.put_block(parent_hash.as_ref(), &vec![42]).unwrap(); let validator_index: usize = 0; let proposer_map = { @@ -224,7 +225,7 @@ pub fn run_block_validation_scenario( validation_last_justified_slot: u64, params: &TestParams, mutator_func: F) - -> Result + -> Result<(BlockStatus, Option), SszBlockValidationError> where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) -> (Block, AttesterMap, ProposerMap, TestStore) { @@ -277,7 +278,7 @@ fn get_simple_params() -> TestParams { } #[test] -fn test_block_validation_simple_scenario_valid() { +fn test_block_validation_simple_scenario_valid_in_canonical_chain() { let params = get_simple_params(); let validation_slot = params.block_slot; @@ -293,7 +294,28 @@ fn test_block_validation_simple_scenario_valid() { ¶ms, no_mutate); - assert_eq!(status.unwrap(), BlockStatus::NewBlock); + assert_eq!(status.unwrap().0, BlockStatus::NewBlockInCanonicalChain); +} + +#[test] +fn test_block_validation_simple_scenario_valid_not_in_canonical_chain() { + let params = get_simple_params(); + + let validation_slot = params.block_slot; + let validation_last_justified_slot = params.attestations_justified_slot; + + let no_mutate = |mut block: Block, attester_map, proposer_map, stores| { + block.parent_hash = Hash256::from("not in canonical chain".as_bytes()); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + validation_slot, + validation_last_justified_slot, + ¶ms, + no_mutate); + + assert_eq!(status.unwrap().0, BlockStatus::NewBlockInForkChain); } #[test] diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 524ce07ba..abd30f7dd 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -19,7 +19,10 @@ use super::{ AttesterMap, ProposerMap, }; -use super::SszBlock; +use super::{ + SszBlock, + Block, +}; use super::db::{ ClientDB, DBError, @@ -37,7 +40,8 @@ use super::utils::types::Hash256; #[derive(Debug, PartialEq)] pub enum BlockStatus { - NewBlock, + NewBlockInCanonicalChain, + NewBlockInForkChain, KnownBlock, } @@ -45,6 +49,7 @@ pub enum BlockStatus { pub enum SszBlockValidationError { FutureSlot, UnknownPoWChainRef, + UnknownParentHash, BadAttestationSsz, AttestationValidationError(AttestationValidationError), AttestationSignatureFailed, @@ -68,6 +73,9 @@ pub enum SszBlockValidationError { /// /// This function will determine if the block is new, already known or invalid (either /// intrinsically or due to some application error.) +/// +/// Note: this function does not implement randao_reveal checking as it is not in the +/// specification. #[allow(dead_code)] pub fn validate_ssz_block(b: &SszBlock, expected_slot: u64, @@ -79,29 +87,28 @@ pub fn validate_ssz_block(b: &SszBlock, block_store: &Arc>, validator_store: &Arc>, pow_store: &Arc>) - -> Result + -> Result<(BlockStatus, Option), SszBlockValidationError> where T: ClientDB + Sized { - /* - * If this block is already known, return immediately. - */ - if block_store.block_exists(&b.block_hash())? { - return Ok(BlockStatus::KnownBlock); - } - - /* - * Copy the block slot (will be used multiple times) - */ - let block_slot = b.slot_number(); /* * If the block slot corresponds to a slot in the future (according to the local time), * drop it. */ + let block_slot = b.slot_number(); if block_slot > expected_slot { return Err(SszBlockValidationError::FutureSlot); } + /* + * If this block is already known, return immediately. + */ + let block_hash = &b.block_hash(); + if block_store.block_exists(&block_hash)? { + return Ok((BlockStatus::KnownBlock, None)); + } + + /* * If the PoW chain hash is not known to us, drop it. * @@ -110,6 +117,7 @@ pub fn validate_ssz_block(b: &SszBlock, * Note: it is not clear what a "known" PoW chain ref is. Likely, * it means "sufficienty deep in the canonical PoW chain". */ + let pow_chain_ref = b.pow_chain_ref(); if !pow_store.block_hash_exists(b.pow_chain_ref())? { return Err(SszBlockValidationError::UnknownPoWChainRef); } @@ -175,21 +183,44 @@ pub fn validate_ssz_block(b: &SszBlock, /* * Verify each other AttestationRecord. * - * Note: this uses the `rayon` library to do "sometimes" parallelization. Put simply, - * if there's some spare threads the verification of attestation records will happen + * This uses the `rayon` library to do "sometimes" parallelization. Put simply, + * if there are some spare threads, the verification of attestation records will happen * concurrently. + * + * There is a thread-safe `failure` variable which is set whenever an attestation fails + * validation. This is so all attestation validation is halted if a single bad attestation + * is found. */ - let failure: Option = None; - let failure = RwLock::new(failure); - other_attestations.par_iter() - .for_each(|attestation| { + let failure: RwLock> = RwLock::new(None); + let deserialized_attestations: Vec = other_attestations + .par_iter() + .filter_map(|attestation_ssz| { + /* + * If some thread has set the `failure` variable to `Some(error)` the abandon + * attestation serialization and validation. + */ if let Some(_) = *failure.read().unwrap() { - () - }; - match AttestationRecord::ssz_decode(&attestation, 0) { - Ok((a, _)) => { + return None; + } + /* + * If there has not been a failure yet, attempt to serialize and validate the + * attestation. + */ + match AttestationRecord::ssz_decode(&attestation_ssz, 0) { + /* + * Deserialization failed, therefore the block is invalid. + */ + Err(e) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::from(e)); + None + } + /* + * Deserialization succeeded and the attestation should be validated. + */ + Ok((attestation, _)) => { let result = validate_attestation( - &a, + &attestation, block_slot, cycle_length, last_justified_slot, @@ -198,23 +229,31 @@ pub fn validate_ssz_block(b: &SszBlock, &validator_store, &attester_map); match result { + /* + * Attestation validation failed with some error. + */ Err(e) => { let mut failure = failure.write().unwrap(); *failure = Some(SszBlockValidationError::from(e)); + None } + /* + * Attestation validation failed due to a bad signature. + */ Ok(None) => { let mut failure = failure.write().unwrap(); *failure = Some(SszBlockValidationError::AttestationSignatureFailed); + None } - _ => () + /* + * Attestation validation succeded. + */ + Ok(_) => Some(attestation) } } - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - } - }; - }); + } + }) + .collect(); match failure.into_inner() { Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), @@ -231,7 +270,27 @@ pub fn validate_ssz_block(b: &SszBlock, * If we have reached this point, the block is a new valid block that is worthy of * processing. */ - Ok(BlockStatus::NewBlock) + + /* + * If the block's parent_hash _is_ in the canonical chain, the block is a + * new block in the canonical chain. Otherwise, it's a new block in a fork chain. + */ + let parent_hash = b.parent_hash(); + let status = if block_store.block_exists_in_canonical_chain(&parent_hash)? { + BlockStatus::NewBlockInCanonicalChain + } else { + BlockStatus::NewBlockInForkChain + }; + let block = Block { + parent_hash: Hash256::from(parent_hash), + slot_number: block_slot, + randao_reveal: Hash256::from(b.randao_reveal()), + attestations: deserialized_attestations, + pow_chain_ref: Hash256::from(pow_chain_ref), + active_state_root: Hash256::from(b.act_state_root()), + crystallized_state_root: Hash256::from(b.cry_state_root()), + }; + Ok((status, Some(block))) } impl From for SszBlockValidationError { From 0b99951bf8d721d2654c1a37e365ae9bc94bef96 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 29 Sep 2018 16:07:59 +0930 Subject: [PATCH 13/29] Refactor block val. into "BlockValidationContext" --- lighthouse/state/block/validation/benches.rs | 25 +- lighthouse/state/block/validation/mod.rs | 2 +- lighthouse/state/block/validation/tests.rs | 41 +- .../block/validation/validate_ssz_block.rs | 429 +++++++++--------- 4 files changed, 252 insertions(+), 245 deletions(-) diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 0aca5a03a..3b93e6786 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -5,7 +5,7 @@ use self::test::Bencher; use std::sync::Arc; use super::{ - validate_ssz_block, + BlockValidationContext, AttesterMap, ProposerMap, }; @@ -50,17 +50,18 @@ fn bench_block_validation_scenario( let proposer_map = Arc::new(proposer_map); let attester_map = Arc::new(attester_map); b.iter(|| { - validate_ssz_block( - &ssz_block, - validation_slot, - params.cycle_length, - validation_last_justified_slot, - &parent_hashes.clone(), - &proposer_map.clone(), - &attester_map.clone(), - &stores.block.clone(), - &stores.validator.clone(), - &stores.pow_chain.clone()) + let context = BlockValidationContext { + present_slot: validation_slot, + cycle_length: params.cycle_length, + last_justified_slot: validation_last_justified_slot, + parent_hashes: parent_hashes.clone(), + proposer_map: proposer_map.clone(), + attester_map: attester_map.clone(), + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + pow_store: stores.pow_chain.clone() + }; + context.validate_ssz_block(&ssz_block) }); } diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 23a485aad..e2382da65 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -18,7 +18,7 @@ use super::common::maps::{ ProposerMap, }; pub use self::validate_ssz_block::{ - validate_ssz_block, + BlockValidationContext, SszBlockValidationError, BlockStatus, }; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 52975959f..a70a25ece 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -5,7 +5,7 @@ use self::ssz::{ }; use std::sync::Arc; use super::{ - validate_ssz_block, + BlockValidationContext, SszBlockValidationError, BlockStatus, AttesterMap, @@ -101,21 +101,7 @@ pub fn setup_block_validation_scenario(params: &TestParams) proposer_map.insert(block_slot, validator_index); proposer_map }; - /* - let attestation_slot = block_slot - 1; - let (attester_map, attestations, _keypairs) = - generate_attestations_for_slot( - attestation_slot, - block_slot, - shards_per_slot, - validators_per_shard, - cycle_length, - &parent_hashes, - &Hash256::from("shard_hash".as_bytes()), - &justified_block_hash, - attestations_justified_slot, - &stores); - */ + let (attester_map, attestations, _keypairs) = { let mut i = 0; let attestation_slot = block_slot - 1; @@ -244,17 +230,18 @@ pub fn run_block_validation_scenario( let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) .unwrap(); - validate_ssz_block( - &ssz_block, - validation_slot, - params.cycle_length, - validation_last_justified_slot, - &Arc::new(parent_hashes), - &Arc::new(proposer_map), - &Arc::new(attester_map), - &stores.block.clone(), - &stores.validator.clone(), - &stores.pow_chain.clone()) + let context = BlockValidationContext { + present_slot: validation_slot, + cycle_length: params.cycle_length, + last_justified_slot: validation_last_justified_slot, + parent_hashes: Arc::new(parent_hashes), + proposer_map: Arc::new(proposer_map), + attester_map: Arc::new(attester_map), + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + pow_store: stores.pow_chain.clone() + }; + context.validate_ssz_block(&ssz_block) } fn get_simple_params() -> TestParams { diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index abd30f7dd..0ad688145 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -60,237 +60,256 @@ pub enum SszBlockValidationError { DatabaseError(String), } -/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure -/// that reads directly from encoded SSZ. -/// -/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is -/// a suspicion that the block might be invalid. Such a suspicion should be applied to -/// all blocks coming from the network. -/// -/// Of course, this function will only be more efficient if a block is already serialized. -/// Serializing a complete block and then validating with this function will be less -/// efficient than just validating the original block. -/// -/// This function will determine if the block is new, already known or invalid (either -/// intrinsically or due to some application error.) -/// -/// Note: this function does not implement randao_reveal checking as it is not in the -/// specification. -#[allow(dead_code)] -pub fn validate_ssz_block(b: &SszBlock, - expected_slot: u64, - cycle_length: u8, - last_justified_slot: u64, - parent_hashes: &Arc>, - proposer_map: &Arc, - attester_map: &Arc, - block_store: &Arc>, - validator_store: &Arc>, - pow_store: &Arc>) - -> Result<(BlockStatus, Option), SszBlockValidationError> +/// The context against which a block should be validated. +pub struct BlockValidationContext where T: ClientDB + Sized { + /// The slot as determined by the system time. + pub present_slot: u64, + /// The cycle_length as determined by the chain configuration. + pub cycle_length: u8, + /// The last justified slot as per the client's view of the canonical chain. + pub last_justified_slot: u64, + /// A vec of the hashes of the blocks preceeding the present slot. + pub parent_hashes: Arc>, + /// A map of slots to a block proposer validation index. + pub proposer_map: Arc, + /// A map of (slot, shard_id) to the attestation set of validation indices. + pub attester_map: Arc, + /// The store containing block information. + pub block_store: Arc>, + /// The store containing validator information. + pub validator_store: Arc>, + /// The store containing information about the proof-of-work chain. + pub pow_store: Arc>, +} - /* - * If the block slot corresponds to a slot in the future (according to the local time), - * drop it. - */ - let block_slot = b.slot_number(); - if block_slot > expected_slot { - return Err(SszBlockValidationError::FutureSlot); - } +impl BlockValidationContext + where T: ClientDB +{ + /// Validate some SszBlock against a block validation context. An SszBlock varies from a Block in + /// that is a read-only structure that reads directly from encoded SSZ. + /// + /// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is + /// a suspicion that the block might be invalid. Such a suspicion should be applied to + /// all blocks coming from the network. + /// + /// Of course, this function will only be more efficient if a block is already serialized. + /// Serializing a complete block and then validating with this function will be less + /// efficient than just validating the original block. + /// + /// This function will determine if the block is new, already known or invalid (either + /// intrinsically or due to some application error.) + /// + /// Note: this function does not implement randao_reveal checking as it is not in the + /// specification. + #[allow(dead_code)] + pub fn validate_ssz_block(&self, b: &SszBlock) + -> Result<(BlockStatus, Option), SszBlockValidationError> + where T: ClientDB + Sized + { - /* - * If this block is already known, return immediately. - */ - let block_hash = &b.block_hash(); - if block_store.block_exists(&block_hash)? { - return Ok((BlockStatus::KnownBlock, None)); - } + /* + * If the block slot corresponds to a slot in the future (according to the local time), + * drop it. + */ + let block_slot = b.slot_number(); + if block_slot > self.present_slot { + return Err(SszBlockValidationError::FutureSlot); + } + + /* + * If this block is already known, return immediately. + */ + let block_hash = &b.block_hash(); + if self.block_store.block_exists(&block_hash)? { + return Ok((BlockStatus::KnownBlock, None)); + } - /* - * If the PoW chain hash is not known to us, drop it. - * - * We only accept blocks that reference a known PoW hash. - * - * Note: it is not clear what a "known" PoW chain ref is. Likely, - * it means "sufficienty deep in the canonical PoW chain". - */ - let pow_chain_ref = b.pow_chain_ref(); - if !pow_store.block_hash_exists(b.pow_chain_ref())? { - return Err(SszBlockValidationError::UnknownPoWChainRef); - } + /* + * If the PoW chain hash is not known to us, drop it. + * + * We only accept blocks that reference a known PoW hash. + * + * Note: it is not clear what a "known" PoW chain ref is. Likely, + * it means "sufficienty deep in the canonical PoW chain". + */ + let pow_chain_ref = b.pow_chain_ref(); + if !self.pow_store.block_hash_exists(b.pow_chain_ref())? { + return Err(SszBlockValidationError::UnknownPoWChainRef); + } - /* - * Store a reference to the serialized attestations from the block. - */ - let attestations_ssz = &b.attestations(); + /* + * Store a reference to the serialized attestations from the block. + */ + let attestations_ssz = &b.attestations(); - /* - * Get a slice of the first serialized attestation (the 0th) and decode it into - * a full AttestationRecord object. - */ - let (first_attestation_ssz, next_index) = split_one_attestation( - &attestations_ssz, - 0)?; - let (first_attestation, _) = AttestationRecord::ssz_decode( - &first_attestation_ssz, 0)?; + /* + * Get a slice of the first serialized attestation (the 0th) and decode it into + * a full AttestationRecord object. + */ + let (first_attestation_ssz, next_index) = split_one_attestation( + &attestations_ssz, + 0)?; + let (first_attestation, _) = AttestationRecord::ssz_decode( + &first_attestation_ssz, 0)?; - /* - * Validate this first attestation. - * - * It is a requirement that the block proposer for this slot - * must have signed the 0th attestation record. - */ - let attestation_voters = validate_attestation( - &first_attestation, - block_slot, - cycle_length, - last_justified_slot, - &parent_hashes, - &block_store, - &validator_store, - &attester_map)?; + /* + * Validate this first attestation. + * + * It is a requirement that the block proposer for this slot + * must have signed the 0th attestation record. + */ + let attestation_voters = validate_attestation( + &first_attestation, + block_slot, + self.cycle_length, + self.last_justified_slot, + &self.parent_hashes, + &self.block_store, + &self.validator_store, + &self.attester_map)?; - /* - * If the set of voters is None, the attestation was invalid. - */ - let attestation_voters = attestation_voters - .ok_or(SszBlockValidationError:: - FirstAttestationSignatureFailed)?; + /* + * If the set of voters is None, the attestation was invalid. + */ + let attestation_voters = attestation_voters + .ok_or(SszBlockValidationError:: + FirstAttestationSignatureFailed)?; - /* - * Read the proposer from the map of slot -> validator index. - */ - let proposer = proposer_map.get(&block_slot) - .ok_or(SszBlockValidationError::BadProposerMap)?; + /* + * Read the proposer from the map of slot -> validator index. + */ + let proposer = self.proposer_map.get(&block_slot) + .ok_or(SszBlockValidationError::BadProposerMap)?; - /* - * If the proposer for this slot was not a voter, reject the block. - */ - if !attestation_voters.contains(&proposer) { - return Err(SszBlockValidationError::NoProposerSignature); - } + /* + * If the proposer for this slot was not a voter, reject the block. + */ + if !attestation_voters.contains(&proposer) { + return Err(SszBlockValidationError::NoProposerSignature); + } - /* - * Split the remaining attestations into a vector of slices, each containing - * a single serialized attestation record. - */ - let other_attestations = split_all_attestations(attestations_ssz, - next_index)?; + /* + * Split the remaining attestations into a vector of slices, each containing + * a single serialized attestation record. + */ + let other_attestations = split_all_attestations(attestations_ssz, + next_index)?; - /* - * Verify each other AttestationRecord. - * - * This uses the `rayon` library to do "sometimes" parallelization. Put simply, - * if there are some spare threads, the verification of attestation records will happen - * concurrently. - * - * There is a thread-safe `failure` variable which is set whenever an attestation fails - * validation. This is so all attestation validation is halted if a single bad attestation - * is found. - */ - let failure: RwLock> = RwLock::new(None); - let deserialized_attestations: Vec = other_attestations - .par_iter() - .filter_map(|attestation_ssz| { - /* - * If some thread has set the `failure` variable to `Some(error)` the abandon - * attestation serialization and validation. - */ - if let Some(_) = *failure.read().unwrap() { - return None; - } - /* - * If there has not been a failure yet, attempt to serialize and validate the - * attestation. - */ - match AttestationRecord::ssz_decode(&attestation_ssz, 0) { + /* + * Verify each other AttestationRecord. + * + * This uses the `rayon` library to do "sometimes" parallelization. Put simply, + * if there are some spare threads, the verification of attestation records will happen + * concurrently. + * + * There is a thread-safe `failure` variable which is set whenever an attestation fails + * validation. This is so all attestation validation is halted if a single bad attestation + * is found. + */ + let failure: RwLock> = RwLock::new(None); + let deserialized_attestations: Vec = other_attestations + .par_iter() + .filter_map(|attestation_ssz| { /* - * Deserialization failed, therefore the block is invalid. + * If some thread has set the `failure` variable to `Some(error)` the abandon + * attestation serialization and validation. */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None + if let Some(_) = *failure.read().unwrap() { + return None; } /* - * Deserialization succeeded and the attestation should be validated. + * If there has not been a failure yet, attempt to serialize and validate the + * attestation. */ - Ok((attestation, _)) => { - let result = validate_attestation( - &attestation, - block_slot, - cycle_length, - last_justified_slot, - &parent_hashes, - &block_store, - &validator_store, - &attester_map); - match result { - /* - * Attestation validation failed with some error. - */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None + match AttestationRecord::ssz_decode(&attestation_ssz, 0) { + /* + * Deserialization failed, therefore the block is invalid. + */ + Err(e) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::from(e)); + None + } + /* + * Deserialization succeeded and the attestation should be validated. + */ + Ok((attestation, _)) => { + let result = validate_attestation( + &attestation, + block_slot, + self.cycle_length, + self.last_justified_slot, + &self.parent_hashes, + &self.block_store, + &self.validator_store, + &self.attester_map); + match result { + /* + * Attestation validation failed with some error. + */ + Err(e) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::from(e)); + None + } + /* + * Attestation validation failed due to a bad signature. + */ + Ok(None) => { + let mut failure = failure.write().unwrap(); + *failure = Some(SszBlockValidationError::AttestationSignatureFailed); + None + } + /* + * Attestation validation succeded. + */ + Ok(_) => Some(attestation) } - /* - * Attestation validation failed due to a bad signature. - */ - Ok(None) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::AttestationSignatureFailed); - None - } - /* - * Attestation validation succeded. - */ - Ok(_) => Some(attestation) } } - } - }) - .collect(); + }) + .collect(); - match failure.into_inner() { - Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), - Ok(failure) => { - match failure { - Some(error) => return Err(error), - _ => () - } + match failure.into_inner() { + Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), + Ok(failure) => { + match failure { + Some(error) => return Err(error), + _ => () + } + } } + + /* + * If we have reached this point, the block is a new valid block that is worthy of + * processing. + */ + + /* + * If the block's parent_hash _is_ in the canonical chain, the block is a + * new block in the canonical chain. Otherwise, it's a new block in a fork chain. + */ + let parent_hash = b.parent_hash(); + let status = if self.block_store.block_exists_in_canonical_chain(&parent_hash)? { + BlockStatus::NewBlockInCanonicalChain + } else { + BlockStatus::NewBlockInForkChain + }; + let block = Block { + parent_hash: Hash256::from(parent_hash), + slot_number: block_slot, + randao_reveal: Hash256::from(b.randao_reveal()), + attestations: deserialized_attestations, + pow_chain_ref: Hash256::from(pow_chain_ref), + active_state_root: Hash256::from(b.act_state_root()), + crystallized_state_root: Hash256::from(b.cry_state_root()), + }; + Ok((status, Some(block))) } - - /* - * If we have reached this point, the block is a new valid block that is worthy of - * processing. - */ - - /* - * If the block's parent_hash _is_ in the canonical chain, the block is a - * new block in the canonical chain. Otherwise, it's a new block in a fork chain. - */ - let parent_hash = b.parent_hash(); - let status = if block_store.block_exists_in_canonical_chain(&parent_hash)? { - BlockStatus::NewBlockInCanonicalChain - } else { - BlockStatus::NewBlockInForkChain - }; - let block = Block { - parent_hash: Hash256::from(parent_hash), - slot_number: block_slot, - randao_reveal: Hash256::from(b.randao_reveal()), - attestations: deserialized_attestations, - pow_chain_ref: Hash256::from(pow_chain_ref), - active_state_root: Hash256::from(b.act_state_root()), - crystallized_state_root: Hash256::from(b.cry_state_root()), - }; - Ok((status, Some(block))) } impl From for SszBlockValidationError { From 385564d637d10924be4864fa0790ef3d50a6ea50 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 11:25:00 +0930 Subject: [PATCH 14/29] Rename block_store functions - Specified that the block is "serialized". --- lighthouse/db/stores/block_store.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/stores/block_store.rs index 2166bea5a..3b5f1dace 100644 --- a/lighthouse/db/stores/block_store.rs +++ b/lighthouse/db/stores/block_store.rs @@ -18,13 +18,13 @@ impl BlockStore { } } - pub fn put_block(&self, hash: &[u8], ssz: &[u8]) + pub fn put_serialized_block(&self, hash: &[u8], ssz: &[u8]) -> Result<(), DBError> { self.db.put(DB_COLUMN, hash, ssz) } - pub fn get_block(&self, hash: &[u8]) + pub fn get_serialized_block(&self, hash: &[u8]) -> Result>, DBError> { self.db.get(DB_COLUMN, hash) @@ -70,7 +70,7 @@ mod tests { for w in 0..wc { let key = (t * w) as u8; let val = 42; - bs.put_block(&vec![key], &vec![val]).unwrap(); + bs.put_serialized_block(&vec![key], &vec![val]).unwrap(); } }); handles.push(handle); @@ -84,7 +84,7 @@ mod tests { for w in 0..write_count { let key = (t * w) as u8; assert!(bs.block_exists(&vec![key]).unwrap()); - let val = bs.get_block(&vec![key]).unwrap().unwrap(); + let val = bs.get_serialized_block(&vec![key]).unwrap().unwrap(); assert_eq!(vec![42], val); } } From 5dd2361d21702e52aa880d951b735f4b7f2d8be2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 11:26:36 +0930 Subject: [PATCH 15/29] Progress further with block validation - Rename errors returned from SszBlock. - Add parent_hash concept to block validation. - Add more comments to block validation. --- lighthouse/state/block/mod.rs | 5 +- lighthouse/state/block/ssz_block.rs | 16 +-- lighthouse/state/block/validation/mod.rs | 1 + lighthouse/state/block/validation/tests.rs | 33 +++-- .../block/validation/validate_ssz_block.rs | 123 ++++++++++++------ 5 files changed, 121 insertions(+), 57 deletions(-) diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 1b8a19439..9a461568d 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -11,4 +11,7 @@ mod ssz_block; mod validation; pub use self::structs::Block; -pub use self::ssz_block::SszBlock; +pub use self::ssz_block::{ + SszBlock, + SszBlockError, +}; diff --git a/lighthouse/state/block/ssz_block.rs b/lighthouse/state/block/ssz_block.rs index 1d4fb1c72..e92291f80 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/lighthouse/state/block/ssz_block.rs @@ -10,7 +10,7 @@ use super::structs::{ use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; #[derive(Debug, PartialEq)] -pub enum BlockValidatorError { +pub enum SszBlockError { TooShort, TooLong, } @@ -46,7 +46,7 @@ impl<'a> SszBlock<'a> { /// how many bytes were read from the slice. In the case of multiple, sequentually serialized /// blocks `len` can be used to assume the location of the next serialized block. pub fn from_slice(vec: &'a [u8]) - -> Result + -> Result { let untrimmed_ssz = &vec[..]; /* @@ -55,19 +55,19 @@ impl<'a> SszBlock<'a> { * attestation record). */ if vec.len() < MIN_SSZ_BLOCK_LENGTH + MIN_SSZ_ATTESTION_RECORD_LENGTH { - return Err(BlockValidatorError::TooShort); + return Err(SszBlockError::TooShort); } /* * Ensure the SSZ slice isn't longer than is possible for a block. */ if vec.len() > MAX_SSZ_BLOCK_LENGTH { - return Err(BlockValidatorError::TooLong); + return Err(SszBlockError::TooLong); } /* * Determine how many bytes are used to store attestation records. */ let attestation_len = decode_length(untrimmed_ssz, 72, LENGTH_BYTES) - .map_err(|_| BlockValidatorError::TooShort)?; + .map_err(|_| SszBlockError::TooShort)?; /* * The block only has one variable field, `attestations`, therefore * the size of the block must be the minimum size, plus the length @@ -77,7 +77,7 @@ impl<'a> SszBlock<'a> { MIN_SSZ_BLOCK_LENGTH + attestation_len }; if vec.len() < block_ssz_len { - return Err(BlockValidatorError::TooShort); + return Err(SszBlockError::TooShort); } Ok(Self{ ssz: &untrimmed_ssz[0..block_ssz_len], @@ -171,7 +171,7 @@ mod tests { assert_eq!( SszBlock::from_slice(&ssz[..]), - Err(BlockValidatorError::TooShort) + Err(SszBlockError::TooShort) ); } @@ -183,7 +183,7 @@ mod tests { assert_eq!( SszBlock::from_slice(&ssz[0..(ssz.len() - 1)]), - Err(BlockValidatorError::TooShort) + Err(SszBlockError::TooShort) ); } diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index e2382da65..29f94c347 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -7,6 +7,7 @@ mod benches; use super::attestation_record; use super::{ SszBlock, + SszBlockError, Block, }; use super::db; diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index a70a25ece..1dc4da2e5 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -92,13 +92,22 @@ pub fn setup_block_validation_scenario(params: &TestParams) let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); - stores.block.put_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - stores.block.put_block(parent_hash.as_ref(), &vec![42]).unwrap(); + stores.block.put_serialized_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); + + /* + * Generate a minimum viable parent block and store it in the database. + */ + let mut parent_block = Block::zero(); + let parent_attestation = AttestationRecord::zero(); + parent_block.slot_number = block_slot - 1; + parent_block.attestations.push(parent_attestation); + let parent_block_ssz = serialize_block(&parent_block); + stores.block.put_serialized_block(parent_hash.as_ref(), &parent_block_ssz).unwrap(); let validator_index: usize = 0; let proposer_map = { let mut proposer_map = ProposerMap::new(); - proposer_map.insert(block_slot, validator_index); + proposer_map.insert(parent_block.slot_number, validator_index); proposer_map }; @@ -209,6 +218,7 @@ pub fn serialize_block(b: &Block) -> Vec { pub fn run_block_validation_scenario( validation_slot: u64, validation_last_justified_slot: u64, + validation_last_finalized_slot: u64, params: &TestParams, mutator_func: F) -> Result<(BlockStatus, Option), SszBlockValidationError> @@ -234,6 +244,7 @@ pub fn run_block_validation_scenario( present_slot: validation_slot, cycle_length: params.cycle_length, last_justified_slot: validation_last_justified_slot, + last_finalized_slot: validation_last_finalized_slot, parent_hashes: Arc::new(parent_hashes), proposer_map: Arc::new(proposer_map), attester_map: Arc::new(attester_map), @@ -265,11 +276,12 @@ fn get_simple_params() -> TestParams { } #[test] -fn test_block_validation_simple_scenario_valid_in_canonical_chain() { +fn test_block_validation_simple_scenario_valid() { let params = get_simple_params(); let validation_slot = params.block_slot; let validation_last_justified_slot = params.attestations_justified_slot; + let validation_last_finalized_slot = 0; let no_mutate = |block, attester_map, proposer_map, stores| { (block, attester_map, proposer_map, stores) @@ -278,31 +290,34 @@ fn test_block_validation_simple_scenario_valid_in_canonical_chain() { let status = run_block_validation_scenario( validation_slot, validation_last_justified_slot, + validation_last_finalized_slot, ¶ms, no_mutate); - assert_eq!(status.unwrap().0, BlockStatus::NewBlockInCanonicalChain); + assert_eq!(status.unwrap().0, BlockStatus::NewBlock); } #[test] -fn test_block_validation_simple_scenario_valid_not_in_canonical_chain() { +fn test_block_validation_simple_scenario_invalid_unknown_parent_block() { let params = get_simple_params(); let validation_slot = params.block_slot; let validation_last_justified_slot = params.attestations_justified_slot; + let validation_last_finalized_slot = 0; let no_mutate = |mut block: Block, attester_map, proposer_map, stores| { - block.parent_hash = Hash256::from("not in canonical chain".as_bytes()); + block.parent_hash = Hash256::from("unknown parent block".as_bytes()); (block, attester_map, proposer_map, stores) }; let status = run_block_validation_scenario( validation_slot, validation_last_justified_slot, + validation_last_finalized_slot, ¶ms, no_mutate); - assert_eq!(status.unwrap().0, BlockStatus::NewBlockInForkChain); + assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); } #[test] @@ -311,6 +326,7 @@ fn test_block_validation_simple_scenario_invalid_2nd_attestation() { let validation_slot = params.block_slot; let validation_last_justified_slot = params.attestations_justified_slot; + let validation_last_finalized_slot = 0; let mutator = |mut block: Block, attester_map, proposer_map, stores| { /* @@ -323,6 +339,7 @@ fn test_block_validation_simple_scenario_invalid_2nd_attestation() { let status = run_block_validation_scenario( validation_slot, validation_last_justified_slot, + validation_last_finalized_slot, ¶ms, mutator); diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index 0ad688145..d2aa3c853 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -1,4 +1,5 @@ extern crate rayon; + use self::rayon::prelude::*; use std::sync::{ @@ -21,6 +22,7 @@ use super::{ }; use super::{ SszBlock, + SszBlockError, Block, }; use super::db::{ @@ -40,24 +42,25 @@ use super::utils::types::Hash256; #[derive(Debug, PartialEq)] pub enum BlockStatus { - NewBlockInCanonicalChain, - NewBlockInForkChain, + NewBlock, KnownBlock, } #[derive(Debug, PartialEq)] pub enum SszBlockValidationError { FutureSlot, + SlotAlreadyFinalized, UnknownPoWChainRef, UnknownParentHash, BadAttestationSsz, AttestationValidationError(AttestationValidationError), AttestationSignatureFailed, FirstAttestationSignatureFailed, + ProposerAttestationHasObliqueHashes, NoProposerSignature, BadProposerMap, RwLockPoisoned, - DatabaseError(String), + DBError(String), } /// The context against which a block should be validated. @@ -70,6 +73,8 @@ pub struct BlockValidationContext pub cycle_length: u8, /// The last justified slot as per the client's view of the canonical chain. pub last_justified_slot: u64, + /// The last finalized slot as per the client's view of the canonical chain. + pub last_finalized_slot: u64, /// A vec of the hashes of the blocks preceeding the present slot. pub parent_hashes: Arc>, /// A map of slots to a block proposer validation index. @@ -110,8 +115,16 @@ impl BlockValidationContext { /* - * If the block slot corresponds to a slot in the future (according to the local time), - * drop it. + * If this block is already known, return immediately and indicate the the block is + * known. Don't attempt to deserialize the block. + */ + let block_hash = &b.block_hash(); + if self.block_store.block_exists(&block_hash)? { + return Ok((BlockStatus::KnownBlock, None)); + } + + /* + * If the block slot corresponds to a slot in the future, drop it. */ let block_slot = b.slot_number(); if block_slot > self.present_slot { @@ -119,21 +132,23 @@ impl BlockValidationContext } /* - * If this block is already known, return immediately. + * If the block is unknown (assumed unknown because we checked the db earlier in this + * function) and it comes from a slot that is already finalized, drop the block. + * + * If a slot is finalized, there's no point in considering any other blocks for that slot. */ - let block_hash = &b.block_hash(); - if self.block_store.block_exists(&block_hash)? { - return Ok((BlockStatus::KnownBlock, None)); + if block_slot <= self.last_finalized_slot { + return Err(SszBlockValidationError::SlotAlreadyFinalized); } - /* * If the PoW chain hash is not known to us, drop it. * * We only accept blocks that reference a known PoW hash. * - * Note: it is not clear what a "known" PoW chain ref is. Likely, - * it means "sufficienty deep in the canonical PoW chain". + * Note: it is not clear what a "known" PoW chain ref is. Likely it means the block hash is + * "sufficienty deep in the canonical PoW chain". This should be clarified as the spec + * crystallizes. */ let pow_chain_ref = b.pow_chain_ref(); if !self.pow_store.block_hash_exists(b.pow_chain_ref())? { @@ -141,13 +156,16 @@ impl BlockValidationContext } /* - * Store a reference to the serialized attestations from the block. + * Store a slice of the serialized attestations from the block SSZ. */ let attestations_ssz = &b.attestations(); /* - * Get a slice of the first serialized attestation (the 0th) and decode it into + * Get a slice of the first serialized attestation (the 0'th) and decode it into * a full AttestationRecord object. + * + * The first attestation must be validated separately as it must contain a signature of the + * proposer of the previous block (this is checked later in this function). */ let (first_attestation_ssz, next_index) = split_one_attestation( &attestations_ssz, @@ -156,10 +174,17 @@ impl BlockValidationContext &first_attestation_ssz, 0)?; /* - * Validate this first attestation. + * The first attestation may not have oblique hashes. * - * It is a requirement that the block proposer for this slot - * must have signed the 0th attestation record. + * The presence of oblique hashes in the first attestation would indicate that the proposer + * of the previous block is attesting to some other block than the one they produced. + */ + if first_attestation.oblique_parent_hashes.len() > 0 { + return Err(SszBlockValidationError::ProposerAttestationHasObliqueHashes); + } + + /* + * Validate this first attestation. */ let attestation_voters = validate_attestation( &first_attestation, @@ -179,16 +204,28 @@ impl BlockValidationContext FirstAttestationSignatureFailed)?; /* - * Read the proposer from the map of slot -> validator index. + * Read the parent hash from the block we are validating then attempt to load + * the parent block ssz from the database. If that parent doesn't exist in + * the database, reject the block. + * + * If the parent does exist in the database, read the slot of that parent. Then, + * determine the proposer of that slot (the parent slot) by looking it up + * in the proposer map. + * + * If that proposer (the proposer of the parent block) was not present in the first (0'th) + * attestation of this block, reject the block. */ - let proposer = self.proposer_map.get(&block_slot) - .ok_or(SszBlockValidationError::BadProposerMap)?; - - /* - * If the proposer for this slot was not a voter, reject the block. - */ - if !attestation_voters.contains(&proposer) { - return Err(SszBlockValidationError::NoProposerSignature); + let parent_hash = b.parent_hash(); + match self.block_store.get_serialized_block(&parent_hash)? { + None => return Err(SszBlockValidationError::UnknownParentHash), + Some(ssz) => { + let parent_block = SszBlock::from_slice(&ssz[..])?; + let proposer = self.proposer_map.get(&parent_block.slot_number()) + .ok_or(SszBlockValidationError::BadProposerMap)?; + if !attestation_voters.contains(&proposer) { + return Err(SszBlockValidationError::NoProposerSignature); + } + } } /* @@ -210,7 +247,7 @@ impl BlockValidationContext * is found. */ let failure: RwLock> = RwLock::new(None); - let deserialized_attestations: Vec = other_attestations + let mut deserialized_attestations: Vec = other_attestations .par_iter() .filter_map(|attestation_ssz| { /* @@ -284,21 +321,16 @@ impl BlockValidationContext } } + /* + * Add the first attestation to the vec of deserialized attestations at + * index 0. + */ + deserialized_attestations.insert(0, first_attestation); + /* * If we have reached this point, the block is a new valid block that is worthy of * processing. */ - - /* - * If the block's parent_hash _is_ in the canonical chain, the block is a - * new block in the canonical chain. Otherwise, it's a new block in a fork chain. - */ - let parent_hash = b.parent_hash(); - let status = if self.block_store.block_exists_in_canonical_chain(&parent_hash)? { - BlockStatus::NewBlockInCanonicalChain - } else { - BlockStatus::NewBlockInForkChain - }; let block = Block { parent_hash: Hash256::from(parent_hash), slot_number: block_slot, @@ -308,13 +340,13 @@ impl BlockValidationContext active_state_root: Hash256::from(b.act_state_root()), crystallized_state_root: Hash256::from(b.cry_state_root()), }; - Ok((status, Some(block))) + Ok((BlockStatus::NewBlock, Some(block))) } } impl From for SszBlockValidationError { fn from(e: DBError) -> Self { - SszBlockValidationError::DatabaseError(e.message) + SszBlockValidationError::DBError(e.message) } } @@ -327,6 +359,17 @@ impl From for SszBlockValidationError { } } +impl From for SszBlockValidationError { + fn from(e: SszBlockError) -> Self { + match e { + SszBlockError::TooShort => + SszBlockValidationError::DBError("Bad parent block in db.".to_string()), + SszBlockError::TooLong => + SszBlockValidationError::DBError("Bad parent block in db.".to_string()), + } + } +} + impl From for SszBlockValidationError { fn from(e: DecodeError) -> Self { match e { From 9642c4b7e1b9a8b62ac070dc5f6737cc4687c3d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 12:21:30 +0930 Subject: [PATCH 16/29] Simplify block validation tests Move more parameters into get_simple_params() --- lighthouse/state/block/validation/tests.rs | 40 +++++++--------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index 1dc4da2e5..a8d6f8fba 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -64,6 +64,9 @@ pub struct TestParams { pub validators_per_shard: usize, pub block_slot: u64, pub attestations_justified_slot: u64, + pub validation_context_slot: u64, + pub validation_context_justified_slot: u64, + pub validation_context_finalized_slot: u64, } type ParentHashes = Vec; @@ -216,9 +219,6 @@ pub fn serialize_block(b: &Block) -> Vec { /// /// Returns the Result returned from the block validation function. pub fn run_block_validation_scenario( - validation_slot: u64, - validation_last_justified_slot: u64, - validation_last_finalized_slot: u64, params: &TestParams, mutator_func: F) -> Result<(BlockStatus, Option), SszBlockValidationError> @@ -241,10 +241,10 @@ pub fn run_block_validation_scenario( .unwrap(); let context = BlockValidationContext { - present_slot: validation_slot, + present_slot: params.validation_context_slot, cycle_length: params.cycle_length, - last_justified_slot: validation_last_justified_slot, - last_finalized_slot: validation_last_finalized_slot, + last_justified_slot: params.validation_context_justified_slot, + last_finalized_slot: params.validation_context_finalized_slot, parent_hashes: Arc::new(parent_hashes), proposer_map: Arc::new(proposer_map), attester_map: Arc::new(attester_map), @@ -264,6 +264,10 @@ fn get_simple_params() -> TestParams { let block_slot = u64::from(cycle_length) * 10000; let attestations_justified_slot = block_slot - u64::from(cycle_length); + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + TestParams { total_validators, cycle_length, @@ -272,6 +276,9 @@ fn get_simple_params() -> TestParams { validators_per_shard, block_slot, attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, } } @@ -279,18 +286,11 @@ fn get_simple_params() -> TestParams { fn test_block_validation_simple_scenario_valid() { let params = get_simple_params(); - let validation_slot = params.block_slot; - let validation_last_justified_slot = params.attestations_justified_slot; - let validation_last_finalized_slot = 0; - let no_mutate = |block, attester_map, proposer_map, stores| { (block, attester_map, proposer_map, stores) }; let status = run_block_validation_scenario( - validation_slot, - validation_last_justified_slot, - validation_last_finalized_slot, ¶ms, no_mutate); @@ -301,19 +301,12 @@ fn test_block_validation_simple_scenario_valid() { fn test_block_validation_simple_scenario_invalid_unknown_parent_block() { let params = get_simple_params(); - let validation_slot = params.block_slot; - let validation_last_justified_slot = params.attestations_justified_slot; - let validation_last_finalized_slot = 0; - let no_mutate = |mut block: Block, attester_map, proposer_map, stores| { block.parent_hash = Hash256::from("unknown parent block".as_bytes()); (block, attester_map, proposer_map, stores) }; let status = run_block_validation_scenario( - validation_slot, - validation_last_justified_slot, - validation_last_finalized_slot, ¶ms, no_mutate); @@ -324,10 +317,6 @@ fn test_block_validation_simple_scenario_invalid_unknown_parent_block() { fn test_block_validation_simple_scenario_invalid_2nd_attestation() { let params = get_simple_params(); - let validation_slot = params.block_slot; - let validation_last_justified_slot = params.attestations_justified_slot; - let validation_last_finalized_slot = 0; - let mutator = |mut block: Block, attester_map, proposer_map, stores| { /* * Set the second attestaion record to have an invalid signature. @@ -337,9 +326,6 @@ fn test_block_validation_simple_scenario_invalid_2nd_attestation() { }; let status = run_block_validation_scenario( - validation_slot, - validation_last_justified_slot, - validation_last_finalized_slot, ¶ms, mutator); From a87fe88d975d353bc1f050c750299669b91c4c8f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 14:24:03 +0930 Subject: [PATCH 17/29] Fix bug in boolean bitfield The length generated when converting from a string was incorrect. --- boolean-bitfield/src/lib.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index 6ac8e082b..60d877e65 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -109,8 +109,8 @@ impl BooleanBitfield { pub fn compute_length(bytes: &[u8]) -> usize { for byte in (0..bytes.len()).rev() { for bit in (0..8).rev() { - if byte & (1 << (bit as u8)) != 0 { - return (byte * 8) + bit + if bytes[byte] & (1 << (bit as u8)) != 0 { + return (byte * 8) + bit + 1 } } } @@ -188,6 +188,25 @@ mod tests { use super::*; use ssz::Decodable; + #[test] + fn test_new_from_slice() { + let s = [0]; + let b = BooleanBitfield::from(&s[..]); + assert_eq!(b.len, 0); + + let s = [255]; + let b = BooleanBitfield::from(&s[..]); + assert_eq!(b.len, 8); + + let s = [0, 1]; + let b = BooleanBitfield::from(&s[..]); + assert_eq!(b.len, 9); + + let s = [31]; + let b = BooleanBitfield::from(&s[..]); + assert_eq!(b.len, 5); + } + #[test] fn test_ssz_encoding() { let mut b = BooleanBitfield::new(); From 496adc0f0b7e12c5842f7f1507933d82a1747d6e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 14:24:42 +0930 Subject: [PATCH 18/29] Fix bug in attestation val. bitfield checking There was a logic error --- .../attestation_record/validation/attestation_validation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index db2a781de..1c0b149d8 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -161,7 +161,7 @@ fn bytes_for_bits(bits: usize) -> usize { fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool { for i in 0..n { - let mask = 0_u8 >> 8_usize - i as usize; + let mask = 1_u8 >> 7_usize.saturating_sub(i as usize); if byte & mask > 0 { return true } From 77b48b98223785196597887c5f2845b007382f69 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 14:25:42 +0930 Subject: [PATCH 19/29] Implement more tests for block validation --- lighthouse/state/block/structs.rs | 2 +- lighthouse/state/block/validation/tests.rs | 170 +++++++++++++++++++-- 2 files changed, 161 insertions(+), 11 deletions(-) diff --git a/lighthouse/state/block/structs.rs b/lighthouse/state/block/structs.rs index f22bc370d..eb8833a40 100644 --- a/lighthouse/state/block/structs.rs +++ b/lighthouse/state/block/structs.rs @@ -13,7 +13,7 @@ pub const MIN_SSZ_BLOCK_LENGTH: usize = { }; pub const MAX_SSZ_BLOCK_LENGTH: usize = MIN_SSZ_BLOCK_LENGTH + (1 << 24); -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct Block { pub parent_hash: Hash256, pub slot_number: u64, diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests.rs index a8d6f8fba..e0fa153c2 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests.rs @@ -64,6 +64,7 @@ pub struct TestParams { pub validators_per_shard: usize, pub block_slot: u64, pub attestations_justified_slot: u64, + pub parent_proposer_index: usize, pub validation_context_slot: u64, pub validation_context_justified_slot: u64, pub validation_context_finalized_slot: u64, @@ -107,10 +108,9 @@ pub fn setup_block_validation_scenario(params: &TestParams) let parent_block_ssz = serialize_block(&parent_block); stores.block.put_serialized_block(parent_hash.as_ref(), &parent_block_ssz).unwrap(); - let validator_index: usize = 0; let proposer_map = { let mut proposer_map = ProposerMap::new(); - proposer_map.insert(parent_block.slot_number, validator_index); + proposer_map.insert(parent_block.slot_number, params.parent_proposer_index); proposer_map }; @@ -252,7 +252,16 @@ pub fn run_block_validation_scenario( validator_store: stores.validator.clone(), pow_store: stores.pow_chain.clone() }; - context.validate_ssz_block(&ssz_block) + let validation_status = context.validate_ssz_block(&ssz_block); + /* + * If validation returned a block, make sure it's the same block we supplied to it. + * + * I.e., there were no errors during the serialization -> deserialization process. + */ + if let Ok((_, Some(returned_block))) = &validation_status { + assert_eq!(*returned_block, block); + }; + validation_status } fn get_simple_params() -> TestParams { @@ -263,6 +272,7 @@ fn get_simple_params() -> TestParams { let total_validators: usize = validators_per_shard * shard_count as usize; let block_slot = u64::from(cycle_length) * 10000; let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; let validation_context_slot = block_slot; let validation_context_justified_slot = attestations_justified_slot; @@ -274,6 +284,7 @@ fn get_simple_params() -> TestParams { shard_count, shards_per_slot, validators_per_shard, + parent_proposer_index, block_slot, attestations_justified_slot, validation_context_slot, @@ -282,39 +293,178 @@ fn get_simple_params() -> TestParams { } } +// TODO: test bad ssz serialization + #[test] -fn test_block_validation_simple_scenario_valid() { +fn test_block_validation_valid() { let params = get_simple_params(); - let no_mutate = |block, attester_map, proposer_map, stores| { + let mutator = |block: Block, attester_map, proposer_map, stores| { + /* + * Do not mutate + */ (block, attester_map, proposer_map, stores) }; let status = run_block_validation_scenario( ¶ms, - no_mutate); + mutator); assert_eq!(status.unwrap().0, BlockStatus::NewBlock); } #[test] -fn test_block_validation_simple_scenario_invalid_unknown_parent_block() { +fn test_block_validation_valid_known_block() { let params = get_simple_params(); - let no_mutate = |mut block: Block, attester_map, proposer_map, stores| { + let mutator = |block: Block, attester_map, proposer_map, stores: TestStore| { + /* + * Pre-store the block in the database + */ + let block_ssz = serialize_block(&block); + let block_hash = canonical_hash(&block_ssz); + stores.block.put_serialized_block(&block_hash, &block_ssz).unwrap(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status.unwrap(), (BlockStatus::KnownBlock, None)); +} + +#[test] +fn test_block_validation_invalid_future_slot() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + block.slot_number = block.slot_number + 1; + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::FutureSlot)); +} + +#[test] +fn test_block_validation_invalid_slot_already_finalized() { + let mut params = get_simple_params(); + + params.validation_context_finalized_slot = params.block_slot; + params.validation_context_justified_slot = params.validation_context_finalized_slot + + u64::from(params.cycle_length); + + let mutator = |block, attester_map, proposer_map, stores| { + /* + * Do not mutate + */ + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::SlotAlreadyFinalized)); +} + +#[test] +fn test_block_validation_invalid_unknown_pow_hash() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + block.pow_chain_ref = Hash256::from("unknown pow hash".as_bytes()); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::UnknownPoWChainRef)); +} + +#[test] +fn test_block_validation_invalid_unknown_parent_hash() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { block.parent_hash = Hash256::from("unknown parent block".as_bytes()); (block, attester_map, proposer_map, stores) }; let status = run_block_validation_scenario( ¶ms, - no_mutate); + mutator); assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); } #[test] -fn test_block_validation_simple_scenario_invalid_2nd_attestation() { +fn test_block_validation_invalid_1st_attestation_signature() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + /* + * Set the second attestaion record to have an invalid signature. + */ + block.attestations[0].aggregate_sig = AggregateSignature::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::FirstAttestationSignatureFailed)); +} + +#[test] +fn test_block_validation_invalid_no_parent_proposer_signature() { + let params = get_simple_params(); + + let mutator = |block: Block, attester_map, mut proposer_map: ProposerMap, stores: TestStore| { + /* + * Set the proposer for this slot to be a validator that does not exist. + */ + let ssz = stores.block.get_serialized_block(&block.parent_hash.as_ref()).unwrap().unwrap(); + let parent_block_slot = SszBlock::from_slice(&ssz[..]).unwrap().slot_number(); + proposer_map.insert(parent_block_slot, params.total_validators + 1); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::NoProposerSignature)); +} + +#[test] +fn test_block_validation_invalid_bad_proposer_map() { + let params = get_simple_params(); + + let mutator = |block, attester_map, _, stores| { + /* + * Initialize a new, empty proposer map + */ + let proposer_map = ProposerMap::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::BadProposerMap)); +} + +#[test] +fn test_block_validation_invalid_2nd_attestation_signature() { let params = get_simple_params(); let mutator = |mut block: Block, attester_map, proposer_map, stores| { From 29ed29cfc3933fb23c4099d346f9f518684759e4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 15:38:22 +0930 Subject: [PATCH 20/29] Update benchmarks as per recent code changes --- lighthouse/state/block/validation/benches.rs | 37 ++++++++++++-------- lighthouse/state/block/validation/mod.rs | 2 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 3b93e6786..32427a2ad 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -24,8 +24,6 @@ use super::super::{ fn bench_block_validation_scenario( b: &mut Bencher, - validation_slot: u64, - validation_last_justified_slot: u64, params: &TestParams, mutator_func: F) where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) @@ -51,9 +49,10 @@ fn bench_block_validation_scenario( let attester_map = Arc::new(attester_map); b.iter(|| { let context = BlockValidationContext { - present_slot: validation_slot, + present_slot: params.validation_context_slot, cycle_length: params.cycle_length, - last_justified_slot: validation_last_justified_slot, + last_justified_slot: params.validation_context_justified_slot, + last_finalized_slot: params.validation_context_finalized_slot, parent_hashes: parent_hashes.clone(), proposer_map: proposer_map.clone(), attester_map: attester_map.clone(), @@ -61,7 +60,8 @@ fn bench_block_validation_scenario( validator_store: stores.validator.clone(), pow_store: stores.pow_chain.clone() }; - context.validate_ssz_block(&ssz_block) + let result = context.validate_ssz_block(&ssz_block); + assert!(result.is_ok()); }); } @@ -74,6 +74,11 @@ fn bench_block_validation_10m_eth(b: &mut Bencher) { let validators_per_shard: usize = total_validators / usize::from(shard_count); let block_slot = u64::from(cycle_length) * 10000; let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; let params = TestParams { total_validators, @@ -81,11 +86,13 @@ fn bench_block_validation_10m_eth(b: &mut Bencher) { shard_count, shards_per_slot, validators_per_shard, + parent_proposer_index, block_slot, attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, }; - let validation_slot = params.block_slot; - let validation_last_justified_slot = params.attestations_justified_slot; let no_mutate = |block, attester_map, proposer_map, stores| { (block, attester_map, proposer_map, stores) @@ -93,8 +100,6 @@ fn bench_block_validation_10m_eth(b: &mut Bencher) { bench_block_validation_scenario( b, - validation_slot, - validation_last_justified_slot, ¶ms, no_mutate); } @@ -108,6 +113,11 @@ fn bench_block_validation_100m_eth(b: &mut Bencher) { let validators_per_shard: usize = total_validators / usize::from(shard_count); let block_slot = u64::from(cycle_length) * 10000; let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; let params = TestParams { total_validators, @@ -115,21 +125,20 @@ fn bench_block_validation_100m_eth(b: &mut Bencher) { shard_count, shards_per_slot, validators_per_shard, + parent_proposer_index, block_slot, attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, }; - let validation_slot = params.block_slot; - let validation_last_justified_slot = params.attestations_justified_slot; - let no_mutate = |block, attester_map, proposer_map, stores| { (block, attester_map, proposer_map, stores) }; bench_block_validation_scenario( b, - validation_slot, - validation_last_justified_slot, ¶ms, no_mutate); } diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index 29f94c347..faf3ad096 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,7 +1,7 @@ mod validate_ssz_block; #[cfg(test)] mod tests; -#[cfg(all(feature = "benches", test))] +#[cfg(test)] mod benches; use super::attestation_record; From b426c9e7243004f817bc107b673de1b6055838e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 16:09:07 +0930 Subject: [PATCH 21/29] Remove "benches" feature Now benches just live with the rest of the code in the default feature. --- Cargo.toml | 3 --- lighthouse/state/block/validation/benches.rs | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 682adbe63..c076461c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,3 @@ ring = { git = "https://github.com/paritytech/ring" } [[bin]] path = "lighthouse/main.rs" name = "lighthouse" - -[features] -benches = [] diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs index 32427a2ad..538ade41f 100644 --- a/lighthouse/state/block/validation/benches.rs +++ b/lighthouse/state/block/validation/benches.rs @@ -66,6 +66,7 @@ fn bench_block_validation_scenario( } #[bench] +#[ignore] fn bench_block_validation_10m_eth(b: &mut Bencher) { let total_validators: usize = 10_000_000 / 32; let cycle_length: u8 = 64; @@ -105,6 +106,7 @@ fn bench_block_validation_10m_eth(b: &mut Bencher) { } #[bench] +#[ignore] fn bench_block_validation_100m_eth(b: &mut Bencher) { let total_validators: usize = 100_000_000 / 32; let cycle_length: u8 = 64; From c3ec8a3407c9218573f56e951eb22b6276d3c6ae Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 16:09:35 +0930 Subject: [PATCH 22/29] Introduce AttestationValidationContext This reduced code duplication. --- lighthouse/state/attestation_record/mod.rs | 2 +- .../validation/attestation_validation.rs | 225 ++++++++++-------- .../attestation_record/validation/mod.rs | 2 +- .../block/validation/validate_ssz_block.rs | 41 ++-- 4 files changed, 141 insertions(+), 129 deletions(-) diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs index 8f09f9f36..d1813b2a5 100644 --- a/lighthouse/state/attestation_record/mod.rs +++ b/lighthouse/state/attestation_record/mod.rs @@ -19,6 +19,6 @@ pub use self::ssz_splitter::{ AttestationSplitError, }; pub use self::validation::{ - validate_attestation, + AttestationValidationContext, AttestationValidationError, }; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index 1c0b149d8..f8ed3bdc0 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -45,114 +45,133 @@ pub enum AttestationValidationError { DBError(String), } -pub fn validate_attestation(a: &AttestationRecord, - block_slot: u64, - cycle_length: u8, - known_last_justified_slot: u64, - known_parent_hashes: &Arc>, - block_store: &Arc>, - validator_store: &Arc>, - attester_map: &Arc) - -> Result>, AttestationValidationError> +pub struct AttestationValidationContext where T: ClientDB + Sized { - /* - * The attesation slot must not be higher than the block that contained it. - */ - if a.slot > block_slot { - return Err(AttestationValidationError::SlotTooHigh); - } + pub block_slot: u64, + pub cycle_length: u8, + pub last_justified_slot: u64, + pub parent_hashes: Arc>, + pub block_store: Arc>, + pub validator_store: Arc>, + pub attester_map: Arc, +} - /* - * The slot of this attestation must not be more than cycle_length + 1 distance - * from the block that contained it. - * - * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. - */ - if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) { - return Err(AttestationValidationError::SlotTooLow); - } - - /* - * The attestation must indicate that its last justified slot is the same as the last justified - * slot known to us. - */ - if a.justified_slot > known_last_justified_slot { - return Err(AttestationValidationError::JustifiedSlotTooHigh); - } - - /* - * There is no need to include more oblique parents hashes than there are blocks - * in a cycle. - */ - if a.oblique_parent_hashes.len() > usize::from(cycle_length) { - return Err(AttestationValidationError::TooManyObliqueHashes); - } - - /* - * Retrieve the set of attestation indices for this slot and shard id. - * - * This is an array mapping the order that validators will appear in the bitfield to the - * canonincal index of a validator. - */ - let attestation_indices = attester_map.get(&(a.slot, a.shard_id)) - .ok_or(AttestationValidationError::BadAttesterMap)?; - - /* - * The bitfield must be no longer than the minimum required to represent each validator in the - * attestation indicies for this slot and shard id. - */ - if a.attester_bitfield.num_bytes() != - bytes_for_bits(attestation_indices.len()) +impl AttestationValidationContext + where T: ClientDB +{ + pub fn validate_attestation(&self, a: &AttestationRecord) + /* + block_slot: u64, + cycle_length: u8, + known_last_justified_slot: u64, + known_parent_hashes: &Arc>, + block_store: &Arc>, + validator_store: &Arc>, + attester_map: &Arc) + */ + -> Result>, AttestationValidationError> + where T: ClientDB + Sized { - return Err(AttestationValidationError::BadBitfieldLength); + /* + * The attesation slot must not be higher than the block that contained it. + */ + if a.slot > self.block_slot { + return Err(AttestationValidationError::SlotTooHigh); + } + + /* + * The slot of this attestation must not be more than cycle_length + 1 distance + * from the block that contained it. + * + * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. + */ + if a.slot < self.block_slot + .saturating_sub(u64::from(self.cycle_length).saturating_add(1)) { + return Err(AttestationValidationError::SlotTooLow); + } + + /* + * The attestation must indicate that its last justified slot is the same as the last justified + * slot known to us. + */ + if a.justified_slot > self.last_justified_slot { + return Err(AttestationValidationError::JustifiedSlotTooHigh); + } + + /* + * There is no need to include more oblique parents hashes than there are blocks + * in a cycle. + */ + if a.oblique_parent_hashes.len() > usize::from(self.cycle_length) { + return Err(AttestationValidationError::TooManyObliqueHashes); + } + + /* + * Retrieve the set of attestation indices for this slot and shard id. + * + * This is an array mapping the order that validators will appear in the bitfield to the + * canonincal index of a validator. + */ + let attestation_indices = self.attester_map.get(&(a.slot, a.shard_id)) + .ok_or(AttestationValidationError::BadAttesterMap)?; + + /* + * The bitfield must be no longer than the minimum required to represent each validator in the + * attestation indicies for this slot and shard id. + */ + if a.attester_bitfield.num_bytes() != + bytes_for_bits(attestation_indices.len()) + { + return Err(AttestationValidationError::BadBitfieldLength); + } + + /* + * If there are excess bits in the bitfield because the number of a validators in not a + * multiple of 8, reject this attestation record. + * + * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to + * refer to the same AttesationRecord. + */ + let last_byte = + a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) + .ok_or(AttestationValidationError::InvalidBitfield)?; + if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) { + return Err(AttestationValidationError::InvalidBitfieldEndBits) + } + + /* + * The specified justified block hash must be known to us + */ + if !self.block_store.block_exists(&a.justified_block_hash)? { + return Err(AttestationValidationError::UnknownJustifiedBlock) + } + + let signed_message = { + let parent_hashes = attestation_parent_hashes( + self.cycle_length, + self.block_slot, + a.slot, + &self.parent_hashes, + &a.oblique_parent_hashes)?; + generate_signed_message( + a.slot, + &parent_hashes, + a.shard_id, + &a.shard_block_hash, + a.justified_slot) + }; + + let voted_hashmap = + verify_aggregate_signature_for_indices( + &signed_message, + &a.aggregate_sig, + &attestation_indices, + &a.attester_bitfield, + &self.validator_store)?; + + Ok(voted_hashmap) } - - /* - * If there are excess bits in the bitfield because the number of a validators in not a - * multiple of 8, reject this attestation record. - * - * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to - * refer to the same AttesationRecord. - */ - let last_byte = - a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) - .ok_or(AttestationValidationError::InvalidBitfield)?; - if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) { - return Err(AttestationValidationError::InvalidBitfieldEndBits) - } - - /* - * The specified justified block hash must be known to us - */ - if !block_store.block_exists(&a.justified_block_hash)? { - return Err(AttestationValidationError::UnknownJustifiedBlock) - } - - let signed_message = { - let parent_hashes = attestation_parent_hashes( - cycle_length, - block_slot, - a.slot, - &known_parent_hashes, - &a.oblique_parent_hashes)?; - generate_signed_message( - a.slot, - &parent_hashes, - a.shard_id, - &a.shard_block_hash, - a.justified_slot) - }; - - let voted_hashmap = - verify_aggregate_signature_for_indices( - &signed_message, - &a.aggregate_sig, - &attestation_indices, - &a.attester_bitfield, - &validator_store)?; - - Ok(voted_hashmap) } fn bytes_for_bits(bits: usize) -> usize { diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs index 43ae6fc0c..dd3c589c5 100644 --- a/lighthouse/state/attestation_record/validation/mod.rs +++ b/lighthouse/state/attestation_record/validation/mod.rs @@ -11,6 +11,6 @@ mod signature_verification; mod message_generation; pub use self::attestation_validation::{ - validate_attestation, + AttestationValidationContext, AttestationValidationError, }; diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/validate_ssz_block.rs index d2aa3c853..11ea54e4d 100644 --- a/lighthouse/state/block/validation/validate_ssz_block.rs +++ b/lighthouse/state/block/validation/validate_ssz_block.rs @@ -7,7 +7,7 @@ use std::sync::{ RwLock, }; use super::attestation_record::{ - validate_attestation, + AttestationValidationContext, AttestationValidationError, }; use super::attestation_record::{ @@ -99,10 +99,6 @@ impl BlockValidationContext /// a suspicion that the block might be invalid. Such a suspicion should be applied to /// all blocks coming from the network. /// - /// Of course, this function will only be more efficient if a block is already serialized. - /// Serializing a complete block and then validating with this function will be less - /// efficient than just validating the original block. - /// /// This function will determine if the block is new, already known or invalid (either /// intrinsically or due to some application error.) /// @@ -183,18 +179,24 @@ impl BlockValidationContext return Err(SszBlockValidationError::ProposerAttestationHasObliqueHashes); } + /* + * Generate the context in which attestations will be validated. + */ + let attestation_validation_context = Arc::new(AttestationValidationContext { + block_slot, + cycle_length: self.cycle_length, + last_justified_slot: self.last_justified_slot, + parent_hashes: self.parent_hashes.clone(), + block_store: self.block_store.clone(), + validator_store: self.validator_store.clone(), + attester_map: self.attester_map.clone(), + }); + /* * Validate this first attestation. */ - let attestation_voters = validate_attestation( - &first_attestation, - block_slot, - self.cycle_length, - self.last_justified_slot, - &self.parent_hashes, - &self.block_store, - &self.validator_store, - &self.attester_map)?; + let attestation_voters = attestation_validation_context + .validate_attestation(&first_attestation)?; /* * If the set of voters is None, the attestation was invalid. @@ -274,16 +276,7 @@ impl BlockValidationContext * Deserialization succeeded and the attestation should be validated. */ Ok((attestation, _)) => { - let result = validate_attestation( - &attestation, - block_slot, - self.cycle_length, - self.last_justified_slot, - &self.parent_hashes, - &self.block_store, - &self.validator_store, - &self.attester_map); - match result { + match attestation_validation_context.validate_attestation(&attestation) { /* * Attestation validation failed with some error. */ From 4d1f730c9556725f4bc02908f8bf687c4a9a73ba Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Sep 2018 16:25:28 +0930 Subject: [PATCH 23/29] Restructure block validation directory --- ...idate_ssz_block.rs => block_validation.rs} | 0 lighthouse/state/block/validation/mod.rs | 4 +- .../validation/{tests.rs => tests/helpers.rs} | 261 +---------------- .../state/block/validation/tests/mod.rs | 275 ++++++++++++++++++ 4 files changed, 286 insertions(+), 254 deletions(-) rename lighthouse/state/block/validation/{validate_ssz_block.rs => block_validation.rs} (100%) rename lighthouse/state/block/validation/{tests.rs => tests/helpers.rs} (52%) create mode 100644 lighthouse/state/block/validation/tests/mod.rs diff --git a/lighthouse/state/block/validation/validate_ssz_block.rs b/lighthouse/state/block/validation/block_validation.rs similarity index 100% rename from lighthouse/state/block/validation/validate_ssz_block.rs rename to lighthouse/state/block/validation/block_validation.rs diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index faf3ad096..d9c67fbcb 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,4 +1,4 @@ -mod validate_ssz_block; +mod block_validation; #[cfg(test)] mod tests; #[cfg(test)] @@ -18,7 +18,7 @@ use super::common::maps::{ AttesterMap, ProposerMap, }; -pub use self::validate_ssz_block::{ +pub use self::block_validation::{ BlockValidationContext, SszBlockValidationError, BlockStatus, diff --git a/lighthouse/state/block/validation/tests.rs b/lighthouse/state/block/validation/tests/helpers.rs similarity index 52% rename from lighthouse/state/block/validation/tests.rs rename to lighthouse/state/block/validation/tests/helpers.rs index e0fa153c2..3b2dda9e3 100644 --- a/lighthouse/state/block/validation/tests.rs +++ b/lighthouse/state/block/validation/tests/helpers.rs @@ -1,8 +1,3 @@ -extern crate ssz; - -use self::ssz::{ - SszStream, -}; use std::sync::Arc; use super::{ BlockValidationContext, @@ -10,27 +5,21 @@ use super::{ BlockStatus, AttesterMap, ProposerMap, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::db::{ MemoryDB, -}; -use super::utils::hash::canonical_hash; -use super::utils::types::{ + canonical_hash, Hash256, Bitfield, -}; -use super::SszBlock; -use super::super::Block; -use super::super::attestation_record::AttestationRecord; -use super::super::super::bls::{ Keypair, Signature, AggregateSignature, + BlockStore, + PoWChainStore, + ValidatorStore, + SszStream, + SszBlock, + Block, + AttestationRecord, + TestParams, }; pub struct TestStore { @@ -55,20 +44,6 @@ impl TestStore { } } -#[derive(Debug)] -pub struct TestParams { - pub total_validators: usize, - pub cycle_length: u8, - pub shard_count: u16, - pub shards_per_slot: u16, - pub validators_per_shard: usize, - pub block_slot: u64, - pub attestations_justified_slot: u64, - pub parent_proposer_index: usize, - pub validation_context_slot: u64, - pub validation_context_justified_slot: u64, - pub validation_context_finalized_slot: u64, -} type ParentHashes = Vec; /// Setup for a block validation function, without actually executing the @@ -263,221 +238,3 @@ pub fn run_block_validation_scenario( }; validation_status } - -fn get_simple_params() -> TestParams { - let validators_per_shard: usize = 5; - let cycle_length: u8 = 2; - let shard_count: u16 = 4; - let shards_per_slot: u16 = shard_count / u16::from(cycle_length); - let total_validators: usize = validators_per_shard * shard_count as usize; - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - } -} - -// TODO: test bad ssz serialization - -#[test] -fn test_block_validation_valid() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap().0, BlockStatus::NewBlock); -} - -#[test] -fn test_block_validation_valid_known_block() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores: TestStore| { - /* - * Pre-store the block in the database - */ - let block_ssz = serialize_block(&block); - let block_hash = canonical_hash(&block_ssz); - stores.block.put_serialized_block(&block_hash, &block_ssz).unwrap(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap(), (BlockStatus::KnownBlock, None)); -} - -#[test] -fn test_block_validation_invalid_future_slot() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.slot_number = block.slot_number + 1; - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::FutureSlot)); -} - -#[test] -fn test_block_validation_invalid_slot_already_finalized() { - let mut params = get_simple_params(); - - params.validation_context_finalized_slot = params.block_slot; - params.validation_context_justified_slot = params.validation_context_finalized_slot + - u64::from(params.cycle_length); - - let mutator = |block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::SlotAlreadyFinalized)); -} - -#[test] -fn test_block_validation_invalid_unknown_pow_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.pow_chain_ref = Hash256::from("unknown pow hash".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownPoWChainRef)); -} - -#[test] -fn test_block_validation_invalid_unknown_parent_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.parent_hash = Hash256::from("unknown parent block".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); -} - -#[test] -fn test_block_validation_invalid_1st_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[0].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::FirstAttestationSignatureFailed)); -} - -#[test] -fn test_block_validation_invalid_no_parent_proposer_signature() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, mut proposer_map: ProposerMap, stores: TestStore| { - /* - * Set the proposer for this slot to be a validator that does not exist. - */ - let ssz = stores.block.get_serialized_block(&block.parent_hash.as_ref()).unwrap().unwrap(); - let parent_block_slot = SszBlock::from_slice(&ssz[..]).unwrap().slot_number(); - proposer_map.insert(parent_block_slot, params.total_validators + 1); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::NoProposerSignature)); -} - -#[test] -fn test_block_validation_invalid_bad_proposer_map() { - let params = get_simple_params(); - - let mutator = |block, attester_map, _, stores| { - /* - * Initialize a new, empty proposer map - */ - let proposer_map = ProposerMap::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::BadProposerMap)); -} - -#[test] -fn test_block_validation_invalid_2nd_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[1].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::AttestationSignatureFailed)); -} diff --git a/lighthouse/state/block/validation/tests/mod.rs b/lighthouse/state/block/validation/tests/mod.rs new file mode 100644 index 000000000..02398dbe0 --- /dev/null +++ b/lighthouse/state/block/validation/tests/mod.rs @@ -0,0 +1,275 @@ +extern crate ssz; + +mod helpers; + +pub use self::helpers::{ + TestStore, + setup_block_validation_scenario, + run_block_validation_scenario, + serialize_block, +}; + +use self::ssz::{ + SszStream, +}; +use super::{ + BlockValidationContext, + SszBlockValidationError, + BlockStatus, + AttesterMap, + ProposerMap, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::db::{ + MemoryDB, +}; +use super::utils::hash::canonical_hash; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use super::SszBlock; +use super::super::Block; +use super::super::attestation_record::AttestationRecord; +use super::super::super::bls::{ + Keypair, + Signature, + AggregateSignature, +}; + +#[derive(Debug)] +pub struct TestParams { + pub total_validators: usize, + pub cycle_length: u8, + pub shard_count: u16, + pub shards_per_slot: u16, + pub validators_per_shard: usize, + pub block_slot: u64, + pub attestations_justified_slot: u64, + pub parent_proposer_index: usize, + pub validation_context_slot: u64, + pub validation_context_justified_slot: u64, + pub validation_context_finalized_slot: u64, +} + +fn get_simple_params() -> TestParams { + let validators_per_shard: usize = 5; + let cycle_length: u8 = 2; + let shard_count: u16 = 4; + let shards_per_slot: u16 = shard_count / u16::from(cycle_length); + let total_validators: usize = validators_per_shard * shard_count as usize; + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + + TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + parent_proposer_index, + block_slot, + attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, + } +} + +// TODO: test bad ssz serialization + +#[test] +fn test_block_validation_valid() { + let params = get_simple_params(); + + let mutator = |block: Block, attester_map, proposer_map, stores| { + /* + * Do not mutate + */ + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status.unwrap().0, BlockStatus::NewBlock); +} + +#[test] +fn test_block_validation_valid_known_block() { + let params = get_simple_params(); + + let mutator = |block: Block, attester_map, proposer_map, stores: TestStore| { + /* + * Pre-store the block in the database + */ + let block_ssz = serialize_block(&block); + let block_hash = canonical_hash(&block_ssz); + stores.block.put_serialized_block(&block_hash, &block_ssz).unwrap(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status.unwrap(), (BlockStatus::KnownBlock, None)); +} + +#[test] +fn test_block_validation_invalid_future_slot() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + block.slot_number = block.slot_number + 1; + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::FutureSlot)); +} + +#[test] +fn test_block_validation_invalid_slot_already_finalized() { + let mut params = get_simple_params(); + + params.validation_context_finalized_slot = params.block_slot; + params.validation_context_justified_slot = params.validation_context_finalized_slot + + u64::from(params.cycle_length); + + let mutator = |block, attester_map, proposer_map, stores| { + /* + * Do not mutate + */ + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::SlotAlreadyFinalized)); +} + +#[test] +fn test_block_validation_invalid_unknown_pow_hash() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + block.pow_chain_ref = Hash256::from("unknown pow hash".as_bytes()); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::UnknownPoWChainRef)); +} + +#[test] +fn test_block_validation_invalid_unknown_parent_hash() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + block.parent_hash = Hash256::from("unknown parent block".as_bytes()); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); +} + +#[test] +fn test_block_validation_invalid_1st_attestation_signature() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + /* + * Set the second attestaion record to have an invalid signature. + */ + block.attestations[0].aggregate_sig = AggregateSignature::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::FirstAttestationSignatureFailed)); +} + +#[test] +fn test_block_validation_invalid_no_parent_proposer_signature() { + let params = get_simple_params(); + + let mutator = |block: Block, attester_map, mut proposer_map: ProposerMap, stores: TestStore| { + /* + * Set the proposer for this slot to be a validator that does not exist. + */ + let ssz = stores.block.get_serialized_block(&block.parent_hash.as_ref()).unwrap().unwrap(); + let parent_block_slot = SszBlock::from_slice(&ssz[..]).unwrap().slot_number(); + proposer_map.insert(parent_block_slot, params.total_validators + 1); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::NoProposerSignature)); +} + +#[test] +fn test_block_validation_invalid_bad_proposer_map() { + let params = get_simple_params(); + + let mutator = |block, attester_map, _, stores| { + /* + * Initialize a new, empty proposer map + */ + let proposer_map = ProposerMap::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::BadProposerMap)); +} + +#[test] +fn test_block_validation_invalid_2nd_attestation_signature() { + let params = get_simple_params(); + + let mutator = |mut block: Block, attester_map, proposer_map, stores| { + /* + * Set the second attestaion record to have an invalid signature. + */ + block.attestations[1].aggregate_sig = AggregateSignature::new(); + (block, attester_map, proposer_map, stores) + }; + + let status = run_block_validation_scenario( + ¶ms, + mutator); + + assert_eq!(status, Err(SszBlockValidationError::AttestationSignatureFailed)); +} From 8f5285875b7a13009338d7134e221886c2f29800 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 1 Oct 2018 16:37:34 +0930 Subject: [PATCH 24/29] Refactor validation tests into integration tests --- Cargo.toml | 3 + benches/block_validation.rs | 148 +++++++++++++++ benches/main.rs | 5 + lighthouse/bls/mod.rs | 1 + lighthouse/lib.rs | 22 +++ .../validation/attestation_validation.rs | 10 - lighthouse/state/block/mod.rs | 2 +- lighthouse/state/block/validation/mod.rs | 6 +- tests/attestation_validation/helpers.rs | 177 ++++++++++++++++++ tests/attestation_validation/mod.rs | 8 + tests/attestation_validation/tests.rs | 31 +++ .../block_validation}/helpers.rs | 142 +++++++------- tests/block_validation/mod.rs | 10 + .../mod.rs => tests/block_validation/tests.rs | 57 ++---- tests/main.rs | 12 ++ 15 files changed, 496 insertions(+), 138 deletions(-) create mode 100644 benches/block_validation.rs create mode 100644 benches/main.rs create mode 100644 lighthouse/lib.rs create mode 100644 tests/attestation_validation/helpers.rs create mode 100644 tests/attestation_validation/mod.rs create mode 100644 tests/attestation_validation/tests.rs rename {lighthouse/state/block/validation/tests => tests/block_validation}/helpers.rs (71%) create mode 100644 tests/block_validation/mod.rs rename lighthouse/state/block/validation/tests/mod.rs => tests/block_validation/tests.rs (87%) create mode 100644 tests/main.rs diff --git a/Cargo.toml b/Cargo.toml index c076461c3..50c7360bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,3 +36,6 @@ ring = { git = "https://github.com/paritytech/ring" } [[bin]] path = "lighthouse/main.rs" name = "lighthouse" + +[lib] +path = "lighthouse/lib.rs" diff --git a/benches/block_validation.rs b/benches/block_validation.rs new file mode 100644 index 000000000..03e9df498 --- /dev/null +++ b/benches/block_validation.rs @@ -0,0 +1,148 @@ +#![feature(test)] + +extern crate test; + +use self::test::Bencher; + +use std::sync::Arc; + +use super::{ + BlockValidationContext, + AttesterMap, + ProposerMap, +}; + +use super::tests::{ + TestStore, + TestParams, + setup_block_validation_scenario, + serialize_block, +}; + +use super::super::{ + Block, + SszBlock, +}; + +fn bench_block_validation_scenario( + b: &mut Bencher, + params: &TestParams, + mutator_func: F) + where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) + -> (Block, AttesterMap, ProposerMap, TestStore) +{ + let (block, + parent_hashes, + attester_map, + proposer_map, + stores) = setup_block_validation_scenario(¶ms); + + let (block, + attester_map, + proposer_map, + stores) = mutator_func(block, attester_map, proposer_map, stores); + + let ssz_bytes = serialize_block(&block); + let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) + .unwrap(); + + let parent_hashes = Arc::new(parent_hashes); + let proposer_map = Arc::new(proposer_map); + let attester_map = Arc::new(attester_map); + b.iter(|| { + let context = BlockValidationContext { + present_slot: params.validation_context_slot, + cycle_length: params.cycle_length, + last_justified_slot: params.validation_context_justified_slot, + last_finalized_slot: params.validation_context_finalized_slot, + parent_hashes: parent_hashes.clone(), + proposer_map: proposer_map.clone(), + attester_map: attester_map.clone(), + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + pow_store: stores.pow_chain.clone() + }; + let result = context.validate_ssz_block(&ssz_block); + assert!(result.is_ok()); + }); +} + +#[bench] +#[ignore] +fn bench_block_validation_10m_eth(b: &mut Bencher) { + let total_validators: usize = 10_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + parent_proposer_index, + block_slot, + attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, + }; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + ¶ms, + no_mutate); +} + +#[bench] +#[ignore] +fn bench_block_validation_100m_eth(b: &mut Bencher) { + let total_validators: usize = 100_000_000 / 32; + let cycle_length: u8 = 64; + let shard_count: u16 = 1024; + let shards_per_slot: u16 = 1024 / u16::from(cycle_length); + let validators_per_shard: usize = total_validators / usize::from(shard_count); + let block_slot = u64::from(cycle_length) * 10000; + let attestations_justified_slot = block_slot - u64::from(cycle_length); + let parent_proposer_index = 0; + + let validation_context_slot = block_slot; + let validation_context_justified_slot = attestations_justified_slot; + let validation_context_finalized_slot = 0; + + let params = TestParams { + total_validators, + cycle_length, + shard_count, + shards_per_slot, + validators_per_shard, + parent_proposer_index, + block_slot, + attestations_justified_slot, + validation_context_slot, + validation_context_justified_slot, + validation_context_finalized_slot, + }; + + let no_mutate = |block, attester_map, proposer_map, stores| { + (block, attester_map, proposer_map, stores) + }; + + bench_block_validation_scenario( + b, + ¶ms, + no_mutate); +} diff --git a/benches/main.rs b/benches/main.rs new file mode 100644 index 000000000..0faf89f6d --- /dev/null +++ b/benches/main.rs @@ -0,0 +1,5 @@ +#![feature(test)] +extern crate lighthouse; +extern crate tests; + +mod block_validation; diff --git a/lighthouse/bls/mod.rs b/lighthouse/bls/mod.rs index fff21940c..ff9a0919a 100644 --- a/lighthouse/bls/mod.rs +++ b/lighthouse/bls/mod.rs @@ -5,5 +5,6 @@ pub use self::bls_aggregates::AggregatePublicKey; pub use self::bls_aggregates::Signature; pub use self::bls_aggregates::Keypair; pub use self::bls_aggregates::PublicKey; +pub use self::bls_aggregates::SecretKey; pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97; diff --git a/lighthouse/lib.rs b/lighthouse/lib.rs new file mode 100644 index 000000000..9abf4184e --- /dev/null +++ b/lighthouse/lib.rs @@ -0,0 +1,22 @@ +#[macro_use] +extern crate slog; +extern crate slog_term; +extern crate slog_async; +extern crate ssz; +extern crate clap; +extern crate network_libp2p; +extern crate futures; + +#[macro_use] +#[allow(dead_code)] +pub mod utils; +#[allow(dead_code)] +pub mod bls; +#[allow(dead_code)] +pub mod db; +pub mod client; +#[allow(dead_code)] +pub mod state; +#[allow(dead_code)] +mod sync; +mod config; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index f8ed3bdc0..077abf2ee 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -61,17 +61,7 @@ impl AttestationValidationContext where T: ClientDB { pub fn validate_attestation(&self, a: &AttestationRecord) - /* - block_slot: u64, - cycle_length: u8, - known_last_justified_slot: u64, - known_parent_hashes: &Arc>, - block_store: &Arc>, - validator_store: &Arc>, - attester_map: &Arc) - */ -> Result>, AttestationValidationError> - where T: ClientDB + Sized { /* * The attesation slot must not be higher than the block that contained it. diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs index 9a461568d..eb6365bdc 100644 --- a/lighthouse/state/block/mod.rs +++ b/lighthouse/state/block/mod.rs @@ -8,7 +8,7 @@ use super::utils; mod structs; mod ssz_block; -mod validation; +pub mod validation; pub use self::structs::Block; pub use self::ssz_block::{ diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs index d9c67fbcb..a18d3934f 100644 --- a/lighthouse/state/block/validation/mod.rs +++ b/lighthouse/state/block/validation/mod.rs @@ -1,8 +1,4 @@ mod block_validation; -#[cfg(test)] -mod tests; -#[cfg(test)] -mod benches; use super::attestation_record; use super::{ @@ -14,7 +10,7 @@ use super::db; use super::ssz; use super::utils; -use super::common::maps::{ +pub use super::common::maps::{ AttesterMap, ProposerMap, }; diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs new file mode 100644 index 000000000..c06c23948 --- /dev/null +++ b/tests/attestation_validation/helpers.rs @@ -0,0 +1,177 @@ +use std::sync::Arc; + +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + ValidatorStore, +}; +use super::state::attestation_record::{ + AttestationRecord, + AttestationValidationContext, + AttestationValidationError, +}; +use super::state::block::validation::AttesterMap; +use super::bls::{ + AggregateSignature, + Keypair, + SecretKey, + Signature, +}; +use super::ssz::SszStream; +use super::utils::types::{ + Hash256, + Bitfield, +}; +use super::utils::hash::{ + canonical_hash, +}; + + +pub struct TestStore { + pub db: Arc, + pub block: Arc>, + pub validator: Arc>, +} + +impl TestStore { + pub fn new() -> Self { + let db = Arc::new(MemoryDB::open()); + let block = Arc::new(BlockStore::new(db.clone())); + let validator = Arc::new(ValidatorStore::new(db.clone())); + Self { + db, + block, + validator, + } + } +} + +fn generate_message_hash(slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) + -> Vec +{ + let mut stream = SszStream::new(); + stream.append(&slot); + stream.append_vec(&parent_hashes.to_vec()); + stream.append(&shard_id); + stream.append(shard_block_hash); + stream.append(&justified_slot); + let bytes = stream.drain(); + canonical_hash(&bytes) +} + +pub fn generate_attestation(shard_id: u16, + shard_block_hash: &Hash256, + block_slot: u64, + attestation_slot: u64, + justified_slot: u64, + justified_block_hash: &Hash256, + cycle_length: u8, + parent_hashes: &[Hash256], + signing_keys: &[Option]) + -> AttestationRecord +{ + let mut attester_bitfield = Bitfield::new(); + let mut aggregate_sig = AggregateSignature::new(); + + let parent_hashes_slice = { + let distance: usize = (block_slot - attestation_slot) as usize; + let last: usize = parent_hashes.len() - distance; + let first: usize = last - usize::from(cycle_length); + &parent_hashes[first..last] + }; + + /* + * Generate the message that will be signed across for this attr record. + */ + let attestation_message = generate_message_hash( + attestation_slot, + parent_hashes_slice, + shard_id, + shard_block_hash, + justified_slot); + + for (i, secret_key) in signing_keys.iter().enumerate() { + /* + * If the signing key is Some, set the bitfield bit to true + * and sign the aggregate sig. + */ + if let Some(sk) = secret_key { + attester_bitfield.set_bit(i, true); + let sig = Signature::new(&attestation_message, sk); + aggregate_sig.add(&sig); + } + } + + AttestationRecord { + slot: attestation_slot, + shard_id, + oblique_parent_hashes: vec![], + shard_block_hash: shard_block_hash.clone(), + attester_bitfield, + justified_slot, + justified_block_hash: justified_block_hash.clone(), + aggregate_sig, + } +} + +/* +fn get_valid_attestation_and_context(shard_id: u16, + shard_block_hash: Hash256, + attester_count: usize, + signing_attesters: &[usize]) + -> (AttestationRecord, AttestationValidationContext) +{ + let stores = TestStore::new(); + + let block_slot = 10000; + let cycle_length: u8 = 64; + let last_justified_slot = block_slot - u64::from(cycle_length); + let parent_hashes: Vec = (0..(cycle_length * 2)) + .map(|i| Hash256::from(i as u64)) + .collect(); + let parent_hashes = Arc::new(parent_hashes); + let attester_map = Arc::new(AttesterMap::new()); + let justified_block_hash = Hash256::from("justified_block".as_bytes()); + + stores.block.put_serialized_block(&justified_block_hash.as_ref(), &[42]).unwrap(); + + let aggregate_sig = AggregateSignature::new(); + let attester_bitfield = Bitfield::new(); + + let mut attestation_indices = vec![]; + for attester_index in 0..attester_count { + let kp = Keypair::random(); + let validator_index = attester_count - attester_index; + attestation_indices.push(validator_index); + stores.validator.put_public_key_by_index(validator_index, &kp.pk); + } + + let context: AttestationValidationContext = AttestationValidationContext { + block_slot, + cycle_length, + last_justified_slot, + parent_hashes, + block_store: stores.block.clone(), + validator_store: stores.validator.clone(), + attester_map, + }; + + let attestation = AttestationRecord { + slot: block_slot - 1, + shard_id, + oblique_parent_hashes: vec![], + shard_block_hash, + attester_bitfield, + justified_slot: last_justified_slot, + justified_block_hash, + aggregate_sig, + }; + (attestation, context) +} +*/ diff --git a/tests/attestation_validation/mod.rs b/tests/attestation_validation/mod.rs new file mode 100644 index 000000000..0555fbe3a --- /dev/null +++ b/tests/attestation_validation/mod.rs @@ -0,0 +1,8 @@ +pub mod helpers; +mod tests; + +use super::bls; +use super::db; +use super::state; +use super::ssz; +use super::utils; diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs new file mode 100644 index 000000000..a57192d8d --- /dev/null +++ b/tests/attestation_validation/tests.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use super::helpers::{ + TestStore, +}; +use super::state::attestation_record::{ + AttestationRecord, + AttestationValidationContext, + AttestationValidationError, +}; +use super::state::block::validation::AttesterMap; +use super::bls::{ + AggregateSignature, + Keypair, +}; +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + ValidatorStore, +}; +use super::utils::types::{ + Hash256, + Bitfield, +}; + +#[test] +fn test_attestation_validation_valid() { + // TODO +} diff --git a/lighthouse/state/block/validation/tests/helpers.rs b/tests/block_validation/helpers.rs similarity index 71% rename from lighthouse/state/block/validation/tests/helpers.rs rename to tests/block_validation/helpers.rs index 3b2dda9e3..62fffc071 100644 --- a/lighthouse/state/block/validation/tests/helpers.rs +++ b/tests/block_validation/helpers.rs @@ -1,26 +1,52 @@ use std::sync::Arc; -use super::{ + +use super::generate_attestation; +use super::bls::{ + Keypair, +}; +use super::db::{ + MemoryDB, +}; +use super::db::stores::{ + BlockStore, + PoWChainStore, + ValidatorStore, +}; +use super::state::attestation_record::{ + AttestationRecord, +}; +use super::state::block::{ + SszBlock, + Block, +}; +use super::state::block::validation::{ BlockValidationContext, SszBlockValidationError, BlockStatus, AttesterMap, ProposerMap, - MemoryDB, - canonical_hash, - Hash256, - Bitfield, - Keypair, - Signature, - AggregateSignature, - BlockStore, - PoWChainStore, - ValidatorStore, - SszStream, - SszBlock, - Block, - AttestationRecord, - TestParams, }; +use super::ssz::{ + SszStream, +}; +use super::utils::types::{ + Hash256, +}; + +#[derive(Debug)] +pub struct BlockTestParams { + pub total_validators: usize, + pub cycle_length: u8, + pub shard_count: u16, + pub shards_per_slot: u16, + pub validators_per_shard: usize, + pub block_slot: u64, + pub attestations_justified_slot: u64, + pub parent_proposer_index: usize, + pub validation_context_slot: u64, + pub validation_context_justified_slot: u64, + pub validation_context_finalized_slot: u64, +} pub struct TestStore { pub db: Arc, @@ -48,7 +74,7 @@ type ParentHashes = Vec; /// Setup for a block validation function, without actually executing the /// block validation function. -pub fn setup_block_validation_scenario(params: &TestParams) +pub fn setup_block_validation_scenario(params: &BlockTestParams) -> (Block, ParentHashes, AttesterMap, ProposerMap, TestStore) { let stores = TestStore::new(); @@ -95,72 +121,36 @@ pub fn setup_block_validation_scenario(params: &TestParams) let mut attester_map = AttesterMap::new(); let mut attestations = vec![]; let mut keypairs = vec![]; + /* + * For each shard in this slot, generate an attestation. + */ for shard in 0..shards_per_slot { + let mut signing_keys = vec![]; let mut attesters = vec![]; - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - let attestation_message = { - let mut stream = SszStream::new(); - stream.append(&attestation_slot); - stream.append_vec(&parent_hashes_slice.to_vec()); - stream.append(&shard); - stream.append(&shard_block_hash); - stream.append(&attestations_justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) - }; - - - - for attestation_index in 0..validators_per_shard { - /* - * Add the attester to the attestation indices for this shard. - */ - attesters.push(i); - /* - * Set the voters bit on the bitfield to true. - */ - attester_bitfield.set_bit(attestation_index, true); - /* - * Generate a random keypair for this validatior and clone it into the - * list of keypairs. - */ + /* + * Generate a random keypair for each validator and clone it into the + * list of keypairs. Store it in the database. + */ + for _ in 0..validators_per_shard { let keypair = Keypair::random(); keypairs.push(keypair.clone()); - /* - * Store the validators public key in the database. - */ stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - /* - * Generate a new signature and aggregate it on the rolling signature. - */ - let sig = Signature::new(&attestation_message, &keypair.sk); - aggregate_sig.add(&sig); - /* - * Increment the validator counter to monotonically assign validators. - */ + signing_keys.push(Some(keypair.sk.clone())); + attesters.push(i); i += 1; } - attester_map.insert((attestation_slot, shard), attesters); - let attestation = AttestationRecord { - slot: attestation_slot, - shard_id: shard, - oblique_parent_hashes: vec![], - shard_block_hash, - attester_bitfield, - justified_slot: attestations_justified_slot, - justified_block_hash, - aggregate_sig, - }; + + let attestation = generate_attestation( + shard, + &shard_block_hash, + block_slot, + attestation_slot, + attestations_justified_slot, + &justified_block_hash, + cycle_length, + &parent_hashes, + &signing_keys[..]); attestations.push(attestation); } (attester_map, attestations, keypairs) @@ -194,7 +184,7 @@ pub fn serialize_block(b: &Block) -> Vec { /// /// Returns the Result returned from the block validation function. pub fn run_block_validation_scenario( - params: &TestParams, + params: &BlockTestParams, mutator_func: F) -> Result<(BlockStatus, Option), SszBlockValidationError> where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) diff --git a/tests/block_validation/mod.rs b/tests/block_validation/mod.rs new file mode 100644 index 000000000..3ff5741cb --- /dev/null +++ b/tests/block_validation/mod.rs @@ -0,0 +1,10 @@ +mod helpers; +mod tests; + +use super::bls; +use super::db; +use super::ssz; +use super::state; +use super::utils; + +use super::attestation_validation::helpers::generate_attestation; diff --git a/lighthouse/state/block/validation/tests/mod.rs b/tests/block_validation/tests.rs similarity index 87% rename from lighthouse/state/block/validation/tests/mod.rs rename to tests/block_validation/tests.rs index 02398dbe0..6adf026b0 100644 --- a/lighthouse/state/block/validation/tests/mod.rs +++ b/tests/block_validation/tests.rs @@ -1,62 +1,27 @@ -extern crate ssz; - -mod helpers; - -pub use self::helpers::{ +use super::bls::{ + AggregateSignature, +}; +use super::helpers::{ + BlockTestParams, TestStore, - setup_block_validation_scenario, run_block_validation_scenario, serialize_block, }; - -use self::ssz::{ - SszStream, +use super::state::block::{ + SszBlock, + Block, }; -use super::{ - BlockValidationContext, +use super::state::block::validation::{ SszBlockValidationError, BlockStatus, - AttesterMap, ProposerMap, }; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::db::{ - MemoryDB, -}; use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, - Bitfield, -}; -use super::SszBlock; -use super::super::Block; -use super::super::attestation_record::AttestationRecord; -use super::super::super::bls::{ - Keypair, - Signature, - AggregateSignature, }; -#[derive(Debug)] -pub struct TestParams { - pub total_validators: usize, - pub cycle_length: u8, - pub shard_count: u16, - pub shards_per_slot: u16, - pub validators_per_shard: usize, - pub block_slot: u64, - pub attestations_justified_slot: u64, - pub parent_proposer_index: usize, - pub validation_context_slot: u64, - pub validation_context_justified_slot: u64, - pub validation_context_finalized_slot: u64, -} - -fn get_simple_params() -> TestParams { +fn get_simple_params() -> BlockTestParams { let validators_per_shard: usize = 5; let cycle_length: u8 = 2; let shard_count: u16 = 4; @@ -70,7 +35,7 @@ fn get_simple_params() -> TestParams { let validation_context_justified_slot = attestations_justified_slot; let validation_context_finalized_slot = 0; - TestParams { + BlockTestParams { total_validators, cycle_length, shard_count, diff --git a/tests/main.rs b/tests/main.rs new file mode 100644 index 000000000..caa601105 --- /dev/null +++ b/tests/main.rs @@ -0,0 +1,12 @@ +extern crate lighthouse; +extern crate ssz; + +#[cfg(test)] +mod attestation_validation; +#[cfg(test)] +mod block_validation; + +use lighthouse::bls; +use lighthouse::db; +use lighthouse::state; +use lighthouse::utils; From 8e094b358f9cfcc71aa983b4477c251b06340991 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 1 Oct 2018 17:19:08 +0930 Subject: [PATCH 25/29] Implement framework for testing attestation val. --- lighthouse/state/block/validation/benches.rs | 146 ------------------ .../block/validation/block_validation.rs | 5 + tests/attestation_validation/helpers.rs | 62 ++++---- tests/attestation_validation/tests.rs | 7 +- tests/block_validation/helpers.rs | 2 +- 5 files changed, 45 insertions(+), 177 deletions(-) delete mode 100644 lighthouse/state/block/validation/benches.rs diff --git a/lighthouse/state/block/validation/benches.rs b/lighthouse/state/block/validation/benches.rs deleted file mode 100644 index 538ade41f..000000000 --- a/lighthouse/state/block/validation/benches.rs +++ /dev/null @@ -1,146 +0,0 @@ -extern crate test; - -use self::test::Bencher; - -use std::sync::Arc; - -use super::{ - BlockValidationContext, - AttesterMap, - ProposerMap, -}; - -use super::tests::{ - TestStore, - TestParams, - setup_block_validation_scenario, - serialize_block, -}; - -use super::super::{ - Block, - SszBlock, -}; - -fn bench_block_validation_scenario( - b: &mut Bencher, - params: &TestParams, - mutator_func: F) - where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) - -> (Block, AttesterMap, ProposerMap, TestStore) -{ - let (block, - parent_hashes, - attester_map, - proposer_map, - stores) = setup_block_validation_scenario(¶ms); - - let (block, - attester_map, - proposer_map, - stores) = mutator_func(block, attester_map, proposer_map, stores); - - let ssz_bytes = serialize_block(&block); - let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) - .unwrap(); - - let parent_hashes = Arc::new(parent_hashes); - let proposer_map = Arc::new(proposer_map); - let attester_map = Arc::new(attester_map); - b.iter(|| { - let context = BlockValidationContext { - present_slot: params.validation_context_slot, - cycle_length: params.cycle_length, - last_justified_slot: params.validation_context_justified_slot, - last_finalized_slot: params.validation_context_finalized_slot, - parent_hashes: parent_hashes.clone(), - proposer_map: proposer_map.clone(), - attester_map: attester_map.clone(), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - pow_store: stores.pow_chain.clone() - }; - let result = context.validate_ssz_block(&ssz_block); - assert!(result.is_ok()); - }); -} - -#[bench] -#[ignore] -fn bench_block_validation_10m_eth(b: &mut Bencher) { - let total_validators: usize = 10_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} - -#[bench] -#[ignore] -fn bench_block_validation_100m_eth(b: &mut Bencher) { - let total_validators: usize = 100_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} diff --git a/lighthouse/state/block/validation/block_validation.rs b/lighthouse/state/block/validation/block_validation.rs index 11ea54e4d..555a4228b 100644 --- a/lighthouse/state/block/validation/block_validation.rs +++ b/lighthouse/state/block/validation/block_validation.rs @@ -379,3 +379,8 @@ impl From for SszBlockValidationError { SszBlockValidationError::AttestationValidationError(e) } } + +/* + * Tests for block validation are contained in the root directory "tests" directory (AKA + * "integration tests directory"). + */ diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs index c06c23948..a93d8d531 100644 --- a/tests/attestation_validation/helpers.rs +++ b/tests/attestation_validation/helpers.rs @@ -120,12 +120,8 @@ pub fn generate_attestation(shard_id: u16, } } -/* -fn get_valid_attestation_and_context(shard_id: u16, - shard_block_hash: Hash256, - attester_count: usize, - signing_attesters: &[usize]) - -> (AttestationRecord, AttestationValidationContext) +pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize) + -> (AttestationRecord, AttestationValidationContext, TestStore) { let stores = TestStore::new(); @@ -136,42 +132,50 @@ fn get_valid_attestation_and_context(shard_id: u16, .map(|i| Hash256::from(i as u64)) .collect(); let parent_hashes = Arc::new(parent_hashes); - let attester_map = Arc::new(AttesterMap::new()); let justified_block_hash = Hash256::from("justified_block".as_bytes()); + let shard_block_hash = Hash256::from("shard_block".as_bytes()); stores.block.put_serialized_block(&justified_block_hash.as_ref(), &[42]).unwrap(); - let aggregate_sig = AggregateSignature::new(); - let attester_bitfield = Bitfield::new(); + let attestation_slot = block_slot - 1; - let mut attestation_indices = vec![]; - for attester_index in 0..attester_count { - let kp = Keypair::random(); - let validator_index = attester_count - attester_index; - attestation_indices.push(validator_index); - stores.validator.put_public_key_by_index(validator_index, &kp.pk); + let mut keypairs = vec![]; + let mut signing_keys = vec![]; + let mut attester_map = AttesterMap::new(); + let mut attesters = vec![]; + + /* + * Generate a random keypair for each validator and clone it into the + * list of keypairs. Store it in the database. + */ + for i in 0..attester_count { + let keypair = Keypair::random(); + keypairs.push(keypair.clone()); + stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); + signing_keys.push(Some(keypair.sk.clone())); + attesters.push(i); } + attester_map.insert((attestation_slot, shard_id), attesters); let context: AttestationValidationContext = AttestationValidationContext { block_slot, cycle_length, last_justified_slot, - parent_hashes, + parent_hashes: parent_hashes.clone(), block_store: stores.block.clone(), validator_store: stores.validator.clone(), - attester_map, + attester_map: Arc::new(attester_map), }; - - let attestation = AttestationRecord { - slot: block_slot - 1, + let attestation = generate_attestation( shard_id, - oblique_parent_hashes: vec![], - shard_block_hash, - attester_bitfield, - justified_slot: last_justified_slot, - justified_block_hash, - aggregate_sig, - }; - (attestation, context) + &shard_block_hash, + block_slot, + attestation_slot, + last_justified_slot, + &justified_block_hash, + cycle_length, + &parent_hashes.clone(), + &signing_keys); + + (attestation, context, stores) } -*/ diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs index a57192d8d..6ea3d722b 100644 --- a/tests/attestation_validation/tests.rs +++ b/tests/attestation_validation/tests.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use super::helpers::{ TestStore, + setup_attestation_validation_test, }; use super::state::attestation_record::{ AttestationRecord, @@ -27,5 +28,9 @@ use super::utils::types::{ #[test] fn test_attestation_validation_valid() { - // TODO + let (a, c, _stores) = setup_attestation_validation_test(10, 2); + + let result = c.validate_attestation(&a); + + assert!(result.unwrap().is_some()); } diff --git a/tests/block_validation/helpers.rs b/tests/block_validation/helpers.rs index 62fffc071..7e085320d 100644 --- a/tests/block_validation/helpers.rs +++ b/tests/block_validation/helpers.rs @@ -126,7 +126,7 @@ pub fn setup_block_validation_scenario(params: &BlockTestParams) */ for shard in 0..shards_per_slot { let mut signing_keys = vec![]; - let mut attesters = vec![]; + let mut attesters = vec![]; /* * Generate a random keypair for each validator and clone it into the * list of keypairs. Store it in the database. From 6b7677a20635518ce75617d7275a9454eeb18361 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Oct 2018 09:46:14 +1000 Subject: [PATCH 26/29] Update block and attestation validation code --- .../validation/attestation_validation.rs | 37 +++++++------------ .../block/validation/block_validation.rs | 16 -------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs index 077abf2ee..c96dfb479 100644 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ b/lighthouse/state/attestation_record/validation/attestation_validation.rs @@ -27,7 +27,7 @@ use super::signature_verification::{ pub enum AttestationValidationError { SlotTooHigh, SlotTooLow, - JustifiedSlotTooHigh, + JustifiedSlotIncorrect, UnknownJustifiedBlock, TooManyObliqueHashes, BadCurrentHashes, @@ -41,7 +41,7 @@ pub enum AttestationValidationError { InvalidBitfieldEndBits, NoSignatures, NonZeroTrailingBits, - AggregateSignatureFail, + BadAggregateSignature, DBError(String), } @@ -61,7 +61,7 @@ impl AttestationValidationContext where T: ClientDB { pub fn validate_attestation(&self, a: &AttestationRecord) - -> Result>, AttestationValidationError> + -> Result, AttestationValidationError> { /* * The attesation slot must not be higher than the block that contained it. @@ -73,8 +73,6 @@ impl AttestationValidationContext /* * The slot of this attestation must not be more than cycle_length + 1 distance * from the block that contained it. - * - * The below code stays overflow-safe as long as cycle length is a < 64 bit integer. */ if a.slot < self.block_slot .saturating_sub(u64::from(self.cycle_length).saturating_add(1)) { @@ -85,8 +83,8 @@ impl AttestationValidationContext * The attestation must indicate that its last justified slot is the same as the last justified * slot known to us. */ - if a.justified_slot > self.last_justified_slot { - return Err(AttestationValidationError::JustifiedSlotTooHigh); + if a.justified_slot != self.last_justified_slot { + return Err(AttestationValidationError::JustifiedSlotIncorrect); } /* @@ -114,7 +112,7 @@ impl AttestationValidationContext bytes_for_bits(attestation_indices.len()) { return Err(AttestationValidationError::BadBitfieldLength); - } + } /* * If there are excess bits in the bitfield because the number of a validators in not a @@ -123,10 +121,7 @@ impl AttestationValidationContext * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to * refer to the same AttesationRecord. */ - let last_byte = - a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1) - .ok_or(AttestationValidationError::InvalidBitfield)?; - if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) { + if a.attester_bitfield.len() > attestation_indices.len() { return Err(AttestationValidationError::InvalidBitfieldEndBits) } @@ -160,7 +155,13 @@ impl AttestationValidationContext &a.attester_bitfield, &self.validator_store)?; - Ok(voted_hashmap) + /* + * If the hashmap of voters is None, the signature verification failed. + */ + match voted_hashmap { + None => Err(AttestationValidationError::BadAggregateSignature), + Some(hashmap) => Ok(hashmap), + } } } @@ -168,16 +169,6 @@ fn bytes_for_bits(bits: usize) -> usize { (bits.saturating_sub(1) / 8) + 1 } -fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool { - for i in 0..n { - let mask = 1_u8 >> 7_usize.saturating_sub(i as usize); - if byte & mask > 0 { - return true - } - } - false -} - impl From for AttestationValidationError { fn from(e: ParentHashesError) -> Self { match e { diff --git a/lighthouse/state/block/validation/block_validation.rs b/lighthouse/state/block/validation/block_validation.rs index 555a4228b..2b6a8906f 100644 --- a/lighthouse/state/block/validation/block_validation.rs +++ b/lighthouse/state/block/validation/block_validation.rs @@ -55,7 +55,6 @@ pub enum SszBlockValidationError { BadAttestationSsz, AttestationValidationError(AttestationValidationError), AttestationSignatureFailed, - FirstAttestationSignatureFailed, ProposerAttestationHasObliqueHashes, NoProposerSignature, BadProposerMap, @@ -198,13 +197,6 @@ impl BlockValidationContext let attestation_voters = attestation_validation_context .validate_attestation(&first_attestation)?; - /* - * If the set of voters is None, the attestation was invalid. - */ - let attestation_voters = attestation_voters - .ok_or(SszBlockValidationError:: - FirstAttestationSignatureFailed)?; - /* * Read the parent hash from the block we are validating then attempt to load * the parent block ssz from the database. If that parent doesn't exist in @@ -285,14 +277,6 @@ impl BlockValidationContext *failure = Some(SszBlockValidationError::from(e)); None } - /* - * Attestation validation failed due to a bad signature. - */ - Ok(None) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::AttestationSignatureFailed); - None - } /* * Attestation validation succeded. */ From cd3b2f537179d9b9c1bb62ba7732d028d695f94b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Oct 2018 09:47:20 +1000 Subject: [PATCH 27/29] Add test for attestation msg generation --- .../validation/message_generation.rs | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs index 3bcc4c16a..6299802e9 100644 --- a/lighthouse/state/attestation_record/validation/message_generation.rs +++ b/lighthouse/state/attestation_record/validation/message_generation.rs @@ -5,11 +5,12 @@ use super::utils::types::Hash256; /// Generates the message used to validate the signature provided with an AttestationRecord. /// /// Ensures that the signer of the message has a view of the chain that is compatible with ours. -pub fn generate_signed_message(slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) +pub fn generate_signed_message( + slot: u64, + parent_hashes: &[Hash256], + shard_id: u16, + shard_block_hash: &Hash256, + justified_slot: u64) -> Vec { /* @@ -30,3 +31,40 @@ pub fn generate_signed_message(slot: u64, let bytes = ssz_stream.drain(); canonical_hash(&bytes) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_signed_message() { + let slot = 93; + let parent_hashes: Vec = (0..12) + .map(|i| Hash256::from(i as u64)) + .collect(); + let shard_id = 15; + let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); + let justified_slot = 18; + + let output = generate_signed_message( + slot, + &parent_hashes, + shard_id, + &shard_block_hash, + justified_slot); + + /* + * Note: this is not some well-known test vector, it's simply the result of running + * this and printing the output. + * + * Once well-known test vectors are established, they should be placed here. + */ + let expected = vec![ + 149, 99, 94, 229, 72, 144, 233, 14, 164, 16, 143, 53, 94, 48, + 118, 179, 33, 181, 172, 215, 2, 191, 176, 18, 188, 172, 137, + 178, 236, 66, 74, 120 + ]; + + assert_eq!(output, expected); + } +} From 07bfd7e97da0e268a975ac59588e8c513d878626 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Oct 2018 09:47:45 +1000 Subject: [PATCH 28/29] Add tests for attestation_validation --- lighthouse/main.rs | 2 - tests/attestation_validation/helpers.rs | 17 +++- tests/attestation_validation/tests.rs | 121 +++++++++++++++++++++--- tests/block_validation/tests.rs | 9 +- 4 files changed, 127 insertions(+), 22 deletions(-) diff --git a/lighthouse/main.rs b/lighthouse/main.rs index 33edddd55..c90bcdaea 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -1,5 +1,3 @@ -#![feature(test)] - #[macro_use] extern crate slog; extern crate slog_term; diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs index a93d8d531..3a42db9e1 100644 --- a/tests/attestation_validation/helpers.rs +++ b/tests/attestation_validation/helpers.rs @@ -10,7 +10,6 @@ use super::db::stores::{ use super::state::attestation_record::{ AttestationRecord, AttestationValidationContext, - AttestationValidationError, }; use super::state::block::validation::AttesterMap; use super::bls::{ @@ -48,6 +47,13 @@ impl TestStore { } } +pub struct TestRig { + pub attestation: AttestationRecord, + pub context: AttestationValidationContext, + pub stores: TestStore, + pub attester_count: usize, +} + fn generate_message_hash(slot: u64, parent_hashes: &[Hash256], shard_id: u16, @@ -121,7 +127,7 @@ pub fn generate_attestation(shard_id: u16, } pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize) - -> (AttestationRecord, AttestationValidationContext, TestStore) + -> TestRig { let stores = TestStore::new(); @@ -177,5 +183,10 @@ pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize) &parent_hashes.clone(), &signing_keys); - (attestation, context, stores) + TestRig { + attestation, + context, + stores, + attester_count, + } } diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs index 6ea3d722b..f486cab37 100644 --- a/tests/attestation_validation/tests.rs +++ b/tests/attestation_validation/tests.rs @@ -1,36 +1,127 @@ use std::sync::Arc; use super::helpers::{ - TestStore, + TestRig, setup_attestation_validation_test, }; use super::state::attestation_record::{ - AttestationRecord, - AttestationValidationContext, AttestationValidationError, }; use super::state::block::validation::AttesterMap; use super::bls::{ AggregateSignature, - Keypair, -}; -use super::db::{ - MemoryDB, -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, }; use super::utils::types::{ Hash256, - Bitfield, }; +fn generic_rig() -> TestRig { + let shard_id = 10; + let validator_count = 2; + setup_attestation_validation_test(shard_id, validator_count) +} + #[test] fn test_attestation_validation_valid() { - let (a, c, _stores) = setup_attestation_validation_test(10, 2); + let rig = generic_rig(); - let result = c.validate_attestation(&a); + let result = rig.context.validate_attestation(&rig.attestation); - assert!(result.unwrap().is_some()); + let voter_map = result.unwrap(); + assert_eq!(voter_map.len(), 2); +} + +#[test] +fn test_attestation_validation_invalid_slot_too_high() { + let mut rig = generic_rig(); + + rig.attestation.slot = rig.context.block_slot + 1; + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); +} + +#[test] +fn test_attestation_validation_invalid_slot_too_low() { + let mut rig = generic_rig(); + + rig.attestation.slot = rig.context.block_slot - u64::from(rig.context.cycle_length) - 2; + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); +} + +#[test] +fn test_attestation_validation_invalid_justified_slot_incorrect() { + let mut rig = generic_rig(); + + let original = rig.attestation.justified_slot; + rig.attestation.justified_slot = original - 1; + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); + + rig.attestation.justified_slot = original + 1; + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); +} + +#[test] +fn test_attestation_validation_invalid_too_many_oblique() { + let mut rig = generic_rig(); + + let obliques: Vec = (0..(rig.context.cycle_length + 1)) + .map(|i| Hash256::from((i * 2) as u64)) + .collect(); + + rig.attestation.oblique_parent_hashes = obliques; + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::TooManyObliqueHashes)); +} + +#[test] +fn test_attestation_validation_invalid_bad_attester_map() { + let mut rig = generic_rig(); + + rig.context.attester_map = Arc::new(AttesterMap::new()); + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::BadAttesterMap)); +} + +#[test] +fn test_attestation_validation_invalid_bad_bitfield_length() { + let mut rig = generic_rig(); + + /* + * Extend the bitfield by one byte + * + * This is a little hacky and makes assumptions about the internals + * of the bitfield. + */ + let one_byte_higher = rig.attester_count + 8; + rig.attestation.attester_bitfield.set_bit(one_byte_higher, true); + rig.attestation.attester_bitfield.set_bit(one_byte_higher, false); + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::BadBitfieldLength)); +} + +#[test] +fn test_attestation_validation_invalid_unknown_justfied_block_hash() { + let mut rig = generic_rig(); + + rig.attestation.justified_block_hash = Hash256::from("unknown block hash".as_bytes()); + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::UnknownJustifiedBlock)); +} + +#[test] +fn test_attestation_validation_invalid_empty_signature() { + let mut rig = generic_rig(); + + rig.attestation.aggregate_sig = AggregateSignature::new(); + + let result = rig.context.validate_attestation(&rig.attestation); + assert_eq!(result, Err(AttestationValidationError::BadAggregateSignature)); } diff --git a/tests/block_validation/tests.rs b/tests/block_validation/tests.rs index 6adf026b0..d3c18af84 100644 --- a/tests/block_validation/tests.rs +++ b/tests/block_validation/tests.rs @@ -16,6 +16,9 @@ use super::state::block::validation::{ BlockStatus, ProposerMap, }; +use super::state::attestation_record::{ + AttestationValidationError, +}; use super::utils::hash::canonical_hash; use super::utils::types::{ Hash256, @@ -177,7 +180,8 @@ fn test_block_validation_invalid_1st_attestation_signature() { ¶ms, mutator); - assert_eq!(status, Err(SszBlockValidationError::FirstAttestationSignatureFailed)); + assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( + AttestationValidationError::BadAggregateSignature))); } #[test] @@ -236,5 +240,6 @@ fn test_block_validation_invalid_2nd_attestation_signature() { ¶ms, mutator); - assert_eq!(status, Err(SszBlockValidationError::AttestationSignatureFailed)); + assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( + AttestationValidationError::BadAggregateSignature))); } From 0fbe4179b31938bf565a5cff1813dd2563a39f94 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Oct 2018 16:41:10 +1000 Subject: [PATCH 29/29] Heavily restructure repo Separate most modules into crates --- Cargo.toml | 20 +- beacon_chain/types/Cargo.toml | 11 + .../types/src}/active_state.rs | 2 +- .../types/src/attestation_record.rs | 2 +- .../types/src/block.rs | 2 +- .../types/src}/chain_config.rs | 0 .../src}/common/delegation/block_hash.rs | 0 .../types/src}/common/delegation/mod.rs | 0 .../types/src}/common/maps.rs | 0 .../types/src}/common/mod.rs | 0 .../types/src}/common/shuffling/README.md | 0 .../types/src}/common/shuffling/mod.rs | 0 .../types/src}/common/shuffling/rng.rs | 0 .../types/src}/crosslink_record.rs | 2 +- .../types/src}/crystallized_state.rs | 2 +- beacon_chain/types/src/lib.rs | 49 +++ .../state => beacon_chain/types/src}/mod.rs | 0 .../types/src}/shard_and_committee.rs | 1 - .../types/src}/validator_record.rs | 17 +- beacon_chain/utils/bls/Cargo.toml | 7 + .../utils/bls/src/lib.rs | 0 .../utils/boolean-bitfield}/Cargo.toml | 0 .../utils/boolean-bitfield}/README.md | 0 .../utils/boolean-bitfield}/src/lib.rs | 2 + beacon_chain/utils/hashing/Cargo.toml | 7 + .../utils/hashing/src/lib.rs | 4 +- {ssz => beacon_chain/utils/ssz}/Cargo.toml | 0 {ssz => beacon_chain/utils/ssz}/README.md | 0 {ssz => beacon_chain/utils/ssz}/src/decode.rs | 0 {ssz => beacon_chain/utils/ssz}/src/encode.rs | 0 .../utils/ssz}/src/impl_decode.rs | 2 +- .../utils/ssz}/src/impl_encode.rs | 0 {ssz => beacon_chain/utils/ssz}/src/lib.rs | 0 beacon_chain/utils/ssz_helpers/Cargo.toml | 10 + .../src/attestation_ssz_splitter.rs | 6 +- beacon_chain/utils/ssz_helpers/src/lib.rs | 7 + .../utils/ssz_helpers/src}/ssz_block.rs | 14 +- benches/block_validation.rs | 148 ------- benches/main.rs | 5 - lighthouse/db/Cargo.toml | 10 + lighthouse/db/{ => src}/disk_db.rs | 0 lighthouse/db/{mod.rs => src/lib.rs} | 4 +- lighthouse/db/{ => src}/memory_db.rs | 0 lighthouse/db/{ => src}/stores/block_store.rs | 0 lighthouse/db/{ => src}/stores/mod.rs | 0 .../db/{ => src}/stores/pow_chain_store.rs | 0 .../db/{ => src}/stores/validator_store.rs | 0 lighthouse/db/{ => src}/traits.rs | 0 lighthouse/lib.rs | 22 -- lighthouse/main.rs | 14 +- lighthouse/state/attestation_record/mod.rs | 24 -- .../validation/attestation_validation.rs | 208 ---------- .../validation/message_generation.rs | 70 ---- .../attestation_record/validation/mod.rs | 16 - .../validation/signature_verification.rs | 183 --------- lighthouse/state/block/mod.rs | 17 - .../block/validation/block_validation.rs | 370 ------------------ lighthouse/state/block/validation/mod.rs | 21 - .../state/common/attestation_parent_hashes.rs | 226 ----------- lighthouse/utils/errors.rs | 8 - lighthouse/utils/logging.rs | 22 -- lighthouse/utils/macros.rs | 8 - lighthouse/utils/mod.rs | 11 - lighthouse/utils/types.rs | 12 - tests/attestation_validation/helpers.rs | 192 --------- tests/attestation_validation/mod.rs | 8 - tests/attestation_validation/tests.rs | 127 ------ tests/block_validation/helpers.rs | 230 ----------- tests/block_validation/mod.rs | 10 - tests/block_validation/tests.rs | 245 ------------ tests/main.rs | 12 - 71 files changed, 150 insertions(+), 2240 deletions(-) create mode 100644 beacon_chain/types/Cargo.toml rename {lighthouse/state => beacon_chain/types/src}/active_state.rs (94%) rename lighthouse/state/attestation_record/structs.rs => beacon_chain/types/src/attestation_record.rs (98%) rename lighthouse/state/block/structs.rs => beacon_chain/types/src/block.rs (98%) rename {lighthouse/state => beacon_chain/types/src}/chain_config.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/delegation/block_hash.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/delegation/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/maps.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/README.md (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/rng.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/crosslink_record.rs (93%) rename {lighthouse/state => beacon_chain/types/src}/crystallized_state.rs (98%) create mode 100644 beacon_chain/types/src/lib.rs rename {lighthouse/state => beacon_chain/types/src}/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/shard_and_committee.rs (99%) rename {lighthouse/state => beacon_chain/types/src}/validator_record.rs (89%) create mode 100644 beacon_chain/utils/bls/Cargo.toml rename lighthouse/bls/mod.rs => beacon_chain/utils/bls/src/lib.rs (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/Cargo.toml (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/README.md (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/src/lib.rs (99%) create mode 100644 beacon_chain/utils/hashing/Cargo.toml rename lighthouse/utils/hash.rs => beacon_chain/utils/hashing/src/lib.rs (66%) rename {ssz => beacon_chain/utils/ssz}/Cargo.toml (100%) rename {ssz => beacon_chain/utils/ssz}/README.md (100%) rename {ssz => beacon_chain/utils/ssz}/src/decode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/encode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/impl_decode.rs (99%) rename {ssz => beacon_chain/utils/ssz}/src/impl_encode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/lib.rs (100%) create mode 100644 beacon_chain/utils/ssz_helpers/Cargo.toml rename lighthouse/state/attestation_record/ssz_splitter.rs => beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs (96%) create mode 100644 beacon_chain/utils/ssz_helpers/src/lib.rs rename {lighthouse/state/block => beacon_chain/utils/ssz_helpers/src}/ssz_block.rs (97%) delete mode 100644 benches/block_validation.rs delete mode 100644 benches/main.rs create mode 100644 lighthouse/db/Cargo.toml rename lighthouse/db/{ => src}/disk_db.rs (100%) rename lighthouse/db/{mod.rs => src/lib.rs} (93%) rename lighthouse/db/{ => src}/memory_db.rs (100%) rename lighthouse/db/{ => src}/stores/block_store.rs (100%) rename lighthouse/db/{ => src}/stores/mod.rs (100%) rename lighthouse/db/{ => src}/stores/pow_chain_store.rs (100%) rename lighthouse/db/{ => src}/stores/validator_store.rs (100%) rename lighthouse/db/{ => src}/traits.rs (100%) delete mode 100644 lighthouse/lib.rs delete mode 100644 lighthouse/state/attestation_record/mod.rs delete mode 100644 lighthouse/state/attestation_record/validation/attestation_validation.rs delete mode 100644 lighthouse/state/attestation_record/validation/message_generation.rs delete mode 100644 lighthouse/state/attestation_record/validation/mod.rs delete mode 100644 lighthouse/state/attestation_record/validation/signature_verification.rs delete mode 100644 lighthouse/state/block/mod.rs delete mode 100644 lighthouse/state/block/validation/block_validation.rs delete mode 100644 lighthouse/state/block/validation/mod.rs delete mode 100644 lighthouse/state/common/attestation_parent_hashes.rs delete mode 100644 lighthouse/utils/errors.rs delete mode 100644 lighthouse/utils/logging.rs delete mode 100644 lighthouse/utils/macros.rs delete mode 100644 lighthouse/utils/mod.rs delete mode 100644 lighthouse/utils/types.rs delete mode 100644 tests/attestation_validation/helpers.rs delete mode 100644 tests/attestation_validation/mod.rs delete mode 100644 tests/attestation_validation/tests.rs delete mode 100644 tests/block_validation/helpers.rs delete mode 100644 tests/block_validation/mod.rs delete mode 100644 tests/block_validation/tests.rs delete mode 100644 tests/main.rs diff --git a/Cargo.toml b/Cargo.toml index 50c7360bd..d9f9d14bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,26 +4,20 @@ version = "0.0.1" authors = ["Paul Hauner "] [dependencies] -# TODO: remove "blake2" in favor of "blake2-rfc" -blake2 = "^0.7.1" blake2-rfc = "0.2.18" bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } -boolean-bitfield = { path = "boolean-bitfield" } bytes = "" crypto-mac = "^0.6.2" clap = "2.32.0" +db = { path = "lighthouse/db" } dirs = "1.0.3" -ethereum-types = "0.4.0" futures = "0.1.23" network-libp2p = { path = "network-libp2p" } rand = "0.3" -rayon = "1.0.2" -rocksdb = "0.10.1" rlp = { git = "https://github.com/paritytech/parity-common" } slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" -ssz = { path = "ssz" } tokio = "0.1" [dependencies.pairing] @@ -37,5 +31,13 @@ ring = { git = "https://github.com/paritytech/ring" } path = "lighthouse/main.rs" name = "lighthouse" -[lib] -path = "lighthouse/lib.rs" +[workspace] +members = [ + "beacon_chain/types", + "beacon_chain/utils/bls", + "beacon_chain/utils/boolean-bitfield", + "beacon_chain/utils/hashing", + "beacon_chain/utils/ssz", + "beacon_chain/utils/ssz_helpers", + "lighthouse/db", +] diff --git a/beacon_chain/types/Cargo.toml b/beacon_chain/types/Cargo.toml new file mode 100644 index 000000000..cf20f69c0 --- /dev/null +++ b/beacon_chain/types/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "types" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../utils/bls" } +boolean-bitfield = { path = "../utils/boolean-bitfield" } +ethereum-types = "0.4.0" +rand = "0.3" +ssz = { path = "../utils/ssz" } diff --git a/lighthouse/state/active_state.rs b/beacon_chain/types/src/active_state.rs similarity index 94% rename from lighthouse/state/active_state.rs rename to beacon_chain/types/src/active_state.rs index d107bc299..6c71cadb8 100644 --- a/lighthouse/state/active_state.rs +++ b/beacon_chain/types/src/active_state.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; use super::attestation_record::AttestationRecord; pub struct ActiveState { diff --git a/lighthouse/state/attestation_record/structs.rs b/beacon_chain/types/src/attestation_record.rs similarity index 98% rename from lighthouse/state/attestation_record/structs.rs rename to beacon_chain/types/src/attestation_record.rs index 9cfa53516..77631372b 100644 --- a/lighthouse/state/attestation_record/structs.rs +++ b/beacon_chain/types/src/attestation_record.rs @@ -1,4 +1,4 @@ -use super::utils::types::{ Hash256, Bitfield }; +use super::{ Hash256, Bitfield }; use super::bls::{ AggregateSignature, BLS_AGG_SIG_BYTE_SIZE, diff --git a/lighthouse/state/block/structs.rs b/beacon_chain/types/src/block.rs similarity index 98% rename from lighthouse/state/block/structs.rs rename to beacon_chain/types/src/block.rs index eb8833a40..ee84773d0 100644 --- a/lighthouse/state/block/structs.rs +++ b/beacon_chain/types/src/block.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; use super::attestation_record::AttestationRecord; use super::ssz::{ Encodable, SszStream }; diff --git a/lighthouse/state/chain_config.rs b/beacon_chain/types/src/chain_config.rs similarity index 100% rename from lighthouse/state/chain_config.rs rename to beacon_chain/types/src/chain_config.rs diff --git a/lighthouse/state/common/delegation/block_hash.rs b/beacon_chain/types/src/common/delegation/block_hash.rs similarity index 100% rename from lighthouse/state/common/delegation/block_hash.rs rename to beacon_chain/types/src/common/delegation/block_hash.rs diff --git a/lighthouse/state/common/delegation/mod.rs b/beacon_chain/types/src/common/delegation/mod.rs similarity index 100% rename from lighthouse/state/common/delegation/mod.rs rename to beacon_chain/types/src/common/delegation/mod.rs diff --git a/lighthouse/state/common/maps.rs b/beacon_chain/types/src/common/maps.rs similarity index 100% rename from lighthouse/state/common/maps.rs rename to beacon_chain/types/src/common/maps.rs diff --git a/lighthouse/state/common/mod.rs b/beacon_chain/types/src/common/mod.rs similarity index 100% rename from lighthouse/state/common/mod.rs rename to beacon_chain/types/src/common/mod.rs diff --git a/lighthouse/state/common/shuffling/README.md b/beacon_chain/types/src/common/shuffling/README.md similarity index 100% rename from lighthouse/state/common/shuffling/README.md rename to beacon_chain/types/src/common/shuffling/README.md diff --git a/lighthouse/state/common/shuffling/mod.rs b/beacon_chain/types/src/common/shuffling/mod.rs similarity index 100% rename from lighthouse/state/common/shuffling/mod.rs rename to beacon_chain/types/src/common/shuffling/mod.rs diff --git a/lighthouse/state/common/shuffling/rng.rs b/beacon_chain/types/src/common/shuffling/rng.rs similarity index 100% rename from lighthouse/state/common/shuffling/rng.rs rename to beacon_chain/types/src/common/shuffling/rng.rs diff --git a/lighthouse/state/crosslink_record.rs b/beacon_chain/types/src/crosslink_record.rs similarity index 93% rename from lighthouse/state/crosslink_record.rs rename to beacon_chain/types/src/crosslink_record.rs index 37dfe7500..a28284681 100644 --- a/lighthouse/state/crosslink_record.rs +++ b/beacon_chain/types/src/crosslink_record.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; #[derive(Clone)] pub struct CrosslinkRecord { diff --git a/lighthouse/state/crystallized_state.rs b/beacon_chain/types/src/crystallized_state.rs similarity index 98% rename from lighthouse/state/crystallized_state.rs rename to beacon_chain/types/src/crystallized_state.rs index 8769bdcaa..e6d901495 100644 --- a/lighthouse/state/crystallized_state.rs +++ b/beacon_chain/types/src/crystallized_state.rs @@ -2,7 +2,7 @@ use super::validator_record::ValidatorRecord; use super::crosslink_record::CrosslinkRecord; use super::shard_and_committee::ShardAndCommittee; use super::ethereum_types::U256; -use super::utils::types::{ Hash256 }; +use super::Hash256; pub struct CrystallizedState { diff --git a/beacon_chain/types/src/lib.rs b/beacon_chain/types/src/lib.rs new file mode 100644 index 000000000..683560ab0 --- /dev/null +++ b/beacon_chain/types/src/lib.rs @@ -0,0 +1,49 @@ +extern crate ethereum_types; +extern crate bls; +extern crate boolean_bitfield; +extern crate ssz; + +pub mod active_state; +pub mod attestation_record; +pub mod crystallized_state; +pub mod chain_config; +pub mod block; +pub mod crosslink_record; +pub mod shard_and_committee; +pub mod validator_record; + +use self::ethereum_types::{ + H256, + H160, + U256 +}; +use self::boolean_bitfield::BooleanBitfield; +use std::collections::HashMap; + +pub use active_state::ActiveState; +pub use attestation_record::AttestationRecord; +pub use crystallized_state::CrystallizedState; +pub use chain_config::ChainConfig; +pub use block::Block; +pub use crosslink_record::CrosslinkRecord; +pub use shard_and_committee::ShardAndCommittee; +pub use validator_record::ValidatorRecord; + +pub type Hash256 = H256; +pub type Address = H160; +pub type EthBalance = U256; +pub type Bitfield = BooleanBitfield; + +/// Maps a (slot, shard_id) to attestation_indices. +pub type AttesterMap = HashMap<(u64, u16), Vec>; + +/// Maps a slot to a block proposer. +pub type ProposerMap = HashMap; + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/lighthouse/state/mod.rs b/beacon_chain/types/src/mod.rs similarity index 100% rename from lighthouse/state/mod.rs rename to beacon_chain/types/src/mod.rs diff --git a/lighthouse/state/shard_and_committee.rs b/beacon_chain/types/src/shard_and_committee.rs similarity index 99% rename from lighthouse/state/shard_and_committee.rs rename to beacon_chain/types/src/shard_and_committee.rs index 7f57fa6c8..3b2bcd0fd 100644 --- a/lighthouse/state/shard_and_committee.rs +++ b/beacon_chain/types/src/shard_and_committee.rs @@ -1,4 +1,3 @@ - #[derive(Clone,Debug)] pub struct ShardAndCommittee { pub shard_id: u16, diff --git a/lighthouse/state/validator_record.rs b/beacon_chain/types/src/validator_record.rs similarity index 89% rename from lighthouse/state/validator_record.rs rename to beacon_chain/types/src/validator_record.rs index 323c9e88d..4949e13f4 100644 --- a/lighthouse/state/validator_record.rs +++ b/beacon_chain/types/src/validator_record.rs @@ -1,14 +1,19 @@ -extern crate rand; - -use super::utils::types::{ Hash256, Address, U256 }; -use super::bls::{ PublicKey, Keypair }; +use super::{ + Hash256, + Address, + EthBalance, +}; +use super::bls::{ + PublicKey, + Keypair +}; pub struct ValidatorRecord { pub pubkey: PublicKey, pub withdrawal_shard: u16, pub withdrawal_address: Address, pub randao_commitment: Hash256, - pub balance: U256, + pub balance: EthBalance, pub start_dynasty: u64, pub end_dynasty: u64, } @@ -25,7 +30,7 @@ impl ValidatorRecord { withdrawal_shard: 0, withdrawal_address: Address::zero(), randao_commitment: Hash256::zero(), - balance: U256::zero(), + balance: EthBalance::zero(), start_dynasty: 0, end_dynasty: 0, }; diff --git a/beacon_chain/utils/bls/Cargo.toml b/beacon_chain/utils/bls/Cargo.toml new file mode 100644 index 000000000..eef0ec0aa --- /dev/null +++ b/beacon_chain/utils/bls/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bls" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } diff --git a/lighthouse/bls/mod.rs b/beacon_chain/utils/bls/src/lib.rs similarity index 100% rename from lighthouse/bls/mod.rs rename to beacon_chain/utils/bls/src/lib.rs diff --git a/boolean-bitfield/Cargo.toml b/beacon_chain/utils/boolean-bitfield/Cargo.toml similarity index 100% rename from boolean-bitfield/Cargo.toml rename to beacon_chain/utils/boolean-bitfield/Cargo.toml diff --git a/boolean-bitfield/README.md b/beacon_chain/utils/boolean-bitfield/README.md similarity index 100% rename from boolean-bitfield/README.md rename to beacon_chain/utils/boolean-bitfield/README.md diff --git a/boolean-bitfield/src/lib.rs b/beacon_chain/utils/boolean-bitfield/src/lib.rs similarity index 99% rename from boolean-bitfield/src/lib.rs rename to beacon_chain/utils/boolean-bitfield/src/lib.rs index 60d877e65..c9a4382f4 100644 --- a/boolean-bitfield/src/lib.rs +++ b/beacon_chain/utils/boolean-bitfield/src/lib.rs @@ -218,6 +218,7 @@ mod tests { assert_eq!(stream.drain(), vec![0, 0, 0, 2, 0, 1]); } + /* #[test] fn test_ssz_decoding() { /* @@ -246,6 +247,7 @@ mod tests { let res = BooleanBitfield::ssz_decode(&input, 0); assert_eq!(res, Err(ssz::DecodeError::TooShort)); } + */ #[test] fn test_new_bitfield_len() { diff --git a/beacon_chain/utils/hashing/Cargo.toml b/beacon_chain/utils/hashing/Cargo.toml new file mode 100644 index 000000000..36cbc41ef --- /dev/null +++ b/beacon_chain/utils/hashing/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "hashing" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +blake2-rfc = "0.2.18" diff --git a/lighthouse/utils/hash.rs b/beacon_chain/utils/hashing/src/lib.rs similarity index 66% rename from lighthouse/utils/hash.rs rename to beacon_chain/utils/hashing/src/lib.rs index 7b52eb9d1..8e2bd37a9 100644 --- a/lighthouse/utils/hash.rs +++ b/beacon_chain/utils/hashing/src/lib.rs @@ -1,4 +1,6 @@ -use super::blake2::blake2b::blake2b; +extern crate blake2_rfc; + +use self::blake2_rfc::blake2b::blake2b; pub fn canonical_hash(input: &[u8]) -> Vec { let result = blake2b(64, &[], input); diff --git a/ssz/Cargo.toml b/beacon_chain/utils/ssz/Cargo.toml similarity index 100% rename from ssz/Cargo.toml rename to beacon_chain/utils/ssz/Cargo.toml diff --git a/ssz/README.md b/beacon_chain/utils/ssz/README.md similarity index 100% rename from ssz/README.md rename to beacon_chain/utils/ssz/README.md diff --git a/ssz/src/decode.rs b/beacon_chain/utils/ssz/src/decode.rs similarity index 100% rename from ssz/src/decode.rs rename to beacon_chain/utils/ssz/src/decode.rs diff --git a/ssz/src/encode.rs b/beacon_chain/utils/ssz/src/encode.rs similarity index 100% rename from ssz/src/encode.rs rename to beacon_chain/utils/ssz/src/encode.rs diff --git a/ssz/src/impl_decode.rs b/beacon_chain/utils/ssz/src/impl_decode.rs similarity index 99% rename from ssz/src/impl_decode.rs rename to beacon_chain/utils/ssz/src/impl_decode.rs index 12e3e04c9..2833037f3 100644 --- a/ssz/src/impl_decode.rs +++ b/beacon_chain/utils/ssz/src/impl_decode.rs @@ -214,7 +214,7 @@ mod tests { &vec![1], 2 ); - assert_eq!(err, Err(DecodeError::OutOfBounds)); + assert_eq!(err, Err(DecodeError::TooShort)); let err: Result<(u16,usize), DecodeError> = decode_ssz( &vec![0, 0, 0, 0], diff --git a/ssz/src/impl_encode.rs b/beacon_chain/utils/ssz/src/impl_encode.rs similarity index 100% rename from ssz/src/impl_encode.rs rename to beacon_chain/utils/ssz/src/impl_encode.rs diff --git a/ssz/src/lib.rs b/beacon_chain/utils/ssz/src/lib.rs similarity index 100% rename from ssz/src/lib.rs rename to beacon_chain/utils/ssz/src/lib.rs diff --git a/beacon_chain/utils/ssz_helpers/Cargo.toml b/beacon_chain/utils/ssz_helpers/Cargo.toml new file mode 100644 index 000000000..a8189ce60 --- /dev/null +++ b/beacon_chain/utils/ssz_helpers/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ssz_helpers" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../bls" } +hashing = { path = "../hashing" } +types = { path = "../../types" } +ssz = { path = "../ssz" } diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs similarity index 96% rename from lighthouse/state/attestation_record/ssz_splitter.rs rename to beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs index e9e1b5ff8..ba11e1815 100644 --- a/lighthouse/state/attestation_record/ssz_splitter.rs +++ b/beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs @@ -1,4 +1,4 @@ -use super::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH; +use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH; use super::ssz::LENGTH_BYTES; use super::ssz::decode::decode_length; @@ -54,8 +54,8 @@ pub fn split_one_attestation(full_ssz: &[u8], index: usize) #[cfg(test)] mod tests { use super::*; - use super::super::AttestationRecord; - use super::super::utils::types::{ + use super::super::types::{ + AttestationRecord, Hash256, Bitfield, }; diff --git a/beacon_chain/utils/ssz_helpers/src/lib.rs b/beacon_chain/utils/ssz_helpers/src/lib.rs new file mode 100644 index 000000000..6f313e8a0 --- /dev/null +++ b/beacon_chain/utils/ssz_helpers/src/lib.rs @@ -0,0 +1,7 @@ +extern crate bls; +extern crate hashing; +extern crate types; +extern crate ssz; + +pub mod attestation_ssz_splitter; +pub mod ssz_block; diff --git a/lighthouse/state/block/ssz_block.rs b/beacon_chain/utils/ssz_helpers/src/ssz_block.rs similarity index 97% rename from lighthouse/state/block/ssz_block.rs rename to beacon_chain/utils/ssz_helpers/src/ssz_block.rs index e92291f80..7dd2ddc6d 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/beacon_chain/utils/ssz_helpers/src/ssz_block.rs @@ -2,12 +2,12 @@ use super::ssz::decode::{ decode_length, Decodable, }; -use super::utils::hash::canonical_hash; -use super::structs::{ +use super::hashing::canonical_hash; +use super::types::block::{ MIN_SSZ_BLOCK_LENGTH, MAX_SSZ_BLOCK_LENGTH, }; -use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; +use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; #[derive(Debug, PartialEq)] pub enum SszBlockError { @@ -146,10 +146,12 @@ impl<'a> SszBlock<'a> { #[cfg(test)] mod tests { use super::*; - use super::super::structs::Block; - use super::super::attestation_record::AttestationRecord; + use super::super::types::{ + AttestationRecord, + Block, + }; use super::super::ssz::SszStream; - use super::super::utils::types::Hash256; + use super::super::types::Hash256; fn get_block_ssz(b: &Block) -> Vec { let mut ssz_stream = SszStream::new(); diff --git a/benches/block_validation.rs b/benches/block_validation.rs deleted file mode 100644 index 03e9df498..000000000 --- a/benches/block_validation.rs +++ /dev/null @@ -1,148 +0,0 @@ -#![feature(test)] - -extern crate test; - -use self::test::Bencher; - -use std::sync::Arc; - -use super::{ - BlockValidationContext, - AttesterMap, - ProposerMap, -}; - -use super::tests::{ - TestStore, - TestParams, - setup_block_validation_scenario, - serialize_block, -}; - -use super::super::{ - Block, - SszBlock, -}; - -fn bench_block_validation_scenario( - b: &mut Bencher, - params: &TestParams, - mutator_func: F) - where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) - -> (Block, AttesterMap, ProposerMap, TestStore) -{ - let (block, - parent_hashes, - attester_map, - proposer_map, - stores) = setup_block_validation_scenario(¶ms); - - let (block, - attester_map, - proposer_map, - stores) = mutator_func(block, attester_map, proposer_map, stores); - - let ssz_bytes = serialize_block(&block); - let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) - .unwrap(); - - let parent_hashes = Arc::new(parent_hashes); - let proposer_map = Arc::new(proposer_map); - let attester_map = Arc::new(attester_map); - b.iter(|| { - let context = BlockValidationContext { - present_slot: params.validation_context_slot, - cycle_length: params.cycle_length, - last_justified_slot: params.validation_context_justified_slot, - last_finalized_slot: params.validation_context_finalized_slot, - parent_hashes: parent_hashes.clone(), - proposer_map: proposer_map.clone(), - attester_map: attester_map.clone(), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - pow_store: stores.pow_chain.clone() - }; - let result = context.validate_ssz_block(&ssz_block); - assert!(result.is_ok()); - }); -} - -#[bench] -#[ignore] -fn bench_block_validation_10m_eth(b: &mut Bencher) { - let total_validators: usize = 10_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} - -#[bench] -#[ignore] -fn bench_block_validation_100m_eth(b: &mut Bencher) { - let total_validators: usize = 100_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} diff --git a/benches/main.rs b/benches/main.rs deleted file mode 100644 index 0faf89f6d..000000000 --- a/benches/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![feature(test)] -extern crate lighthouse; -extern crate tests; - -mod block_validation; diff --git a/lighthouse/db/Cargo.toml b/lighthouse/db/Cargo.toml new file mode 100644 index 000000000..898df0c9b --- /dev/null +++ b/lighthouse/db/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "db" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../../beacon_chain/utils/bls" } +bytes = "0.4.10" +rocksdb = "0.10.1" +blake2-rfc = "0.2.18" diff --git a/lighthouse/db/disk_db.rs b/lighthouse/db/src/disk_db.rs similarity index 100% rename from lighthouse/db/disk_db.rs rename to lighthouse/db/src/disk_db.rs diff --git a/lighthouse/db/mod.rs b/lighthouse/db/src/lib.rs similarity index 93% rename from lighthouse/db/mod.rs rename to lighthouse/db/src/lib.rs index 40cab486d..be3d986b9 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/src/lib.rs @@ -1,12 +1,12 @@ -extern crate rocksdb; extern crate blake2_rfc as blake2; +extern crate bls; +extern crate rocksdb; mod disk_db; mod memory_db; mod traits; pub mod stores; -use super::bls; use self::stores::COLUMNS; pub use self::disk_db::DiskDB; diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/src/memory_db.rs similarity index 100% rename from lighthouse/db/memory_db.rs rename to lighthouse/db/src/memory_db.rs diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/src/stores/block_store.rs similarity index 100% rename from lighthouse/db/stores/block_store.rs rename to lighthouse/db/src/stores/block_store.rs diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/src/stores/mod.rs similarity index 100% rename from lighthouse/db/stores/mod.rs rename to lighthouse/db/src/stores/mod.rs diff --git a/lighthouse/db/stores/pow_chain_store.rs b/lighthouse/db/src/stores/pow_chain_store.rs similarity index 100% rename from lighthouse/db/stores/pow_chain_store.rs rename to lighthouse/db/src/stores/pow_chain_store.rs diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/src/stores/validator_store.rs similarity index 100% rename from lighthouse/db/stores/validator_store.rs rename to lighthouse/db/src/stores/validator_store.rs diff --git a/lighthouse/db/traits.rs b/lighthouse/db/src/traits.rs similarity index 100% rename from lighthouse/db/traits.rs rename to lighthouse/db/src/traits.rs diff --git a/lighthouse/lib.rs b/lighthouse/lib.rs deleted file mode 100644 index 9abf4184e..000000000 --- a/lighthouse/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[macro_use] -extern crate slog; -extern crate slog_term; -extern crate slog_async; -extern crate ssz; -extern crate clap; -extern crate network_libp2p; -extern crate futures; - -#[macro_use] -#[allow(dead_code)] -pub mod utils; -#[allow(dead_code)] -pub mod bls; -#[allow(dead_code)] -pub mod db; -pub mod client; -#[allow(dead_code)] -pub mod state; -#[allow(dead_code)] -mod sync; -mod config; diff --git a/lighthouse/main.rs b/lighthouse/main.rs index c90bcdaea..ee90649f5 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -2,22 +2,14 @@ extern crate slog; extern crate slog_term; extern crate slog_async; -extern crate ssz; +// extern crate ssz; extern crate clap; extern crate network_libp2p; extern crate futures; -#[macro_use] -#[allow(dead_code)] -mod utils; -#[allow(dead_code)] -mod bls; -#[allow(dead_code)] -mod db; +extern crate db; + mod client; -#[allow(dead_code)] -mod state; -#[allow(dead_code)] mod sync; mod config; diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs deleted file mode 100644 index d1813b2a5..000000000 --- a/lighthouse/state/attestation_record/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -use super::bls; -use super::common; -use super::db; -use super::ssz; -use super::utils; - - -mod structs; -mod ssz_splitter; -mod validation; - -pub use self::structs::{ - AttestationRecord, - MIN_SSZ_ATTESTION_RECORD_LENGTH, -}; -pub use self::ssz_splitter::{ - split_all_attestations, - split_one_attestation, - AttestationSplitError, -}; -pub use self::validation::{ - AttestationValidationContext, - AttestationValidationError, -}; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs deleted file mode 100644 index c96dfb479..000000000 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::collections::HashSet; -use std::sync::Arc; -use super::structs::AttestationRecord; -use super::AttesterMap; -use super::attestation_parent_hashes::{ - attestation_parent_hashes, - ParentHashesError, -}; -use super::db::{ - ClientDB, - DBError -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, -}; -use super::utils::types::{ - Hash256, -}; -use super::message_generation::generate_signed_message; -use super::signature_verification::{ - verify_aggregate_signature_for_indices, - SignatureVerificationError, -}; - -#[derive(Debug,PartialEq)] -pub enum AttestationValidationError { - SlotTooHigh, - SlotTooLow, - JustifiedSlotIncorrect, - UnknownJustifiedBlock, - TooManyObliqueHashes, - BadCurrentHashes, - BadObliqueHashes, - BadAttesterMap, - IntWrapping, - PublicKeyCorrupt, - NoPublicKeyForValidator, - BadBitfieldLength, - InvalidBitfield, - InvalidBitfieldEndBits, - NoSignatures, - NonZeroTrailingBits, - BadAggregateSignature, - DBError(String), -} - -pub struct AttestationValidationContext - where T: ClientDB + Sized -{ - pub block_slot: u64, - pub cycle_length: u8, - pub last_justified_slot: u64, - pub parent_hashes: Arc>, - pub block_store: Arc>, - pub validator_store: Arc>, - pub attester_map: Arc, -} - -impl AttestationValidationContext - where T: ClientDB -{ - pub fn validate_attestation(&self, a: &AttestationRecord) - -> Result, AttestationValidationError> - { - /* - * The attesation slot must not be higher than the block that contained it. - */ - if a.slot > self.block_slot { - return Err(AttestationValidationError::SlotTooHigh); - } - - /* - * The slot of this attestation must not be more than cycle_length + 1 distance - * from the block that contained it. - */ - if a.slot < self.block_slot - .saturating_sub(u64::from(self.cycle_length).saturating_add(1)) { - return Err(AttestationValidationError::SlotTooLow); - } - - /* - * The attestation must indicate that its last justified slot is the same as the last justified - * slot known to us. - */ - if a.justified_slot != self.last_justified_slot { - return Err(AttestationValidationError::JustifiedSlotIncorrect); - } - - /* - * There is no need to include more oblique parents hashes than there are blocks - * in a cycle. - */ - if a.oblique_parent_hashes.len() > usize::from(self.cycle_length) { - return Err(AttestationValidationError::TooManyObliqueHashes); - } - - /* - * Retrieve the set of attestation indices for this slot and shard id. - * - * This is an array mapping the order that validators will appear in the bitfield to the - * canonincal index of a validator. - */ - let attestation_indices = self.attester_map.get(&(a.slot, a.shard_id)) - .ok_or(AttestationValidationError::BadAttesterMap)?; - - /* - * The bitfield must be no longer than the minimum required to represent each validator in the - * attestation indicies for this slot and shard id. - */ - if a.attester_bitfield.num_bytes() != - bytes_for_bits(attestation_indices.len()) - { - return Err(AttestationValidationError::BadBitfieldLength); - } - - /* - * If there are excess bits in the bitfield because the number of a validators in not a - * multiple of 8, reject this attestation record. - * - * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to - * refer to the same AttesationRecord. - */ - if a.attester_bitfield.len() > attestation_indices.len() { - return Err(AttestationValidationError::InvalidBitfieldEndBits) - } - - /* - * The specified justified block hash must be known to us - */ - if !self.block_store.block_exists(&a.justified_block_hash)? { - return Err(AttestationValidationError::UnknownJustifiedBlock) - } - - let signed_message = { - let parent_hashes = attestation_parent_hashes( - self.cycle_length, - self.block_slot, - a.slot, - &self.parent_hashes, - &a.oblique_parent_hashes)?; - generate_signed_message( - a.slot, - &parent_hashes, - a.shard_id, - &a.shard_block_hash, - a.justified_slot) - }; - - let voted_hashmap = - verify_aggregate_signature_for_indices( - &signed_message, - &a.aggregate_sig, - &attestation_indices, - &a.attester_bitfield, - &self.validator_store)?; - - /* - * If the hashmap of voters is None, the signature verification failed. - */ - match voted_hashmap { - None => Err(AttestationValidationError::BadAggregateSignature), - Some(hashmap) => Ok(hashmap), - } - } -} - -fn bytes_for_bits(bits: usize) -> usize { - (bits.saturating_sub(1) / 8) + 1 -} - -impl From for AttestationValidationError { - fn from(e: ParentHashesError) -> Self { - match e { - ParentHashesError::BadCurrentHashes - => AttestationValidationError::BadCurrentHashes, - ParentHashesError::BadObliqueHashes - => AttestationValidationError::BadObliqueHashes, - ParentHashesError::SlotTooLow - => AttestationValidationError::SlotTooLow, - ParentHashesError::SlotTooHigh - => AttestationValidationError::SlotTooHigh, - ParentHashesError::IntWrapping - => AttestationValidationError::IntWrapping - } - } -} - -impl From for AttestationValidationError { - fn from(e: DBError) -> Self { - AttestationValidationError::DBError(e.message) - } -} - -impl From for AttestationValidationError { - fn from(e: SignatureVerificationError) -> Self { - match e { - SignatureVerificationError::BadValidatorIndex - => AttestationValidationError::BadAttesterMap, - SignatureVerificationError::PublicKeyCorrupt - => AttestationValidationError::PublicKeyCorrupt, - SignatureVerificationError::NoPublicKeyForValidator - => AttestationValidationError::NoPublicKeyForValidator, - SignatureVerificationError::DBError(s) - => AttestationValidationError::DBError(s), - } - } -} diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs deleted file mode 100644 index 6299802e9..000000000 --- a/lighthouse/state/attestation_record/validation/message_generation.rs +++ /dev/null @@ -1,70 +0,0 @@ -use super::ssz::SszStream; -use super::utils::hash::canonical_hash; -use super::utils::types::Hash256; - -/// Generates the message used to validate the signature provided with an AttestationRecord. -/// -/// Ensures that the signer of the message has a view of the chain that is compatible with ours. -pub fn generate_signed_message( - slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - /* - * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ - * (for example, SSZ might change whilst this doesn't). - * - * I have suggested switching this to ssz here: - * https://github.com/ethereum/eth2.0-specs/issues/5 - * - * If this doesn't happen, it would be safer to not use SSZ at all. - */ - let mut ssz_stream = SszStream::new(); - ssz_stream.append(&slot); - ssz_stream.append_vec(&parent_hashes.to_vec()); - ssz_stream.append(&shard_id); - ssz_stream.append(shard_block_hash); - ssz_stream.append(&justified_slot); - let bytes = ssz_stream.drain(); - canonical_hash(&bytes) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_generate_signed_message() { - let slot = 93; - let parent_hashes: Vec = (0..12) - .map(|i| Hash256::from(i as u64)) - .collect(); - let shard_id = 15; - let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); - let justified_slot = 18; - - let output = generate_signed_message( - slot, - &parent_hashes, - shard_id, - &shard_block_hash, - justified_slot); - - /* - * Note: this is not some well-known test vector, it's simply the result of running - * this and printing the output. - * - * Once well-known test vectors are established, they should be placed here. - */ - let expected = vec![ - 149, 99, 94, 229, 72, 144, 233, 14, 164, 16, 143, 53, 94, 48, - 118, 179, 33, 181, 172, 215, 2, 191, 176, 18, 188, 172, 137, - 178, 236, 66, 74, 120 - ]; - - assert_eq!(output, expected); - } -} diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs deleted file mode 100644 index dd3c589c5..000000000 --- a/lighthouse/state/attestation_record/validation/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -use super::common::maps::AttesterMap; -use super::db; -use super::bls; -use super::structs; -use super::ssz; -use super::common::attestation_parent_hashes; -use super::utils; - -mod attestation_validation; -mod signature_verification; -mod message_generation; - -pub use self::attestation_validation::{ - AttestationValidationContext, - AttestationValidationError, -}; diff --git a/lighthouse/state/attestation_record/validation/signature_verification.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs deleted file mode 100644 index c33dc2806..000000000 --- a/lighthouse/state/attestation_record/validation/signature_verification.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::HashSet; -use super::bls::{ - AggregateSignature, - AggregatePublicKey, -}; -use super::db::ClientDB; -use super::db::stores::{ - ValidatorStore, - ValidatorStoreError, -}; -use super::utils::types::Bitfield; - -#[derive(Debug, PartialEq)] -pub enum SignatureVerificationError { - BadValidatorIndex, - PublicKeyCorrupt, - NoPublicKeyForValidator, - DBError(String), -} - -/// Verify an aggregate signature across the supplied message. -/// -/// The public keys used for verification are collected by mapping -/// each true bitfield bit to canonical ValidatorRecord index through -/// the attestation_indicies map. -/// -/// Each public key is loaded from the store on-demand. -pub fn verify_aggregate_signature_for_indices( - message: &[u8], - agg_sig: &AggregateSignature, - attestation_indices: &[usize], - bitfield: &Bitfield, - validator_store: &ValidatorStore) - -> Result>, SignatureVerificationError> - where T: ClientDB + Sized -{ - let mut voters = HashSet::new(); - let mut agg_pub_key = AggregatePublicKey::new(); - - for i in 0..attestation_indices.len() { - let voted = bitfield.get_bit(i); - if voted { - /* - * De-reference the attestation index into a canonical ValidatorRecord index. - */ - let validator = *attestation_indices.get(i) - .ok_or(SignatureVerificationError::BadValidatorIndex)?; - /* - * Load the validators public key from our store. - */ - let pub_key = validator_store - .get_public_key_by_index(validator)? - .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; - /* - * Add the validators public key to the aggregate public key. - */ - agg_pub_key.add(&pub_key); - /* - * Add to the validator to the set of voters for this attestation record. - */ - voters.insert(validator); - } - } - /* - * Verify the aggregate public key against the aggregate signature. - * - * This verification will only succeed if the exact set of public keys - * were added to the aggregate public key as those that signed the aggregate signature. - */ - if agg_sig.verify(&message, &agg_pub_key) { - Ok(Some(voters)) - } else { - Ok(None) - } -} - -impl From for SignatureVerificationError { - fn from(error: ValidatorStoreError) -> Self { - match error { - ValidatorStoreError::DBError(s) => - SignatureVerificationError::DBError(s), - ValidatorStoreError::DecodeError => - SignatureVerificationError::PublicKeyCorrupt, - } - } -} - - -#[cfg(test)] -mod tests { - use super::*; - use super::super::bls::{ - Keypair, - Signature, - }; - use super::super::db::MemoryDB; - use std::sync::Arc; - - /* - * Cases that still need testing: - * - * - No signatures. - * - Database failure. - * - Unknown validator index. - * - Extra validator on signature. - */ - - #[test] - fn test_signature_verification() { - let message = "cats".as_bytes(); - let signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - let non_signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - /* - * Signing keypairs first, then non-signing - */ - let mut all_keypairs = signing_keypairs.clone(); - all_keypairs.append(&mut non_signing_keypairs.clone()); - - let attestation_indices: Vec = (0..all_keypairs.len()) - .collect(); - let mut bitfield = Bitfield::new(); - for i in 0..signing_keypairs.len() { - bitfield.set_bit(i, true); - } - - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - for (i, keypair) in all_keypairs.iter().enumerate() { - store.put_public_key_by_index(i, &keypair.pk).unwrap(); - } - - let mut agg_sig = AggregateSignature::new(); - for keypair in &signing_keypairs { - let sig = Signature::new(&message, &keypair.sk); - agg_sig.add(&sig); - } - - /* - * Test using all valid parameters. - */ - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - let voters = voters.unwrap(); - (0..signing_keypairs.len()) - .for_each(|i| assert!(voters.contains(&i))); - (signing_keypairs.len()..non_signing_keypairs.len()) - .for_each(|i| assert!(!voters.contains(&i))); - - /* - * Add another validator to the bitfield, run validation will all other - * parameters the same and assert that it fails. - */ - bitfield.set_bit(signing_keypairs.len() + 1, true); - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - assert_eq!(voters, None); - } -} diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs deleted file mode 100644 index eb6365bdc..000000000 --- a/lighthouse/state/block/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -extern crate blake2_rfc; - -use super::attestation_record; -use super::common; -use super::db; -use super::ssz; -use super::utils; - -mod structs; -mod ssz_block; -pub mod validation; - -pub use self::structs::Block; -pub use self::ssz_block::{ - SszBlock, - SszBlockError, -}; diff --git a/lighthouse/state/block/validation/block_validation.rs b/lighthouse/state/block/validation/block_validation.rs deleted file mode 100644 index 2b6a8906f..000000000 --- a/lighthouse/state/block/validation/block_validation.rs +++ /dev/null @@ -1,370 +0,0 @@ -extern crate rayon; - -use self::rayon::prelude::*; - -use std::sync::{ - Arc, - RwLock, -}; -use super::attestation_record::{ - AttestationValidationContext, - AttestationValidationError, -}; -use super::attestation_record::{ - AttestationRecord, - split_one_attestation, - split_all_attestations, - AttestationSplitError, -}; -use super::{ - AttesterMap, - ProposerMap, -}; -use super::{ - SszBlock, - SszBlockError, - Block, -}; -use super::db::{ - ClientDB, - DBError, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::ssz::{ - Decodable, - DecodeError, -}; -use super::utils::types::Hash256; - -#[derive(Debug, PartialEq)] -pub enum BlockStatus { - NewBlock, - KnownBlock, -} - -#[derive(Debug, PartialEq)] -pub enum SszBlockValidationError { - FutureSlot, - SlotAlreadyFinalized, - UnknownPoWChainRef, - UnknownParentHash, - BadAttestationSsz, - AttestationValidationError(AttestationValidationError), - AttestationSignatureFailed, - ProposerAttestationHasObliqueHashes, - NoProposerSignature, - BadProposerMap, - RwLockPoisoned, - DBError(String), -} - -/// The context against which a block should be validated. -pub struct BlockValidationContext - where T: ClientDB + Sized -{ - /// The slot as determined by the system time. - pub present_slot: u64, - /// The cycle_length as determined by the chain configuration. - pub cycle_length: u8, - /// The last justified slot as per the client's view of the canonical chain. - pub last_justified_slot: u64, - /// The last finalized slot as per the client's view of the canonical chain. - pub last_finalized_slot: u64, - /// A vec of the hashes of the blocks preceeding the present slot. - pub parent_hashes: Arc>, - /// A map of slots to a block proposer validation index. - pub proposer_map: Arc, - /// A map of (slot, shard_id) to the attestation set of validation indices. - pub attester_map: Arc, - /// The store containing block information. - pub block_store: Arc>, - /// The store containing validator information. - pub validator_store: Arc>, - /// The store containing information about the proof-of-work chain. - pub pow_store: Arc>, -} - -impl BlockValidationContext - where T: ClientDB -{ - /// Validate some SszBlock against a block validation context. An SszBlock varies from a Block in - /// that is a read-only structure that reads directly from encoded SSZ. - /// - /// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is - /// a suspicion that the block might be invalid. Such a suspicion should be applied to - /// all blocks coming from the network. - /// - /// This function will determine if the block is new, already known or invalid (either - /// intrinsically or due to some application error.) - /// - /// Note: this function does not implement randao_reveal checking as it is not in the - /// specification. - #[allow(dead_code)] - pub fn validate_ssz_block(&self, b: &SszBlock) - -> Result<(BlockStatus, Option), SszBlockValidationError> - where T: ClientDB + Sized - { - - /* - * If this block is already known, return immediately and indicate the the block is - * known. Don't attempt to deserialize the block. - */ - let block_hash = &b.block_hash(); - if self.block_store.block_exists(&block_hash)? { - return Ok((BlockStatus::KnownBlock, None)); - } - - /* - * If the block slot corresponds to a slot in the future, drop it. - */ - let block_slot = b.slot_number(); - if block_slot > self.present_slot { - return Err(SszBlockValidationError::FutureSlot); - } - - /* - * If the block is unknown (assumed unknown because we checked the db earlier in this - * function) and it comes from a slot that is already finalized, drop the block. - * - * If a slot is finalized, there's no point in considering any other blocks for that slot. - */ - if block_slot <= self.last_finalized_slot { - return Err(SszBlockValidationError::SlotAlreadyFinalized); - } - - /* - * If the PoW chain hash is not known to us, drop it. - * - * We only accept blocks that reference a known PoW hash. - * - * Note: it is not clear what a "known" PoW chain ref is. Likely it means the block hash is - * "sufficienty deep in the canonical PoW chain". This should be clarified as the spec - * crystallizes. - */ - let pow_chain_ref = b.pow_chain_ref(); - if !self.pow_store.block_hash_exists(b.pow_chain_ref())? { - return Err(SszBlockValidationError::UnknownPoWChainRef); - } - - /* - * Store a slice of the serialized attestations from the block SSZ. - */ - let attestations_ssz = &b.attestations(); - - /* - * Get a slice of the first serialized attestation (the 0'th) and decode it into - * a full AttestationRecord object. - * - * The first attestation must be validated separately as it must contain a signature of the - * proposer of the previous block (this is checked later in this function). - */ - let (first_attestation_ssz, next_index) = split_one_attestation( - &attestations_ssz, - 0)?; - let (first_attestation, _) = AttestationRecord::ssz_decode( - &first_attestation_ssz, 0)?; - - /* - * The first attestation may not have oblique hashes. - * - * The presence of oblique hashes in the first attestation would indicate that the proposer - * of the previous block is attesting to some other block than the one they produced. - */ - if first_attestation.oblique_parent_hashes.len() > 0 { - return Err(SszBlockValidationError::ProposerAttestationHasObliqueHashes); - } - - /* - * Generate the context in which attestations will be validated. - */ - let attestation_validation_context = Arc::new(AttestationValidationContext { - block_slot, - cycle_length: self.cycle_length, - last_justified_slot: self.last_justified_slot, - parent_hashes: self.parent_hashes.clone(), - block_store: self.block_store.clone(), - validator_store: self.validator_store.clone(), - attester_map: self.attester_map.clone(), - }); - - /* - * Validate this first attestation. - */ - let attestation_voters = attestation_validation_context - .validate_attestation(&first_attestation)?; - - /* - * Read the parent hash from the block we are validating then attempt to load - * the parent block ssz from the database. If that parent doesn't exist in - * the database, reject the block. - * - * If the parent does exist in the database, read the slot of that parent. Then, - * determine the proposer of that slot (the parent slot) by looking it up - * in the proposer map. - * - * If that proposer (the proposer of the parent block) was not present in the first (0'th) - * attestation of this block, reject the block. - */ - let parent_hash = b.parent_hash(); - match self.block_store.get_serialized_block(&parent_hash)? { - None => return Err(SszBlockValidationError::UnknownParentHash), - Some(ssz) => { - let parent_block = SszBlock::from_slice(&ssz[..])?; - let proposer = self.proposer_map.get(&parent_block.slot_number()) - .ok_or(SszBlockValidationError::BadProposerMap)?; - if !attestation_voters.contains(&proposer) { - return Err(SszBlockValidationError::NoProposerSignature); - } - } - } - - /* - * Split the remaining attestations into a vector of slices, each containing - * a single serialized attestation record. - */ - let other_attestations = split_all_attestations(attestations_ssz, - next_index)?; - - /* - * Verify each other AttestationRecord. - * - * This uses the `rayon` library to do "sometimes" parallelization. Put simply, - * if there are some spare threads, the verification of attestation records will happen - * concurrently. - * - * There is a thread-safe `failure` variable which is set whenever an attestation fails - * validation. This is so all attestation validation is halted if a single bad attestation - * is found. - */ - let failure: RwLock> = RwLock::new(None); - let mut deserialized_attestations: Vec = other_attestations - .par_iter() - .filter_map(|attestation_ssz| { - /* - * If some thread has set the `failure` variable to `Some(error)` the abandon - * attestation serialization and validation. - */ - if let Some(_) = *failure.read().unwrap() { - return None; - } - /* - * If there has not been a failure yet, attempt to serialize and validate the - * attestation. - */ - match AttestationRecord::ssz_decode(&attestation_ssz, 0) { - /* - * Deserialization failed, therefore the block is invalid. - */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None - } - /* - * Deserialization succeeded and the attestation should be validated. - */ - Ok((attestation, _)) => { - match attestation_validation_context.validate_attestation(&attestation) { - /* - * Attestation validation failed with some error. - */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None - } - /* - * Attestation validation succeded. - */ - Ok(_) => Some(attestation) - } - } - } - }) - .collect(); - - match failure.into_inner() { - Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), - Ok(failure) => { - match failure { - Some(error) => return Err(error), - _ => () - } - - } - } - - /* - * Add the first attestation to the vec of deserialized attestations at - * index 0. - */ - deserialized_attestations.insert(0, first_attestation); - - /* - * If we have reached this point, the block is a new valid block that is worthy of - * processing. - */ - let block = Block { - parent_hash: Hash256::from(parent_hash), - slot_number: block_slot, - randao_reveal: Hash256::from(b.randao_reveal()), - attestations: deserialized_attestations, - pow_chain_ref: Hash256::from(pow_chain_ref), - active_state_root: Hash256::from(b.act_state_root()), - crystallized_state_root: Hash256::from(b.cry_state_root()), - }; - Ok((BlockStatus::NewBlock, Some(block))) - } -} - -impl From for SszBlockValidationError { - fn from(e: DBError) -> Self { - SszBlockValidationError::DBError(e.message) - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationSplitError) -> Self { - match e { - AttestationSplitError::TooShort => - SszBlockValidationError::BadAttestationSsz - } - } -} - -impl From for SszBlockValidationError { - fn from(e: SszBlockError) -> Self { - match e { - SszBlockError::TooShort => - SszBlockValidationError::DBError("Bad parent block in db.".to_string()), - SszBlockError::TooLong => - SszBlockValidationError::DBError("Bad parent block in db.".to_string()), - } - } -} - -impl From for SszBlockValidationError { - fn from(e: DecodeError) -> Self { - match e { - DecodeError::TooShort => - SszBlockValidationError::BadAttestationSsz, - DecodeError::TooLong => - SszBlockValidationError::BadAttestationSsz, - } - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationValidationError) -> Self { - SszBlockValidationError::AttestationValidationError(e) - } -} - -/* - * Tests for block validation are contained in the root directory "tests" directory (AKA - * "integration tests directory"). - */ diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs deleted file mode 100644 index a18d3934f..000000000 --- a/lighthouse/state/block/validation/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -mod block_validation; - -use super::attestation_record; -use super::{ - SszBlock, - SszBlockError, - Block, -}; -use super::db; -use super::ssz; -use super::utils; - -pub use super::common::maps::{ - AttesterMap, - ProposerMap, -}; -pub use self::block_validation::{ - BlockValidationContext, - SszBlockValidationError, - BlockStatus, -}; diff --git a/lighthouse/state/common/attestation_parent_hashes.rs b/lighthouse/state/common/attestation_parent_hashes.rs deleted file mode 100644 index ff54233bc..000000000 --- a/lighthouse/state/common/attestation_parent_hashes.rs +++ /dev/null @@ -1,226 +0,0 @@ -use super::Hash256; - -#[derive(Debug)] -pub enum ParentHashesError { - BadCurrentHashes, - BadObliqueHashes, - SlotTooHigh, - SlotTooLow, - IntWrapping, -} - -/// This function is used to select the hashes used in -/// the signing of an AttestationRecord. -/// -/// It either returns Result with a vector of length `cycle_length,` or -/// returns an Error. -/// -/// This function corresponds to the `get_signed_parent_hashes` function -/// in the Python reference implentation. -/// -/// See this slide for more information: -/// https://tinyurl.com/ybzn2spw -pub fn attestation_parent_hashes( - cycle_length: u8, - block_slot: u64, - attestation_slot: u64, - current_hashes: &[Hash256], - oblique_hashes: &[Hash256]) - -> Result, ParentHashesError> -{ - // This cast places a limit on cycle_length. If you change it, check math - // for overflow. - let cycle_length: u64 = u64::from(cycle_length); - - if current_hashes.len() as u64 != (cycle_length * 2) { - return Err(ParentHashesError::BadCurrentHashes); - } - if oblique_hashes.len() as u64 > cycle_length { - return Err(ParentHashesError::BadObliqueHashes); - } - if attestation_slot >= block_slot { - return Err(ParentHashesError::SlotTooHigh); - } - - /* - * Cannot underflow as block_slot cannot be less - * than attestation_slot. - */ - let attestation_distance = block_slot - attestation_slot; - - if attestation_distance > cycle_length { - return Err(ParentHashesError::SlotTooLow); - } - - /* - * Cannot underflow because attestation_distance cannot - * be larger than cycle_length. - */ - let start = cycle_length - attestation_distance; - - /* - * Overflow is potentially impossible, but proof is complicated - * enough to just use checked math. - * - * Arithmetic is: - * start + cycle_length - oblique_hashes.len() - */ - let end = start.checked_add(cycle_length) - .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) - .ok_or(ParentHashesError::IntWrapping)?; - - - let mut hashes = Vec::new(); - hashes.extend_from_slice( - ¤t_hashes[(start as usize)..(end as usize)]); - hashes.extend_from_slice(oblique_hashes); - - Ok(hashes) -} - - -#[cfg(test)] -mod tests { - use super::*; - - fn get_range_of_hashes(from: usize, to: usize) -> Vec { - (from..to).map(|i| get_hash(&vec![i as u8])).collect() - } - - fn get_hash(value: &[u8]) -> Hash256 { - Hash256::from_slice(value) - } - - #[test] - fn test_get_signed_hashes_oblique_scenario_1() { - /* - * Two oblique hashes. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = get_range_of_hashes(100, 102); - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - - let mut expected_result = get_range_of_hashes(7, 13); - expected_result.append(&mut get_range_of_hashes(100, 102)); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_oblique_scenario_2() { - /* - * All oblique hashes. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = get_range_of_hashes(100, 108); - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - - let expected_result = get_range_of_hashes(100, 108); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_1() { - /* - * Google Slides example. - * https://tinyurl.com/ybzn2spw - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - let expected_result = get_range_of_hashes(7, 15); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_2() { - /* - * Block 1, attestation 0. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 1; - let attestation_slot: u64 = 0; - let current_hashes = get_range_of_hashes(0, 16); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - let expected_result = get_range_of_hashes(7, 15); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_3() { - /* - * attestation_slot too large - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 100; - let attestation_slot: u64 = 100; - let current_hashes = get_range_of_hashes(0, 16); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_err()); - } - - #[test] - fn test_get_signed_hashes_scenario_4() { - /* - * Current hashes too small - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 100; - let attestation_slot: u64 = 99; - let current_hashes = get_range_of_hashes(0, 15); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_err()); - } -} diff --git a/lighthouse/utils/errors.rs b/lighthouse/utils/errors.rs deleted file mode 100644 index b464f6a48..000000000 --- a/lighthouse/utils/errors.rs +++ /dev/null @@ -1,8 +0,0 @@ -// Collection of custom errors - -#[derive(Debug,PartialEq)] -pub enum ParameterError { - IntWrapping, - OutOfBounds, - InvalidInput(String), -} diff --git a/lighthouse/utils/logging.rs b/lighthouse/utils/logging.rs deleted file mode 100644 index 7dc186d30..000000000 --- a/lighthouse/utils/logging.rs +++ /dev/null @@ -1,22 +0,0 @@ -extern crate slog; -extern crate slog_term; -extern crate slog_async; - -use slog::*; -pub use slog::Logger; - -pub fn test_logger() -> slog::Logger { - let plain = slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter); - Logger::root( - slog_term::FullFormat::new(plain) - .build().fuse(), o!() - ) -} - -pub fn get_logger() -> slog::Logger { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::CompactFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - - slog::Logger::root(drain, o!()) -} diff --git a/lighthouse/utils/macros.rs b/lighthouse/utils/macros.rs deleted file mode 100644 index e9a8c416d..000000000 --- a/lighthouse/utils/macros.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[macro_export] -macro_rules! assert_error { - ($exp: expr, $err: expr) => { - if !$exp { - return Err($err); - } - } -} diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs deleted file mode 100644 index 16e986e22..000000000 --- a/lighthouse/utils/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -extern crate ethereum_types; -extern crate blake2_rfc as blake2; -extern crate crypto_mac; -extern crate boolean_bitfield; - -#[macro_use] -pub mod macros; -pub mod hash; -pub mod types; -pub mod logging; -pub mod errors; diff --git a/lighthouse/utils/types.rs b/lighthouse/utils/types.rs deleted file mode 100644 index 949b7a549..000000000 --- a/lighthouse/utils/types.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate boolean_bitfield; - -use super::ethereum_types::{ H256, H160 }; -use self::boolean_bitfield::BooleanBitfield; - -pub use super::ethereum_types::U256; - -pub type Hash256 = H256; - -pub type Address = H160; - -pub type Bitfield = BooleanBitfield; diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs deleted file mode 100644 index 3a42db9e1..000000000 --- a/tests/attestation_validation/helpers.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::sync::Arc; - -use super::db::{ - MemoryDB, -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, -}; -use super::state::attestation_record::{ - AttestationRecord, - AttestationValidationContext, -}; -use super::state::block::validation::AttesterMap; -use super::bls::{ - AggregateSignature, - Keypair, - SecretKey, - Signature, -}; -use super::ssz::SszStream; -use super::utils::types::{ - Hash256, - Bitfield, -}; -use super::utils::hash::{ - canonical_hash, -}; - - -pub struct TestStore { - pub db: Arc, - pub block: Arc>, - pub validator: Arc>, -} - -impl TestStore { - pub fn new() -> Self { - let db = Arc::new(MemoryDB::open()); - let block = Arc::new(BlockStore::new(db.clone())); - let validator = Arc::new(ValidatorStore::new(db.clone())); - Self { - db, - block, - validator, - } - } -} - -pub struct TestRig { - pub attestation: AttestationRecord, - pub context: AttestationValidationContext, - pub stores: TestStore, - pub attester_count: usize, -} - -fn generate_message_hash(slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - let mut stream = SszStream::new(); - stream.append(&slot); - stream.append_vec(&parent_hashes.to_vec()); - stream.append(&shard_id); - stream.append(shard_block_hash); - stream.append(&justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) -} - -pub fn generate_attestation(shard_id: u16, - shard_block_hash: &Hash256, - block_slot: u64, - attestation_slot: u64, - justified_slot: u64, - justified_block_hash: &Hash256, - cycle_length: u8, - parent_hashes: &[Hash256], - signing_keys: &[Option]) - -> AttestationRecord -{ - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - /* - * Generate the message that will be signed across for this attr record. - */ - let attestation_message = generate_message_hash( - attestation_slot, - parent_hashes_slice, - shard_id, - shard_block_hash, - justified_slot); - - for (i, secret_key) in signing_keys.iter().enumerate() { - /* - * If the signing key is Some, set the bitfield bit to true - * and sign the aggregate sig. - */ - if let Some(sk) = secret_key { - attester_bitfield.set_bit(i, true); - let sig = Signature::new(&attestation_message, sk); - aggregate_sig.add(&sig); - } - } - - AttestationRecord { - slot: attestation_slot, - shard_id, - oblique_parent_hashes: vec![], - shard_block_hash: shard_block_hash.clone(), - attester_bitfield, - justified_slot, - justified_block_hash: justified_block_hash.clone(), - aggregate_sig, - } -} - -pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize) - -> TestRig -{ - let stores = TestStore::new(); - - let block_slot = 10000; - let cycle_length: u8 = 64; - let last_justified_slot = block_slot - u64::from(cycle_length); - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let parent_hashes = Arc::new(parent_hashes); - let justified_block_hash = Hash256::from("justified_block".as_bytes()); - let shard_block_hash = Hash256::from("shard_block".as_bytes()); - - stores.block.put_serialized_block(&justified_block_hash.as_ref(), &[42]).unwrap(); - - let attestation_slot = block_slot - 1; - - let mut keypairs = vec![]; - let mut signing_keys = vec![]; - let mut attester_map = AttesterMap::new(); - let mut attesters = vec![]; - - /* - * Generate a random keypair for each validator and clone it into the - * list of keypairs. Store it in the database. - */ - for i in 0..attester_count { - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - signing_keys.push(Some(keypair.sk.clone())); - attesters.push(i); - } - attester_map.insert((attestation_slot, shard_id), attesters); - - let context: AttestationValidationContext = AttestationValidationContext { - block_slot, - cycle_length, - last_justified_slot, - parent_hashes: parent_hashes.clone(), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - attester_map: Arc::new(attester_map), - }; - let attestation = generate_attestation( - shard_id, - &shard_block_hash, - block_slot, - attestation_slot, - last_justified_slot, - &justified_block_hash, - cycle_length, - &parent_hashes.clone(), - &signing_keys); - - TestRig { - attestation, - context, - stores, - attester_count, - } -} diff --git a/tests/attestation_validation/mod.rs b/tests/attestation_validation/mod.rs deleted file mode 100644 index 0555fbe3a..000000000 --- a/tests/attestation_validation/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod helpers; -mod tests; - -use super::bls; -use super::db; -use super::state; -use super::ssz; -use super::utils; diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs deleted file mode 100644 index f486cab37..000000000 --- a/tests/attestation_validation/tests.rs +++ /dev/null @@ -1,127 +0,0 @@ -use std::sync::Arc; - -use super::helpers::{ - TestRig, - setup_attestation_validation_test, -}; -use super::state::attestation_record::{ - AttestationValidationError, -}; -use super::state::block::validation::AttesterMap; -use super::bls::{ - AggregateSignature, -}; -use super::utils::types::{ - Hash256, -}; - -fn generic_rig() -> TestRig { - let shard_id = 10; - let validator_count = 2; - setup_attestation_validation_test(shard_id, validator_count) -} - -#[test] -fn test_attestation_validation_valid() { - let rig = generic_rig(); - - let result = rig.context.validate_attestation(&rig.attestation); - - let voter_map = result.unwrap(); - assert_eq!(voter_map.len(), 2); -} - -#[test] -fn test_attestation_validation_invalid_slot_too_high() { - let mut rig = generic_rig(); - - rig.attestation.slot = rig.context.block_slot + 1; - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); -} - -#[test] -fn test_attestation_validation_invalid_slot_too_low() { - let mut rig = generic_rig(); - - rig.attestation.slot = rig.context.block_slot - u64::from(rig.context.cycle_length) - 2; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); -} - -#[test] -fn test_attestation_validation_invalid_justified_slot_incorrect() { - let mut rig = generic_rig(); - - let original = rig.attestation.justified_slot; - rig.attestation.justified_slot = original - 1; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); - - rig.attestation.justified_slot = original + 1; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); -} - -#[test] -fn test_attestation_validation_invalid_too_many_oblique() { - let mut rig = generic_rig(); - - let obliques: Vec = (0..(rig.context.cycle_length + 1)) - .map(|i| Hash256::from((i * 2) as u64)) - .collect(); - - rig.attestation.oblique_parent_hashes = obliques; - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::TooManyObliqueHashes)); -} - -#[test] -fn test_attestation_validation_invalid_bad_attester_map() { - let mut rig = generic_rig(); - - rig.context.attester_map = Arc::new(AttesterMap::new()); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadAttesterMap)); -} - -#[test] -fn test_attestation_validation_invalid_bad_bitfield_length() { - let mut rig = generic_rig(); - - /* - * Extend the bitfield by one byte - * - * This is a little hacky and makes assumptions about the internals - * of the bitfield. - */ - let one_byte_higher = rig.attester_count + 8; - rig.attestation.attester_bitfield.set_bit(one_byte_higher, true); - rig.attestation.attester_bitfield.set_bit(one_byte_higher, false); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadBitfieldLength)); -} - -#[test] -fn test_attestation_validation_invalid_unknown_justfied_block_hash() { - let mut rig = generic_rig(); - - rig.attestation.justified_block_hash = Hash256::from("unknown block hash".as_bytes()); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::UnknownJustifiedBlock)); -} - -#[test] -fn test_attestation_validation_invalid_empty_signature() { - let mut rig = generic_rig(); - - rig.attestation.aggregate_sig = AggregateSignature::new(); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadAggregateSignature)); -} diff --git a/tests/block_validation/helpers.rs b/tests/block_validation/helpers.rs deleted file mode 100644 index 7e085320d..000000000 --- a/tests/block_validation/helpers.rs +++ /dev/null @@ -1,230 +0,0 @@ -use std::sync::Arc; - -use super::generate_attestation; -use super::bls::{ - Keypair, -}; -use super::db::{ - MemoryDB, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::state::attestation_record::{ - AttestationRecord, -}; -use super::state::block::{ - SszBlock, - Block, -}; -use super::state::block::validation::{ - BlockValidationContext, - SszBlockValidationError, - BlockStatus, - AttesterMap, - ProposerMap, -}; -use super::ssz::{ - SszStream, -}; -use super::utils::types::{ - Hash256, -}; - -#[derive(Debug)] -pub struct BlockTestParams { - pub total_validators: usize, - pub cycle_length: u8, - pub shard_count: u16, - pub shards_per_slot: u16, - pub validators_per_shard: usize, - pub block_slot: u64, - pub attestations_justified_slot: u64, - pub parent_proposer_index: usize, - pub validation_context_slot: u64, - pub validation_context_justified_slot: u64, - pub validation_context_finalized_slot: u64, -} - -pub struct TestStore { - pub db: Arc, - pub block: Arc>, - pub pow_chain: Arc>, - pub validator: Arc>, -} - -impl TestStore { - pub fn new() -> Self { - let db = Arc::new(MemoryDB::open()); - let block = Arc::new(BlockStore::new(db.clone())); - let pow_chain = Arc::new(PoWChainStore::new(db.clone())); - let validator = Arc::new(ValidatorStore::new(db.clone())); - Self { - db, - block, - pow_chain, - validator, - } - } -} - -type ParentHashes = Vec; - -/// Setup for a block validation function, without actually executing the -/// block validation function. -pub fn setup_block_validation_scenario(params: &BlockTestParams) - -> (Block, ParentHashes, AttesterMap, ProposerMap, TestStore) -{ - let stores = TestStore::new(); - - let cycle_length = params.cycle_length; - let shards_per_slot = params.shards_per_slot; - let validators_per_shard = params.validators_per_shard; - let block_slot = params.block_slot; - let attestations_justified_slot = params.attestations_justified_slot; - - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let parent_hash = Hash256::from("parent_hash".as_bytes()); - let randao_reveal = Hash256::from("randao_reveal".as_bytes()); - let justified_block_hash = Hash256::from("justified_hash".as_bytes()); - let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); - let active_state_root = Hash256::from("active_state".as_bytes()); - let crystallized_state_root = Hash256::from("cry_state".as_bytes()); - let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); - - stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); - stores.block.put_serialized_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - - /* - * Generate a minimum viable parent block and store it in the database. - */ - let mut parent_block = Block::zero(); - let parent_attestation = AttestationRecord::zero(); - parent_block.slot_number = block_slot - 1; - parent_block.attestations.push(parent_attestation); - let parent_block_ssz = serialize_block(&parent_block); - stores.block.put_serialized_block(parent_hash.as_ref(), &parent_block_ssz).unwrap(); - - let proposer_map = { - let mut proposer_map = ProposerMap::new(); - proposer_map.insert(parent_block.slot_number, params.parent_proposer_index); - proposer_map - }; - - let (attester_map, attestations, _keypairs) = { - let mut i = 0; - let attestation_slot = block_slot - 1; - let mut attester_map = AttesterMap::new(); - let mut attestations = vec![]; - let mut keypairs = vec![]; - /* - * For each shard in this slot, generate an attestation. - */ - for shard in 0..shards_per_slot { - let mut signing_keys = vec![]; - let mut attesters = vec![]; - /* - * Generate a random keypair for each validator and clone it into the - * list of keypairs. Store it in the database. - */ - for _ in 0..validators_per_shard { - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - signing_keys.push(Some(keypair.sk.clone())); - attesters.push(i); - i += 1; - } - attester_map.insert((attestation_slot, shard), attesters); - - let attestation = generate_attestation( - shard, - &shard_block_hash, - block_slot, - attestation_slot, - attestations_justified_slot, - &justified_block_hash, - cycle_length, - &parent_hashes, - &signing_keys[..]); - attestations.push(attestation); - } - (attester_map, attestations, keypairs) - }; - - let block = Block { - parent_hash, - slot_number: block_slot, - randao_reveal, - attestations, - pow_chain_ref, - active_state_root, - crystallized_state_root, - }; - - (block, - parent_hashes, - attester_map, - proposer_map, - stores) -} - -/// Helper function to take some Block and SSZ serialize it. -pub fn serialize_block(b: &Block) -> Vec { - let mut stream = SszStream::new(); - stream.append(b); - stream.drain() -} - -/// Setup and run a block validation scenario, given some parameters. -/// -/// Returns the Result returned from the block validation function. -pub fn run_block_validation_scenario( - params: &BlockTestParams, - mutator_func: F) - -> Result<(BlockStatus, Option), SszBlockValidationError> - where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) - -> (Block, AttesterMap, ProposerMap, TestStore) -{ - let (block, - parent_hashes, - attester_map, - proposer_map, - stores) = setup_block_validation_scenario(¶ms); - - let (block, - attester_map, - proposer_map, - stores) = mutator_func(block, attester_map, proposer_map, stores); - - let ssz_bytes = serialize_block(&block); - let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) - .unwrap(); - - let context = BlockValidationContext { - present_slot: params.validation_context_slot, - cycle_length: params.cycle_length, - last_justified_slot: params.validation_context_justified_slot, - last_finalized_slot: params.validation_context_finalized_slot, - parent_hashes: Arc::new(parent_hashes), - proposer_map: Arc::new(proposer_map), - attester_map: Arc::new(attester_map), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - pow_store: stores.pow_chain.clone() - }; - let validation_status = context.validate_ssz_block(&ssz_block); - /* - * If validation returned a block, make sure it's the same block we supplied to it. - * - * I.e., there were no errors during the serialization -> deserialization process. - */ - if let Ok((_, Some(returned_block))) = &validation_status { - assert_eq!(*returned_block, block); - }; - validation_status -} diff --git a/tests/block_validation/mod.rs b/tests/block_validation/mod.rs deleted file mode 100644 index 3ff5741cb..000000000 --- a/tests/block_validation/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -mod helpers; -mod tests; - -use super::bls; -use super::db; -use super::ssz; -use super::state; -use super::utils; - -use super::attestation_validation::helpers::generate_attestation; diff --git a/tests/block_validation/tests.rs b/tests/block_validation/tests.rs deleted file mode 100644 index d3c18af84..000000000 --- a/tests/block_validation/tests.rs +++ /dev/null @@ -1,245 +0,0 @@ -use super::bls::{ - AggregateSignature, -}; -use super::helpers::{ - BlockTestParams, - TestStore, - run_block_validation_scenario, - serialize_block, -}; -use super::state::block::{ - SszBlock, - Block, -}; -use super::state::block::validation::{ - SszBlockValidationError, - BlockStatus, - ProposerMap, -}; -use super::state::attestation_record::{ - AttestationValidationError, -}; -use super::utils::hash::canonical_hash; -use super::utils::types::{ - Hash256, -}; - -fn get_simple_params() -> BlockTestParams { - let validators_per_shard: usize = 5; - let cycle_length: u8 = 2; - let shard_count: u16 = 4; - let shards_per_slot: u16 = shard_count / u16::from(cycle_length); - let total_validators: usize = validators_per_shard * shard_count as usize; - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - BlockTestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - } -} - -// TODO: test bad ssz serialization - -#[test] -fn test_block_validation_valid() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap().0, BlockStatus::NewBlock); -} - -#[test] -fn test_block_validation_valid_known_block() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores: TestStore| { - /* - * Pre-store the block in the database - */ - let block_ssz = serialize_block(&block); - let block_hash = canonical_hash(&block_ssz); - stores.block.put_serialized_block(&block_hash, &block_ssz).unwrap(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap(), (BlockStatus::KnownBlock, None)); -} - -#[test] -fn test_block_validation_invalid_future_slot() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.slot_number = block.slot_number + 1; - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::FutureSlot)); -} - -#[test] -fn test_block_validation_invalid_slot_already_finalized() { - let mut params = get_simple_params(); - - params.validation_context_finalized_slot = params.block_slot; - params.validation_context_justified_slot = params.validation_context_finalized_slot + - u64::from(params.cycle_length); - - let mutator = |block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::SlotAlreadyFinalized)); -} - -#[test] -fn test_block_validation_invalid_unknown_pow_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.pow_chain_ref = Hash256::from("unknown pow hash".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownPoWChainRef)); -} - -#[test] -fn test_block_validation_invalid_unknown_parent_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.parent_hash = Hash256::from("unknown parent block".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); -} - -#[test] -fn test_block_validation_invalid_1st_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[0].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( - AttestationValidationError::BadAggregateSignature))); -} - -#[test] -fn test_block_validation_invalid_no_parent_proposer_signature() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, mut proposer_map: ProposerMap, stores: TestStore| { - /* - * Set the proposer for this slot to be a validator that does not exist. - */ - let ssz = stores.block.get_serialized_block(&block.parent_hash.as_ref()).unwrap().unwrap(); - let parent_block_slot = SszBlock::from_slice(&ssz[..]).unwrap().slot_number(); - proposer_map.insert(parent_block_slot, params.total_validators + 1); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::NoProposerSignature)); -} - -#[test] -fn test_block_validation_invalid_bad_proposer_map() { - let params = get_simple_params(); - - let mutator = |block, attester_map, _, stores| { - /* - * Initialize a new, empty proposer map - */ - let proposer_map = ProposerMap::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::BadProposerMap)); -} - -#[test] -fn test_block_validation_invalid_2nd_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[1].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( - AttestationValidationError::BadAggregateSignature))); -} diff --git a/tests/main.rs b/tests/main.rs deleted file mode 100644 index caa601105..000000000 --- a/tests/main.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate lighthouse; -extern crate ssz; - -#[cfg(test)] -mod attestation_validation; -#[cfg(test)] -mod block_validation; - -use lighthouse::bls; -use lighthouse::db; -use lighthouse::state; -use lighthouse::utils;