From 77bd934efb0fb43d35f5059e7589f4d642d4816c Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 15 May 2018 14:48:13 -0400 Subject: [PATCH] sharding: 93% coverage Former-commit-id: f1a9fcf4fe1d11855e1dcb0a898f389dc669ab64 [formerly d255a71b60d1a939d2c7861039aace088a9fd34d] Former-commit-id: 3671d13ffb1c71cf7c48eebedeab53fd16d4a0fb --- sharding/collation.go | 6 +- sharding/database/inmemory_test.go | 4 + sharding/shard.go | 48 +++++--- sharding/shard_test.go | 184 +++++++++++++++++++++++++---- 4 files changed, 199 insertions(+), 43 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index d93d795fe..b12225c3b 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -74,11 +74,7 @@ func (h *CollationHeader) ChunkRoot() *common.Hash { return h.data.ChunkRoot } // EncodeRLP gives an encoded representation of the collation header. func (h *CollationHeader) EncodeRLP() ([]byte, error) { - encoded, err := rlp.EncodeToBytes(&h.data) - if err != nil { - return nil, err - } - return encoded, nil + return rlp.EncodeToBytes(&h.data) } // DecodeRLP uses an RLP Stream to populate the data field of a collation header. diff --git a/sharding/database/inmemory_test.go b/sharding/database/inmemory_test.go index ca4659b8b..d61c195f2 100644 --- a/sharding/database/inmemory_test.go +++ b/sharding/database/inmemory_test.go @@ -4,8 +4,12 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/sharding" ) +// Verifies that ShardKV implements the ShardBackend interface. +var _ = sharding.ShardBackend(&shardKV{}) + func Test_ShardKVPut(t *testing.T) { kv := NewShardKV() hash := common.StringToHash("ralph merkle") diff --git a/sharding/shard.go b/sharding/shard.go index 809f6e493..eeedc088f 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -9,7 +9,9 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -type shardBackend interface { +// ShardBackend defines an interface for a shardDB's necessary method +// signatures. +type ShardBackend interface { Get(k common.Hash) ([]byte, error) Has(k common.Hash) bool Put(k common.Hash, val []byte) error @@ -18,12 +20,12 @@ type shardBackend interface { // Shard base struct. type Shard struct { - shardDB shardBackend + shardDB ShardBackend shardID *big.Int } // NewShard creates an instance of a Shard struct given a shardID. -func NewShard(shardID *big.Int, shardDB shardBackend) *Shard { +func NewShard(shardID *big.Int, shardDB ShardBackend) *Shard { return &Shard{ shardID: shardID, shardDB: shardDB, @@ -64,28 +66,28 @@ func (s *Shard) HeaderByHash(hash *common.Hash) (*CollationHeader, error) { func (s *Shard) CollationByHash(headerHash *common.Hash) (*Collation, error) { header, err := s.HeaderByHash(headerHash) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot fetch header by hash: %v", err) } body, err := s.BodyByChunkRoot(header.ChunkRoot()) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot fetch body by chunk root: %v", err) } - // TODO: deserializes the body into a tx's object instead of using + // TODO: deserializes the body into a txs slice instead of using // nil as the third arg to MakeCollation. col := NewCollation(header, body, nil) return col, nil } // CanonicalHeaderHash gets a collation header hash that has been set as -// canonical for shardID/period pair +// canonical for shardID/period pair. func (s *Shard) CanonicalHeaderHash(shardID *big.Int, period *big.Int) (*common.Hash, error) { key := canonicalCollationLookupKey(shardID, period) // fetches the RLP encoded collation header corresponding to the key. encoded, err := s.shardDB.Get(key) - if err != nil || len(encoded) == 0 { - return nil, fmt.Errorf("no canonical collation header set for period, shardID pair: %v", err) + if err != nil { + return nil, fmt.Errorf("no canonical collation header set for period=%v, shardID=%v pair: %v", shardID, period, err) } // RLP decodes the header, computes its hash. @@ -104,10 +106,7 @@ func (s *Shard) CanonicalHeaderHash(shardID *big.Int, period *big.Int) (*common. func (s *Shard) CanonicalCollation(shardID *big.Int, period *big.Int) (*Collation, error) { h, err := s.CanonicalHeaderHash(shardID, period) if err != nil { - return nil, fmt.Errorf("error while getting canoncial header hash: %v", err) - } - if h == nil { - return nil, fmt.Errorf("header not found") + return nil, fmt.Errorf("error while getting canonical header hash: %v", err) } collation, err := s.CollationByHash(h) @@ -154,12 +153,17 @@ func (s *Shard) SetAvailability(chunkRoot *common.Hash, availability bool) error // SaveHeader adds the collation header to shardDB. func (s *Shard) SaveHeader(header *CollationHeader) error { + // checks if the header has a chunk root set before saving. + if header.ChunkRoot() == nil { + return fmt.Errorf("header needs to have a chunk root set before saving") + } + encoded, err := header.EncodeRLP() if err != nil { return fmt.Errorf("cannot encode header: %v", err) } - // Uses the hash of the header as the key. + // uses the hash of the header as the key. if err := s.shardDB.Put(header.Hash(), encoded); err != nil { return fmt.Errorf("cannot update shardDB: %v", err) } @@ -186,12 +190,16 @@ func (s *Shard) SaveCollation(collation *Collation) error { if err := s.ValidateShardID(collation.Header()); err != nil { return err } - s.SaveHeader(collation.Header()) - s.SaveBody(collation.Body()) + if err := s.SaveHeader(collation.Header()); err != nil { + return err + } + if err := s.SaveBody(collation.Body()); err != nil { + return err + } return nil } -// SetCanonical sets the collation as canonical in the shardDB. This is called +// SetCanonical sets the collation header as canonical in the shardDB. This is called // after the period is over and over 2/3 notaries voted on the header. func (s *Shard) SetCanonical(header *CollationHeader) error { if err := s.ValidateShardID(header); err != nil { @@ -205,6 +213,12 @@ func (s *Shard) SetCanonical(header *CollationHeader) error { return err } + // checks if the header has a corresponding body in the DB. + body, err := s.BodyByChunkRoot(dbHeader.ChunkRoot()) + if err != nil { + return fmt.Errorf("no corresponding collation body saved in shardDB: %v", body) + } + key := canonicalCollationLookupKey(dbHeader.ShardID(), dbHeader.Period()) encoded, err := dbHeader.EncodeRLP() if err != nil { diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 78c40fa3f..7d9b68b4b 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -2,6 +2,7 @@ package sharding import ( "bytes" + "fmt" "math/big" "testing" @@ -11,6 +12,26 @@ import ( "github.com/ethereum/go-ethereum/sharding/database" ) +type mockShardDB struct { + kv map[common.Hash][]byte +} + +func (m *mockShardDB) Get(k common.Hash) ([]byte, error) { + return nil, nil +} + +func (m *mockShardDB) Has(k common.Hash) bool { + return false +} + +func (m *mockShardDB) Put(k common.Hash, v []byte) error { + return fmt.Errorf("error updating db") +} + +func (m *mockShardDB) Delete(k common.Hash) error { + return fmt.Errorf("error deleting value in db") +} + // Hash returns the hash of a collation's entire contents. Useful for comparison tests. func (c *Collation) Hash() (hash common.Hash) { hw := sha3.NewKeccak256() @@ -41,8 +62,16 @@ func TestShard_HeaderByHash(t *testing.T) { emptyHash := common.StringToHash("") emptyAddr := common.StringToAddress("") header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, []byte{}) + + // creates a mockDB that always returns nil values from .Get and errors in every other method. + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} + + // creates a well-functioning shardDB. shardDB := database.NewShardKV() + + // creates a shard with a functioning DB and another one with a faulty DB. shard := NewShard(big.NewInt(1), shardDB) + errorShard := NewShard(big.NewInt(1), mockDB) if err := shard.SaveHeader(header); err != nil { t.Fatalf("cannot save collation header: %v", err) @@ -53,6 +82,13 @@ func TestShard_HeaderByHash(t *testing.T) { if err != nil { t.Fatalf("could not fetch collation header by hash: %v", err) } + + // checks for errors if shardDB value decoding fails (in the case of the + // shard with the faulty DB). + if _, err := errorShard.HeaderByHash(&hash); err == nil { + t.Errorf("if a faulty db is used, RLP decoding should fail") + } + // Compare the hashes. if header.Hash() != dbHeader.Hash() { t.Errorf("headers do not match. want=%v. got=%v", header, dbHeader) @@ -70,19 +106,25 @@ func TestShard_CollationByHash(t *testing.T) { body: []byte{1, 2, 3}, } - // TODO: check if body by chunk root fails! - - // We set the chunk root. - collation.CalculateChunkRoot() - shardDB := database.NewShardKV() shard := NewShard(big.NewInt(1), shardDB) hash := collation.Header().Hash() + // should throw error if saving the collation before setting the chunk root + // in header. + if err := shard.SaveCollation(collation); err == nil { + t.Errorf("should not be able to save collation before setting header chunk root") + } + + // we set the chunk root. + collation.CalculateChunkRoot() + + // calculate a new hash now that chunk root is set. + newHash := collation.Header().Hash() + // should not be able to fetch collation without saving first. - _, err := shard.CollationByHash(&hash) - if err == nil { + if _, err := shard.CollationByHash(&newHash); err == nil { t.Errorf("should not be able to fetch collation before saving first") } @@ -106,19 +148,22 @@ func TestShard_CanonicalHeaderHash(t *testing.T) { period := big.NewInt(1) proposerAddress := common.StringToAddress("") proposerSignature := []byte{} - emptyHash := common.StringToHash("") - header := NewCollationHeader(shardID, &emptyHash, period, &proposerAddress, proposerSignature) + header := NewCollationHeader(shardID, nil, period, &proposerAddress, proposerSignature) + + collation := NewCollation(header, []byte{1, 2, 3}, nil) + + collation.CalculateChunkRoot() shardDB := database.NewShardKV() shard := NewShard(shardID, shardDB) - // should not be able to set as canonical before saving the header. + // should not be able to set as canonical before saving the header and body first. if err := shard.SetCanonical(header); err == nil { - t.Errorf("cannot set as canonical before saving header first") + t.Errorf("cannot set as canonical before saving header and body first") } - if err := shard.SaveHeader(header); err != nil { - t.Fatalf("failed to save header to shardDB: %v", err) + if err := shard.SaveCollation(collation); err != nil { + t.Fatalf("failed to save collation to shardDB: %v", err) } if err := shard.SetCanonical(header); err != nil { @@ -132,6 +177,10 @@ func TestShard_CanonicalHeaderHash(t *testing.T) { t.Fatalf("failed to get canonical header hash from shardDB: %v", err) } + if _, err := shard.CanonicalHeaderHash(big.NewInt(100), big.NewInt(300)); err == nil { + t.Errorf("should throw error if a non-existent period, shardID pair is used") + } + if canonicalHeaderHash.String() != headerHash.String() { t.Errorf("header hashes do not match. want=%v. got=%v", headerHash.String(), canonicalHeaderHash.String()) } @@ -147,38 +196,64 @@ func TestShard_CanonicalCollation(t *testing.T) { shardDB := database.NewShardKV() shard := NewShard(shardID, shardDB) - otherShard := NewShard(big.NewInt(2), shardDB) collation := &Collation{ header: header, body: []byte{1, 2, 3}, } - // We set the chunk root. + // we set the chunk root. collation.CalculateChunkRoot() + // saves the full collation in the shardDB. if err := shard.SaveCollation(collation); err != nil { t.Fatalf("failed to save collation to shardDB: %v", err) } - if err := shard.SetCanonical(header); err != nil { - t.Fatalf("failed to set header as canonical: %v", err) + // fetching non-existent shardID, period pair should throw error. + if _, err := shard.CanonicalCollation(big.NewInt(100), big.NewInt(100)); err == nil { + t.Errorf("fetching a") } - // should not be allowed to set as canonical in a different shard. - if err := otherShard.SetCanonical(header); err == nil { - t.Errorf("should not be able to set header with ShardID=%v as canonical in other shard=%v", header.ShardID(), big.NewInt(2)) + // sets the correct information as canonical once the collation has been saved to shardDB. + if err := shard.SetCanonical(header); err != nil { + t.Fatalf("was not able to set header as canonical: %v", err) } canonicalCollation, err := shard.CanonicalCollation(shardID, period) if err != nil { t.Fatalf("failed to get canonical collation from shardDB: %v", err) } + if canonicalCollation.Hash() != collation.Hash() { t.Errorf("collations are not equal. want=%v. got=%v.", collation, canonicalCollation) } } +func TestShard_SetCanonical(t *testing.T) { + chunkRoot := common.StringToHash("") + header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) + + shardDB := database.NewShardKV() + shard := NewShard(big.NewInt(1), shardDB) + otherShard := NewShard(big.NewInt(2), shardDB) + + // saving the header but not the full collation, and then trying to fetch + // a canonical collation should throw an error. + if err := shard.SaveHeader(header); err != nil { + t.Fatalf("failed to save header to shardDB: %v", err) + } + + if err := shard.SetCanonical(header); err == nil { + t.Errorf("should not be able to set collation as canonical if header has no corresponding saved body") + } + + // should not be allowed to set as canonical in a different shard. + if err := otherShard.SetCanonical(header); err == nil { + t.Errorf("should not be able to set header with ShardID=%v as canonical in other shard=%v", header.ShardID(), big.NewInt(2)) + } +} + func TestShard_BodyByChunkRoot(t *testing.T) { body := []byte{1, 2, 3, 4, 5} shardID := big.NewInt(1) @@ -199,6 +274,12 @@ func TestShard_BodyByChunkRoot(t *testing.T) { t.Errorf("cannot fetch body by chunk root: %v", err) } + // it should throw error if fetching non-existent chunk root. + emptyHash := common.StringToHash("") + if _, err := shard.BodyByChunkRoot(&emptyHash); err == nil { + t.Errorf("non-existent chunk root should throw error") + } + if !bytes.Equal(body, dbBody) { t.Errorf("bodies not equal. want=%v. got=%v", body, dbBody) } @@ -220,9 +301,14 @@ func TestShard_CheckAvailability(t *testing.T) { body: []byte{1, 2, 3}, } - // We set the chunk root. + // we set the chunk root. collation.CalculateChunkRoot() + // should throw error if checking availability before header is even saved. + if _, err := shard.CheckAvailability(header); err == nil { + t.Errorf("error should be thrown: cannot check availability before saving header first") + } + if err := shard.SaveBody(collation.body); err != nil { t.Fatalf("cannot save body: %v", err) } @@ -236,6 +322,40 @@ func TestShard_CheckAvailability(t *testing.T) { } } +func TestShard_SetAvailability(t *testing.T) { + chunkRoot := common.StringToHash("") + header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) + + // creates a mockDB that always returns nil values from .Get and errors in every other method. + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} + + // creates a well-functioning shardDB. + shardDB := database.NewShardKV() + + // creates a shard with a functioning DB and another one with a faulty DB. + shard := NewShard(big.NewInt(1), shardDB) + errorShard := NewShard(big.NewInt(1), mockDB) + + if err := errorShard.SetAvailability(&chunkRoot, false); err == nil { + t.Errorf("should not be able to set availability when using a faulty DB") + } + + // sets the availability in a well-functioning shard with a good shardDB. + if err := shard.SetAvailability(&chunkRoot, false); err != nil { + t.Errorf("could not set availability for chunk root") + } + + available, err := shard.CheckAvailability(header) + if err != nil { + t.Fatalf("unable to check availability for header: %v", err) + } + + // availability should have been set to false. + if available { + t.Errorf("collation header should have been set to unavailable, instead was saved as available") + } +} + func TestShard_SaveCollation(t *testing.T) { headerShardID := big.NewInt(1) period := big.NewInt(1) @@ -259,3 +379,25 @@ func TestShard_SaveCollation(t *testing.T) { t.Errorf("cannot save collation in shard with wrong shardID") } } + +func TestShard_SaveHeader(t *testing.T) { + // creates a mockDB that always returns nil values from .Get and errors in every other method. + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} + emptyHash := common.StringToHash("") + errorShard := NewShard(big.NewInt(1), mockDB) + + header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), nil, []byte{}) + if err := errorShard.SaveHeader(header); err == nil { + t.Errorf("should not be able to save header if a faulty shardDB is used") + } +} + +func TestShard_SaveBody(t *testing.T) { + // creates a mockDB that always returns nil values from .Get and errors in every other method. + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} + errorShard := NewShard(big.NewInt(1), mockDB) + + if err := errorShard.SaveBody([]byte{1, 2, 3}); err == nil { + t.Errorf("should not be able to save body if a faulty shardDB is used") + } +}