From ea56078a4c02166af4cf88d955431a6ab3a758ae Mon Sep 17 00:00:00 2001 From: ledgerwatch Date: Tue, 16 Nov 2021 00:11:40 +0000 Subject: [PATCH] state aggregator - tests for contract storage (#165) * First test for storage * Debugging * Fixes to delete account test * Fix the lint Co-authored-by: Alexey Sharp Co-authored-by: Alex Sharp --- aggregator/aggregator.go | 242 ++++++++++++++++++++-------------- aggregator/aggregator_test.go | 94 +++++++++++++ compress/compress.go | 6 +- 3 files changed, 244 insertions(+), 98 deletions(-) diff --git a/aggregator/aggregator.go b/aggregator/aggregator.go index 1be9fc059..39614f8d6 100644 --- a/aggregator/aggregator.go +++ b/aggregator/aggregator.go @@ -410,8 +410,10 @@ func buildIndex(datPath, idxPath, tmpDir string, count int) (*compress.Decompres if err = rs.AddKey(word, pos); err != nil { return nil, nil, err } + //fmt.Printf("add key %x to the index %s\n", word, idxPath) // Skip value word, pos = g.Next(word[:0]) + //fmt.Printf("value is [%x]\n", word) } if err = rs.Build(); err != nil { return nil, nil, err @@ -508,13 +510,6 @@ func (c *Changes) aggregateToBtree(bt *btree.BTree, prefixLen int) error { if prefixLen > 0 && !bytes.Equal(prefix, key[:prefixLen]) { prefix = common.Copy(key[:prefixLen]) item := &AggregateItem{k: prefix, count: 0} - if len(after) > 0 { - item.v = make([]byte, 1+len(after)) - if len(before) == 0 { - item.v[0] = 1 - } - copy(item.v[1:], after) - } bt.ReplaceOrInsert(item) } ai.k = key @@ -557,7 +552,7 @@ func btreeToFile(bt *btree.BTree, datPath string, tmpdir string) (int, error) { if err = comp.AddWord(item.k); err != nil { return false } - //fmt.Printf("add key %x to %s\n", item.k, datPath) + //fmt.Printf("add key %x val [%x] to %s\n", item.k, item.v, datPath) count++ // Only counting keys, not values if err = comp.AddWord(item.v); err != nil { return false @@ -824,6 +819,81 @@ func (a *Aggregator) Close() { closeFiles(a.byEndBlock) } +func (a *Aggregator) readAccount(blockNum uint64, addr []byte) []byte { + var val []byte + a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool { + item := i.(*byEndBlockItem) + if item.accountsIdx.Empty() { + return true + } + offset := item.accountsIdx.Lookup(addr) + g := item.accountsD.MakeGetter() // TODO Cache in the reader + g.Reset(offset) + if g.HasNext() { + key, _ := g.Next(nil) // Add special function that just checks the key + if bytes.Equal(key, addr) { + val, _ = g.Next(nil) + return false + } + } + return true + }) + if len(val) > 0 { + return val[1:] + } + return nil +} + +func (a *Aggregator) readCode(blockNum uint64, addr []byte) []byte { + var val []byte + a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool { + item := i.(*byEndBlockItem) + if item.codeIdx.Empty() { + return false + } + offset := item.codeIdx.Lookup(addr) + g := item.codeD.MakeGetter() // TODO Cache in the reader + g.Reset(offset) + if g.HasNext() { + key, _ := g.Next(nil) // Add special function that just checks the key + if bytes.Equal(key, addr) { + val, _ = g.Next(nil) + return false + } + } + return true + }) + if len(val) > 0 { + return val[1:] + } + return nil +} + +func (a *Aggregator) readStorage(blockNum uint64, filekey []byte) []byte { + var val []byte + a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool { + item := i.(*byEndBlockItem) + if item.storageIdx.Empty() { + return false + } + offset := item.storageIdx.Lookup(filekey) + g := item.storageD.MakeGetter() // TODO Cache in the reader + g.Reset(offset) + if g.HasNext() { + key, _ := g.Next(nil) // Add special function that just checks the key + if bytes.Equal(key, filekey) { + val, _ = g.Next(nil) + return false + } + } + return true + }) + if len(val) > 0 { + return val[1:] + } + return nil +} + func (a *Aggregator) MakeStateReader(tx kv.Getter, blockNum uint64) *Reader { r := &Reader{ a: a, @@ -850,33 +920,11 @@ func (r *Reader) ReadAccountData(addr []byte) ([]byte, error) { return v[4:], nil } // Look in the files - var val []byte - //fmt.Printf("Looking up %x, r.a.byEndBlock.Len()=%d\n", addr, r.a.byEndBlock.Len()) - r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool { - item := i.(*byEndBlockItem) - if item.accountsIdx.Empty() { - return true - } - offset := item.accountsIdx.Lookup(addr) - g := item.accountsD.MakeGetter() // TODO Cache in the reader - g.Reset(offset) - if g.HasNext() { - key, _ := g.Next(nil) // Add special function that just checks the key - //fmt.Printf("state file [%d-%d], offset %d, key %x\n", item.startBlock, item.endBlock, offset, key) - if bytes.Equal(key, addr) { - val, _ = g.Next(nil) - return false - } - } - return true - }) - if len(val) > 0 { - return val[1:], nil - } + val := r.a.readAccount(r.blockNum, addr) return val, nil } -func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte) ([]byte, error) { +func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte) (*uint256.Int, error) { // Look in the summary table first dbkey := make([]byte, len(addr)+len(loc)) copy(dbkey[0:], addr) @@ -886,35 +934,18 @@ func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte) return nil, err } if v != nil { + if len(v) == 4 { + return nil, nil + } // First 4 bytes is the number of 1-block state diffs containing the key - return v[4:], nil + return new(uint256.Int).SetBytes(v[4:]), nil } // Look in the files - filekey := make([]byte, len(addr)+len(loc)) - copy(filekey[0:], addr) - copy(filekey[len(addr):], loc) - var val []byte - r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool { - item := i.(*byEndBlockItem) - if item.storageIdx.Empty() { - return false - } - offset := item.storageIdx.Lookup(filekey) - g := item.storageD.MakeGetter() // TODO Cache in the reader - g.Reset(offset) - if g.HasNext() { - key, _ := g.Next(nil) // Add special function that just checks the key - if bytes.Equal(key, filekey) { - val, _ = g.Next(nil) - return false - } - } - return true - }) - if len(val) > 0 { - return val[1:], nil + val := r.a.readStorage(r.blockNum, dbkey) + if val != nil { + return new(uint256.Int).SetBytes(val), nil } - return val, nil + return nil, nil } func (r *Reader) ReadAccountCode(addr []byte, incarnation uint64) ([]byte, error) { @@ -928,27 +959,7 @@ func (r *Reader) ReadAccountCode(addr []byte, incarnation uint64) ([]byte, error return v[4:], nil } // Look in the files - var val []byte - r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool { - item := i.(*byEndBlockItem) - if item.codeIdx.Empty() { - return false - } - offset := item.codeIdx.Lookup(addr) - g := item.codeD.MakeGetter() // TODO Cache in the reader - g.Reset(offset) - if g.HasNext() { - key, _ := g.Next(nil) // Add special function that just checks the key - if bytes.Equal(key, addr) { - val, _ = g.Next(nil) - return false - } - } - return true - }) - if len(val) > 0 { - return val[1:], nil - } + val := r.a.readCode(r.blockNum, addr) return val, nil } @@ -1026,7 +1037,10 @@ func (w *Writer) UpdateAccountData(addr []byte, account []byte) error { return err } var prevNum uint32 - if prevV != nil { + var original []byte + if prevV == nil { + original = w.a.readAccount(w.blockNum, addr) + } else { prevNum = binary.BigEndian.Uint32(prevV[:4]) } v := make([]byte, 4+len(account)) @@ -1035,12 +1049,15 @@ func (w *Writer) UpdateAccountData(addr []byte, account []byte) error { if err = w.tx.Put(kv.StateAccounts, addr, v); err != nil { return err } - if prevV == nil { + if prevV == nil && original == nil { if err = w.a.accountChanges.insert(addr, account); err != nil { return err } } else { - if err = w.a.accountChanges.update(addr, prevV[4:], account); err != nil { + if original == nil { + original = prevV[4:] + } + if err = w.a.accountChanges.update(addr, original, account); err != nil { return err } } @@ -1053,7 +1070,10 @@ func (w *Writer) UpdateAccountCode(addr []byte, code []byte) error { return err } var prevNum uint32 - if prevV != nil { + var original []byte + if prevV == nil { + original = w.a.readCode(w.blockNum, addr) + } else { prevNum = binary.BigEndian.Uint32(prevV[:4]) } v := make([]byte, 4+len(code)) @@ -1062,12 +1082,15 @@ func (w *Writer) UpdateAccountCode(addr []byte, code []byte) error { if err = w.tx.Put(kv.StateCode, addr, v); err != nil { return err } - if prevV == nil { + if prevV == nil && original == nil { if err = w.a.codeChanges.insert(addr, code); err != nil { return err } } else { - if err = w.a.codeChanges.update(addr, prevV[4:], code); err != nil { + if original == nil { + original = prevV[4:] + } + if err = w.a.codeChanges.update(addr, original, code); err != nil { return err } } @@ -1121,17 +1144,23 @@ func (w *Writer) DeleteAccount(addr []byte) error { return err } var prevNum uint32 - if prevV != nil { - prevNum = binary.BigEndian.Uint32(prevV[:4]) + var original []byte + if prevV == nil { + original = w.a.readAccount(w.blockNum, addr) } else { - return fmt.Errorf("deleteAccount no prev value for %x", addr) + prevNum = binary.BigEndian.Uint32(prevV[:4]) } v := make([]byte, 4) binary.BigEndian.PutUint32(v[:4], prevNum+1) if err = w.tx.Put(kv.StateAccounts, addr, v); err != nil { return err } - if err = w.a.accountChanges.delete(addr, prevV[4:]); err != nil { + if prevV == nil && original == nil { + return fmt.Errorf("previous value expected for DeleteAccount") + } else if original == nil { + original = prevV[4:] + } + if err = w.a.accountChanges.delete(addr, original); err != nil { return err } // Find all storage items for this address @@ -1146,7 +1175,7 @@ func (w *Writer) DeleteAccount(addr []byte) error { return err } if k != nil && bytes.HasPrefix(k, addr) { - heap.Push(&cp, CursorItem{file: false, key: k, val: v, c: c, endBlock: w.blockNum}) + heap.Push(&cp, CursorItem{file: false, key: common.Copy(k), val: common.Copy(v), c: c, endBlock: w.blockNum}) } w.a.byEndBlock.Ascend(func(i btree.Item) bool { item := i.(*byEndBlockItem) @@ -1177,9 +1206,9 @@ func (w *Writer) DeleteAccount(addr []byte) error { ci1 := &cp[0] if ci1.file { if ci1.dg.HasNext() { - ci1.key, _ = ci1.dg.Next(ci1.key) + ci1.key, _ = ci1.dg.Next(ci1.key[:0]) if bytes.HasPrefix(ci1.key, addr) { - ci1.val, _ = ci1.dg.Next(ci1.val) + ci1.val, _ = ci1.dg.Next(ci1.val[:0]) heap.Fix(&cp, 0) } else { heap.Pop(&cp) @@ -1193,14 +1222,27 @@ func (w *Writer) DeleteAccount(addr []byte) error { return err } if k != nil && bytes.HasPrefix(k, addr) { - ci1.key = k - ci1.val = v + ci1.key = common.Copy(k) + ci1.val = common.Copy(v) heap.Fix(&cp, 0) } else { heap.Pop(&cp) } } } + prevV, err = w.tx.GetOne(kv.StateStorage, lastKey) + if err != nil { + return err + } + prevNum = 0 + if prevV != nil { + prevNum = binary.BigEndian.Uint32(prevV[:4]) + } + v = make([]byte, 4) + binary.BigEndian.PutUint32(v[:4], prevNum+1) + if err = w.tx.Put(kv.StateStorage, lastKey, v); err != nil { + return err + } if err = w.a.storageChanges.delete(lastKey, lastVal); err != nil { return err } @@ -1208,7 +1250,7 @@ func (w *Writer) DeleteAccount(addr []byte) error { return nil } -func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte, original, value *uint256.Int) error { +func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte, _, value *uint256.Int) error { dbkey := make([]byte, len(addr)+len(loc)) copy(dbkey[0:], addr) copy(dbkey[len(addr):], loc) @@ -1217,14 +1259,17 @@ func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte return err } var prevNum uint32 - if prevV != nil { + var original []byte + if prevV == nil { + original = w.a.readStorage(w.blockNum, dbkey) + } else { prevNum = binary.BigEndian.Uint32(prevV[:4]) } vLen := value.ByteLen() v := make([]byte, 4+vLen) binary.BigEndian.PutUint32(v[:4], prevNum+1) value.WriteToSlice(v[4:]) - if err = w.tx.Put(kv.StateStorage, addr, v); err != nil { + if err = w.tx.Put(kv.StateStorage, dbkey, v); err != nil { return err } if prevV == nil { @@ -1232,7 +1277,10 @@ func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte return err } } else { - if err = w.a.storageChanges.update(dbkey, prevV[4:], v[4:]); err != nil { + if original == nil { + original = prevV[4:] + } + if err = w.a.storageChanges.update(dbkey, original, v[4:]); err != nil { return err } } diff --git a/aggregator/aggregator_test.go b/aggregator/aggregator_test.go index ca23b6f5f..0027fe9d1 100644 --- a/aggregator/aggregator_test.go +++ b/aggregator/aggregator_test.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "testing" + "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" ) @@ -90,6 +91,7 @@ func TestLoopAggregator(t *testing.T) { if err != nil { t.Fatal(err) } + defer a.Close() var account1 = int256(1) var rwTx kv.RwTx defer func() { @@ -154,3 +156,95 @@ func TestLoopAggregator(t *testing.T) { tx.Rollback() a.Close() } + +func TestRecreateAccountWithStorage(t *testing.T) { + tmpDir := t.TempDir() + db := memdb.New() + defer db.Close() + a, err := NewAggregator(tmpDir, 16, 4) + if err != nil { + t.Fatal(err) + } + defer a.Close() + accountKey := int160(1) + var account1 = int256(1) + var rwTx kv.RwTx + defer func() { + rwTx.Rollback() + }() + var tx kv.Tx + defer func() { + tx.Rollback() + }() + for blockNum := uint64(0); blockNum < 100; blockNum++ { + if rwTx, err = db.BeginRw(context.Background()); err != nil { + t.Fatal(err) + } + var w *Writer + if w, err = a.MakeStateWriter(rwTx, blockNum); err != nil { + t.Fatal(err) + } + switch blockNum { + case 1: + if err = w.UpdateAccountData(accountKey, account1); err != nil { + t.Fatal(err) + } + for s := uint64(0); s < 100; s++ { + if err = w.WriteAccountStorage(accountKey, 1, int256(s), nil, uint256.NewInt(s+1)); err != nil { + t.Fatal(err) + } + } + case 22: + if err = w.DeleteAccount(accountKey); err != nil { + t.Fatal(err) + } + } + if err = w.Finish(); err != nil { + t.Fatal(err) + } + if err = rwTx.Commit(); err != nil { + t.Fatal(err) + } + if tx, err = db.BeginRo(context.Background()); err != nil { + t.Fatal(err) + } + r := a.MakeStateReader(tx, blockNum+1) + switch blockNum { + case 1: + var acc []byte + if acc, err = r.ReadAccountData(accountKey); err != nil { + t.Fatal(err) + } + if !bytes.Equal(account1, acc) { + t.Errorf("wrong account after block %d, expected %x, got %x", blockNum, account1, acc) + } + for s := uint64(0); s < 100; s++ { + var v *uint256.Int + if v, err = r.ReadAccountStorage(accountKey, 1, int256(s)); err != nil { + t.Fatal(err) + } + if !uint256.NewInt(s + 1).Eq(v) { + t.Errorf("wrong storage value after block %d, expected %d, got %s", blockNum, s, v) + } + } + case 22, 44: + var acc []byte + if acc, err = r.ReadAccountData(accountKey); err != nil { + t.Fatal(err) + } + if len(acc) > 0 { + t.Errorf("wrong account after block %d, expected nil, got %x", blockNum, acc) + } + for s := uint64(0); s < 100; s++ { + var v *uint256.Int + if v, err = r.ReadAccountStorage(accountKey, 1, int256(s)); err != nil { + t.Fatal(err) + } + if v != nil { + t.Errorf("wrong storage value after block %d, expected nil, got %s", blockNum, v) + } + } + } + tx.Rollback() + } +} diff --git a/compress/compress.go b/compress/compress.go index 2c346bdbc..f99067f62 100644 --- a/compress/compress.go +++ b/compress/compress.go @@ -986,7 +986,11 @@ func (c *Compressor) optimiseCodes() error { return e } } - if l > 0 { + if l == 0 { + if e = hc.flush(); e != nil { + return e + } + } else { var pNum uint64 // Number of patterns if pNum, e = binary.ReadUvarint(r); e != nil { return e