mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-29 07:07:16 +00:00
state aggregator - tests for contract storage (#165)
* First test for storage * Debugging * Fixes to delete account test * Fix the lint Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local> Co-authored-by: Alex Sharp <alexsharp@Alexs-MacBook-Pro.local>
This commit is contained in:
ledgerwatch
GitHub
parent
5b7f67deae
commit
ea56078a4c
@ -410,8 +410,10 @@ func buildIndex(datPath, idxPath, tmpDir string, count int) (*compress.Decompres
|
||||
if err = rs.AddKey(word, pos); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
//fmt.Printf("add key %x to the index %s\n", word, idxPath)
|
||||
// Skip value
|
||||
word, pos = g.Next(word[:0])
|
||||
//fmt.Printf("value is [%x]\n", word)
|
||||
}
|
||||
if err = rs.Build(); err != nil {
|
||||
return nil, nil, err
|
||||
@ -508,13 +510,6 @@ func (c *Changes) aggregateToBtree(bt *btree.BTree, prefixLen int) error {
|
||||
if prefixLen > 0 && !bytes.Equal(prefix, key[:prefixLen]) {
|
||||
prefix = common.Copy(key[:prefixLen])
|
||||
item := &AggregateItem{k: prefix, count: 0}
|
||||
if len(after) > 0 {
|
||||
item.v = make([]byte, 1+len(after))
|
||||
if len(before) == 0 {
|
||||
item.v[0] = 1
|
||||
}
|
||||
copy(item.v[1:], after)
|
||||
}
|
||||
bt.ReplaceOrInsert(item)
|
||||
}
|
||||
ai.k = key
|
||||
@ -557,7 +552,7 @@ func btreeToFile(bt *btree.BTree, datPath string, tmpdir string) (int, error) {
|
||||
if err = comp.AddWord(item.k); err != nil {
|
||||
return false
|
||||
}
|
||||
//fmt.Printf("add key %x to %s\n", item.k, datPath)
|
||||
//fmt.Printf("add key %x val [%x] to %s\n", item.k, item.v, datPath)
|
||||
count++ // Only counting keys, not values
|
||||
if err = comp.AddWord(item.v); err != nil {
|
||||
return false
|
||||
@ -824,6 +819,81 @@ func (a *Aggregator) Close() {
|
||||
closeFiles(a.byEndBlock)
|
||||
}
|
||||
|
||||
func (a *Aggregator) readAccount(blockNum uint64, addr []byte) []byte {
|
||||
var val []byte
|
||||
a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.accountsIdx.Empty() {
|
||||
return true
|
||||
}
|
||||
offset := item.accountsIdx.Lookup(addr)
|
||||
g := item.accountsD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
if bytes.Equal(key, addr) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Aggregator) readCode(blockNum uint64, addr []byte) []byte {
|
||||
var val []byte
|
||||
a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.codeIdx.Empty() {
|
||||
return false
|
||||
}
|
||||
offset := item.codeIdx.Lookup(addr)
|
||||
g := item.codeD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
if bytes.Equal(key, addr) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Aggregator) readStorage(blockNum uint64, filekey []byte) []byte {
|
||||
var val []byte
|
||||
a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.storageIdx.Empty() {
|
||||
return false
|
||||
}
|
||||
offset := item.storageIdx.Lookup(filekey)
|
||||
g := item.storageD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
if bytes.Equal(key, filekey) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Aggregator) MakeStateReader(tx kv.Getter, blockNum uint64) *Reader {
|
||||
r := &Reader{
|
||||
a: a,
|
||||
@ -850,33 +920,11 @@ func (r *Reader) ReadAccountData(addr []byte) ([]byte, error) {
|
||||
return v[4:], nil
|
||||
}
|
||||
// Look in the files
|
||||
var val []byte
|
||||
//fmt.Printf("Looking up %x, r.a.byEndBlock.Len()=%d\n", addr, r.a.byEndBlock.Len())
|
||||
r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.accountsIdx.Empty() {
|
||||
return true
|
||||
}
|
||||
offset := item.accountsIdx.Lookup(addr)
|
||||
g := item.accountsD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
//fmt.Printf("state file [%d-%d], offset %d, key %x\n", item.startBlock, item.endBlock, offset, key)
|
||||
if bytes.Equal(key, addr) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:], nil
|
||||
}
|
||||
val := r.a.readAccount(r.blockNum, addr)
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte) ([]byte, error) {
|
||||
func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte) (*uint256.Int, error) {
|
||||
// Look in the summary table first
|
||||
dbkey := make([]byte, len(addr)+len(loc))
|
||||
copy(dbkey[0:], addr)
|
||||
@ -886,35 +934,18 @@ func (r *Reader) ReadAccountStorage(addr []byte, incarnation uint64, loc []byte)
|
||||
return nil, err
|
||||
}
|
||||
if v != nil {
|
||||
if len(v) == 4 {
|
||||
return nil, nil
|
||||
}
|
||||
// First 4 bytes is the number of 1-block state diffs containing the key
|
||||
return v[4:], nil
|
||||
return new(uint256.Int).SetBytes(v[4:]), nil
|
||||
}
|
||||
// Look in the files
|
||||
filekey := make([]byte, len(addr)+len(loc))
|
||||
copy(filekey[0:], addr)
|
||||
copy(filekey[len(addr):], loc)
|
||||
var val []byte
|
||||
r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.storageIdx.Empty() {
|
||||
return false
|
||||
}
|
||||
offset := item.storageIdx.Lookup(filekey)
|
||||
g := item.storageD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
if bytes.Equal(key, filekey) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:], nil
|
||||
val := r.a.readStorage(r.blockNum, dbkey)
|
||||
if val != nil {
|
||||
return new(uint256.Int).SetBytes(val), nil
|
||||
}
|
||||
return val, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *Reader) ReadAccountCode(addr []byte, incarnation uint64) ([]byte, error) {
|
||||
@ -928,27 +959,7 @@ func (r *Reader) ReadAccountCode(addr []byte, incarnation uint64) ([]byte, error
|
||||
return v[4:], nil
|
||||
}
|
||||
// Look in the files
|
||||
var val []byte
|
||||
r.a.byEndBlock.DescendLessOrEqual(&byEndBlockItem{endBlock: r.blockNum}, func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
if item.codeIdx.Empty() {
|
||||
return false
|
||||
}
|
||||
offset := item.codeIdx.Lookup(addr)
|
||||
g := item.codeD.MakeGetter() // TODO Cache in the reader
|
||||
g.Reset(offset)
|
||||
if g.HasNext() {
|
||||
key, _ := g.Next(nil) // Add special function that just checks the key
|
||||
if bytes.Equal(key, addr) {
|
||||
val, _ = g.Next(nil)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(val) > 0 {
|
||||
return val[1:], nil
|
||||
}
|
||||
val := r.a.readCode(r.blockNum, addr)
|
||||
return val, nil
|
||||
}
|
||||
|
||||
@ -1026,7 +1037,10 @@ func (w *Writer) UpdateAccountData(addr []byte, account []byte) error {
|
||||
return err
|
||||
}
|
||||
var prevNum uint32
|
||||
if prevV != nil {
|
||||
var original []byte
|
||||
if prevV == nil {
|
||||
original = w.a.readAccount(w.blockNum, addr)
|
||||
} else {
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
}
|
||||
v := make([]byte, 4+len(account))
|
||||
@ -1035,12 +1049,15 @@ func (w *Writer) UpdateAccountData(addr []byte, account []byte) error {
|
||||
if err = w.tx.Put(kv.StateAccounts, addr, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if prevV == nil {
|
||||
if prevV == nil && original == nil {
|
||||
if err = w.a.accountChanges.insert(addr, account); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = w.a.accountChanges.update(addr, prevV[4:], account); err != nil {
|
||||
if original == nil {
|
||||
original = prevV[4:]
|
||||
}
|
||||
if err = w.a.accountChanges.update(addr, original, account); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -1053,7 +1070,10 @@ func (w *Writer) UpdateAccountCode(addr []byte, code []byte) error {
|
||||
return err
|
||||
}
|
||||
var prevNum uint32
|
||||
if prevV != nil {
|
||||
var original []byte
|
||||
if prevV == nil {
|
||||
original = w.a.readCode(w.blockNum, addr)
|
||||
} else {
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
}
|
||||
v := make([]byte, 4+len(code))
|
||||
@ -1062,12 +1082,15 @@ func (w *Writer) UpdateAccountCode(addr []byte, code []byte) error {
|
||||
if err = w.tx.Put(kv.StateCode, addr, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if prevV == nil {
|
||||
if prevV == nil && original == nil {
|
||||
if err = w.a.codeChanges.insert(addr, code); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = w.a.codeChanges.update(addr, prevV[4:], code); err != nil {
|
||||
if original == nil {
|
||||
original = prevV[4:]
|
||||
}
|
||||
if err = w.a.codeChanges.update(addr, original, code); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -1121,17 +1144,23 @@ func (w *Writer) DeleteAccount(addr []byte) error {
|
||||
return err
|
||||
}
|
||||
var prevNum uint32
|
||||
if prevV != nil {
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
var original []byte
|
||||
if prevV == nil {
|
||||
original = w.a.readAccount(w.blockNum, addr)
|
||||
} else {
|
||||
return fmt.Errorf("deleteAccount no prev value for %x", addr)
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
}
|
||||
v := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(v[:4], prevNum+1)
|
||||
if err = w.tx.Put(kv.StateAccounts, addr, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.a.accountChanges.delete(addr, prevV[4:]); err != nil {
|
||||
if prevV == nil && original == nil {
|
||||
return fmt.Errorf("previous value expected for DeleteAccount")
|
||||
} else if original == nil {
|
||||
original = prevV[4:]
|
||||
}
|
||||
if err = w.a.accountChanges.delete(addr, original); err != nil {
|
||||
return err
|
||||
}
|
||||
// Find all storage items for this address
|
||||
@ -1146,7 +1175,7 @@ func (w *Writer) DeleteAccount(addr []byte) error {
|
||||
return err
|
||||
}
|
||||
if k != nil && bytes.HasPrefix(k, addr) {
|
||||
heap.Push(&cp, CursorItem{file: false, key: k, val: v, c: c, endBlock: w.blockNum})
|
||||
heap.Push(&cp, CursorItem{file: false, key: common.Copy(k), val: common.Copy(v), c: c, endBlock: w.blockNum})
|
||||
}
|
||||
w.a.byEndBlock.Ascend(func(i btree.Item) bool {
|
||||
item := i.(*byEndBlockItem)
|
||||
@ -1177,9 +1206,9 @@ func (w *Writer) DeleteAccount(addr []byte) error {
|
||||
ci1 := &cp[0]
|
||||
if ci1.file {
|
||||
if ci1.dg.HasNext() {
|
||||
ci1.key, _ = ci1.dg.Next(ci1.key)
|
||||
ci1.key, _ = ci1.dg.Next(ci1.key[:0])
|
||||
if bytes.HasPrefix(ci1.key, addr) {
|
||||
ci1.val, _ = ci1.dg.Next(ci1.val)
|
||||
ci1.val, _ = ci1.dg.Next(ci1.val[:0])
|
||||
heap.Fix(&cp, 0)
|
||||
} else {
|
||||
heap.Pop(&cp)
|
||||
@ -1193,14 +1222,27 @@ func (w *Writer) DeleteAccount(addr []byte) error {
|
||||
return err
|
||||
}
|
||||
if k != nil && bytes.HasPrefix(k, addr) {
|
||||
ci1.key = k
|
||||
ci1.val = v
|
||||
ci1.key = common.Copy(k)
|
||||
ci1.val = common.Copy(v)
|
||||
heap.Fix(&cp, 0)
|
||||
} else {
|
||||
heap.Pop(&cp)
|
||||
}
|
||||
}
|
||||
}
|
||||
prevV, err = w.tx.GetOne(kv.StateStorage, lastKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prevNum = 0
|
||||
if prevV != nil {
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
}
|
||||
v = make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(v[:4], prevNum+1)
|
||||
if err = w.tx.Put(kv.StateStorage, lastKey, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.a.storageChanges.delete(lastKey, lastVal); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -1208,7 +1250,7 @@ func (w *Writer) DeleteAccount(addr []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte, original, value *uint256.Int) error {
|
||||
func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte, _, value *uint256.Int) error {
|
||||
dbkey := make([]byte, len(addr)+len(loc))
|
||||
copy(dbkey[0:], addr)
|
||||
copy(dbkey[len(addr):], loc)
|
||||
@ -1217,14 +1259,17 @@ func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte
|
||||
return err
|
||||
}
|
||||
var prevNum uint32
|
||||
if prevV != nil {
|
||||
var original []byte
|
||||
if prevV == nil {
|
||||
original = w.a.readStorage(w.blockNum, dbkey)
|
||||
} else {
|
||||
prevNum = binary.BigEndian.Uint32(prevV[:4])
|
||||
}
|
||||
vLen := value.ByteLen()
|
||||
v := make([]byte, 4+vLen)
|
||||
binary.BigEndian.PutUint32(v[:4], prevNum+1)
|
||||
value.WriteToSlice(v[4:])
|
||||
if err = w.tx.Put(kv.StateStorage, addr, v); err != nil {
|
||||
if err = w.tx.Put(kv.StateStorage, dbkey, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if prevV == nil {
|
||||
@ -1232,7 +1277,10 @@ func (w *Writer) WriteAccountStorage(addr []byte, incarnation uint64, loc []byte
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = w.a.storageChanges.update(dbkey, prevV[4:], v[4:]); err != nil {
|
||||
if original == nil {
|
||||
original = prevV[4:]
|
||||
}
|
||||
if err = w.a.storageChanges.update(dbkey, original, v[4:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"github.com/holiman/uint256"
|
||||
"github.com/ledgerwatch/erigon-lib/kv"
|
||||
"github.com/ledgerwatch/erigon-lib/kv/memdb"
|
||||
)
|
||||
@ -90,6 +91,7 @@ func TestLoopAggregator(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
var account1 = int256(1)
|
||||
var rwTx kv.RwTx
|
||||
defer func() {
|
||||
@ -154,3 +156,95 @@ func TestLoopAggregator(t *testing.T) {
|
||||
tx.Rollback()
|
||||
a.Close()
|
||||
}
|
||||
|
||||
func TestRecreateAccountWithStorage(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
db := memdb.New()
|
||||
defer db.Close()
|
||||
a, err := NewAggregator(tmpDir, 16, 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Close()
|
||||
accountKey := int160(1)
|
||||
var account1 = int256(1)
|
||||
var rwTx kv.RwTx
|
||||
defer func() {
|
||||
rwTx.Rollback()
|
||||
}()
|
||||
var tx kv.Tx
|
||||
defer func() {
|
||||
tx.Rollback()
|
||||
}()
|
||||
for blockNum := uint64(0); blockNum < 100; blockNum++ {
|
||||
if rwTx, err = db.BeginRw(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w *Writer
|
||||
if w, err = a.MakeStateWriter(rwTx, blockNum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch blockNum {
|
||||
case 1:
|
||||
if err = w.UpdateAccountData(accountKey, account1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for s := uint64(0); s < 100; s++ {
|
||||
if err = w.WriteAccountStorage(accountKey, 1, int256(s), nil, uint256.NewInt(s+1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
case 22:
|
||||
if err = w.DeleteAccount(accountKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = w.Finish(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = rwTx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tx, err = db.BeginRo(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := a.MakeStateReader(tx, blockNum+1)
|
||||
switch blockNum {
|
||||
case 1:
|
||||
var acc []byte
|
||||
if acc, err = r.ReadAccountData(accountKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(account1, acc) {
|
||||
t.Errorf("wrong account after block %d, expected %x, got %x", blockNum, account1, acc)
|
||||
}
|
||||
for s := uint64(0); s < 100; s++ {
|
||||
var v *uint256.Int
|
||||
if v, err = r.ReadAccountStorage(accountKey, 1, int256(s)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !uint256.NewInt(s + 1).Eq(v) {
|
||||
t.Errorf("wrong storage value after block %d, expected %d, got %s", blockNum, s, v)
|
||||
}
|
||||
}
|
||||
case 22, 44:
|
||||
var acc []byte
|
||||
if acc, err = r.ReadAccountData(accountKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(acc) > 0 {
|
||||
t.Errorf("wrong account after block %d, expected nil, got %x", blockNum, acc)
|
||||
}
|
||||
for s := uint64(0); s < 100; s++ {
|
||||
var v *uint256.Int
|
||||
if v, err = r.ReadAccountStorage(accountKey, 1, int256(s)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v != nil {
|
||||
t.Errorf("wrong storage value after block %d, expected nil, got %s", blockNum, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.Rollback()
|
||||
}
|
||||
}
|
||||
|
||||
@ -986,7 +986,11 @@ func (c *Compressor) optimiseCodes() error {
|
||||
return e
|
||||
}
|
||||
}
|
||||
if l > 0 {
|
||||
if l == 0 {
|
||||
if e = hc.flush(); e != nil {
|
||||
return e
|
||||
}
|
||||
} else {
|
||||
var pNum uint64 // Number of patterns
|
||||
if pNum, e = binary.ReadUvarint(r); e != nil {
|
||||
return e
|
||||
|
||||
Loading…
Reference in New Issue
Block a user