erigon-pulse/core/state/db_state_writer.go

253 lines
7.4 KiB
Go
Raw Normal View History

package state
import (
"bytes"
"context"
"encoding/binary"
"fmt"
Intermediate hash phase 3 (#377) * #remove debug prints * remove storage-mode="i" * minnet re-execute hack with checkpoints * minnet re-execute hack with checkpoints * rollback to master setup * mainnet re-exec hack * rollback some changes * v0 of "push down" functionality * move all logic to own functions * handle case when re-created account already has some storage * clear path for storage * try to rely on tree structure (but maybe need to rely on DB because can be intra-block re-creations of account) * fix some bugs with indexes, moving to tests * tests added * make linter happy * make linter happy * simplify logic * adjust comparison of keys with and without incarnation * test for keyIsBefore * test for keyIsBefore * better nibbles alignment * better nibbles alignment * cleanup * continue work on tests * simplify test * check tombstone existence before pushing it down. * put tombstone only when account deleted, not created * put tombstone only when account has storage * make linter happy * test for storage resolver * make fixedbytes work without incarnation * fix panic on short keys * use special comparison only when working with keys from cache * add blockNr for better tracing * fix: incorrect tombstone check * fix: incorrect tombstone check * trigger ci * hack for problem block * more test-cases * add test case for too long keys * speedup cached resolver by removing bucket creation transaction * remove parent type check in pruning, remove unused copy from mutation.put * dump resolving info on fail * dump resolving info on fail * set tombstone everytime for now to check if it will help * on unload: check parent type, not type of node * fix wrong order of checking node type * fix wrong order of checking node type * rebase to new master * make linter happy * rebase to new master * place tombstone only if acc has storage * rebase master * rebase master * rebase master * rebase master Co-authored-by: alex.sharov <alex.sharov@lazada.com>
2020-03-11 10:31:49 +00:00
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/core/rawdb"
"github.com/ledgerwatch/turbo-geth/core/types/accounts"
"github.com/ledgerwatch/turbo-geth/ethdb"
"github.com/ledgerwatch/turbo-geth/trie"
)
func NewDbStateWriter(db ethdb.Database, blockNr uint64, incarnationMap map[common.Address]uint64) *DbStateWriter {
return &DbStateWriter{
db: db,
blockNr: blockNr,
pw: &PreimageWriter{db: db, savePreimages: false},
csw: NewChangeSetWriter(),
incarnations: incarnationMap,
}
}
type DbStateWriter struct {
db ethdb.Database
pw *PreimageWriter
blockNr uint64
csw *ChangeSetWriter
incarnations map[common.Address]uint64
}
func originalAccountData(original *accounts.Account, omitHashes bool) []byte {
var originalData []byte
if !original.Initialised {
originalData = []byte{}
} else if omitHashes {
testAcc := original.SelfCopy()
copy(testAcc.CodeHash[:], emptyCodeHash)
testAcc.Root = trie.EmptyRoot
originalDataLen := testAcc.EncodingLengthForStorage()
originalData = make([]byte, originalDataLen)
testAcc.EncodeForStorage(originalData)
} else {
originalDataLen := original.EncodingLengthForStorage()
originalData = make([]byte, originalDataLen)
original.EncodeForStorage(originalData)
}
return originalData
}
func (dsw *DbStateWriter) UpdateAccountData(ctx context.Context, address common.Address, original, account *accounts.Account) error {
if err := dsw.csw.UpdateAccountData(ctx, address, original, account); err != nil {
return err
}
addrHash, err := dsw.pw.HashAddress(address, true /*save*/)
if err != nil {
return err
}
if err := rawdb.WriteAccount(dsw.db, addrHash, *account); err != nil {
return err
}
return nil
}
func (dsw *DbStateWriter) DeleteAccount(ctx context.Context, address common.Address, original *accounts.Account) error {
if err := dsw.csw.DeleteAccount(ctx, address, original); err != nil {
return err
}
addrHash, err := dsw.pw.HashAddress(address, true /*save*/)
if err != nil {
return err
}
if err := rawdb.DeleteAccount(dsw.db, addrHash); err != nil {
return err
}
if original.Incarnation > 0 {
dsw.incarnations[address] = original.Incarnation + 1 // next incarnation
}
return nil
}
func (dsw *DbStateWriter) UpdateAccountCode(addrHash common.Hash, incarnation uint64, codeHash common.Hash, code []byte) error {
if err := dsw.csw.UpdateAccountCode(addrHash, incarnation, codeHash, code); err != nil {
return err
}
//save contract code mapping
if err := dsw.db.Put(dbutils.CodeBucket, codeHash[:], code); err != nil {
return err
}
//save contract to codeHash mapping
return dsw.db.Put(dbutils.ContractCodeBucket, dbutils.GenerateStoragePrefix(addrHash[:], incarnation), codeHash[:])
}
func (dsw *DbStateWriter) WriteAccountStorage(ctx context.Context, address common.Address, incarnation uint64, key, original, value *common.Hash) error {
// We delegate here first to let the changeSetWrite make its own decision on whether to proceed in case *original == *value
if err := dsw.csw.WriteAccountStorage(ctx, address, incarnation, key, original, value); err != nil {
return err
}
if *original == *value {
return nil
}
seckey, err := dsw.pw.HashKey(key, true /*save*/)
if err != nil {
return err
}
v := bytes.TrimLeft(value[:], "\x00")
vv := make([]byte, len(v))
copy(vv, v)
addrHash, err := dsw.pw.HashAddress(address, false /*save*/)
if err != nil {
return err
}
compositeKey := dbutils.GenerateCompositeStorageKey(addrHash, incarnation, seckey)
if len(v) == 0 {
return dsw.db.Delete(dbutils.CurrentStateBucket, compositeKey)
} else {
return dsw.db.Put(dbutils.CurrentStateBucket, compositeKey, vv)
}
}
Intermediate hash phase 3 (#377) * #remove debug prints * remove storage-mode="i" * minnet re-execute hack with checkpoints * minnet re-execute hack with checkpoints * rollback to master setup * mainnet re-exec hack * rollback some changes * v0 of "push down" functionality * move all logic to own functions * handle case when re-created account already has some storage * clear path for storage * try to rely on tree structure (but maybe need to rely on DB because can be intra-block re-creations of account) * fix some bugs with indexes, moving to tests * tests added * make linter happy * make linter happy * simplify logic * adjust comparison of keys with and without incarnation * test for keyIsBefore * test for keyIsBefore * better nibbles alignment * better nibbles alignment * cleanup * continue work on tests * simplify test * check tombstone existence before pushing it down. * put tombstone only when account deleted, not created * put tombstone only when account has storage * make linter happy * test for storage resolver * make fixedbytes work without incarnation * fix panic on short keys * use special comparison only when working with keys from cache * add blockNr for better tracing * fix: incorrect tombstone check * fix: incorrect tombstone check * trigger ci * hack for problem block * more test-cases * add test case for too long keys * speedup cached resolver by removing bucket creation transaction * remove parent type check in pruning, remove unused copy from mutation.put * dump resolving info on fail * dump resolving info on fail * set tombstone everytime for now to check if it will help * on unload: check parent type, not type of node * fix wrong order of checking node type * fix wrong order of checking node type * rebase to new master * make linter happy * rebase to new master * place tombstone only if acc has storage * rebase master * rebase master * rebase master * rebase master Co-authored-by: alex.sharov <alex.sharov@lazada.com>
2020-03-11 10:31:49 +00:00
func (dsw *DbStateWriter) CreateContract(address common.Address) (uint64, error) {
if _, err := dsw.csw.CreateContract(address); err != nil {
return 0, err
}
if incarnation, ok := dsw.incarnations[address]; ok {
return incarnation, nil
}
addrHash, err := dsw.pw.HashAddress(address, false /*save*/)
if err != nil {
return 0, err
}
return dsw.nextIncarnation(addrHash)
}
// nextIncarnation determines what should be the next incarnation of an account (i.e. how many time it has existed before at this address)
func (dsw *DbStateWriter) nextIncarnation(addrHash common.Hash) (uint64, error) {
var found bool
var incarnationBytes [common.IncarnationLength]byte
startkey := make([]byte, common.HashLength+common.IncarnationLength+common.HashLength)
var fixedbits uint = 8 * common.HashLength
copy(startkey, addrHash[:])
if err := dsw.db.Walk(dbutils.CurrentStateBucket, startkey, fixedbits, func(k, v []byte) (bool, error) {
copy(incarnationBytes[:], k[common.HashLength:])
found = true
return false, nil
}); err != nil {
return 0, err
}
if found {
return (^binary.BigEndian.Uint64(incarnationBytes[:])) + 1, nil
}
return FirstContractIncarnation, nil
}
// WriteChangeSets causes accumulated change sets to be written into
// the database (or batch) associated with the `dsw`
func (dsw *DbStateWriter) WriteChangeSets() error {
accountChanges, err := dsw.csw.GetAccountChanges()
if err != nil {
return err
}
var accountSerialised []byte
accountSerialised, err = changeset.EncodeAccounts(accountChanges)
if err != nil {
return err
}
key := dbutils.EncodeTimestamp(dsw.blockNr)
if err = dsw.db.Put(dbutils.AccountChangeSetBucket, key, accountSerialised); err != nil {
return err
}
storageChanges, err := dsw.csw.GetStorageChanges()
if err != nil {
return err
}
var storageSerialized []byte
if storageChanges.Len() > 0 {
storageSerialized, err = changeset.EncodeStorage(storageChanges)
if err != nil {
return err
}
if err = dsw.db.Put(dbutils.StorageChangeSetBucket, key, storageSerialized); err != nil {
return err
}
}
return nil
}
func (dsw *DbStateWriter) WriteHistory() error {
accountChanges, err := dsw.csw.GetAccountChanges()
if err != nil {
return err
}
err = dsw.writeIndex(accountChanges, dbutils.AccountsHistoryBucket)
if err != nil {
return err
}
storageChanges, err := dsw.csw.GetStorageChanges()
if err != nil {
return err
}
err = dsw.writeIndex(storageChanges, dbutils.StorageHistoryBucket)
if err != nil {
return err
}
return nil
}
func (dsw *DbStateWriter) writeIndex(changes *changeset.ChangeSet, bucket []byte) error {
for _, change := range changes.Changes {
currentChunkKey := dbutils.IndexChunkKey(change.Key, ^uint64(0))
indexBytes, err := dsw.db.Get(bucket, currentChunkKey)
if err != nil && err != ethdb.ErrKeyNotFound {
return fmt.Errorf("find chunk failed: %w", err)
}
v := dsw.blockNr
var index dbutils.HistoryIndexBytes
if len(indexBytes) == 0 {
index = dbutils.NewHistoryIndex()
} else if dbutils.CheckNewIndexChunk(indexBytes, v) {
// Chunk overflow, need to write the "old" current chunk under its key derived from the last element
index = dbutils.WrapHistoryIndex(indexBytes)
indexKey, err := index.Key(change.Key)
if err != nil {
return err
}
// Flush the old chunk
if err := dsw.db.Put(bucket, indexKey, index); err != nil {
return err
}
// Start a new chunk
index = dbutils.NewHistoryIndex()
} else {
index = dbutils.WrapHistoryIndex(indexBytes)
}
index = index.Append(v, len(change.Value) == 0)
if err := dsw.db.Put(bucket, currentChunkKey, index); err != nil {
return err
}
}
return nil
}