erigon-pulse/common/changeset/storage_changeset.go

97 lines
2.9 KiB
Go
Raw Normal View History

package changeset
import (
"errors"
Merge account and storage resolvers (#504) * add_incarnation_to_acc_root_in_ih * merge cached resolver into stateful resolver * - move account root set to "post iteration" of resolver - rename "cache" to IntermediateHash * remove blockNR and bucket params from walker * fix out of range panic * calc acc.Root on the fly * remove fieldSet field from resolver, make logic of root - lazy * remove 2 parameters * working version of forward-only walk over Acc and Storage * improve test * rebase master * save progress - more tests for PrepareResolveParams, add dedicated ResolveSet for storage. Problem: See duplicates in ResolveSet hexes. Next test failing: oracle_test.go * skip old incarnations * don't rebuild when 0 requests * fix tests * start from account key when need resolve storage * Error: stateless prototype faced hashNode when extracting witness * Statless works: copy touches * Remove getAccRoot function * Remove "isAccount" parameter from resolver signature * Fix: use correct storageResolveSet in finaliseStorageRoot * Fix: when startKey changed - reset storage buffers also * Fix: if account incarnation=0 - set EmptyRoot * Fix: remove account roots by default from IntermediateHash bucket * Fix: skip abandoned storage - which appeared just after startKey * Fix: did reset acc key incorrectly * Fix: clean previous key if receive IH * Fix: IH observer - subscribe only to branch nodes (was subscribed to value nodes also) * Add DISABLE_IH and STORE_ACCOUNT_ROOT env variables for tests * Remove accNode from IH cycle * Fix flags * Fix: reset succStorage also * Fix: skip IH if it has wrong incarnation * Fix: skip IH if it has wrong incarnation * Fix: skip IH if it has wrong incarnation * Fix: use rssStorage to HashOnly check * Fix: remove termination symbol from resolveRequest * cleanup * Fix: skip abandoned storage after IH * Debug This reverts commit 9c5eb69465f25607d546b03359b2cbcb1bd46689. * Fix linters * add_incarnation_to_acc_root_in_ih * merge cached resolver into stateful resolver * - move account root set to "post iteration" of resolver - rename "cache" to IntermediateHash * remove blockNR and bucket params from walker * fix out of range panic * calc acc.Root on the fly * remove fieldSet field from resolver, make logic of root - lazy * remove 2 parameters * working version of forward-only walk over Acc and Storage * improve test * rebase master * save progress - more tests for PrepareResolveParams, add dedicated ResolveSet for storage. Problem: See duplicates in ResolveSet hexes. Next test failing: oracle_test.go * skip old incarnations * don't rebuild when 0 requests * fix tests * start from account key when need resolve storage * Error: stateless prototype faced hashNode when extracting witness * Statless works: copy touches * Remove getAccRoot function * Remove "isAccount" parameter from resolver signature * Fix: use correct storageResolveSet in finaliseStorageRoot * Fix: when startKey changed - reset storage buffers also * Fix: if account incarnation=0 - set EmptyRoot * Fix: remove account roots by default from IntermediateHash bucket * Fix: skip abandoned storage - which appeared just after startKey * Fix: did reset acc key incorrectly * Fix: clean previous key if receive IH * Fix: IH observer - subscribe only to branch nodes (was subscribed to value nodes also) * Add DISABLE_IH and STORE_ACCOUNT_ROOT env variables for tests * Remove accNode from IH cycle * Fix flags * Fix: reset succStorage also * Fix: skip IH if it has wrong incarnation * Fix: skip IH if it has wrong incarnation * Fix: skip IH if it has wrong incarnation * Fix: use rssStorage to HashOnly check * Fix: remove termination symbol from resolveRequest * cleanup * Fix: skip abandoned storage after IH * remove inc * remove inc from rss * tr.succStorage.Reset() * remove inc from rss * Remove hard-coding * succ.Reset * Enable CalcTrieRoots * Proper dumping of the trie * Debug * Fix for CalcTrieRoot * Fix another inteference bug * Temp * Fix test * Cleanup * remove STORE_ACCOUNT_ROOT=true flag * Fix linter * Fix linter * Disable getnodedata by default * Fix test * Fix test Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-05-02 18:00:57 +00:00
"github.com/ledgerwatch/turbo-geth/common"
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
"github.com/ledgerwatch/turbo-geth/common/dbutils"
2021-04-26 09:41:31 +00:00
"github.com/ledgerwatch/turbo-geth/common/etl"
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
"github.com/ledgerwatch/turbo-geth/ethdb"
)
const (
DefaultIncarnation = uint64(1)
)
var (
ErrNotFound = errors.New("not found")
ErrFindValue = errors.New("find value error")
)
2020-02-26 22:36:34 +00:00
/* Plain changesets (key is a common.Address) */
2020-02-26 22:36:34 +00:00
func NewStorageChangeSetPlain() *ChangeSet {
return &ChangeSet{
Changes: make([]Change, 0),
keyLen: common.AddressLength + common.HashLength + common.IncarnationLength,
}
}
2020-02-26 22:36:34 +00:00
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
func EncodeStoragePlain(blockN uint64, s *ChangeSet, f func(k, v []byte) error) error {
return encodeStorage2(blockN, s, common.AddressLength, f)
}
type StorageChangeSetPlain struct{ c ethdb.CursorDupSort }
func (b StorageChangeSetPlain) Find(blockNumber uint64, k []byte) ([]byte, error) {
return findWithoutIncarnationInStorageChangeSet2(b.c, blockNumber, common.AddressLength, k[:common.AddressLength], k[common.AddressLength:])
}
func (b StorageChangeSetPlain) FindWithIncarnation(blockNumber uint64, k []byte) ([]byte, error) {
return findInStorageChangeSet2(b.c, blockNumber, common.AddressLength, k)
}
func (b StorageChangeSetPlain) FindWithoutIncarnation(blockNumber uint64, addressToFind []byte, keyToFind []byte) ([]byte, error) {
return findWithoutIncarnationInStorageChangeSet2(b.c, blockNumber, common.AddressLength, addressToFind, keyToFind)
}
2020-02-26 22:36:34 +00:00
// RewindDataPlain generates rewind data for all plain buckets between the timestamp
// timestapSrc is the current timestamp, and timestamp Dst is where we rewind
2021-04-26 09:41:31 +00:00
func RewindData(db ethdb.Tx, timestampSrc, timestampDst uint64, tmpdir string, quit <-chan struct{}) (*etl.Collector, error) {
// Collect list of buckets and keys that need to be considered
2021-04-26 09:41:31 +00:00
changes := etl.NewCollector(tmpdir, etl.NewOldestEntryBuffer(etl.BufferOptimalSize))
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
if err := walkAndCollect(
2021-04-26 09:41:31 +00:00
changes.Collect,
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
db, dbutils.PlainAccountChangeSetBucket,
timestampDst+1, timestampSrc,
quit,
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
); err != nil {
2021-04-26 09:41:31 +00:00
return nil, err
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
}
if err := walkAndCollect(
2021-04-26 09:41:31 +00:00
changes.Collect,
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
db, dbutils.PlainStorageChangeSetBucket,
timestampDst+1, timestampSrc,
quit,
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
); err != nil {
2021-04-26 09:41:31 +00:00
return nil, err
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
}
2021-04-26 09:41:31 +00:00
return changes, nil
}
2020-02-28 19:48:52 +00:00
2021-04-02 11:22:25 +00:00
func walkAndCollect(collectorFunc func([]byte, []byte) error, db ethdb.Tx, bucket string, timestampDst, timestampSrc uint64, quit <-chan struct{}) error {
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
fromDBFormat := FromDBFormat(Mapper[bucket].KeySize)
2021-04-02 11:22:25 +00:00
c, err := db.Cursor(bucket)
if err != nil {
return err
}
defer c.Close()
return ethdb.Walk(c, dbutils.EncodeBlockNumber(timestampDst), 0, func(dbKey, dbValue []byte) (bool, error) {
if err := common.Stopped(quit); err != nil {
return false, err
}
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
timestamp, k, v := fromDBFormat(dbKey, dbValue)
if timestamp > timestampSrc {
return false, nil
}
if innerErr := collectorFunc(common.CopyBytes(k), common.CopyBytes(v)); innerErr != nil {
return false, innerErr
}
return true, nil
})
}