mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

e3: getLogs on iterators (#6683)

Browse Source
This commit is contained in:
Alex Sharov 2023-01-25 16:29:41 +07:00 committed by GitHub
parent 04253eda42
commit 82c478a419
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 184 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/erigon-el/backend/backend.go
View File

@ -888,7 +888,7 @@ func (s *Ethereum) setUpBlockReader(ctx context.Context, dirs datadir.Dirs, snCo
}
dir.MustExist(dirs.SnapHistory)
agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB)
agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB)
if err != nil {
return nil, nil, nil, err
}

2
cmd/integration/commands/reset_state.go
View File

@ -7,9 +7,9 @@ import (
"text/tabwriter"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/erigon/core/rawdb/rawdbhelpers"
reset2 "github.com/ledgerwatch/erigon/core/rawdb/rawdbreset"

6
cmd/integration/commands/stages.go
View File

@ -13,9 +13,9 @@ import (
"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
libstate "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb"
"github.com/ledgerwatch/erigon/cmd/sentry/sentry"
@ -487,7 +487,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context) error {
if reset {
dirs := datadir.New(datadirCli)
if err := reset2.ResetBlocks(tx, db, sn, br, dirs, *chainConfig, engine); err != nil {
if err := reset2.ResetBlocks(tx, db, sn, agg, br, dirs, *chainConfig, engine); err != nil {
return err
}
return nil
@ -1149,7 +1149,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB) (*snapshotsync.RoSnapshots, *
_allSnapshotsSingleton = snapshotsync.NewRoSnapshots(snapCfg, dirs.Snap)
var err error
_aggSingleton, err = libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
_aggSingleton, err = libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
if err != nil {
panic(err)
}

7
cmd/rpcdaemon/cli/config.go
View File

@ -17,13 +17,12 @@ import (
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
libstate "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/erigon/core/systemcontracts"
"github.com/ledgerwatch/erigon/core/state/historyv2read"
"github.com/ledgerwatch/erigon/core/state/temporal"
"github.com/ledgerwatch/erigon/core/systemcontracts"
"github.com/ledgerwatch/erigon/core/types/accounts"
"github.com/ledgerwatch/erigon/eth/ethconfig"
"github.com/ledgerwatch/erigon/rpc/rpccfg"
@ -350,7 +349,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger,
allSnapshots.OptimisticReopenWithDB(db)
allSnapshots.LogStat()
if agg, err = libstate.NewAggregator22(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil {
if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil {
return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("create aggregator: %w", err)
}
_ = agg.ReopenFiles()

2
cmd/rpcdaemon/commands/debug_api.go
View File

@ -6,8 +6,8 @@ import (
jsoniter "github.com/json-iterator/go"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/common/changeset"
"github.com/ledgerwatch/erigon/common/hexutil"

2
cmd/rpcdaemon/commands/erigon_receipts_test.go
View File

@ -57,7 +57,7 @@ func TestGetLogs(t *testing.T) {
//
//api := NewErigonAPI(baseApi, m.DB, nil)
//logs, err := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)})
//assert.NoError(err)
//assert.NoError(err)s
//assert.Equal(uint64(10), logs[0].BlockNumber)
//
////filer by wrong address

115
cmd/rpcdaemon/commands/eth_receipts.go
View File

@ -8,33 +8,30 @@ import (
"math/big"
"github.com/RoaringBitmap/roaring"
"github.com/RoaringBitmap/roaring/roaring64"
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/bitmapdb"
"github.com/ledgerwatch/erigon-lib/kv/iter"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/log/v3"
"github.com/ledgerwatch/erigon/consensus"
"github.com/ledgerwatch/erigon/core/vm/evmtypes"
"github.com/ledgerwatch/erigon/turbo/services"
"github.com/ledgerwatch/erigon/core/state/temporal"
"github.com/ledgerwatch/erigon/common/hexutil"
"github.com/ledgerwatch/erigon/consensus"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/state"
"github.com/ledgerwatch/erigon/core/state/temporal"
"github.com/ledgerwatch/erigon/core/types"
"github.com/ledgerwatch/erigon/core/vm"
"github.com/ledgerwatch/erigon/core/vm/evmtypes"
"github.com/ledgerwatch/erigon/eth/filters"
"github.com/ledgerwatch/erigon/ethdb/cbor"
"github.com/ledgerwatch/erigon/rpc"
"github.com/ledgerwatch/erigon/turbo/rpchelper"
"github.com/ledgerwatch/erigon/turbo/services"
"github.com/ledgerwatch/erigon/turbo/transactions"
)
@ -301,6 +298,8 @@ func applyFilters(out *roaring.Bitmap, tx kv.Tx, begin, end uint64, crit filters
return nil
}
/*
func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) error {
//[from,to)
var fromTxNum, toTxNum uint64
@ -334,18 +333,54 @@ func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64,
}
return nil
}
*/
func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) (out iter.U64, err error) {
//[from,to)
var fromTxNum, toTxNum uint64
if begin > 0 {
fromTxNum, err = rawdbv3.TxNums.Min(tx, begin)
if err != nil {
return out, err
}
}
toTxNum, err = rawdbv3.TxNums.Max(tx, end)
if err != nil {
return out, err
}
toTxNum++
topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum)
if err != nil {
return out, err
}
if topicsBitmap != nil {
out = topicsBitmap
}
addrBitmap, err := getAddrsBitmapV3(tx, crit.Addresses, fromTxNum, toTxNum)
if err != nil {
return out, err
}
if addrBitmap != nil {
if out == nil {
out = addrBitmap
} else {
out = iter.Intersect[uint64](out, addrBitmap)
}
}
if out == nil {
out = iter.Range[uint64](fromTxNum, toTxNum)
}
return out, nil
}
func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) ([]*types.Log, error) {
logs := []*types.Log{}
txNumbers := bitmapdb.NewBitmap64()
defer bitmapdb.ReturnToPool64(txNumbers)
if err := applyFiltersV3(txNumbers, tx, begin, end, crit); err != nil {
txNumbers, err := applyFiltersV3(tx, begin, end, crit)
if err != nil {
return logs, err
}
if txNumbers.IsEmpty() {
return logs, nil
}
addrMap := make(map[libcommon.Address]struct{}, len(crit.Addresses))
for _, v := range crit.Addresses {
@ -356,12 +391,12 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end
if err != nil {
return nil, err
}
exec := newIntraBlockExec(tx, chainConfig, api.engine())
exec := newIntraBlockExec(tx, chainConfig, api.engine(), api._blockReader)
var blockHash libcommon.Hash
var header *types.Header
iter := MapTxNum2BlockNum(tx, bitmapdb.ToIter(txNumbers.Iterator()))
iter := MapTxNum2BlockNum(tx, txNumbers)
for iter.HasNext() {
if err = ctx.Err(); err != nil {
return nil, err
@ -380,7 +415,7 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end
return nil, err
}
if header == nil {
log.Warn("header is nil", "blockNum", blockNum)
log.Warn("[rpc] header is nil", "blockNum", blockNum)
continue
}
blockHash = header.Hash()
@ -439,12 +474,13 @@ type intraBlockExec struct {
vmConfig *vm.Config
}
func newIntraBlockExec(tx kv.TemporalTx, chainConfig *chain.Config, engine consensus.EngineReader) *intraBlockExec {
func newIntraBlockExec(tx kv.TemporalTx, chainConfig *chain.Config, engine consensus.EngineReader, br services.FullBlockReader) *intraBlockExec {
stateReader := state.NewHistoryReaderV3()
stateReader.SetTx(tx)
return &intraBlockExec{
engine: engine,
chainConfig: chainConfig,
br: br,
stateReader: stateReader,
evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}),
vmConfig: &vm.Config{},
@ -492,6 +528,48 @@ func (e *intraBlockExec) execTx(txNum uint64, txIndex int, txn types.Transaction
// {{}, {B}} matches any topic in first position AND B in second position
// {{A}, {B}} matches topic A in first position AND B in second position
// {{A, B}, {C, D}} matches topic (A OR B) in first position AND (C OR D) in second position
func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (iter.U64, error) {
var result iter.U64
for _, sub := range topics {
var bitmapForORing iter.U64 = iter.Array[uint64]([]uint64{})
for _, topic := range sub {
it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to, true, -1)
if err != nil {
return nil, err
}
bitmapForORing = iter.Union[uint64](bitmapForORing, it)
}
if result == nil {
result = bitmapForORing
continue
}
result = iter.Intersect[uint64](result, bitmapForORing)
}
return result, nil
}
func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint64) (iter.U64, error) {
if len(addrs) == 0 {
return nil, nil
}
var rx iter.U64
for _, addr := range addrs {
it, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], from, to, true, -1)
if err != nil {
return nil, err
}
if rx == nil {
rx = it
} else {
rx = iter.Union[uint64](rx, it)
}
}
return rx, nil
}
/*
func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (*roaring64.Bitmap, error) {
var result *roaring64.Bitmap
for _, sub := range topics {
@ -544,6 +622,7 @@ func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint
}
return roaring64.FastOr(rx...), nil
}
*/
// GetTransactionReceipt implements eth_getTransactionReceipt. Returns the receipt of a transaction given the transaction's hash.
func (api *APIImpl) GetTransactionReceipt(ctx context.Context, txnHash libcommon.Hash) (map[string]interface{}, error) {

6
cmd/rpcdaemon/commands/otterscan_api.go
View File

@ -10,9 +10,9 @@ import (
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/iter"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/core/state/temporal"
"github.com/ledgerwatch/log/v3"
@ -277,7 +277,7 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co
txNums := iter.Union[uint64](itFrom, itTo)
txNumsIter := MapDescendTxNum2BlockNum(tx, txNums)
exec := newIntraBlockExec(tx, chainConfig, api.engine())
exec := newIntraBlockExec(tx, chainConfig, api.engine(), api._blockReader)
var blockHash libcommon.Hash
var header *types.Header
txs := make([]*RPCTransaction, 0, pageSize)
@ -298,7 +298,7 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co
return nil, err
}
if header == nil {
log.Warn("header is nil", "blockNum", blockNum)
log.Warn("[rpc] header is nil", "blockNum", blockNum)
continue
}
blockHash = header.Hash()

2
cmd/rpcdaemon/commands/trace_filtering.go
View File

@ -10,9 +10,9 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/bitmapdb"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/hexutil"
"github.com/ledgerwatch/erigon/consensus/ethash"

4
cmd/state/exec3/state.go
View File

@ -30,8 +30,8 @@ type Worker struct {
background bool // if true - worker does manage RoTx (begin/rollback) in .ResetTx()
blockReader services.FullBlockReader
rs *state.StateV3
stateWriter *state.StateWriter22
stateReader *state.StateReader22
stateWriter *state.StateWriterV3
stateReader *state.StateReaderV3
chainConfig *chain.Config
getHeader func(hash libcommon.Hash, number uint64) *types.Header

4
core/chain_makers.go
View File

@ -394,7 +394,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E
c.Close()
if GenerateTrace {
fmt.Printf("State after %d================\n", b.header.Number)
it, err := tx.Stream(kv.HashedAccounts, nil, nil)
it, err := tx.Range(kv.HashedAccounts, nil, nil)
if err != nil {
return nil, nil, err
}
@ -406,7 +406,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E
fmt.Printf("%x: %x\n", k, v)
}
fmt.Printf("..................\n")
it, err = tx.Stream(kv.HashedStorage, nil, nil)
it, err = tx.Range(kv.HashedStorage, nil, nil)
if err != nil {
return nil, nil, err
}

7
core/genesis.go
View File

@ -32,12 +32,9 @@ import (
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/mdbx"
"github.com/ledgerwatch/log/v3"
"golang.org/x/exp/slices"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/hexutil"
"github.com/ledgerwatch/erigon/common/math"
@ -50,6 +47,8 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/params/networkname"
"github.com/ledgerwatch/erigon/turbo/trie"
"github.com/ledgerwatch/log/v3"
"golang.org/x/exp/slices"
)
//go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go

5
core/rawdb/rawdbreset/reset_stages.go
View File

@ -12,6 +12,7 @@ import (
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"
"go.uber.org/atomic"
"golang.org/x/sync/errgroup"
@ -55,7 +56,7 @@ func ResetState(db kv.RwDB, ctx context.Context, chain string) error {
return nil
}
func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, br services.FullBlockReader, dirs datadir.Dirs, cc chain.Config, engine consensus.Engine) error {
func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, agg *state.AggregatorV3, br services.FullBlockReader, dirs datadir.Dirs, cc chain.Config, engine consensus.Engine) error {
// keep Genesis
if err := rawdb.TruncateBlocks(context.Background(), tx, 1); err != nil {
return err
@ -105,7 +106,7 @@ func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, br
}
if snapshots != nil && snapshots.Cfg().Enabled && snapshots.BlocksAvailable() > 0 {
if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, snapshots, br, cc, engine); err != nil {
if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, snapshots, br, cc, engine, agg); err != nil {
return err
}
_ = stages.SaveStageProgress(tx, stages.Snapshots, snapshots.BlocksAvailable())

50
core/state/rw_v3.go
View File

@ -582,7 +582,7 @@ func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool {
return true
}
type StateWriter22 struct {
type StateWriterV3 struct {
rs *StateV3
txNum uint64
writeLists map[string]*exec22.KvList
@ -592,8 +592,8 @@ type StateWriter22 struct {
codePrevs map[string]uint64
}
func NewStateWriter22(rs *StateV3) *StateWriter22 {
return &StateWriter22{
func NewStateWriter22(rs *StateV3) *StateWriterV3 {
return &StateWriterV3{
rs: rs,
writeLists: newWriteList(),
accountPrevs: map[string][]byte{},
@ -603,11 +603,11 @@ func NewStateWriter22(rs *StateV3) *StateWriter22 {
}
}
func (w *StateWriter22) SetTxNum(txNum uint64) {
func (w *StateWriterV3) SetTxNum(txNum uint64) {
w.txNum = txNum
}
func (w *StateWriter22) ResetWriteSet() {
func (w *StateWriterV3) ResetWriteSet() {
w.writeLists = newWriteList()
w.accountPrevs = map[string][]byte{}
w.accountDels = map[string]*accounts.Account{}
@ -615,15 +615,15 @@ func (w *StateWriter22) ResetWriteSet() {
w.codePrevs = map[string]uint64{}
}
func (w *StateWriter22) WriteSet() map[string]*exec22.KvList {
func (w *StateWriterV3) WriteSet() map[string]*exec22.KvList {
return w.writeLists
}
func (w *StateWriter22) PrevAndDels() (map[string][]byte, map[string]*accounts.Account, map[string][]byte, map[string]uint64) {
func (w *StateWriterV3) PrevAndDels() (map[string][]byte, map[string]*accounts.Account, map[string][]byte, map[string]uint64) {
return w.accountPrevs, w.accountDels, w.storagePrevs, w.codePrevs
}
func (w *StateWriter22) UpdateAccountData(address common2.Address, original, account *accounts.Account) error {
func (w *StateWriterV3) UpdateAccountData(address common2.Address, original, account *accounts.Account) error {
addressBytes := address.Bytes()
value := make([]byte, account.EncodingLengthForStorage())
account.EncodeForStorage(value)
@ -638,7 +638,7 @@ func (w *StateWriter22) UpdateAccountData(address common2.Address, original, acc
return nil
}
func (w *StateWriter22) UpdateAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash, code []byte) error {
func (w *StateWriterV3) UpdateAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash, code []byte) error {
addressBytes, codeHashBytes := address.Bytes(), codeHash.Bytes()
w.writeLists[kv.Code].Keys = append(w.writeLists[kv.Code].Keys, codeHashBytes)
w.writeLists[kv.Code].Vals = append(w.writeLists[kv.Code].Vals, code)
@ -651,7 +651,7 @@ func (w *StateWriter22) UpdateAccountCode(address common2.Address, incarnation u
return nil
}
func (w *StateWriter22) DeleteAccount(address common2.Address, original *accounts.Account) error {
func (w *StateWriterV3) DeleteAccount(address common2.Address, original *accounts.Account) error {
addressBytes := address.Bytes()
w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, addressBytes)
w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, []byte{})
@ -667,7 +667,7 @@ func (w *StateWriter22) DeleteAccount(address common2.Address, original *account
return nil
}
func (w *StateWriter22) WriteAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash, original, value *uint256.Int) error {
func (w *StateWriterV3) WriteAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash, original, value *uint256.Int) error {
if *original == *value {
return nil
}
@ -679,11 +679,11 @@ func (w *StateWriter22) WriteAccountStorage(address common2.Address, incarnation
return nil
}
func (w *StateWriter22) CreateContract(address common2.Address) error {
func (w *StateWriterV3) CreateContract(address common2.Address) error {
return nil
}
type StateReader22 struct {
type StateReaderV3 struct {
tx kv.Tx
txNum uint64
trace bool
@ -692,34 +692,34 @@ type StateReader22 struct {
readLists map[string]*exec22.KvList
}
func NewStateReader22(rs *StateV3) *StateReader22 {
return &StateReader22{
func NewStateReader22(rs *StateV3) *StateReaderV3 {
return &StateReaderV3{
rs: rs,
readLists: newReadList(),
}
}
func (r *StateReader22) SetTxNum(txNum uint64) {
func (r *StateReaderV3) SetTxNum(txNum uint64) {
r.txNum = txNum
}
func (r *StateReader22) SetTx(tx kv.Tx) {
func (r *StateReaderV3) SetTx(tx kv.Tx) {
r.tx = tx
}
func (r *StateReader22) ResetReadSet() {
func (r *StateReaderV3) ResetReadSet() {
r.readLists = newReadList()
}
func (r *StateReader22) ReadSet() map[string]*exec22.KvList {
func (r *StateReaderV3) ReadSet() map[string]*exec22.KvList {
return r.readLists
}
func (r *StateReader22) SetTrace(trace bool) {
func (r *StateReaderV3) SetTrace(trace bool) {
r.trace = trace
}
func (r *StateReader22) ReadAccountData(address common2.Address) (*accounts.Account, error) {
func (r *StateReaderV3) ReadAccountData(address common2.Address) (*accounts.Account, error) {
addr := address.Bytes()
enc := r.rs.Get(kv.PlainState, addr)
if enc == nil {
@ -745,7 +745,7 @@ func (r *StateReader22) ReadAccountData(address common2.Address) (*accounts.Acco
return &a, nil
}
func (r *StateReader22) ReadAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash) ([]byte, error) {
func (r *StateReaderV3) ReadAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash) ([]byte, error) {
composite := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes())
enc := r.rs.Get(kv.PlainState, composite)
if enc == nil {
@ -770,7 +770,7 @@ func (r *StateReader22) ReadAccountStorage(address common2.Address, incarnation
return enc, nil
}
func (r *StateReader22) ReadAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash) ([]byte, error) {
func (r *StateReaderV3) ReadAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash) ([]byte, error) {
addr, codeHashBytes := address.Bytes(), codeHash.Bytes()
enc := r.rs.Get(kv.Code, codeHashBytes)
if enc == nil {
@ -788,7 +788,7 @@ func (r *StateReader22) ReadAccountCode(address common2.Address, incarnation uin
return enc, nil
}
func (r *StateReader22) ReadAccountCodeSize(address common2.Address, incarnation uint64, codeHash common2.Hash) (int, error) {
func (r *StateReaderV3) ReadAccountCodeSize(address common2.Address, incarnation uint64, codeHash common2.Hash) (int, error) {
codeHashBytes := codeHash.Bytes()
enc := r.rs.Get(kv.Code, codeHashBytes)
if enc == nil {
@ -809,7 +809,7 @@ func (r *StateReader22) ReadAccountCodeSize(address common2.Address, incarnation
return size, nil
}
func (r *StateReader22) ReadAccountIncarnation(address common2.Address) (uint64, error) {
func (r *StateReaderV3) ReadAccountIncarnation(address common2.Address) (uint64, error) {
enc := r.rs.Get(kv.IncarnationMap, address.Bytes())
if enc == nil {
var err error

21
core/state/temporal/kv_temporal.go
View File

@ -8,10 +8,11 @@ import (
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/iter"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/order"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/state"
)
@ -168,11 +169,11 @@ func (tx *Tx) DomainRangeAscend(name kv.Domain, k1, fromKey []byte, asOfTs uint6
copy(toPrefix, k1)
binary.BigEndian.PutUint64(toPrefix[length.Addr:], inc+1)
it2, err := tx.StreamAscend(kv.PlainState, startkey, toPrefix, limit)
it2, err := tx.RangeAscend(kv.PlainState, startkey, toPrefix, limit)
if err != nil {
return nil, err
}
it3 := iter.TransformPairs(it2, func(k, v []byte) ([]byte, []byte) {
it3 := iter.TransformKV(it2, func(k, v []byte) ([]byte, []byte) {
return append(append([]byte{}, k[:20]...), k[28:]...), v
})
//TODO: seems MergePairs can't handle "amount" request
@ -270,36 +271,36 @@ type Cursor struct {
hitoryV3 bool
}
func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, orderAscend bool, limit int) (timestamps iter.U64, err error) {
return tx.IndexStream(name, key, fromTs, toTs, orderAscend, limit)
func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, asc order.By, limit int) (timestamps iter.U64, err error) {
return tx.IndexStream(name, key, fromTs, toTs, asc, limit)
}
// [fromTs, toTs)
func (tx *Tx) IndexStream(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, orderAscend bool, limit int) (timestamps iter.U64, err error) {
func (tx *Tx) IndexStream(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, asc order.By, limit int) (timestamps iter.U64, err error) {
switch name {
case LogTopicIdx:
t, err := tx.agg.LogTopicIterator(key, fromTs, toTs, orderAscend, limit, tx)
t, err := tx.agg.LogTopicIterator(key, fromTs, toTs, asc, limit, tx)
if err != nil {
return nil, err
}
tx.resourcesToClose = append(tx.resourcesToClose, t)
return t, nil
case LogAddrIdx:
t, err := tx.agg.LogAddrIterator(key, fromTs, toTs, orderAscend, limit, tx)
t, err := tx.agg.LogAddrIterator(key, fromTs, toTs, asc, limit, tx)
if err != nil {
return nil, err
}
tx.resourcesToClose = append(tx.resourcesToClose, t)
return t, nil
case TracesFromIdx:
t, err := tx.agg.TraceFromIterator(key, fromTs, toTs, orderAscend, limit, tx)
t, err := tx.agg.TraceFromIterator(key, fromTs, toTs, asc, limit, tx)
if err != nil {
return nil, err
}
tx.resourcesToClose = append(tx.resourcesToClose, t)
return t, nil
case TracesToIdx:
t, err := tx.agg.TraceToIterator(key, fromTs, toTs, orderAscend, limit, tx)
t, err := tx.agg.TraceToIterator(key, fromTs, toTs, asc, limit, tx)
if err != nil {
return nil, err
}

2
eth/backend.go
View File

@ -1002,7 +1002,7 @@ func (s *Ethereum) setUpBlockReader(ctx context.Context, dirs datadir.Dirs, snCo
}
dir.MustExist(dirs.SnapHistory)
agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB)
agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB)
if err != nil {
return nil, nil, nil, err
}

2
eth/stagedsync/exec3.go
View File

@ -21,10 +21,10 @@ import (
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
libstate "github.com/ledgerwatch/erigon-lib/state"
state2 "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"

2
eth/stagedsync/stage_bodies.go
View File

@ -9,8 +9,8 @@ import (
"github.com/ledgerwatch/erigon-lib/chain"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/log/v3"
"github.com/ledgerwatch/erigon/core/rawdb"

2
eth/stagedsync/stage_execute.go
View File

@ -17,9 +17,9 @@ import (
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2"
libstate "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"

4
eth/stagedsync/stage_execute_test.go
View File

@ -7,9 +7,9 @@ import (
"testing"
"time"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/memdb"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2"
libstate "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/erigon/cmd/state/exec22"
@ -165,7 +165,7 @@ func apply(tx kv.RwTx, agg *libstate.AggregatorV3) (beforeBlock, afterBlock test
func newAgg(t *testing.T) *libstate.AggregatorV3 {
t.Helper()
dir, ctx := t.TempDir(), context.Background()
agg, err := libstate.NewAggregator22(ctx, dir, dir, ethconfig.HistoryV3AggregationStep, nil)
agg, err := libstate.NewAggregatorV3(ctx, dir, dir, ethconfig.HistoryV3AggregationStep, nil)
require.NoError(t, err)
err = agg.ReopenFiles()
require.NoError(t, err)

2
eth/stagedsync/stage_hashstate.go
View File

@ -15,9 +15,9 @@ import (
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2"
"github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"

2
eth/stagedsync/stage_interhashes.go
View File

@ -10,9 +10,9 @@ import (
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/hexutility"
"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2"
"github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"

9
eth/stagedsync/stage_snapshots.go
View File

@ -15,12 +15,12 @@ import (
"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
"github.com/ledgerwatch/erigon-lib/etl"
proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/log/v3"
"golang.org/x/sync/semaphore"
@ -183,13 +183,13 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
s.BlockNumber = blocksAvailable
}
if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.snapshots, cfg.blockReader, cfg.chainConfig, cfg.engine); err != nil {
if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.snapshots, cfg.blockReader, cfg.chainConfig, cfg.engine, cfg.agg); err != nil {
return err
}
return nil
}
func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs datadir.Dirs, sn *snapshotsync.RoSnapshots, blockReader services.FullBlockReader, chainConfig chain.Config, engine consensus.Engine) error {
func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs datadir.Dirs, sn *snapshotsync.RoSnapshots, blockReader services.FullBlockReader, chainConfig chain.Config, engine consensus.Engine, agg *state.AggregatorV3) error {
blocksAvailable := sn.BlocksAvailable()
logEvery := time.NewTicker(logInterval)
defer logEvery.Stop()
@ -327,6 +327,9 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs
return fmt.Errorf("build txNum => blockNum mapping: %w", err)
}
}
if err := rawdb.WriteSnapshots(tx, sn.Files(), agg.Files()); err != nil {
return err
}
}
}
return nil

2
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.18
require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b
github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da
github.com/ledgerwatch/log/v3 v3.7.0
github.com/ledgerwatch/secp256k1 v1.0.0

4
go.sum
View File

@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b h1:WyiyLh4CgS3ppKA8Q52x8X6hH8prIDaGx7WAGJm/sxA=
github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w=
github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7 h1:pUTs7rXO2WZTJGJ6tKvmdpOJcpF5BNTsB/9JYdB1rFI=
github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=

2
migrations/reset_blocks.go
View File

@ -113,7 +113,7 @@ var resetBlocks4 = Migration{
}
cc := tool.ChainConfig(tx)
if err := rawdbreset.ResetBlocks(tx, db, nil, nil, dirs, *cc, nil); err != nil {
if err := rawdbreset.ResetBlocks(tx, db, nil, nil, nil, dirs, *cc, nil); err != nil {
return err
}

6
turbo/app/snapshots.go
View File

@ -16,13 +16,13 @@ import (
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/compress"
"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/mdbx"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
libstate "github.com/ledgerwatch/erigon-lib/state"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/log/v3"
@ -220,7 +220,7 @@ func doIndicesCommand(cliCtx *cli.Context) error {
if err := snapshotsync.BuildMissedIndices("Indexing", ctx, dirs, *chainID, sem); err != nil {
return err
}
agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB)
agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB)
if err != nil {
return err
}
@ -348,7 +348,7 @@ func doRetireCommand(cliCtx *cli.Context) error {
}
br := snapshotsync.NewBlockRetire(estimate.CompressSnapshot.Workers(), dirs.Tmp, snapshots, db, nil, nil)
agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
if err != nil {
return err
}

2
turbo/rpchelper/helper.go
View File

@ -5,9 +5,9 @@ import (
"fmt"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/rawdbv3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcache"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/state"
"github.com/ledgerwatch/erigon/core/systemcontracts"

2
turbo/stages/mock_sentry.go
View File

@ -246,7 +246,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey
var agg *libstate.AggregatorV3
if cfg.HistoryV3 {
dir.MustExist(dirs.SnapHistory)
agg, err = libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
agg, err = libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db)
if err != nil {
panic(err)
}

4
turbo/transactions/call.go
View File

@ -117,6 +117,10 @@ func MakeHeaderGetter(requireCanonical bool, tx kv.Tx, headerReader services.Hea
log.Error("Can't get block hash by number", "number", n, "only-canonical", requireCanonical)
return libcommon.Hash{}
}
if h == nil {
log.Warn("[evm] header is nil", "blockNum", n)
return libcommon.Hash{}
}
return h.Hash()
}
}