diff --git a/cmd/rpcdaemon/commands/erigon_receipts.go b/cmd/rpcdaemon/commands/erigon_receipts.go index fe0e55861..e022d5377 100644 --- a/cmd/rpcdaemon/commands/erigon_receipts.go +++ b/cmd/rpcdaemon/commands/erigon_receipts.go @@ -101,23 +101,9 @@ func (api *ErigonImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) } blockNumbers := bitmapdb.NewBitmap() defer bitmapdb.ReturnToPool(blockNumbers) - blockNumbers.AddRange(begin, end+1) // [min,max) - - topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, begin, end) - if err != nil { + if err := applyFilters(blockNumbers, tx, begin, end, crit); err != nil { return nil, err } - if topicsBitmap != nil { - blockNumbers.And(topicsBitmap) - } - addrBitmap, err := getAddrsBitmap(tx, crit.Addresses, begin, end) - if err != nil { - return nil, err - } - if addrBitmap != nil { - blockNumbers.And(addrBitmap) - } - if blockNumbers.IsEmpty() { return erigonLogs, nil } @@ -128,7 +114,7 @@ func (api *ErigonImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) } iter := blockNumbers.Iterator() for iter.HasNext() { - if err = ctx.Err(); err != nil { + if err := ctx.Err(); err != nil { return nil, err } @@ -250,20 +236,8 @@ func (api *ErigonImpl) GetLatestLogs(ctx context.Context, crit filters.FilterCri blockNumbers := bitmapdb.NewBitmap() defer bitmapdb.ReturnToPool(blockNumbers) - blockNumbers.AddRange(0, latest) - topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, 0, latest) - if err != nil { - return nil, err - } - if topicsBitmap != nil { - blockNumbers.And(topicsBitmap) - } - addrBitmap, err := getAddrsBitmap(tx, crit.Addresses, 0, latest) - if err != nil { - return nil, err - } - if addrBitmap != nil { - blockNumbers.And(addrBitmap) + if err := applyFilters(blockNumbers, tx, 0, latest, crit); err != nil { + return erigonLogs, err } if blockNumbers.IsEmpty() { return erigonLogs, nil diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index 7530b3957..787968b7d 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -136,22 +136,9 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t blockNumbers := bitmapdb.NewBitmap() defer bitmapdb.ReturnToPool(blockNumbers) - blockNumbers.AddRange(begin, end+1) // [min,max) - topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, begin, end) - if err != nil { - return nil, err + if err := applyFilters(blockNumbers, tx, begin, end, crit); err != nil { + return logs, err } - if topicsBitmap != nil { - blockNumbers.And(topicsBitmap) - } - addrBitmap, err := getAddrsBitmap(tx, crit.Addresses, begin, end) - if err != nil { - return nil, err - } - if addrBitmap != nil { - blockNumbers.And(addrBitmap) - } - if blockNumbers.IsEmpty() { return logs, nil } @@ -161,7 +148,7 @@ func (api *APIImpl) GetLogs(ctx context.Context, crit filters.FilterCriteria) (t } iter := blockNumbers.Iterator() for iter.HasNext() { - if err = ctx.Err(); err != nil { + if err := ctx.Err(); err != nil { return nil, err } @@ -284,6 +271,25 @@ func getAddrsBitmap(tx kv.Tx, addrs []common.Address, from, to uint64) (*roaring return roaring.FastOr(rx...), nil } +func applyFilters(out *roaring.Bitmap, tx kv.Tx, begin, end uint64, crit filters.FilterCriteria) error { + out.AddRange(begin, end+1) // [from,to) + topicsBitmap, err := getTopicsBitmap(tx, crit.Topics, begin, end) + if err != nil { + return err + } + if topicsBitmap != nil { + out.And(topicsBitmap) + } + addrBitmap, err := getAddrsBitmap(tx, crit.Addresses, begin, end) + if err != nil { + return err + } + if addrBitmap != nil { + out.And(addrBitmap) + } + return nil +} + func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) ([]*types.Log, error) { logs := []*types.Log{} diff --git a/cmd/rpcdaemon/commands/trace_filtering.go b/cmd/rpcdaemon/commands/trace_filtering.go index a54add394..a27d2b6bd 100644 --- a/cmd/rpcdaemon/commands/trace_filtering.go +++ b/cmd/rpcdaemon/commands/trace_filtering.go @@ -233,6 +233,59 @@ func (api *TraceAPIImpl) Block(ctx context.Context, blockNr rpc.BlockNumber) (Pa return out, err } +func traceFilterBitmaps(tx kv.Tx, req TraceFilterRequest, fromBlock, toBlock uint64) (fromAddresses, toAddresses map[common.Address]struct{}, allBlocks *roaring64.Bitmap, err error) { + fromAddresses = make(map[common.Address]struct{}, len(req.FromAddress)) + toAddresses = make(map[common.Address]struct{}, len(req.ToAddress)) + allBlocks = roaring64.New() + var blocksTo roaring64.Bitmap + for _, addr := range req.FromAddress { + if addr != nil { + b, err := bitmapdb.Get64(tx, kv.CallFromIndex, addr.Bytes(), fromBlock, toBlock) + if err != nil { + if errors.Is(err, ethdb.ErrKeyNotFound) { + continue + } + return nil, nil, nil, err + } + allBlocks.Or(b) + fromAddresses[*addr] = struct{}{} + } + } + + for _, addr := range req.ToAddress { + if addr != nil { + b, err := bitmapdb.Get64(tx, kv.CallToIndex, addr.Bytes(), fromBlock, toBlock) + if err != nil { + if errors.Is(err, ethdb.ErrKeyNotFound) { + continue + } + return nil, nil, nil, err + } + blocksTo.Or(b) + toAddresses[*addr] = struct{}{} + } + } + + switch req.Mode { + case TraceFilterModeIntersection: + allBlocks.And(&blocksTo) + case TraceFilterModeUnion: + fallthrough + default: + allBlocks.Or(&blocksTo) + } + + // Special case - if no addresses specified, take all traces + if len(req.FromAddress) == 0 && len(req.ToAddress) == 0 { + allBlocks.AddRange(fromBlock, toBlock) + } else { + allBlocks.RemoveRange(0, fromBlock) + allBlocks.RemoveRange(toBlock, uint64(0x100000000)) + } + + return fromAddresses, toAddresses, allBlocks, nil +} + // Filter implements trace_filter // NOTE: We do not store full traces - we just store index for each address // Pull blocks which have txs with matching address @@ -265,60 +318,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str return api.filterV3(ctx, dbtx.(kv.TemporalTx), fromBlock, toBlock, req, stream) } toBlock++ //+1 because internally Erigon using semantic [from, to), but some RPC have different semantic - - fromAddresses := make(map[common.Address]struct{}, len(req.FromAddress)) - toAddresses := make(map[common.Address]struct{}, len(req.ToAddress)) - - var ( - allBlocks roaring64.Bitmap - blocksTo roaring64.Bitmap - ) - - for _, addr := range req.FromAddress { - if addr != nil { - b, err := bitmapdb.Get64(dbtx, kv.CallFromIndex, addr.Bytes(), fromBlock, toBlock) - if err != nil { - if errors.Is(err, ethdb.ErrKeyNotFound) { - continue - } - return err - } - allBlocks.Or(b) - fromAddresses[*addr] = struct{}{} - } - } - - for _, addr := range req.ToAddress { - if addr != nil { - b, err := bitmapdb.Get64(dbtx, kv.CallToIndex, addr.Bytes(), fromBlock, toBlock) - if err != nil { - if errors.Is(err, ethdb.ErrKeyNotFound) { - continue - } - - return err - } - blocksTo.Or(b) - toAddresses[*addr] = struct{}{} - } - } - - switch req.Mode { - case TraceFilterModeIntersection: - allBlocks.And(&blocksTo) - case TraceFilterModeUnion: - fallthrough - default: - allBlocks.Or(&blocksTo) - } - - // Special case - if no addresses specified, take all traces - if len(req.FromAddress) == 0 && len(req.ToAddress) == 0 { - allBlocks.AddRange(fromBlock, toBlock) - } else { - allBlocks.RemoveRange(0, fromBlock) - allBlocks.RemoveRange(toBlock, uint64(0x100000000)) - } + fromAddresses, toAddresses, allBlocks, err := traceFilterBitmaps(dbtx, req, fromBlock, toBlock) chainConfig, err := api.chainConfig(dbtx) if err != nil { diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index d44d0c506..41fd714aa 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/roaring64" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" @@ -268,23 +270,41 @@ func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64) ( return nil, fmt.Errorf("unexpected history name: %s", name) } } else { - var table string + var bm *roaring64.Bitmap switch name { case LogTopicIdx: - table = kv.LogTopicIndex + bm32, err := bitmapdb.Get(tx, kv.LogTopicIndex, key, uint32(fromTs), uint32(toTs)) + if err != nil { + return nil, err + } + bm = castBitmapTo64(bm32) case LogAddrIdx: - table = kv.LogAddressIdx + bm32, err := bitmapdb.Get(tx, kv.LogAddressIndex, key, uint32(fromTs), uint32(toTs)) + if err != nil { + return nil, err + } + bm = castBitmapTo64(bm32) case TracesFromIdx: - table = kv.TracesFromIdx + bm, err = bitmapdb.Get64(tx, kv.TracesFromIdx, key, fromTs, toTs) + if err != nil { + return nil, err + } case TracesToIdx: - table = kv.TracesToIdx + bm, err = bitmapdb.Get64(tx, kv.CallToIndex, key, fromTs, toTs) + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("unexpected history name: %s", name) } - bm, err := bitmapdb.Get64(tx, table, key, fromTs, toTs) - if err != nil { - return nil, err - } return kv.StreamArray(bm.ToArray()), nil } } + +func castBitmapTo64(in *roaring.Bitmap) *roaring64.Bitmap { + bm := roaring64.New() + for _, v := range in.ToArray() { + bm.Add(uint64(v)) + } + return bm +}