From 896be5fbe35cef5935174f81ff79e0991f30c380 Mon Sep 17 00:00:00 2001 From: ledgerwatch Date: Sun, 16 May 2021 20:01:04 +0100 Subject: [PATCH] Trace Filter 7 (#1941) * use roaring64 * Fix bitmap finalisation Co-authored-by: Alex Sharp Co-authored-by: Alexey Sharp --- cmd/rpcdaemon/commands/trace_filtering.go | 15 +++++++++++---- eth/stagedsync/stage_call_traces.go | 10 ++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cmd/rpcdaemon/commands/trace_filtering.go b/cmd/rpcdaemon/commands/trace_filtering.go index 1d29ccbf5..10865619a 100644 --- a/cmd/rpcdaemon/commands/trace_filtering.go +++ b/cmd/rpcdaemon/commands/trace_filtering.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/roaring64" jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/common/dbutils" @@ -218,11 +218,12 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str fromAddresses := make(map[common.Address]struct{}, len(req.FromAddress)) toAddresses := make(map[common.Address]struct{}, len(req.ToAddress)) - var allBlocks roaring.Bitmap + var allBlocks roaring64.Bitmap for _, addr := range req.FromAddress { if addr != nil { - b, err := bitmapdb.Get(dbtx, dbutils.CallFromIndex, addr.Bytes(), uint32(fromBlock), uint32(toBlock)) + b, err := bitmapdb.Get64(dbtx, dbutils.CallFromIndex, addr.Bytes(), fromBlock, toBlock) if err != nil { + stream.WriteNil() return err } allBlocks.Or(b) @@ -231,8 +232,9 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str } for _, addr := range req.ToAddress { if addr != nil { - b, err := bitmapdb.Get(dbtx, dbutils.CallToIndex, addr.Bytes(), uint32(fromBlock), uint32(toBlock)) + b, err := bitmapdb.Get64(dbtx, dbutils.CallToIndex, addr.Bytes(), fromBlock, toBlock) if err != nil { + stream.WriteNil() return err } allBlocks.Or(b) @@ -244,6 +246,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str chainConfig, err := api.chainConfig(dbtx) if err != nil { + stream.WriteNil() return err } var json = jsoniter.ConfigCompatibleWithStandardLibrary @@ -262,6 +265,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str block, _, bErr := rawdb.ReadBlockWithSenders(dbtx, hash, b) if bErr != nil { + stream.WriteNil() return bErr } if block == nil { @@ -287,6 +291,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str pt.TransactionPosition = &txPosition b, err := json.Marshal(pt) if err != nil { + stream.WriteNil() return err } if first { @@ -314,6 +319,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str tr.TraceAddress = []int{} b, err := json.Marshal(tr) if err != nil { + stream.WriteNil() return err } if first { @@ -340,6 +346,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str tr.TraceAddress = []int{} b, err := json.Marshal(tr) if err != nil { + stream.WriteNil() return err } if first { diff --git a/eth/stagedsync/stage_call_traces.go b/eth/stagedsync/stage_call_traces.go index b68f7b0be..c84fac8f6 100644 --- a/eth/stagedsync/stage_call_traces.go +++ b/eth/stagedsync/stage_call_traces.go @@ -106,6 +106,7 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin if err != nil { return fmt.Errorf("%s: failed to create cursor for call traces: %w", logPrefix, err) } + defer traceCursor.Close() var k, v []byte prev := startBlock @@ -176,22 +177,23 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin return err } - if err := finaliseCallTraces(froms, tos, collectorFrom, collectorTo, logPrefix, tx, quit); err != nil { + if err := finaliseCallTraces(collectorFrom, collectorTo, logPrefix, tx, quit); err != nil { return fmt.Errorf("[%s] %w", logPrefix, err) } return nil } -func finaliseCallTraces(froms, tos map[string]*roaring64.Bitmap, collectorFrom, collectorTo *etl.Collector, logPrefix string, tx ethdb.RwTx, quit <-chan struct{}) error { +func finaliseCallTraces(collectorFrom, collectorTo *etl.Collector, logPrefix string, tx ethdb.RwTx, quit <-chan struct{}) error { var currentBitmap = roaring64.New() var buf = bytes.NewBuffer(nil) + lastChunkKey := make([]byte, 128) var loaderFunc = func(k []byte, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error { if _, err := currentBitmap.ReadFrom(bytes.NewReader(v)); err != nil { return err } - lastChunkKey := make([]byte, len(k)+4) + lastChunkKey = lastChunkKey[:len(k)+8] copy(lastChunkKey, k) - binary.BigEndian.PutUint32(lastChunkKey[len(k):], ^uint32(0)) + binary.BigEndian.PutUint64(lastChunkKey[len(k):], ^uint64(0)) lastChunkBytes, err := table.Get(lastChunkKey) if err != nil { return fmt.Errorf("find last chunk failed: %w", err)