mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-25 21:17:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Trace Filter 7 (#1941)

Browse Source 
* use roaring64

* Fix bitmap finalisation

Co-authored-by: Alex Sharp <alexsharp@Alexs-MacBook-Pro.local>
Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
This commit is contained in:
ledgerwatch 2021-05-16 20:01:04 +01:00 committed by GitHub
parent f6ac04e53e
commit 896be5fbe3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/rpcdaemon/commands/trace_filtering.go
View File

@ -4,7 +4,7 @@ import (
"context"
"fmt"
"github.com/RoaringBitmap/roaring"
"github.com/RoaringBitmap/roaring/roaring64"
jsoniter "github.com/json-iterator/go"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
@ -218,11 +218,12 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
fromAddresses := make(map[common.Address]struct{}, len(req.FromAddress))
toAddresses := make(map[common.Address]struct{}, len(req.ToAddress))
var allBlocks roaring.Bitmap
var allBlocks roaring64.Bitmap
for _, addr := range req.FromAddress {
if addr != nil {
b, err := bitmapdb.Get(dbtx, dbutils.CallFromIndex, addr.Bytes(), uint32(fromBlock), uint32(toBlock))
b, err := bitmapdb.Get64(dbtx, dbutils.CallFromIndex, addr.Bytes(), fromBlock, toBlock)
if err != nil {
stream.WriteNil()
return err
}
allBlocks.Or(b)
@ -231,8 +232,9 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
}
for _, addr := range req.ToAddress {
if addr != nil {
b, err := bitmapdb.Get(dbtx, dbutils.CallToIndex, addr.Bytes(), uint32(fromBlock), uint32(toBlock))
b, err := bitmapdb.Get64(dbtx, dbutils.CallToIndex, addr.Bytes(), fromBlock, toBlock)
if err != nil {
stream.WriteNil()
return err
}
allBlocks.Or(b)
@ -244,6 +246,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
chainConfig, err := api.chainConfig(dbtx)
if err != nil {
stream.WriteNil()
return err
}
var json = jsoniter.ConfigCompatibleWithStandardLibrary
@ -262,6 +265,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
block, _, bErr := rawdb.ReadBlockWithSenders(dbtx, hash, b)
if bErr != nil {
stream.WriteNil()
return bErr
}
if block == nil {
@ -287,6 +291,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
pt.TransactionPosition = &txPosition
b, err := json.Marshal(pt)
if err != nil {
stream.WriteNil()
return err
}
if first {
@ -314,6 +319,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
tr.TraceAddress = []int{}
b, err := json.Marshal(tr)
if err != nil {
stream.WriteNil()
return err
}
if first {
@ -340,6 +346,7 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest, str
tr.TraceAddress = []int{}
b, err := json.Marshal(tr)
if err != nil {
stream.WriteNil()
return err
}
if first {

10
eth/stagedsync/stage_call_traces.go
View File

@ -106,6 +106,7 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
if err != nil {
return fmt.Errorf("%s: failed to create cursor for call traces: %w", logPrefix, err)
}
defer traceCursor.Close()
var k, v []byte
prev := startBlock
@ -176,22 +177,23 @@ func promoteCallTraces(logPrefix string, tx ethdb.RwTx, startBlock, endBlock uin
return err
}
if err := finaliseCallTraces(froms, tos, collectorFrom, collectorTo, logPrefix, tx, quit); err != nil {
if err := finaliseCallTraces(collectorFrom, collectorTo, logPrefix, tx, quit); err != nil {
return fmt.Errorf("[%s] %w", logPrefix, err)
}
return nil
}
func finaliseCallTraces(froms, tos map[string]*roaring64.Bitmap, collectorFrom, collectorTo *etl.Collector, logPrefix string, tx ethdb.RwTx, quit <-chan struct{}) error {
func finaliseCallTraces(collectorFrom, collectorTo *etl.Collector, logPrefix string, tx ethdb.RwTx, quit <-chan struct{}) error {
var currentBitmap = roaring64.New()
var buf = bytes.NewBuffer(nil)
lastChunkKey := make([]byte, 128)
var loaderFunc = func(k []byte, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
if _, err := currentBitmap.ReadFrom(bytes.NewReader(v)); err != nil {
return err
}
lastChunkKey := make([]byte, len(k)+4)
lastChunkKey = lastChunkKey[:len(k)+8]
copy(lastChunkKey, k)
binary.BigEndian.PutUint32(lastChunkKey[len(k):], ^uint32(0))
binary.BigEndian.PutUint64(lastChunkKey[len(k):], ^uint64(0))
lastChunkBytes, err := table.Get(lastChunkKey)
if err != nil {
return fmt.Errorf("find last chunk failed: %w", err)