add eth_getFilterLogs, fix filter subscription ids (#6514)

this pr does the following:

1. adds new function to ApiImpl `GetFilterLogs` which should implement
`eth_getFilterLogs` (eth_getFilterChanges except with only logs filters)

2. changes the ID generator of rpchelper.Filters to use crypto/rand.

3. switched logs subscriptions to use the secure ID instead of number

4. changes subcription ids from an 8 byte hex string to a 16 byte

cmd/rpcdaemon/commands/eth_api.go

@@ -65,7 +65,8 @@ type EthAPI interface {
 	NewBlockFilter(_ context.Context) (string, error)
 	NewFilter(_ context.Context, crit ethFilters.FilterCriteria) (string, error)
 	UninstallFilter(_ context.Context, index string) (bool, error)
-	GetFilterChanges(_ context.Context, index string) ([]interface{}, error)
+	GetFilterChanges(_ context.Context, index string) ([]any, error)
+	GetFilterLogs(_ context.Context, index string) ([]*types.Log, error)
 
 	// Account related (see ./eth_accounts.go)
 	Accounts(ctx context.Context) ([]common.Address, error)

cmd/rpcdaemon/commands/eth_filters.go

@@ -2,11 +2,12 @@ package commands
 
 import (
 	"context"
+	"strings"
 
 	"github.com/ledgerwatch/log/v3"
 
 	"github.com/ledgerwatch/erigon/common/debug"
-	"github.com/ledgerwatch/erigon/common/hexutil"
+	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/eth/filters"
 	"github.com/ledgerwatch/erigon/rpc"
 	"github.com/ledgerwatch/erigon/turbo/rpchelper"
@@ -51,42 +52,38 @@ func (api *APIImpl) NewFilter(_ context.Context, crit filters.FilterCriteria) (s
 			api.filters.AddLogs(id, lg)
 		}
 	}()
-	return hexutil.EncodeUint64(uint64(id)), nil
+	return "0x" + string(id), nil
 }
 
 // UninstallFilter new transaction filter
-func (api *APIImpl) UninstallFilter(_ context.Context, index string) (bool, error) {
+func (api *APIImpl) UninstallFilter(_ context.Context, index string) (isDeleted bool, err error) {
 	if api.filters == nil {
 		return false, rpc.ErrNotificationsUnsupported
 	}
-	var isDeleted bool
 	// remove 0x
-	cutIndex := index
-	if len(index) >= 2 && index[0] == '0' && (index[1] == 'x' || index[1] == 'X') {
-		cutIndex = index[2:]
+	cutIndex := strings.TrimPrefix(index, "0x")
+	if ok := api.filters.UnsubscribeHeads(rpchelper.HeadsSubID(cutIndex)); ok {
+		isDeleted = true
 	}
-	isDeleted = api.filters.UnsubscribeHeads(rpchelper.HeadsSubID(cutIndex)) ||
-		api.filters.UnsubscribePendingTxs(rpchelper.PendingTxsSubID(cutIndex))
-	id, err := hexutil.DecodeUint64(index)
-	if err == nil {
-		return isDeleted || api.filters.UnsubscribeLogs(rpchelper.LogsSubID(id)), nil
+	if ok := api.filters.UnsubscribePendingTxs(rpchelper.PendingTxsSubID(cutIndex)); ok {
+		isDeleted = true
 	}
-
-	return isDeleted, nil
+	if ok := api.filters.UnsubscribeLogs(rpchelper.LogsSubID(cutIndex)); ok {
+		isDeleted = true
+	}
+	return
 }
 
-// GetFilterChanges implements eth_getFilterChanges. Polling method for a previously-created filter, which returns an array of logs which occurred since last poll.
-func (api *APIImpl) GetFilterChanges(_ context.Context, index string) ([]interface{}, error) {
+// GetFilterChanges implements eth_getFilterChanges.
+// Polling method for a previously-created filter
+// returns an array of logs, block headers, or pending transactions which occurred since last poll.
+func (api *APIImpl) GetFilterChanges(_ context.Context, index string) ([]any, error) {
 	if api.filters == nil {
 		return nil, rpc.ErrNotificationsUnsupported
 	}
-	stub := make([]interface{}, 0)
-
+	stub := make([]any, 0)
 	// remove 0x
-	cutIndex := index
-	if len(index) >= 2 && index[0] == '0' && (index[1] == 'x' || index[1] == 'X') {
-		cutIndex = index[2:]
-	}
+	cutIndex := strings.TrimPrefix(index, "0x")
 	if blocks, ok := api.filters.ReadPendingBlocks(rpchelper.HeadsSubID(cutIndex)); ok {
 		for _, v := range blocks {
 			stub = append(stub, v.Hash())
@@ -102,11 +99,7 @@ func (api *APIImpl) GetFilterChanges(_ context.Context, index string) ([]interfa
 		}
 		return stub, nil
 	}
-	id, err := hexutil.DecodeUint64(index)
-	if err != nil {
-		return stub, nil
-	}
-	if logs, ok := api.filters.ReadLogs(rpchelper.LogsSubID(id)); ok {
+	if logs, ok := api.filters.ReadLogs(rpchelper.LogsSubID(cutIndex)); ok {
 		for _, v := range logs {
 			stub = append(stub, v)
 		}
@@ -115,6 +108,21 @@ func (api *APIImpl) GetFilterChanges(_ context.Context, index string) ([]interfa
 	return stub, nil
 }
 
+// GetFilterLogs implements eth_getFilterLogs.
+// Polling method for a previously-created filter
+// returns an array of logs which occurred since last poll.
+func (api *APIImpl) GetFilterLogs(_ context.Context, index string) ([]*types.Log, error) {
+	if api.filters == nil {
+		return nil, rpc.ErrNotificationsUnsupported
+	}
+	cutIndex := strings.TrimPrefix(index, "0x")
+	logs, ok := api.filters.ReadLogs(rpchelper.LogsSubID(cutIndex))
+	if len(logs) == 0 || !ok {
+		return []*types.Log{}, nil
+	}
+	return logs, nil
+}
+
 // NewHeads send a notification each time a new (header) block is appended to the chain.
 func (api *APIImpl) NewHeads(ctx context.Context) (*rpc.Subscription, error) {
 	if api.filters == nil {
@@ -131,7 +139,6 @@ func (api *APIImpl) NewHeads(ctx context.Context) (*rpc.Subscription, error) {
 		defer debug.LogPanic()
 		headers, id := api.filters.SubscribeNewHeads(32)
 		defer api.filters.UnsubscribeHeads(id)
-
 		for {
 			select {
 			case h, ok := <-headers:

rpc/subscription.go

@@ -57,7 +57,6 @@ func randomIDGenerator() func() ID {
 	} else {
 		seed = int64(time.Now().Nanosecond())
 	}
-
 	var (
 		mu  sync.Mutex
 		rng = rand.New(rand.NewSource(seed)) // nolint: gosec

turbo/rpchelper/filter_id.go

@@ -1,7 +1,10 @@
 package rpchelper
 
 import (
-	"fmt"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"strings"
 	"sync/atomic"
 )
 
@@ -11,12 +14,24 @@ type (
 	PendingLogsSubID  SubscriptionID
 	PendingBlockSubID SubscriptionID
 	PendingTxsSubID   SubscriptionID
-	LogsSubID         uint64
+	LogsSubID         SubscriptionID
 )
 
 var globalSubscriptionId uint64
 
 func generateSubscriptionID() SubscriptionID {
-	id := atomic.AddUint64(&globalSubscriptionId, 1)
-	return SubscriptionID(fmt.Sprintf("%016x", id))
+	id := [16]byte{}
+	sb := new(strings.Builder)
+	hex := hex.NewEncoder(sb)
+	binary.LittleEndian.PutUint64(id[:], atomic.AddUint64(&globalSubscriptionId, 1))
+	// try 4 times to generate an id
+	for i := 0; i < 4; i++ {
+		_, err := rand.Read(id[8:])
+		if err == nil {
+			break
+		}
+	}
+	// if the computer has no functioning secure rand source, it will just use the incrementing number
+	hex.Write(id[:])
+	return SubscriptionID(sb.String())
 }

turbo/rpchelper/logsfilter.go

@@ -14,7 +14,6 @@ type LogsFilterAggregator struct {
 	aggLogsFilter  LogsFilter                       // Aggregation of all current log filters
 	logsFilters    *SyncMap[LogsSubID, *LogsFilter] // Filter for each subscriber, keyed by filterID
 	logsFilterLock sync.RWMutex
-	nextFilterId   LogsSubID
 }
 
 // LogsFilter is used for both representing log filter for a specific subscriber (RPC daemon usually)
@@ -44,14 +43,12 @@ func NewLogsFilterAggregator() *LogsFilterAggregator {
 			addrs:  make(map[libcommon.Address]int),
 			topics: make(map[libcommon.Hash]int),
 		},
-		logsFilters:  NewSyncMap[LogsSubID, *LogsFilter](),
-		nextFilterId: 0,
+		logsFilters: NewSyncMap[LogsSubID, *LogsFilter](),
 	}
 }
 
 func (a *LogsFilterAggregator) insertLogsFilter(sender Sub[*types2.Log]) (LogsSubID, *LogsFilter) {
-	filterId := a.nextFilterId
-	a.nextFilterId++
+	filterId := LogsSubID(generateSubscriptionID())
 	filter := &LogsFilter{addrs: map[libcommon.Address]int{}, topics: map[libcommon.Hash]int{}, sender: sender}
 	a.logsFilters.Put(filterId, filter)
 	return filterId, filter