erigon-pulse/ethdb/walk.go
Alex Sharov e02d6acc7d
bitmap indices for logs (#1124)
* save progress

* try now

* don't create bloom inside rlpDecode

* don't create bloom inside ApplyTransaction

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* rename method

* print timings

* print timings

* print timings

* sort before flush

* fix err lint

* clean

* move tests to transactions

* compressed version

* up bound

* up bound

* more tests

* more tests

* more tests

* more tests

* better removal

* clean

* better performance of get/put methods

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* optimize rpcdaemon

* fix test

* fix rpcdaemon

* fix test

* simplify

* simplify

* fix nil pointer

* clean

* revert some changes

* add some logs

* clean

* try without optimize

* clean

* clean

* clean

* clean

* try

* move log_index to own stage

* move log_index to own stage

* integration add log_index stage

* integration add log_index stage

* clean

* clean

* print timing

* remove duplicates at unwind

* extract truncateBitmaps func

* try detect

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* add blackList of topics

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* clean

* sharding 1

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 2

* sharded 3

* sharded 3

* sharded 3

* speedup things by putCurrent and putReserve

* clean

* optimize trim

* clean

* remove blacklist

* add more info to err

* ?

* clean

* clean

* clean

* clean

* clean

* working version

* switch to cgo version of roaring bitmaps

* clean

* clean

* clean

* clean

* more docs

* clean

* clean

* fix logs bloom field

* Fix debug_getModifiedAccountsByNumber

* Try to fix crash

* fix problem with "absent block"

* fix problem with "absent block"

* remove optimize method call

* remove roaring iterator

* fix problem with rebuild indicess

* remove debug prints

* tests for eth_getLogs involving topics

* add tests for new stage, speparate topics into 2 buckets

* version up

* remove debug logs

* remove debug logs

* remove bloom filter implementation

* Optimisation

* Optimisatin not required, make rpctest lenient to geth errors

* Lenient to geth failures

Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 18:18:36 +01:00

132 lines
3.8 KiB
Go

// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package ethdb
import (
"bytes"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
)
// splitCursor implements cursor with two keys
// it is used to ignore incarnations in the middle
// of composite storage key, but without
// reconstructing the key
// Instead, the key is split into two parts and
// functions `Seek` and `Next` deliver both
// parts as well as the corresponding value
type splitCursor struct {
c Cursor // Unlerlying cursor
startkey []byte // Starting key (also contains bits that need to be preserved)
matchBytes int
mask uint8
part1end int // Position in the key where the first part ends
part2start int // Position in the key where the second part starts
part3start int // Position in the key where the third part starts
}
func NewSplitCursor(c Cursor, startkey []byte, matchBits int, part1end, part2start, part3start int) *splitCursor {
var sc splitCursor
sc.c = c
sc.startkey = startkey
sc.part1end = part1end
sc.part2start = part2start
sc.part3start = part3start
sc.matchBytes, sc.mask = Bytesmask(matchBits)
return &sc
}
func (sc *splitCursor) matchKey(k []byte) bool {
if k == nil {
return false
}
if sc.matchBytes == 0 {
return true
}
if len(k) < sc.matchBytes {
return false
}
if !bytes.Equal(k[:sc.matchBytes-1], sc.startkey[:sc.matchBytes-1]) {
return false
}
return (k[sc.matchBytes-1] & sc.mask) == (sc.startkey[sc.matchBytes-1] & sc.mask)
}
func (sc *splitCursor) Seek() (key1, key2, key3, val []byte, err error) {
k, v, err1 := sc.c.Seek(sc.startkey)
if err1 != nil {
return nil, nil, nil, nil, err1
}
if !sc.matchKey(k) {
return nil, nil, nil, nil, nil
}
return k[:sc.part1end], k[sc.part2start:sc.part3start], k[sc.part3start:], v, nil
}
func (sc *splitCursor) Next() (key1, key2, key3, val []byte, err error) {
k, v, err1 := sc.c.Next()
if err1 != nil {
return nil, nil, nil, nil, err1
}
if !sc.matchKey(k) {
return nil, nil, nil, nil, nil
}
return k[:sc.part1end], k[sc.part2start:sc.part3start], k[sc.part3start:], v, nil
}
var EndSuffix = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
func GetModifiedAccounts(db Getter, startTimestamp, endTimestamp uint64) ([]common.Address, error) {
keys := make(map[common.Address]struct{})
startCode := dbutils.EncodeTimestamp(startTimestamp)
if err := db.Walk(dbutils.PlainAccountChangeSetBucket, startCode, 0, func(k, v []byte) (bool, error) {
keyTimestamp, _ := dbutils.DecodeTimestamp(k)
if keyTimestamp > endTimestamp {
return false, nil
}
walker := func(addr, _ []byte) error {
keys[common.BytesToAddress(addr)] = struct{}{}
return nil
}
if innerErr := changeset.AccountChangeSetPlainBytes(v).Walk(walker); innerErr != nil {
return false, innerErr
}
return true, nil
}); err != nil {
return nil, err
}
if len(keys) == 0 {
return nil, nil
}
accounts := make([]common.Address, len(keys))
idx := 0
for key := range keys {
copy(accounts[idx][:], key[:])
idx++
}
return accounts, nil
}