erigon-pulse/ethdb/tx_db.go

380 lines
8.5 KiB
Go
Raw Normal View History

package ethdb
import (
"context"
"fmt"
"github.com/google/btree"
"github.com/ledgerwatch/erigon/log"
)
2021-03-30 09:53:54 +00:00
// Implements ethdb.Getter for Tx
type roTxDb struct {
tx Tx
top bool
2021-03-30 09:53:54 +00:00
}
func NewRoTxDb(tx Tx) *roTxDb {
return &roTxDb{tx: tx, top: true}
}
func (m *roTxDb) GetOne(bucket string, key []byte) ([]byte, error) {
c, err := m.tx.Cursor(bucket)
if err != nil {
return nil, err
}
2021-03-30 09:53:54 +00:00
defer c.Close()
_, v, err := c.SeekExact(key)
return v, err
}
func (m *roTxDb) Get(bucket string, key []byte) ([]byte, error) {
dat, err := m.GetOne(bucket, key)
return getOneWrapper(dat, err)
2021-03-30 09:53:54 +00:00
}
func (m *roTxDb) Has(bucket string, key []byte) (bool, error) {
c, err := m.tx.Cursor(bucket)
if err != nil {
return false, err
}
2021-03-30 09:53:54 +00:00
defer c.Close()
_, v, err := c.SeekExact(key)
return v != nil, err
}
func (m *roTxDb) Walk(bucket string, startkey []byte, fixedbits int, walker func([]byte, []byte) (bool, error)) error {
c, err := m.tx.Cursor(bucket)
if err != nil {
return err
}
2021-03-30 09:53:54 +00:00
defer c.Close()
return Walk(c, startkey, fixedbits, walker)
}
func (m *roTxDb) ForEach(bucket string, fromPrefix []byte, walker func(k, v []byte) error) error {
return m.tx.ForEach(bucket, fromPrefix, walker)
}
func (m *roTxDb) ForPrefix(bucket string, prefix []byte, walker func(k, v []byte) error) error {
return m.tx.ForPrefix(bucket, prefix, walker)
}
func (m *roTxDb) ForAmount(bucket string, prefix []byte, amount uint32, walker func(k, v []byte) error) error {
return m.tx.ForAmount(bucket, prefix, amount, walker)
}
2021-04-03 06:26:00 +00:00
func (m *roTxDb) BeginGetter(ctx context.Context) (GetterTx, error) {
2021-03-30 09:53:54 +00:00
return &roTxDb{tx: m.tx, top: false}, nil
}
func (m *roTxDb) Rollback() {
if m.top {
m.tx.Rollback()
}
}
func (m *roTxDb) Tx() Tx {
return m.tx
}
func NewRwTxDb(tx Tx) *TxDb {
return &TxDb{tx: tx, cursors: map[string]Cursor{}}
}
// TxDb - provides Database interface around ethdb.Tx
// It's not thread-safe!
// TxDb not usable after .Commit()/.Rollback() call, but usable after .CommitAndBegin() call
2021-03-22 06:47:01 +00:00
// you can put unlimited amount of data into this class
// Walk and MultiWalk methods - work outside of Tx object yet, will implement it later
type TxDb struct {
2021-02-10 17:04:22 +00:00
db Database
tx Tx
cursors map[string]Cursor
txFlags TxFlags
2021-02-10 17:04:22 +00:00
len uint64
}
func WrapIntoTxDB(tx RwTx) *TxDb {
return &TxDb{tx: tx, cursors: map[string]Cursor{}}
}
func (m *TxDb) Close() {
panic("don't call me")
}
// NewTxDbWithoutTransaction creates TxDb object without opening transaction,
// such TxDb not usable before .Begin() call on it
// It allows inject TxDb object into class hierarchy, but open write transaction later
2021-03-21 05:20:44 +00:00
func NewTxDbWithoutTransaction(db Database, flags TxFlags) DbWithPendingMutations {
return &TxDb{db: db, txFlags: flags}
}
func (m *TxDb) Begin(ctx context.Context, flags TxFlags) (DbWithPendingMutations, error) {
batch := m
if m.tx != nil {
2021-02-10 17:04:22 +00:00
panic("nested transactions not supported")
}
2021-02-10 17:04:22 +00:00
if err := batch.begin(ctx, flags); err != nil {
return nil, err
}
return batch, nil
}
2021-04-03 06:26:00 +00:00
func (m *TxDb) BeginGetter(ctx context.Context) (GetterTx, error) {
2021-03-30 09:53:54 +00:00
batch := m
if m.tx != nil {
panic("nested transactions not supported")
}
if err := batch.begin(ctx, RO); err != nil {
return nil, err
}
return batch, nil
}
func (m *TxDb) cursor(bucket string) (Cursor, error) {
c, ok := m.cursors[bucket]
if !ok {
var err error
c, err = m.tx.Cursor(bucket)
if err != nil {
return nil, err
}
m.cursors[bucket] = c
}
return c, nil
}
func (m *TxDb) IncrementSequence(bucket string, amount uint64) (res uint64, err error) {
return m.tx.(RwTx).IncrementSequence(bucket, amount)
}
func (m *TxDb) ReadSequence(bucket string) (res uint64, err error) {
return m.tx.ReadSequence(bucket)
}
func (m *TxDb) Put(bucket string, key []byte, value []byte) error {
m.len += uint64(len(key) + len(value))
c, err := m.cursor(bucket)
if err != nil {
return err
}
return c.(RwCursor).Put(key, value)
}
func (m *TxDb) Append(bucket string, key []byte, value []byte) error {
m.len += uint64(len(key) + len(value))
c, err := m.cursor(bucket)
if err != nil {
return err
}
return c.(RwCursor).Append(key, value)
}
func (m *TxDb) AppendDup(bucket string, key []byte, value []byte) error {
m.len += uint64(len(key) + len(value))
c, err := m.cursor(bucket)
if err != nil {
return err
}
return c.(RwCursorDupSort).AppendDup(key, value)
}
func (m *TxDb) Delete(bucket string, k, v []byte) error {
m.len += uint64(len(k))
c, err := m.cursor(bucket)
if err != nil {
return err
}
return c.(RwCursor).Delete(k, v)
}
func (m *TxDb) NewBatch() DbWithPendingMutations {
return &mutation{
db: m,
puts: btree.New(32),
}
}
2021-02-10 17:04:22 +00:00
func (m *TxDb) begin(ctx context.Context, flags TxFlags) error {
2021-03-30 09:53:54 +00:00
kv := m.db.(HasRwKV).RwKV()
var tx Tx
var err error
if flags&RO != 0 {
2021-04-03 06:26:00 +00:00
tx, err = kv.BeginRo(ctx)
} else {
tx, err = kv.BeginRw(ctx)
}
if err != nil {
return err
}
m.tx = tx
m.cursors = make(map[string]Cursor, 16)
return nil
}
2021-03-30 09:53:54 +00:00
func (m *TxDb) RwKV() RwKV {
panic("not allowed to get KV interface because you will loose transaction, please use .Tx() method")
}
// Last can only be called from the transaction thread
func (m *TxDb) Last(bucket string) ([]byte, []byte, error) {
c, err := m.cursor(bucket)
if err != nil {
return []byte{}, nil, err
}
return c.Last()
}
func (m *TxDb) GetOne(bucket string, key []byte) ([]byte, error) {
c, err := m.cursor(bucket)
if err != nil {
return nil, err
}
_, v, err := c.SeekExact(key)
return v, err
}
func (m *TxDb) Get(bucket string, key []byte) ([]byte, error) {
dat, err := m.GetOne(bucket, key)
return getOneWrapper(dat, err)
}
func (m *TxDb) Has(bucket string, key []byte) (bool, error) {
v, err := m.Get(bucket, key)
if err != nil {
return false, err
}
return v != nil, nil
}
func (m *TxDb) MultiPut(tuples ...[]byte) (uint64, error) {
return 0, MultiPut(m.tx.(RwTx), tuples...)
}
func (m *TxDb) BatchSize() int {
return int(m.len)
}
func (m *TxDb) Walk(bucket string, startkey []byte, fixedbits int, walker func([]byte, []byte) (bool, error)) error {
// get cursor out of pool, then calls txDb.Put/Get/Delete on same bucket inside Walk callback - will not affect state of Walk
c, ok := m.cursors[bucket]
if ok {
delete(m.cursors, bucket)
} else {
var err error
c, err = m.tx.Cursor(bucket)
if err != nil {
return err
}
}
defer func() { // put cursor back to pool if can
if _, ok = m.cursors[bucket]; ok {
c.Close()
} else {
m.cursors[bucket] = c
}
}()
return Walk(c, startkey, fixedbits, walker)
}
func (m *TxDb) ForEach(bucket string, fromPrefix []byte, walker func(k, v []byte) error) error {
return m.tx.ForEach(bucket, fromPrefix, walker)
}
func (m *TxDb) ForPrefix(bucket string, prefix []byte, walker func(k, v []byte) error) error {
return m.tx.ForPrefix(bucket, prefix, walker)
}
func (m *TxDb) ForAmount(bucket string, prefix []byte, amount uint32, walker func(k, v []byte) error) error {
return m.tx.ForAmount(bucket, prefix, amount, walker)
}
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
func (m *TxDb) CommitAndBegin(ctx context.Context) error {
err := m.Commit()
if err != nil {
return err
}
2021-02-10 17:04:22 +00:00
return m.begin(ctx, m.txFlags)
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
}
func (m *TxDb) RollbackAndBegin(ctx context.Context) error {
m.Rollback()
2021-02-10 17:04:22 +00:00
return m.begin(ctx, m.txFlags)
}
func (m *TxDb) Commit() error {
if m.tx == nil {
return fmt.Errorf("second call .Commit() on same transaction")
}
2021-04-03 06:26:00 +00:00
if err := m.tx.Commit(); err != nil {
return err
}
m.tx = nil
m.cursors = nil
m.len = 0
return nil
}
func (m *TxDb) Rollback() {
if m.tx == nil {
return
}
m.tx.Rollback()
m.cursors = nil
m.tx = nil
m.len = 0
}
func (m *TxDb) Tx() Tx {
return m.tx
}
func (m *TxDb) Keys() ([][]byte, error) {
panic("don't use me")
}
2020-09-08 19:39:43 +00:00
func (m *TxDb) BucketExists(name string) (bool, error) {
exists := false
migrator, ok := m.tx.(BucketMigrator)
if !ok {
return false, fmt.Errorf("%T doesn't implement ethdb.TxMigrator interface", m.tx)
}
exists = migrator.ExistsBucket(name)
return exists, nil
}
func (m *TxDb) ClearBuckets(buckets ...string) error {
for i := range buckets {
name := buckets[i]
migrator, ok := m.tx.(BucketMigrator)
if !ok {
return fmt.Errorf("%T doesn't implement ethdb.TxMigrator interface", m.tx)
}
if err := migrator.ClearBucket(name); err != nil {
return err
}
}
return nil
}
func (m *TxDb) DropBuckets(buckets ...string) error {
for i := range buckets {
name := buckets[i]
log.Info("Dropping bucket", "name", name)
migrator, ok := m.tx.(BucketMigrator)
if !ok {
return fmt.Errorf("%T doesn't implement ethdb.TxMigrator interface", m.tx)
}
if err := migrator.DropBucket(name); err != nil {
return err
}
}
return nil
}