mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Run CI on bolt with prefix_compression_without_allocation (fixes) (#380)

Browse Source
This commit is contained in:
Alex Sharov 2020-04-11 20:28:15 +07:00 committed by GitHub
parent 346a79444f
commit 535d73be5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 33 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/prometheus/docker-compose.yml
View File

@ -5,7 +5,7 @@ services:
turbo-geth:
image: turbo-geth:latest
build: ./../..
command: --nousb --metrics --metrics.expensive --pprof --pprofaddr=0.0.0.0 --remote-db-listen-addr 0.0.0.0:9999
command: --nousb --metrics --metrics.expensive --pprof --pprofaddr=0.0.0.0 --pprofport=6060 --remote-db-listen-addr=0.0.0.0:9999
stop_grace_period: 2m
volumes:
- ${TGETH_DATADIR:-~/Library/Ethereum}:/root/.ethereum/

7
core/pruner.go
View File

@ -185,7 +185,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error {
return false, nil
}
keysToRemove.AccountChangeSet = append(keysToRemove.AccountChangeSet, key)
keysToRemove.AccountChangeSet = append(keysToRemove.AccountChangeSet, common.CopyBytes(key))
innerErr := changeset.Walk(v, func(cKey, _ []byte) error {
compKey, _ := dbutils.CompositeKeySuffix(cKey, timestamp)
@ -209,7 +209,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error {
return false, nil
}
keysToRemove.StorageChangeSet = append(keysToRemove.StorageChangeSet, key)
keysToRemove.StorageChangeSet = append(keysToRemove.StorageChangeSet, common.CopyBytes(key))
var innerErr error
if debug.IsThinHistory() {
innerErr = changeset.StorageChangeSetBytes(v).Walk(func(cKey, _ []byte) error {
@ -219,7 +219,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error {
} else {
innerErr = changeset.Walk(v, func(cKey, _ []byte) error {
compKey, _ := dbutils.CompositeKeySuffix(cKey, timestamp)
keysToRemove.StorageHistoryKeys = append(keysToRemove.StorageHistoryKeys, compKey)
keysToRemove.StorageHistoryKeys = append(keysToRemove.StorageHistoryKeys, common.CopyBytes(compKey))
return nil
})
}
@ -351,6 +351,7 @@ func (i *limitIterator) HasMore() bool {
if bytes.Equal(i.currentBucket, lastBatch.bucket) && len(lastBatch.keys) == i.currentNum {
return false
}
return true
}

11
core/state/history_test.go
View File

@ -3,7 +3,6 @@ package state
import (
"bytes"
"context"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"math/big"
"math/rand"
"reflect"
@ -13,6 +12,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/common/debug"
"github.com/ledgerwatch/turbo-geth/core/types/accounts"
@ -724,12 +724,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) {
var err error
var startKey [72]byte
err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 2, func(k []byte, v []byte) (b bool, e error) {
err = block2.Add(k, v)
err = block2.Add(common.CopyBytes(k), common.CopyBytes(v))
if err != nil {
t.Fatal(err)
}
//fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v))
return true, nil
})
if err != nil {
@ -737,12 +736,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) {
}
err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 4, func(k []byte, v []byte) (b bool, e error) {
err = block4.Add(k, v)
err = block4.Add(common.CopyBytes(k), common.CopyBytes(v))
if err != nil {
t.Fatal(err)
}
//fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v))
return true, nil
})
if err != nil {
@ -750,12 +748,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) {
}
err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 6, func(k []byte, v []byte) (b bool, e error) {
err = block6.Add(k, v)
err = block6.Add(common.CopyBytes(k), common.CopyBytes(v))
if err != nil {
t.Fatal(err)
}
//fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v))
return true, nil
})
if err != nil {

2
ethdb/database_test.go
View File

@ -279,7 +279,7 @@ func testWalk(db Database, t *testing.T) {
var gotKeys [][]byte
err := db.Walk(testBucket, startKey, fixedBits, func(key, val []byte) (bool, error) {
gotKeys = append(gotKeys, key)
gotKeys = append(gotKeys, common.CopyBytes(key))
return true, nil
})
assert.NoError(t, err)

3
ethdb/dbtest/testsuite.go
View File

@ -22,6 +22,7 @@ import (
"sort"
"testing"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/ethdb"
)
@ -256,7 +257,7 @@ func iterateKeys(db ethdb.Database) []string {
func iterateKeysFromKey(db ethdb.Database, fromKey []byte) []string {
keys := []string{}
db.Walk(nil, fromKey, 0, func(key, value []byte) (bool, error) {
keys = append(keys, string(key))
keys = append(keys, string(common.CopyBytes(key)))
return true, nil
})
sort.Strings(keys)

3
ethdb/memory_database.go
View File

@ -18,6 +18,7 @@ package ethdb
import (
"github.com/ledgerwatch/bolt"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/log"
)
@ -83,7 +84,7 @@ func (db *BoltDatabase) MemCopy() Database {
return err
}
return b.ForEach(func(k, v []byte) error {
if err := newBucketToWrite.Put(k, v); err != nil {
if err := newBucketToWrite.Put(common.CopyBytes(k), common.CopyBytes(v)); err != nil {
return err
}
return nil

10
migrations/changeset_migration.go
View File

@ -67,7 +67,7 @@ func splitChangeSetMigration(batchSize int) Migration {
}
for currentKey != nil {
changesetsToRemove = append(changesetsToRemove, currentKey)
changesetsToRemove = append(changesetsToRemove, common.CopyBytes(currentKey))
ts, bucket := dbutils.DecodeTimestamp(currentKey)
encTS := dbutils.EncodeTimestamp(ts)
@ -83,9 +83,9 @@ func splitChangeSetMigration(batchSize int) Migration {
return innerErr
}
accChangesets = append(accChangesets, encTS, v)
accChangesets = append(accChangesets, encTS, common.CopyBytes(v))
} else {
accChangesets = append(accChangesets, encTS, currentValue)
accChangesets = append(accChangesets, encTS, common.CopyBytes(currentValue))
}
case bytes.Equal(dbutils.StorageHistoryBucket, bucket):
@ -100,10 +100,10 @@ func splitChangeSetMigration(batchSize int) Migration {
log.Error("Error on encode storage changeset", "err", innerErr)
return innerErr
}
storageChangesets = append(storageChangesets, encTS, v)
storageChangesets = append(storageChangesets, encTS, common.CopyBytes(v))
} else {
storageChangesets = append(storageChangesets, encTS, currentValue)
storageChangesets = append(storageChangesets, encTS, common.CopyBytes(currentValue))
}
}

19
migrations/changeset_migration_test.go
View File

@ -2,12 +2,13 @@ package migrations
import (
"fmt"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/ethdb"
"testing"
)
func TestChangeSetMigrationSuccess(t *testing.T) {
@ -19,7 +20,7 @@ func TestChangeSetMigrationSuccess(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -29,7 +30,7 @@ func TestChangeSetMigrationSuccess(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -105,7 +106,7 @@ func TestChangeSetMigrationThinHistorySuccess(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -116,7 +117,7 @@ func TestChangeSetMigrationThinHistorySuccess(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -196,12 +197,12 @@ func TestChangeSetMigrationFail(t *testing.T) {
}
if i == 25 {
//incorrect value
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc[:5])
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc[:5]))
if err != nil {
t.Error(err)
}
} else {
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -213,7 +214,7 @@ func TestChangeSetMigrationFail(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}
@ -236,7 +237,7 @@ func TestChangeSetMigrationFail(t *testing.T) {
if err != nil {
t.Error(err)
}
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(25)), dbutils.AccountsHistoryBucket), enc)
err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(25)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc))
if err != nil {
t.Error(err)
}

7
tests/pruner_test.go
View File

@ -3,12 +3,13 @@ package tests
import (
"context"
"crypto/ecdsa"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"math/big"
"reflect"
"testing"
"time"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/davecgh/go-spew/spew"
"github.com/ledgerwatch/turbo-geth/accounts/abi/bind"
"github.com/ledgerwatch/turbo-geth/accounts/abi/bind/backends"
@ -667,14 +668,14 @@ func getStat(db ethdb.Database) (stateStats, error) {
StorageSuffixRecordsByTimestamp: make(map[uint64]uint32),
}
err := db.Walk(dbutils.AccountChangeSetBucket, []byte{}, 0, func(key, v []byte) (b bool, e error) {
timestamp, _ := dbutils.DecodeTimestamp(key)
timestamp, _ := dbutils.DecodeTimestamp(common.CopyBytes(key))
if _, ok := stat.AccountSuffixRecordsByTimestamp[timestamp]; ok {
panic("multiple account suffix records")
}
stat.AccountSuffixRecordsByTimestamp[timestamp] = uint32(changeset.Len(v))
innerErr := changeset.Walk(v, func(k, _ []byte) error {
compKey, _ := dbutils.CompositeKeySuffix(k, timestamp)
compKey, _ := dbutils.CompositeKeySuffix(common.CopyBytes(k), timestamp)
_, err := db.Get(dbutils.AccountsHistoryBucket, compKey)
if err != nil {
stat.ErrAccountsInHistory++