diff --git a/cmd/prometheus/docker-compose.yml b/cmd/prometheus/docker-compose.yml index 129f1a7e3..bce679350 100644 --- a/cmd/prometheus/docker-compose.yml +++ b/cmd/prometheus/docker-compose.yml @@ -5,7 +5,7 @@ services: turbo-geth: image: turbo-geth:latest build: ./../.. - command: --nousb --metrics --metrics.expensive --pprof --pprofaddr=0.0.0.0 --remote-db-listen-addr 0.0.0.0:9999 + command: --nousb --metrics --metrics.expensive --pprof --pprofaddr=0.0.0.0 --pprofport=6060 --remote-db-listen-addr=0.0.0.0:9999 stop_grace_period: 2m volumes: - ${TGETH_DATADIR:-~/Library/Ethereum}:/root/.ethereum/ diff --git a/core/pruner.go b/core/pruner.go index 6a03107c1..9168e0699 100644 --- a/core/pruner.go +++ b/core/pruner.go @@ -185,7 +185,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error { return false, nil } - keysToRemove.AccountChangeSet = append(keysToRemove.AccountChangeSet, key) + keysToRemove.AccountChangeSet = append(keysToRemove.AccountChangeSet, common.CopyBytes(key)) innerErr := changeset.Walk(v, func(cKey, _ []byte) error { compKey, _ := dbutils.CompositeKeySuffix(cKey, timestamp) @@ -209,7 +209,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error { return false, nil } - keysToRemove.StorageChangeSet = append(keysToRemove.StorageChangeSet, key) + keysToRemove.StorageChangeSet = append(keysToRemove.StorageChangeSet, common.CopyBytes(key)) var innerErr error if debug.IsThinHistory() { innerErr = changeset.StorageChangeSetBytes(v).Walk(func(cKey, _ []byte) error { @@ -219,7 +219,7 @@ func Prune(db ethdb.Database, blockNumFrom uint64, blockNumTo uint64) error { } else { innerErr = changeset.Walk(v, func(cKey, _ []byte) error { compKey, _ := dbutils.CompositeKeySuffix(cKey, timestamp) - keysToRemove.StorageHistoryKeys = append(keysToRemove.StorageHistoryKeys, compKey) + keysToRemove.StorageHistoryKeys = append(keysToRemove.StorageHistoryKeys, common.CopyBytes(compKey)) return nil }) } @@ -351,6 +351,7 @@ func (i *limitIterator) HasMore() bool { if bytes.Equal(i.currentBucket, lastBatch.bucket) && len(lastBatch.keys) == i.currentNum { return false } + return true } diff --git a/core/state/history_test.go b/core/state/history_test.go index c8c84403a..ffc634bec 100644 --- a/core/state/history_test.go +++ b/core/state/history_test.go @@ -3,7 +3,6 @@ package state import ( "bytes" "context" - "github.com/ledgerwatch/turbo-geth/common/changeset" "math/big" "math/rand" "reflect" @@ -13,6 +12,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ledgerwatch/turbo-geth/common" + "github.com/ledgerwatch/turbo-geth/common/changeset" "github.com/ledgerwatch/turbo-geth/common/dbutils" "github.com/ledgerwatch/turbo-geth/common/debug" "github.com/ledgerwatch/turbo-geth/core/types/accounts" @@ -724,12 +724,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) { var err error var startKey [72]byte err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 2, func(k []byte, v []byte) (b bool, e error) { - err = block2.Add(k, v) + err = block2.Add(common.CopyBytes(k), common.CopyBytes(v)) if err != nil { t.Fatal(err) } - //fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v)) return true, nil }) if err != nil { @@ -737,12 +736,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) { } err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 4, func(k []byte, v []byte) (b bool, e error) { - err = block4.Add(k, v) + err = block4.Add(common.CopyBytes(k), common.CopyBytes(v)) if err != nil { t.Fatal(err) } - //fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v)) return true, nil }) if err != nil { @@ -750,12 +748,11 @@ func TestBoltDB_WalkAsOf1(t *testing.T) { } err = db.WalkAsOf(dbutils.StorageBucket, dbutils.StorageHistoryBucket, startKey[:], 0, 6, func(k []byte, v []byte) (b bool, e error) { - err = block6.Add(k, v) + err = block6.Add(common.CopyBytes(k), common.CopyBytes(v)) if err != nil { t.Fatal(err) } - //fmt.Printf("%v - %v \n", common.BytesToHash(k).String(), string(v)) return true, nil }) if err != nil { diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 0f92d2eaa..c55eca0b2 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -279,7 +279,7 @@ func testWalk(db Database, t *testing.T) { var gotKeys [][]byte err := db.Walk(testBucket, startKey, fixedBits, func(key, val []byte) (bool, error) { - gotKeys = append(gotKeys, key) + gotKeys = append(gotKeys, common.CopyBytes(key)) return true, nil }) assert.NoError(t, err) diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 6a875a75b..0dde31148 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -22,6 +22,7 @@ import ( "sort" "testing" + "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/ethdb" ) @@ -256,7 +257,7 @@ func iterateKeys(db ethdb.Database) []string { func iterateKeysFromKey(db ethdb.Database, fromKey []byte) []string { keys := []string{} db.Walk(nil, fromKey, 0, func(key, value []byte) (bool, error) { - keys = append(keys, string(key)) + keys = append(keys, string(common.CopyBytes(key))) return true, nil }) sort.Strings(keys) diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 0c4e88f05..ab3eed43b 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -18,6 +18,7 @@ package ethdb import ( "github.com/ledgerwatch/bolt" + "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/common/dbutils" "github.com/ledgerwatch/turbo-geth/log" ) @@ -83,7 +84,7 @@ func (db *BoltDatabase) MemCopy() Database { return err } return b.ForEach(func(k, v []byte) error { - if err := newBucketToWrite.Put(k, v); err != nil { + if err := newBucketToWrite.Put(common.CopyBytes(k), common.CopyBytes(v)); err != nil { return err } return nil diff --git a/migrations/changeset_migration.go b/migrations/changeset_migration.go index ec6a18044..e17e8a732 100644 --- a/migrations/changeset_migration.go +++ b/migrations/changeset_migration.go @@ -67,7 +67,7 @@ func splitChangeSetMigration(batchSize int) Migration { } for currentKey != nil { - changesetsToRemove = append(changesetsToRemove, currentKey) + changesetsToRemove = append(changesetsToRemove, common.CopyBytes(currentKey)) ts, bucket := dbutils.DecodeTimestamp(currentKey) encTS := dbutils.EncodeTimestamp(ts) @@ -83,9 +83,9 @@ func splitChangeSetMigration(batchSize int) Migration { return innerErr } - accChangesets = append(accChangesets, encTS, v) + accChangesets = append(accChangesets, encTS, common.CopyBytes(v)) } else { - accChangesets = append(accChangesets, encTS, currentValue) + accChangesets = append(accChangesets, encTS, common.CopyBytes(currentValue)) } case bytes.Equal(dbutils.StorageHistoryBucket, bucket): @@ -100,10 +100,10 @@ func splitChangeSetMigration(batchSize int) Migration { log.Error("Error on encode storage changeset", "err", innerErr) return innerErr } - storageChangesets = append(storageChangesets, encTS, v) + storageChangesets = append(storageChangesets, encTS, common.CopyBytes(v)) } else { - storageChangesets = append(storageChangesets, encTS, currentValue) + storageChangesets = append(storageChangesets, encTS, common.CopyBytes(currentValue)) } } diff --git a/migrations/changeset_migration_test.go b/migrations/changeset_migration_test.go index 1a634c311..767012a5d 100644 --- a/migrations/changeset_migration_test.go +++ b/migrations/changeset_migration_test.go @@ -2,12 +2,13 @@ package migrations import ( "fmt" + "testing" + "github.com/davecgh/go-spew/spew" "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/common/changeset" "github.com/ledgerwatch/turbo-geth/common/dbutils" "github.com/ledgerwatch/turbo-geth/ethdb" - "testing" ) func TestChangeSetMigrationSuccess(t *testing.T) { @@ -19,7 +20,7 @@ func TestChangeSetMigrationSuccess(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -29,7 +30,7 @@ func TestChangeSetMigrationSuccess(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -105,7 +106,7 @@ func TestChangeSetMigrationThinHistorySuccess(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -116,7 +117,7 @@ func TestChangeSetMigrationThinHistorySuccess(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -196,12 +197,12 @@ func TestChangeSetMigrationFail(t *testing.T) { } if i == 25 { //incorrect value - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc[:5]) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc[:5])) if err != nil { t.Error(err) } } else { - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -213,7 +214,7 @@ func TestChangeSetMigrationFail(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } @@ -236,7 +237,7 @@ func TestChangeSetMigrationFail(t *testing.T) { if err != nil { t.Error(err) } - err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(25)), dbutils.AccountsHistoryBucket), enc) + err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(25)), dbutils.AccountsHistoryBucket), common.CopyBytes(enc)) if err != nil { t.Error(err) } diff --git a/tests/pruner_test.go b/tests/pruner_test.go index 1e77f40fd..e28062e96 100644 --- a/tests/pruner_test.go +++ b/tests/pruner_test.go @@ -3,12 +3,13 @@ package tests import ( "context" "crypto/ecdsa" - "github.com/ledgerwatch/turbo-geth/common/changeset" "math/big" "reflect" "testing" "time" + "github.com/ledgerwatch/turbo-geth/common/changeset" + "github.com/davecgh/go-spew/spew" "github.com/ledgerwatch/turbo-geth/accounts/abi/bind" "github.com/ledgerwatch/turbo-geth/accounts/abi/bind/backends" @@ -667,14 +668,14 @@ func getStat(db ethdb.Database) (stateStats, error) { StorageSuffixRecordsByTimestamp: make(map[uint64]uint32), } err := db.Walk(dbutils.AccountChangeSetBucket, []byte{}, 0, func(key, v []byte) (b bool, e error) { - timestamp, _ := dbutils.DecodeTimestamp(key) + timestamp, _ := dbutils.DecodeTimestamp(common.CopyBytes(key)) if _, ok := stat.AccountSuffixRecordsByTimestamp[timestamp]; ok { panic("multiple account suffix records") } stat.AccountSuffixRecordsByTimestamp[timestamp] = uint32(changeset.Len(v)) innerErr := changeset.Walk(v, func(k, _ []byte) error { - compKey, _ := dbutils.CompositeKeySuffix(k, timestamp) + compKey, _ := dbutils.CompositeKeySuffix(common.CopyBytes(k), timestamp) _, err := db.Get(dbutils.AccountsHistoryBucket, compKey) if err != nil { stat.ErrAccountsInHistory++