erigon-pulse/cmd/integration/commands/reset_state.go

271 lines
6.0 KiB
Go
Raw Normal View History

2020-07-14 01:56:29 +00:00
package commands
import (
"context"
"fmt"
"os"
"text/tabwriter"
2020-10-28 03:18:10 +00:00
"github.com/ledgerwatch/turbo-geth/cmd/utils"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/core"
2020-10-28 03:18:10 +00:00
"github.com/ledgerwatch/turbo-geth/core/rawdb"
"github.com/ledgerwatch/turbo-geth/eth/stagedsync"
"github.com/ledgerwatch/turbo-geth/eth/stagedsync/stages"
"github.com/ledgerwatch/turbo-geth/ethdb"
"github.com/ledgerwatch/turbo-geth/log"
"github.com/spf13/cobra"
)
var cmdResetState = &cobra.Command{
Use: "reset_state",
Short: "Reset StateStages (5,6,7,8,9,10) and buckets",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := utils.RootContext()
2020-10-28 03:18:10 +00:00
db := openDatabase(chaindata, true)
defer db.Close()
err := resetState(db, ctx)
if err != nil {
log.Error(err.Error())
return err
}
return nil
},
}
2020-08-10 23:38:19 +00:00
var cmdClearUnwindStack = &cobra.Command{
Use: "clear_unwind_stack",
Short: "Clear unwind stack",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := utils.RootContext()
2020-10-28 03:18:10 +00:00
db := openDatabase(chaindata, true)
defer db.Close()
err := clearUnwindStack(db, ctx)
2020-08-10 23:38:19 +00:00
if err != nil {
log.Error(err.Error())
return err
}
return nil
},
}
func init() {
withChaindata(cmdResetState)
rootCmd.AddCommand(cmdResetState)
2020-08-10 23:38:19 +00:00
withChaindata(cmdClearUnwindStack)
rootCmd.AddCommand(cmdClearUnwindStack)
}
2020-10-28 03:18:10 +00:00
func clearUnwindStack(db rawdb.DatabaseWriter, _ context.Context) error {
for _, stage := range stages.AllStages {
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stage, 0); err != nil {
2020-08-10 23:38:19 +00:00
return err
}
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetState(db ethdb.Database, _ context.Context) error {
fmt.Printf("Before reset: \n")
if err := printStages(db); err != nil {
return err
}
// don't reset senders here
if err := resetExec(db); err != nil {
return err
}
if err := stagedsync.ResetHashState(db); err != nil {
return err
}
if err := stagedsync.ResetIH(db); err != nil {
return err
}
if err := resetHistory(db); err != nil {
return err
}
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
if err := resetLogIndex(db); err != nil {
return err
}
if err := resetCallTraces(db); err != nil {
return err
}
if err := resetTxLookup(db); err != nil {
return err
}
if err := resetTxPool(db); err != nil {
return err
}
2020-09-11 10:33:46 +00:00
if err := resetFinish(db); err != nil {
return err
}
// set genesis after reset all buckets
if _, _, err := core.DefaultGenesisBlock().CommitGenesisState(db, false); err != nil {
return err
}
fmt.Printf("After reset: \n")
if err := printStages(db); err != nil {
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetSenders(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
dbutils.Senders,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.Senders, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.Senders, 0); err != nil {
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetExec(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
dbutils.HashedAccountsBucket,
dbutils.HashedStorageBucket,
dbutils.ContractCodeBucket,
dbutils.PlainStateBucket,
dbutils.PlainAccountChangeSetBucket,
dbutils.PlainStorageChangeSetBucket,
dbutils.PlainContractCodeBucket,
dbutils.BlockReceiptsPrefix,
dbutils.Log,
dbutils.IncarnationMapBucket,
dbutils.CodeBucket,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.Execution, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.Execution, 0); err != nil {
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetHistory(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
dbutils.AccountsHistoryBucket,
dbutils.StorageHistoryBucket,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.AccountHistoryIndex, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.StorageHistoryIndex, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.AccountHistoryIndex, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.StorageHistoryIndex, 0); err != nil {
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetLogIndex(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
dbutils.LogAddressIndex,
dbutils.LogTopicIndex,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.LogIndex, 0); err != nil {
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.LogIndex, 0); err != nil {
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetCallTraces(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
dbutils.CallFromIndex,
dbutils.CallToIndex,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.CallTraces, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.CallTraces, 0); err != nil {
return err
}
return nil
}
2020-10-28 03:18:10 +00:00
func resetTxLookup(db rawdb.DatabaseWriter) error {
if err := db.(ethdb.BucketsMigrator).ClearBuckets(
dbutils.TxLookupPrefix,
); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.TxLookup, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.TxLookup, 0); err != nil {
return err
}
return nil
}
func resetTxPool(db ethdb.Putter) error {
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.TxPool, 0); err != nil {
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.TxPool, 0); err != nil {
return err
}
return nil
}
2020-09-11 10:33:46 +00:00
func resetFinish(db ethdb.Putter) error {
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageProgress(db, stages.Finish, 0); err != nil {
2020-09-11 10:33:46 +00:00
return err
}
2020-12-21 11:01:37 +00:00
if err := stages.SaveStageUnwind(db, stages.Finish, 0); err != nil {
2020-09-11 10:33:46 +00:00
return err
}
return nil
}
func printStages(db ethdb.Getter) error {
var err error
var progress uint64
w := new(tabwriter.Writer)
defer w.Flush()
w.Init(os.Stdout, 8, 8, 0, '\t', 0)
for _, stage := range stages.AllStages {
2020-12-21 11:01:37 +00:00
if progress, err = stages.GetStageProgress(db, stage); err != nil {
return err
}
fmt.Fprintf(w, "%s \t %d\n", string(stage), progress)
}
return nil
}