erigon-pulse/cmd/integration/commands/reset_state.go

258 lines
6.3 KiB
Go
Raw Normal View History

2020-07-14 01:56:29 +00:00
package commands
import (
"context"
"fmt"
"os"
"text/tabwriter"
2021-07-29 11:53:13 +00:00
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/cmd/utils"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
"github.com/ledgerwatch/erigon/ethdb/prune"
2021-07-29 10:23:23 +00:00
"github.com/ledgerwatch/log/v3"
"github.com/spf13/cobra"
)
var cmdResetState = &cobra.Command{
Use: "reset_state",
Short: "Reset StateStages (5,6,7,8,9,10) and buckets",
RunE: func(cmd *cobra.Command, args []string) error {
ctx, _ := utils.RootContext()
logger := log.New()
db := openDB(chaindata, logger, true)
2020-10-28 03:18:10 +00:00
defer db.Close()
err := resetState(db, logger, ctx)
if err != nil {
log.Error(err.Error())
return err
}
2020-08-10 23:38:19 +00:00
return nil
},
}
func init() {
withDatadir(cmdResetState)
2021-06-16 04:31:25 +00:00
withChain(cmdResetState)
rootCmd.AddCommand(cmdResetState)
}
func resetState(db kv.RwDB, logger log.Logger, ctx context.Context) error {
if err := db.View(ctx, func(tx kv.Tx) error { return printStages(tx) }); err != nil {
return err
}
// don't reset senders here
if err := db.Update(ctx, stagedsync.ResetHashState); err != nil {
return err
}
if err := db.Update(ctx, stagedsync.ResetIH); err != nil {
return err
}
if err := db.Update(ctx, resetHistory); err != nil {
return err
}
if err := db.Update(ctx, resetLogIndex); err != nil {
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return err
}
if err := db.Update(ctx, resetCallTraces); err != nil {
return err
}
if err := db.Update(ctx, resetTxLookup); err != nil {
return err
}
if err := db.Update(ctx, resetFinish); err != nil {
2020-09-11 10:33:46 +00:00
return err
}
genesis, _ := byChain()
if err := db.Update(ctx, func(tx kv.RwTx) error { return resetExec(tx, genesis) }); err != nil {
return err
}
// set genesis after reset all buckets
fmt.Printf("After reset: \n")
if err := db.View(ctx, func(tx kv.Tx) error { return printStages(tx) }); err != nil {
return err
}
return nil
}
func resetSenders(tx kv.RwTx) error {
if err := tx.ClearBucket(kv.Senders); err != nil {
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.Senders, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.Senders, 0); err != nil {
return err
}
return nil
}
func resetExec(tx kv.RwTx, g *core.Genesis) error {
if err := tx.ClearBucket(kv.HashedAccounts); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.HashedStorage); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.ContractCode); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 03:43:51 +00:00
if err := tx.ClearBucket(kv.PlainState); err != nil {
return err
}
if err := tx.ClearBucket(kv.AccountChangeSet); err != nil {
return err
}
if err := tx.ClearBucket(kv.StorageChangeSet); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.PlainContractCode); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.Receipts); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.Log); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.IncarnationMap); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 03:43:51 +00:00
if err := tx.ClearBucket(kv.Code); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.CallTraceSet); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.Epoch); err != nil {
return err
}
if err := tx.ClearBucket(kv.PendingEpoch); err != nil {
return err
}
if err := stages.SaveStageProgress(tx, stages.Execution, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.Execution, 0); err != nil {
return err
}
if _, _, err := g.WriteGenesisState(tx); err != nil {
return err
}
return nil
}
func resetHistory(tx kv.RwTx) error {
if err := tx.ClearBucket(kv.AccountsHistory); err != nil {
return err
}
if err := tx.ClearBucket(kv.StorageHistory); err != nil {
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.AccountHistoryIndex, 0); err != nil {
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.StorageHistoryIndex, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.AccountHistoryIndex, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.StorageHistoryIndex, 0); err != nil {
return err
}
return nil
}
func resetLogIndex(tx kv.RwTx) error {
if err := tx.ClearBucket(kv.LogAddressIndex); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.LogTopicIndex); err != nil {
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.LogIndex, 0); err != nil {
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.LogIndex, 0); err != nil {
return err
}
bitmap indices for logs (#1124) * save progress * try now * don't create bloom inside rlpDecode * don't create bloom inside ApplyTransaction * clean * clean * clean * clean * clean * clean * clean * clean * rename method * print timings * print timings * print timings * sort before flush * fix err lint * clean * move tests to transactions * compressed version * up bound * up bound * more tests * more tests * more tests * more tests * better removal * clean * better performance of get/put methods * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * optimize rpcdaemon * fix test * fix rpcdaemon * fix test * simplify * simplify * fix nil pointer * clean * revert some changes * add some logs * clean * try without optimize * clean * clean * clean * clean * try * move log_index to own stage * move log_index to own stage * integration add log_index stage * integration add log_index stage * clean * clean * print timing * remove duplicates at unwind * extract truncateBitmaps func * try detect * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * clean * add blackList of topics * clean * clean * clean * clean * clean * clean * clean * clean * sharding 1 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 2 * sharded 3 * sharded 3 * sharded 3 * speedup things by putCurrent and putReserve * clean * optimize trim * clean * remove blacklist * add more info to err * ? * clean * clean * clean * clean * clean * working version * switch to cgo version of roaring bitmaps * clean * clean * clean * clean * more docs * clean * clean * fix logs bloom field * Fix debug_getModifiedAccountsByNumber * Try to fix crash * fix problem with "absent block" * fix problem with "absent block" * remove optimize method call * remove roaring iterator * fix problem with rebuild indicess * remove debug prints * tests for eth_getLogs involving topics * add tests for new stage, speparate topics into 2 buckets * version up * remove debug logs * remove debug logs * remove bloom filter implementation * Optimisation * Optimisatin not required, make rpctest lenient to geth errors * Lenient to geth failures Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-09-28 17:18:36 +00:00
return nil
}
func resetCallTraces(tx kv.RwTx) error {
if err := tx.ClearBucket(kv.CallFromIndex); err != nil {
2021-04-05 07:48:16 +00:00
return err
}
if err := tx.ClearBucket(kv.CallToIndex); err != nil {
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.CallTraces, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.CallTraces, 0); err != nil {
return err
}
return nil
}
func resetTxLookup(tx kv.RwTx) error {
if err := tx.ClearBucket(kv.TxLookup); err != nil {
return err
}
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.TxLookup, 0); err != nil {
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.TxLookup, 0); err != nil {
return err
}
return nil
}
func resetFinish(tx kv.RwTx) error {
2021-04-05 07:48:16 +00:00
if err := stages.SaveStageProgress(tx, stages.Finish, 0); err != nil {
2020-09-11 10:33:46 +00:00
return err
}
if err := stages.SaveStagePruneProgress(tx, stages.Finish, 0); err != nil {
return err
}
2020-09-11 10:33:46 +00:00
return nil
}
func printStages(db kv.Getter) error {
var err error
var progress uint64
w := new(tabwriter.Writer)
defer w.Flush()
w.Init(os.Stdout, 8, 8, 0, '\t', 0)
fmt.Fprintf(w, "Note: prune_at doesn't mean 'all data before were deleted' - it just mean stage.Prune function were run to this block. Because 1 stage may prune multiple data types to different prune distance.\n")
fmt.Fprint(w, "\n \t stage_at \t prune_at\n")
for _, stage := range stages.AllStages {
2020-12-21 11:01:37 +00:00
if progress, err = stages.GetStageProgress(db, stage); err != nil {
return err
}
prunedTo, err := stages.GetStagePruneProgress(db, stage)
if err != nil {
return err
}
fmt.Fprintf(w, "%s \t %d \t %d\n", string(stage), progress, prunedTo)
}
pm, err := prune.Get(db)
if err != nil {
return err
}
fmt.Fprintf(w, "--\n")
2021-07-24 09:50:42 +00:00
fmt.Fprintf(w, "prune distance: %s\n\n", pm.String())
return nil
}