2020-07-14 01:56:29 +00:00
package commands
2020-07-07 04:00:25 +00:00
import (
"context"
"fmt"
2020-07-25 07:35:08 +00:00
"os"
2020-08-05 10:13:35 +00:00
"text/tabwriter"
2020-07-07 04:00:25 +00:00
2021-07-29 11:53:13 +00:00
"github.com/ledgerwatch/erigon-lib/kv"
2021-05-20 18:25:53 +00:00
"github.com/ledgerwatch/erigon/cmd/utils"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/eth/stagedsync"
"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
"github.com/ledgerwatch/erigon/ethdb/prune"
2021-07-29 10:23:23 +00:00
"github.com/ledgerwatch/log/v3"
2020-07-07 04:00:25 +00:00
"github.com/spf13/cobra"
)
var cmdResetState = & cobra . Command {
Use : "reset_state" ,
2020-08-10 17:46:06 +00:00
Short : "Reset StateStages (5,6,7,8,9,10) and buckets" ,
2020-07-07 04:00:25 +00:00
RunE : func ( cmd * cobra . Command , args [ ] string ) error {
2021-04-26 12:39:34 +00:00
ctx , _ := utils . RootContext ( )
2021-07-28 02:47:38 +00:00
logger := log . New ( )
db := openDB ( chaindata , logger , true )
2020-10-28 03:18:10 +00:00
defer db . Close ( )
2021-07-28 02:47:38 +00:00
err := resetState ( db , logger , ctx )
2020-07-07 04:00:25 +00:00
if err != nil {
log . Error ( err . Error ( ) )
return err
}
2020-08-10 23:38:19 +00:00
return nil
} ,
}
2020-07-07 04:00:25 +00:00
func init ( ) {
2021-04-19 07:25:26 +00:00
withDatadir ( cmdResetState )
2021-06-16 04:31:25 +00:00
withChain ( cmdResetState )
2020-07-07 04:00:25 +00:00
rootCmd . AddCommand ( cmdResetState )
}
2021-07-28 02:47:38 +00:00
func resetState ( db kv . RwDB , logger log . Logger , ctx context . Context ) error {
if err := db . View ( ctx , func ( tx kv . Tx ) error { return printStages ( tx ) } ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
// don't reset senders here
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , stagedsync . ResetHashState ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , stagedsync . ResetIH ) ; err != nil {
2021-02-21 18:41:59 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , resetHistory ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , resetLogIndex ) ; err != nil {
2020-09-28 17:18:36 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , resetCallTraces ) ; err != nil {
2020-10-12 08:39:04 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , resetTxLookup ) ; err != nil {
2020-07-07 10:07:14 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , resetFinish ) ; err != nil {
2020-09-11 10:33:46 +00:00
return err
}
2020-07-07 04:00:25 +00:00
2021-06-25 18:13:40 +00:00
genesis , _ := byChain ( )
2021-07-28 02:47:38 +00:00
if err := db . Update ( ctx , func ( tx kv . RwTx ) error { return resetExec ( tx , genesis ) } ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-06-25 18:13:40 +00:00
// set genesis after reset all buckets
2020-07-07 04:00:25 +00:00
fmt . Printf ( "After reset: \n" )
2021-07-28 02:47:38 +00:00
if err := db . View ( ctx , func ( tx kv . Tx ) error { return printStages ( tx ) } ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
return nil
}
2020-07-07 10:07:14 +00:00
2021-07-28 02:47:38 +00:00
func resetSenders ( tx kv . RwTx ) error {
if err := tx . ClearBucket ( kv . Senders ) ; err != nil {
2020-07-07 07:11:10 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . Senders , 0 ) ; err != nil {
2020-07-15 11:53:12 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . Senders , 0 ) ; err != nil {
return err
}
2020-07-15 11:53:12 +00:00
return nil
2020-07-07 07:11:10 +00:00
}
2020-07-07 04:00:25 +00:00
2021-07-28 02:47:38 +00:00
func resetExec ( tx kv . RwTx , g * core . Genesis ) error {
if err := tx . ClearBucket ( kv . HashedAccounts ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . HashedStorage ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . ContractCode ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 03:43:51 +00:00
if err := tx . ClearBucket ( kv . PlainState ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . AccountChangeSet ) ; err != nil {
2020-07-15 11:53:12 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . StorageChangeSet ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . PlainContractCode ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . Receipts ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . Log ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . IncarnationMap ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 03:43:51 +00:00
if err := tx . ClearBucket ( kv . Code ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . CallTraceSet ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . Epoch ) ; err != nil {
2021-07-12 15:27:25 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . PendingEpoch ) ; err != nil {
2021-07-21 11:13:26 +00:00
return err
}
2021-05-10 15:07:22 +00:00
if err := stages . SaveStageProgress ( tx , stages . Execution , 0 ) ; err != nil {
2020-07-15 11:53:12 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . Execution , 0 ) ; err != nil {
return err
}
2021-06-25 18:13:40 +00:00
2021-11-30 12:36:01 +00:00
if _ , _ , err := g . WriteGenesisState ( tx ) ; err != nil {
2021-06-25 18:13:40 +00:00
return err
}
2020-07-15 11:53:12 +00:00
return nil
2020-07-07 04:00:25 +00:00
}
2021-07-28 02:47:38 +00:00
func resetHistory ( tx kv . RwTx ) error {
if err := tx . ClearBucket ( kv . AccountsHistory ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . StorageHistory ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . AccountHistoryIndex , 0 ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . StorageHistoryIndex , 0 ) ; err != nil {
2020-07-15 11:53:12 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . AccountHistoryIndex , 0 ) ; err != nil {
return err
}
if err := stages . SaveStagePruneProgress ( tx , stages . StorageHistoryIndex , 0 ) ; err != nil {
return err
}
2020-07-07 04:00:25 +00:00
return nil
}
2021-07-28 02:47:38 +00:00
func resetLogIndex ( tx kv . RwTx ) error {
if err := tx . ClearBucket ( kv . LogAddressIndex ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . LogTopicIndex ) ; err != nil {
2020-09-28 17:18:36 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . LogIndex , 0 ) ; err != nil {
2020-09-28 17:18:36 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . LogIndex , 0 ) ; err != nil {
return err
}
2020-09-28 17:18:36 +00:00
return nil
}
2021-07-28 02:47:38 +00:00
func resetCallTraces ( tx kv . RwTx ) error {
if err := tx . ClearBucket ( kv . CallFromIndex ) ; err != nil {
2021-04-05 07:48:16 +00:00
return err
}
2021-07-28 02:47:38 +00:00
if err := tx . ClearBucket ( kv . CallToIndex ) ; err != nil {
2020-10-12 08:39:04 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . CallTraces , 0 ) ; err != nil {
2020-10-12 08:39:04 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . CallTraces , 0 ) ; err != nil {
return err
}
2020-10-12 08:39:04 +00:00
return nil
}
2021-07-28 02:47:38 +00:00
func resetTxLookup ( tx kv . RwTx ) error {
if err := tx . ClearBucket ( kv . TxLookup ) ; err != nil {
2020-07-07 10:07:14 +00:00
return err
}
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . TxLookup , 0 ) ; err != nil {
2020-07-07 10:07:14 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . TxLookup , 0 ) ; err != nil {
return err
}
2020-07-07 10:07:14 +00:00
return nil
}
2020-09-03 08:20:04 +00:00
2021-07-28 02:47:38 +00:00
func resetFinish ( tx kv . RwTx ) error {
2021-04-05 07:48:16 +00:00
if err := stages . SaveStageProgress ( tx , stages . Finish , 0 ) ; err != nil {
2020-09-11 10:33:46 +00:00
return err
}
2021-07-21 04:03:58 +00:00
if err := stages . SaveStagePruneProgress ( tx , stages . Finish , 0 ) ; err != nil {
return err
}
2020-09-11 10:33:46 +00:00
return nil
}
2021-07-28 02:47:38 +00:00
func printStages ( db kv . Getter ) error {
2020-07-07 04:00:25 +00:00
var err error
var progress uint64
2020-08-05 10:13:35 +00:00
w := new ( tabwriter . Writer )
defer w . Flush ( )
w . Init ( os . Stdout , 8 , 8 , 0 , '\t' , 0 )
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
fmt . Fprintf ( w , "Note: prune_at doesn't mean 'all data before were deleted' - it just mean stage.Prune function were run to this block. Because 1 stage may prune multiple data types to different prune distance.\n" )
2021-07-21 11:13:26 +00:00
fmt . Fprint ( w , "\n \t stage_at \t prune_at\n" )
2020-09-05 16:07:27 +00:00
for _ , stage := range stages . AllStages {
2020-12-21 11:01:37 +00:00
if progress , err = stages . GetStageProgress ( db , stage ) ; err != nil {
2020-07-07 04:00:25 +00:00
return err
}
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
prunedTo , err := stages . GetStagePruneProgress ( db , stage )
if err != nil {
return err
}
2021-07-21 11:13:26 +00:00
fmt . Fprintf ( w , "%s \t %d \t %d\n" , string ( stage ) , progress , prunedTo )
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
}
pm , err := prune . Get ( db )
if err != nil {
return err
2020-07-07 04:00:25 +00:00
}
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
fmt . Fprintf ( w , "--\n" )
2021-07-24 09:50:42 +00:00
fmt . Fprintf ( w , "prune distance: %s\n\n" , pm . String ( ) )
2020-07-07 04:00:25 +00:00
return nil
}