From 09788dbc02cf1164a160aa66c341c79206ee99b8 Mon Sep 17 00:00:00 2001 From: Artem Vorotnikov Date: Sat, 8 May 2021 00:07:49 +0300 Subject: [PATCH] Pruning flag, incremental staged sync (#1899) --- cmd/cons/commands/clique.go | 2 +- cmd/devp2p/internal/ethtest/chain.go | 2 +- cmd/headers/download/downloader.go | 5 ++ cmd/utils/flags.go | 52 ------------- common/dbutils/bucket.go | 5 +- consensus/clique/clique_test.go | 4 +- consensus/clique/snapshot_test.go | 4 +- core/blockchain.go | 33 +-------- core/genesis.go | 13 +--- core/state/database.go | 6 -- eth/backend.go | 16 +--- eth/ethconfig/config.go | 14 +--- eth/protocols/eth/handler_test.go | 2 +- eth/stagedsync/stage_headers_new.go | 28 ++++++- ethdb/storage_mode.go | 22 +++++- ethdb/storage_mode_test.go | 2 + params/eip_ctx.go | 107 --------------------------- params/network_params.go | 6 -- tests/state_test_util.go | 2 +- turbo/cli/default_flags.go | 1 + turbo/cli/flags.go | 10 +++ turbo/stages/blockchain_test.go | 34 ++------- turbo/stages/genesis_test.go | 4 +- 23 files changed, 94 insertions(+), 280 deletions(-) diff --git a/cmd/cons/commands/clique.go b/cmd/cons/commands/clique.go index ae50f34b2..9146584f1 100644 --- a/cmd/cons/commands/clique.go +++ b/cmd/cons/commands/clique.go @@ -264,7 +264,7 @@ func (cs *CliqueServerImpl) initAndConfig(configuration []byte) error { } } var genesisBlock *types.Block - if genesisBlock, _, err = genesis.ToBlock(false /* history */); err != nil { + if genesisBlock, _, err = genesis.ToBlock(); err != nil { return fmt.Errorf("creating genesis block: %w", err) } log.Info("Created genesis block", "hash", genesisBlock.Hash()) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 81255316b..0beae87c1 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -122,7 +122,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { if err = json.Unmarshal(chainConfig, &gen); err != nil { return nil, err } - gblock, _, err := gen.ToBlock(false) + gblock, _, err := gen.ToBlock() if err != nil { return nil, err } diff --git a/cmd/headers/download/downloader.go b/cmd/headers/download/downloader.go index 1d616a37f..71f5c7e54 100644 --- a/cmd/headers/download/downloader.go +++ b/cmd/headers/download/downloader.go @@ -300,6 +300,10 @@ func NewStagedSync( tmpdir string, txPool *core.TxPool, ) (*stagedsync.StagedSync, error) { + var increment uint64 + if sm.Pruning { + increment = params.FullImmutabilityThreshold + } return stages.NewStagedSync(ctx, sm, stagedsync.StageHeadersCfg( @@ -311,6 +315,7 @@ func NewStagedSync( controlServer.penalize, controlServer.requestWakeUpBodies, batchSize, + increment, ), stagedsync.StageBodiesCfg( db, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f84d147b5..79010c326 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -28,7 +28,6 @@ import ( "strings" "text/tabwriter" "text/template" - "time" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -147,51 +146,14 @@ var ( Name: "nocode", Usage: "Exclude contract code (save db lookups)", } - GCModePruningFlag = cli.BoolFlag{ - Name: "pruning", - Usage: `Enable storage pruning`, - } - GCModeLimitFlag = cli.Uint64Flag{ - Name: "pruning.stop_limit", - Usage: `Blockchain pruning limit`, - Value: 1024, - } - GCModeBlockToPruneFlag = cli.Uint64Flag{ - Name: "pruning.processing_limit", - Usage: `Block to prune per tick`, - Value: 20, - } - GCModeTickTimeout = cli.DurationFlag{ - Name: "pruning.tick", - Usage: `Time of tick`, - Value: time.Second * 2, - } - LightServFlag = cli.IntFlag{ - Name: "lightserv", - Usage: "Maximum percentage of time allowed for serving LES requests (0-90)", - Value: 0, - } - LightKDFFlag = cli.BoolFlag{ - Name: "lightkdf", - Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", - } WhitelistFlag = cli.StringFlag{ Name: "whitelist", Usage: "Comma separated block number-to-hash mappings to enforce (=)", } - BloomFilterSizeFlag = cli.Uint64Flag{ - Name: "bloomfilter.size", - Usage: "Megabytes of memory allocated to bloom-filter for pruning", - Value: 2048, - } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", Usage: "Manually specify Berlin fork-block, overriding the bundled setting", } - DownloadOnlyFlag = cli.BoolFlag{ - Name: "download-only", - Usage: "Run in download only mode - only fetch blocks but not process them", - } DebugProtocolFlag = cli.BoolFlag{ Name: "debug-protocol", Usage: "Enable the DBG (debug) protocol", @@ -343,10 +305,6 @@ var ( Name: "fakepow", Usage: "Disables proof-of-work verification", } - NoCompactionFlag = cli.BoolFlag{ - Name: "nocompaction", - Usage: "Disables db compaction after import", - } // RPC settings IPCDisabledFlag = cli.BoolFlag{ Name: "ipcdisable", @@ -1132,16 +1090,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.NetworkID = ctx.GlobalUint64(NetworkIdFlag.Name) } - // todo uncomment after fix pruning - //cfg.Pruning = ctx.GlobalBool(GCModePruningFlag.Name) - cfg.Pruning = false - cfg.BlocksBeforePruning = ctx.GlobalUint64(GCModeLimitFlag.Name) - cfg.BlocksToPrune = ctx.GlobalUint64(GCModeBlockToPruneFlag.Name) - cfg.PruningTimeout = ctx.GlobalDuration(GCModeTickTimeout.Name) - - // Read the value from the flag no matter if it's set or not. - cfg.DownloadOnly = ctx.GlobalBoolT(DownloadOnlyFlag.Name) - cfg.EnableDebugProtocol = ctx.GlobalBool(DebugProtocolFlag.Name) if ctx.GlobalIsSet(DocRootFlag.Name) { diff --git a/common/dbutils/bucket.go b/common/dbutils/bucket.go index 2ac48eeeb..59d6e7f18 100644 --- a/common/dbutils/bucket.go +++ b/common/dbutils/bucket.go @@ -219,9 +219,8 @@ const ( // Keys var ( - // last block that was pruned - // it's saved one in 5 minutes - LastPrunedBlockKey = []byte("LastPrunedBlock") + //StorageModePruning - does node prune. + StorageModePruning = []byte("smPruning") //StorageModeHistory - does node save history. StorageModeHistory = []byte("smHistory") //StorageModeReceipts - does node save receipts. diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 4ae31b5bc..d397aa34b 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -62,7 +62,7 @@ func TestReimportMirroredState(t *testing.T) { // Generate a batch of blocks, each properly signed txCacher := core.NewTxSenderCacher(1) - chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, txCacher) + chain, _ := core.NewBlockChain(db, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, txCacher) defer chain.Stop() blocks, _, err := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) { @@ -112,7 +112,7 @@ func TestReimportMirroredState(t *testing.T) { // flushing the dirty states out. Insert the last block, triggering a sidechain // reimport. txCacher2 := core.NewTxSenderCacher(1) - chain2, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, txCacher2) + chain2, _ := core.NewBlockChain(db, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, txCacher2) defer chain2.Stop() if _, err := stagedsync.InsertBlocksInStages(db, ethdb.DefaultStorageMode, params.AllCliqueProtocolChanges, &vm.Config{}, engine, blocks[2:], true /* checkRoot */); err != nil { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 2e9bed5d7..d22313be9 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -431,14 +431,14 @@ func TestClique(t *testing.T) { engine.fakeDiff = true txCacher := core.NewTxSenderCacher(runtime.NumCPU()) - chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, txCacher) + chain, err := core.NewBlockChain(db, &config, engine, vm.Config{}, nil, txCacher) if err != nil { t.Errorf("test %d: failed to create test chain: %v", i, err) engine.Close() return } - genesisBlock, _, _ := genesis.ToBlock(false) + genesisBlock, _, _ := genesis.ToBlock() blocks, _, err := core.GenerateChain(&config, genesisBlock, engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block gen.SetCoinbase(accounts.address(tt.votes[j].voted)) diff --git a/core/blockchain.go b/core/blockchain.go index 26ed53e1a..08e5bf07a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -85,27 +85,6 @@ const ( TriesInMemory = 128 ) -// CacheConfig contains the configuration values for the trie caching/pruning -// that's resident in a blockchain. -type CacheConfig struct { - Pruning bool - - BlocksBeforePruning uint64 - BlocksToPrune uint64 - PruneTimeout time.Duration - DownloadOnly bool - NoHistory bool -} - -// defaultCacheConfig are the default caching values if none are specified by the -// user (also used during testing). -var defaultCacheConfig = &CacheConfig{ - Pruning: false, - BlocksBeforePruning: 1024, - DownloadOnly: false, - NoHistory: false, -} - // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -122,7 +101,6 @@ var defaultCacheConfig = &CacheConfig{ // canonical chain. type BlockChain struct { chainConfig *params.ChainConfig // Chain & network configuration - cacheConfig *CacheConfig // Cache configuration for pruning db ethdb.Database // Low level persistent database to store final content in @@ -165,17 +143,12 @@ type BlockChain struct { // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, senderCacher *TxSenderCacher) (*BlockChain, error) { - if cacheConfig == nil { - cacheConfig = defaultCacheConfig - } - +func NewBlockChain(db ethdb.Database, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, senderCacher *TxSenderCacher) (*BlockChain, error) { receiptsCache, _ := lru.New(receiptsCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) bc := &BlockChain{ chainConfig: chainConfig, - cacheConfig: cacheConfig, db: db, quit: make(chan struct{}), shouldPreserve: shouldPreserve, @@ -513,10 +486,6 @@ func (bc *BlockChain) ChainDb() ethdb.Database { return bc.db } -func (bc *BlockChain) NoHistory() bool { - return bc.cacheConfig.NoHistory -} - type Pruner interface { Start() error Stop() diff --git a/core/genesis.go b/core/genesis.go index 632cdb585..5d808d935 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -158,10 +158,6 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis, history bool, overwrite bool) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, genesis, nil, history, overwrite) -} - -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideBerlin *big.Int, history bool, overwrite bool) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, ErrGenesisNoConfig } @@ -188,7 +184,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override } // Check whether the genesis block is already written. if genesis != nil { - block, _, err1 := genesis.ToBlock(history) + block, _, err1 := genesis.ToBlock() if err1 != nil { return genesis.Config, common.Hash{}, err1 } @@ -199,9 +195,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideBerlin != nil { - newcfg.BerlinBlock = overrideBerlin - } if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } @@ -261,7 +254,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // ToBlock creates the genesis block and writes state of a genesis specification // to the given database (or discards it if nil). -func (g *Genesis) ToBlock(history bool) (*types.Block, *state.IntraBlockState, error) { +func (g *Genesis) ToBlock() (*types.Block, *state.IntraBlockState, error) { tmpDB := ethdb.NewMemDatabase() defer tmpDB.Close() tx, err := tmpDB.Begin(context.Background(), ethdb.RW) @@ -321,7 +314,7 @@ func (g *Genesis) ToBlock(history bool) (*types.Block, *state.IntraBlockState, e } func (g *Genesis) WriteGenesisState(tx ethdb.RwTx, history bool) (*types.Block, *state.IntraBlockState, error) { - block, statedb, err := g.ToBlock(history) + block, statedb, err := g.ToBlock() if err != nil { return nil, nil, err } diff --git a/core/state/database.go b/core/state/database.go index e30d29a92..bb453c576 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -221,7 +221,6 @@ type TrieDbState struct { aggregateBuffer *Buffer // Merge of all buffers currentBuffer *Buffer historical bool - noHistory bool resolveReads bool retainListBuilder *trie.RetainListBuilder newStream trie.Stream @@ -261,10 +260,6 @@ func (tds *TrieDbState) SetResolveReads(rr bool) { tds.resolveReads = rr } -func (tds *TrieDbState) SetNoHistory(nh bool) { - tds.noHistory = nh -} - func (tds *TrieDbState) Copy() *TrieDbState { tds.tMu.Lock() tcopy := *tds.t @@ -322,7 +317,6 @@ func (tds *TrieDbState) WithNewBuffer() *TrieDbState { aggregateBuffer: aggregateBuffer, currentBuffer: currentBuffer, historical: tds.historical, - noHistory: tds.noHistory, resolveReads: tds.resolveReads, retainListBuilder: tds.retainListBuilder, pw: tds.pw, diff --git a/eth/backend.go b/eth/backend.go index 6d490149a..2742afcbf 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -134,7 +134,7 @@ func New(stack *node.Node, config *ethconfig.Config, gitCommit string) (*Ethereu return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideBerlin, config.StorageMode.History, false /* overwrite */) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis, config.StorageMode.History, false /* overwrite */) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } @@ -309,7 +309,7 @@ func New(stack *node.Node, config *ethconfig.Config, gitCommit string) (*Ethereu return nil, err } - vmConfig, _ := BlockchainRuntimeConfig(config) + vmConfig := BlockchainRuntimeConfig(config) txCacher := core.NewTxSenderCacher(runtime.NumCPU()) if config.TxPool.Journal != "" { @@ -537,22 +537,14 @@ func SendPendingTxsToRpcDaemon(txPool *core.TxPool, notifier *remotedbserver.Eve } } -func BlockchainRuntimeConfig(config *ethconfig.Config) (vm.Config, *core.CacheConfig) { +func BlockchainRuntimeConfig(config *ethconfig.Config) vm.Config { var ( vmConfig = vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, NoReceipts: !config.StorageMode.Receipts, } - cacheConfig = &core.CacheConfig{ - Pruning: config.Pruning, - BlocksBeforePruning: config.BlocksBeforePruning, - BlocksToPrune: config.BlocksToPrune, - PruneTimeout: config.PruningTimeout, - DownloadOnly: config.DownloadOnly, - NoHistory: !config.StorageMode.History, - } ) - return vmConfig, cacheConfig + return vmConfig } // func makeExtraData(extra []byte) []byte { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f533299ee..8bb6e2c24 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -105,7 +105,7 @@ func init() { //go:generate gencodec -type Config -formats toml -out gen_config.go -// Config contains configuration options for of the ETH and LES protocols. +// Config contains configuration options for ETH protocol. type Config struct { // The genesis block, which is inserted if the database is empty. // If nil, the Ethereum main net block is used. @@ -118,7 +118,7 @@ type Config struct { // for nodes to connect to. EthDiscoveryURLs []string - Pruning bool // Whether to disable pruning and flush everything to disk + Pruning bool EnableDownloadV2 bool P2PEnabled bool @@ -132,13 +132,6 @@ type Config struct { // empty if you want to use internal bittorrent snapshot downloader ExternalSnapshotDownloaderAddr string - // DownloadOnly is set when the node does not need to process the blocks, but simply - // download them - DownloadOnly bool - BlocksBeforePruning uint64 - BlocksToPrune uint64 - PruningTimeout time.Duration - // Whitelist of required block number -> hash values to accept Whitelist map[uint64]common.Hash `toml:"-"` @@ -181,8 +174,7 @@ type Config struct { CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` // Berlin block override (TODO: remove after the fork) - StagedSync *stagedsync.StagedSync `toml:"-"` - OverrideBerlin *big.Int `toml:",omitempty"` + StagedSync *stagedsync.StagedSync `toml:"-"` } func CreateConsensusEngine(chainConfig *params.ChainConfig, config interface{}, notify []string, noverify bool) consensus.Engine { diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 501cb6e16..88595dec8 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -81,7 +81,7 @@ func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen) txconfig := core.DefaultTxPoolConfig txconfig.Journal = "" // Don't litter the disk with test journals - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) txCacher := core.NewTxSenderCacher(1) return &testBackend{ db: db, diff --git a/eth/stagedsync/stage_headers_new.go b/eth/stagedsync/stage_headers_new.go index a066915ac..d595def3c 100644 --- a/eth/stagedsync/stage_headers_new.go +++ b/eth/stagedsync/stage_headers_new.go @@ -26,6 +26,7 @@ type HeadersCfg struct { penalize func(context.Context, []headerdownload.PenaltyItem) wakeUpChan chan struct{} batchSize datasize.ByteSize + increment uint64 } func StageHeadersCfg( @@ -37,6 +38,7 @@ func StageHeadersCfg( penalize func(context.Context, []headerdownload.PenaltyItem), wakeUpChan chan struct{}, batchSize datasize.ByteSize, + increment uint64, ) HeadersCfg { return HeadersCfg{ db: db, @@ -47,6 +49,7 @@ func StageHeadersCfg( penalize: penalize, wakeUpChan: wakeUpChan, batchSize: batchSize, + increment: increment, } } @@ -93,7 +96,12 @@ func HeadersForward( return nil } - log.Info(fmt.Sprintf("[%s] Processing headers...", logPrefix), "from", headerProgress) + incrementalTarget := headerProgress + cfg.increment + if cfg.increment > 0 { + log.Info(fmt.Sprintf("[%s] Processing headers...", logPrefix), "from", headerProgress, "target", incrementalTarget) + } else { + log.Info(fmt.Sprintf("[%s] Processing headers...", logPrefix), "from", headerProgress) + } batch := ethdb.NewBatch(tx) defer batch.Rollback() logEvery := time.NewTicker(logInterval) @@ -110,7 +118,23 @@ func HeadersForward( stopped := false timer := time.NewTimer(1 * time.Second) // Check periodically even in the absence of incoming messages prevProgress := headerProgress + + // FIXME: remove this hack + if cfg.increment > 0 { + if cfg.hd.TopSeenHeight() > incrementalTarget { + initialCycle = true + } + } + for !stopped { + if cfg.increment > 0 { + progress := cfg.hd.Progress() + if progress > incrementalTarget { + log.Info(fmt.Sprintf("[%s] Target reached, exiting cycle", logPrefix), "progress", progress, "target", incrementalTarget) + break + } + } + currentTime := uint64(time.Now().Unix()) req, penalties := cfg.hd.RequestMoreHeaders(currentTime) if req != nil { @@ -218,7 +242,7 @@ func HeadersForward( return err } } - log.Info("Processed", "highest", headerInserter.GetHighest()) + log.Info(fmt.Sprintf("[%s] Processed", logPrefix), "highest", headerInserter.GetHighest()) if stopped { return fmt.Errorf("interrupted") } diff --git a/ethdb/storage_mode.go b/ethdb/storage_mode.go index 09a306b09..4ece6eca7 100644 --- a/ethdb/storage_mode.go +++ b/ethdb/storage_mode.go @@ -7,16 +7,20 @@ import ( ) type StorageMode struct { + Pruning bool History bool Receipts bool TxIndex bool CallTraces bool } -var DefaultStorageMode = StorageMode{History: true, Receipts: true, TxIndex: true, CallTraces: false} +var DefaultStorageMode = StorageMode{Pruning: false, History: true, Receipts: true, TxIndex: true, CallTraces: false} func (m StorageMode) ToString() string { modeString := "" + if m.Pruning { + modeString += "p" + } if m.History { modeString += "h" } @@ -58,6 +62,12 @@ func GetStorageModeFromDB(db KVGetter) (StorageMode, error) { v []byte err error ) + v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModePruning) + if err != nil { + return StorageMode{}, err + } + sm.Pruning = len(v) == 1 && v[0] == 1 + v, err = db.GetOne(dbutils.DatabaseInfoBucket, dbutils.StorageModeHistory) if err != nil { return StorageMode{}, err @@ -88,6 +98,11 @@ func OverrideStorageMode(db Database, sm StorageMode) error { var ( err error ) + err = setMode(db, dbutils.StorageModePruning, sm.Pruning) + if err != nil { + return err + } + err = setMode(db, dbutils.StorageModeHistory, sm.History) if err != nil { return err @@ -115,6 +130,11 @@ func SetStorageModeIfNotExist(db Database, sm StorageMode) error { var ( err error ) + err = setModeOnEmpty(db, dbutils.StorageModePruning, sm.Pruning) + if err != nil { + return err + } + err = setModeOnEmpty(db, dbutils.StorageModeHistory, sm.History) if err != nil { return err diff --git a/ethdb/storage_mode_test.go b/ethdb/storage_mode_test.go index 96e76ac4a..aa421a163 100644 --- a/ethdb/storage_mode_test.go +++ b/ethdb/storage_mode_test.go @@ -23,6 +23,7 @@ func TestSetStorageModeIfNotExist(t *testing.T) { true, true, true, + true, }) if err != nil { t.Fatal(err) @@ -38,6 +39,7 @@ func TestSetStorageModeIfNotExist(t *testing.T) { true, true, true, + true, }) { spew.Dump(sm) t.Fatal("not equal") diff --git a/params/eip_ctx.go b/params/eip_ctx.go index bc3b61f55..f60449a03 100644 --- a/params/eip_ctx.go +++ b/params/eip_ctx.go @@ -16,7 +16,6 @@ const ( IsConstantinopleEnabled IsPetersburgEnabled BlockNumber - NoHistory WithHistoryHighest ) @@ -32,10 +31,6 @@ func (c *ChainConfig) WithEIPsFlags(ctx context.Context, blockNum uint64) contex return ctx } -func WithNoHistory(ctx context.Context, defaultValue bool, f noHistFunc) context.Context { - return context.WithValue(ctx, NoHistory, getIsNoHistory(defaultValue, f)) -} - func GetForkFlag(ctx context.Context, name configKey) bool { b := ctx.Value(name) if b == nil { @@ -57,105 +52,3 @@ func GetBlockNumber(ctx context.Context) *big.Int { } return nil } - -func GetNoHistoryByBlock(ctx context.Context, currentBlock *big.Int) (context.Context, bool) { - if currentBlock == nil { - return ctx, false - } - - key := getNoHistoryByBlock(currentBlock) - v, ok := getNoHistoryByKey(ctx, key) - if ok { - if !v { - ctx = updateHighestWithHistory(ctx, currentBlock) - } - return ctx, v - } - - return withNoHistory(ctx, currentBlock, key) -} - -func withNoHistory(ctx context.Context, currentBlock *big.Int, key configKey) (context.Context, bool) { - v := getNoHistory(ctx, currentBlock) - ctx = context.WithValue(ctx, key, v) - if !v { - ctx = updateHighestWithHistory(ctx, currentBlock) - } - - return ctx, v -} - -func updateHighestWithHistory(ctx context.Context, currentBlock *big.Int) context.Context { - highestWithHistory := getWithHistoryHighest(ctx) - var currentIsLower bool - if highestWithHistory != nil { - currentIsLower = currentBlock.Cmp(highestWithHistory) < 0 - } - if !currentIsLower { - ctx = setWithHistoryHighestByBlock(ctx, currentBlock) - } - return ctx -} - -func getNoHistory(ctx context.Context, currentBlock *big.Int) bool { - v := ctx.Value(NoHistory) - if v == nil { - return false - } - if val, ok := v.(noHistFunc); ok { - return val(currentBlock) - } - return false -} - -func getWithHistoryHighest(ctx context.Context) *big.Int { - v := ctx.Value(WithHistoryHighest) - if v == nil { - return nil - } - if val, ok := v.(*big.Int); ok { - return val - } - return nil -} - -func setWithHistoryHighestByBlock(ctx context.Context, block *big.Int) context.Context { - return context.WithValue(ctx, WithHistoryHighest, block) -} - -func getNoHistoryByKey(ctx context.Context, key configKey) (bool, bool) { - if key < 0 { - return false, false - } - v := ctx.Value(key) - if v == nil { - return false, false - } - if val, ok := v.(bool); ok { - return val, true - } - return false, false -} - -func getNoHistoryByBlock(block *big.Int) configKey { - if block == nil { - return configKey(-1) - } - return configKey(block.Uint64() + 1000) -} - -func GetNoHistory(ctx context.Context) (context.Context, bool) { - return GetNoHistoryByBlock(ctx, GetBlockNumber(ctx)) -} - -type noHistFunc func(currentBlock *big.Int) bool - -func getIsNoHistory(defaultValue bool, f noHistFunc) noHistFunc { - return func(currentBlock *big.Int) bool { - if f == nil { - return defaultValue - } - - return f(currentBlock) - } -} diff --git a/params/network_params.go b/params/network_params.go index 9311b5e2d..e914ff51a 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -58,10 +58,4 @@ const ( // hard limit against deep ancestors, by the blockchain against deep reorgs, by // the freezer as the cutoff threshold and by clique as the snapshot trust limit. FullImmutabilityThreshold = 90000 - - // LightImmutabilityThreshold is the number of blocks after which a header chain - // segment is considered immutable for light client(i.e. soft finality). It is used by - // the downloader as a hard limit against deep ancestors, by the blockchain against deep - // reorgs, by the light pruner as the pruning validity guarantee. - LightImmutabilityThreshold = 30000 ) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 49d545860..468725f26 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -175,7 +175,7 @@ func (t *StateTest) RunNoVerify(ctx context.Context, tx ethdb.Database, subtest return nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips - block, _, err := t.genesis(config).ToBlock(false) + block, _, err := t.genesis(config).ToBlock() if err != nil { return nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index 92f35a320..3ab0f1029 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -21,6 +21,7 @@ var DefaultFlags = []cli.Flag{ utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, + PruningFlag, StorageModeFlag, SnapshotModeFlag, SeedSnapshotsFlag, diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index c2327150e..ddd8bbce4 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -55,6 +55,11 @@ var ( Usage: "enable experimental downloader v2", } + PruningFlag = cli.BoolFlag{ + Name: "prune", + Usage: "Enable pruning ancient data", + } + StorageModeFlag = cli.StringFlag{ Name: "storage-mode", Usage: `Configures the storage mode of the app: @@ -125,6 +130,11 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config) { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } cfg.StorageMode = mode + cfg.StorageMode.Pruning = ctx.GlobalBool(PruningFlag.Name) + if cfg.StorageMode.Pruning && !cfg.EnableDownloadV2 { + log.Info("Pruning is on, switching to new downloader") + cfg.EnableDownloadV2 = true + } snMode, err := snapshotsync.SnapshotModeFromString(ctx.GlobalString(SnapshotModeFlag.Name)) if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index dd8581762..5e7d2aaad 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -78,17 +78,6 @@ func newCanonical(engine consensus.Engine, n int, full bool) (*ethdb.ObjectDatab panic(err) } - // Initialize a fresh chain with only a genesis block - //cacheConfig := &core.CacheConfig{ - // TrieCleanLimit: 256, - // TrieDirtyLimit: 256, - // TrieTimeLimit: 5 * time.Minute, - // NoHistory: false, - // Pruning: false, - //} - //txCacher := core.NewTxSenderCacher(1) - //blockchain, _ := core.NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, txCacher) - // Create and inject the requested chain if n == 0 { return db, genesis, nil @@ -693,12 +682,8 @@ func TestLogReorgs(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - cacheConfig := &core.CacheConfig{ - NoHistory: false, - Pruning: false, - } txCacher := core.NewTxSenderCacher(1) - blockchain, _ := core.NewBlockChain(db, cacheConfig, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) + blockchain, _ := core.NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) blockchain.EnableReceipts(true) defer blockchain.Stop() @@ -764,7 +749,7 @@ func TestLogRebirth(t *testing.T) { genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, txCacher) + blockchain, _ = core.NewBlockChain(db, gspec.Config, engine, vm.Config{}, nil, txCacher) ) blockchain.EnableReceipts(true) defer blockchain.Stop() @@ -850,7 +835,7 @@ func TestSideLogRebirth(t *testing.T) { ) txCacher := core.NewTxSenderCacher(1) - blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) + blockchain, _ := core.NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) defer blockchain.Stop() newLogCh := make(chan []*types.Log, 10) @@ -974,7 +959,7 @@ func TestEIP155Transition(t *testing.T) { ) txCacher := core.NewTxSenderCacher(1) - blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) + blockchain, _ := core.NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) defer blockchain.Stop() blocks, _, chainErr := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *core.BlockGen) { @@ -1090,15 +1075,8 @@ func doModesTest(history, preimages, receipts, txlookup bool) error { genesis, _, _ = gspec.Commit(db, history) ) - cacheConfig := &core.CacheConfig{ - Pruning: false, - BlocksBeforePruning: 1024, - DownloadOnly: false, - NoHistory: !history, - } - txCacher := core.NewTxSenderCacher(1) - blockchain, _ := core.NewBlockChain(db, cacheConfig, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) + blockchain, _ := core.NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) blockchain.EnableReceipts(receipts) blockchain.EnablePreimages(preimages) blockchain.EnableTxLookupIndex(txlookup) @@ -1314,7 +1292,7 @@ func TestDoubleAccountRemoval(t *testing.T) { ) txCacher := core.NewTxSenderCacher(1) - blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) + blockchain, _ := core.NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, txCacher) defer blockchain.Stop() var theAddr common.Address diff --git a/turbo/stages/genesis_test.go b/turbo/stages/genesis_test.go index 90ddcaf29..ede59effd 100644 --- a/turbo/stages/genesis_test.go +++ b/turbo/stages/genesis_test.go @@ -35,14 +35,14 @@ import ( func TestDefaultGenesisBlock(t *testing.T) { db := ethdb.NewMemDatabase() defer db.Close() - block, _, _ := core.DefaultGenesisBlock().ToBlock(true) + block, _, _ := core.DefaultGenesisBlock().ToBlock() if block.Hash() != params.MainnetGenesisHash { t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash) } var err error db1 := ethdb.NewMemDatabase() defer db1.Close() - block, _, err = core.DefaultRopstenGenesisBlock().ToBlock(true) + block, _, err = core.DefaultRopstenGenesisBlock().ToBlock() if err != nil { t.Errorf("error: %w", err) }