From d7159cd8bd07f358b68389366a1abaeb30f0e5eb Mon Sep 17 00:00:00 2001 From: Alexandr Borodulin Date: Thu, 23 Sep 2021 05:13:19 +0300 Subject: [PATCH] More convenient pruning for non-PoW consensus: add --prune.*.before flags (#2714) --- cmd/integration/commands/flags.go | 2 + cmd/integration/commands/stages.go | 15 +- eth/ethconfig/gen_config.go | 7 +- eth/stagedsync/stage_execute_test.go | 8 +- ethdb/prune/storage_mode.go | 231 ++++++++++++++++++--------- ethdb/prune/storage_mode_test.go | 58 ++++++- go.mod | 2 +- go.sum | 5 +- migrations/migrations.go | 1 + migrations/set_prune_type.go | 41 +++++ turbo/cli/default_flags.go | 4 + turbo/cli/flags.go | 39 ++++- turbo/stages/blockchain_test.go | 11 +- 13 files changed, 331 insertions(+), 93 deletions(-) create mode 100644 migrations/set_prune_type.go diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go index 76d2d6725..a3ed21b8e 100644 --- a/cmd/integration/commands/flags.go +++ b/cmd/integration/commands/flags.go @@ -25,6 +25,8 @@ var ( txtrace bool // Whether to trace the execution (should only be used together eith `block`) pruneFlag string pruneH, pruneR, pruneT, pruneC uint64 + pruneHBefore, pruneRBefore uint64 + pruneTBefore, pruneCBefore uint64 experiments []string chain string // Which chain to use (mainnet, ropsten, rinkeby, goerli, etc.) ) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 76968f660..12244132e 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -364,10 +364,14 @@ func init() { withDatadir(cmdSetPrune) withChain(cmdSetPrune) cmdSetPrune.Flags().StringVar(&pruneFlag, "prune", "hrtc", "") - cmdSetPrune.Flags().Uint64Var(&pruneH, "--prune.h.older", 0, "") - cmdSetPrune.Flags().Uint64Var(&pruneR, "--prune.r.older", 0, "") - cmdSetPrune.Flags().Uint64Var(&pruneT, "--prune.t.older", 0, "") - cmdSetPrune.Flags().Uint64Var(&pruneC, "--prune.c.older", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneH, "prune.h.older", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneR, "prune.r.older", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneT, "prune.t.older", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneC, "prune.c.older", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneHBefore, "prune.h.before", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneRBefore, "prune.r.before", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneTBefore, "prune.t.before", 0, "") + cmdSetPrune.Flags().Uint64Var(&pruneCBefore, "prune.c.before", 0, "") cmdSetPrune.Flags().StringSliceVar(&experiments, "experiments", nil, "Storage mode to override database") rootCmd.AddCommand(cmdSetPrune) } @@ -1035,7 +1039,8 @@ func stage(st *stagedsync.Sync, tx kv.Tx, db kv.RoDB, stage stages.SyncStage) *s } func overrideStorageMode(db kv.RwDB) error { - pm, err := prune.FromCli(pruneFlag, pruneH, pruneR, pruneT, pruneC, experiments) + pm, err := prune.FromCli(pruneFlag, pruneH, pruneR, pruneT, pruneC, + pruneHBefore, pruneRBefore, pruneTBefore, pruneCBefore, experiments) if err != nil { return err } diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 7ef951273..baa9d077b 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -54,6 +54,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { PruneR *uint64 PruneT *uint64 PruneC *uint64 + PruneBeforeH *uint64 + PruneBeforeR *uint64 + PruneBeforeT *uint64 + PruneBeforeC *uint64 Experiments *[]string OnlyAnnounce *bool SkipBcVersionCheck *bool `toml:"-"` @@ -83,7 +87,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { c.Whitelist = dec.Whitelist } if dec.Mode != nil { - mode, err := prune.FromCli(*dec.Mode, *dec.PruneH, *dec.PruneR, *dec.PruneT, *dec.PruneC, *dec.Experiments) + mode, err := prune.FromCli(*dec.Mode, *dec.PruneH, *dec.PruneR, *dec.PruneT, *dec.PruneC, + *dec.PruneBeforeH, *dec.PruneBeforeR, *dec.PruneBeforeT, *dec.PruneBeforeC, *dec.Experiments) if err != nil { return err } diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index b50cfb157..10f64653c 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -87,7 +87,7 @@ func TestPruneExecution(t *testing.T) { s := &PruneState{ID: stages.Execution, ForwardProgress: 50} // check pruning distance > than current stage progress - err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 100, Receipts: 101, CallTraces: 200}}, ctx, false) + err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(100), Receipts: prune.Distance(101), CallTraces: prune.Distance(200)}}, ctx, false) assert.NoError(err) available, err = changeset.AvailableFrom(tx) @@ -98,7 +98,8 @@ func TestPruneExecution(t *testing.T) { assert.Equal(uint64(1), available) // pruning distance, first run - err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false) + err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(5), + Receipts: prune.Distance(15), CallTraces: prune.Distance(25)}}, ctx, false) assert.NoError(err) available, err = changeset.AvailableFrom(tx) @@ -109,7 +110,8 @@ func TestPruneExecution(t *testing.T) { assert.Equal(uint64(45), available) // pruning distance, second run - err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false) + err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(5), + Receipts: prune.Distance(15), CallTraces: prune.Distance(25)}}, ctx, false) assert.NoError(err) available, err = changeset.AvailableFrom(tx) diff --git a/ethdb/prune/storage_mode.go b/ethdb/prune/storage_mode.go index 2ab2930c6..1042e3c25 100644 --- a/ethdb/prune/storage_mode.go +++ b/ethdb/prune/storage_mode.go @@ -11,10 +11,10 @@ import ( var DefaultMode = Mode{ Initialised: true, - History: math.MaxUint64, // all off - Receipts: math.MaxUint64, - TxIndex: math.MaxUint64, - CallTraces: math.MaxUint64, + History: Distance(math.MaxUint64), // all off + Receipts: Distance(math.MaxUint64), + TxIndex: Distance(math.MaxUint64), + CallTraces: Distance(math.MaxUint64), Experiments: Experiments{}, // all off } @@ -22,7 +22,8 @@ type Experiments struct { TEVM bool } -func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallTraces uint64, experiments []string) (Mode, error) { +func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallTraces, + beforeH, beforeR, beforeT, beforeC uint64, experiments []string) (Mode, error) { mode := DefaultMode if flags == "default" || flags == "disabled" { return DefaultMode, nil @@ -31,13 +32,13 @@ func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallT for _, flag := range flags { switch flag { case 'h': - mode.History = params.FullImmutabilityThreshold + mode.History = Distance(params.FullImmutabilityThreshold) case 'r': - mode.Receipts = params.FullImmutabilityThreshold + mode.Receipts = Distance(params.FullImmutabilityThreshold) case 't': - mode.TxIndex = params.FullImmutabilityThreshold + mode.TxIndex = Distance(params.FullImmutabilityThreshold) case 'c': - mode.CallTraces = params.FullImmutabilityThreshold + mode.CallTraces = Distance(params.FullImmutabilityThreshold) default: return DefaultMode, fmt.Errorf("unexpected flag found: %c", flag) } @@ -56,6 +57,19 @@ func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallT mode.CallTraces = Distance(exactCallTraces) } + if beforeH > 0 { + mode.History = Before(beforeH) + } + if beforeR > 0 { + mode.Receipts = Before(beforeR) + } + if beforeT > 0 { + mode.TxIndex = Before(beforeT) + } + if beforeC > 0 { + mode.CallTraces = Before(beforeC) + } + for _, ex := range experiments { switch ex { case "tevm": @@ -74,61 +88,64 @@ func Get(db kv.Getter) (Mode, error) { prune := DefaultMode prune.Initialised = true - v, err := db.GetOne(kv.DatabaseInfo, kv.PruneDistanceHistory) + blockAmount, err := get(db, kv.PruneHistory) if err != nil { return prune, err } - if v != nil { - prune.History = Distance(binary.BigEndian.Uint64(v)) - } else { - prune.History = math.MaxUint64 - } - v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceReceipts) - if err != nil { - return prune, err - } - if v != nil { - prune.Receipts = Distance(binary.BigEndian.Uint64(v)) - } else { - prune.Receipts = math.MaxUint64 - } - v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceTxIndex) - if err != nil { - return prune, err - } - if v != nil { - prune.TxIndex = Distance(binary.BigEndian.Uint64(v)) - } else { - prune.TxIndex = math.MaxUint64 + if blockAmount != nil { + prune.History = blockAmount } - v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceCallTraces) + blockAmount, err = get(db, kv.PruneReceipts) if err != nil { return prune, err } - if v != nil { - prune.CallTraces = Distance(binary.BigEndian.Uint64(v)) - } else { - prune.CallTraces = math.MaxUint64 + if blockAmount != nil { + prune.Receipts = blockAmount } - v, err = db.GetOne(kv.DatabaseInfo, kv.StorageModeTEVM) + blockAmount, err = get(db, kv.PruneTxIndex) + if err != nil { + return prune, err + } + if blockAmount != nil { + prune.TxIndex = blockAmount + } + + blockAmount, err = get(db, kv.PruneCallTraces) + if err != nil { + return prune, err + } + if blockAmount != nil { + prune.CallTraces = blockAmount + } + + v, err := db.GetOne(kv.DatabaseInfo, kv.StorageModeTEVM) if err != nil { return prune, err } prune.Experiments.TEVM = len(v) == 1 && v[0] == 1 + return prune, nil } type Mode struct { Initialised bool // Set when the values are initialised (not default) - History Distance - Receipts Distance - TxIndex Distance - CallTraces Distance + History BlockAmount + Receipts BlockAmount + TxIndex BlockAmount + CallTraces BlockAmount Experiments Experiments } +type BlockAmount interface { + PruneTo(stageHead uint64) uint64 + Enabled() bool + toValue() uint64 + dbType() []byte + useDefaultValue() bool +} + // Distance amount of blocks to keep in DB // but manual manipulation with such distance is very unsafe // for example: @@ -136,7 +153,10 @@ type Mode struct { // may delete whole db - because of uint64 underflow when pruningDistance > currentStageProgress type Distance uint64 -func (p Distance) Enabled() bool { return p != math.MaxUint64 } +func (p Distance) Enabled() bool { return p != math.MaxUint64 } +func (p Distance) toValue() uint64 { return uint64(p) } +func (p Distance) useDefaultValue() bool { return uint64(p) == params.FullImmutabilityThreshold } +func (p Distance) dbType() []byte { return kv.PruneTypeOlder } func (p Distance) PruneTo(stageHead uint64) uint64 { if p == 0 { @@ -148,6 +168,22 @@ func (p Distance) PruneTo(stageHead uint64) uint64 { return stageHead - uint64(p) } +// Before number after which keep in DB +type Before uint64 + +func (b Before) Enabled() bool { return b > 0 } +func (b Before) toValue() uint64 { return uint64(b) } +func (b Before) useDefaultValue() bool { return uint64(b) == 0 } +func (b Before) dbType() []byte { return kv.PruneTypeBefore } + +func (b Before) PruneTo(uint64) uint64 { + if b == 0 { + return uint64(b) + } + + return uint64(b) - 1 +} + func (m Mode) String() string { if !m.Initialised { return "default" @@ -155,31 +191,31 @@ func (m Mode) String() string { long := "" short := "--prune=" if m.History.Enabled() { - if m.History == params.FullImmutabilityThreshold { + if m.History.useDefaultValue() { short += "h" } else { - long += fmt.Sprintf(" --prune.h.older=%d", m.History) + long += fmt.Sprintf(" --prune.h.%s=%d", m.History.dbType(), m.History.toValue()) } } if m.Receipts.Enabled() { - if m.Receipts == params.FullImmutabilityThreshold { + if m.Receipts.useDefaultValue() { short += "r" } else { - long += fmt.Sprintf(" --prune.r.older=%d", m.Receipts) + long += fmt.Sprintf(" --prune.r.%s=%d", m.Receipts.dbType(), m.Receipts.toValue()) } } if m.TxIndex.Enabled() { - if m.TxIndex == params.FullImmutabilityThreshold { + if m.TxIndex.useDefaultValue() { short += "t" } else { - long += fmt.Sprintf(" --prune.t.older=%d", m.TxIndex) + long += fmt.Sprintf(" --prune.t.%s=%d", m.TxIndex.dbType(), m.TxIndex.toValue()) } } if m.CallTraces.Enabled() { - if m.CallTraces == params.FullImmutabilityThreshold { + if m.CallTraces.useDefaultValue() { short += "c" } else { - long += fmt.Sprintf(" --prune.c.older=%d", m.CallTraces) + long += fmt.Sprintf(" --prune.c.%s=%d", m.CallTraces.dbType(), m.CallTraces.toValue()) } } if m.Experiments.TEVM { @@ -193,22 +229,22 @@ func Override(db kv.RwTx, sm Mode) error { err error ) - err = setDistance(db, kv.PruneDistanceHistory, sm.History) + err = set(db, kv.PruneHistory, sm.History) if err != nil { return err } - err = setDistance(db, kv.PruneDistanceReceipts, sm.Receipts) + err = set(db, kv.PruneReceipts, sm.Receipts) if err != nil { return err } - err = setDistance(db, kv.PruneDistanceTxIndex, sm.TxIndex) + err = set(db, kv.PruneTxIndex, sm.TxIndex) if err != nil { return err } - err = setDistance(db, kv.PruneDistanceCallTraces, sm.CallTraces) + err = set(db, kv.PruneCallTraces, sm.CallTraces) if err != nil { return err } @@ -229,24 +265,18 @@ func SetIfNotExist(db kv.GetPut, pm Mode) error { pm = DefaultMode } - err = setDistanceOnEmpty(db, kv.PruneDistanceHistory, pm.History) - if err != nil { - return err + pruneDBData := map[string]BlockAmount{ + string(kv.PruneHistory): pm.History, + string(kv.PruneReceipts): pm.Receipts, + string(kv.PruneTxIndex): pm.TxIndex, + string(kv.PruneCallTraces): pm.CallTraces, } - err = setDistanceOnEmpty(db, kv.PruneDistanceReceipts, pm.Receipts) - if err != nil { - return err - } - - err = setDistanceOnEmpty(db, kv.PruneDistanceTxIndex, pm.TxIndex) - if err != nil { - return err - } - - err = setDistanceOnEmpty(db, kv.PruneDistanceCallTraces, pm.CallTraces) - if err != nil { - return err + for key, value := range pruneDBData { + err = setOnEmpty(db, []byte(key), value) + if err != nil { + return err + } } err = setModeOnEmpty(db, kv.StorageModeTEVM, pm.Experiments.TEVM) @@ -257,26 +287,77 @@ func SetIfNotExist(db kv.GetPut, pm Mode) error { return nil } -func setDistance(db kv.Putter, key []byte, distance Distance) error { +func createBlockAmount(pruneType []byte, v []byte) (BlockAmount, error) { + var blockAmount BlockAmount + + switch string(pruneType) { + case string(kv.PruneTypeOlder): + blockAmount = Distance(binary.BigEndian.Uint64(v)) + case string(kv.PruneTypeBefore): + blockAmount = Before(binary.BigEndian.Uint64(v)) + default: + return nil, fmt.Errorf("unexpected block amount type: %s", string(pruneType)) + } + + return blockAmount, nil +} + +func get(db kv.Getter, key []byte) (BlockAmount, error) { + v, err := db.GetOne(kv.DatabaseInfo, key) + if err != nil { + return nil, err + } + + vType, err := db.GetOne(kv.DatabaseInfo, keyType(key)) + if err != nil { + return nil, err + } + + if v != nil { + blockAmount, err := createBlockAmount(vType, v) + if err != nil { + return nil, err + } + return blockAmount, nil + } + + return nil, nil +} + +func set(db kv.Putter, key []byte, blockAmount BlockAmount) error { v := make([]byte, 8) - binary.BigEndian.PutUint64(v, uint64(distance)) + binary.BigEndian.PutUint64(v, blockAmount.toValue()) if err := db.Put(kv.DatabaseInfo, key, v); err != nil { return err } + + keyType := keyType(key) + + if err := db.Put(kv.DatabaseInfo, keyType, blockAmount.dbType()); err != nil { + return err + } + return nil } -func setDistanceOnEmpty(db kv.GetPut, key []byte, distance Distance) error { +func keyType(name []byte) []byte { + return append(name, []byte("Type")...) +} + +func setOnEmpty(db kv.GetPut, key []byte, blockAmount BlockAmount) error { mode, err := db.GetOne(kv.DatabaseInfo, key) if err != nil { return err } if len(mode) == 0 || binary.BigEndian.Uint64(mode) == math.MaxUint64 { v := make([]byte, 8) - binary.BigEndian.PutUint64(v, uint64(distance)) + binary.BigEndian.PutUint64(v, blockAmount.toValue()) if err = db.Put(kv.DatabaseInfo, key, v); err != nil { return err } + if err = db.Put(kv.DatabaseInfo, keyType(key), blockAmount.dbType()); err != nil { + return err + } } return nil diff --git a/ethdb/prune/storage_mode_test.go b/ethdb/prune/storage_mode_test.go index d5fad1331..0df75df1c 100644 --- a/ethdb/prune/storage_mode_test.go +++ b/ethdb/prune/storage_mode_test.go @@ -1,6 +1,8 @@ package prune import ( + "math/rand" + "strconv" "testing" "github.com/ledgerwatch/erigon-lib/kv/memdb" @@ -12,12 +14,62 @@ func TestSetStorageModeIfNotExist(t *testing.T) { _, tx := memdb.NewTestTx(t) prune, err := Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64, Experiments{TEVM: false}}, prune) + assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64), + Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{TEVM: false}}, prune) - err = SetIfNotExist(tx, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}}) + err = SetIfNotExist(tx, Mode{true, Distance(1), Distance(2), + Before(3), Before(4), Experiments{TEVM: false}}) assert.NoError(t, err) prune, err = Get(tx) assert.NoError(t, err) - assert.Equal(t, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}}, prune) + assert.Equal(t, Mode{true, Distance(1), Distance(2), + Before(3), Before(4), Experiments{TEVM: false}}, prune) +} + +var distanceTests = []struct { + stageHead uint64 + pruneTo uint64 + expected uint64 +}{ + {3_000_000, 1, 2_999_999}, + {3_000_000, 4_000_000, 0}, + {3_000_000, math.MaxUint64, 0}, + {3_000_000, 1_000_000, 2_000_000}, +} + +func TestDistancePruneTo(t *testing.T) { + for _, tt := range distanceTests { + t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) { + stageHead := tt.stageHead + d := Distance(tt.pruneTo) + pruneTo := d.PruneTo(stageHead) + + if pruneTo != tt.expected { + t.Errorf("got %d, want %d", pruneTo, tt.expected) + } + }) + } +} + +var beforeTests = []struct { + pruneTo uint64 + expected uint64 +}{ + {0, 0}, + {1_000_000, 999_999}, +} + +func TestBeforePruneTo(t *testing.T) { + for _, tt := range beforeTests { + t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) { + stageHead := uint64(rand.Int63n(10_000_000)) + b := Before(tt.pruneTo) + pruneTo := b.PruneTo(stageHead) + + if pruneTo != tt.expected { + t.Errorf("got %d, want %d", pruneTo, tt.expected) + } + }) + } } diff --git a/go.mod b/go.mod index 7b96cab19..8520408f8 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/json-iterator/go v1.1.11 github.com/julienschmidt/httprouter v1.3.0 github.com/kevinburke/go-bindata v3.21.0+incompatible - github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e + github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6 github.com/ledgerwatch/log/v3 v3.3.1 github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d github.com/logrusorgru/aurora/v3 v3.0.0 diff --git a/go.sum b/go.sum index 7327725bc..93b17301f 100644 --- a/go.sum +++ b/go.sum @@ -493,8 +493,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e h1:szu+TNE3OxZazhRA+LC/naHQ8xeeAh0ISymx1deoWd8= -github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e/go.mod h1:WgyjBACSDhgfepaaDJIbzd2TV868EjOrp2ILnEMKspY= +github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6 h1:jlAd4fIbjy+82GJUa/D+RGr5ZgDBQomsmaON0p9BGAM= +github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6/go.mod h1:WgyjBACSDhgfepaaDJIbzd2TV868EjOrp2ILnEMKspY= github.com/ledgerwatch/log/v3 v3.3.1 h1:HmvLeTEvtCtqSvtu4t/a5MAdcLfeBcbIeowXbLYuzLc= github.com/ledgerwatch/log/v3 v3.3.1/go.mod h1:S3VJqhhVX32rbp1JyyvhJou12twtFwNEPESBgpbNkRk= github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d h1:/IKMrJdfRsoYNc36PXqP4xMH3vhW/8IQyBKGQbKZUno= @@ -579,7 +579,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 h1:KcEvVBAvyHkUdFAygKAzwB6LAcZ6LS32WHmRD2VyXMI= github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0= diff --git a/migrations/migrations.go b/migrations/migrations.go index f6d254c50..cec0d5a41 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -36,6 +36,7 @@ var migrations = map[kv.Label][]Migration{ dbSchemaVersion, fixSequences, storageMode, + setPruneType, }, kv.TxPoolDB: {}, kv.SentryDB: {}, diff --git a/migrations/set_prune_type.go b/migrations/set_prune_type.go new file mode 100644 index 000000000..fac208775 --- /dev/null +++ b/migrations/set_prune_type.go @@ -0,0 +1,41 @@ +package migrations + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/kv" +) + +var setPruneType = Migration{ + Name: "set_prune_type", + Up: func(db kv.RwDB, tmpdir string, progress []byte, BeforeCommit Callback) (err error) { + tx, err := db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + + var pruneTypeKeys = [4][]byte{kv.PruneHistoryType, kv.PruneReceiptsType, kv.PruneTxIndexType, kv.PruneCallTracesType} + + for _, key := range pruneTypeKeys { + pruneType, getErr := tx.GetOne(kv.DatabaseInfo, key) + if getErr != nil { + return getErr + } + + if pruneType != nil { + continue + } + + putErr := tx.Put(kv.DatabaseInfo, key, kv.PruneTypeOlder) + if putErr != nil { + return putErr + } + } + + if err := BeforeCommit(tx, nil, true); err != nil { + return err + } + return tx.Commit() + }, +} diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index 640beac78..10e14ce4e 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -28,6 +28,10 @@ var DefaultFlags = []cli.Flag{ PruneReceiptFlag, PruneTxIndexFlag, PruneCallTracesFlag, + PruneHistoryBeforeFlag, + PruneReceiptBeforeFlag, + PruneTxIndexBeforeFlag, + PruneCallTracesBeforeFlag, SnapshotModeFlag, SeedSnapshotsFlag, SnapshotDatabaseLayoutFlag, diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index acf51ce51..575caa22a 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -88,6 +88,24 @@ var ( Name: "prune.c.older", Usage: `Prune data after this amount of blocks (if --prune flag has 'c', then default is 90K)`, } + + PruneHistoryBeforeFlag = cli.Uint64Flag{ + Name: "prune.h.before", + Usage: `Prune data before this block`, + } + PruneReceiptBeforeFlag = cli.Uint64Flag{ + Name: "prune.r.before", + Usage: `Prune data before this block`, + } + PruneTxIndexBeforeFlag = cli.Uint64Flag{ + Name: "prune.t.before", + Usage: `Prune data before this block`, + } + PruneCallTracesBeforeFlag = cli.Uint64Flag{ + Name: "prune.c.before", + Usage: `Prune data before this block`, + } + ExperimentsFlag = cli.StringFlag{ Name: "experiments", Usage: `Enable some experimental stages: @@ -166,6 +184,10 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config) { ctx.GlobalUint64(PruneReceiptFlag.Name), ctx.GlobalUint64(PruneTxIndexFlag.Name), ctx.GlobalUint64(PruneCallTracesFlag.Name), + ctx.GlobalUint64(PruneHistoryBeforeFlag.Name), + ctx.GlobalUint64(PruneReceiptBeforeFlag.Name), + ctx.GlobalUint64(PruneTxIndexBeforeFlag.Name), + ctx.GlobalUint64(PruneCallTracesBeforeFlag.Name), strings.Split(ctx.GlobalString(ExperimentsFlag.Name), ","), ) if err != nil { @@ -239,7 +261,22 @@ func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) { if v := f.Uint64(PruneCallTracesFlag.Name, PruneCallTracesFlag.Value, PruneCallTracesFlag.Usage); v != nil { exactC = *v } - mode, err := prune.FromCli(*v, exactH, exactR, exactT, exactC, experiments) + + var beforeH, beforeR, beforeT, beforeC uint64 + if v := f.Uint64(PruneHistoryBeforeFlag.Name, PruneHistoryBeforeFlag.Value, PruneHistoryBeforeFlag.Usage); v != nil { + beforeH = *v + } + if v := f.Uint64(PruneReceiptBeforeFlag.Name, PruneReceiptBeforeFlag.Value, PruneReceiptBeforeFlag.Usage); v != nil { + beforeR = *v + } + if v := f.Uint64(PruneTxIndexBeforeFlag.Name, PruneTxIndexBeforeFlag.Value, PruneTxIndexBeforeFlag.Usage); v != nil { + beforeT = *v + } + if v := f.Uint64(PruneCallTracesBeforeFlag.Name, PruneCallTracesBeforeFlag.Value, PruneCallTracesBeforeFlag.Usage); v != nil { + beforeC = *v + } + + mode, err := prune.FromCli(*v, exactH, exactR, exactT, exactC, beforeH, beforeR, beforeT, beforeC, experiments) if err != nil { utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err)) } diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index 7da01ed60..a2e6ae5d6 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -665,6 +665,15 @@ func TestModes(t *testing.T) { ) } +func TestBeforeModes(t *testing.T) { + mode := prune.DefaultMode + mode.History = prune.Before(0) + mode.Receipts = prune.Before(1) + mode.TxIndex = prune.Before(2) + mode.CallTraces = prune.Before(3) + doModesTest(t, mode) +} + func doModesTest(t *testing.T, pm prune.Mode) error { fmt.Printf("h=%v, r=%v, t=%v\n", pm.History.Enabled(), pm.Receipts.Enabled(), pm.TxIndex.Enabled()) require := require.New(t) @@ -837,7 +846,7 @@ func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, c if err := runPermutation(t, testFunc, current+1, pm); err != nil { return err } - invert := func(a prune.Distance) prune.Distance { + invert := func(a prune.BlockAmount) prune.Distance { if a.Enabled() { return math.MaxUint64 }