mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

More convenient pruning for non-PoW consensus: add --prune.*.before flags (#2714)

Browse Source
This commit is contained in:
Alexandr Borodulin 2021-09-23 05:13:19 +03:00 committed by GitHub
parent 0b776eead4
commit d7159cd8bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 331 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/integration/commands/flags.go
View File

@ -25,6 +25,8 @@ var (
txtrace bool // Whether to trace the execution (should only be used together eith `block`)
pruneFlag string
pruneH, pruneR, pruneT, pruneC uint64
pruneHBefore, pruneRBefore uint64
pruneTBefore, pruneCBefore uint64
experiments []string
chain string // Which chain to use (mainnet, ropsten, rinkeby, goerli, etc.)
)

15
cmd/integration/commands/stages.go
View File

@ -364,10 +364,14 @@ func init() {
withDatadir(cmdSetPrune)
withChain(cmdSetPrune)
cmdSetPrune.Flags().StringVar(&pruneFlag, "prune", "hrtc", "")
cmdSetPrune.Flags().Uint64Var(&pruneH, "--prune.h.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneR, "--prune.r.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneT, "--prune.t.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneC, "--prune.c.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneH, "prune.h.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneR, "prune.r.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneT, "prune.t.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneC, "prune.c.older", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneHBefore, "prune.h.before", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneRBefore, "prune.r.before", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneTBefore, "prune.t.before", 0, "")
cmdSetPrune.Flags().Uint64Var(&pruneCBefore, "prune.c.before", 0, "")
cmdSetPrune.Flags().StringSliceVar(&experiments, "experiments", nil, "Storage mode to override database")
rootCmd.AddCommand(cmdSetPrune)
}
@ -1035,7 +1039,8 @@ func stage(st *stagedsync.Sync, tx kv.Tx, db kv.RoDB, stage stages.SyncStage) *s
}
func overrideStorageMode(db kv.RwDB) error {
pm, err := prune.FromCli(pruneFlag, pruneH, pruneR, pruneT, pruneC, experiments)
pm, err := prune.FromCli(pruneFlag, pruneH, pruneR, pruneT, pruneC,
pruneHBefore, pruneRBefore, pruneTBefore, pruneCBefore, experiments)
if err != nil {
return err
}

7
eth/ethconfig/gen_config.go
View File

@ -54,6 +54,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
PruneR *uint64
PruneT *uint64
PruneC *uint64
PruneBeforeH *uint64
PruneBeforeR *uint64
PruneBeforeT *uint64
PruneBeforeC *uint64
Experiments *[]string
OnlyAnnounce *bool
SkipBcVersionCheck *bool `toml:"-"`
@ -83,7 +87,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
c.Whitelist = dec.Whitelist
}
if dec.Mode != nil {
mode, err := prune.FromCli(*dec.Mode, *dec.PruneH, *dec.PruneR, *dec.PruneT, *dec.PruneC, *dec.Experiments)
mode, err := prune.FromCli(*dec.Mode, *dec.PruneH, *dec.PruneR, *dec.PruneT, *dec.PruneC,
*dec.PruneBeforeH, *dec.PruneBeforeR, *dec.PruneBeforeT, *dec.PruneBeforeC, *dec.Experiments)
if err != nil {
return err
}

8
eth/stagedsync/stage_execute_test.go
View File

@ -87,7 +87,7 @@ func TestPruneExecution(t *testing.T) {
s := &PruneState{ID: stages.Execution, ForwardProgress: 50}
// check pruning distance > than current stage progress
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 100, Receipts: 101, CallTraces: 200}}, ctx, false)
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(100), Receipts: prune.Distance(101), CallTraces: prune.Distance(200)}}, ctx, false)
assert.NoError(err)
available, err = changeset.AvailableFrom(tx)
@ -98,7 +98,8 @@ func TestPruneExecution(t *testing.T) {
assert.Equal(uint64(1), available)
// pruning distance, first run
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false)
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(5),
Receipts: prune.Distance(15), CallTraces: prune.Distance(25)}}, ctx, false)
assert.NoError(err)
available, err = changeset.AvailableFrom(tx)
@ -109,7 +110,8 @@ func TestPruneExecution(t *testing.T) {
assert.Equal(uint64(45), available)
// pruning distance, second run
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: 5, Receipts: 15, CallTraces: 25}}, ctx, false)
err = PruneExecutionStage(s, tx, ExecuteBlockCfg{prune: prune.Mode{History: prune.Distance(5),
Receipts: prune.Distance(15), CallTraces: prune.Distance(25)}}, ctx, false)
assert.NoError(err)
available, err = changeset.AvailableFrom(tx)

231
ethdb/prune/storage_mode.go
View File

@ -11,10 +11,10 @@ import (
var DefaultMode = Mode{
Initialised: true,
History: math.MaxUint64, // all off
Receipts: math.MaxUint64,
TxIndex: math.MaxUint64,
CallTraces: math.MaxUint64,
History: Distance(math.MaxUint64), // all off
Receipts: Distance(math.MaxUint64),
TxIndex: Distance(math.MaxUint64),
CallTraces: Distance(math.MaxUint64),
Experiments: Experiments{}, // all off
}
@ -22,7 +22,8 @@ type Experiments struct {
TEVM bool
}
func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallTraces uint64, experiments []string) (Mode, error) {
func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallTraces,
beforeH, beforeR, beforeT, beforeC uint64, experiments []string) (Mode, error) {
mode := DefaultMode
if flags == "default" || flags == "disabled" {
return DefaultMode, nil
@ -31,13 +32,13 @@ func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallT
for _, flag := range flags {
switch flag {
case 'h':
mode.History = params.FullImmutabilityThreshold
mode.History = Distance(params.FullImmutabilityThreshold)
case 'r':
mode.Receipts = params.FullImmutabilityThreshold
mode.Receipts = Distance(params.FullImmutabilityThreshold)
case 't':
mode.TxIndex = params.FullImmutabilityThreshold
mode.TxIndex = Distance(params.FullImmutabilityThreshold)
case 'c':
mode.CallTraces = params.FullImmutabilityThreshold
mode.CallTraces = Distance(params.FullImmutabilityThreshold)
default:
return DefaultMode, fmt.Errorf("unexpected flag found: %c", flag)
}
@ -56,6 +57,19 @@ func FromCli(flags string, exactHistory, exactReceipts, exactTxIndex, exactCallT
mode.CallTraces = Distance(exactCallTraces)
}
if beforeH > 0 {
mode.History = Before(beforeH)
}
if beforeR > 0 {
mode.Receipts = Before(beforeR)
}
if beforeT > 0 {
mode.TxIndex = Before(beforeT)
}
if beforeC > 0 {
mode.CallTraces = Before(beforeC)
}
for _, ex := range experiments {
switch ex {
case "tevm":
@ -74,61 +88,64 @@ func Get(db kv.Getter) (Mode, error) {
prune := DefaultMode
prune.Initialised = true
v, err := db.GetOne(kv.DatabaseInfo, kv.PruneDistanceHistory)
blockAmount, err := get(db, kv.PruneHistory)
if err != nil {
return prune, err
}
if v != nil {
prune.History = Distance(binary.BigEndian.Uint64(v))
} else {
prune.History = math.MaxUint64
}
v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceReceipts)
if err != nil {
return prune, err
}
if v != nil {
prune.Receipts = Distance(binary.BigEndian.Uint64(v))
} else {
prune.Receipts = math.MaxUint64
}
v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceTxIndex)
if err != nil {
return prune, err
}
if v != nil {
prune.TxIndex = Distance(binary.BigEndian.Uint64(v))
} else {
prune.TxIndex = math.MaxUint64
if blockAmount != nil {
prune.History = blockAmount
}
v, err = db.GetOne(kv.DatabaseInfo, kv.PruneDistanceCallTraces)
blockAmount, err = get(db, kv.PruneReceipts)
if err != nil {
return prune, err
}
if v != nil {
prune.CallTraces = Distance(binary.BigEndian.Uint64(v))
} else {
prune.CallTraces = math.MaxUint64
if blockAmount != nil {
prune.Receipts = blockAmount
}
v, err = db.GetOne(kv.DatabaseInfo, kv.StorageModeTEVM)
blockAmount, err = get(db, kv.PruneTxIndex)
if err != nil {
return prune, err
}
if blockAmount != nil {
prune.TxIndex = blockAmount
}
blockAmount, err = get(db, kv.PruneCallTraces)
if err != nil {
return prune, err
}
if blockAmount != nil {
prune.CallTraces = blockAmount
}
v, err := db.GetOne(kv.DatabaseInfo, kv.StorageModeTEVM)
if err != nil {
return prune, err
}
prune.Experiments.TEVM = len(v) == 1 && v[0] == 1
return prune, nil
}
type Mode struct {
Initialised bool // Set when the values are initialised (not default)
History Distance
Receipts Distance
TxIndex Distance
CallTraces Distance
History BlockAmount
Receipts BlockAmount
TxIndex BlockAmount
CallTraces BlockAmount
Experiments Experiments
}
type BlockAmount interface {
PruneTo(stageHead uint64) uint64
Enabled() bool
toValue() uint64
dbType() []byte
useDefaultValue() bool
}
// Distance amount of blocks to keep in DB
// but manual manipulation with such distance is very unsafe
// for example:
@ -136,7 +153,10 @@ type Mode struct {
// may delete whole db - because of uint64 underflow when pruningDistance > currentStageProgress
type Distance uint64
func (p Distance) Enabled() bool { return p != math.MaxUint64 }
func (p Distance) Enabled() bool { return p != math.MaxUint64 }
func (p Distance) toValue() uint64 { return uint64(p) }
func (p Distance) useDefaultValue() bool { return uint64(p) == params.FullImmutabilityThreshold }
func (p Distance) dbType() []byte { return kv.PruneTypeOlder }
func (p Distance) PruneTo(stageHead uint64) uint64 {
if p == 0 {
@ -148,6 +168,22 @@ func (p Distance) PruneTo(stageHead uint64) uint64 {
return stageHead - uint64(p)
}
// Before number after which keep in DB
type Before uint64
func (b Before) Enabled() bool { return b > 0 }
func (b Before) toValue() uint64 { return uint64(b) }
func (b Before) useDefaultValue() bool { return uint64(b) == 0 }
func (b Before) dbType() []byte { return kv.PruneTypeBefore }
func (b Before) PruneTo(uint64) uint64 {
if b == 0 {
return uint64(b)
}
return uint64(b) - 1
}
func (m Mode) String() string {
if !m.Initialised {
return "default"
@ -155,31 +191,31 @@ func (m Mode) String() string {
long := ""
short := "--prune="
if m.History.Enabled() {
if m.History == params.FullImmutabilityThreshold {
if m.History.useDefaultValue() {
short += "h"
} else {
long += fmt.Sprintf(" --prune.h.older=%d", m.History)
long += fmt.Sprintf(" --prune.h.%s=%d", m.History.dbType(), m.History.toValue())
}
}
if m.Receipts.Enabled() {
if m.Receipts == params.FullImmutabilityThreshold {
if m.Receipts.useDefaultValue() {
short += "r"
} else {
long += fmt.Sprintf(" --prune.r.older=%d", m.Receipts)
long += fmt.Sprintf(" --prune.r.%s=%d", m.Receipts.dbType(), m.Receipts.toValue())
}
}
if m.TxIndex.Enabled() {
if m.TxIndex == params.FullImmutabilityThreshold {
if m.TxIndex.useDefaultValue() {
short += "t"
} else {
long += fmt.Sprintf(" --prune.t.older=%d", m.TxIndex)
long += fmt.Sprintf(" --prune.t.%s=%d", m.TxIndex.dbType(), m.TxIndex.toValue())
}
}
if m.CallTraces.Enabled() {
if m.CallTraces == params.FullImmutabilityThreshold {
if m.CallTraces.useDefaultValue() {
short += "c"
} else {
long += fmt.Sprintf(" --prune.c.older=%d", m.CallTraces)
long += fmt.Sprintf(" --prune.c.%s=%d", m.CallTraces.dbType(), m.CallTraces.toValue())
}
}
if m.Experiments.TEVM {
@ -193,22 +229,22 @@ func Override(db kv.RwTx, sm Mode) error {
err error
)
err = setDistance(db, kv.PruneDistanceHistory, sm.History)
err = set(db, kv.PruneHistory, sm.History)
if err != nil {
return err
}
err = setDistance(db, kv.PruneDistanceReceipts, sm.Receipts)
err = set(db, kv.PruneReceipts, sm.Receipts)
if err != nil {
return err
}
err = setDistance(db, kv.PruneDistanceTxIndex, sm.TxIndex)
err = set(db, kv.PruneTxIndex, sm.TxIndex)
if err != nil {
return err
}
err = setDistance(db, kv.PruneDistanceCallTraces, sm.CallTraces)
err = set(db, kv.PruneCallTraces, sm.CallTraces)
if err != nil {
return err
}
@ -229,24 +265,18 @@ func SetIfNotExist(db kv.GetPut, pm Mode) error {
pm = DefaultMode
}
err = setDistanceOnEmpty(db, kv.PruneDistanceHistory, pm.History)
if err != nil {
return err
pruneDBData := map[string]BlockAmount{
string(kv.PruneHistory): pm.History,
string(kv.PruneReceipts): pm.Receipts,
string(kv.PruneTxIndex): pm.TxIndex,
string(kv.PruneCallTraces): pm.CallTraces,
}
err = setDistanceOnEmpty(db, kv.PruneDistanceReceipts, pm.Receipts)
if err != nil {
return err
}
err = setDistanceOnEmpty(db, kv.PruneDistanceTxIndex, pm.TxIndex)
if err != nil {
return err
}
err = setDistanceOnEmpty(db, kv.PruneDistanceCallTraces, pm.CallTraces)
if err != nil {
return err
for key, value := range pruneDBData {
err = setOnEmpty(db, []byte(key), value)
if err != nil {
return err
}
}
err = setModeOnEmpty(db, kv.StorageModeTEVM, pm.Experiments.TEVM)
@ -257,26 +287,77 @@ func SetIfNotExist(db kv.GetPut, pm Mode) error {
return nil
}
func setDistance(db kv.Putter, key []byte, distance Distance) error {
func createBlockAmount(pruneType []byte, v []byte) (BlockAmount, error) {
var blockAmount BlockAmount
switch string(pruneType) {
case string(kv.PruneTypeOlder):
blockAmount = Distance(binary.BigEndian.Uint64(v))
case string(kv.PruneTypeBefore):
blockAmount = Before(binary.BigEndian.Uint64(v))
default:
return nil, fmt.Errorf("unexpected block amount type: %s", string(pruneType))
}
return blockAmount, nil
}
func get(db kv.Getter, key []byte) (BlockAmount, error) {
v, err := db.GetOne(kv.DatabaseInfo, key)
if err != nil {
return nil, err
}
vType, err := db.GetOne(kv.DatabaseInfo, keyType(key))
if err != nil {
return nil, err
}
if v != nil {
blockAmount, err := createBlockAmount(vType, v)
if err != nil {
return nil, err
}
return blockAmount, nil
}
return nil, nil
}
func set(db kv.Putter, key []byte, blockAmount BlockAmount) error {
v := make([]byte, 8)
binary.BigEndian.PutUint64(v, uint64(distance))
binary.BigEndian.PutUint64(v, blockAmount.toValue())
if err := db.Put(kv.DatabaseInfo, key, v); err != nil {
return err
}
keyType := keyType(key)
if err := db.Put(kv.DatabaseInfo, keyType, blockAmount.dbType()); err != nil {
return err
}
return nil
}
func setDistanceOnEmpty(db kv.GetPut, key []byte, distance Distance) error {
func keyType(name []byte) []byte {
return append(name, []byte("Type")...)
}
func setOnEmpty(db kv.GetPut, key []byte, blockAmount BlockAmount) error {
mode, err := db.GetOne(kv.DatabaseInfo, key)
if err != nil {
return err
}
if len(mode) == 0 || binary.BigEndian.Uint64(mode) == math.MaxUint64 {
v := make([]byte, 8)
binary.BigEndian.PutUint64(v, uint64(distance))
binary.BigEndian.PutUint64(v, blockAmount.toValue())
if err = db.Put(kv.DatabaseInfo, key, v); err != nil {
return err
}
if err = db.Put(kv.DatabaseInfo, keyType(key), blockAmount.dbType()); err != nil {
return err
}
}
return nil

58
ethdb/prune/storage_mode_test.go
View File

@ -1,6 +1,8 @@
package prune
import (
"math/rand"
"strconv"
"testing"
"github.com/ledgerwatch/erigon-lib/kv/memdb"
@ -12,12 +14,62 @@ func TestSetStorageModeIfNotExist(t *testing.T) {
_, tx := memdb.NewTestTx(t)
prune, err := Get(tx)
assert.NoError(t, err)
assert.Equal(t, Mode{true, math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64, Experiments{TEVM: false}}, prune)
assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64),
Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{TEVM: false}}, prune)
err = SetIfNotExist(tx, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}})
err = SetIfNotExist(tx, Mode{true, Distance(1), Distance(2),
Before(3), Before(4), Experiments{TEVM: false}})
assert.NoError(t, err)
prune, err = Get(tx)
assert.NoError(t, err)
assert.Equal(t, Mode{true, 1, 2, 3, 4, Experiments{TEVM: false}}, prune)
assert.Equal(t, Mode{true, Distance(1), Distance(2),
Before(3), Before(4), Experiments{TEVM: false}}, prune)
}
var distanceTests = []struct {
stageHead uint64
pruneTo uint64
expected uint64
}{
{3_000_000, 1, 2_999_999},
{3_000_000, 4_000_000, 0},
{3_000_000, math.MaxUint64, 0},
{3_000_000, 1_000_000, 2_000_000},
}
func TestDistancePruneTo(t *testing.T) {
for _, tt := range distanceTests {
t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) {
stageHead := tt.stageHead
d := Distance(tt.pruneTo)
pruneTo := d.PruneTo(stageHead)
if pruneTo != tt.expected {
t.Errorf("got %d, want %d", pruneTo, tt.expected)
}
})
}
}
var beforeTests = []struct {
pruneTo uint64
expected uint64
}{
{0, 0},
{1_000_000, 999_999},
}
func TestBeforePruneTo(t *testing.T) {
for _, tt := range beforeTests {
t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) {
stageHead := uint64(rand.Int63n(10_000_000))
b := Before(tt.pruneTo)
pruneTo := b.PruneTo(stageHead)
if pruneTo != tt.expected {
t.Errorf("got %d, want %d", pruneTo, tt.expected)
}
})
}
}

2
go.mod
View File

@ -36,7 +36,7 @@ require (
github.com/json-iterator/go v1.1.11
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e
github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6
github.com/ledgerwatch/log/v3 v3.3.1
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d
github.com/logrusorgru/aurora/v3 v3.0.0

5
go.sum
View File

@ -493,8 +493,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e h1:szu+TNE3OxZazhRA+LC/naHQ8xeeAh0ISymx1deoWd8=
github.com/ledgerwatch/erigon-lib v0.0.0-20210922080429-58ed1c72e16e/go.mod h1:WgyjBACSDhgfepaaDJIbzd2TV868EjOrp2ILnEMKspY=
github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6 h1:jlAd4fIbjy+82GJUa/D+RGr5ZgDBQomsmaON0p9BGAM=
github.com/ledgerwatch/erigon-lib v0.0.0-20210922103429-7740382188a6/go.mod h1:WgyjBACSDhgfepaaDJIbzd2TV868EjOrp2ILnEMKspY=
github.com/ledgerwatch/log/v3 v3.3.1 h1:HmvLeTEvtCtqSvtu4t/a5MAdcLfeBcbIeowXbLYuzLc=
github.com/ledgerwatch/log/v3 v3.3.1/go.mod h1:S3VJqhhVX32rbp1JyyvhJou12twtFwNEPESBgpbNkRk=
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d h1:/IKMrJdfRsoYNc36PXqP4xMH3vhW/8IQyBKGQbKZUno=
@ -579,7 +579,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 h1:KcEvVBAvyHkUdFAygKAzwB6LAcZ6LS32WHmRD2VyXMI=
github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4=
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=

1
migrations/migrations.go
View File

@ -36,6 +36,7 @@ var migrations = map[kv.Label][]Migration{
dbSchemaVersion,
fixSequences,
storageMode,
setPruneType,
},
kv.TxPoolDB: {},
kv.SentryDB: {},

41
migrations/set_prune_type.go Normal file
View File

@ -0,0 +1,41 @@
package migrations
import (
"context"
"github.com/ledgerwatch/erigon-lib/kv"
)
var setPruneType = Migration{
Name: "set_prune_type",
Up: func(db kv.RwDB, tmpdir string, progress []byte, BeforeCommit Callback) (err error) {
tx, err := db.BeginRw(context.Background())
if err != nil {
return err
}
defer tx.Rollback()
var pruneTypeKeys = [4][]byte{kv.PruneHistoryType, kv.PruneReceiptsType, kv.PruneTxIndexType, kv.PruneCallTracesType}
for _, key := range pruneTypeKeys {
pruneType, getErr := tx.GetOne(kv.DatabaseInfo, key)
if getErr != nil {
return getErr
}
if pruneType != nil {
continue
}
putErr := tx.Put(kv.DatabaseInfo, key, kv.PruneTypeOlder)
if putErr != nil {
return putErr
}
}
if err := BeforeCommit(tx, nil, true); err != nil {
return err
}
return tx.Commit()
},
}

4
turbo/cli/default_flags.go
View File

@ -28,6 +28,10 @@ var DefaultFlags = []cli.Flag{
PruneReceiptFlag,
PruneTxIndexFlag,
PruneCallTracesFlag,
PruneHistoryBeforeFlag,
PruneReceiptBeforeFlag,
PruneTxIndexBeforeFlag,
PruneCallTracesBeforeFlag,
SnapshotModeFlag,
SeedSnapshotsFlag,
SnapshotDatabaseLayoutFlag,

39
turbo/cli/flags.go
View File

@ -88,6 +88,24 @@ var (
Name: "prune.c.older",
Usage: `Prune data after this amount of blocks (if --prune flag has 'c', then default is 90K)`,
}
PruneHistoryBeforeFlag = cli.Uint64Flag{
Name: "prune.h.before",
Usage: `Prune data before this block`,
}
PruneReceiptBeforeFlag = cli.Uint64Flag{
Name: "prune.r.before",
Usage: `Prune data before this block`,
}
PruneTxIndexBeforeFlag = cli.Uint64Flag{
Name: "prune.t.before",
Usage: `Prune data before this block`,
}
PruneCallTracesBeforeFlag = cli.Uint64Flag{
Name: "prune.c.before",
Usage: `Prune data before this block`,
}
ExperimentsFlag = cli.StringFlag{
Name: "experiments",
Usage: `Enable some experimental stages:
@ -166,6 +184,10 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config) {
ctx.GlobalUint64(PruneReceiptFlag.Name),
ctx.GlobalUint64(PruneTxIndexFlag.Name),
ctx.GlobalUint64(PruneCallTracesFlag.Name),
ctx.GlobalUint64(PruneHistoryBeforeFlag.Name),
ctx.GlobalUint64(PruneReceiptBeforeFlag.Name),
ctx.GlobalUint64(PruneTxIndexBeforeFlag.Name),
ctx.GlobalUint64(PruneCallTracesBeforeFlag.Name),
strings.Split(ctx.GlobalString(ExperimentsFlag.Name), ","),
)
if err != nil {
@ -239,7 +261,22 @@ func ApplyFlagsForEthConfigCobra(f *pflag.FlagSet, cfg *ethconfig.Config) {
if v := f.Uint64(PruneCallTracesFlag.Name, PruneCallTracesFlag.Value, PruneCallTracesFlag.Usage); v != nil {
exactC = *v
}
mode, err := prune.FromCli(*v, exactH, exactR, exactT, exactC, experiments)
var beforeH, beforeR, beforeT, beforeC uint64
if v := f.Uint64(PruneHistoryBeforeFlag.Name, PruneHistoryBeforeFlag.Value, PruneHistoryBeforeFlag.Usage); v != nil {
beforeH = *v
}
if v := f.Uint64(PruneReceiptBeforeFlag.Name, PruneReceiptBeforeFlag.Value, PruneReceiptBeforeFlag.Usage); v != nil {
beforeR = *v
}
if v := f.Uint64(PruneTxIndexBeforeFlag.Name, PruneTxIndexBeforeFlag.Value, PruneTxIndexBeforeFlag.Usage); v != nil {
beforeT = *v
}
if v := f.Uint64(PruneCallTracesBeforeFlag.Name, PruneCallTracesBeforeFlag.Value, PruneCallTracesBeforeFlag.Usage); v != nil {
beforeC = *v
}
mode, err := prune.FromCli(*v, exactH, exactR, exactT, exactC, beforeH, beforeR, beforeT, beforeC, experiments)
if err != nil {
utils.Fatalf(fmt.Sprintf("error while parsing mode: %v", err))
}

11
turbo/stages/blockchain_test.go
View File

@ -665,6 +665,15 @@ func TestModes(t *testing.T) {
)
}
func TestBeforeModes(t *testing.T) {
mode := prune.DefaultMode
mode.History = prune.Before(0)
mode.Receipts = prune.Before(1)
mode.TxIndex = prune.Before(2)
mode.CallTraces = prune.Before(3)
doModesTest(t, mode)
}
func doModesTest(t *testing.T, pm prune.Mode) error {
fmt.Printf("h=%v, r=%v, t=%v\n", pm.History.Enabled(), pm.Receipts.Enabled(), pm.TxIndex.Enabled())
require := require.New(t)
@ -837,7 +846,7 @@ func runPermutation(t *testing.T, testFunc func(*testing.T, prune.Mode) error, c
if err := runPermutation(t, testFunc, current+1, pm); err != nil {
return err
}
invert := func(a prune.Distance) prune.Distance {
invert := func(a prune.BlockAmount) prune.Distance {
if a.Enabled() {
return math.MaxUint64
}