Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
package prune
|
|
|
|
|
|
|
|
import (
|
2021-09-23 02:13:19 +00:00
|
|
|
"math/rand"
|
|
|
|
"strconv"
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
"testing"
|
|
|
|
|
2021-07-29 11:53:13 +00:00
|
|
|
"github.com/ledgerwatch/erigon-lib/kv/memdb"
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
"github.com/ledgerwatch/erigon/common/math"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestSetStorageModeIfNotExist(t *testing.T) {
|
2021-07-28 02:47:38 +00:00
|
|
|
_, tx := memdb.NewTestTx(t)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
prune, err := Get(tx)
|
|
|
|
assert.NoError(t, err)
|
2021-09-23 02:13:19 +00:00
|
|
|
assert.Equal(t, Mode{true, Distance(math.MaxUint64), Distance(math.MaxUint64),
|
2022-09-01 18:49:29 +00:00
|
|
|
Distance(math.MaxUint64), Distance(math.MaxUint64), Experiments{}}, prune)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
|
2022-01-22 04:18:24 +00:00
|
|
|
err = setIfNotExist(tx, Mode{true, Distance(1), Distance(2),
|
2022-09-01 18:49:29 +00:00
|
|
|
Before(3), Before(4), Experiments{}})
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
prune, err = Get(tx)
|
|
|
|
assert.NoError(t, err)
|
2021-09-23 02:13:19 +00:00
|
|
|
assert.Equal(t, Mode{true, Distance(1), Distance(2),
|
2022-09-01 18:49:29 +00:00
|
|
|
Before(3), Before(4), Experiments{}}, prune)
|
2021-09-23 02:13:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var distanceTests = []struct {
|
|
|
|
stageHead uint64
|
|
|
|
pruneTo uint64
|
|
|
|
expected uint64
|
|
|
|
}{
|
|
|
|
{3_000_000, 1, 2_999_999},
|
|
|
|
{3_000_000, 4_000_000, 0},
|
|
|
|
{3_000_000, math.MaxUint64, 0},
|
|
|
|
{3_000_000, 1_000_000, 2_000_000},
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestDistancePruneTo(t *testing.T) {
|
|
|
|
for _, tt := range distanceTests {
|
|
|
|
t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) {
|
|
|
|
stageHead := tt.stageHead
|
|
|
|
d := Distance(tt.pruneTo)
|
|
|
|
pruneTo := d.PruneTo(stageHead)
|
|
|
|
|
|
|
|
if pruneTo != tt.expected {
|
|
|
|
t.Errorf("got %d, want %d", pruneTo, tt.expected)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var beforeTests = []struct {
|
|
|
|
pruneTo uint64
|
|
|
|
expected uint64
|
|
|
|
}{
|
|
|
|
{0, 0},
|
|
|
|
{1_000_000, 999_999},
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestBeforePruneTo(t *testing.T) {
|
|
|
|
for _, tt := range beforeTests {
|
|
|
|
t.Run(strconv.FormatUint(tt.pruneTo, 10), func(t *testing.T) {
|
|
|
|
stageHead := uint64(rand.Int63n(10_000_000))
|
|
|
|
b := Before(tt.pruneTo)
|
|
|
|
pruneTo := b.PruneTo(stageHead)
|
|
|
|
|
|
|
|
if pruneTo != tt.expected {
|
|
|
|
t.Errorf("got %d, want %d", pruneTo, tt.expected)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
}
|