Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
package migrations
|
|
|
|
|
|
|
|
import (
|
2021-07-24 04:28:05 +00:00
|
|
|
"context"
|
|
|
|
|
2021-07-29 11:53:13 +00:00
|
|
|
"github.com/ledgerwatch/erigon-lib/kv"
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
"github.com/ledgerwatch/erigon/common/math"
|
|
|
|
"github.com/ledgerwatch/erigon/ethdb/prune"
|
|
|
|
"github.com/ledgerwatch/erigon/params"
|
|
|
|
)
|
|
|
|
|
|
|
|
var storageMode = Migration{
|
|
|
|
Name: "storage_mode",
|
2021-07-28 02:47:38 +00:00
|
|
|
Up: func(db kv.RwDB, tmpdir string, progress []byte, BeforeCommit Callback) (err error) {
|
2021-07-24 04:28:05 +00:00
|
|
|
tx, err := db.BeginRw(context.Background())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer tx.Rollback()
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
var ( // old db keys
|
|
|
|
//StorageModeHistory - does node save history.
|
|
|
|
StorageModeHistory = []byte("smHistory")
|
|
|
|
//StorageModeReceipts - does node save receipts.
|
|
|
|
StorageModeReceipts = []byte("smReceipts")
|
|
|
|
//StorageModeTxIndex - does node save transactions index.
|
|
|
|
StorageModeTxIndex = []byte("smTxIndex")
|
|
|
|
//StorageModeCallTraces - does not build index of call traces
|
|
|
|
StorageModeCallTraces = []byte("smCallTraces")
|
|
|
|
)
|
|
|
|
pm := prune.Mode{Initialised: true}
|
|
|
|
castToPruneDistance := func(v []byte) prune.Distance {
|
|
|
|
if len(v) == 1 && v[0] == 2 {
|
|
|
|
return params.FullImmutabilityThreshold // means, prune enabled
|
|
|
|
}
|
|
|
|
return math.MaxUint64 // means, prune disabled
|
|
|
|
}
|
|
|
|
{
|
2021-07-28 02:47:38 +00:00
|
|
|
v, err := tx.GetOne(kv.DatabaseInfo, StorageModeHistory)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-11-04 10:23:45 +00:00
|
|
|
if v == nil { // if no records in db - means Erigon just started first time and nothing to migrate. Noop.
|
|
|
|
if err := BeforeCommit(tx, nil, true); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return tx.Commit()
|
|
|
|
}
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
pm.History = castToPruneDistance(v)
|
|
|
|
}
|
|
|
|
{
|
2021-07-28 02:47:38 +00:00
|
|
|
v, err := tx.GetOne(kv.DatabaseInfo, StorageModeReceipts)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pm.Receipts = castToPruneDistance(v)
|
|
|
|
}
|
|
|
|
{
|
2021-07-28 02:47:38 +00:00
|
|
|
v, err := tx.GetOne(kv.DatabaseInfo, StorageModeTxIndex)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pm.TxIndex = castToPruneDistance(v)
|
|
|
|
}
|
|
|
|
{
|
2021-07-28 02:47:38 +00:00
|
|
|
v, err := tx.GetOne(kv.DatabaseInfo, StorageModeCallTraces)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pm.CallTraces = castToPruneDistance(v)
|
|
|
|
}
|
|
|
|
{
|
2021-07-28 02:47:38 +00:00
|
|
|
v, err := tx.GetOne(kv.DatabaseInfo, kv.StorageModeTEVM)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
pm.Experiments.TEVM = len(v) == 1 && v[0] == 1
|
|
|
|
}
|
|
|
|
|
2021-07-24 04:28:05 +00:00
|
|
|
err = prune.SetIfNotExist(tx, pm)
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-07-24 04:28:05 +00:00
|
|
|
if err := BeforeCommit(tx, nil, true); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return tx.Commit()
|
Pruning for: exec, log_index, tx_lookup, history stages (#2399)
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* Pruning for: exec, log_index, tx_lookup, history stages
* add tvm flag
* save
* db migration for storage mode
add flag --prune=
remove flag --storage-mode=
add flag --experiments=tevm,...
rename integration set_storage_mode to set_prune
* fix
* forward move of stages must skip everything before PruneTo
* keep in db progress of prune method
* keep in db progress of prune method
* simplify logs
* simplify logs
* simplify logs
* fix test
* simplify logs
* simplify logs
* simplify logs
* simplify logs
* remove callTraceSet as dupsort
use etl transform for txlookup prune
remove some logs
* cleanup tests a bit
* print_stages and eth_sync to show prune progress
* fix print_stages
* add readme about --prune.to flag
* more docs
* add --prune.history.older and other flags support
* fix migration on empty db
* better toString
* better toString
2021-07-20 20:03:19 +00:00
|
|
|
},
|
|
|
|
}
|