mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-22 11:41:19 +00:00
79ed8cad35
This change introduces additional processes to manage snapshot uploading for E2 snapshots: ## erigon snapshots upload The `snapshots uploader` command starts a version of erigon customized for uploading snapshot files to a remote location. It breaks the stage execution process after the senders stage and then uses the snapshot stage to send uploaded headers, bodies and (in the case of polygon) bor spans and events to snapshot files. Because this process avoids execution in run signifigantly faster than a standard erigon configuration. The uploader uses rclone to send seedable (100K or 500K blocks) to a remote storage location specified in the rclone config file. The **uploader** is configured to minimize disk usage by doing the following: * It removes snapshots once they are loaded * It aggressively prunes the database once entities are transferred to snapshots in addition to this it has the following performance related features: * maximizes the workers allocated to snapshot processing to improve throughput * Can be started from scratch by downloading the latest snapshots from the remote location to seed processing ## snapshots command Is a stand alone command for managing remote snapshots it has the following sub commands * **cmp** - compare snapshots * **copy** - copy snapshots * **verify** - verify snapshots * **manifest** - manage the manifest file in the root of remote snapshot locations * **torrent** - manage snapshot torrent files
178 lines
5.6 KiB
Go
178 lines
5.6 KiB
Go
// Package app contains framework for building a command-line based Erigon node.
|
|
package app
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/ledgerwatch/erigon-lib/common/datadir"
|
|
"github.com/ledgerwatch/erigon/turbo/logging"
|
|
"github.com/ledgerwatch/log/v3"
|
|
"github.com/urfave/cli/v2"
|
|
|
|
"github.com/ledgerwatch/erigon/cmd/utils"
|
|
"github.com/ledgerwatch/erigon/node"
|
|
"github.com/ledgerwatch/erigon/node/nodecfg"
|
|
"github.com/ledgerwatch/erigon/params"
|
|
cli2 "github.com/ledgerwatch/erigon/turbo/cli"
|
|
"github.com/ledgerwatch/erigon/turbo/debug"
|
|
)
|
|
|
|
// MakeApp creates a cli application (based on `github.com/urlfave/cli` package).
|
|
// The application exits when `action` returns.
|
|
// Parameters:
|
|
// * action: the main function for the application. receives `*cli.Context` with parsed command-line flags.
|
|
// * cliFlags: the list of flags `cli.Flag` that the app should set and parse. By default, use `DefaultFlags()`. If you want to specify your own flag, use `append(DefaultFlags(), myFlag)` for this parameter.
|
|
func MakeApp(name string, action cli.ActionFunc, cliFlags []cli.Flag) *cli.App {
|
|
app := cli2.NewApp(params.GitCommit, "erigon")
|
|
app.Name = name
|
|
app.UsageText = app.Name + ` [command] [flags]`
|
|
app.Action = func(context *cli.Context) error {
|
|
// handle case: unknown sub-command
|
|
if context.Args().Present() {
|
|
var goodNames []string
|
|
for _, c := range app.VisibleCommands() {
|
|
goodNames = append(goodNames, c.Name)
|
|
}
|
|
log.Error(fmt.Sprintf("Command '%s' not found. Available commands: %s", context.Args().First(), goodNames))
|
|
cli.ShowAppHelpAndExit(context, 1)
|
|
}
|
|
|
|
// handle case: config flag
|
|
configFilePath := context.String(utils.ConfigFlag.Name)
|
|
if configFilePath != "" {
|
|
if err := cli2.SetFlagsFromConfigFile(context, configFilePath); err != nil {
|
|
log.Error("failed setting config flags from yaml/toml file", "err", err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
// run default action
|
|
return action(context)
|
|
}
|
|
|
|
app.Flags = appFlags(cliFlags)
|
|
|
|
app.After = func(ctx *cli.Context) error {
|
|
debug.Exit()
|
|
return nil
|
|
}
|
|
app.Commands = []*cli.Command{
|
|
&initCommand,
|
|
&importCommand,
|
|
&snapshotCommand,
|
|
&supportCommand,
|
|
//&backupCommand,
|
|
}
|
|
return app
|
|
}
|
|
|
|
func appFlags(cliFlags []cli.Flag) []cli.Flag {
|
|
|
|
flags := append(cliFlags, debug.Flags...) // debug flags are required
|
|
flags = append(flags, utils.MetricFlags...)
|
|
flags = append(flags, logging.Flags...)
|
|
flags = append(flags, &utils.ConfigFlag)
|
|
|
|
// remove exact duplicate flags, keeping only the first one. this will allow easier composition later down the line
|
|
allFlags := flags
|
|
newFlags := make([]cli.Flag, 0, len(allFlags))
|
|
seen := map[string]struct{}{}
|
|
for _, vv := range allFlags {
|
|
v := vv
|
|
if _, ok := seen[v.String()]; ok {
|
|
continue
|
|
}
|
|
newFlags = append(newFlags, v)
|
|
}
|
|
|
|
return newFlags
|
|
}
|
|
|
|
// MigrateFlags makes all global flag values available in the
|
|
// context. This should be called as early as possible in app.Before.
|
|
//
|
|
// Example:
|
|
//
|
|
// geth account new --keystore /tmp/mykeystore --lightkdf
|
|
//
|
|
// is equivalent after calling this method with:
|
|
//
|
|
// geth --keystore /tmp/mykeystore --lightkdf account new
|
|
//
|
|
// i.e. in the subcommand Action function of 'account new', ctx.Bool("lightkdf)
|
|
// will return true even if --lightkdf is set as a global option.
|
|
//
|
|
// This function may become unnecessary when https://github.com/urfave/cli/pull/1245 is merged.
|
|
func MigrateFlags(action cli.ActionFunc) cli.ActionFunc {
|
|
return func(ctx *cli.Context) error {
|
|
doMigrateFlags(ctx)
|
|
return action(ctx)
|
|
}
|
|
}
|
|
|
|
func doMigrateFlags(ctx *cli.Context) {
|
|
// Figure out if there are any aliases of commands. If there are, we want
|
|
// to ignore them when iterating over the flags.
|
|
var aliases = make(map[string]bool)
|
|
for _, fl := range ctx.Command.Flags {
|
|
for _, alias := range fl.Names()[1:] {
|
|
aliases[alias] = true
|
|
}
|
|
}
|
|
for _, name := range ctx.FlagNames() {
|
|
for _, parent := range ctx.Lineage()[1:] {
|
|
if parent.IsSet(name) {
|
|
// When iterating across the lineage, we will be served both
|
|
// the 'canon' and alias formats of all commands. In most cases,
|
|
// it's fine to set it in the ctx multiple times (one for each
|
|
// name), however, the Slice-flags are not fine.
|
|
// The slice-flags accumulate, so if we set it once as
|
|
// "foo" and once as alias "F", then both will be present in the slice.
|
|
if _, isAlias := aliases[name]; isAlias {
|
|
continue
|
|
}
|
|
// If it is a string-slice, we need to set it as
|
|
// "alfa, beta, gamma" instead of "[alfa beta gamma]", in order
|
|
// for the backing StringSlice to parse it properly.
|
|
if result := parent.StringSlice(name); len(result) > 0 {
|
|
ctx.Set(name, strings.Join(result, ","))
|
|
} else {
|
|
ctx.Set(name, parent.String(name))
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func NewNodeConfig(ctx *cli.Context) *nodecfg.Config {
|
|
nodeConfig := nodecfg.DefaultConfig
|
|
// see simiar changes in `cmd/geth/config.go#defaultNodeConfig`
|
|
if commit := params.GitCommit; commit != "" {
|
|
nodeConfig.Version = params.VersionWithCommit(commit)
|
|
} else {
|
|
nodeConfig.Version = params.Version
|
|
}
|
|
nodeConfig.IPCPath = "" // force-disable IPC endpoint
|
|
nodeConfig.Name = "erigon"
|
|
if ctx.IsSet(utils.DataDirFlag.Name) {
|
|
nodeConfig.Dirs = datadir.New(ctx.String(utils.DataDirFlag.Name))
|
|
}
|
|
return &nodeConfig
|
|
}
|
|
|
|
func MakeConfigNodeDefault(cliCtx *cli.Context, logger log.Logger) *node.Node {
|
|
return makeConfigNode(cliCtx.Context, NewNodeConfig(cliCtx), logger)
|
|
}
|
|
|
|
func makeConfigNode(ctx context.Context, config *nodecfg.Config, logger log.Logger) *node.Node {
|
|
stack, err := node.New(ctx, config, logger)
|
|
if err != nil {
|
|
utils.Fatalf("Failed to create Erigon node: %v", err)
|
|
}
|
|
|
|
return stack
|
|
}
|