From 85e47d83d2b8a58c12dcd79d0ca4f0ccdc964203 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 24 Mar 2022 12:17:31 +0700 Subject: [PATCH] Snapshots: remove "experimental" prefix from cli flag (#3760) --- cmd/downloader/downloader/downloader.go | 2 +- cmd/downloader/readme.md | 20 +++++++++++------- cmd/state/commands/global_flags_vars.go | 2 +- cmd/utils/flags.go | 2 +- docker-compose.yml | 2 +- eth/ethconfig/config.go | 4 ++-- eth/stagedsync/stage_headers.go | 28 ++++++++++++------------- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cmd/downloader/downloader/downloader.go b/cmd/downloader/downloader/downloader.go index a0f5c7ea6..64c9e8958 100644 --- a/cmd/downloader/downloader/downloader.go +++ b/cmd/downloader/downloader/downloader.go @@ -81,7 +81,7 @@ func (cli *Protocols) PeerID() []byte { } func LoggingLoop(ctx context.Context, torrentClient *torrent.Client) { - interval := time.Second * 5 + interval := time.Second * 20 logEvery := time.NewTicker(interval) defer logEvery.Stop() var m runtime.MemStats diff --git a/cmd/downloader/readme.md b/cmd/downloader/readme.md index e67322ac1..fb6798893 100644 --- a/cmd/downloader/readme.md +++ b/cmd/downloader/readme.md @@ -4,15 +4,13 @@ Service to seed/download historical data (immutable .seg files) ## How to Start Erigon in snapshot sync mode -Only Mainnet, Goerli and BSC networks are supported now. - ```shell -# 1. Downloader by default run inside Erigon, by `--experimental.snapshot` flag: -erigon --experimental.snapshot --datadir= +# 1. Downloader by default run inside Erigon, by `--snapshot` flag: +erigon --snapshot --datadir= ``` ```shell -# 2. It's possible to start Downloader as independent process, by `--experimental.snapshot --downloader.api.addr=127.0.0.1:9093` flags: +# 2. It's possible to start Downloader as independent process, by `--snapshot --downloader.api.addr=127.0.0.1:9093` flags: make erigon downloader # Start downloader (can limit network usage by 512mb/sec: --download.rate=512mb --upload.rate=512mb) @@ -21,10 +19,18 @@ downloader --downloader.api.addr=127.0.0.1:9093 --torrent.port=42068 --datadir=< # --torrent.port=42068 - is for public BitTorrent protocol listen # Erigon on startup does send list of .torrent files to Downloader and wait for 100% download accomplishment -erigon --experimental.snapshot --downloader.api.addr=127.0.0.1:9093 --datadir= +erigon --snapshot --downloader.api.addr=127.0.0.1:9093 --datadir= ``` -Use `--experimental.snapshot.keepblocks=true` to don't delete retired blocks from DB +Use `--snapshot.keepblocks=true` to don't delete retired blocks from DB + +Any network/chain can start with snapshot sync: + +- node will download only snapshots registered in next repo https://github.com/ledgerwatch/erigon-snapshot +- node will move old blocks from DB to snapshots of 1K blocks size, then merge snapshots to bigger range, until + snapshots of 500K blocks, then automatically start seeding new snapshot + +Flag `--snapshot` is compatible with `--prune` flag ## How to create new network or bootnode diff --git a/cmd/state/commands/global_flags_vars.go b/cmd/state/commands/global_flags_vars.go index f65108cc9..c7443457a 100644 --- a/cmd/state/commands/global_flags_vars.go +++ b/cmd/state/commands/global_flags_vars.go @@ -50,7 +50,7 @@ func withIndexBucket(cmd *cobra.Command) { } func withSnapshotBlocks(cmd *cobra.Command) { - cmd.Flags().BoolVar(&snapshotBlocks, "experimental.snapshot", false, "") + cmd.Flags().BoolVar(&snapshotBlocks, "snapshot", false, "") } func withChain(cmd *cobra.Command) { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ac1b97472..80254ed39 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -622,7 +622,7 @@ var ( } SnapshotSyncFlag = cli.BoolFlag{ - Name: "experimental.snapshot", + Name: "snapshot", Usage: "Enabling experimental snapshot sync", } SnapshotKeepBlocksFlag = cli.BoolFlag{ diff --git a/docker-compose.yml b/docker-compose.yml index c69879fbd..b04b8c61d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: - "8551:8551" restart: unless-stopped - downloader: # Service to download/seed historical data (need only if you use --experimental.snapshot) + downloader: # Service to download/seed historical data (need only if you use --snapshot) image: thorax/erigon:${TAG:-latest} command: downloader ${DOWNLOADER_FLAGS-} --datadir=/home/erigon/.local/share/erigon --downloader.api.addr=0.0.0.0:9093 pid: service:erigon # Use erigon's PID namespace. It's required to open Erigon's DB from another process (RPCDaemon local-mode) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8d2213766..4b50c24dc 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -139,8 +139,8 @@ func (s Snapshot) String() string { } var ( - FlagSnapshot = "experimental.snapshot" - FlagSnapshotKeepBlocks = "experimental.snapshot.keepblocks" + FlagSnapshot = "snapshot" + FlagSnapshotKeepBlocks = "snapshot.keepblocks" ) func NewSnapshotCfg(enabled, keepBlocks bool) Snapshot { diff --git a/eth/stagedsync/stage_headers.go b/eth/stagedsync/stage_headers.go index c70ade477..e12f91c86 100644 --- a/eth/stagedsync/stage_headers.go +++ b/eth/stagedsync/stage_headers.go @@ -962,24 +962,24 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R if err := cfg.snapshots.ReopenIndices(); err != nil { return fmt.Errorf("ReopenIndices: %w", err) } - } - // Create .idx files - if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() { - if !cfg.snapshots.SegmentsReady() { - return fmt.Errorf("not all snapshot segments are available") - } - - // wait for Downloader service to download all expected snapshots + // Create .idx files if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() { - chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID) - if err := snapshotsync.BuildIndices(ctx, cfg.snapshots, cfg.snapshotDir, *chainID, cfg.tmpdir, cfg.snapshots.IndicesAvailable(), log.LvlInfo); err != nil { - return err + if !cfg.snapshots.SegmentsReady() { + return fmt.Errorf("not all snapshot segments are available") } - } - if err := cfg.snapshots.ReopenIndices(); err != nil { - return fmt.Errorf("ReopenIndices: %w", err) + // wait for Downloader service to download all expected snapshots + if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() { + chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID) + if err := snapshotsync.BuildIndices(ctx, cfg.snapshots, cfg.snapshotDir, *chainID, cfg.tmpdir, cfg.snapshots.IndicesAvailable(), log.LvlInfo); err != nil { + return err + } + } + + if err := cfg.snapshots.ReopenIndices(); err != nil { + return fmt.Errorf("ReopenIndices: %w", err) + } } }