Snapshots: remove "experimental" prefix from cli flag (#3760)

cmd/downloader/downloader/downloader.go

@@ -81,7 +81,7 @@ func (cli *Protocols) PeerID() []byte {
 }
 
 func LoggingLoop(ctx context.Context, torrentClient *torrent.Client) {
-	interval := time.Second * 5
+	interval := time.Second * 20
 	logEvery := time.NewTicker(interval)
 	defer logEvery.Stop()
 	var m runtime.MemStats

cmd/downloader/readme.md

@@ -4,15 +4,13 @@ Service to seed/download historical data (immutable .seg files)
 
 ## How to Start Erigon in snapshot sync mode
 
-Only Mainnet, Goerli and BSC networks are supported now.
-
 ```shell
-# 1. Downloader by default run inside Erigon, by `--experimental.snapshot` flag:
+# 1. Downloader by default run inside Erigon, by `--snapshot` flag:
-erigon --experimental.snapshot --datadir=<your_datadir> 
+erigon --snapshot --datadir=<your_datadir> 
 ```
 
 ```shell
-# 2. It's possible to start Downloader as independent process, by `--experimental.snapshot --downloader.api.addr=127.0.0.1:9093` flags:
+# 2. It's possible to start Downloader as independent process, by `--snapshot --downloader.api.addr=127.0.0.1:9093` flags:
 make erigon downloader 
 
 # Start downloader (can limit network usage by 512mb/sec: --download.rate=512mb --upload.rate=512mb)
@@ -21,10 +19,18 @@ downloader --downloader.api.addr=127.0.0.1:9093 --torrent.port=42068 --datadir=<
 # --torrent.port=42068  - is for public BitTorrent protocol listen 
 
 # Erigon on startup does send list of .torrent files to Downloader and wait for 100% download accomplishment
-erigon --experimental.snapshot --downloader.api.addr=127.0.0.1:9093 --datadir=<your_datadir> 
+erigon --snapshot --downloader.api.addr=127.0.0.1:9093 --datadir=<your_datadir> 
 ```
 
-Use `--experimental.snapshot.keepblocks=true` to don't delete retired blocks from DB
+Use `--snapshot.keepblocks=true` to don't delete retired blocks from DB
+
+Any network/chain can start with snapshot sync:
+
+- node will download only snapshots registered in next repo https://github.com/ledgerwatch/erigon-snapshot
+- node will move old blocks from DB to snapshots of 1K blocks size, then merge snapshots to bigger range, until
+  snapshots of 500K blocks, then automatically start seeding new snapshot
+
+Flag `--snapshot` is compatible with `--prune` flag
 
 ## How to create new network or bootnode
 

cmd/state/commands/global_flags_vars.go

@@ -50,7 +50,7 @@ func withIndexBucket(cmd *cobra.Command) {
 }
 
 func withSnapshotBlocks(cmd *cobra.Command) {
-	cmd.Flags().BoolVar(&snapshotBlocks, "experimental.snapshot", false, "")
+	cmd.Flags().BoolVar(&snapshotBlocks, "snapshot", false, "")
 }
 
 func withChain(cmd *cobra.Command) {

cmd/utils/flags.go

@@ -622,7 +622,7 @@ var (
 	}
 
 	SnapshotSyncFlag = cli.BoolFlag{
-		Name:  "experimental.snapshot",
+		Name:  "snapshot",
 		Usage: "Enabling experimental snapshot sync",
 	}
 	SnapshotKeepBlocksFlag = cli.BoolFlag{

docker-compose.yml

@@ -56,7 +56,7 @@ services:
       - "8551:8551"
     restart: unless-stopped
 
-  downloader: # Service to download/seed historical data (need only if you use --experimental.snapshot)
+  downloader: # Service to download/seed historical data (need only if you use --snapshot)
     image: thorax/erigon:${TAG:-latest}
     command: downloader ${DOWNLOADER_FLAGS-} --datadir=/home/erigon/.local/share/erigon --downloader.api.addr=0.0.0.0:9093
     pid: service:erigon # Use erigon's PID namespace. It's required to open Erigon's DB from another process (RPCDaemon local-mode)

eth/ethconfig/config.go

@@ -139,8 +139,8 @@ func (s Snapshot) String() string {
 }
 
 var (
-	FlagSnapshot           = "experimental.snapshot"
+	FlagSnapshot           = "snapshot"
-	FlagSnapshotKeepBlocks = "experimental.snapshot.keepblocks"
+	FlagSnapshotKeepBlocks = "snapshot.keepblocks"
 )
 
 func NewSnapshotCfg(enabled, keepBlocks bool) Snapshot {

eth/stagedsync/stage_headers.go

@@ -962,24 +962,24 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
 		if err := cfg.snapshots.ReopenIndices(); err != nil {
 			return fmt.Errorf("ReopenIndices: %w", err)
 		}
-	}
 
-	// Create .idx files
+		// Create .idx files
-	if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() {
-		if !cfg.snapshots.SegmentsReady() {
-			return fmt.Errorf("not all snapshot segments are available")
-		}
-
-		// wait for Downloader service to download all expected snapshots
 		if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() {
-			chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID)
+			if !cfg.snapshots.SegmentsReady() {
-			if err := snapshotsync.BuildIndices(ctx, cfg.snapshots, cfg.snapshotDir, *chainID, cfg.tmpdir, cfg.snapshots.IndicesAvailable(), log.LvlInfo); err != nil {
+				return fmt.Errorf("not all snapshot segments are available")
-				return err
 			}
-		}
 
-		if err := cfg.snapshots.ReopenIndices(); err != nil {
+			// wait for Downloader service to download all expected snapshots
-			return fmt.Errorf("ReopenIndices: %w", err)
+			if cfg.snapshots.IndicesAvailable() < cfg.snapshots.SegmentsAvailable() {
+				chainID, _ := uint256.FromBig(cfg.chainConfig.ChainID)
+				if err := snapshotsync.BuildIndices(ctx, cfg.snapshots, cfg.snapshotDir, *chainID, cfg.tmpdir, cfg.snapshots.IndicesAvailable(), log.LvlInfo); err != nil {
+					return err
+				}
+			}
+
+			if err := cfg.snapshots.ReopenIndices(); err != nil {
+				return fmt.Errorf("ReopenIndices: %w", err)
+			}
 		}
 	}