diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index e593e4f4d..4a0012e13 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -7,7 +7,15 @@ import ( "strings" "time" + lg "github.com/anacrolix/log" + "github.com/ledgerwatch/erigon-lib/direct" + downloader3 "github.com/ledgerwatch/erigon-lib/downloader" + state2 "github.com/ledgerwatch/erigon-lib/state" + + "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/erigon-lib/chain/snapcfg" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/downloader" "github.com/ledgerwatch/erigon/cl/abstract" "github.com/ledgerwatch/erigon/cl/clparams" @@ -15,9 +23,12 @@ import ( persistence2 "github.com/ledgerwatch/erigon/cl/persistence" "github.com/ledgerwatch/erigon/cmd/caplin/caplin1" "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/params" + "github.com/ledgerwatch/erigon/turbo/snapshotsync" "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" @@ -49,9 +60,10 @@ var CLI struct { Blocks Blocks `cmd:"" help:"download blocks from reqresp network"` Epochs Epochs `cmd:"" help:"download epochs from reqresp network"` - Chain Chain `cmd:"" help:"download the entire chain from reqresp network"` - DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"` - CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"` + Chain Chain `cmd:"" help:"download the entire chain from reqresp network"` + DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"` + CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"` + DownloadSnapshots DownloadSnapshots `cmd:"" help:"download snapshots from webseed"` } type chainCfg struct { @@ -514,3 +526,63 @@ func (c *CheckSnapshots) Run(ctx *Context) error { } return nil } + +type DownloadSnapshots struct { + chainCfg + outputFolder +} + +func (d *DownloadSnapshots) Run(ctx *Context) error { + webSeeds := snapcfg.KnownWebseeds[d.Chain] + dirs := datadir.New(d.Datadir) + + _, _, beaconConfig, _, err := clparams.GetConfigsByNetworkName(d.Chain) + if err != nil { + return err + } + + rawDB := persistence.AferoRawBeaconBlockChainFromOsPath(beaconConfig, dirs.CaplinHistory) + + log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler)) + + _, db, err := caplin1.OpenCaplinDatabase(ctx, db_config.DatabaseConfiguration{PruneDepth: math.MaxUint64}, beaconConfig, rawDB, dirs.CaplinIndexing, nil, false) + if err != nil { + return err + } + tx, err := db.BeginRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + downloadRate, err := datasize.ParseString("16mb") + if err != nil { + return err + } + + uploadRate, err := datasize.ParseString("0mb") + if err != nil { + return err + } + version := "erigon: " + params.VersionWithCommit(params.GitCommit) + + downloaderCfg, err := downloadercfg.New(dirs, version, lg.Info, downloadRate, uploadRate, 42069, 10, 3, nil, webSeeds, d.Chain) + if err != nil { + return err + } + downloaderCfg.DownloadTorrentFilesFromWebseed = true + downlo, err := downloader.New(ctx, downloaderCfg, dirs, log.Root(), log.LvlInfo) + if err != nil { + return err + } + s, err := state2.NewAggregatorV3(ctx, dirs.Tmp, dirs.Tmp, 200000, db, log.Root()) + if err != nil { + return err + } + downlo.MainLoopInBackground(false) + bittorrentServer, err := downloader3.NewGrpcServer(downlo) + if err != nil { + return fmt.Errorf("new server: %w", err) + } + return snapshotsync.WaitForDownloader("CapCliDownloader", ctx, false, snapshotsync.OnlyCaplin, s, tx, freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root()), freezeblocks.NewBorRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root())), nil, params.ChainConfigByChainName(d.Chain), direct.NewDownloaderClient(bittorrentServer)) + +} diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index df0dd76ce..9498479ce 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -102,7 +102,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger folder: m, torrentClient: torrentClient, statsLock: &sync.RWMutex{}, - webseeds: &WebSeeds{logger: logger, verbosity: verbosity, downloadTorrentFile: cfg.DownloadTorrentFilesFromWebseed, chainName: cfg.ChainName}, + webseeds: &WebSeeds{logger: logger, verbosity: verbosity, downloadTorrentFile: cfg.DownloadTorrentFilesFromWebseed}, logger: logger, verbosity: verbosity, } diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index e1112499f..45947deda 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -36,7 +36,6 @@ type WebSeeds struct { torrentUrls snaptype.TorrentUrls // HTTP urls of .torrent files downloadTorrentFile bool - chainName string logger log.Logger verbosity log.Lvl } diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index c3558dbb1..bb94dcd66 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon-lib go 1.20 require ( - github.com/erigontech/mdbx-go v0.27.20 + github.com/erigontech/mdbx-go v0.27.19 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3 github.com/ledgerwatch/interfaces v0.0.0-20231011121315-f58b806039f0 github.com/ledgerwatch/log/v3 v3.9.0 diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index e93c0f750..5c684a438 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -202,8 +202,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erigontech/mdbx-go v0.27.20 h1:b6nhop/nEtXLR0s8Vu1NvoQeiI6GNRoPVA5YbUPUGU0= -github.com/erigontech/mdbx-go v0.27.20/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= +github.com/erigontech/mdbx-go v0.27.19 h1:3jugN7wNmVZ4zDk4heLMPAGuOXuCG/0/DfzjlGB48kk= +github.com/erigontech/mdbx-go v0.27.19/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 0ea782fba..5332ae54c 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -120,7 +120,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R return nil } - if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { + if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, snapshotsync.NoCaplin, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { return err } diff --git a/go.mod b/go.mod index a16b18a67..fa42248c0 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/VictoriaMetrics/fastcache v1.12.1 github.com/VictoriaMetrics/metrics v1.23.1 github.com/alecthomas/kong v0.8.0 + github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4 github.com/anacrolix/sync v0.4.0 github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b @@ -112,7 +113,6 @@ require ( github.com/anacrolix/envpprof v1.3.0 // indirect github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 // indirect github.com/anacrolix/go-libutp v1.3.1 // indirect - github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect @@ -186,7 +186,7 @@ require ( github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3 // indirect + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect diff --git a/go.sum b/go.sum index 9bf0275ef..2f85410ce 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3 h1:59TKwBsS+Fn4iGh+PCxw7s73+/0WmDeK6bskDe3tFzY= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb h1:Y6eZ4D8rMrAdDoy2za2Lkf8qSHbNNSPGnSdl/B78G4Y= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 0661bf843..c1cfd628d 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -24,6 +24,16 @@ import ( "github.com/ledgerwatch/log/v3" ) +type CaplinMode int + +const ( + + // CaplinModeNone - no caplin mode + NoCaplin CaplinMode = 1 + OnlyCaplin CaplinMode = 2 + AlsoCaplin CaplinMode = 3 +) + func BuildProtoRequest(downloadRequest []services.DownloadRequest) *proto_downloader.DownloadRequest { req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, 0, len(snaptype.AllSnapshotTypes))} for _, r := range downloadRequest { @@ -72,7 +82,7 @@ func RequestSnapshotsDownload(ctx context.Context, downloadRequest []services.Do // WaitForDownloader - wait for Downloader service to download all expected snapshots // for MVP we sync with Downloader only once, in future will send new snapshots also -func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, agg *state.AggregatorV3, tx kv.RwTx, blockReader services.FullBlockReader, notifier services.DBEventNotifier, cc *chain.Config, snapshotDownloader proto_downloader.DownloaderClient) error { +func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, caplin CaplinMode, agg *state.AggregatorV3, tx kv.RwTx, blockReader services.FullBlockReader, notifier services.DBEventNotifier, cc *chain.Config, snapshotDownloader proto_downloader.DownloaderClient) error { snapshots := blockReader.Snapshots() borSnapshots := blockReader.BorSnapshots() if blockReader.FreezingCfg().NoDownloader { @@ -131,6 +141,12 @@ func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, agg * continue } } + if caplin == NoCaplin && strings.Contains(p.Name, "beaconblocks") { + continue + } + if caplin == OnlyCaplin && !strings.Contains(p.Name, "beaconblocks") { + continue + } _, exists := existingFilesMap[p.Name] _, borExists := borExistingFilesMap[p.Name]