mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added beacon snapshots download (#8601)

Browse Source
This commit is contained in:
Giulio rebuffo 2023-10-28 17:41:50 +02:00 committed by GitHub
parent 9adf31b8eb
commit 0e5af0a69c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 101 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
cmd/capcli/cli.go
View File

@ -7,7 +7,15 @@ import (
"strings"
"time"
lg "github.com/anacrolix/log"
"github.com/ledgerwatch/erigon-lib/direct"
downloader3 "github.com/ledgerwatch/erigon-lib/downloader"
state2 "github.com/ledgerwatch/erigon-lib/state"
"github.com/c2h5oh/datasize"
"github.com/ledgerwatch/erigon-lib/chain/snapcfg"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/downloader"
"github.com/ledgerwatch/erigon/cl/abstract"
"github.com/ledgerwatch/erigon/cl/clparams"
@ -15,9 +23,12 @@ import (
persistence2 "github.com/ledgerwatch/erigon/cl/persistence"
"github.com/ledgerwatch/erigon/cmd/caplin/caplin1"
"github.com/ledgerwatch/erigon/eth/ethconfig"
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
"github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/downloader/downloadercfg"
"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/mdbx"
@ -49,9 +60,10 @@ var CLI struct {
Blocks Blocks `cmd:"" help:"download blocks from reqresp network"`
Epochs Epochs `cmd:"" help:"download epochs from reqresp network"`
Chain Chain `cmd:"" help:"download the entire chain from reqresp network"`
DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"`
CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"`
Chain Chain `cmd:"" help:"download the entire chain from reqresp network"`
DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"`
CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"`
DownloadSnapshots DownloadSnapshots `cmd:"" help:"download snapshots from webseed"`
}
type chainCfg struct {
@ -514,3 +526,63 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
}
return nil
}
type DownloadSnapshots struct {
chainCfg
outputFolder
}
func (d *DownloadSnapshots) Run(ctx *Context) error {
webSeeds := snapcfg.KnownWebseeds[d.Chain]
dirs := datadir.New(d.Datadir)
_, _, beaconConfig, _, err := clparams.GetConfigsByNetworkName(d.Chain)
if err != nil {
return err
}
rawDB := persistence.AferoRawBeaconBlockChainFromOsPath(beaconConfig, dirs.CaplinHistory)
log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler))
_, db, err := caplin1.OpenCaplinDatabase(ctx, db_config.DatabaseConfiguration{PruneDepth: math.MaxUint64}, beaconConfig, rawDB, dirs.CaplinIndexing, nil, false)
if err != nil {
return err
}
tx, err := db.BeginRw(ctx)
if err != nil {
return err
}
defer tx.Rollback()
downloadRate, err := datasize.ParseString("16mb")
if err != nil {
return err
}
uploadRate, err := datasize.ParseString("0mb")
if err != nil {
return err
}
version := "erigon: " + params.VersionWithCommit(params.GitCommit)
downloaderCfg, err := downloadercfg.New(dirs, version, lg.Info, downloadRate, uploadRate, 42069, 10, 3, nil, webSeeds, d.Chain)
if err != nil {
return err
}
downloaderCfg.DownloadTorrentFilesFromWebseed = true
downlo, err := downloader.New(ctx, downloaderCfg, dirs, log.Root(), log.LvlInfo)
if err != nil {
return err
}
s, err := state2.NewAggregatorV3(ctx, dirs.Tmp, dirs.Tmp, 200000, db, log.Root())
if err != nil {
return err
}
downlo.MainLoopInBackground(false)
bittorrentServer, err := downloader3.NewGrpcServer(downlo)
if err != nil {
return fmt.Errorf("new server: %w", err)
}
return snapshotsync.WaitForDownloader("CapCliDownloader", ctx, false, snapshotsync.OnlyCaplin, s, tx, freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root()), freezeblocks.NewBorRoSnapshots(ethconfig.NewSnapCfg(false, false, false), dirs.Snap, log.Root())), nil, params.ChainConfigByChainName(d.Chain), direct.NewDownloaderClient(bittorrentServer))
}

2
erigon-lib/downloader/downloader.go
View File

@ -102,7 +102,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger
folder: m,
torrentClient: torrentClient,
statsLock: &sync.RWMutex{},
webseeds: &WebSeeds{logger: logger, verbosity: verbosity, downloadTorrentFile: cfg.DownloadTorrentFilesFromWebseed, chainName: cfg.ChainName},
webseeds: &WebSeeds{logger: logger, verbosity: verbosity, downloadTorrentFile: cfg.DownloadTorrentFilesFromWebseed},
logger: logger,
verbosity: verbosity,
}

1
erigon-lib/downloader/webseed.go
View File

@ -36,7 +36,6 @@ type WebSeeds struct {
torrentUrls snaptype.TorrentUrls // HTTP urls of .torrent files
downloadTorrentFile bool
chainName string
logger log.Logger
verbosity log.Lvl
}

2
erigon-lib/go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon-lib
go 1.20
require (
github.com/erigontech/mdbx-go v0.27.20
github.com/erigontech/mdbx-go v0.27.19
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3
github.com/ledgerwatch/interfaces v0.0.0-20231011121315-f58b806039f0
github.com/ledgerwatch/log/v3 v3.9.0

4
erigon-lib/go.sum
View File

@ -202,8 +202,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erigontech/mdbx-go v0.27.20 h1:b6nhop/nEtXLR0s8Vu1NvoQeiI6GNRoPVA5YbUPUGU0=
github.com/erigontech/mdbx-go v0.27.20/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4=
github.com/erigontech/mdbx-go v0.27.19 h1:3jugN7wNmVZ4zDk4heLMPAGuOXuCG/0/DfzjlGB48kk=
github.com/erigontech/mdbx-go v0.27.19/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4=
github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

2
eth/stagedsync/stage_snapshots.go
View File

@ -120,7 +120,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
return nil
}
if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil {
if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, snapshotsync.NoCaplin, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil {
return err
}

4
go.mod
View File

@ -19,6 +19,7 @@ require (
github.com/VictoriaMetrics/fastcache v1.12.1
github.com/VictoriaMetrics/metrics v1.23.1
github.com/alecthomas/kong v0.8.0
github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4
github.com/anacrolix/sync v0.4.0
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b
@ -112,7 +113,6 @@ require (
github.com/anacrolix/envpprof v1.3.0 // indirect
github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 // indirect
github.com/anacrolix/go-libutp v1.3.1 // indirect
github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect
@ -186,7 +186,7 @@ require (
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect

4
go.sum
View File

@ -539,8 +539,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3 h1:59TKwBsS+Fn4iGh+PCxw7s73+/0WmDeK6bskDe3tFzY=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231019214918-3eb2303a41f3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb h1:Y6eZ4D8rMrAdDoy2za2Lkf8qSHbNNSPGnSdl/B78G4Y=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231027092055-45ee9d86a6cb/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

18
turbo/snapshotsync/snapshotsync.go
View File

@ -24,6 +24,16 @@ import (
"github.com/ledgerwatch/log/v3"
)
type CaplinMode int
const (
// CaplinModeNone - no caplin mode
NoCaplin CaplinMode = 1
OnlyCaplin CaplinMode = 2
AlsoCaplin CaplinMode = 3
)
func BuildProtoRequest(downloadRequest []services.DownloadRequest) *proto_downloader.DownloadRequest {
req := &proto_downloader.DownloadRequest{Items: make([]*proto_downloader.DownloadItem, 0, len(snaptype.AllSnapshotTypes))}
for _, r := range downloadRequest {
@ -72,7 +82,7 @@ func RequestSnapshotsDownload(ctx context.Context, downloadRequest []services.Do
// WaitForDownloader - wait for Downloader service to download all expected snapshots
// for MVP we sync with Downloader only once, in future will send new snapshots also
func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, agg *state.AggregatorV3, tx kv.RwTx, blockReader services.FullBlockReader, notifier services.DBEventNotifier, cc *chain.Config, snapshotDownloader proto_downloader.DownloaderClient) error {
func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, caplin CaplinMode, agg *state.AggregatorV3, tx kv.RwTx, blockReader services.FullBlockReader, notifier services.DBEventNotifier, cc *chain.Config, snapshotDownloader proto_downloader.DownloaderClient) error {
snapshots := blockReader.Snapshots()
borSnapshots := blockReader.BorSnapshots()
if blockReader.FreezingCfg().NoDownloader {
@ -131,6 +141,12 @@ func WaitForDownloader(logPrefix string, ctx context.Context, histV3 bool, agg *
continue
}
}
if caplin == NoCaplin && strings.Contains(p.Name, "beaconblocks") {
continue
}
if caplin == OnlyCaplin && !strings.Contains(p.Name, "beaconblocks") {
continue
}
_, exists := existingFilesMap[p.Name]
_, borExists := borExistingFilesMap[p.Name]