mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

update caplin snapshots hashes (#8663)

Browse Source 
This PR also adds snippets to download caplin snapshots
This commit is contained in:
Giulio rebuffo 2023-11-06 21:05:07 +01:00 committed by GitHub
parent e67db34145
commit 4b580dcc2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 26 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cl/persistence/block_saver.go
View File

@ -2,6 +2,7 @@ package persistence
import (
"context"
"errors"
"fmt"
"io"
"path"
@ -18,6 +19,8 @@ import (
"github.com/spf13/afero"
)
const subDivisionFolderSize = 10_000
type beaconChainDatabaseFilesystem struct {
rawDB RawBeaconBlockChain
cfg *clparams.BeaconChainConfig
@ -73,6 +76,9 @@ func (b beaconChainDatabaseFilesystem) GetRange(ctx context.Context, tx kv.Tx, f
slot := slots[idx]
r, err := b.rawDB.BlockReader(ctx, slot, blockRoot)
if errors.Is(err, afero.ErrFileNotFound) {
continue
}
if err != nil {
return nil, err
}
@ -98,7 +104,7 @@ func (b beaconChainDatabaseFilesystem) PurgeRange(ctx context.Context, tx kv.RwT
return err
}
return beacon_indicies.PruneBlockRoots(ctx, tx, from, from+count)
return nil
}
func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwTx, block *cltypes.SignedBeaconBlock, canonical bool) error {
@ -156,28 +162,9 @@ func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwT
// SlotToPaths define the file structure to store a block
//
// superEpoch = floor(slot / (epochSize ^ 2))
// epoch = floot(slot / epochSize)
// file is to be stored at
// "/signedBeaconBlock/{superEpoch}/{epoch}/{root}.ssz_snappy"
// "/signedBeaconBlock/{slot/10_000}/{root}.ssz_snappy"
func RootToPaths(slot uint64, root libcommon.Hash, config *clparams.BeaconChainConfig) (folderPath string, filePath string) {
folderPath = path.Clean(fmt.Sprintf("%d/%d", slot/(config.SlotsPerEpoch*config.SlotsPerEpoch), slot/config.SlotsPerEpoch))
folderPath = path.Clean(fmt.Sprintf("%d", slot/subDivisionFolderSize))
filePath = path.Clean(fmt.Sprintf("%s/%x.sz", folderPath, root))
return
}
func ValidateEpoch(fs afero.Fs, epoch uint64, config *clparams.BeaconChainConfig) error {
superEpoch := epoch / (config.SlotsPerEpoch)
// the folder path is superEpoch/epoch
folderPath := path.Clean(fmt.Sprintf("%d/%d", superEpoch, epoch))
fi, err := afero.ReadDir(fs, folderPath)
if err != nil {
return err
}
for _, fn := range fi {
fn.Name()
}
return nil
}

2
erigon-lib/go.mod
View File

@ -4,7 +4,7 @@ go 1.20
require (
github.com/erigontech/mdbx-go v0.35.2-0.20231101074031-9f999220e9ed
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520
github.com/ledgerwatch/log/v3 v3.9.0
github.com/ledgerwatch/secp256k1 v1.0.0

4
erigon-lib/go.sum
View File

@ -291,8 +291,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 h1:2av0zql2zwx8PL/mZWhjsuWOpHbDMK2tsiH003KriXs=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520 h1:j/PRJWbPrbk8wpVjU77SWS8xJ/N+dcxPs1relNSolUs=
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=

9
eth/stagedsync/stage_snapshots.go
View File

@ -40,6 +40,7 @@ type SnapshotsCfg struct {
dbEventNotifier services.DBEventNotifier
historyV3 bool
caplin bool
agg *state.AggregatorV3
silkworm *silkworm.Silkworm
}
@ -53,6 +54,7 @@ func StageSnapshotsCfg(db kv.RwDB,
dbEventNotifier services.DBEventNotifier,
historyV3 bool,
agg *state.AggregatorV3,
caplin bool,
silkworm *silkworm.Silkworm,
) SnapshotsCfg {
return SnapshotsCfg{
@ -64,6 +66,7 @@ func StageSnapshotsCfg(db kv.RwDB,
blockReader: blockReader,
dbEventNotifier: dbEventNotifier,
historyV3: historyV3,
caplin: caplin,
agg: agg,
silkworm: silkworm,
}
@ -119,8 +122,12 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
if !cfg.blockReader.FreezingCfg().Enabled {
return nil
}
cstate := snapshotsync.NoCaplin
// if cfg.caplin { //TODO(Giulio2002): uncomment
// cstate = snapshotsync.AlsoCaplin
// }
if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, snapshotsync.NoCaplin, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil {
if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cstate, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil {
return err
}

2
go.mod
View File

@ -186,7 +186,7 @@ require (
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 // indirect
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect

4
go.sum
View File

@ -539,8 +539,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 h1:2av0zql2zwx8PL/mZWhjsuWOpHbDMK2tsiH003KriXs=
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

2
turbo/stages/mock/mock_sentry.go
View File

@ -431,7 +431,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK
blockRetire := freezeblocks.NewBlockRetire(1, dirs, mock.BlockReader, blockWriter, mock.DB, mock.Notifications.Events, logger)
mock.Sync = stagedsync.New(
stagedsync.DefaultStages(mock.Ctx,
stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, nil),
stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, false, nil),
stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications, engine_helpers.NewForkValidatorMock(1), nil),
stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil /* heimdallClient */, mock.BlockReader, nil, nil, recents, signatures),
stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter),

4
turbo/stages/stageloop.go
View File

@ -484,7 +484,7 @@ func NewDefaultStages(ctx context.Context,
}
return stagedsync.DefaultStages(ctx,
stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm),
stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm),
stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications, forkValidator, loopBreakCheck),
stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures),
stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter),
@ -542,7 +542,7 @@ func NewPipelineStages(ctx context.Context,
runInTestMode := cfg.ImportMode
return stagedsync.PipelineStages(ctx,
stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm),
stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm),
stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter),
stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd),
stagedsync.StageExecuteBlocksCfg(