diff --git a/cl/persistence/block_saver.go b/cl/persistence/block_saver.go index 6bc35ae3c..bbbaecdaf 100644 --- a/cl/persistence/block_saver.go +++ b/cl/persistence/block_saver.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "errors" "fmt" "io" "path" @@ -18,6 +19,8 @@ import ( "github.com/spf13/afero" ) +const subDivisionFolderSize = 10_000 + type beaconChainDatabaseFilesystem struct { rawDB RawBeaconBlockChain cfg *clparams.BeaconChainConfig @@ -73,6 +76,9 @@ func (b beaconChainDatabaseFilesystem) GetRange(ctx context.Context, tx kv.Tx, f slot := slots[idx] r, err := b.rawDB.BlockReader(ctx, slot, blockRoot) + if errors.Is(err, afero.ErrFileNotFound) { + continue + } if err != nil { return nil, err } @@ -98,7 +104,7 @@ func (b beaconChainDatabaseFilesystem) PurgeRange(ctx context.Context, tx kv.RwT return err } - return beacon_indicies.PruneBlockRoots(ctx, tx, from, from+count) + return nil } func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwTx, block *cltypes.SignedBeaconBlock, canonical bool) error { @@ -156,28 +162,9 @@ func (b beaconChainDatabaseFilesystem) WriteBlock(ctx context.Context, tx kv.RwT // SlotToPaths define the file structure to store a block // -// superEpoch = floor(slot / (epochSize ^ 2)) -// epoch = floot(slot / epochSize) -// file is to be stored at -// "/signedBeaconBlock/{superEpoch}/{epoch}/{root}.ssz_snappy" +// "/signedBeaconBlock/{slot/10_000}/{root}.ssz_snappy" func RootToPaths(slot uint64, root libcommon.Hash, config *clparams.BeaconChainConfig) (folderPath string, filePath string) { - folderPath = path.Clean(fmt.Sprintf("%d/%d", slot/(config.SlotsPerEpoch*config.SlotsPerEpoch), slot/config.SlotsPerEpoch)) + folderPath = path.Clean(fmt.Sprintf("%d", slot/subDivisionFolderSize)) filePath = path.Clean(fmt.Sprintf("%s/%x.sz", folderPath, root)) return } - -func ValidateEpoch(fs afero.Fs, epoch uint64, config *clparams.BeaconChainConfig) error { - superEpoch := epoch / (config.SlotsPerEpoch) - - // the folder path is superEpoch/epoch - folderPath := path.Clean(fmt.Sprintf("%d/%d", superEpoch, epoch)) - - fi, err := afero.ReadDir(fs, folderPath) - if err != nil { - return err - } - for _, fn := range fi { - fn.Name() - } - return nil -} diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 21986c37e..9f0c6da0c 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/erigontech/mdbx-go v0.35.2-0.20231101074031-9f999220e9ed - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 03c789a7c..ebe0a6632 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -291,8 +291,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 h1:2av0zql2zwx8PL/mZWhjsuWOpHbDMK2tsiH003KriXs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520 h1:j/PRJWbPrbk8wpVjU77SWS8xJ/N+dcxPs1relNSolUs= github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 3da1d60ba..1d36be124 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -40,6 +40,7 @@ type SnapshotsCfg struct { dbEventNotifier services.DBEventNotifier historyV3 bool + caplin bool agg *state.AggregatorV3 silkworm *silkworm.Silkworm } @@ -53,6 +54,7 @@ func StageSnapshotsCfg(db kv.RwDB, dbEventNotifier services.DBEventNotifier, historyV3 bool, agg *state.AggregatorV3, + caplin bool, silkworm *silkworm.Silkworm, ) SnapshotsCfg { return SnapshotsCfg{ @@ -64,6 +66,7 @@ func StageSnapshotsCfg(db kv.RwDB, blockReader: blockReader, dbEventNotifier: dbEventNotifier, historyV3: historyV3, + caplin: caplin, agg: agg, silkworm: silkworm, } @@ -119,8 +122,12 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R if !cfg.blockReader.FreezingCfg().Enabled { return nil } + cstate := snapshotsync.NoCaplin + // if cfg.caplin { //TODO(Giulio2002): uncomment + // cstate = snapshotsync.AlsoCaplin + // } - if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, snapshotsync.NoCaplin, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { + if err := snapshotsync.WaitForDownloader(s.LogPrefix(), ctx, cfg.historyV3, cstate, cfg.agg, tx, cfg.blockReader, cfg.dbEventNotifier, &cfg.chainConfig, cfg.snapshotDownloader); err != nil { return err } diff --git a/go.mod b/go.mod index c004d8dd1..934bb0dd8 100644 --- a/go.mod +++ b/go.mod @@ -186,7 +186,7 @@ require ( github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 // indirect + github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect diff --git a/go.sum b/go.sum index e64dfc0bf..26825776a 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46 h1:yt3/AcefMQOzY/P05jyeaKpqMQvrCbL6OJWALsjKp5U= -github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231102060711-19219b948f46/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2 h1:2av0zql2zwx8PL/mZWhjsuWOpHbDMK2tsiH003KriXs= +github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231106151827-2653fe9782b2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 525358828..fc92b99ec 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -431,7 +431,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK blockRetire := freezeblocks.NewBlockRetire(1, dirs, mock.BlockReader, blockWriter, mock.DB, mock.Notifications.Events, logger) mock.Sync = stagedsync.New( stagedsync.DefaultStages(mock.Ctx, - stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, nil), + stagedsync.StageSnapshotsCfg(mock.DB, *mock.ChainConfig, dirs, blockRetire, snapshotsDownloader, mock.BlockReader, mock.Notifications.Events, mock.HistoryV3, mock.agg, false, nil), stagedsync.StageHeadersCfg(mock.DB, mock.sentriesClient.Hd, mock.sentriesClient.Bd, *mock.ChainConfig, sendHeaderRequest, propagateNewBlockHashes, penalize, cfg.BatchSize, false, mock.BlockReader, blockWriter, dirs.Tmp, mock.Notifications, engine_helpers.NewForkValidatorMock(1), nil), stagedsync.StageBorHeimdallCfg(mock.DB, snapDb, stagedsync.MiningState{}, *mock.ChainConfig, nil /* heimdallClient */, mock.BlockReader, nil, nil, recents, signatures), stagedsync.StageBlockHashesCfg(mock.DB, mock.Dirs.Tmp, mock.ChainConfig, blockWriter), diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index fa76d7a42..eff427be6 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -484,7 +484,7 @@ func NewDefaultStages(ctx context.Context, } return stagedsync.DefaultStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm), stagedsync.StageHeadersCfg(db, controlServer.Hd, controlServer.Bd, *controlServer.ChainConfig, controlServer.SendHeaderRequest, controlServer.PropagateNewBlockHashes, controlServer.Penalize, cfg.BatchSize, p2pCfg.NoDiscovery, blockReader, blockWriter, dirs.Tmp, notifications, forkValidator, loopBreakCheck), stagedsync.StageBorHeimdallCfg(db, snapDb, stagedsync.MiningState{}, *controlServer.ChainConfig, heimdallClient, blockReader, controlServer.Hd, controlServer.Penalize, recents, signatures), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), @@ -542,7 +542,7 @@ func NewPipelineStages(ctx context.Context, runInTestMode := cfg.ImportMode return stagedsync.PipelineStages(ctx, - stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, silkworm), + stagedsync.StageSnapshotsCfg(db, *controlServer.ChainConfig, dirs, blockRetire, snapDownloader, blockReader, notifications.Events, cfg.HistoryV3, agg, cfg.InternalCL, silkworm), stagedsync.StageBlockHashesCfg(db, dirs.Tmp, controlServer.ChainConfig, blockWriter), stagedsync.StageSendersCfg(db, controlServer.ChainConfig, false, dirs.Tmp, cfg.Prune, blockReader, controlServer.Hd), stagedsync.StageExecuteBlocksCfg(