mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 19:50:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Snapshots: use kv.ReadAhead helper (#3909)

Browse Source
This commit is contained in:
Alex Sharov 2022-04-18 10:48:09 +07:00 committed by GitHub
parent 5a805abdd4
commit 9305efeaa7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/downloader/main.go
View File

@ -181,11 +181,8 @@ var printTorrentHashes = &cobra.Command{
return downloader.VerifyDtaFiles(ctx, snapshotDir)
}
lockedSnapshotDir := &dir.Rw{Path: snapshotDir}
if forceRebuild { // remove and create .torrent files (will re-read all snapshots)
lockedSnapshotDir, err := dir.OpenRw(snapshotDir)
if err != nil {
return err
}
defer lockedSnapshotDir.Close()
removeChunksStorage(lockedSnapshotDir)
@ -198,9 +195,9 @@ var printTorrentHashes = &cobra.Command{
return err
}
}
if err := downloader.BuildTorrentFilesIfNeed(ctx, lockedSnapshotDir); err != nil {
return err
}
}
if err := downloader.BuildTorrentFilesIfNeed(ctx, lockedSnapshotDir); err != nil {
return err
}
res := map[string]string{}

2
go.mod
View File

@ -41,7 +41,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
github.com/ledgerwatch/erigon-lib v0.0.0-20220415104858-ee6e50ab671d
github.com/ledgerwatch/erigon-lib v0.0.0-20220418023534-87a7a2124418
github.com/ledgerwatch/log/v3 v3.4.1
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/magiconair/properties v1.8.6 // indirect

4
go.sum
View File

@ -612,8 +612,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20220415104858-ee6e50ab671d h1:r+guNbIyUTF4BFGMcTekkv5+ckMHdKJYtyxmA1+WRGQ=
github.com/ledgerwatch/erigon-lib v0.0.0-20220415104858-ee6e50ab671d/go.mod h1:IQjw8YqJuqQqKqOBJ8av6TxaL+reYU6+VXWVbTrfpCw=
github.com/ledgerwatch/erigon-lib v0.0.0-20220418023534-87a7a2124418 h1:qjyseSyJJqWIlHaoPeiVFPIwjvvozTrkkEQ9YohcxNs=
github.com/ledgerwatch/erigon-lib v0.0.0-20220418023534-87a7a2124418/go.mod h1:N0SNhcFu4P+uHJNOP3Di1RzLqKeql5RyjozseoqI69E=
github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

30
turbo/snapshotsync/block_snapshots.go
View File

@ -1158,6 +1158,7 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF
firstIDSaved := false
doWarmup, warmupTxs, warmupSenders := blockTo-blockFrom >= 100_000 && workers > 4, atomic.NewBool(false), atomic.NewBool(false)
from := dbutils.EncodeBlockNumber(blockFrom)
var lastBody types.BodyForStorage
if err := kv.BigChunks(db, kv.HeaderCanonical, from, func(tx kv.Tx, k, v []byte) (bool, error) {
@ -1179,6 +1180,12 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF
if body.TxAmount == 0 {
return true, nil
}
if doWarmup && !warmupSenders.Load() && blockNum%1_000 == 0 {
kv.ReadAhead(ctx, db, warmupSenders, kv.Senders, dbutils.EncodeBlockNumber(blockNum), 10_000)
}
if doWarmup && !warmupTxs.Load() && blockNum%1_000 == 0 {
kv.ReadAhead(ctx, db, warmupTxs, kv.EthTx, dbutils.EncodeBlockNumber(body.BaseTxId), 100*10_000)
}
senders, err := rawdb.ReadSenders(tx, h, blockNum)
if err != nil {
return false, err
@ -1217,17 +1224,6 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF
count++
j++
select {
case <-ctx.Done():
return ctx.Err()
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Log(lvl, "[snapshots] Dumping txs", "block num", blockNum,
"alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys),
)
default:
}
return nil
}); err != nil {
return false, fmt.Errorf("ForAmount: %w", err)
@ -1238,6 +1234,18 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF
}
prevTxID++
count++
select {
case <-ctx.Done():
return false, ctx.Err()
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Log(lvl, "[snapshots] Dumping txs", "block num", blockNum,
"alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys),
)
default:
}
return true, nil
}); err != nil {
return 0, fmt.Errorf("BigChunks: %w", err)

2
turbo/snapshotsync/snapshothashes/erigon-snapshots

@ -1 +1 @@
Subproject commit ed7651831fb50f45861e636853ef077691726a94
Subproject commit 32182ada25c07c14155b8a3d01fe55dda10bb4f5