mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 11:41:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

faster opening of snapshots and indices (#7757)

Browse Source 
- mostly by MADV_SEQUENTIAL during opening 
- see https://github.com/ledgerwatch/erigon-lib/pull/1025
This commit is contained in:
Alex Sharov 2023-06-19 11:34:01 +07:00 committed by GitHub
parent eae2d9a79c
commit a77edd7a32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.19
require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475
github.com/ledgerwatch/log/v3 v3.8.0
github.com/ledgerwatch/secp256k1 v1.0.0

4
go.sum
View File

@ -417,8 +417,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6 h1:GnKL1d8xQqVbf9nEt7T1qUR6TvtX5LTU+7al6VG5ugA=
github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6 h1:x86x6UIBiz2V2XYBUGQ74QT6KaZ1RWU0T4oE/o3oyGo=
github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475 h1:1BvWA6agTUS4RZUHx79f45HpvelMVv4iEddaURUYcC8=
github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.8.0 h1:gCpp7uGtIerEz1jKVPeDnbIopFPud9ZnCpBLlLBGqPU=

79
turbo/snapshotsync/freezeblocks/block_snapshots.go
View File

@ -556,20 +556,21 @@ Loop:
switch f.T {
case snaptype.Headers:
for _, sn := range s.Headers.segments {
if sn.seg == nil { // it's ok if some segment was not able to open
var sn *HeaderSegment
var exists bool
for _, sn2 := range s.Headers.segments {
if sn2.seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.seg.FileName() {
sn = sn2
exists = true
break
}
}
sn := &HeaderSegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &HeaderSegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
@ -586,27 +587,30 @@ Loop:
}
}
// it's possible to iterate over .seg file even if you don't have index
// then make segment available even if index open may fail
s.Headers.segments = append(s.Headers.segments, sn)
if !exists {
// it's possible to iterate over .seg file even if you don't have index
// then make segment available even if index open may fail
s.Headers.segments = append(s.Headers.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
case snaptype.Bodies:
for _, sn := range s.Bodies.segments {
if sn.seg == nil {
var sn *BodySegment
var exists bool
for _, sn2 := range s.Bodies.segments {
if sn2.seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.seg.FileName() {
sn = sn2
exists = true
break
}
}
sn := &BodySegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &BodySegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
@ -622,25 +626,28 @@ Loop:
return err
}
}
s.Bodies.segments = append(s.Bodies.segments, sn)
if !exists {
s.Bodies.segments = append(s.Bodies.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
case snaptype.Transactions:
for _, sn := range s.Txs.segments {
if sn.Seg == nil {
var sn *TxnSegment
var exists bool
for _, sn2 := range s.Txs.segments {
if sn2.Seg == nil { // it's ok if some segment was not able to open
continue
}
_, name := filepath.Split(sn.Seg.FilePath())
if fName == name {
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
continue Loop
if fName == sn2.Seg.FileName() {
sn = sn2
exists = true
break
}
}
sn := &TxnSegment{ranges: Range{f.From, f.To}}
if !exists {
sn = &TxnSegment{ranges: Range{f.From, f.To}}
}
if err := sn.reopenSeg(s.dir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if optimistic {
@ -656,7 +663,9 @@ Loop:
return err
}
}
s.Txs.segments = append(s.Txs.segments, sn)
if !exists {
s.Txs.segments = append(s.Txs.segments, sn)
}
if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}