mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2025-01-08 03:51:20 +00:00
Merge branch 'stable'
This commit is contained in:
commit
0d0bcd0071
@ -68,7 +68,7 @@ type AggStats struct {
|
||||
Progress float32
|
||||
|
||||
BytesCompleted, BytesTotal uint64
|
||||
DroppedCompleted, DroppedTotal atomic.Uint64
|
||||
DroppedCompleted, DroppedTotal uint64
|
||||
|
||||
BytesDownload, BytesUpload uint64
|
||||
UploadRate, DownloadRate uint64
|
||||
@ -158,8 +158,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error {
|
||||
case <-t.GotInfo():
|
||||
}
|
||||
if t.Complete.Bool() {
|
||||
d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted()))
|
||||
d.stats.DroppedTotal.Add(uint64(t.Length()))
|
||||
atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted()))
|
||||
atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length()))
|
||||
//t.Drop()
|
||||
torrentMap[t.InfoHash()] = struct{}{}
|
||||
continue
|
||||
@ -179,8 +179,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error {
|
||||
return
|
||||
case <-t.Complete.On():
|
||||
}
|
||||
d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted()))
|
||||
d.stats.DroppedTotal.Add(uint64(t.Length()))
|
||||
atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted()))
|
||||
atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length()))
|
||||
//t.Drop()
|
||||
}(t)
|
||||
}
|
||||
@ -188,6 +188,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error {
|
||||
goto DownloadLoop
|
||||
}
|
||||
|
||||
atomic.StoreUint64(&d.stats.DroppedCompleted, 0)
|
||||
atomic.StoreUint64(&d.stats.DroppedTotal, 0)
|
||||
if err := d.addSegments(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
@ -203,8 +205,6 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error {
|
||||
case <-t.GotInfo():
|
||||
}
|
||||
if t.Complete.Bool() {
|
||||
d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted()))
|
||||
d.stats.DroppedTotal.Add(uint64(t.Length()))
|
||||
//t.Drop()
|
||||
torrentMap[t.InfoHash()] = struct{}{}
|
||||
continue
|
||||
@ -224,9 +224,6 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error {
|
||||
return
|
||||
case <-t.Complete.On():
|
||||
}
|
||||
d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted()))
|
||||
d.stats.DroppedTotal.Add(uint64(t.Length()))
|
||||
//t.Drop()
|
||||
}(t)
|
||||
}
|
||||
if len(torrents) != len(d.Torrent().Torrents()) { //if amount of torrents changed - keep downloading
|
||||
@ -309,7 +306,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) {
|
||||
stats.BytesDownload = uint64(connStats.BytesReadUsefulIntendedData.Int64())
|
||||
stats.BytesUpload = uint64(connStats.BytesWrittenData.Int64())
|
||||
|
||||
stats.BytesTotal, stats.BytesCompleted, stats.ConnectionsTotal, stats.MetadataReady = stats.DroppedTotal.Load(), stats.DroppedCompleted.Load(), 0, 0
|
||||
stats.BytesTotal, stats.BytesCompleted, stats.ConnectionsTotal, stats.MetadataReady = atomic.LoadUint64(&stats.DroppedTotal), atomic.LoadUint64(&stats.DroppedCompleted), 0, 0
|
||||
for _, t := range torrents {
|
||||
select {
|
||||
case <-t.GotInfo():
|
||||
|
@ -121,16 +121,15 @@ func IsCorrectHistoryFileName(name string) bool {
|
||||
return len(parts) == 3
|
||||
}
|
||||
|
||||
func ParseFileName(dir, fileName string) (res FileInfo, err error) {
|
||||
func ParseFileName(dir, fileName string) (res FileInfo, ok bool) {
|
||||
ext := filepath.Ext(fileName)
|
||||
onlyName := fileName[:len(fileName)-len(ext)]
|
||||
parts := strings.Split(onlyName, "-")
|
||||
if len(parts) < 4 {
|
||||
return res, fmt.Errorf("expected format: v1-001500-002000-bodies.seg got: %s. %w", fileName, ErrInvalidFileName)
|
||||
}
|
||||
if parts[0] != "v1" {
|
||||
return res, fmt.Errorf("version: %s. %w", parts[0], ErrInvalidFileName)
|
||||
return res, ok
|
||||
}
|
||||
version := parts[0]
|
||||
_ = version
|
||||
from, err := strconv.ParseUint(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
return
|
||||
@ -139,26 +138,11 @@ func ParseFileName(dir, fileName string) (res FileInfo, err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var snapshotType Type
|
||||
ft, ok := ParseFileType(parts[3])
|
||||
if !ok {
|
||||
return res, fmt.Errorf("unexpected snapshot suffix: %s,%w", parts[2], ErrInvalidFileName)
|
||||
return res, ok
|
||||
}
|
||||
switch ft {
|
||||
case Headers:
|
||||
snapshotType = Headers
|
||||
case Bodies:
|
||||
snapshotType = Bodies
|
||||
case Transactions:
|
||||
snapshotType = Transactions
|
||||
case BorEvents:
|
||||
snapshotType = BorEvents
|
||||
case BorSpans:
|
||||
snapshotType = BorSpans
|
||||
default:
|
||||
return res, fmt.Errorf("unexpected snapshot suffix: %s,%w", parts[2], ErrInvalidFileName)
|
||||
}
|
||||
return FileInfo{From: from * 1_000, To: to * 1_000, Path: filepath.Join(dir, fileName), T: snapshotType, Ext: ext}, nil
|
||||
return FileInfo{From: from * 1_000, To: to * 1_000, Path: filepath.Join(dir, fileName), T: ft, Ext: ext}, ok
|
||||
}
|
||||
|
||||
const Erigon3SeedableSteps = 32
|
||||
@ -217,12 +201,9 @@ func ParseDir(dir string) (res []FileInfo, err error) {
|
||||
continue
|
||||
}
|
||||
|
||||
meta, err := ParseFileName(dir, f.Name())
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrInvalidFileName) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
meta, ok := ParseFileName(dir, f.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
res = append(res, meta)
|
||||
}
|
||||
|
@ -137,9 +137,9 @@ func seedableSegmentFiles(dir string) ([]string, error) {
|
||||
if filepath.Ext(f.Name()) != ".seg" { // filter out only compressed files
|
||||
continue
|
||||
}
|
||||
ff, err := snaptype.ParseFileName(dir, f.Name())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ParseFileName: %w", err)
|
||||
ff, ok := snaptype.ParseFileName(dir, f.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !ff.Seedable() {
|
||||
continue
|
||||
@ -174,7 +174,7 @@ func seedableHistorySnapshots(dir string) ([]string, error) {
|
||||
continue
|
||||
}
|
||||
ext := filepath.Ext(f.Name())
|
||||
if ext != ".v" && ext != ".ef" { // filter out only compressed files
|
||||
if ext != ".v" && ext != ".ef" && ext != ".kv" { // filter out only compressed files
|
||||
continue
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user