From 3cee4ac5c8dc13f13d1cb90abd2324d3a3221be1 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Sun, 10 Sep 2023 15:46:27 +0700 Subject: [PATCH] Downloader: don't fail when see unusual file, skip it (backward/forward compatibility) (#1117) --- downloader/downloader.go | 19 ++++++++---------- downloader/snaptype/files.go | 37 +++++++++--------------------------- downloader/util.go | 8 ++++---- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index cf1296a45..ee0a4c1be 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -68,7 +68,7 @@ type AggStats struct { Progress float32 BytesCompleted, BytesTotal uint64 - DroppedCompleted, DroppedTotal atomic.Uint64 + DroppedCompleted, DroppedTotal uint64 BytesDownload, BytesUpload uint64 UploadRate, DownloadRate uint64 @@ -158,8 +158,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { case <-t.GotInfo(): } if t.Complete.Bool() { - d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted())) - d.stats.DroppedTotal.Add(uint64(t.Length())) + atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) + atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) //t.Drop() torrentMap[t.InfoHash()] = struct{}{} continue @@ -179,8 +179,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { return case <-t.Complete.On(): } - d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted())) - d.stats.DroppedTotal.Add(uint64(t.Length())) + atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) + atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) //t.Drop() }(t) } @@ -188,6 +188,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { goto DownloadLoop } + atomic.StoreUint64(&d.stats.DroppedCompleted, 0) + atomic.StoreUint64(&d.stats.DroppedTotal, 0) if err := d.addSegments(ctx); err != nil { return } @@ -203,8 +205,6 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { case <-t.GotInfo(): } if t.Complete.Bool() { - d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted())) - d.stats.DroppedTotal.Add(uint64(t.Length())) //t.Drop() torrentMap[t.InfoHash()] = struct{}{} continue @@ -224,9 +224,6 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { return case <-t.Complete.On(): } - d.stats.DroppedCompleted.Add(uint64(t.BytesCompleted())) - d.stats.DroppedTotal.Add(uint64(t.Length())) - //t.Drop() }(t) } if len(torrents) != len(d.Torrent().Torrents()) { //if amount of torrents changed - keep downloading @@ -309,7 +306,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { stats.BytesDownload = uint64(connStats.BytesReadUsefulIntendedData.Int64()) stats.BytesUpload = uint64(connStats.BytesWrittenData.Int64()) - stats.BytesTotal, stats.BytesCompleted, stats.ConnectionsTotal, stats.MetadataReady = stats.DroppedTotal.Load(), stats.DroppedCompleted.Load(), 0, 0 + stats.BytesTotal, stats.BytesCompleted, stats.ConnectionsTotal, stats.MetadataReady = atomic.LoadUint64(&stats.DroppedTotal), atomic.LoadUint64(&stats.DroppedCompleted), 0, 0 for _, t := range torrents { select { case <-t.GotInfo(): diff --git a/downloader/snaptype/files.go b/downloader/snaptype/files.go index 424596e07..01b0cc95b 100644 --- a/downloader/snaptype/files.go +++ b/downloader/snaptype/files.go @@ -121,16 +121,15 @@ func IsCorrectHistoryFileName(name string) bool { return len(parts) == 3 } -func ParseFileName(dir, fileName string) (res FileInfo, err error) { +func ParseFileName(dir, fileName string) (res FileInfo, ok bool) { ext := filepath.Ext(fileName) onlyName := fileName[:len(fileName)-len(ext)] parts := strings.Split(onlyName, "-") if len(parts) < 4 { - return res, fmt.Errorf("expected format: v1-001500-002000-bodies.seg got: %s. %w", fileName, ErrInvalidFileName) - } - if parts[0] != "v1" { - return res, fmt.Errorf("version: %s. %w", parts[0], ErrInvalidFileName) + return res, ok } + version := parts[0] + _ = version from, err := strconv.ParseUint(parts[1], 10, 64) if err != nil { return @@ -139,26 +138,11 @@ func ParseFileName(dir, fileName string) (res FileInfo, err error) { if err != nil { return } - var snapshotType Type ft, ok := ParseFileType(parts[3]) if !ok { - return res, fmt.Errorf("unexpected snapshot suffix: %s,%w", parts[2], ErrInvalidFileName) + return res, ok } - switch ft { - case Headers: - snapshotType = Headers - case Bodies: - snapshotType = Bodies - case Transactions: - snapshotType = Transactions - case BorEvents: - snapshotType = BorEvents - case BorSpans: - snapshotType = BorSpans - default: - return res, fmt.Errorf("unexpected snapshot suffix: %s,%w", parts[2], ErrInvalidFileName) - } - return FileInfo{From: from * 1_000, To: to * 1_000, Path: filepath.Join(dir, fileName), T: snapshotType, Ext: ext}, nil + return FileInfo{From: from * 1_000, To: to * 1_000, Path: filepath.Join(dir, fileName), T: ft, Ext: ext}, ok } const Erigon3SeedableSteps = 32 @@ -217,12 +201,9 @@ func ParseDir(dir string) (res []FileInfo, err error) { continue } - meta, err := ParseFileName(dir, f.Name()) - if err != nil { - if errors.Is(err, ErrInvalidFileName) { - continue - } - return nil, err + meta, ok := ParseFileName(dir, f.Name()) + if !ok { + continue } res = append(res, meta) } diff --git a/downloader/util.go b/downloader/util.go index f35ecb788..88cd38293 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -137,9 +137,9 @@ func seedableSegmentFiles(dir string) ([]string, error) { if filepath.Ext(f.Name()) != ".seg" { // filter out only compressed files continue } - ff, err := snaptype.ParseFileName(dir, f.Name()) - if err != nil { - return nil, fmt.Errorf("ParseFileName: %w", err) + ff, ok := snaptype.ParseFileName(dir, f.Name()) + if !ok { + continue } if !ff.Seedable() { continue @@ -174,7 +174,7 @@ func seedableHistorySnapshots(dir string) ([]string, error) { continue } ext := filepath.Ext(f.Name()) - if ext != ".v" && ext != ".ef" { // filter out only compressed files + if ext != ".v" && ext != ".ef" && ext != ".kv" { // filter out only compressed files continue }