From 9db82fee5b215d178c5d5f0c79685339900ddb18 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Sat, 14 Oct 2023 17:11:39 +0700 Subject: [PATCH] downloader: don't loose debug logs (#8473) --- erigon-lib/downloader/downloader.go | 13 +-- .../downloader/downloadercfg/downloadercfg.go | 10 +- erigon-lib/downloader/downloadercfg/logger.go | 106 +++++++++++------- erigon-lib/downloader/util.go | 10 +- 4 files changed, 84 insertions(+), 55 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 3a0caa150..4159f8447 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -216,7 +216,6 @@ func (d *Downloader) mainLoop(silent bool) error { return case <-t.GotInfo(): } - t.AllowDataDownload() t.DownloadAll() d.wg.Add(1) go func(t *torrent.Torrent) { @@ -470,11 +469,7 @@ func (d *Downloader) AddNewSeedableFile(ctx context.Context, name string) error if err != nil { return err } - wsUrls, ok := d.webseeds.ByFileName(ts.DisplayName) - if ok { - ts.Webseeds = append(ts.Webseeds, wsUrls...) - } - err = addTorrentFile(ctx, ts, d.torrentClient) + err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds) if err != nil { return fmt.Errorf("addTorrentFile: %w", err) } @@ -557,11 +552,7 @@ func (d *Downloader) addTorrentFilesFromDisk(quiet bool) error { return err } for i, ts := range files { - ws, ok := d.webseeds.ByFileName(ts.DisplayName) - if ok { - ts.Webseeds = append(ts.Webseeds, ws...) - } - err := addTorrentFile(d.ctx, ts, d.torrentClient) + err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds) if err != nil { return err } diff --git a/erigon-lib/downloader/downloadercfg/downloadercfg.go b/erigon-lib/downloader/downloadercfg/downloadercfg.go index 39f4acaea..f8737d2f6 100644 --- a/erigon-lib/downloader/downloadercfg/downloadercfg.go +++ b/erigon-lib/downloader/downloadercfg/downloadercfg.go @@ -23,6 +23,7 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/anacrolix/dht/v2" lg "github.com/anacrolix/log" @@ -59,6 +60,9 @@ func Default() *torrent.ClientConfig { torrentConfig := torrent.NewDefaultClientConfig() torrentConfig.PieceHashersPerTorrent = runtime.NumCPU() + torrentConfig.MinDialTimeout = 6 * time.Second //default: 3s + torrentConfig.HandshakesTimeout = 8 * time.Second //default: 4s + // enable dht torrentConfig.NoDHT = true //torrentConfig.DisableTrackers = true @@ -100,9 +104,9 @@ func New(dirs datadir.Dirs, version string, verbosity lg.Level, downloadRate, up } // debug - // torrentConfig.Debug = false - torrentConfig.Logger.WithFilterLevel(verbosity) - torrentConfig.Logger.Handlers = []lg.Handler{adapterHandler{}} + //torrentConfig.Debug = true + torrentConfig.Logger = torrentConfig.Logger.WithFilterLevel(verbosity) + torrentConfig.Logger.SetHandlers(adapterHandler{}) if len(staticPeers) > 0 { torrentConfig.NoDHT = false diff --git a/erigon-lib/downloader/downloadercfg/logger.go b/erigon-lib/downloader/downloadercfg/logger.go index 59bff4811..8878e9004 100644 --- a/erigon-lib/downloader/downloadercfg/logger.go +++ b/erigon-lib/downloader/downloadercfg/logger.go @@ -61,63 +61,91 @@ func (b adapterHandler) Handle(r lg.Record) { switch lvl { case lg.Debug: - log.Info("[downloader] " + r.String()) + str := r.String() + skip := strings.Contains(str, "completion change") || strings.Contains(str, "hashed piece") || + strings.Contains(str, "set torrent=") || + strings.Contains(str, "all initial dials failed") || + strings.Contains(str, "local and remote peer ids are the same") || + strings.Contains(str, "connection at") || strings.Contains(str, "don't want conns right now") || + strings.Contains(str, "is mutually complete") || + strings.Contains(str, "sending PEX message") || strings.Contains(str, "received pex message") || + strings.Contains(str, "announce to") || strings.Contains(str, "announcing to") || + strings.Contains(str, "EOF") || strings.Contains(str, "closed") || strings.Contains(str, "connection reset by peer") || strings.Contains(str, "use of closed network connection") || strings.Contains(str, "broken pipe") || + strings.Contains(str, "inited with remoteAddr") + if skip { + break + } + log.Debug(str) case lg.Info: str := r.String() - if strings.Contains(str, "EOF") || - strings.Contains(str, "spurious timer") || - strings.Contains(str, "banning ip ") { // suppress useless errors + skip := false //strings.Contains(str, "EOF") + //strings.Contains(str, "banning ip ") || + //strings.Contains(str, "spurious timer") { // suppress useless errors + if skip { break } log.Info(str) case lg.Warning: str := r.String() - if strings.Contains(str, "could not find offer for id") { // suppress useless errors - break - } - if strings.Contains(str, "webrtc conn for unloaded torrent") { // suppress useless errors - break - } - if strings.Contains(str, "TrackerClient closed") { // suppress useless errors - break - } - if strings.Contains(str, "banned ip") { // suppress useless errors - break - } - if strings.Contains(str, "being sole dirtier of piece") { // suppress useless errors - break - } - if strings.Contains(str, "requested chunk too long") { // suppress useless errors - break - } - if strings.Contains(str, "reservation cancelled") { // suppress useless errors - break - } - if strings.Contains(str, "received invalid reject") { // suppress useless errors - break - } + skip := false + //if strings.Contains(str, "could not find offer for id") { // suppress useless errors + // break + //} + //if strings.Contains(str, "webrtc conn for unloaded torrent") { // suppress useless errors + // break + //} + //if strings.Contains(str, "TrackerClient closed") { // suppress useless errors + // break + //} + //if strings.Contains(str, "banned ip") { // suppress useless errors + // break + //} + //if strings.Contains(str, "being sole dirtier of piece") { // suppress useless errors + // break + //} + //if strings.Contains(str, "requested chunk too long") { // suppress useless errors + // break + //} + //if strings.Contains(str, "reservation cancelled") { // suppress useless errors + // break + //} + //if strings.Contains(str, "received invalid reject") { // suppress useless errors + // break + //} + + if skip { + break + } log.Warn(str) case lg.Error: str := r.String() - if strings.Contains(str, "EOF") { // suppress useless errors + skip := false + //if strings.Contains(str, "EOF") { // suppress useless errors + // break + //} + + if skip { break } - log.Error(str) case lg.Critical: str := r.String() - if strings.Contains(str, "EOF") { // suppress useless errors - break - } - if strings.Contains(str, "don't want conns") { // suppress useless errors - break - } - if strings.Contains(str, "torrent closed") { // suppress useless errors - break - } + skip := false + //if strings.Contains(str, "EOF") { // suppress useless errors + // break + //} + //if strings.Contains(str, "don't want conns") { // suppress useless errors + // break + //} + //if strings.Contains(str, "torrent closed") { // suppress useless errors + // break + //} + if skip { + break + } log.Error(str) default: log.Info("[downloader] "+r.String(), "torrent_log_type", "unknown", "or", lvl.LogString()) diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index 9639611e7..7bf8f5827 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -314,13 +314,19 @@ func saveTorrent(torrentFilePath string, res []byte) error { // added first time - pieces verification process will start (disk IO heavy) - Progress // kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. // Don't need call torrent.VerifyData manually -func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client) error { +func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) error { select { case <-ctx.Done(): return ctx.Err() default: } - if _, ok := torrentClient.Torrent(ts.InfoHash); !ok { // can set ChunkSize only for new torrents + wsUrls, ok := webseeds.ByFileName(ts.DisplayName) + if ok { + ts.Webseeds = append(ts.Webseeds, wsUrls...) + } + + _, ok = torrentClient.Torrent(ts.InfoHash) + if !ok { // can set ChunkSize only for new torrents ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize } else { ts.ChunkSize = 0