mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-11 21:40:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

downloader: don't loose debug logs ()

Browse Source
This commit is contained in:
Alex Sharov 2023-10-14 17:11:39 +07:00 committed by GitHub
parent 71ee11af90
commit 9db82fee5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 84 additions and 55 deletions
erigon-lib/downloader
downloader.go
downloadercfg
downloadercfg.gologger.go
util.go

13
erigon-lib/downloader/downloader.go
View File

@ -216,7 +216,6 @@ func (d *Downloader) mainLoop(silent bool) error {
return
case <-t.GotInfo():
}
t.AllowDataDownload()
t.DownloadAll()
d.wg.Add(1)
go func(t *torrent.Torrent) {
@ -470,11 +469,7 @@ func (d *Downloader) AddNewSeedableFile(ctx context.Context, name string) error
if err != nil {
return err
}
wsUrls, ok := d.webseeds.ByFileName(ts.DisplayName)
if ok {
ts.Webseeds = append(ts.Webseeds, wsUrls...)
}
err = addTorrentFile(ctx, ts, d.torrentClient)
err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds)
if err != nil {
return fmt.Errorf("addTorrentFile: %w", err)
}
@ -557,11 +552,7 @@ func (d *Downloader) addTorrentFilesFromDisk(quiet bool) error {
return err
}
for i, ts := range files {
ws, ok := d.webseeds.ByFileName(ts.DisplayName)
if ok {
ts.Webseeds = append(ts.Webseeds, ws...)
}
err := addTorrentFile(d.ctx, ts, d.torrentClient)
err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds)
if err != nil {
return err
}

10
erigon-lib/downloader/downloadercfg/downloadercfg.go
View File

@ -23,6 +23,7 @@ import (
"path/filepath"
"runtime"
"strings"
"time"
"github.com/anacrolix/dht/v2"
lg "github.com/anacrolix/log"
@ -59,6 +60,9 @@ func Default() *torrent.ClientConfig {
torrentConfig := torrent.NewDefaultClientConfig()
torrentConfig.PieceHashersPerTorrent = runtime.NumCPU()
torrentConfig.MinDialTimeout = 6 * time.Second //default: 3s
torrentConfig.HandshakesTimeout = 8 * time.Second //default: 4s
// enable dht
torrentConfig.NoDHT = true
//torrentConfig.DisableTrackers = true
@ -100,9 +104,9 @@ func New(dirs datadir.Dirs, version string, verbosity lg.Level, downloadRate, up
}
// debug
// torrentConfig.Debug = false
torrentConfig.Logger.WithFilterLevel(verbosity)
torrentConfig.Logger.Handlers = []lg.Handler{adapterHandler{}}
//torrentConfig.Debug = true
torrentConfig.Logger = torrentConfig.Logger.WithFilterLevel(verbosity)
torrentConfig.Logger.SetHandlers(adapterHandler{})
if len(staticPeers) > 0 {
torrentConfig.NoDHT = false

106
erigon-lib/downloader/downloadercfg/logger.go
View File

@ -61,63 +61,91 @@ func (b adapterHandler) Handle(r lg.Record) {
switch lvl {
case lg.Debug:
log.Info("[downloader] " + r.String())
str := r.String()
skip := strings.Contains(str, "completion change") || strings.Contains(str, "hashed piece") ||
strings.Contains(str, "set torrent=") ||
strings.Contains(str, "all initial dials failed") ||
strings.Contains(str, "local and remote peer ids are the same") ||
strings.Contains(str, "connection at") || strings.Contains(str, "don't want conns right now") ||
strings.Contains(str, "is mutually complete") ||
strings.Contains(str, "sending PEX message") || strings.Contains(str, "received pex message") ||
strings.Contains(str, "announce to") || strings.Contains(str, "announcing to") ||
strings.Contains(str, "EOF") || strings.Contains(str, "closed") || strings.Contains(str, "connection reset by peer") || strings.Contains(str, "use of closed network connection") || strings.Contains(str, "broken pipe") ||
strings.Contains(str, "inited with remoteAddr")
if skip {
break
}
log.Debug(str)
case lg.Info:
str := r.String()
if strings.Contains(str, "EOF") ||
strings.Contains(str, "spurious timer") ||
strings.Contains(str, "banning ip <nil>") { // suppress useless errors
skip := false //strings.Contains(str, "EOF")
//strings.Contains(str, "banning ip <nil>") ||
//strings.Contains(str, "spurious timer") { // suppress useless errors
if skip {
break
}
log.Info(str)
case lg.Warning:
str := r.String()
if strings.Contains(str, "could not find offer for id") { // suppress useless errors
break
}
if strings.Contains(str, "webrtc conn for unloaded torrent") { // suppress useless errors
break
}
if strings.Contains(str, "TrackerClient closed") { // suppress useless errors
break
}
if strings.Contains(str, "banned ip") { // suppress useless errors
break
}
if strings.Contains(str, "being sole dirtier of piece") { // suppress useless errors
break
}
if strings.Contains(str, "requested chunk too long") { // suppress useless errors
break
}
if strings.Contains(str, "reservation cancelled") { // suppress useless errors
break
}
if strings.Contains(str, "received invalid reject") { // suppress useless errors
break
}
skip := false
//if strings.Contains(str, "could not find offer for id") { // suppress useless errors
// break
//}
//if strings.Contains(str, "webrtc conn for unloaded torrent") { // suppress useless errors
// break
//}
//if strings.Contains(str, "TrackerClient closed") { // suppress useless errors
// break
//}
//if strings.Contains(str, "banned ip") { // suppress useless errors
// break
//}
//if strings.Contains(str, "being sole dirtier of piece") { // suppress useless errors
// break
//}
//if strings.Contains(str, "requested chunk too long") { // suppress useless errors
// break
//}
//if strings.Contains(str, "reservation cancelled") { // suppress useless errors
// break
//}
//if strings.Contains(str, "received invalid reject") { // suppress useless errors
// break
//}
if skip {
break
}
log.Warn(str)
case lg.Error:
str := r.String()
if strings.Contains(str, "EOF") { // suppress useless errors
skip := false
//if strings.Contains(str, "EOF") { // suppress useless errors
// break
//}
if skip {
break
}
log.Error(str)
case lg.Critical:
str := r.String()
if strings.Contains(str, "EOF") { // suppress useless errors
break
}
if strings.Contains(str, "don't want conns") { // suppress useless errors
break
}
if strings.Contains(str, "torrent closed") { // suppress useless errors
break
}
skip := false
//if strings.Contains(str, "EOF") { // suppress useless errors
// break
//}
//if strings.Contains(str, "don't want conns") { // suppress useless errors
// break
//}
//if strings.Contains(str, "torrent closed") { // suppress useless errors
// break
//}
if skip {
break
}
log.Error(str)
default:
log.Info("[downloader] "+r.String(), "torrent_log_type", "unknown", "or", lvl.LogString())

10
erigon-lib/downloader/util.go
View File

@ -314,13 +314,19 @@ func saveTorrent(torrentFilePath string, res []byte) error {
// added first time - pieces verification process will start (disk IO heavy) - Progress
// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again.
// Don't need call torrent.VerifyData manually
func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client) error {
func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if _, ok := torrentClient.Torrent(ts.InfoHash); !ok { // can set ChunkSize only for new torrents
wsUrls, ok := webseeds.ByFileName(ts.DisplayName)
if ok {
ts.Webseeds = append(ts.Webseeds, wsUrls...)
}
_, ok = torrentClient.Torrent(ts.InfoHash)
if !ok { // can set ChunkSize only for new torrents
ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize
} else {
ts.ChunkSize = 0