mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-22 03:30:37 +00:00
downloader: download rates per peer with webseeds (#8879)
This commit is contained in:
Alex Sharov
GitHub
parent
421118378a
commit
ad48ecdcbb
@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -328,24 +329,45 @@ func (d *Downloader) ReCalcStats(interval time.Duration) {
|
||||
select {
|
||||
case <-t.GotInfo():
|
||||
stats.MetadataReady++
|
||||
for _, peer := range t.PeerConns() {
|
||||
peersOfThisFile := t.PeerConns()
|
||||
weebseedPeersOfThisFile := t.WebseedPeerConns()
|
||||
for _, peer := range peersOfThisFile {
|
||||
stats.ConnectionsTotal++
|
||||
peers[peer.PeerID] = struct{}{}
|
||||
}
|
||||
stats.BytesCompleted += uint64(t.BytesCompleted())
|
||||
stats.BytesTotal += uint64(t.Length())
|
||||
if !t.Complete.Bool() {
|
||||
progress := float32(float64(100) * (float64(t.BytesCompleted()) / float64(t.Length())))
|
||||
if progress == 0 {
|
||||
zeroProgress = append(zeroProgress, t.Name())
|
||||
} else {
|
||||
peersOfThisFile := make(map[torrent.PeerID]struct{}, 16)
|
||||
for _, peer := range t.PeerConns() {
|
||||
peersOfThisFile[peer.PeerID] = struct{}{}
|
||||
if t.Complete.Bool() {
|
||||
break //of switch
|
||||
}
|
||||
|
||||
progress := float32(float64(100) * (float64(t.BytesCompleted()) / float64(t.Length())))
|
||||
if progress == 0 {
|
||||
zeroProgress = append(zeroProgress, t.Name())
|
||||
break //of switch
|
||||
}
|
||||
|
||||
d.logger.Log(d.verbosity, "[snapshots] progress", "file", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "peers", len(peersOfThisFile), "webseeds", len(weebseedPeersOfThisFile))
|
||||
if d.verbosity < log.LvlInfo {
|
||||
break //of switch
|
||||
}
|
||||
|
||||
// more detailed statistic: download rate of each peer (for each file)
|
||||
webseedRates := make([]interface{}, 0, len(weebseedPeersOfThisFile)*2)
|
||||
for _, peer := range weebseedPeersOfThisFile {
|
||||
urlS := strings.Trim(strings.TrimPrefix(peer.String(), "webseed peer for "), "\"")
|
||||
if urlObj, err := url.Parse(urlS); err == nil {
|
||||
if shortUrl, err := url.JoinPath(urlObj.Host, urlObj.Path); err == nil {
|
||||
webseedRates = append(webseedRates, shortUrl, fmt.Sprintf("%s/s", common.ByteCount(uint64(peer.DownloadRate()))))
|
||||
}
|
||||
d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile))
|
||||
}
|
||||
}
|
||||
d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...)
|
||||
rates := make([]interface{}, 0, len(peersOfThisFile)*2)
|
||||
for _, peer := range peersOfThisFile {
|
||||
rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(uint64(peer.DownloadRate()))))
|
||||
}
|
||||
d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...)
|
||||
default:
|
||||
noMetadata = append(noMetadata, t.Name())
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@ go 1.20
|
||||
|
||||
require (
|
||||
github.com/erigontech/mdbx-go v0.27.21
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b
|
||||
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520
|
||||
github.com/ledgerwatch/log/v3 v3.9.0
|
||||
github.com/ledgerwatch/secp256k1 v1.0.0
|
||||
@ -15,7 +15,7 @@ require (
|
||||
github.com/anacrolix/dht/v2 v2.20.0
|
||||
github.com/anacrolix/go-libutp v1.3.1
|
||||
github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8
|
||||
github.com/aws/aws-sdk-go-v2 v1.21.2
|
||||
github.com/aws/aws-sdk-go-v2/config v1.19.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.43
|
||||
|
||||
@ -71,8 +71,8 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC
|
||||
github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
|
||||
github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
|
||||
github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe h1:kqJye1x6GGJWNC8mq9ESPwMVMvUYkdHyxum9bX7Soe0=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe/go.mod h1:Ma/WtLey9lU97u2i55LUJ8AnXaL2GfEK6pWh7/9v1hI=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8 h1:6EyYT2DsEOZ/WwTDsQ0HXHI996IdT0MZCGP2L6xvfNg=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8/go.mod h1:Ma/WtLey9lU97u2i55LUJ8AnXaL2GfEK6pWh7/9v1hI=
|
||||
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA=
|
||||
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96/go.mod h1:Wa6n8cYIdaG35x15aH3Zy6d03f7P728QfdcDeD/IEOs=
|
||||
github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4=
|
||||
@ -291,8 +291,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27 h1:1iLvIq2oxLgPaz0BSVGa9Dmu750+G2puOeLxTWROoAg=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b h1:7T/Rk0mRuuHBQ+llajSJY7k59Ku7mxwU5Eb6uytiePE=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
|
||||
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520 h1:j/PRJWbPrbk8wpVjU77SWS8xJ/N+dcxPs1relNSolUs=
|
||||
github.com/ledgerwatch/interfaces v0.0.0-20231031050643-c86352e41520/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
|
||||
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
|
||||
|
||||
4
go.mod
4
go.mod
@ -21,7 +21,7 @@ require (
|
||||
github.com/alecthomas/kong v0.8.0
|
||||
github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4
|
||||
github.com/anacrolix/sync v0.4.0
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8
|
||||
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.1.3
|
||||
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
|
||||
@ -186,7 +186,7 @@ require (
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27 // indirect
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
|
||||
|
||||
8
go.sum
8
go.sum
@ -129,8 +129,8 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC
|
||||
github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
|
||||
github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
|
||||
github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe h1:kqJye1x6GGJWNC8mq9ESPwMVMvUYkdHyxum9bX7Soe0=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20230926121951-11833b45cfbe/go.mod h1:Ma/WtLey9lU97u2i55LUJ8AnXaL2GfEK6pWh7/9v1hI=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8 h1:6EyYT2DsEOZ/WwTDsQ0HXHI996IdT0MZCGP2L6xvfNg=
|
||||
github.com/anacrolix/torrent v1.52.6-0.20231201115409-7ea994b6bbd8/go.mod h1:Ma/WtLey9lU97u2i55LUJ8AnXaL2GfEK6pWh7/9v1hI=
|
||||
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA=
|
||||
github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96/go.mod h1:Wa6n8cYIdaG35x15aH3Zy6d03f7P728QfdcDeD/IEOs=
|
||||
github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4=
|
||||
@ -540,8 +540,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
|
||||
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
|
||||
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27 h1:1iLvIq2oxLgPaz0BSVGa9Dmu750+G2puOeLxTWROoAg=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231130092351-fc8d6b8c7b27/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b h1:7T/Rk0mRuuHBQ+llajSJY7k59Ku7mxwU5Eb6uytiePE=
|
||||
github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20231201114238-3b4e78673c4b/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
|
||||
github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
|
||||
github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
|
||||
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user