trackerslist: convert from git submodule to go package (#4761)

.gitmodules

@@ -1,10 +1,6 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
-[submodule "cmd/downloader/trackers/trackerslist"]
-	path = cmd/downloader/trackers/trackerslist
-	url = https://github.com/ngosang/trackerslist.git
-	
 [submodule "libmdbx"]
 	path = libmdbx
 	url = https://github.com/torquem-ch/libmdbx.git

cmd/downloader/trackers/embed.go

@@ -2,29 +2,18 @@ package trackers
 
 import (
 	"bufio"
-	_ "embed"
 	"strings"
+
+	"github.com/ledgerwatch/trackerslist"
 )
 
-//go:embed trackerslist/trackers_best.txt
+var (
-var best string
+	Best  = split(trackerslist.Best)
-var Best = split(best)
+	Https = split(trackerslist.Https)
-
+	Http  = split(trackerslist.Http)
-//go:embed trackerslist/trackers_all_https.txt
+	Udp   = split(trackerslist.Udp)
-var https string
+	Ws    = split(trackerslist.Ws)
-var Https = split(https)
+)
-
-//go:embed trackerslist/trackers_all_http.txt
-var http string
-var Http = split(http)
-
-//go:embed trackerslist/trackers_all_udp.txt
-var udp string
-var Udp = split(udp)
-
-//go:embed trackerslist/trackers_all_ws.txt
-var ws string
-var Ws = split(ws)
 
 func split(txt string) (lines []string) {
 	sc := bufio.NewScanner(strings.NewReader(txt))

cmd/downloader/trackers/trackerslist

@@ -1 +0,0 @@
-Subproject commit 17f277f376286f5a99db386421897d5f82031f57

go.mod

@@ -5,6 +5,9 @@ go 1.18
 require (
 	github.com/ledgerwatch/erigon-lib v0.0.0-20220719140506-af5355ee9286
 	github.com/ledgerwatch/erigon-snapshot v1.0.0
+	github.com/ledgerwatch/log/v3 v3.4.1
+	github.com/ledgerwatch/secp256k1 v1.0.0
+	github.com/ledgerwatch/trackerslist v1.0.0
 )
 
 require (
@@ -41,8 +44,6 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kevinburke/go-bindata v3.21.0+incompatible
-	github.com/ledgerwatch/log/v3 v3.4.1
-	github.com/ledgerwatch/secp256k1 v1.0.0
 	github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pelletier/go-toml/v2 v2.0.2

go.sum

@@ -398,6 +398,8 @@ github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY
 github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
 github.com/ledgerwatch/secp256k1 v1.0.0/go.mod h1:SPmqJFciiF/Q0mPt2jVs2dTr/1TZBTIA+kPMmKgBAak=
+github.com/ledgerwatch/trackerslist v1.0.0 h1:6gnQu93WCTL4jPcdmc8UEmw56Cb8IFQHLGnevfIeLwo=
+github.com/ledgerwatch/trackerslist v1.0.0/go.mod h1:pCC+eEw8izNcnBBiSwvIq8kKsxDLInAafSW275jqFrg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lispad/go-generics-tools v1.1.0 h1:mbSgcxdFVmpoyso1X/MJHXbSbSL3dD+qhRryyxk+/XY=

turbo/snapshotsync/parallelcompress/decompress.go

@@ -1,62 +0,0 @@
-package parallelcompress
-
-import (
-	"bufio"
-	"encoding/binary"
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/ledgerwatch/erigon-lib/compress"
-	"github.com/ledgerwatch/erigon-lib/etl"
-	"github.com/ledgerwatch/log/v3"
-)
-
-func Decompress(logPrefix, segFilePath, datFilePath string) error {
-	d, err := compress.NewDecompressor(segFilePath)
-	if err != nil {
-		return err
-	}
-	defer d.Close()
-	logEvery := time.NewTicker(20 * time.Second)
-	defer logEvery.Stop()
-	var df *os.File
-	if df, err = os.Create(datFilePath); err != nil {
-		return err
-	}
-	dw := bufio.NewWriterSize(df, etl.BufIOSize)
-	var word = make([]byte, 0, 256)
-	numBuf := make([]byte, binary.MaxVarintLen64)
-	var decodeTime time.Duration
-	g := d.MakeGetter()
-	start := time.Now()
-	wc := 0
-	for g.HasNext() {
-		word, _ = g.Next(word[:0])
-		decodeTime += time.Since(start)
-		n := binary.PutUvarint(numBuf, uint64(len(word)))
-		if _, e := dw.Write(numBuf[:n]); e != nil {
-			return e
-		}
-		if len(word) > 0 {
-			if _, e := dw.Write(word); e != nil {
-				return e
-			}
-		}
-		wc++
-		select {
-		default:
-		case <-logEvery.C:
-			log.Info(fmt.Sprintf("[%s] Decompress", logPrefix), "millions", wc/1_000_000)
-		}
-		start = time.Now()
-	}
-	log.Info(fmt.Sprintf("[%s] Average decoding time", logPrefix), "per word", time.Duration(int64(decodeTime)/int64(wc)))
-	if err = dw.Flush(); err != nil {
-		return err
-	}
-	if err = df.Close(); err != nil {
-		return err
-	}
-	return nil
-}