/* Copyright 2021 Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package downloader import ( "context" //nolint:gosec "fmt" "net" "os" "path/filepath" "regexp" "runtime" "strconv" "sync" "sync/atomic" "time" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/metainfo" common2 "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/cmp" dir2 "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/semaphore" ) // `github.com/anacrolix/torrent` library spawning several goroutines and producing many requests for each tracker. So we limit amout of trackers by 7 var udpOrHttpTrackers = []string{ "udp://tracker.opentrackr.org:1337/announce", "udp://9.rarbg.com:2810/announce", "udp://tracker.openbittorrent.com:6969/announce", "http://tracker.openbittorrent.com:80/announce", "udp://opentracker.i2p.rocks:6969/announce", "https://opentracker.i2p.rocks:443/announce", "udp://tracker.torrent.eu.org:451/announce", "udp://tracker.moeking.me:6969/announce", } // nolint var websocketTrackers = []string{ "wss://tracker.btorrent.xyz", } // Trackers - break down by priority tier var Trackers = [][]string{ udpOrHttpTrackers, //websocketTrackers // TODO: Ws protocol producing too many errors and flooding logs. But it's also very fast and reactive. } func AllTorrentPaths(dir string) ([]string, error) { files, err := AllTorrentFiles(dir) if err != nil { return nil, err } histDir := filepath.Join(dir, "history") files2, err := AllTorrentFiles(histDir) if err != nil { return nil, err } res := make([]string, 0, len(files)+len(files2)) for _, f := range files { torrentFilePath := filepath.Join(dir, f) res = append(res, torrentFilePath) } for _, f := range files2 { torrentFilePath := filepath.Join(histDir, f) res = append(res, torrentFilePath) } return res, nil } func AllTorrentFiles(dir string) ([]string, error) { files, err := os.ReadDir(dir) if err != nil { return nil, err } res := make([]string, 0, len(files)) for _, f := range files { if filepath.Ext(f.Name()) != ".torrent" { // filter out only compressed files continue } fileInfo, err := f.Info() if err != nil { return nil, err } if fileInfo.Size() == 0 { continue } res = append(res, f.Name()) } return res, nil } func seedableSegmentFiles(dir string) ([]string, error) { files, err := os.ReadDir(dir) if err != nil { return nil, err } res := make([]string, 0, len(files)) for _, f := range files { if f.IsDir() { continue } if !f.Type().IsRegular() { continue } if !snaptype.IsCorrectFileName(f.Name()) { continue } fileInfo, err := f.Info() if err != nil { return nil, err } if fileInfo.Size() == 0 { continue } if filepath.Ext(f.Name()) != ".seg" { // filter out only compressed files continue } ff, ok := snaptype.ParseFileName(dir, f.Name()) if !ok { continue } if !ff.Seedable() { continue } res = append(res, f.Name()) } return res, nil } var historyFileRegex = regexp.MustCompile("^([[:lower:]]+).([0-9]+)-([0-9]+).(v|ef)$") func seedableHistorySnapshots(dir string) ([]string, error) { historyDir := filepath.Join(dir, "history") dir2.MustExist(historyDir) files, err := os.ReadDir(historyDir) if err != nil { return nil, err } res := make([]string, 0, len(files)) for _, f := range files { if f.IsDir() { continue } if !f.Type().IsRegular() { continue } fileInfo, err := f.Info() if err != nil { return nil, err } if fileInfo.Size() == 0 { continue } ext := filepath.Ext(f.Name()) if ext != ".v" && ext != ".ef" && ext != ".kv" { // filter out only compressed files continue } subs := historyFileRegex.FindStringSubmatch(f.Name()) if len(subs) != 5 { continue } from, err := strconv.ParseUint(subs[2], 10, 64) if err != nil { return nil, fmt.Errorf("ParseFileName: %w", err) } to, err := strconv.ParseUint(subs[3], 10, 64) if err != nil { return nil, fmt.Errorf("ParseFileName: %w", err) } if to-from != snaptype.Erigon3SeedableSteps { continue } res = append(res, filepath.Join("history", f.Name())) } return res, nil } func buildTorrentIfNeed(fName, root string) (err error) { fPath := filepath.Join(root, fName) if dir2.FileExist(fPath + ".torrent") { return } if !dir2.FileExist(fPath) { return } info := &metainfo.Info{PieceLength: downloadercfg.DefaultPieceSize, Name: fName} if err := info.BuildFromFilePath(fPath); err != nil { return fmt.Errorf("createTorrentFileFromSegment: %w", err) } info.Name = fName return createTorrentFileFromInfo(root, info, nil) } // AddSegment - add existing .seg file, create corresponding .torrent if need func AddSegment(originalFileName, snapDir string, client *torrent.Client) (bool, error) { fPath := filepath.Join(snapDir, originalFileName) if !dir2.FileExist(fPath + ".torrent") { return false, nil } _, err := AddTorrentFile(fPath+".torrent", client) if err != nil { return false, fmt.Errorf("AddTorrentFile: %w", err) } return true, nil } // BuildTorrentFilesIfNeed - create .torrent files from .seg files (big IO) - if .seg files were added manually func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, error) { logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() files, err := seedableSegmentFiles(snapDir) if err != nil { return nil, err } files2, err := seedableHistorySnapshots(snapDir) if err != nil { return nil, err } files = append(files, files2...) errs := make(chan error, len(files)*2) wg := &sync.WaitGroup{} workers := cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 2 var sem = semaphore.NewWeighted(int64(workers)) i := atomic.Int32{} for _, file := range files { wg.Add(1) if err := sem.Acquire(ctx, 1); err != nil { return nil, err } go func(f string) { defer i.Add(1) defer sem.Release(1) defer wg.Done() if err := buildTorrentIfNeed(f, snapDir); err != nil { errs <- err } select { default: case <-ctx.Done(): errs <- ctx.Err() case <-logEvery.C: log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files))) } }(file) } go func() { wg.Wait() close(errs) }() for err := range errs { if err != nil { return nil, err } } return files, nil } func CreateTorrentFileIfNotExists(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error { fPath := filepath.Join(root, info.Name) if dir2.FileExist(fPath + ".torrent") { return nil } if err := createTorrentFileFromInfo(root, info, mi); err != nil { return err } return nil } func createTorrentFileFromInfo(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error { if mi == nil { infoBytes, err := bencode.Marshal(info) if err != nil { return err } mi = &metainfo.MetaInfo{ CreationDate: time.Now().Unix(), CreatedBy: "erigon", InfoBytes: infoBytes, AnnounceList: Trackers, } } else { mi.AnnounceList = Trackers } torrentFileName := filepath.Join(root, info.Name+".torrent") file, err := os.Create(torrentFileName) if err != nil { return err } defer file.Close() if err := mi.Write(file); err != nil { return err } file.Sync() return nil } // nolint func segmentFileNameFromTorrentFileName(in string) string { ext := filepath.Ext(in) return in[0 : len(in)-len(ext)] } // AddTorrentFile - adding .torrent file to torrentClient (and checking their hashes), if .torrent file // added first time - pieces verification process will start (disk IO heavy) - Progress // kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. // Don't need call torrent.VerifyData manually func AddTorrentFile(torrentFilePath string, torrentClient *torrent.Client) (*torrent.Torrent, error) { mi, err := metainfo.LoadFromFile(torrentFilePath) if err != nil { return nil, err } mi.AnnounceList = Trackers ts, err := torrent.TorrentSpecFromMetaInfoErr(mi) if err != nil { return nil, err } if _, ok := torrentClient.Torrent(ts.InfoHash); !ok { // can set ChunkSize only for new torrents ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize } else { ts.ChunkSize = 0 } ts.DisallowDataDownload = true t, _, err := torrentClient.AddTorrentSpec(ts) if err != nil { return nil, err } t.DisallowDataDownload() t.AllowDataUpload() return t, nil } var ErrSkip = fmt.Errorf("skip") func portMustBeTCPAndUDPOpen(port int) error { tcpAddr := &net.TCPAddr{ Port: port, IP: net.ParseIP("127.0.0.1"), } ln, err := net.ListenTCP("tcp", tcpAddr) if err != nil { return fmt.Errorf("please open port %d for TCP and UDP. %w", port, err) } _ = ln.Close() udpAddr := &net.UDPAddr{ Port: port, IP: net.ParseIP("127.0.0.1"), } ser, err := net.ListenUDP("udp", udpAddr) if err != nil { return fmt.Errorf("please open port %d for UDP. %w", port, err) } _ = ser.Close() return nil } func savePeerID(db kv.RwDB, peerID torrent.PeerID) error { return db.Update(context.Background(), func(tx kv.RwTx) error { return tx.Put(kv.BittorrentInfo, []byte(kv.BittorrentPeerID), peerID[:]) }) } func readPeerID(db kv.RoDB) (peerID []byte, err error) { if err = db.View(context.Background(), func(tx kv.Tx) error { peerIDFromDB, err := tx.GetOne(kv.BittorrentInfo, []byte(kv.BittorrentPeerID)) if err != nil { return fmt.Errorf("get peer id: %w", err) } peerID = common2.Copy(peerIDFromDB) return nil }); err != nil { return nil, err } return peerID, nil }