erigon-pulse/cmd/state/generate/seeder.go
b00ris 4ebade3583
Add sync by bittorrent snapshots (#1160)
* save state

* torrent experiments

* torrent experiment passed

* fixes after merge

* snapshot headers processing passed

* save state

* save state

* download headers works after snapshot processing

* save state

* save state

* save state

* save state

* add lazy load tx to snapshots, increase number of trackers

* save state

* speedup getting info

* change logging

* move to turbo package

* save state

* save state

* save state

* cleanup

* save state

* add test test

* save state

* lmdb debugging

* fix readonly mode

* save state

* fix build

* sync works

* save state

* save state

* save state

* allow cmd stages stageSenders use snapshots

* debugging failed hashing

* remove experimental tests

* remove torrent experimental tests

* fix lint

* extract snapshot wrapper

* metainfo checker

* add remote seeder

* add logs

* update gomod

* remove useless code

* fix lint&remove useless code

* extract verify snapshot to separated command

* skip debug test

* fix test

* change type of seedSnapshot flag

* add eth logger to torrent lib

* skip debug test

* add Close method

* review fixes

* fix lint

Co-authored-by: alex.sharov <AskAlexSharov@gmail.com>
Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-10-06 20:24:48 +01:00

127 lines
2.5 KiB
Go

package generate
import (
"context"
"errors"
"fmt"
lg "github.com/anacrolix/log"
"github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/torrent/metainfo"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/log"
trnt "github.com/ledgerwatch/turbo-geth/turbo/torrent"
"os"
"os/signal"
"time"
)
func Seed(pathes []string) error {
if len(pathes) != 1 {
return errors.New("you must provide snapshots dir")
}
ctx, cancel := context.WithCancel(context.Background())
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
<-c
cancel()
}()
cfg := torrent.NewDefaultClientConfig()
cfg.NoDHT = false
cfg.DisableTrackers = false
cfg.Seed = true
cfg.Debug = false
cfg.Logger = cfg.Logger.FilterLevel(lg.Info)
cfg.DataDir = pathes[0]
cfg.DataDir = "/media/b00ris/nvme/snapshots"
pathes = []string{
cfg.DataDir + "/headers",
cfg.DataDir + "/bodies",
//cfg.DataDir+"/state/",
//cfg.DataDir+"/receipts/",
}
//cfg.Logger=cfg.Logger.FilterLevel(trlog.Info)
cl, err := torrent.NewClient(cfg)
if err != nil {
return err
}
defer cl.Close()
torrents := make([]*torrent.Torrent, len(pathes))
for i, v := range pathes {
i := i
mi := &metainfo.MetaInfo{
CreationDate: time.Now().Unix(),
CreatedBy: "turbogeth",
AnnounceList: trnt.Trackers,
}
if _, err := os.Stat(v); os.IsNotExist(err) {
fmt.Println(err)
continue
} else if err != nil {
return err
}
if common.IsCanceled(ctx) {
return common.ErrStopped
}
info, err := trnt.BuildInfoBytesForLMDBSnapshot(v)
if err != nil {
return err
}
mi.InfoBytes, err = bencode.Marshal(info)
if err != nil {
return err
}
tt := time.Now()
torrents[i], _, err = cl.AddTorrentSpec(&torrent.TorrentSpec{
Trackers: trnt.Trackers,
InfoHash: mi.HashInfoBytes(),
InfoBytes: mi.InfoBytes,
ChunkSize: trnt.DefaultChunkSize,
})
if err != nil {
return err
}
log.Info("Torrent added", "name", torrents[i].Info().Name, "path", v, "t", time.Since(tt))
if !torrents[i].Seeding() {
log.Warn(torrents[i].Name() + " not seeding")
}
if common.IsCanceled(ctx) {
return common.ErrStopped
}
torrents[i].VerifyData()
}
go func() {
ticker := time.NewTicker(10 * time.Second)
for range ticker.C {
for _, t := range cl.Torrents() {
log.Info("Snapshot stats", "snapshot", t.Name(), "active peers", t.Stats().ActivePeers, "seeding", t.Seeding())
}
if common.IsCanceled(ctx) {
ticker.Stop()
return
}
}
}()
<-ctx.Done()
return nil
}