mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 11:41:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

downloader: speedup "--downloader.verify" mode (#7529)

Browse Source 
deduplicate logic
create more producer goroutines (torrent lib does limiting internally
amount of consumers/disk-readers/hashers by 2, and it's enough because
we can verify multiple files in parallel)
move flag from "downloader torrent_hashes --verify" to "downloader
--verify"
This commit is contained in:
Alex Sharov 2023-05-17 12:33:59 +07:00 committed by GitHub
parent ea9712f858
commit a2ad68d634
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/downloader/main.go
View File

@ -71,10 +71,10 @@ func init() {
rootCmd.Flags().StringVar(&staticPeersStr, utils.TorrentStaticPeersFlag.Name, utils.TorrentStaticPeersFlag.Value, utils.TorrentStaticPeersFlag.Usage)
rootCmd.Flags().BoolVar(&disableIPV6, "downloader.disable.ipv6", utils.DisableIPV6.Value, utils.DisableIPV6.Usage)
rootCmd.Flags().BoolVar(&disableIPV4, "downloader.disable.ipv4", utils.DisableIPV4.Value, utils.DisableIPV6.Usage)
rootCmd.PersistentFlags().BoolVar(&forceVerify, "verify", false, "Force verify data files if have .torrent files")
withDataDir(printTorrentHashes)
printTorrentHashes.PersistentFlags().BoolVar(&forceRebuild, "rebuild", false, "Force re-create .torrent files")
printTorrentHashes.PersistentFlags().BoolVar(&forceVerify, "verify", false, "Force verify data files if have .torrent files")
printTorrentHashes.Flags().StringVar(&targetFile, "targetfile", "", "write output to file")
if err := printTorrentHashes.MarkFlagFilename("targetfile"); err != nil {
panic(err)
@ -103,7 +103,7 @@ func main() {
var rootCmd = &cobra.Command{
Use: "",
Short: "snapshot downloader",
Example: "go run ./cmd/snapshots --datadir <your_datadir> --downloader.api.addr 127.0.0.1:9093",
Example: "go run ./cmd/downloader --datadir <your_datadir> --downloader.api.addr 127.0.0.1:9093",
PersistentPostRun: func(cmd *cobra.Command, args []string) {
debug.Exit()
},
@ -162,6 +162,7 @@ func Downloader(ctx context.Context, logger log.Logger) error {
}
defer d.Close()
logger.Info("[torrent] Start", "my peerID", fmt.Sprintf("%x", d.Torrent().PeerID()))
d.MainLoopInBackground(ctx, false)
bittorrentServer, err := downloader.NewGrpcServer(d)
@ -175,6 +176,12 @@ func Downloader(ctx context.Context, logger log.Logger) error {
}
defer grpcServer.GracefulStop()
if forceVerify { // remove and create .torrent files (will re-read all snapshots)
if err = d.VerifyData(ctx); err != nil {
return err
}
}
<-ctx.Done()
return nil
}
@ -192,10 +199,6 @@ var printTorrentHashes = &cobra.Command{
dirs := datadir.New(datadirCli)
ctx := cmd.Context()
if forceVerify { // remove and create .torrent files (will re-read all snapshots)
return downloader.VerifyDtaFiles(ctx, dirs.Snap)
}
if forceRebuild { // remove and create .torrent files (will re-read all snapshots)
//removePieceCompletionStorage(snapDir)
files, err := downloader.AllTorrentPaths(dirs.Snap)

2
cmd/downloader/readme.md
View File

@ -99,7 +99,7 @@ Technical details:
```
# Use it if you see weird behavior, bugs, bans, hardware issues, etc...
downloader torrent_hashes --verify --datadir=<your_datadir>
downloader --verify --datadir=<your_datadir>
```
## Faster rsync

4
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.19
require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335
github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336
github.com/ledgerwatch/log/v3 v3.7.0
github.com/ledgerwatch/secp256k1 v1.0.0
@ -110,7 +110,7 @@ require (
github.com/anacrolix/chansync v0.3.0 // indirect
github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 // indirect
github.com/anacrolix/envpprof v1.2.1 // indirect
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 // indirect
github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 // indirect
github.com/anacrolix/go-libutp v1.2.0 // indirect
github.com/anacrolix/log v0.13.2-0.20221123232138-02e2764801c3 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect

8
go.sum
View File

@ -53,8 +53,8 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g
github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4=
github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE=
github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4=
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 h1:k4/h2B1gGF+PJGyGHxs8nmHHt1pzWXZWBj6jn4OBlRc=
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8=
github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 h1:fyXlBfnlFzZSFckJ8QLb2lfmWfY++4RiUnae7ZMuv0A=
github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8=
github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0=
github.com/anacrolix/go-libutp v1.2.0/go.mod h1:RrJ3KcaDcf9Jqp33YL5V/5CBEc6xMc7aJL8wXfuWL50=
github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU=
@ -440,8 +440,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335 h1:Y5OMv07tL3Ncucs9APplTNj85/wY5hNfZgVhb3rggNM=
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335/go.mod h1:gKgJ2AjOOIG9qO38mbFX4GX8EHHgRDuvOc4MqSj29LY=
github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35 h1:WMpT5ASdfKvRqxEwfkNWfjDEB4J5xF2g+Ix0EFPyqVI=
github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35/go.mod h1:gKgJ2AjOOIG9qO38mbFX4GX8EHHgRDuvOc4MqSj29LY=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=