mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 11:41:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

downloader: speedup "--downloader.verify" mode (#7529)

Browse Source 
deduplicate logic
create more producer goroutines (torrent lib does limiting internally
amount of consumers/disk-readers/hashers by 2, and it's enough because
we can verify multiple files in parallel)
move flag from "downloader torrent_hashes --verify" to "downloader
--verify"
This commit is contained in:
Alex Sharov 2023-05-17 12:33:59 +07:00 committed by GitHub
parent ea9712f858
commit a2ad68d634
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/downloader/main.go
View File

@ -71,10 +71,10 @@ func init() {
rootCmd.Flags().StringVar(&staticPeersStr, utils.TorrentStaticPeersFlag.Name, utils.TorrentStaticPeersFlag.Value, utils.TorrentStaticPeersFlag.Usage) rootCmd.Flags().StringVar(&staticPeersStr, utils.TorrentStaticPeersFlag.Name, utils.TorrentStaticPeersFlag.Value, utils.TorrentStaticPeersFlag.Usage)
rootCmd.Flags().BoolVar(&disableIPV6, "downloader.disable.ipv6", utils.DisableIPV6.Value, utils.DisableIPV6.Usage) rootCmd.Flags().BoolVar(&disableIPV6, "downloader.disable.ipv6", utils.DisableIPV6.Value, utils.DisableIPV6.Usage)
rootCmd.Flags().BoolVar(&disableIPV4, "downloader.disable.ipv4", utils.DisableIPV4.Value, utils.DisableIPV6.Usage) rootCmd.Flags().BoolVar(&disableIPV4, "downloader.disable.ipv4", utils.DisableIPV4.Value, utils.DisableIPV6.Usage)
rootCmd.PersistentFlags().BoolVar(&forceVerify, "verify", false, "Force verify data files if have .torrent files")
withDataDir(printTorrentHashes) withDataDir(printTorrentHashes)
printTorrentHashes.PersistentFlags().BoolVar(&forceRebuild, "rebuild", false, "Force re-create .torrent files") printTorrentHashes.PersistentFlags().BoolVar(&forceRebuild, "rebuild", false, "Force re-create .torrent files")
printTorrentHashes.PersistentFlags().BoolVar(&forceVerify, "verify", false, "Force verify data files if have .torrent files")
printTorrentHashes.Flags().StringVar(&targetFile, "targetfile", "", "write output to file") printTorrentHashes.Flags().StringVar(&targetFile, "targetfile", "", "write output to file")
if err := printTorrentHashes.MarkFlagFilename("targetfile"); err != nil { if err := printTorrentHashes.MarkFlagFilename("targetfile"); err != nil {
panic(err) panic(err)
@ -103,7 +103,7 @@ func main() {
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "", Use: "",
Short: "snapshot downloader", Short: "snapshot downloader",
Example: "go run ./cmd/snapshots --datadir <your_datadir> --downloader.api.addr 127.0.0.1:9093", Example: "go run ./cmd/downloader --datadir <your_datadir> --downloader.api.addr 127.0.0.1:9093",
PersistentPostRun: func(cmd *cobra.Command, args []string) { PersistentPostRun: func(cmd *cobra.Command, args []string) {
debug.Exit() debug.Exit()
}, },
@ -162,6 +162,7 @@ func Downloader(ctx context.Context, logger log.Logger) error {
} }
defer d.Close() defer d.Close()
logger.Info("[torrent] Start", "my peerID", fmt.Sprintf("%x", d.Torrent().PeerID())) logger.Info("[torrent] Start", "my peerID", fmt.Sprintf("%x", d.Torrent().PeerID()))
d.MainLoopInBackground(ctx, false) d.MainLoopInBackground(ctx, false)
bittorrentServer, err := downloader.NewGrpcServer(d) bittorrentServer, err := downloader.NewGrpcServer(d)
@ -175,6 +176,12 @@ func Downloader(ctx context.Context, logger log.Logger) error {
} }
defer grpcServer.GracefulStop() defer grpcServer.GracefulStop()
if forceVerify { // remove and create .torrent files (will re-read all snapshots)
if err = d.VerifyData(ctx); err != nil {
return err
}
}
<-ctx.Done() <-ctx.Done()
return nil return nil
} }
@ -192,10 +199,6 @@ var printTorrentHashes = &cobra.Command{
dirs := datadir.New(datadirCli) dirs := datadir.New(datadirCli)
ctx := cmd.Context() ctx := cmd.Context()
if forceVerify { // remove and create .torrent files (will re-read all snapshots)
return downloader.VerifyDtaFiles(ctx, dirs.Snap)
}
if forceRebuild { // remove and create .torrent files (will re-read all snapshots) if forceRebuild { // remove and create .torrent files (will re-read all snapshots)
//removePieceCompletionStorage(snapDir) //removePieceCompletionStorage(snapDir)
files, err := downloader.AllTorrentPaths(dirs.Snap) files, err := downloader.AllTorrentPaths(dirs.Snap)

2
cmd/downloader/readme.md
View File

@ -99,7 +99,7 @@ Technical details:
``` ```
# Use it if you see weird behavior, bugs, bans, hardware issues, etc... # Use it if you see weird behavior, bugs, bans, hardware issues, etc...
downloader torrent_hashes --verify --datadir=<your_datadir> downloader --verify --datadir=<your_datadir>
``` ```
## Faster rsync ## Faster rsync

4
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.19 go 1.19
require ( require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335 github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336
github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/log/v3 v3.7.0
github.com/ledgerwatch/secp256k1 v1.0.0 github.com/ledgerwatch/secp256k1 v1.0.0
@ -110,7 +110,7 @@ require (
github.com/anacrolix/chansync v0.3.0 // indirect github.com/anacrolix/chansync v0.3.0 // indirect
github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 // indirect github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 // indirect
github.com/anacrolix/envpprof v1.2.1 // indirect github.com/anacrolix/envpprof v1.2.1 // indirect
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 // indirect github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 // indirect
github.com/anacrolix/go-libutp v1.2.0 // indirect github.com/anacrolix/go-libutp v1.2.0 // indirect
github.com/anacrolix/log v0.13.2-0.20221123232138-02e2764801c3 // indirect github.com/anacrolix/log v0.13.2-0.20221123232138-02e2764801c3 // indirect
github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect

8
go.sum
View File

@ -53,8 +53,8 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g
github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4=
github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE= github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE=
github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4=
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 h1:k4/h2B1gGF+PJGyGHxs8nmHHt1pzWXZWBj6jn4OBlRc= github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68 h1:fyXlBfnlFzZSFckJ8QLb2lfmWfY++4RiUnae7ZMuv0A=
github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/generics v0.0.0-20230428105757-683593396d68/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8=
github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0= github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0=
github.com/anacrolix/go-libutp v1.2.0/go.mod h1:RrJ3KcaDcf9Jqp33YL5V/5CBEc6xMc7aJL8wXfuWL50= github.com/anacrolix/go-libutp v1.2.0/go.mod h1:RrJ3KcaDcf9Jqp33YL5V/5CBEc6xMc7aJL8wXfuWL50=
github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU=
@ -440,8 +440,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335 h1:Y5OMv07tL3Ncucs9APplTNj85/wY5hNfZgVhb3rggNM= github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35 h1:WMpT5ASdfKvRqxEwfkNWfjDEB4J5xF2g+Ix0EFPyqVI=
github.com/ledgerwatch/erigon-lib v0.0.0-20230514014037-6442bdc61335/go.mod h1:gKgJ2AjOOIG9qO38mbFX4GX8EHHgRDuvOc4MqSj29LY= github.com/ledgerwatch/erigon-lib v0.0.0-20230517035724-358b538efe35/go.mod h1:gKgJ2AjOOIG9qO38mbFX4GX8EHHgRDuvOc4MqSj29LY=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=