From 156da607e7495d709c141aec40f66a2556d35dc0 Mon Sep 17 00:00:00 2001 From: ledgerwatch Date: Wed, 13 Apr 2022 16:29:44 +0100 Subject: [PATCH] More compact representation of huffman trees in the seg files (#3875) * More compact representation of huffman trees in the seg files * Update to latest erigon-lib * recompress * recompress * recompress * uncompress: buf fix * Update to latest erigon-lib * Revert default sync mode Co-authored-by: Alexey Sharp Co-authored-by: Alex Sharp Co-authored-by: alex.sharov --- cmd/downloader/recompress.sh | 2 +- eth/ethconfig/config.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- turbo/app/snapshots.go | 4 ++-- turbo/snapshotsync/snapshothashes/erigon-snapshots | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/downloader/recompress.sh b/cmd/downloader/recompress.sh index c4967cee7..4118e47dc 100755 --- a/cmd/downloader/recompress.sh +++ b/cmd/downloader/recompress.sh @@ -2,7 +2,7 @@ set -e dir=$1 -files=$(ls $dir/snapshots | grep -v tor | grep seg | sort -n -t$'-' -k1) +files=$(ls $dir/snapshots | grep -v old | grep -v tor | grep seg | sort -n -t$'-' -k1) for file in $files do from=$dir/snapshots/$file diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index afb4dcf04..de5d8d993 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -303,7 +303,7 @@ func SyncModeByChainName(chain, syncCliFlag string) SyncMode { } switch chain { case networkname.MainnetChainName, networkname.BSCChainName, networkname.GoerliChainName: - return FastSync + return FastSync // TODO change nack to snap mode when snapshots are re-created default: return FastSync } diff --git a/go.mod b/go.mod index 481c75369..897d5d97d 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/kevinburke/go-bindata v3.21.0+incompatible - github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0 + github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7 github.com/ledgerwatch/log/v3 v3.4.1 github.com/ledgerwatch/secp256k1 v1.0.0 github.com/magiconair/properties v1.8.6 // indirect diff --git a/go.sum b/go.sum index d20365a00..49af6034d 100644 --- a/go.sum +++ b/go.sum @@ -612,8 +612,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0 h1:jRyKux3YgcOAuBDMt78Cu4/TY0MH+7cOz2Gwg+O5Lfk= -github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0/go.mod h1:IQjw8YqJuqQqKqOBJ8av6TxaL+reYU6+VXWVbTrfpCw= +github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7 h1:eERQdNzf4+qOPkWMMlhn2iWHpq94hCXXu+GCJU/Npto= +github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7/go.mod h1:IQjw8YqJuqQqKqOBJ8av6TxaL+reYU6+VXWVbTrfpCw= github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc= github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/turbo/app/snapshots.go b/turbo/app/snapshots.go index 3e10b916a..56774fa10 100644 --- a/turbo/app/snapshots.go +++ b/turbo/app/snapshots.go @@ -197,12 +197,12 @@ func doCompress(cliCtx *cli.Context) error { f := args[0] datadir := cliCtx.String(utils.DataDirFlag.Name) tmpDir := filepath.Join(datadir, etl.TmpDirName) - c, err := compress.NewCompressor(ctx, "", f, tmpDir, compress.MinPatternScore, runtime.NumCPU()-2, log.LvlInfo) + c, err := compress.NewCompressor(ctx, "", f, tmpDir, compress.MinPatternScore, runtime.NumCPU()-1, log.LvlInfo) if err != nil { return err } scanner := bufio.NewScanner(os.Stdin) - buf := make([]byte, 0, 16*1024*1024) + buf := make([]byte, 0, 32*1024*1024) scanner.Buffer(buf, cap(buf)) for scanner.Scan() { if err := c.AddWord(scanner.Bytes()); err != nil { diff --git a/turbo/snapshotsync/snapshothashes/erigon-snapshots b/turbo/snapshotsync/snapshothashes/erigon-snapshots index 31625103b..fe0d6e307 160000 --- a/turbo/snapshotsync/snapshothashes/erigon-snapshots +++ b/turbo/snapshotsync/snapshothashes/erigon-snapshots @@ -1 +1 @@ -Subproject commit 31625103bfd3c5bb24e3aad1851a8260bd5dc841 +Subproject commit fe0d6e307aef4edd30f19223f5dbd37c9516c6c3