mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

More compact representation of huffman trees in the seg files (#3875)

Browse Source 
* More compact representation of huffman trees in the seg files

* Update to latest erigon-lib

* recompress

* recompress

* recompress

* uncompress: buf fix

* Update to latest erigon-lib

* Revert default sync mode

Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
Co-authored-by: Alex Sharp <alexsharp@Alexs-MacBook-Pro.local>
Co-authored-by: alex.sharov <AskAlexSharov@gmail.com>
This commit is contained in:
ledgerwatch 2022-04-13 16:29:44 +01:00 committed by GitHub
parent 8db37cfd97
commit 156da607e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/downloader/recompress.sh
View File

@ -2,7 +2,7 @@
set -e
dir=$1
files=$(ls $dir/snapshots | grep -v tor | grep seg | sort -n -t$'-' -k1)
files=$(ls $dir/snapshots | grep -v old | grep -v tor | grep seg | sort -n -t$'-' -k1)
for file in $files
do
from=$dir/snapshots/$file

2
eth/ethconfig/config.go
View File

@ -303,7 +303,7 @@ func SyncModeByChainName(chain, syncCliFlag string) SyncMode {
}
switch chain {
case networkname.MainnetChainName, networkname.BSCChainName, networkname.GoerliChainName:
return FastSync
return FastSync // TODO change nack to snap mode when snapshots are re-created
default:
return FastSync
}

2
go.mod
View File

@ -41,7 +41,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0
github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7
github.com/ledgerwatch/log/v3 v3.4.1
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/magiconair/properties v1.8.6 // indirect

4
go.sum
View File

@ -612,8 +612,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0 h1:jRyKux3YgcOAuBDMt78Cu4/TY0MH+7cOz2Gwg+O5Lfk=
github.com/ledgerwatch/erigon-lib v0.0.0-20220412020057-c5504fb943d0/go.mod h1:IQjw8YqJuqQqKqOBJ8av6TxaL+reYU6+VXWVbTrfpCw=
github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7 h1:eERQdNzf4+qOPkWMMlhn2iWHpq94hCXXu+GCJU/Npto=
github.com/ledgerwatch/erigon-lib v0.0.0-20220413115515-f18e05186dd7/go.mod h1:IQjw8YqJuqQqKqOBJ8av6TxaL+reYU6+VXWVbTrfpCw=
github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

4
turbo/app/snapshots.go
View File

@ -197,12 +197,12 @@ func doCompress(cliCtx *cli.Context) error {
f := args[0]
datadir := cliCtx.String(utils.DataDirFlag.Name)
tmpDir := filepath.Join(datadir, etl.TmpDirName)
c, err := compress.NewCompressor(ctx, "", f, tmpDir, compress.MinPatternScore, runtime.NumCPU()-2, log.LvlInfo)
c, err := compress.NewCompressor(ctx, "", f, tmpDir, compress.MinPatternScore, runtime.NumCPU()-1, log.LvlInfo)
if err != nil {
return err
}
scanner := bufio.NewScanner(os.Stdin)
buf := make([]byte, 0, 16*1024*1024)
buf := make([]byte, 0, 32*1024*1024)
scanner.Buffer(buf, cap(buf))
for scanner.Scan() {
if err := c.AddWord(scanner.Bytes()); err != nil {

2
turbo/snapshotsync/snapshothashes/erigon-snapshots

@ -1 +1 @@
Subproject commit 31625103bfd3c5bb24e3aad1851a8260bd5dc841
Subproject commit fe0d6e307aef4edd30f19223f5dbd37c9516c6c3