mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

erigon method to test decompression speed (#5464)

Browse Source
This commit is contained in:
Alex Sharov 2022-09-22 14:11:39 +07:00 committed by GitHub
parent 7fc7b91270
commit ef9e296efd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 65 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.18
require (
github.com/ledgerwatch/erigon-lib v0.0.0-20220920040819-2928d1d9903a
github.com/ledgerwatch/erigon-lib v0.0.0-20220922070856-dac8c207fc16
github.com/ledgerwatch/erigon-snapshot v1.0.1-0.20220913092204-de54ee30c7b9
github.com/ledgerwatch/log/v3 v3.4.2
github.com/ledgerwatch/secp256k1 v1.0.0
@ -103,7 +103,6 @@ require (
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
github.com/docker/docker v20.10.17+incompatible
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/flanglet/kanzi-go v1.9.1-0.20211212184056-72dda96261ee // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect
github.com/go-kit/kit v0.10.0 // indirect

6
go.sum
View File

@ -202,8 +202,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0=
github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
github.com/flanglet/kanzi-go v1.9.1-0.20211212184056-72dda96261ee h1:CaVlPeoz5kJQ+cAOV+ZDdlr3J2FmKyNkGu9LY+x7cDM=
github.com/flanglet/kanzi-go v1.9.1-0.20211212184056-72dda96261ee/go.mod h1:/sUSVgDcbjsisuW42GPDgaMqvJ0McZERNICnD7b1nRA=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
@ -394,8 +392,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20220920040819-2928d1d9903a h1:tQGONWCt52w9kgxLg6asnAFDeFagtp1v+9LSdvly5pI=
github.com/ledgerwatch/erigon-lib v0.0.0-20220920040819-2928d1d9903a/go.mod h1:r+WpLFk+Xv7OXoY6ML5KO9OzMKzu02ArBf9Kg62B/EI=
github.com/ledgerwatch/erigon-lib v0.0.0-20220922070856-dac8c207fc16 h1:uzTPkp/TAJ71/AMlocA0cd/G8N9XaGwHcwnqu81xa+g=
github.com/ledgerwatch/erigon-lib v0.0.0-20220922070856-dac8c207fc16/go.mod h1:OGQ4UVTeSdjTc5B9qxqauXhtdfHpOLJ5dVG4ABz31EI=
github.com/ledgerwatch/erigon-snapshot v1.0.1-0.20220913092204-de54ee30c7b9 h1:iWjzYLtOsp/Wpo9ZWV/eMIlnFzk8bm7POSzrXAILw24=
github.com/ledgerwatch/erigon-snapshot v1.0.1-0.20220913092204-de54ee30c7b9/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.4.2 h1:chvjB7c100rlIFgPv+Col2eerxIrHL88OiZRuPZDkxw=

64
turbo/app/snapshots.go
View File

@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"runtime"
"time"
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/common"
@ -92,6 +93,12 @@ var snapshotCommand = cli.Command{
Before: func(ctx *cli.Context) error { return debug.Setup(ctx) },
Flags: append([]cli.Flag{utils.DataDirFlag}, debug.Flags...),
},
{
Name: "decompress_speed",
Action: doDecompressSpeed,
Before: func(ctx *cli.Context) error { return debug.Setup(ctx) },
Flags: append([]cli.Flag{utils.DataDirFlag}, debug.Flags...),
},
},
}
@ -122,6 +129,54 @@ var (
}
)
func preloadFileAsync(name string) {
go func() {
ff, _ := os.Open(name)
_, _ = io.CopyBuffer(io.Discard, bufio.NewReaderSize(ff, 64*1024*1024), make([]byte, 64*1024*1024))
}()
}
func doDecompressSpeed(cliCtx *cli.Context) error {
args := cliCtx.Args()
if len(args) != 1 {
return fmt.Errorf("expecting .seg file path")
}
f := args[0]
compress.SetDecompressionTableCondensity(9)
preloadFileAsync(f)
decompressor, err := compress.NewDecompressor(f)
if err != nil {
return err
}
defer decompressor.Close()
t := time.Now()
if err := decompressor.WithReadAhead(func() error {
g := decompressor.MakeGetter()
buf := make([]byte, 0, 16*etl.BufIOSize)
for g.HasNext() {
buf, _ = g.Next(buf[:0])
}
return nil
}); err != nil {
return err
}
log.Info("decompress speed", "took", time.Since(t))
t = time.Now()
if err := decompressor.WithReadAhead(func() error {
g := decompressor.MakeGetter()
for g.HasNext() {
_ = g.Skip()
}
return nil
}); err != nil {
return err
}
log.Info("decompress skip speed", "took", time.Since(t))
return nil
}
func doRam(cliCtx *cli.Context) error {
args := cliCtx.Args()
if len(args) != 1 {
@ -171,12 +226,15 @@ func doUncompress(cliCtx *cli.Context) error {
return fmt.Errorf("expecting .seg file path")
}
f := args[0]
preloadFileAsync(f)
decompressor, err := compress.NewDecompressor(f)
if err != nil {
return err
}
defer decompressor.Close()
wr := bufio.NewWriterSize(os.Stdout, 16*etl.BufIOSize)
wr := bufio.NewWriterSize(os.Stdout, 512*1024*1024)
defer wr.Flush()
var numBuf [binary.MaxVarintLen64]byte
if err := decompressor.WithReadAhead(func() error {
@ -220,7 +278,9 @@ func doCompress(cliCtx *cli.Context) error {
if err != nil {
return err
}
r := bufio.NewReaderSize(os.Stdin, 16*etl.BufIOSize)
defer c.Close()
r := bufio.NewReaderSize(os.Stdin, 512*1024*1024)
buf := make([]byte, 0, 32*1024*1024)
var l uint64
for l, err = binary.ReadUvarint(r); err == nil; l, err = binary.ReadUvarint(r) {