mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-28 14:47:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

mdbx: use OS pagesize by default (but > 4Kb, and < 64Kb) #530

Browse Source
This commit is contained in:
Alex Sharov 2022-07-19 10:39:48 +07:00 committed by GitHub
parent 8b2c02d1fb
commit b9efe56597
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
kv/helpers.go
View File

@ -3,12 +3,25 @@ package kv
import (
"context"
"fmt"
"os"
"time"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/torquem-ch/mdbx-go/mdbx"
"go.uber.org/atomic"
)
func DefaultPageSize() uint64 {
osPageSize := os.Getpagesize()
if osPageSize < 4096 { // reduce further may lead to errors (because some data is just big)
osPageSize = 4096
} else if osPageSize > mdbx.MaxPageSize {
osPageSize = mdbx.MaxPageSize
}
osPageSize = osPageSize / 4096 * 4096 // ensure it's rounded
return uint64(osPageSize)
}
// BigChunks - read `table` by big chunks - restart read transaction after each 1 minutes
func BigChunks(db RoDB, table string, from []byte, walker func(tx Tx, k, v []byte) (bool, error)) error {
rollbackEvery := time.NewTicker(1 * time.Minute)

44
kv/mdbx/kv_mdbx.go
View File

@ -75,7 +75,7 @@ func NewMDBX(log log.Logger) MdbxOpts {
bucketsCfg: WithChaindataTables,
flags: mdbx.NoReadahead | mdbx.Coalesce | mdbx.Durable,
log: log,
pageSize: 4096,
pageSize: kv.DefaultPageSize(),
}
}
@ -208,11 +208,24 @@ func (opts MdbxOpts) Open() (kv.RwDB, error) {
}
}
defaultDirtyPagesLimit, err := env.GetOption(mdbx.OptTxnDpLimit)
err = env.Open(opts.path, opts.flags, 0664)
if err != nil {
return nil, err
if err != nil {
return nil, fmt.Errorf("%w, label: %s, trace: %s", err, opts.label.String(), stack2.Trace().String())
}
}
// mdbx will not change pageSize if db already exists. means need read real value after env.open()
in, err := env.Info(nil)
if err != nil {
if err != nil {
return nil, fmt.Errorf("%w, label: %s, trace: %s", err, opts.label.String(), stack2.Trace().String())
}
}
opts.pageSize = uint64(in.PageSize)
// erigon using big transactions
// increase "page measured" options. need do it after env.Open() because default are depend on pageSize known only after env.Open()
if opts.flags&mdbx.Accede == 0 && opts.flags&mdbx.Readonly == 0 {
// 1/8 is good for transactions with a lot of modifications - to reduce invalidation size.
// But Erigon app now using Batch and etl.Collectors to avoid writing to DB frequently changing data.
@ -220,10 +233,24 @@ func (opts MdbxOpts) Open() (kv.RwDB, error) {
//if err = env.SetOption(mdbx.OptSpillMinDenominator, 8); err != nil {
// return nil, err
//}
if err = env.SetOption(mdbx.OptTxnDpInitial, 16*1024); err != nil {
txnDpInitial, err := env.GetOption(mdbx.OptTxnDpInitial)
if err != nil {
return nil, err
}
if err = env.SetOption(mdbx.OptDpReverseLimit, 16*1024); err != nil {
if err = env.SetOption(mdbx.OptTxnDpInitial, txnDpInitial*2); err != nil {
return nil, err
}
dpReserveLimit, err := env.GetOption(mdbx.OptDpReverseLimit)
if err != nil {
return nil, err
}
if err = env.SetOption(mdbx.OptDpReverseLimit, dpReserveLimit*2); err != nil {
return nil, err
}
defaultDirtyPagesLimit, err := env.GetOption(mdbx.OptTxnDpLimit)
if err != nil {
return nil, err
}
if err = env.SetOption(mdbx.OptTxnDpLimit, defaultDirtyPagesLimit*2); err != nil { // default is RAM/42
@ -241,13 +268,6 @@ func (opts MdbxOpts) Open() (kv.RwDB, error) {
return nil, err
}
err = env.Open(opts.path, opts.flags, 0664)
if err != nil {
if err != nil {
return nil, fmt.Errorf("%w, label: %s, trace: %s", err, opts.label.String(), stack2.Trace().String())
}
}
if opts.syncPeriod != 0 {
if err = env.SetSyncPeriod(opts.syncPeriod); err != nil {
env.Close()