mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-03 09:37:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

mdbx: hard dirtyPages limit (#9178)

Browse Source 
logic: 1/42 of ram, but not more than 2Gb for chandata and not more than
256mb for other databases.

---------

Co-authored-by: battlmonstr <battlmonstr@users.noreply.github.com>
This commit is contained in:
Alex Sharov 2024-01-10 15:15:27 +07:00 committed by GitHub
parent 0cfe6617db
commit ceeb090f48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 34 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
erigon-lib/kv/mdbx/kv_mdbx.go
View File

@ -314,7 +314,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
// erigon using big transactions // erigon using big transactions
// increase "page measured" options. need do it after env.Open() because default are depend on pageSize known only after env.Open() // increase "page measured" options. need do it after env.Open() because default are depend on pageSize known only after env.Open()
if !opts.HasFlag(mdbx.Accede) && !opts.HasFlag(mdbx.Readonly) { if !opts.HasFlag(mdbx.Readonly) {
// 1/8 is good for transactions with a lot of modifications - to reduce invalidation size. // 1/8 is good for transactions with a lot of modifications - to reduce invalidation size.
// But Erigon app now using Batch and etl.Collectors to avoid writing to DB frequently changing data. // But Erigon app now using Batch and etl.Collectors to avoid writing to DB frequently changing data.
// It means most of our writes are: APPEND or "single UPSERT per key during transaction" // It means most of our writes are: APPEND or "single UPSERT per key during transaction"
@ -326,6 +326,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if opts.label == kv.ChainDB {
if err = env.SetOption(mdbx.OptTxnDpInitial, txnDpInitial*2); err != nil { if err = env.SetOption(mdbx.OptTxnDpInitial, txnDpInitial*2); err != nil {
return nil, err return nil, err
} }
@ -336,10 +337,33 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
if err = env.SetOption(mdbx.OptDpReverseLimit, dpReserveLimit*2); err != nil { if err = env.SetOption(mdbx.OptDpReverseLimit, dpReserveLimit*2); err != nil {
return nil, err return nil, err
} }
}
if err = env.SetOption(mdbx.OptTxnDpLimit, opts.dirtySpace/opts.pageSize); err != nil { var dirtySpace uint64
if opts.dirtySpace > 0 {
dirtySpace = opts.dirtySpace
} else {
// the default value is based on the RAM amount
dirtyPagesLimit, err := env.GetOption(mdbx.OptTxnDpLimit)
if err != nil {
return nil, err return nil, err
} }
dirtySpace = dirtyPagesLimit * opts.pageSize
// clamp to max size
const dirtySpaceMaxChainDB = uint64(2 * datasize.GB)
const dirtySpaceMaxDefault = uint64(256 * datasize.MB)
if opts.label == kv.ChainDB && dirtySpace > dirtySpaceMaxChainDB {
dirtySpace = dirtySpaceMaxChainDB
} else if opts.label != kv.ChainDB && dirtySpace > dirtySpaceMaxDefault {
dirtySpace = dirtySpaceMaxDefault
}
}
if err = env.SetOption(mdbx.OptTxnDpLimit, dirtySpace/opts.pageSize); err != nil {
return nil, err
}
// must be in the range from 12.5% (almost empty) to 50% (half empty) // must be in the range from 12.5% (almost empty) to 50% (half empty)
// which corresponds to the range from 8192 and to 32768 in units respectively // which corresponds to the range from 8192 and to 32768 in units respectively
if err = env.SetOption(mdbx.OptMergeThreshold16dot16Percent, opts.mergeThreshold); err != nil { if err = env.SetOption(mdbx.OptMergeThreshold16dot16Percent, opts.mergeThreshold); err != nil {