mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-01 00:31:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

e3: wal ram configure (#732)

Browse Source
This commit is contained in:
Alex Sharov 2022-11-11 11:28:18 +07:00 committed by GitHub
parent 875b19b2b4
commit 9e1963e40d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
state/history.go
View File

@ -505,7 +505,7 @@ func (h *History) newWriter(tmpdir string) *historyWAL {
historyKey: make([]byte, 0, 128), historyKey: make([]byte, 0, 128),
buffered: true, buffered: true,
historyVals: etl.NewCollector(h.historyValsTable, tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize/16)), historyVals: etl.NewCollector(h.historyValsTable, tmpdir, etl.NewSortableBuffer(WALCollectorRam)),
} }
w.historyVals.LogLvl(log.LvlTrace) w.historyVals.LogLvl(log.LvlTrace)

18
state/inverted_index.go
View File

@ -33,6 +33,7 @@ import (
"time" "time"
"github.com/RoaringBitmap/roaring/roaring64" "github.com/RoaringBitmap/roaring/roaring64"
"github.com/c2h5oh/datasize"
"github.com/google/btree" "github.com/google/btree"
"github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/common/dir"
@ -374,14 +375,27 @@ func (ii *invertedIndexWAL) close() {
ii.indexKeys.Close() ii.indexKeys.Close()
} }
var WALCollectorRam = etl.BufferOptimalSize / 16
func init() {
v, _ := os.LookupEnv("ERIGON_WAL_COLLETOR_RAM")
if v != "" {
var err error
WALCollectorRam, err = datasize.ParseString(v)
if err != nil {
panic(err)
}
}
}
func (ii *InvertedIndex) newWriter(tmpdir string) *invertedIndexWAL { func (ii *InvertedIndex) newWriter(tmpdir string) *invertedIndexWAL {
w := &invertedIndexWAL{ii: ii, w := &invertedIndexWAL{ii: ii,
buffered: true, buffered: true,
tmpdir: tmpdir, tmpdir: tmpdir,
// 3 history + 4 indices = 10 etl collectors, 10*256Mb/16 = 256mb - for all indices buffers // 3 history + 4 indices = 10 etl collectors, 10*256Mb/16 = 256mb - for all indices buffers
// etl collector doesn't fsync: means if have enough ram, all files produced by all collectors will be in ram // etl collector doesn't fsync: means if have enough ram, all files produced by all collectors will be in ram
index: etl.NewCollector(ii.indexTable, tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize/16)), index: etl.NewCollector(ii.indexTable, tmpdir, etl.NewSortableBuffer(WALCollectorRam)),
indexKeys: etl.NewCollector(ii.indexKeysTable, tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize/16)), indexKeys: etl.NewCollector(ii.indexKeysTable, tmpdir, etl.NewSortableBuffer(WALCollectorRam)),
} }
w.index.LogLvl(log.LvlTrace) w.index.LogLvl(log.LvlTrace)
w.indexKeys.LogLvl(log.LvlTrace) w.indexKeys.LogLvl(log.LvlTrace)