mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2025-01-14 14:58:21 +00:00
increase IO buffers size (#686)
* increase IO buffers size * found SSD threshold
This commit is contained in:
parent
daba929bd8
commit
b26b00f72b
common/etl
@ -16,6 +16,7 @@ const (
|
||||
SortableOldestAppearedBuffer
|
||||
|
||||
BufferOptimalSize = 256 * 1024 * 1024 /* 256 mb | var because we want to sometimes change it from tests */
|
||||
BufIOSize = 64 * 4096 // 64 pages | default is 1 page | increasing further doesn't show speedup on SSD
|
||||
)
|
||||
|
||||
type Buffer interface {
|
||||
|
@ -35,12 +35,11 @@ func FlushToDisk(encoder Encoder, currentKey []byte, b Buffer, datadir string) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer bufferFile.Sync() //nolint:errcheck
|
||||
|
||||
w := bufio.NewWriter(bufferFile)
|
||||
w := bufio.NewWriterSize(bufferFile, BufIOSize)
|
||||
defer w.Flush() //nolint:errcheck
|
||||
defer func() {
|
||||
bufferFile.Sync() //nolint:errcheck
|
||||
}()
|
||||
|
||||
encoder.Reset(w)
|
||||
for _, entry := range b.GetEntries() {
|
||||
err = writeToDisk(encoder, entry.key, entry.value)
|
||||
@ -66,7 +65,7 @@ func (p *fileDataProvider) Next(decoder Decoder) ([]byte, []byte, error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
p.reader = bufio.NewReader(p.file)
|
||||
p.reader = bufio.NewReaderSize(p.file, BufIOSize)
|
||||
}
|
||||
decoder.Reset(p.reader)
|
||||
return readElementFromDisk(decoder)
|
||||
|
Loading…
Reference in New Issue
Block a user