mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2025-01-03 17:44:29 +00:00
[WIP] Switch to boltDB with Yield, and yield boltdb transaction regularly to not block memory map resizing (#239)
* Switch to boltDB with Yield Yield boltdb transaction regularly to not block memory map resizing * Add test for Yield * Refer to tagged version of bolt
This commit is contained in:
ledgerwatch
GitHub
parent
9d5949a0b9
commit
3a2111ead2
@ -211,6 +211,10 @@ func Server(ctx context.Context, db *bolt.DB, in io.Reader, out io.Writer, close
|
||||
cursorsByBucket := make(map[uint64][]uint64)
|
||||
var c Command
|
||||
for {
|
||||
// Make sure we are not blocking the resizing of the memory map
|
||||
for _, tx := range transactions {
|
||||
tx.Yield()
|
||||
}
|
||||
if err := decoder.Decode(&c); err != nil {
|
||||
if err == io.EOF {
|
||||
// Graceful termination when the end of the input is reached
|
||||
|
||||
@ -19,6 +19,7 @@ package remote
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"github.com/ledgerwatch/bolt"
|
||||
@ -536,3 +537,52 @@ func TestCmdFirst(t *testing.T) {
|
||||
assert.Nil(t, decoder.Decode(&value), "Could not decode response from CmdCursorNext")
|
||||
assert.Nil(t, value, "Unexpected value")
|
||||
}
|
||||
|
||||
func TestTxYield(t *testing.T) {
|
||||
db, err := bolt.Open("in-memory", 0600, &bolt.Options{MemOnly: true})
|
||||
if err != nil {
|
||||
t.Errorf("Could not create database: %v", err)
|
||||
}
|
||||
// Create bucket
|
||||
if err = db.Update(func(tx *bolt.Tx) error {
|
||||
_, err1 := tx.CreateBucket([]byte("bucket"), false)
|
||||
return err1
|
||||
}); err != nil {
|
||||
t.Errorf("Could not create bucket: %v", err)
|
||||
}
|
||||
var readFinished bool
|
||||
go func() {
|
||||
// Long read-only transaction
|
||||
if err1 := db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("bucket"))
|
||||
var keyBuf [8]byte
|
||||
for i := 0; i < 100000; i++ {
|
||||
binary.BigEndian.PutUint64(keyBuf[:], uint64(i))
|
||||
b.Get(keyBuf[:])
|
||||
tx.Yield()
|
||||
}
|
||||
return nil
|
||||
}); err1 != nil {
|
||||
t.Fatal(err1)
|
||||
}
|
||||
readFinished = true
|
||||
}()
|
||||
// Expand the database
|
||||
if err = db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("bucket"))
|
||||
var keyBuf, valBuf [8]byte
|
||||
for i := 0; i < 10000; i++ {
|
||||
binary.BigEndian.PutUint64(keyBuf[:], uint64(i))
|
||||
binary.BigEndian.PutUint64(valBuf[:], uint64(i))
|
||||
if err2 := b.Put(keyBuf[:], valBuf[:]); err2 != nil {
|
||||
return err2
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Errorf("Could not execute update: %v", err)
|
||||
}
|
||||
if readFinished {
|
||||
t.Errorf("Read should not finished here, if it did, it means the writes were blocked by it")
|
||||
}
|
||||
}
|
||||
|
||||
3
go.mod
3
go.mod
@ -10,7 +10,6 @@ require (
|
||||
github.com/VictoriaMetrics/fastcache v1.5.4
|
||||
github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847
|
||||
github.com/blend/go-sdk v2.0.0+incompatible // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6
|
||||
github.com/cespare/cp v0.1.0
|
||||
github.com/cloudflare/cloudflare-go v0.10.6
|
||||
@ -37,7 +36,7 @@ require (
|
||||
github.com/julienschmidt/httprouter v1.2.0
|
||||
github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/ledgerwatch/bolt v1.4.1
|
||||
github.com/ledgerwatch/bolt v1.4.2
|
||||
github.com/llgcode/draw2d v0.0.0-20180825133448-f52c8a71aff0
|
||||
github.com/mattn/go-colorable v0.1.2
|
||||
github.com/mattn/go-isatty v0.0.9
|
||||
|
||||
6
go.sum
6
go.sum
@ -41,8 +41,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/blend/go-sdk v2.0.0+incompatible h1:FL9X/of4ZYO5D2JJNI4vHrbXPfuSDbUa7h8JP9+E92w=
|
||||
github.com/blend/go-sdk v2.0.0+incompatible/go.mod h1:3GUb0YsHFNTJ6hsJTpzdmCUl05o8HisKjx5OAlzYKdw=
|
||||
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
|
||||
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
|
||||
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI=
|
||||
github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
|
||||
@ -141,8 +139,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/ledgerwatch/bolt v1.4.1 h1:yQz1erm+SHuNpeG20I4wyDe1DBoH4oHlacIPNRaqMRo=
|
||||
github.com/ledgerwatch/bolt v1.4.1/go.mod h1:N5ODe9aUsis79NxhdSRR39nlRQJORCyB/IDsfwuG+tE=
|
||||
github.com/ledgerwatch/bolt v1.4.2 h1:UdouTBMi89CQC9dH+1w2AmsCu9uNtQoT2Dvf4vWdVB8=
|
||||
github.com/ledgerwatch/bolt v1.4.2/go.mod h1:N5ODe9aUsis79NxhdSRR39nlRQJORCyB/IDsfwuG+tE=
|
||||
github.com/llgcode/draw2d v0.0.0-20180825133448-f52c8a71aff0 h1:2vp6ESimuT8pCuZHThVyV0hlfa9oPL06HnGCL9pbUgc=
|
||||
github.com/llgcode/draw2d v0.0.0-20180825133448-f52c8a71aff0/go.mod h1:mVa0dA29Db2S4LVqDYLlsePDzRJLDfdhVZiI15uY0FA=
|
||||
github.com/llgcode/ps v0.0.0-20150911083025-f1443b32eedb h1:61ndUreYSlWFeCY44JxDDkngVoI7/1MVhEl98Nm0KOk=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user