mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-15 15:28:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Configurable temporary directory for in-memory DB (#676)

Browse Source 
See https://github.com/ledgerwatch/erigon/pull/5702
This commit is contained in:
Andrew Ashikhmin 2022-10-11 16:57:32 +02:00 committed by GitHub
parent d0efd3c1ca
commit a09e268abe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 41 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
kv/mdbx/kv_abstract_test.go
View File

@ -166,7 +166,7 @@ func TestRemoteKvVersion(t *testing.T) {
f := func(defaultBuckets kv.TableCfg) kv.TableCfg {
return defaultBuckets
}
writeDB := mdbx.NewMDBX(logger).InMem().WithTableCfg(f).MustOpen()
writeDB := mdbx.NewMDBX(logger).InMem("").WithTableCfg(f).MustOpen()
defer writeDB.Close()
conn := bufconn.Listen(1024 * 1024)
grpcServer := grpc.NewServer()
@ -210,8 +210,8 @@ func setupDatabases(t *testing.T, logger log.Logger, f mdbx.TableCfgFunc) (write
t.Helper()
ctx := context.Background()
writeDBs = []kv.RwDB{
mdbx.NewMDBX(logger).InMem().WithTableCfg(f).MustOpen(),
mdbx.NewMDBX(logger).InMem().WithTableCfg(f).MustOpen(), // for remote db
mdbx.NewMDBX(logger).InMem("").WithTableCfg(f).MustOpen(),
mdbx.NewMDBX(logger).InMem("").WithTableCfg(f).MustOpen(), // for remote db
}
conn := bufconn.Listen(1024 * 1024)

25
kv/mdbx/kv_mdbx.go
View File

@ -68,14 +68,6 @@ type MdbxOpts struct {
roTxsLimiter *semaphore.Weighted
}
func testKVPath() string {
dir, err := os.MkdirTemp(os.TempDir(), "erigon-test-db")
if err != nil {
panic(err)
}
return dir
}
func NewMDBX(log log.Logger) MdbxOpts {
return MdbxOpts{
bucketsCfg: WithChaindataTables,
@ -125,7 +117,17 @@ func (opts MdbxOpts) Set(opt MdbxOpts) MdbxOpts {
return opt
}
func (opts MdbxOpts) InMem() MdbxOpts {
func (opts MdbxOpts) InMem(tmpDir string) MdbxOpts {
if tmpDir != "" {
if err := os.MkdirAll(tmpDir, 0755); err != nil {
panic(err)
}
}
path, err := os.MkdirTemp(tmpDir, "erigon-memdb-")
if err != nil {
panic(err)
}
opts.path = path
opts.inMem = true
opts.flags = mdbx.UtterlyNoSync | mdbx.NoMetaSync | mdbx.LifoReclaim | mdbx.WriteMap
opts.mapSize = 512 * datasize.MB
@ -178,11 +180,6 @@ func (opts MdbxOpts) WithTableCfg(f TableCfgFunc) MdbxOpts {
}
func (opts MdbxOpts) Open() (kv.RwDB, error) {
var err error
if opts.inMem {
opts.path = testKVPath()
}
env, err := mdbx.NewEnv()
if err != nil {
return nil, err

8
kv/memdb/memory_database.go
View File

@ -26,17 +26,17 @@ import (
)
func New() kv.RwDB {
return mdbx.NewMDBX(log.New()).InMem().MustOpen()
return mdbx.NewMDBX(log.New()).InMem("").MustOpen()
}
func NewPoolDB() kv.RwDB {
return mdbx.NewMDBX(log.New()).InMem().Label(kv.TxPoolDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }).MustOpen()
return mdbx.NewMDBX(log.New()).InMem("").Label(kv.TxPoolDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.TxpoolTablesCfg }).MustOpen()
}
func NewDownloaderDB() kv.RwDB {
return mdbx.NewMDBX(log.New()).InMem().Label(kv.DownloaderDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.DownloaderTablesCfg }).MustOpen()
return mdbx.NewMDBX(log.New()).InMem("").Label(kv.DownloaderDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.DownloaderTablesCfg }).MustOpen()
}
func NewSentryDB() kv.RwDB {
return mdbx.NewMDBX(log.New()).InMem().Label(kv.SentryDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.SentryTablesCfg }).MustOpen()
return mdbx.NewMDBX(log.New()).InMem("").Label(kv.SentryDB).WithTableCfg(func(_ kv.TableCfg) kv.TableCfg { return kv.SentryTablesCfg }).MustOpen()
}
func NewTestDB(tb testing.TB) kv.RwDB {

8
kv/memdb/memory_mutation.go
View File

@ -32,16 +32,16 @@ type MemoryMutation struct {
statelessCursors map[string]kv.RwCursor
}
// NewBatch - starts in-mem batch
// NewMemoryBatch - starts in-mem batch
//
// Common pattern:
//
// batch := db.NewBatch()
// batch := NewMemoryBatch(db, tmpDir)
// defer batch.Rollback()
// ... some calculations on `batch`
// batch.Commit()
func NewMemoryBatch(tx kv.Tx) *MemoryMutation {
tmpDB := mdbx.NewMDBX(log.New()).InMem().MustOpen()
func NewMemoryBatch(tx kv.Tx, tmpDir string) *MemoryMutation {
tmpDB := mdbx.NewMDBX(log.New()).InMem(tmpDir).MustOpen()
memTx, err := tmpDB.BeginRw(context.Background())
if err != nil {
panic(err)

38
kv/memdb/memory_mutation_test.go
View File

@ -36,7 +36,7 @@ func TestPutAppendHas(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
require.NoError(t, batch.Append(kv.HashedAccounts, []byte("AAAA"), []byte("value1.5")))
require.Error(t, batch.Append(kv.HashedAccounts, []byte("AAAA"), []byte("value1.3")))
require.NoError(t, batch.Put(kv.HashedAccounts, []byte("AAAA"), []byte("value1.3")))
@ -65,7 +65,7 @@ func TestLastMiningDB(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("BCAA"), []byte("value5"))
@ -90,7 +90,7 @@ func TestLastMiningMem(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("DCAA"), []byte("value5"))
@ -114,7 +114,7 @@ func TestDeleteMining(t *testing.T) {
require.NoError(t, err)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("DCAA"), []byte("value5"))
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
@ -141,7 +141,7 @@ func TestFlush(t *testing.T) {
require.NoError(t, err)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("AAAA"), []byte("value5"))
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
@ -163,7 +163,7 @@ func TestForEach(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
require.NoError(t, batch.Flush(rwTx))
@ -206,7 +206,7 @@ func TestForPrefix(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
var keys1 []string
var values1 []string
@ -246,7 +246,7 @@ func TestForAmount(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
var keys []string
@ -280,7 +280,7 @@ func TestGetOneAfterClearBucket(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
err = batch.ClearBucket(kv.HashedAccounts)
@ -304,7 +304,7 @@ func TestSeekExactAfterClearBucket(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
err = batch.ClearBucket(kv.HashedAccounts)
@ -341,7 +341,7 @@ func TestFirstAfterClearBucket(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
err = batch.ClearBucket(kv.HashedAccounts)
@ -370,7 +370,7 @@ func TestIncReadSequence(t *testing.T) {
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
_, err = batch.IncrementSequence(kv.HashedAccounts, uint64(12))
@ -394,7 +394,7 @@ func TestNext(t *testing.T) {
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
batch.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.2"))
@ -439,7 +439,7 @@ func TestNextNoDup(t *testing.T) {
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
batch.Put(kv.AccountChangeSet, []byte("key2"), []byte("value2.1"))
@ -467,7 +467,7 @@ func TestDeleteCurrentDuplicates(t *testing.T) {
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
cursor, err := batch.RwCursorDupSort(kv.AccountChangeSet)
@ -503,7 +503,7 @@ func TestSeekBothRange(t *testing.T) {
rwTx.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.1"))
rwTx.Put(kv.AccountChangeSet, []byte("key3"), []byte("value3.3"))
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
cursor, err := batch.RwCursorDupSort(kv.AccountChangeSet)
@ -538,7 +538,7 @@ func TestAutoConversion(t *testing.T) {
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
c, err := batch.RwCursor(kv.PlainState)
@ -595,7 +595,7 @@ func TestAutoConversionDelete(t *testing.T) {
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
c, err := batch.RwCursor(kv.PlainState)
@ -633,7 +633,7 @@ func TestAutoConversionSeekBothRange(t *testing.T) {
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx)
batch := NewMemoryBatch(rwTx, "")
defer batch.Close()
c, err := batch.RwCursorDupSort(kv.PlainState)