erigon-pulse/erigon-lib/kv/membatchwithdb/memory_mutation_test.go
Mark Holt 19bc328a07
Added db loggers to all db callers and fixed flag settings (#9099)
Mdbx now takes a logger - but this has not been pushed to all callers -
meaning it had an invalid logger

This fixes the log propagation.

It also fixed a start-up issue for http.enabled and txpool.disable
created by a previous merge
2023-12-31 17:10:08 +07:00

669 lines
19 KiB
Go

/*
Copyright 2022 Erigon contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package membatchwithdb
import (
"testing"
"github.com/ledgerwatch/erigon-lib/kv/memdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/log/v3"
)
func initializeDbNonDupSort(rwTx kv.RwTx) {
rwTx.Put(kv.HashedAccounts, []byte("AAAA"), []byte("value"))
rwTx.Put(kv.HashedAccounts, []byte("CAAA"), []byte("value1"))
rwTx.Put(kv.HashedAccounts, []byte("CBAA"), []byte("value2"))
rwTx.Put(kv.HashedAccounts, []byte("CCAA"), []byte("value3"))
}
func TestPutAppendHas(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
require.NoError(t, batch.Append(kv.HashedAccounts, []byte("AAAA"), []byte("value1.5")))
require.Error(t, batch.Append(kv.HashedAccounts, []byte("AAAA"), []byte("value1.3")))
require.NoError(t, batch.Put(kv.HashedAccounts, []byte("AAAA"), []byte("value1.3")))
require.NoError(t, batch.Append(kv.HashedAccounts, []byte("CBAA"), []byte("value3.5")))
require.Error(t, batch.Append(kv.HashedAccounts, []byte("CBAA"), []byte("value3.1")))
require.NoError(t, batch.AppendDup(kv.HashedAccounts, []byte("CBAA"), []byte("value3.1")))
require.Error(t, batch.Append(kv.HashedAccounts, []byte("AAAA"), []byte("value1.3")))
require.Nil(t, batch.Flush(rwTx))
exist, err := batch.Has(kv.HashedAccounts, []byte("AAAA"))
require.Nil(t, err)
require.Equal(t, exist, true)
val, err := batch.GetOne(kv.HashedAccounts, []byte("AAAA"))
require.Nil(t, err)
require.Equal(t, val, []byte("value1.3"))
exist, err = batch.Has(kv.HashedAccounts, []byte("KKKK"))
require.Nil(t, err)
require.Equal(t, exist, false)
}
func TestLastMiningDB(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("BCAA"), []byte("value5"))
cursor, err := batch.Cursor(kv.HashedAccounts)
require.NoError(t, err)
key, value, err := cursor.Last()
require.NoError(t, err)
require.Equal(t, key, []byte("CCAA"))
require.Equal(t, value, []byte("value3"))
key, value, err = cursor.Next()
require.NoError(t, err)
require.Equal(t, key, []byte(nil))
require.Equal(t, value, []byte(nil))
}
func TestLastMiningMem(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("DCAA"), []byte("value5"))
cursor, err := batch.Cursor(kv.HashedAccounts)
require.NoError(t, err)
key, value, err := cursor.Last()
require.NoError(t, err)
require.Equal(t, key, []byte("DCAA"))
require.Equal(t, value, []byte("value5"))
key, value, err = cursor.Next()
require.NoError(t, err)
require.Equal(t, key, []byte(nil))
require.Equal(t, value, []byte(nil))
}
func TestDeleteMining(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("DCAA"), []byte("value5"))
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
batch.Delete(kv.HashedAccounts, []byte("BAAA"))
batch.Delete(kv.HashedAccounts, []byte("CBAA"))
cursor, err := batch.Cursor(kv.HashedAccounts)
require.NoError(t, err)
key, value, err := cursor.SeekExact([]byte("BAAA"))
require.NoError(t, err)
require.Equal(t, key, []byte(nil))
require.Equal(t, value, []byte(nil))
key, value, err = cursor.SeekExact([]byte("CBAA"))
require.NoError(t, err)
require.Equal(t, key, []byte(nil))
require.Equal(t, value, []byte(nil))
}
func TestFlush(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
batch.Put(kv.HashedAccounts, []byte("BAAA"), []byte("value4"))
batch.Put(kv.HashedAccounts, []byte("AAAA"), []byte("value5"))
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
require.NoError(t, batch.Flush(rwTx))
value, err := rwTx.GetOne(kv.HashedAccounts, []byte("BAAA"))
require.NoError(t, err)
require.Equal(t, value, []byte("value4"))
value, err = rwTx.GetOne(kv.HashedAccounts, []byte("AAAA"))
require.NoError(t, err)
require.Equal(t, value, []byte("value5"))
}
func TestForEach(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
batch.Put(kv.HashedAccounts, []byte("FCAA"), []byte("value5"))
require.NoError(t, batch.Flush(rwTx))
var keys []string
var values []string
err := batch.ForEach(kv.HashedAccounts, []byte("XYAZ"), func(k, v []byte) error {
keys = append(keys, string(k))
values = append(values, string(v))
return nil
})
require.Nil(t, err)
require.Nil(t, keys)
require.Nil(t, values)
err = batch.ForEach(kv.HashedAccounts, []byte("CC"), func(k, v []byte) error {
keys = append(keys, string(k))
values = append(values, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"CCAA", "FCAA"}, keys)
require.Equal(t, []string{"value3", "value5"}, values)
var keys1 []string
var values1 []string
err = batch.ForEach(kv.HashedAccounts, []byte("A"), func(k, v []byte) error {
keys1 = append(keys1, string(k))
values1 = append(values1, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"AAAA", "CAAA", "CBAA", "CCAA", "FCAA"}, keys1)
require.Equal(t, []string{"value", "value1", "value2", "value3", "value5"}, values1)
}
func TestForPrefix(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
var keys1 []string
var values1 []string
err := batch.ForPrefix(kv.HashedAccounts, []byte("AB"), func(k, v []byte) error {
keys1 = append(keys1, string(k))
values1 = append(values1, string(v))
return nil
})
require.Nil(t, err)
require.Nil(t, keys1)
require.Nil(t, values1)
err = batch.ForPrefix(kv.HashedAccounts, []byte("AAAA"), func(k, v []byte) error {
keys1 = append(keys1, string(k))
values1 = append(values1, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"AAAA"}, keys1)
require.Equal(t, []string{"value"}, values1)
var keys []string
var values []string
err = batch.ForPrefix(kv.HashedAccounts, []byte("C"), func(k, v []byte) error {
keys = append(keys, string(k))
values = append(values, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"CAAA", "CBAA", "CCAA"}, keys)
require.Equal(t, []string{"value1", "value2", "value3"}, values)
}
func TestForAmount(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
var keys []string
var values []string
err := batch.ForAmount(kv.HashedAccounts, []byte("C"), uint32(3), func(k, v []byte) error {
keys = append(keys, string(k))
values = append(values, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"CAAA", "CBAA", "CCAA"}, keys)
require.Equal(t, []string{"value1", "value2", "value3"}, values)
var keys1 []string
var values1 []string
err = batch.ForAmount(kv.HashedAccounts, []byte("C"), uint32(10), func(k, v []byte) error {
keys1 = append(keys1, string(k))
values1 = append(values1, string(v))
return nil
})
require.Nil(t, err)
require.Equal(t, []string{"CAAA", "CBAA", "CCAA"}, keys1)
require.Equal(t, []string{"value1", "value2", "value3"}, values1)
}
func TestGetOneAfterClearBucket(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
err := batch.ClearBucket(kv.HashedAccounts)
require.Nil(t, err)
cond := batch.isTableCleared(kv.HashedAccounts)
require.True(t, cond)
val, err := batch.GetOne(kv.HashedAccounts, []byte("A"))
require.Nil(t, err)
require.Nil(t, val)
val, err = batch.GetOne(kv.HashedAccounts, []byte("AAAA"))
require.Nil(t, err)
require.Nil(t, val)
}
func TestSeekExactAfterClearBucket(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
err := batch.ClearBucket(kv.HashedAccounts)
require.Nil(t, err)
cond := batch.isTableCleared(kv.HashedAccounts)
require.True(t, cond)
cursor, err := batch.RwCursor(kv.HashedAccounts)
require.NoError(t, err)
key, val, err := cursor.SeekExact([]byte("AAAA"))
require.Nil(t, err)
assert.Nil(t, key)
assert.Nil(t, val)
err = cursor.Put([]byte("AAAA"), []byte("valueX"))
require.Nil(t, err)
key, val, err = cursor.SeekExact([]byte("AAAA"))
require.Nil(t, err)
assert.Equal(t, []byte("AAAA"), key)
assert.Equal(t, []byte("valueX"), val)
key, val, err = cursor.SeekExact([]byte("BBBB"))
require.Nil(t, err)
assert.Nil(t, key)
assert.Nil(t, val)
}
func TestFirstAfterClearBucket(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
err := batch.ClearBucket(kv.HashedAccounts)
require.Nil(t, err)
err = batch.Put(kv.HashedAccounts, []byte("BBBB"), []byte("value5"))
require.Nil(t, err)
cursor, err := batch.Cursor(kv.HashedAccounts)
require.NoError(t, err)
key, val, err := cursor.First()
require.Nil(t, err)
assert.Equal(t, []byte("BBBB"), key)
assert.Equal(t, []byte("value5"), val)
key, val, err = cursor.Next()
require.Nil(t, err)
assert.Nil(t, key)
assert.Nil(t, val)
}
func TestIncReadSequence(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbNonDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
_, err := batch.IncrementSequence(kv.HashedAccounts, uint64(12))
require.Nil(t, err)
val, err := batch.ReadSequence(kv.HashedAccounts)
require.Nil(t, err)
require.Equal(t, val, uint64(12))
}
func initializeDbDupSort(rwTx kv.RwTx) {
rwTx.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.1"))
rwTx.Put(kv.AccountChangeSet, []byte("key3"), []byte("value3.1"))
rwTx.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.3"))
rwTx.Put(kv.AccountChangeSet, []byte("key3"), []byte("value3.3"))
}
func TestNext(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
batch.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.2"))
cursor, err := batch.CursorDupSort(kv.AccountChangeSet)
require.NoError(t, err)
k, v, err := cursor.First()
require.Nil(t, err)
assert.Equal(t, []byte("key1"), k)
assert.Equal(t, []byte("value1.1"), v)
k, v, err = cursor.Next()
require.Nil(t, err)
assert.Equal(t, []byte("key1"), k)
assert.Equal(t, []byte("value1.2"), v)
k, v, err = cursor.Next()
require.Nil(t, err)
assert.Equal(t, []byte("key1"), k)
assert.Equal(t, []byte("value1.3"), v)
k, v, err = cursor.Next()
require.Nil(t, err)
assert.Equal(t, []byte("key3"), k)
assert.Equal(t, []byte("value3.1"), v)
k, v, err = cursor.Next()
require.Nil(t, err)
assert.Equal(t, []byte("key3"), k)
assert.Equal(t, []byte("value3.3"), v)
k, v, err = cursor.Next()
require.Nil(t, err)
assert.Nil(t, k)
assert.Nil(t, v)
}
func TestNextNoDup(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
batch.Put(kv.AccountChangeSet, []byte("key2"), []byte("value2.1"))
batch.Put(kv.AccountChangeSet, []byte("key2"), []byte("value2.2"))
cursor, err := batch.CursorDupSort(kv.AccountChangeSet)
require.NoError(t, err)
k, _, err := cursor.First()
require.Nil(t, err)
assert.Equal(t, []byte("key1"), k)
k, _, err = cursor.NextNoDup()
require.Nil(t, err)
assert.Equal(t, []byte("key2"), k)
k, _, err = cursor.NextNoDup()
require.Nil(t, err)
assert.Equal(t, []byte("key3"), k)
}
func TestDeleteCurrentDuplicates(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbDupSort(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
cursor, err := batch.RwCursorDupSort(kv.AccountChangeSet)
require.NoError(t, err)
require.NoError(t, cursor.Put([]byte("key3"), []byte("value3.2")))
key, _, err := cursor.SeekExact([]byte("key3"))
require.NoError(t, err)
require.Equal(t, []byte("key3"), key)
require.NoError(t, cursor.DeleteCurrentDuplicates())
require.NoError(t, batch.Flush(rwTx))
var keys []string
var values []string
err = rwTx.ForEach(kv.AccountChangeSet, nil, func(k, v []byte) error {
keys = append(keys, string(k))
values = append(values, string(v))
return nil
})
require.NoError(t, err)
require.Equal(t, []string{"key1", "key1"}, keys)
require.Equal(t, []string{"value1.1", "value1.3"}, values)
}
func TestSeekBothRange(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
rwTx.Put(kv.AccountChangeSet, []byte("key1"), []byte("value1.1"))
rwTx.Put(kv.AccountChangeSet, []byte("key3"), []byte("value3.3"))
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
cursor, err := batch.RwCursorDupSort(kv.AccountChangeSet)
require.NoError(t, err)
require.NoError(t, cursor.Put([]byte("key3"), []byte("value3.1")))
require.NoError(t, cursor.Put([]byte("key1"), []byte("value1.3")))
v, err := cursor.SeekBothRange([]byte("key2"), []byte("value1.2"))
require.NoError(t, err)
// SeekBothRange does exact match of the key, but range match of the value, so we get nil here
require.Nil(t, v)
v, err = cursor.SeekBothRange([]byte("key3"), []byte("value3.2"))
require.NoError(t, err)
require.Equal(t, "value3.3", string(v))
}
func initializeDbAutoConversion(rwTx kv.RwTx) {
rwTx.Put(kv.PlainState, []byte("A"), []byte("0"))
rwTx.Put(kv.PlainState, []byte("A..........................._______________________________A"), []byte("1"))
rwTx.Put(kv.PlainState, []byte("A..........................._______________________________C"), []byte("2"))
rwTx.Put(kv.PlainState, []byte("B"), []byte("8"))
rwTx.Put(kv.PlainState, []byte("C"), []byte("9"))
rwTx.Put(kv.PlainState, []byte("D..........................._______________________________A"), []byte("3"))
rwTx.Put(kv.PlainState, []byte("D..........................._______________________________C"), []byte("4"))
}
func TestAutoConversion(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
c, err := batch.RwCursor(kv.PlainState)
require.NoError(t, err)
// key length conflict
require.Error(t, c.Put([]byte("A..........................."), []byte("?")))
require.NoError(t, c.Delete([]byte("A..........................._______________________________A")))
require.NoError(t, c.Put([]byte("B"), []byte("7")))
require.NoError(t, c.Delete([]byte("C")))
require.NoError(t, c.Put([]byte("D..........................._______________________________C"), []byte("6")))
require.NoError(t, c.Put([]byte("D..........................._______________________________E"), []byte("5")))
k, v, err := c.First()
require.NoError(t, err)
assert.Equal(t, []byte("A"), k)
assert.Equal(t, []byte("0"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("A..........................._______________________________C"), k)
assert.Equal(t, []byte("2"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("B"), k)
assert.Equal(t, []byte("7"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("D..........................._______________________________A"), k)
assert.Equal(t, []byte("3"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("D..........................._______________________________C"), k)
assert.Equal(t, []byte("6"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("D..........................._______________________________E"), k)
assert.Equal(t, []byte("5"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Nil(t, k)
assert.Nil(t, v)
}
func TestAutoConversionDelete(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
c, err := batch.RwCursor(kv.PlainState)
require.NoError(t, err)
require.NoError(t, c.Delete([]byte("A..........................._______________________________A")))
require.NoError(t, c.Delete([]byte("A..........................._______________________________C")))
require.NoError(t, c.Delete([]byte("B")))
require.NoError(t, c.Delete([]byte("C")))
k, v, err := c.First()
require.NoError(t, err)
assert.Equal(t, []byte("A"), k)
assert.Equal(t, []byte("0"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("D..........................._______________________________A"), k)
assert.Equal(t, []byte("3"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Equal(t, []byte("D..........................._______________________________C"), k)
assert.Equal(t, []byte("4"), v)
k, v, err = c.Next()
require.NoError(t, err)
assert.Nil(t, k)
assert.Nil(t, v)
}
func TestAutoConversionSeekBothRange(t *testing.T) {
_, rwTx := memdb.NewTestTx(t)
initializeDbAutoConversion(rwTx)
batch := NewMemoryBatch(rwTx, "", log.Root())
defer batch.Close()
c, err := batch.RwCursorDupSort(kv.PlainState)
require.NoError(t, err)
require.NoError(t, c.Delete([]byte("A..........................._______________________________A")))
require.NoError(t, c.Put([]byte("D..........................._______________________________C"), []byte("6")))
require.NoError(t, c.Put([]byte("D..........................._______________________________E"), []byte("5")))
v, err := c.SeekBothRange([]byte("A..........................."), []byte("_______________________________A"))
require.NoError(t, err)
assert.Equal(t, []byte("_______________________________C2"), v)
_, v, err = c.NextDup()
require.NoError(t, err)
assert.Nil(t, v)
v, err = c.SeekBothRange([]byte("A..........................."), []byte("_______________________________X"))
require.NoError(t, err)
assert.Nil(t, v)
v, err = c.SeekBothRange([]byte("B..........................."), []byte(""))
require.NoError(t, err)
assert.Nil(t, v)
v, err = c.SeekBothRange([]byte("C..........................."), []byte(""))
require.NoError(t, err)
assert.Nil(t, v)
v, err = c.SeekBothRange([]byte("D..........................."), []byte(""))
require.NoError(t, err)
assert.Equal(t, []byte("_______________________________A3"), v)
_, v, err = c.NextDup()
require.NoError(t, err)
assert.Equal(t, []byte("_______________________________C6"), v)
_, v, err = c.NextDup()
require.NoError(t, err)
assert.Equal(t, []byte("_______________________________E5"), v)
_, v, err = c.NextDup()
require.NoError(t, err)
assert.Nil(t, v)
v, err = c.SeekBothRange([]byte("X..........................."), []byte("_______________________________Y"))
require.NoError(t, err)
assert.Nil(t, v)
}