mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-28 14:47:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

coherent state cache: change btree type (#813)

Browse Source
This commit is contained in:
Alex Sharov 2022-12-30 19:47:21 +07:00 committed by GitHub
parent d0af20cb8a
commit 745aacef74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 51 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
go.mod
View File

@ -2,6 +2,14 @@ module github.com/ledgerwatch/erigon-lib
go 1.18
require (
github.com/ledgerwatch/interfaces v0.0.0-20221226080656-9ea2ff13ca12
github.com/ledgerwatch/log/v3 v3.6.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/ledgerwatch/trackerslist v1.0.0
github.com/torquem-ch/mdbx-go v0.27.4
)
require (
github.com/RoaringBitmap/roaring v1.2.1
github.com/VictoriaMetrics/metrics v1.23.0
@ -15,17 +23,12 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
github.com/holiman/uint256 v1.2.1
github.com/ledgerwatch/interfaces v0.0.0-20221226080656-9ea2ff13ca12
github.com/ledgerwatch/log/v3 v3.6.0
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/ledgerwatch/trackerslist v1.0.0
github.com/matryer/moq v0.3.0
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/quasilyte/go-ruleguard/dsl v0.3.21
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.8.1
github.com/tidwall/btree v1.5.0
github.com/torquem-ch/mdbx-go v0.27.0
go.uber.org/atomic v1.10.0
golang.org/x/crypto v0.4.0
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15

4
go.sum
View File

@ -381,8 +381,8 @@ github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYms
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/torquem-ch/mdbx-go v0.27.0 h1:FquhRvKL2zweMdk1R6UdOx3h6DiHgJ0+P9yQvSouURI=
github.com/torquem-ch/mdbx-go v0.27.0/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E=
github.com/torquem-ch/mdbx-go v0.27.4 h1:XDqtNuCoY6RMqfcyB0pWAPtDavCFxkMjcSUd4g6HelY=
github.com/torquem-ch/mdbx-go v0.27.4/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=

73
kv/kvcache/cache.go
View File

@ -27,7 +27,7 @@ import (
"github.com/VictoriaMetrics/metrics"
"github.com/c2h5oh/datasize"
"github.com/google/btree"
btree2 "github.com/tidwall/btree"
"go.uber.org/atomic"
"golang.org/x/crypto/sha3"
@ -120,8 +120,8 @@ type Coherent struct {
}
type CoherentRoot struct {
cache *btree.BTreeG[*Element]
codeCache *btree.BTreeG[*Element]
cache *btree2.BTreeG[*Element]
codeCache *btree2.BTreeG[*Element]
ready chan struct{} // close when ready
readyChanClosed atomic.Bool // protecting `ready` field from double-close (on unwind). Consumers don't need check this field.
@ -177,6 +177,7 @@ func New(cfg CoherentConfig) *Coherent {
if cfg.KeepViews == 0 {
panic("empty config passed")
}
return &Coherent{
roots: map[uint64]*CoherentRoot{},
stateEvict: &ThreadSafeEvictionList{l: NewList()},
@ -206,8 +207,8 @@ func (c *Coherent) selectOrCreateRoot(versionID uint64) *CoherentRoot {
r = &CoherentRoot{
ready: make(chan struct{}),
cache: btree.NewG[*Element](DEGREE, Less),
codeCache: btree.NewG[*Element](DEGREE, Less),
cache: btree2.NewBTreeG[*Element](Less),
codeCache: btree2.NewBTreeG[*Element](Less),
}
c.roots[versionID] = r
return r
@ -229,22 +230,26 @@ func (c *Coherent) advanceRoot(stateVersionID uint64) (r *CoherentRoot) {
if prevView, ok := c.roots[stateVersionID-1]; ok && prevView.isCanonical {
//log.Info("advance: clone", "from", viewID-1, "to", viewID)
r.cache = prevView.cache.Clone()
r.codeCache = prevView.codeCache.Clone()
r.cache = prevView.cache.Copy()
r.codeCache = prevView.codeCache.Copy()
} else {
c.stateEvict.Init()
c.codeEvict.Init()
if r.cache == nil {
//log.Info("advance: new", "to", viewID)
r.cache = btree.NewG[*Element](DEGREE, Less)
r.codeCache = btree.NewG[*Element](DEGREE, Less)
r.cache = btree2.NewBTreeG[*Element](Less)
r.codeCache = btree2.NewBTreeG[*Element](Less)
} else {
r.cache.Ascend(func(i *Element) bool {
c.stateEvict.PushFront(i)
r.cache.Walk(func(items []*Element) bool {
for _, i := range items {
c.stateEvict.PushFront(i)
}
return true
})
r.codeCache.Ascend(func(i *Element) bool {
c.codeEvict.PushFront(i)
r.codeCache.Walk(func(items []*Element) bool {
for _, i := range items {
c.codeEvict.PushFront(i)
}
return true
})
}
@ -445,7 +450,7 @@ func (c *Coherent) removeOldestCode(r *CoherentRoot) {
}
func (c *Coherent) add(k, v []byte, r *CoherentRoot, id uint64) *Element {
it := &Element{K: k, V: v}
replaced, _ := r.cache.ReplaceOrInsert(it)
replaced, _ := r.cache.Set(it)
if c.latestStateVersionID != id {
//fmt.Printf("add to non-last viewID: %d<%d\n", c.latestViewID, id)
return it
@ -464,7 +469,7 @@ func (c *Coherent) add(k, v []byte, r *CoherentRoot, id uint64) *Element {
}
func (c *Coherent) addCode(k, v []byte, r *CoherentRoot, id uint64) *Element {
it := &Element{K: k, V: v}
replaced, _ := r.codeCache.ReplaceOrInsert(it)
replaced, _ := r.codeCache.Set(it)
if c.latestStateVersionID != id {
//fmt.Printf("add to non-last viewID: %d<%d\n", c.latestViewID, id)
return it
@ -527,11 +532,11 @@ func (c *Coherent) ValidateCurrentRoot(ctx context.Context, tx kv.Tx) (*CacheVal
clearCache := false
compare := func(cache *btree.BTreeG[*Element], bucket string) (bool, [][]byte, error) {
compare := func(cache *btree2.BTreeG[*Element], bucket string) (bool, [][]byte, error) {
keys := make([][]byte, 0)
for {
val, ok := cache.DeleteMax()
val, ok := cache.PopMax()
if !ok {
break
}
@ -587,19 +592,19 @@ func (c *Coherent) ValidateCurrentRoot(ctx context.Context, tx kv.Tx) (*CacheVal
return result, nil
}
func (c *Coherent) cloneCaches(r *CoherentRoot) (cache *btree.BTreeG[*Element], codeCache *btree.BTreeG[*Element]) {
func (c *Coherent) cloneCaches(r *CoherentRoot) (cache *btree2.BTreeG[*Element], codeCache *btree2.BTreeG[*Element]) {
c.lock.Lock()
defer c.lock.Unlock()
cache = r.cache.Clone()
codeCache = r.codeCache.Clone()
cache = r.cache.Copy()
codeCache = r.codeCache.Copy()
return cache, codeCache
}
func (c *Coherent) clearCaches(r *CoherentRoot) {
c.lock.Lock()
defer c.lock.Unlock()
r.cache.Clear(false)
r.codeCache.Clear(false)
r.cache.Clear()
r.codeCache.Clear()
}
type Stat struct {
@ -647,18 +652,20 @@ func AssertCheckValues(ctx context.Context, tx kv.Tx, cache Cache) (int, error)
if !ok {
return 0, nil
}
root.cache.Ascend(func(i *Element) bool {
k, v := i.K, i.V
var dbV []byte
dbV, err = tx.GetOne(kv.PlainState, k)
if err != nil {
return false
root.cache.Walk(func(items []*Element) bool {
for _, i := range items {
k, v := i.K, i.V
var dbV []byte
dbV, err = tx.GetOne(kv.PlainState, k)
if err != nil {
return false
}
if !bytes.Equal(dbV, v) {
err = fmt.Errorf("key: %x, has different values: %x != %x", k, v, dbV)
return false
}
checked++
}
if !bytes.Equal(dbV, v) {
err = fmt.Errorf("key: %x, has different values: %x != %x", k, v, dbV)
return false
}
checked++
return true
})
return checked, err

3
kv/kvcache/cache_test.go
View File

@ -22,13 +22,12 @@ import (
"sync"
"testing"
"github.com/stretchr/testify/require"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/gointerfaces"
"github.com/ledgerwatch/erigon-lib/gointerfaces/remote"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/memdb"
"github.com/stretchr/testify/require"
)
func TestEvictionInUnexpectedOrder(t *testing.T) {