mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

move more parts to lru2 (#7098)

Browse Source
This commit is contained in:
Alex Sharov 2023-03-14 14:37:23 +07:00 committed by GitHub
parent 158fb2b606
commit bbe56620a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
go 1.18
require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d
github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3
github.com/ledgerwatch/log/v3 v3.7.0
github.com/ledgerwatch/secp256k1 v1.0.0
@ -45,7 +45,7 @@ require (
github.com/gorilla/websocket v1.5.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
github.com/hashicorp/golang-lru/v2 v2.0.1
github.com/hashicorp/golang-lru/v2 v2.0.2
github.com/holiman/uint256 v1.2.1
github.com/huandu/xstrings v1.4.0
github.com/huin/goupnp v1.1.0

8
go.sum
View File

@ -426,8 +426,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d h1:TEbIpIPOXtLvBd9WLCGFRHmSGAF+GJ0y1TX3g5L78jo=
github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44=
github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53 h1:afhHLD++/zGz6IHKgflvvJWcKgP5F7T5wfb8DPN74V4=
github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw=
github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=

8
p2p/discover/v5wire/session.go
View File

@ -26,7 +26,7 @@ import (
"github.com/ledgerwatch/erigon/crypto"
"github.com/ledgerwatch/erigon/p2p/enode"
"github.com/hashicorp/golang-lru/simplelru"
"github.com/hashicorp/golang-lru/v2/simplelru"
)
const handshakeTimeout = time.Second
@ -34,7 +34,7 @@ const handshakeTimeout = time.Second
// The SessionCache keeps negotiated encryption keys and
// state for in-progress handshakes in the Discovery v5 wire protocol.
type SessionCache struct {
sessions *simplelru.LRU
sessions *simplelru.LRU[sessionID, *session]
handshakes map[sessionID]*Whoareyou
clock mclock.Clock
@ -63,7 +63,7 @@ func (s *session) keysFlipped() *session {
}
func NewSessionCache(maxItems int, clock mclock.Clock) *SessionCache {
cache, err := simplelru.NewLRU(maxItems, nil)
cache, err := simplelru.NewLRU[sessionID, *session](maxItems, nil)
if err != nil {
panic("can't create session cache")
}
@ -100,7 +100,7 @@ func (sc *SessionCache) session(id enode.ID, addr string) *session {
if !ok {
return nil
}
return item.(*session)
return item
}
// readKey returns the current read key for the given node.

17
turbo/stages/bodydownload/prefetched_blocks.go
View File

@ -1,32 +1,29 @@
package bodydownload
import (
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/hashicorp/golang-lru/v2"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon/core/types"
lru "github.com/hashicorp/golang-lru"
)
type PrefetchedBlocks struct {
blocks *lru.Cache
blocks *lru.Cache[common.Hash, types.RawBlock]
}
func NewPrefetchedBlocks() *PrefetchedBlocks {
// Setting this to 2500 as `erigon import` imports blocks in batches of 2500
// and the import command makes use of PrefetchedBlocks.
cache, err := lru.New(2500)
cache, err := lru.New[common.Hash, types.RawBlock](2500)
if err != nil {
panic("error creating prefetching cache for blocks")
}
return &PrefetchedBlocks{blocks: cache}
}
func (pb *PrefetchedBlocks) Get(hash libcommon.Hash) (*types.Header, *types.RawBody) {
if val, ok := pb.blocks.Get(hash); ok && val != nil {
if block, ok := val.(types.RawBlock); ok {
return block.Header, block.Body
}
func (pb *PrefetchedBlocks) Get(hash common.Hash) (*types.Header, *types.RawBody) {
if block, ok := pb.blocks.Get(hash); ok {
return block.Header, block.Body
}
return nil, nil
}

4
turbo/stages/headerdownload/header_algos.go
View File

@ -797,7 +797,7 @@ func (hi *HeaderInserter) ForkingPoint(db kv.StatelessRwTx, header, parent *type
blockHeight := header.Number.Uint64()
var ch libcommon.Hash
if fromCache, ok := hi.canonicalCache.Get(blockHeight - 1); ok {
ch = fromCache.(libcommon.Hash)
ch = fromCache
} else {
if ch, err = hi.headerReader.CanonicalHash(context.Background(), db, blockHeight-1); err != nil {
return 0, fmt.Errorf("reading canonical hash for height %d: %w", blockHeight-1, err)
@ -811,7 +811,7 @@ func (hi *HeaderInserter) ForkingPoint(db kv.StatelessRwTx, header, parent *type
ancestorHeight := blockHeight - 2
// Look in the cache first
for fromCache, ok := hi.canonicalCache.Get(ancestorHeight); ok; fromCache, ok = hi.canonicalCache.Get(ancestorHeight) {
ch = fromCache.(libcommon.Hash)
ch = fromCache
if ch == ancestorHash {
break
}

76
turbo/stages/headerdownload/header_data_struct.go
View File

@ -8,8 +8,8 @@ import (
"time"
"github.com/google/btree"
lru "github.com/hashicorp/golang-lru"
libcommon "github.com/ledgerwatch/erigon-lib/common"
"github.com/hashicorp/golang-lru/v2"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/etl"
"github.com/ledgerwatch/erigon/consensus"
@ -36,9 +36,9 @@ const (
type Link struct {
header *types.Header
headerRaw []byte
fChild *Link // Pointer to the first child, further children can be found by following `next` pointers to the siblings
next *Link // Pointer to the next sibling, or nil if there are no siblings
hash libcommon.Hash // Hash of the header
fChild *Link // Pointer to the first child, further children can be found by following `next` pointers to the siblings
next *Link // Pointer to the next sibling, or nil if there are no siblings
hash common.Hash // Hash of the header
blockHeight uint64
persisted bool // Whether this link comes from the database record
verified bool // Ancestor of pre-verified header or verified by consensus engine
@ -105,8 +105,8 @@ func (lq *LinkQueue) Pop() interface{} {
// more than one pointer.
type Anchor struct {
peerID [64]byte
fLink *Link // Links attached immediately to this anchor (pointer to the first one, the rest can be found by following `next` fields)
parentHash libcommon.Hash // Hash of the header this anchor can be connected to (to disappear)
fLink *Link // Links attached immediately to this anchor (pointer to the first one, the rest can be found by following `next` fields)
parentHash common.Hash // Hash of the header this anchor can be connected to (to disappear)
blockHeight uint64
nextRetryTime time.Time // Zero when anchor has just been created, otherwise time when anchor needs to be check to see if retry is needed
timeouts int // Number of timeout that this anchor has experiences - after certain threshold, it gets invalidated
@ -115,7 +115,7 @@ type Anchor struct {
type ChainSegmentHeader struct {
HeaderRaw rlp.RawValue
Header *types.Header
Hash libcommon.Hash
Hash common.Hash
Number uint64
}
@ -149,7 +149,7 @@ type PeerPenalty struct {
// Request for chain segment starting with hash and going to its parent, etc, with length headers in total
type HeaderRequest struct {
Hash libcommon.Hash
Hash common.Hash
Number uint64
Length uint64
Skip uint64
@ -162,12 +162,12 @@ type PenaltyItem struct {
Penalty Penalty
}
type Announce struct {
Hash libcommon.Hash
Hash common.Hash
Number uint64
}
type VerifySealFunc func(header *types.Header) error
type CalcDifficultyFunc func(childTimestamp uint64, parentTime uint64, parentDifficulty, parentNumber *big.Int, parentHash, parentUncleHash libcommon.Hash) *big.Int
type CalcDifficultyFunc func(childTimestamp uint64, parentTime uint64, parentDifficulty, parentNumber *big.Int, parentHash, parentUncleHash common.Hash) *big.Int
// InsertQueue keeps the links before they are inserted in the database
// It priorities them by block height (the lowest block height on the top),
@ -227,9 +227,9 @@ type Stats struct {
}
type HeaderDownload struct {
badHeaders map[libcommon.Hash]struct{}
anchors map[libcommon.Hash]*Anchor // Mapping from parentHash to collection of anchors
links map[libcommon.Hash]*Link // Links by header hash
badHeaders map[common.Hash]struct{}
anchors map[common.Hash]*Anchor // Mapping from parentHash to collection of anchors
links map[common.Hash]*Link // Links by header hash
engine consensus.Engine
insertQueue InsertQueue // Priority queue of non-persisted links that need to be verified and can be inserted
seenAnnounces *SeenAnnounces // External announcement hashes, after header verification if hash is in this set - will broadcast it further
@ -259,17 +259,17 @@ type HeaderDownload struct {
requestId int
posAnchor *Anchor
posStatus SyncStatus
posSync bool // Whether the chain is syncing in the PoS mode
headersCollector *etl.Collector // ETL collector for headers
BeaconRequestList *engineapi.RequestList // Requests from ethbackend to staged sync
PayloadStatusCh chan engineapi.PayloadStatus // Responses (validation/execution status)
ShutdownCh chan struct{} // Channel to signal shutdown
pendingPayloadHash libcommon.Hash // Header whose status we still should send to PayloadStatusCh
pendingPayloadStatus *engineapi.PayloadStatus // Alternatively, there can be an already prepared response to send to PayloadStatusCh
unsettledForkChoice *engineapi.ForkChoiceMessage // Forkchoice to process after unwind
unsettledHeadHeight uint64 // Height of unsettledForkChoice.headBlockHash
posDownloaderTip libcommon.Hash // See https://hackmd.io/GDc0maGsQeKfP8o2C7L52w
badPoSHeaders map[libcommon.Hash]libcommon.Hash // Invalid Tip -> Last Valid Ancestor
posSync bool // Whether the chain is syncing in the PoS mode
headersCollector *etl.Collector // ETL collector for headers
BeaconRequestList *engineapi.RequestList // Requests from ethbackend to staged sync
PayloadStatusCh chan engineapi.PayloadStatus // Responses (validation/execution status)
ShutdownCh chan struct{} // Channel to signal shutdown
pendingPayloadHash common.Hash // Header whose status we still should send to PayloadStatusCh
pendingPayloadStatus *engineapi.PayloadStatus // Alternatively, there can be an already prepared response to send to PayloadStatusCh
unsettledForkChoice *engineapi.ForkChoiceMessage // Forkchoice to process after unwind
unsettledHeadHeight uint64 // Height of unsettledForkChoice.headBlockHash
posDownloaderTip common.Hash // See https://hackmd.io/GDc0maGsQeKfP8o2C7L52w
badPoSHeaders map[common.Hash]common.Hash // Invalid Tip -> Last Valid Ancestor
}
// HeaderRecord encapsulates two forms of the same header - raw RLP encoding (to avoid duplicated decodings and encodings), and parsed value types.Header
@ -287,13 +287,13 @@ func NewHeaderDownload(
persistentLinkLimit := linkLimit / 16
hd := &HeaderDownload{
initialCycle: true,
badHeaders: make(map[libcommon.Hash]struct{}),
anchors: make(map[libcommon.Hash]*Anchor),
badHeaders: make(map[common.Hash]struct{}),
anchors: make(map[common.Hash]*Anchor),
persistedLinkLimit: persistentLinkLimit,
linkLimit: linkLimit - persistentLinkLimit,
anchorLimit: anchorLimit,
engine: engine,
links: make(map[libcommon.Hash]*Link),
links: make(map[common.Hash]*Link),
anchorTree: btree.NewG[*Anchor](32, func(a, b *Anchor) bool { return a.blockHeight < b.blockHeight }),
seenAnnounces: NewSeenAnnounces(),
DeliveryNotify: make(chan struct{}, 1),
@ -302,7 +302,7 @@ func NewHeaderDownload(
PayloadStatusCh: make(chan engineapi.PayloadStatus, 1),
ShutdownCh: make(chan struct{}),
headerReader: headerReader,
badPoSHeaders: make(map[libcommon.Hash]libcommon.Hash),
badPoSHeaders: make(map[common.Hash]common.Hash),
}
heap.Init(&hd.persistedLinkQueue)
heap.Init(&hd.linkQueue)
@ -371,14 +371,14 @@ func (hd *HeaderDownload) moveLinkToQueue(link *Link, queueId QueueID) {
type HeaderInserter struct {
localTd *big.Int
logPrefix string
prevHash libcommon.Hash // Hash of previously seen header - to filter out potential duplicates
highestHash libcommon.Hash
prevHash common.Hash // Hash of previously seen header - to filter out potential duplicates
highestHash common.Hash
newCanonical bool
unwind bool
unwindPoint uint64
highest uint64
highestTimestamp uint64
canonicalCache *lru.Cache
canonicalCache *lru.Cache[uint64, common.Hash]
headerReader services.HeaderAndCanonicalReader
}
@ -389,24 +389,24 @@ func NewHeaderInserter(logPrefix string, localTd *big.Int, headerProgress uint64
unwindPoint: headerProgress,
headerReader: headerReader,
}
hi.canonicalCache, _ = lru.New(1000)
hi.canonicalCache, _ = lru.New[uint64, common.Hash](1000)
return hi
}
// SeenAnnounces - external announcement hashes, after header verification if hash is in this set - will broadcast it further
type SeenAnnounces struct {
hashes *lru.Cache
hashes *lru.Cache[common.Hash, struct{}]
}
func NewSeenAnnounces() *SeenAnnounces {
cache, err := lru.New(1000)
cache, err := lru.New[common.Hash, struct{}](1000)
if err != nil {
panic("error creating prefetching cache for blocks")
}
return &SeenAnnounces{hashes: cache}
}
func (s *SeenAnnounces) Pop(hash libcommon.Hash) bool {
func (s *SeenAnnounces) Pop(hash common.Hash) bool {
_, ok := s.hashes.Get(hash)
if ok {
s.hashes.Remove(hash)
@ -414,11 +414,11 @@ func (s *SeenAnnounces) Pop(hash libcommon.Hash) bool {
return ok
}
func (s SeenAnnounces) Seen(hash libcommon.Hash) bool {
func (s SeenAnnounces) Seen(hash common.Hash) bool {
_, ok := s.hashes.Get(hash)
return ok
}
func (s *SeenAnnounces) Add(b libcommon.Hash) {
func (s *SeenAnnounces) Add(b common.Hash) {
s.hashes.ContainsOrAdd(b, struct{}{})
}