mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

RemoteRPC: coherentCache for kv.Code; LocalRPC: enable small blocksLRU (#2815)

Browse Source
This commit is contained in:
Alex Sharov 2021-10-12 12:04:04 +07:00 committed by GitHub
parent 21d024b06a
commit da00e949c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 114 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -129,7 +129,7 @@ lintci:
lintci-deps:
rm -f ./build/bin/golangci-lint
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.41.1
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.42.1
clean:
env GO111MODULE=on go clean -cache

2
cmd/rpcdaemon/cli/config.go
View File

@ -69,7 +69,7 @@ var rootCmd = &cobra.Command{
func RootCommand() (*cobra.Command, *Flags) {
utils.CobraFlags(rootCmd, append(debug.Flags, utils.MetricFlags...))
cfg := &Flags{}
cfg := &Flags{StateCache: kvcache.DefaultCoherentConfig}
rootCmd.PersistentFlags().StringVar(&cfg.PrivateApiAddr, "private.api.addr", "127.0.0.1:9090", "private api network address, for example: 127.0.0.1:9090")
rootCmd.PersistentFlags().StringVar(&cfg.Datadir, "datadir", "", "path to Erigon working directory")
rootCmd.PersistentFlags().StringVar(&cfg.Chaindata, "chaindata", "", "path to the database")

14
cmd/rpcdaemon/commands/eth_api.go
View File

@ -8,16 +8,15 @@ import (
lru "github.com/hashicorp/golang-lru"
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcache"
"github.com/ledgerwatch/erigon/consensus/misc"
"github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/filters"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/services"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/hexutil"
"github.com/ledgerwatch/erigon/common/math"
"github.com/ledgerwatch/erigon/consensus/misc"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/types"
ethFilters "github.com/ledgerwatch/erigon/eth/filters"
@ -110,10 +109,15 @@ type BaseAPI struct {
}
func NewBaseApi(f *filters.Filters, stateCache kvcache.Cache, singleNodeMode bool) *BaseAPI {
var blocksLRU *lru.Cache
blocksLRUSize := 128 // ~32Mb
if !singleNodeMode {
blocksLRU, _ = lru.New(256)
blocksLRUSize = 512
}
blocksLRU, err := lru.New(blocksLRUSize)
if err != nil {
panic(err)
}
return &BaseAPI{filters: f, stateCache: stateCache, blocksLRU: blocksLRU}
}

33
cmd/rpcdaemon/commands/eth_call.go
View File

@ -18,6 +18,7 @@ import (
"github.com/ledgerwatch/erigon/internal/ethapi"
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rpc"
"github.com/ledgerwatch/erigon/turbo/rpchelper"
"github.com/ledgerwatch/erigon/turbo/transactions"
"github.com/ledgerwatch/log/v3"
)
@ -43,7 +44,20 @@ func (api *APIImpl) Call(ctx context.Context, args ethapi.CallArgs, blockNrOrHas
if api.TevmEnabled {
contractHasTEVM = ethdb.GetHasTEVM(tx)
}
result, err := transactions.DoCall(ctx, args, tx, blockNrOrHash, overrides, api.GasCap, chainConfig, api.filters, api.stateCache, contractHasTEVM)
blockNumber, hash, err := rpchelper.GetCanonicalBlockNumber(blockNrOrHash, tx, api.filters) // DoCall cannot be executed on non-canonical blocks
if err != nil {
return nil, err
}
block, err := api.BaseAPI.blockWithSenders(tx, hash, blockNumber)
if err != nil {
return nil, err
}
if block == nil {
return nil, nil
}
result, err := transactions.DoCall(ctx, args, tx, blockNrOrHash, block, overrides, api.GasCap, chainConfig, api.stateCache, contractHasTEVM)
if err != nil {
return nil, err
}
@ -181,8 +195,21 @@ func (api *APIImpl) EstimateGas(ctx context.Context, args ethapi.CallArgs, block
executable := func(gas uint64) (bool, *core.ExecutionResult, error) {
args.Gas = (*hexutil.Uint64)(&gas)
result, err := transactions.DoCall(ctx, args, dbtx, rpc.BlockNumberOrHash{BlockNumber: &lastBlockNum}, nil,
api.GasCap, chainConfig, api.filters, api.stateCache, contractHasTEVM)
numOrHash := rpc.BlockNumberOrHash{BlockNumber: &lastBlockNum}
blockNumber, hash, err := rpchelper.GetCanonicalBlockNumber(numOrHash, dbtx, api.filters) // DoCall cannot be executed on non-canonical blocks
if err != nil {
return false, nil, err
}
block, err := api.BaseAPI.blockWithSenders(dbtx, hash, blockNumber)
if err != nil {
return false, nil, err
}
if block == nil {
return false, nil, nil
}
result, err := transactions.DoCall(ctx, args, dbtx, numOrHash, block, nil,
api.GasCap, chainConfig, api.stateCache, contractHasTEVM)
if err != nil {
if errors.Is(err, core.ErrIntrinsicGas) {
// Special case, raise gas limit

8
cmd/rpctest/main.go
View File

@ -22,6 +22,7 @@ func main() {
erigonURL string
blockFrom uint64
blockTo uint64
latest bool
recordFile string
errorFile string
)
@ -35,6 +36,9 @@ func main() {
cmd.Flags().Uint64Var(&blockFrom, "blockFrom", 2000000, "Block number to start test generation from")
cmd.Flags().Uint64Var(&blockTo, "blockTo", 2101000, "Block number to end test generation at")
}
withLatest := func(cmd *cobra.Command) {
cmd.Flags().BoolVar(&latest, "latest", false, "Exec on latest ")
}
withNeedCompare := func(cmd *cobra.Command) {
cmd.Flags().BoolVar(&needCompare, "needCompare", false, "need compare with geth")
}
@ -55,10 +59,10 @@ func main() {
Short: "",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
rpctest.BenchEthCall(erigonURL, gethURL, needCompare, blockFrom, blockTo, recordFile, errorFile)
rpctest.BenchEthCall(erigonURL, gethURL, needCompare, latest, blockFrom, blockTo, recordFile, errorFile)
},
}
with(benchEthCallCmd, withErigonUrl, withGethUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile)
with(benchEthCallCmd, withErigonUrl, withGethUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile, withLatest)
var bench1Cmd = &cobra.Command{
Use: "bench1",

9
cmd/rpctest/rpctest/bench_ethcall.go
View File

@ -15,7 +15,7 @@ import (
// false value - to generate vegeta files, it's faster but we can generate vegeta files for Geth and Erigon
// recordFile stores all eth_call returned with success
// errorFile stores information when erigon and geth doesn't return same data
func BenchEthCall(erigonURL, gethURL string, needCompare bool, blockFrom uint64, blockTo uint64, recordFile string, errorFile string) {
func BenchEthCall(erigonURL, gethURL string, needCompare, latest bool, blockFrom, blockTo uint64, recordFile string, errorFile string) {
setRoutes(erigonURL, gethURL)
var client = &http.Client{
Timeout: time.Second * 600,
@ -103,7 +103,12 @@ func BenchEthCall(erigonURL, gethURL string, needCompare bool, blockFrom uint64,
reqGen.reqID++
nTransactions = nTransactions + 1
request := reqGen.ethCall(tx.From, tx.To, &tx.Gas, &tx.GasPrice, &tx.Value, tx.Input, bn-1)
var request string
if latest {
request = reqGen.ethCallLatest(tx.From, tx.To, &tx.Gas, &tx.GasPrice, &tx.Value, tx.Input)
} else {
request = reqGen.ethCall(tx.From, tx.To, &tx.Gas, &tx.GasPrice, &tx.Value, tx.Input, bn-1)
}
errCtx := fmt.Sprintf(" bn=%d hash=%s", bn, tx.Hash)
if err := requestAndCompare(request, "eth_call", errCtx, reqGen, needCompare, rec, errs, resultsCh); err != nil {

22
cmd/rpctest/rpctest/request_generator.go
View File

@ -241,6 +241,28 @@ func (g *RequestGenerator) ethCall(from common.Address, to *common.Address, gas
return sb.String()
}
func (g *RequestGenerator) ethCallLatest(from common.Address, to *common.Address, gas *hexutil.Big, gasPrice *hexutil.Big, value *hexutil.Big, data hexutil.Bytes) string {
var sb strings.Builder
fmt.Fprintf(&sb, `{ "jsonrpc": "2.0", "method": "eth_call", "params": [{"from":"0x%x"`, from)
if to != nil {
fmt.Fprintf(&sb, `,"to":"0x%x"`, *to)
}
if gas != nil {
fmt.Fprintf(&sb, `,"gas":"%s"`, gas)
}
if gasPrice != nil {
fmt.Fprintf(&sb, `,"gasPrice":"%s"`, gasPrice)
}
if len(data) > 0 {
fmt.Fprintf(&sb, `,"data":"%s"`, data)
}
if value != nil {
fmt.Fprintf(&sb, `,"value":"%s"`, value)
}
fmt.Fprintf(&sb, `},"latest"], "id":%d}`, g.reqID)
return sb.String()
}
func (g *RequestGenerator) call(target string, method, body string, response interface{}) CallResult {
start := time.Now()
err := post(g.client, routes[target], body, response)

4
core/state/cached_reader2.go
View File

@ -25,7 +25,7 @@ func NewCachedReader2(cache kvcache.CacheView, tx kv.Tx) *CachedReader2 {
// ReadAccountData is called when an account needs to be fetched from the state
func (r *CachedReader2) ReadAccountData(address common.Address) (*accounts.Account, error) {
enc, err := r.cache.Get(address.Bytes())
enc, err := r.cache.Get(address[:])
if err != nil {
return nil, err
}
@ -55,7 +55,7 @@ func (r *CachedReader2) ReadAccountCode(address common.Address, incarnation uint
if bytes.Equal(codeHash.Bytes(), emptyCodeHash) {
return nil, nil
}
code, err := r.db.GetOne(kv.Code, codeHash.Bytes())
code, err := r.cache.GetCode(codeHash.Bytes())
if len(code) == 0 {
return nil, nil
}

1
crypto/blake2b/blake2bAVX2_amd64.go
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.7 && amd64 && !gccgo && !appengine
// +build go1.7,amd64,!gccgo,!appengine
package blake2b

1
crypto/blake2b/register.go
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.9
// +build go1.9
package blake2b

1
crypto/bn256/bn256_fast.go
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
//go:build amd64 || arm64
// +build amd64 arm64
// Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve.

1
crypto/bn256/cloudflare/gfp_decl.go
View File

@ -1,3 +1,4 @@
//go:build (amd64 && !generic) || (arm64 && !generic)
// +build amd64,!generic arm64,!generic
package bn256

1
crypto/signature_cgo.go
View File

@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build !nacl && !js && cgo && !gofuzz
// +build !nacl,!js,cgo,!gofuzz
package crypto

24
eth/backend.go
View File

@ -499,17 +499,19 @@ func New(stack *node.Node, config *ethconfig.Config, logger log.Logger) (*Ethere
}
if backend.config.TxPool.V2 {
if err := backend.txPool2DB.View(context.Background(), func(tx kv.Tx) error {
pendingBaseFee := misc.CalcBaseFee(chainConfig, hh)
return backend.txPool2.OnNewBlock(context.Background(), &remote.StateChangeBatch{
PendingBlockBaseFee: pendingBaseFee.Uint64(),
DatabaseViewID: tx.ViewID(),
ChangeBatch: []*remote.StateChange{
{BlockHeight: hh.Number.Uint64(), BlockHash: gointerfaces.ConvertHashToH256(hh.Hash())},
},
}, txpool2.TxSlots{}, txpool2.TxSlots{}, tx)
}); err != nil {
return nil, err
if hh != nil {
if err := backend.txPool2DB.View(context.Background(), func(tx kv.Tx) error {
pendingBaseFee := misc.CalcBaseFee(chainConfig, hh)
return backend.txPool2.OnNewBlock(context.Background(), &remote.StateChangeBatch{
PendingBlockBaseFee: pendingBaseFee.Uint64(),
DatabaseViewID: tx.ViewID(),
ChangeBatch: []*remote.StateChange{
{BlockHeight: hh.Number.Uint64(), BlockHash: gointerfaces.ConvertHashToH256(hh.Hash())},
},
}, txpool2.TxSlots{}, txpool2.TxSlots{}, tx)
}); err != nil {
return nil, err
}
}
} else {
if hh != nil {

1
ethdb/olddb/database_test.go
View File

@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build !js
// +build !js
package olddb

2
go.mod
View File

@ -36,7 +36,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
github.com/ledgerwatch/erigon-lib v0.0.0-20211011173149-083ee839067f
github.com/ledgerwatch/erigon-lib v0.0.0-20211012041733-a745f2391c49
github.com/ledgerwatch/log/v3 v3.3.1
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d
github.com/logrusorgru/aurora/v3 v3.0.0

4
go.sum
View File

@ -497,8 +497,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20211011173149-083ee839067f h1:gxkNMQ0Bx4+VzT83xMvKshvvSrD2tkz4pzLM/RBSyoQ=
github.com/ledgerwatch/erigon-lib v0.0.0-20211011173149-083ee839067f/go.mod h1:kM8TzB/YifxKol66U0bNLsmdopypP+ag6REvUGU62s4=
github.com/ledgerwatch/erigon-lib v0.0.0-20211012041733-a745f2391c49 h1:cpbGDRm6kkCvcZ9WjejziSTqgxRqYGvZcpIG5AmJxe8=
github.com/ledgerwatch/erigon-lib v0.0.0-20211012041733-a745f2391c49/go.mod h1:kM8TzB/YifxKol66U0bNLsmdopypP+ag6REvUGU62s4=
github.com/ledgerwatch/log/v3 v3.3.1 h1:HmvLeTEvtCtqSvtu4t/a5MAdcLfeBcbIeowXbLYuzLc=
github.com/ledgerwatch/log/v3 v3.3.1/go.mod h1:S3VJqhhVX32rbp1JyyvhJou12twtFwNEPESBgpbNkRk=
github.com/ledgerwatch/secp256k1 v0.0.0-20210626115225-cd5cd00ed72d h1:/IKMrJdfRsoYNc36PXqP4xMH3vhW/8IQyBKGQbKZUno=

1
internal/debug/loudpanic.go
View File

@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build go1.6
// +build go1.6
package debug

1
internal/debug/signal.go
View File

@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package debug

3
internal/debug/trace.go
View File

@ -14,7 +14,8 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//+build go1.5
//go:build go1.5
// +build go1.5
package debug

1
metrics/cpu_enabled.go
View File

@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build !ios
// +build !ios
package metrics

1
metrics/cpu_syscall.go
View File

@ -14,6 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build !windows
// +build !windows
package metrics

4
metrics/runtime_cgo.go
View File

@ -1,5 +1,5 @@
// +build cgo
// +build !appengine
//go:build cgo && !appengine
// +build cgo,!appengine
package metrics

1
metrics/syslog.go
View File

@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package metrics

3
p2p/netutil/toobig_notwindows.go
View File

@ -14,7 +14,8 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//+build !windows
//go:build !windows
// +build !windows
package netutil

16
turbo/transactions/call.go
View File

@ -9,7 +9,6 @@ import (
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcache"
"github.com/ledgerwatch/erigon/cmd/rpcdaemon/filters"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/core/rawdb"
@ -19,15 +18,12 @@ import (
"github.com/ledgerwatch/erigon/internal/ethapi"
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/rpc"
"github.com/ledgerwatch/erigon/turbo/rpchelper"
"github.com/ledgerwatch/log/v3"
)
const callTimeout = 5 * time.Minute
func DoCall(ctx context.Context, args ethapi.CallArgs, tx kv.Tx, blockNrOrHash rpc.BlockNumberOrHash,
overrides *map[common.Address]ethapi.Account, gasCap uint64, chainConfig *params.ChainConfig,
filters *filters.Filters, stateCache kvcache.Cache, contractHasTEVM func(hash common.Hash) (bool, error)) (*core.ExecutionResult, error) {
func DoCall(ctx context.Context, args ethapi.CallArgs, tx kv.Tx, blockNrOrHash rpc.BlockNumberOrHash, block *types.Block, overrides *map[common.Address]ethapi.Account, gasCap uint64, chainConfig *params.ChainConfig, stateCache kvcache.Cache, contractHasTEVM func(hash common.Hash) (bool, error)) (*core.ExecutionResult, error) {
// todo: Pending state is only known by the miner
/*
if blockNrOrHash.BlockNumber != nil && *blockNrOrHash.BlockNumber == rpc.PendingBlockNumber {
@ -35,10 +31,7 @@ func DoCall(ctx context.Context, args ethapi.CallArgs, tx kv.Tx, blockNrOrHash r
return state, block.Header(), nil
}
*/
blockNumber, hash, err := rpchelper.GetCanonicalBlockNumber(blockNrOrHash, tx, filters) // DoCall cannot be executed on non-canonical blocks
if err != nil {
return nil, err
}
blockNumber := block.NumberU64()
var stateReader state.StateReader
if num, ok := blockNrOrHash.Number(); ok && num == rpc.LatestBlockNumber {
cacheView, err := stateCache.View(ctx, tx)
@ -51,10 +44,7 @@ func DoCall(ctx context.Context, args ethapi.CallArgs, tx kv.Tx, blockNrOrHash r
}
state := state.New(stateReader)
header := rawdb.ReadHeader(tx, hash, blockNumber)
if header == nil {
return nil, fmt.Errorf("block %d(%x) not found", blockNumber, hash)
}
header := block.Header()
// Override the fields of specified contracts before execution.
if overrides != nil {