rpcdaemon: fix remote bor engine for trace transaction (#9214)

This PR fixes the below error when running a remote rpcdaemon for
Polygon debug_traceTransaction.

Request:
```
curl http://localhost:9545/ \
-X POST \
-H "Content-Type: application/json" \
--data '{
  "jsonrpc": "2.0",
  "id": 0,
  "method": "debug_traceTransaction",
  "params": [
    "0xf265c04fc31c91eec9971e118f059322a910776ce40584114c351732ab01aec7",
    {
      "tracer": "callTracer",
      "borTraceEnabled": true,
      "borTx": true
    }
  ]
}'
```

Response:
```
{"jsonrpc":"2.0","id":0,"result":null,"error":{"code":-32000,"message":"method handler crashed"}}
```

Logs:
```
EROR[01-11|18:45:14.087] RPC method debug_traceTransaction crashed: interface conversion: *cli.remoteConsensusEngine is not consensus.Engine: missing method APIs
```

cmd/rpcdaemon/cli/config.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -13,32 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ledgerwatch/erigon/polygon/heimdall/span"
-
-	"github.com/ledgerwatch/erigon-lib/chain"
-	"github.com/ledgerwatch/erigon-lib/chain/snapcfg"
-	libcommon "github.com/ledgerwatch/erigon-lib/common"
-	"github.com/ledgerwatch/erigon-lib/common/datadir"
-	"github.com/ledgerwatch/erigon-lib/common/dir"
-	"github.com/ledgerwatch/erigon-lib/common/hexutility"
-	"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
-	"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
-	libstate "github.com/ledgerwatch/erigon-lib/state"
-	"github.com/ledgerwatch/erigon/consensus"
-	"github.com/ledgerwatch/erigon/consensus/ethash"
-	"github.com/ledgerwatch/erigon/core/state/temporal"
-	"github.com/ledgerwatch/erigon/core/systemcontracts"
-	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/ethconfig"
-	"github.com/ledgerwatch/erigon/polygon/bor"
-	"github.com/ledgerwatch/erigon/polygon/bor/borcfg"
-	"github.com/ledgerwatch/erigon/polygon/bor/contract"
-	"github.com/ledgerwatch/erigon/rpc/rpccfg"
-	"github.com/ledgerwatch/erigon/turbo/debug"
-	"github.com/ledgerwatch/erigon/turbo/logging"
-	"github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks"
-	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snap"
-
 	"github.com/ledgerwatch/log/v3"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/semaphore"
@@ -46,6 +21,12 @@ import (
 	grpcHealth "google.golang.org/grpc/health"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
+	"github.com/ledgerwatch/erigon-lib/chain"
+	"github.com/ledgerwatch/erigon-lib/chain/snapcfg"
+	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon-lib/common/datadir"
+	"github.com/ledgerwatch/erigon-lib/common/dir"
+	"github.com/ledgerwatch/erigon-lib/common/hexutility"
 	"github.com/ledgerwatch/erigon-lib/direct"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil"
@@ -53,10 +34,12 @@ import (
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/kvcache"
+	"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
 	kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx"
+	"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
 	"github.com/ledgerwatch/erigon-lib/kv/remotedb"
 	"github.com/ledgerwatch/erigon-lib/kv/remotedbserver"
-
+	libstate "github.com/ledgerwatch/erigon-lib/state"
 	"github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli/httpcfg"
 	"github.com/ledgerwatch/erigon/cmd/rpcdaemon/graphql"
 	"github.com/ledgerwatch/erigon/cmd/rpcdaemon/health"
@@ -65,12 +48,28 @@ import (
 	"github.com/ledgerwatch/erigon/cmd/utils/flags"
 	"github.com/ledgerwatch/erigon/common"
 	"github.com/ledgerwatch/erigon/common/paths"
+	"github.com/ledgerwatch/erigon/consensus"
+	"github.com/ledgerwatch/erigon/consensus/ethash"
 	"github.com/ledgerwatch/erigon/core/rawdb"
+	"github.com/ledgerwatch/erigon/core/state"
+	"github.com/ledgerwatch/erigon/core/state/temporal"
+	"github.com/ledgerwatch/erigon/core/systemcontracts"
+	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/eth/ethconfig"
 	"github.com/ledgerwatch/erigon/node"
 	"github.com/ledgerwatch/erigon/node/nodecfg"
+	"github.com/ledgerwatch/erigon/polygon/bor"
+	"github.com/ledgerwatch/erigon/polygon/bor/borcfg"
+	"github.com/ledgerwatch/erigon/polygon/bor/contract"
+	"github.com/ledgerwatch/erigon/polygon/heimdall/span"
 	"github.com/ledgerwatch/erigon/rpc"
+	"github.com/ledgerwatch/erigon/rpc/rpccfg"
+	"github.com/ledgerwatch/erigon/turbo/debug"
+	"github.com/ledgerwatch/erigon/turbo/logging"
 	"github.com/ledgerwatch/erigon/turbo/rpchelper"
 	"github.com/ledgerwatch/erigon/turbo/services"
+	"github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks"
+	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snap"
 
 	// Force-load native and js packages, to trigger registration
 	_ "github.com/ledgerwatch/erigon/eth/tracers/js"
@@ -785,7 +784,7 @@ func isWebsocket(r *http.Request) bool {
 		strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade")
 }
 
-// obtainJWTSecret loads the jwt-secret, either from the provided config,
+// ObtainJWTSecret loads the jwt-secret, either from the provided config,
 // or from the default location. If neither of those are present, it generates
 // a new secret and stores to the default location.
 func ObtainJWTSecret(cfg *httpcfg.HttpCfg, logger log.Logger) ([]byte, error) {
@@ -877,8 +876,10 @@ func createEngineListener(cfg *httpcfg.HttpCfg, engineApi []rpc.API, logger log.
 	return engineListener, engineSrv, engineAddr.String(), nil
 }
 
+var remoteConsensusEngineNotReadyErr = errors.New("remote consensus engine not ready")
+
 type remoteConsensusEngine struct {
-	engine consensus.EngineReader
+	engine consensus.Engine
 }
 
 func (e *remoteConsensusEngine) HasEngine() bool {
@@ -889,6 +890,18 @@ func (e *remoteConsensusEngine) Engine() consensus.EngineReader {
 	return e.engine
 }
 
+func (e *remoteConsensusEngine) validateEngineReady() error {
+	if !e.HasEngine() {
+		return remoteConsensusEngineNotReadyErr
+	}
+
+	return nil
+}
+
+// init - reasoning behind init is that we would like to initialise the remote consensus engine either post rpcdaemon
+// service startup or in a background goroutine, so that we do not depend on the liveness of other services when
+// starting up rpcdaemon and do not block startup (avoiding "cascade outage" scenario). In this case the DB dependency
+// can be a remote DB service running on another machine.
 func (e *remoteConsensusEngine) init(db kv.RoDB, blockReader services.FullBlockReader, remoteKV remote.KVClient, logger log.Logger) bool {
 	var cc *chain.Config
 
@@ -928,41 +941,89 @@ func (e *remoteConsensusEngine) init(db kv.RoDB, blockReader services.FullBlockR
 }
 
 func (e *remoteConsensusEngine) Author(header *types.Header) (libcommon.Address, error) {
-	if e.engine != nil {
+	if err := e.validateEngineReady(); err != nil {
-		return e.engine.Author(header)
+		return libcommon.Address{}, err
 	}
 
-	return libcommon.Address{}, fmt.Errorf("remote consensus engine not iinitialized")
+	return e.engine.Author(header)
 }
 
 func (e *remoteConsensusEngine) IsServiceTransaction(sender libcommon.Address, syscall consensus.SystemCall) bool {
-	if e.engine != nil {
+	if err := e.validateEngineReady(); err != nil {
-		return e.engine.IsServiceTransaction(sender, syscall)
+		panic(err)
 	}
 
-	return false
+	return e.engine.IsServiceTransaction(sender, syscall)
 }
 
 func (e *remoteConsensusEngine) Type() chain.ConsensusName {
-	if e.engine != nil {
+	if err := e.validateEngineReady(); err != nil {
-		return e.engine.Type()
+		panic(err)
 	}
 
-	return ""
+	return e.engine.Type()
 }
 
 func (e *remoteConsensusEngine) CalculateRewards(config *chain.Config, header *types.Header, uncles []*types.Header, syscall consensus.SystemCall) ([]consensus.Reward, error) {
-	if e.engine != nil {
+	if err := e.validateEngineReady(); err != nil {
-		return e.engine.CalculateRewards(config, header, uncles, syscall)
+		return nil, err
 	}
 
-	return nil, fmt.Errorf("remote consensus engine not iinitialized")
+	return e.engine.CalculateRewards(config, header, uncles, syscall)
 }
 
 func (e *remoteConsensusEngine) Close() error {
-	if e.engine != nil {
+	if err := e.validateEngineReady(); err != nil {
-		return e.engine.Close()
+		return err
 	}
 
-	return nil
+	return e.engine.Close()
+}
+
+func (e *remoteConsensusEngine) Initialize(config *chain.Config, chain consensus.ChainHeaderReader, header *types.Header, state *state.IntraBlockState, syscall consensus.SysCallCustom, logger log.Logger) {
+	if err := e.validateEngineReady(); err != nil {
+		panic(err)
+	}
+
+	e.engine.Initialize(config, chain, header, state, syscall, logger)
+}
+
+func (e *remoteConsensusEngine) VerifyHeader(_ consensus.ChainHeaderReader, _ *types.Header, _ bool) error {
+	panic("remoteConsensusEngine.VerifyHeader not supported")
+}
+
+func (e *remoteConsensusEngine) VerifyUncles(_ consensus.ChainReader, _ *types.Header, _ []*types.Header) error {
+	panic("remoteConsensusEngine.VerifyUncles not supported")
+}
+
+func (e *remoteConsensusEngine) Prepare(_ consensus.ChainHeaderReader, _ *types.Header, _ *state.IntraBlockState) error {
+	panic("remoteConsensusEngine.Prepare not supported")
+}
+
+func (e *remoteConsensusEngine) Finalize(_ *chain.Config, _ *types.Header, _ *state.IntraBlockState, _ types.Transactions, _ []*types.Header, _ types.Receipts, _ []*types.Withdrawal, _ consensus.ChainReader, _ consensus.SystemCall, _ log.Logger) (types.Transactions, types.Receipts, error) {
+	panic("remoteConsensusEngine.Finalize not supported")
+}
+
+func (e *remoteConsensusEngine) FinalizeAndAssemble(_ *chain.Config, _ *types.Header, _ *state.IntraBlockState, _ types.Transactions, _ []*types.Header, _ types.Receipts, _ []*types.Withdrawal, _ consensus.ChainReader, _ consensus.SystemCall, _ consensus.Call, _ log.Logger) (*types.Block, types.Transactions, types.Receipts, error) {
+	panic("remoteConsensusEngine.FinalizeAndAssemble not supported")
+}
+
+func (e *remoteConsensusEngine) Seal(_ consensus.ChainHeaderReader, _ *types.Block, _ chan<- *types.Block, _ <-chan struct{}) error {
+	panic("remoteConsensusEngine.Seal not supported")
+}
+
+func (e *remoteConsensusEngine) SealHash(_ *types.Header) libcommon.Hash {
+	panic("remoteConsensusEngine.SealHash not supported")
+}
+
+func (e *remoteConsensusEngine) CalcDifficulty(_ consensus.ChainHeaderReader, _ uint64, _ uint64, _ *big.Int, _ uint64, _ libcommon.Hash, _ libcommon.Hash, _ uint64) *big.Int {
+	panic("remoteConsensusEngine.CalcDifficulty not supported")
+}
+
+func (e *remoteConsensusEngine) GenerateSeal(_ consensus.ChainHeaderReader, _ *types.Header, _ *types.Header, _ consensus.Call) []byte {
+	panic("remoteConsensusEngine.GenerateSeal not supported")
+}
+
+func (e *remoteConsensusEngine) APIs(_ consensus.ChainHeaderReader) []rpc.API {
+	panic("remoteConsensusEngine.APIs not supported")
 }