From 2aa1ebf9f6acd6de89aec42b9cf2642b23ebb59e Mon Sep 17 00:00:00 2001 From: Enrique Jose Avila Asapche Date: Tue, 25 Jan 2022 19:44:35 +0300 Subject: [PATCH] Separate Server For Engine API (#3332) --- Dockerfile | 2 +- README.md | 5 +- cmd/rpcdaemon/cli/config.go | 165 +++++++++++++++++++++++++++--------- cmd/rpcdaemon/main.go | 1 + docker-compose.yml | 1 + node/defaults.go | 13 +-- 6 files changed, 137 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 36c29cfde..ac5bfdf20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN chown -R erigon:erigon /home/erigon USER erigon -EXPOSE 8545 8546 30303 30303/udp 30304 30304/udp 8080 9090 6060 +EXPOSE 8545 8550 8546 30303 30303/udp 30304 30304/udp 8080 9090 6060 # https://github.com/opencontainers/image-spec/blob/main/annotations.md ARG BUILD_DATE diff --git a/README.md b/README.md index 6eac4dc7a..04a2b52bc 100644 --- a/README.md +++ b/README.md @@ -298,8 +298,11 @@ internally for rpcdaemon or other connections, (e.g. rpcdaemon -> erigon) | Port | Protocol | Purpose | Expose | |:-----:|:---------:|:-----------------:|:-------:| | 8545 | TCP | HTTP & WebSockets | Private | +|:-----:|:---------:|:-----------------:|:-------:| +| 8550 | TCP | HTTP | Private | -Typically 8545 is exposed only interally for JSON-RPC queries. Both HTTP and WebSocket connections are on the same port. +Typically 8545 is exposed only internally for JSON-RPC queries. Both HTTP and WebSocket connections are on the same port. +Typically 8550 is exposed only internally for the engineApi JSON-RPC queries #### `sentry` ports diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 41cecd0a5..f7a777a17 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -41,35 +41,37 @@ import ( ) type Flags struct { - PrivateApiAddr string - SingleNodeMode bool // Erigon's database can be read by separated processes on same machine - in read-only mode - with full support of transactions. It will share same "OS PageCache" with Erigon process. - Datadir string - Chaindata string - HttpListenAddress string - TLSCertfile string - TLSCACert string - TLSKeyFile string - HttpPort int - HttpCORSDomain []string - HttpVirtualHost []string - HttpCompression bool - API []string - Gascap uint64 - MaxTraces uint64 - WebsocketEnabled bool - WebsocketCompression bool - RpcAllowListFilePath string - RpcBatchConcurrency uint - TraceCompatibility bool // Bug for bug compatibility for trace_ routines with OpenEthereum - TxPoolApiAddr string - TevmEnabled bool - StateCache kvcache.CoherentConfig - Snapshot ethconfig.Snapshot - GRPCServerEnabled bool - GRPCListenAddress string - GRPCPort int - GRPCHealthCheckEnabled bool - StarknetGRPCAddress string + PrivateApiAddr string + SingleNodeMode bool // Erigon's database can be read by separated processes on same machine - in read-only mode - with full support of transactions. It will share same "OS PageCache" with Erigon process. + Datadir string + Chaindata string + HttpListenAddress string + EngineHTTPListenAddress string + TLSCertfile string + TLSCACert string + TLSKeyFile string + HttpPort int + EnginePort int + HttpCORSDomain []string + HttpVirtualHost []string + HttpCompression bool + API []string + Gascap uint64 + MaxTraces uint64 + WebsocketEnabled bool + WebsocketCompression bool + RpcAllowListFilePath string + RpcBatchConcurrency uint + TraceCompatibility bool // Bug for bug compatibility for trace_ routines with OpenEthereum + TxPoolApiAddr string + TevmEnabled bool + StateCache kvcache.CoherentConfig + Snapshot ethconfig.Snapshot + GRPCServerEnabled bool + GRPCListenAddress string + GRPCPort int + GRPCHealthCheckEnabled bool + StarknetGRPCAddress string } var rootCmd = &cobra.Command{ @@ -85,10 +87,12 @@ func RootCommand() (*cobra.Command, *Flags) { rootCmd.PersistentFlags().StringVar(&cfg.Datadir, "datadir", "", "path to Erigon working directory") rootCmd.PersistentFlags().StringVar(&cfg.Chaindata, "chaindata", "", "path to the database") rootCmd.PersistentFlags().StringVar(&cfg.HttpListenAddress, "http.addr", node.DefaultHTTPHost, "HTTP-RPC server listening interface") + rootCmd.PersistentFlags().StringVar(&cfg.EngineHTTPListenAddress, "engine.addr", node.DefaultHTTPHost, "HTTP-RPC server listening interface for engineAPI") rootCmd.PersistentFlags().StringVar(&cfg.TLSCertfile, "tls.cert", "", "certificate for client side TLS handshake") rootCmd.PersistentFlags().StringVar(&cfg.TLSKeyFile, "tls.key", "", "key file for client side TLS handshake") rootCmd.PersistentFlags().StringVar(&cfg.TLSCACert, "tls.cacert", "", "CA certificate for client side TLS handshake") rootCmd.PersistentFlags().IntVar(&cfg.HttpPort, "http.port", node.DefaultHTTPPort, "HTTP-RPC server listening port") + rootCmd.PersistentFlags().IntVar(&cfg.EnginePort, "engine.port", node.DefaultEngineHTTPPort, "HTTP-RPC server listening port for the engineAPI") rootCmd.PersistentFlags().StringSliceVar(&cfg.HttpCORSDomain, "http.corsdomain", []string{}, "Comma separated list of domains from which to accept cross origin requests (browser enforced)") rootCmd.PersistentFlags().StringSliceVar(&cfg.HttpVirtualHost, "http.vhosts", node.DefaultConfig.HTTPVirtualHosts, "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.") rootCmd.PersistentFlags().BoolVar(&cfg.HttpCompression, "http.compression", true, "Disable http compression") @@ -359,6 +363,10 @@ func RemoteServices(ctx context.Context, cfg Flags, logger log.Logger, rootCance } func StartRpcServer(ctx context.Context, cfg Flags, rpcAPI []rpc.API) error { + var engineListener *http.Server + var enginesrv *rpc.Server + var engineHttpEndpoint string + // register apis and create handler stack httpEndpoint := fmt.Sprintf("%s:%d", cfg.HttpListenAddress, cfg.HttpPort) @@ -370,7 +378,29 @@ func StartRpcServer(ctx context.Context, cfg Flags, rpcAPI []rpc.API) error { } srv.SetAllowList(allowListForRPC) - if err := node.RegisterApisFromWhitelist(rpcAPI, cfg.API, srv, false); err != nil { + var defaultAPIList []rpc.API + var engineAPI []rpc.API + + for _, api := range rpcAPI { + if api.Namespace != "engine" { + defaultAPIList = append(defaultAPIList, api) + } else { + engineAPI = append(engineAPI, api) + } + } + + var apiFlags []string + var engineFlag []string + + for _, flag := range cfg.API { + if flag != "engine" { + apiFlags = append(apiFlags, flag) + } else { + engineFlag = append(engineFlag, flag) + } + } + + if err := node.RegisterApisFromWhitelist(defaultAPIList, apiFlags, srv, false); err != nil { return fmt.Errorf("could not start register RPC apis: %w", err) } @@ -380,24 +410,22 @@ func StartRpcServer(ctx context.Context, cfg Flags, rpcAPI []rpc.API) error { wsHandler = srv.WebsocketHandler([]string{"*"}, cfg.WebsocketCompression) } - var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // adding a healthcheck here - if health.ProcessHealthcheckIfNeeded(w, r, rpcAPI) { - return - } - if cfg.WebsocketEnabled && r.Method == "GET" { - wsHandler.ServeHTTP(w, r) - return - } - httpHandler.ServeHTTP(w, r) - }) + apiHandler := createHandler(cfg, defaultAPIList, httpHandler, wsHandler) - listener, _, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler) + listener, _, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, apiHandler) if err != nil { return fmt.Errorf("could not start RPC api: %w", err) } info := []interface{}{"url", httpEndpoint, "ws", cfg.WebsocketEnabled, "ws.compression", cfg.WebsocketCompression, "grpc", cfg.GRPCServerEnabled} + + if len(engineAPI) > 0 { + engineListener, enginesrv, engineHttpEndpoint, err = createEngineListener(cfg, engineAPI, engineFlag) + if err != nil { + return fmt.Errorf("could not start RPC api for engine: %w", err) + } + } + var ( healthServer *grpcHealth.Server grpcServer *grpc.Server @@ -422,11 +450,19 @@ func StartRpcServer(ctx context.Context, cfg Flags, rpcAPI []rpc.API) error { defer func() { srv.Stop() + if enginesrv != nil { + enginesrv.Stop() + } shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _ = listener.Shutdown(shutdownCtx) log.Info("HTTP endpoint closed", "url", httpEndpoint) + if engineListener != nil { + _ = engineListener.Shutdown(shutdownCtx) + log.Info("Engine HTTP endpoint close", "url", engineHttpEndpoint) + } + if cfg.GRPCServerEnabled { if cfg.GRPCHealthCheckEnabled { healthServer.Shutdown() @@ -440,3 +476,48 @@ func StartRpcServer(ctx context.Context, cfg Flags, rpcAPI []rpc.API) error { log.Info("Exiting...") return nil } + +func createHandler(cfg Flags, apiList []rpc.API, httpHandler http.Handler, wsHandler http.Handler) http.Handler { + var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // adding a healthcheck here + if health.ProcessHealthcheckIfNeeded(w, r, apiList) { + return + } + if cfg.WebsocketEnabled && wsHandler != nil && r.Method == "GET" { + wsHandler.ServeHTTP(w, r) + return + } + httpHandler.ServeHTTP(w, r) + }) + + return handler +} + +func createEngineListener(cfg Flags, engineApi []rpc.API, engineFlag []string) (*http.Server, *rpc.Server, string, error) { + engineHttpEndpoint := fmt.Sprintf("%s:%d", cfg.EngineHTTPListenAddress, cfg.EnginePort) + + enginesrv := rpc.NewServer(cfg.RpcBatchConcurrency) + + allowListForRPC, err := parseAllowListForRPC(cfg.RpcAllowListFilePath) + if err != nil { + return nil, nil, "", err + } + enginesrv.SetAllowList(allowListForRPC) + + if err := node.RegisterApisFromWhitelist(engineApi, engineFlag, enginesrv, false); err != nil { + return nil, nil, "", fmt.Errorf("could not start register RPC engine api: %w", err) + } + + engineHttpHandler := node.NewHTTPHandlerStack(enginesrv, cfg.HttpCORSDomain, cfg.HttpVirtualHost, cfg.HttpCompression) + engineApiHandler := createHandler(cfg, engineApi, engineHttpHandler, nil) + + engineListener, _, err := node.StartHTTPEndpoint(engineHttpEndpoint, rpc.DefaultHTTPTimeouts, engineApiHandler) + if err != nil { + return nil, nil, "", fmt.Errorf("could not start RPC api: %w", err) + } + engineInfo := []interface{}{"url", engineHttpEndpoint} + log.Info("HTTP endpoint opened for engine", engineInfo...) + + return engineListener, enginesrv, engineHttpEndpoint, nil + +} diff --git a/cmd/rpcdaemon/main.go b/cmd/rpcdaemon/main.go index 25a6b2259..959b39cf7 100644 --- a/cmd/rpcdaemon/main.go +++ b/cmd/rpcdaemon/main.go @@ -34,6 +34,7 @@ func main() { log.Error(err.Error()) return nil } + return nil } diff --git a/docker-compose.yml b/docker-compose.yml index bdf91817b..8550ee05c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,5 +46,6 @@ services: - ${XDG_DATA_HOME:-~/.local/share}/erigon:/home/erigon/.local/share/erigon ports: - "8545:8545" + - "8550:8550" restart: unless-stopped diff --git a/node/defaults.go b/node/defaults.go index f105872cd..951571f89 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -24,12 +24,13 @@ import ( ) const ( - DefaultHTTPHost = "localhost" // Default host interface for the HTTP RPC server - DefaultHTTPPort = 8545 // Default TCP port for the HTTP RPC server - DefaultWSHost = "localhost" // Default host interface for the websocket RPC server - DefaultWSPort = 8546 // Default TCP port for the websocket RPC server - DefaultGRPCHost = "localhost" // Default host interface for the GRPC server - DefaultGRPCPort = 8547 // Default TCP port for the GRPC server + DefaultHTTPHost = "localhost" // Default host interface for the HTTP RPC server + DefaultHTTPPort = 8545 // Default TCP port for the HTTP RPC server + DefaultEngineHTTPPort = 8550 // Default TCP port for the engineApi HTTP RPC server + DefaultWSHost = "localhost" // Default host interface for the websocket RPC server + DefaultWSPort = 8546 // Default TCP port for the websocket RPC server + DefaultGRPCHost = "localhost" // Default host interface for the GRPC server + DefaultGRPCPort = 8547 // Default TCP port for the GRPC server ) // DefaultConfig contains reasonable default settings.