Add metrics for EE PayloadStatus returns (#3486)

## Issue Addressed

NA

## Proposed Changes

Adds some metrics so we can track payload status responses from the EE. I think this will be useful for troubleshooting and alerting.

I also bumped the `BecaonChain::per_slot_task` to `debug` since it doesn't seem too noisy and would have helped us with some things we were debugging in the past.

## Additional Info

NA

Cargo.lock

@@ -2083,6 +2083,7 @@ dependencies = [
  "slot_clock",
  "ssz-rs",
  "state_processing",
+ "strum",
  "task_executor",
  "tempfile",
  "tokio",

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -4274,8 +4274,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// it contains a call to `fork_choice` which may eventually call
     /// `tokio::runtime::block_on` in certain cases.
     pub async fn per_slot_task(self: &Arc<Self>) {
-        trace!(self.log, "Running beacon chain per slot tasks");
         if let Some(slot) = self.slot_clock.now() {
+            debug!(
+                self.log,
+                "Running beacon chain per slot tasks";
+                "slot" => ?slot
+            );
+
             // Always run the light-weight pruning tasks (these structures should be empty during
             // sync anyway).
             self.naive_aggregation_pool.write().prune(slot);

beacon_node/execution_layer/Cargo.toml

@@ -43,4 +43,4 @@ fork_choice = { path = "../../consensus/fork_choice" }
 mev-build-rs = {git = "https://github.com/ralexstokes/mev-rs", tag = "v0.2.0"}
 ethereum-consensus = {git = "https://github.com/ralexstokes/ethereum-consensus"}
 ssz-rs = {git = "https://github.com/ralexstokes/ssz-rs"}
-
+strum = "0.24.0"

beacon_node/execution_layer/src/engine_api.rs

@@ -4,6 +4,7 @@ use http::deposit_methods::RpcError;
 pub use json_structures::TransitionConfigurationV1;
 use reqwest::StatusCode;
 use serde::{Deserialize, Serialize};
+use strum::IntoStaticStr;
 pub use types::{
     Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, FixedVector,
     Hash256, Uint256, VariableList,
@@ -71,7 +72,8 @@ impl From<builder_client::Error> for Error {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, IntoStaticStr)]
+#[strum(serialize_all = "snake_case")]
 pub enum PayloadStatusV1Status {
     Valid,
     Invalid,

beacon_node/execution_layer/src/lib.rs

@@ -893,6 +893,13 @@ impl<T: EthSpec> ExecutionLayer<T> {
             .request(|engine| engine.api.new_payload_v1(execution_payload.clone()))
             .await;
 
+        if let Ok(status) = &result {
+            metrics::inc_counter_vec(
+                &metrics::EXECUTION_LAYER_PAYLOAD_STATUS,
+                &["new_payload", status.status.into()],
+            );
+        }
+
         process_payload_status(execution_payload.block_hash, result, self.log())
             .map_err(Box::new)
             .map_err(Error::EngineError)
@@ -1032,6 +1039,13 @@ impl<T: EthSpec> ExecutionLayer<T> {
             })
             .await;
 
+        if let Ok(status) = &result {
+            metrics::inc_counter_vec(
+                &metrics::EXECUTION_LAYER_PAYLOAD_STATUS,
+                &["forkchoice_updated", status.payload_status.status.into()],
+            );
+        }
+
         process_payload_status(
             head_block_hash,
             result.map(|response| response.payload_status),

beacon_node/execution_layer/src/metrics.rs

@@ -36,4 +36,9 @@ lazy_static::lazy_static! {
         "execution_layer_get_payload_by_block_hash_time",
         "Time to reconstruct a payload from the EE using eth_getBlockByHash"
     );
+    pub static ref EXECUTION_LAYER_PAYLOAD_STATUS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "execution_layer_payload_status",
+        "Indicates the payload status returned for a particular method",
+        &["method", "status"]
+    );
 }