Add circuit breaker for relayer/builder (#11215)

* Add circuit breaker for relayer

* Update mainnet_config.go

* Renamings

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

beacon-chain/node/config.go

@@ -65,6 +65,24 @@ func configureSafeSlotsToImportOptimistically(cliCtx *cli.Context) error {
 	return nil
 }
 
+func configureBuilderCircuitBreaker(cliCtx *cli.Context) error {
+	if cliCtx.IsSet(flags.MaxBuilderConsecutiveMissedSlots.Name) {
+		c := params.BeaconConfig().Copy()
+		c.MaxBuilderConsecutiveMissedSlots = types.Slot(cliCtx.Int(flags.MaxBuilderConsecutiveMissedSlots.Name))
+		if err := params.SetActive(c); err != nil {
+			return err
+		}
+	}
+	if cliCtx.IsSet(flags.MaxBuilderEpochMissedSlots.Name) {
+		c := params.BeaconConfig().Copy()
+		c.MaxBuilderEpochMissedSlots = types.Slot(cliCtx.Int(flags.MaxBuilderEpochMissedSlots.Name))
+		if err := params.SetActive(c); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func configureSlotsPerArchivedPoint(cliCtx *cli.Context) error {
 	if cliCtx.IsSet(flags.SlotsPerArchivedPoint.Name) {
 		c := params.BeaconConfig().Copy()

beacon-chain/node/node.go

@@ -136,6 +136,10 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 	if err := configureSafeSlotsToImportOptimistically(cliCtx); err != nil {
 		return nil, err
 	}
+	err := configureBuilderCircuitBreaker(cliCtx)
+	if err != nil {
+		return nil, err
+	}
 	if err := configureSlotsPerArchivedPoint(cliCtx); err != nil {
 		return nil, err
 	}

beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel

@@ -134,6 +134,7 @@ go_test(
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/execution/testing:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",
         "//beacon-chain/operations/slashings:go_default_library",
         "//beacon-chain/operations/synccommittee:go_default_library",
@@ -143,6 +144,7 @@ go_test(
         "//beacon-chain/state/stategen:go_default_library",
         "//beacon-chain/state/stategen/mock:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
+        "//beacon-chain/state/v3:go_default_library",
         "//beacon-chain/sync/initial-sync/testing:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go

@@ -295,6 +295,54 @@ func (vs *Server) readyForBuilder(ctx context.Context) (bool, error) {
 	return blocks.IsExecutionBlock(b.Block().Body())
 }
 
+// circuitBreakBuilder returns true if the builder is not allowed to be used due to circuit breaker conditions.
+func (vs *Server) circuitBreakBuilder(s types.Slot) (bool, error) {
+	if vs.ForkFetcher == nil || vs.ForkFetcher.ForkChoicer() == nil {
+		return true, errors.New("no fork choicer configured")
+	}
+
+	// Circuit breaker is active if the missing consecutive slots greater than `MaxBuilderConsecutiveMissedSlots`.
+	highestReceivedSlot := vs.ForkFetcher.ForkChoicer().HighestReceivedBlockSlot()
+	fallbackSlots := params.BeaconConfig().MaxBuilderConsecutiveMissedSlots
+	diff, err := s.SafeSubSlot(highestReceivedSlot)
+	if err != nil {
+		return true, err
+	}
+	if diff > fallbackSlots {
+		log.WithFields(logrus.Fields{
+			"currentSlot":         s,
+			"highestReceivedSlot": highestReceivedSlot,
+			"fallBackSkipSlots":   fallbackSlots,
+		}).Warn("Builder circuit breaker activated due to missing consecutive slot")
+		return true, nil
+	}
+
+	// Not much reason to check missed slots epoch rolling window if input slot is less than epoch.
+	if s < params.BeaconConfig().SlotsPerEpoch {
+		return false, nil
+	}
+
+	// Circuit breaker is active if the missing slots per epoch (rolling window) greater than `MaxBuilderEpochMissedSlots`.
+	receivedCount, err := vs.ForkFetcher.ForkChoicer().ReceivedBlocksLastEpoch()
+	if err != nil {
+		return true, err
+	}
+	fallbackSlotsLastEpoch := params.BeaconConfig().MaxBuilderEpochMissedSlots
+	diff, err = params.BeaconConfig().SlotsPerEpoch.SafeSub(receivedCount)
+	if err != nil {
+		return true, err
+	}
+	if diff > fallbackSlotsLastEpoch {
+		log.WithFields(logrus.Fields{
+			"totalMissed":                receivedCount,
+			"fallBackSkipSlotsLastEpoch": fallbackSlotsLastEpoch,
+		}).Warn("Builder circuit breaker activated due to missing enough slots last epoch")
+		return true, nil
+	}
+
+	return false, nil
+}
+
 // Get and build blind block from builder network. Returns a boolean status, built block and error.
 // If the status is false that means builder the header block is disallowed.
 // This routine is time limited by `blockBuilderTimeout`.
@@ -313,6 +361,15 @@ func (vs *Server) getAndBuildBlindBlock(ctx context.Context, b *ethpb.BeaconBloc
 	if !ready {
 		return false, nil, nil
 	}
+
+	circuitBreak, err := vs.circuitBreakBuilder(b.Slot)
+	if err != nil {
+		return false, nil, errors.Wrap(err, "could not determine if builder circuit breaker condition")
+	}
+	if circuitBreak {
+		return false, nil, nil
+	}
+
 	h, err := vs.getPayloadHeaderFromBuilder(ctx, b.Slot, b.ProposerIndex)
 	if err != nil {
 		return false, nil, errors.Wrap(err, "could not get payload header")

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix_test.go

@@ -18,11 +18,14 @@ import (
 	prysmtime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	mockExecution "github.com/prysmaticlabs/prysm/beacon-chain/execution/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/synccommittee"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	v3 "github.com/prysmaticlabs/prysm/beacon-chain/state/v3"
 	mockSync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -31,6 +34,7 @@ import (
 	types "github.com/prysmaticlabs/prysm/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	v1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/testing/require"
@@ -331,6 +335,7 @@ func TestServer_getAndBuildHeaderBlock(t *testing.T) {
 	vs.FinalizationFetcher = &blockchainTest.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Root: wbr1[:]}}
 	vs.HeadFetcher = &blockchainTest.ChainService{Block: wb1}
 	vs.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, ErrGetHeader: errors.New("could not get payload")}
+	vs.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()}
 	ready, _, err = vs.getAndBuildBlindBlock(ctx, &ethpb.BeaconBlockAltair{})
 	require.ErrorContains(t, "could not get payload", err)
 	require.Equal(t, false, ready)
@@ -406,6 +411,7 @@ func TestServer_getAndBuildHeaderBlock(t *testing.T) {
 	}
 	vs.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, Bid: sBid}
 	vs.TimeFetcher = &blockchainTest.ChainService{Genesis: time.Now()}
+	vs.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()}
 	ready, builtBlk, err := vs.getAndBuildBlindBlock(ctx, altairBlk.Block)
 	require.NoError(t, err)
 	require.Equal(t, true, ready)
@@ -644,12 +650,18 @@ func TestServer_GetBellatrixBeaconBlock_BuilderCase(t *testing.T) {
 		Signature: sk.Sign(sr[:]).Marshal(),
 	}
 	proposerServer.BlockBuilder = &builderTest.MockBuilderService{HasConfigured: true, Bid: sBid}
-
+	proposerServer.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()}
 	randaoReveal, err := util.RandaoReveal(beaconState, 0, privKeys)
 	require.NoError(t, err)
 
 	require.NoError(t, proposerServer.BeaconDB.SaveRegistrationsByValidatorIDs(ctx, []types.ValidatorIndex{40},
 		[]*ethpb.ValidatorRegistrationV1{{FeeRecipient: bytesutil.PadTo([]byte{}, fieldparams.FeeRecipientLength), Pubkey: bytesutil.PadTo([]byte{}, fieldparams.BLSPubkeyLength)}}))
+
+	params.SetupTestConfigCleanup(t)
+	cfg.MaxBuilderConsecutiveMissedSlots = bellatrixSlot + 1
+	cfg.MaxBuilderEpochMissedSlots = 32
+	params.OverrideBeaconConfig(cfg)
+
 	block, err := proposerServer.getBellatrixBeaconBlock(ctx, &ethpb.BlockRequest{
 		Slot:         bellatrixSlot + 1,
 		RandaoReveal: randaoReveal,
@@ -721,3 +733,75 @@ func TestServer_validateBuilderSignature(t *testing.T) {
 	sBid.Message.Value = make([]byte, 32)
 	require.ErrorIs(t, s.validateBuilderSignature(sBid), signing.ErrSigFailedToVerify)
 }
+
+func TestServer_circuitBreakBuilder(t *testing.T) {
+	hook := logTest.NewGlobal()
+	s := &Server{}
+	_, err := s.circuitBreakBuilder(0)
+	require.ErrorContains(t, "no fork choicer configured", err)
+
+	s.ForkFetcher = &blockchainTest.ChainService{ForkChoiceStore: protoarray.New()}
+	s.ForkFetcher.ForkChoicer().SetGenesisTime(uint64(time.Now().Unix()))
+	b, err := s.circuitBreakBuilder(params.BeaconConfig().MaxBuilderConsecutiveMissedSlots + 1)
+	require.NoError(t, err)
+	require.Equal(t, true, b)
+	require.LogsContain(t, hook, "Builder circuit breaker activated due to missing consecutive slot")
+
+	ojc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	ofc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	ctx := context.Background()
+	st, blkRoot, err := createState(1, [32]byte{'a'}, [32]byte{}, params.BeaconConfig().ZeroHash, ojc, ofc)
+	require.NoError(t, err)
+	require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot))
+	b, err = s.circuitBreakBuilder(params.BeaconConfig().MaxBuilderConsecutiveMissedSlots + 1)
+	require.NoError(t, err)
+	require.Equal(t, false, b)
+
+	params.SetupTestConfigCleanup(t)
+	params.OverrideBeaconConfig(params.MainnetConfig())
+	st, blkRoot, err = createState(params.BeaconConfig().SlotsPerEpoch, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, ojc, ofc)
+	require.NoError(t, err)
+	require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot))
+	b, err = s.circuitBreakBuilder(params.BeaconConfig().SlotsPerEpoch + 1)
+	require.NoError(t, err)
+	require.Equal(t, true, b)
+	require.LogsContain(t, hook, "Builder circuit breaker activated due to missing enough slots last epoch")
+
+	want := params.BeaconConfig().SlotsPerEpoch - params.BeaconConfig().MaxBuilderEpochMissedSlots
+	for i := types.Slot(2); i <= want+2; i++ {
+		st, blkRoot, err = createState(i, [32]byte{byte(i)}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, ojc, ofc)
+		require.NoError(t, err)
+		require.NoError(t, s.ForkFetcher.ForkChoicer().InsertNode(ctx, st, blkRoot))
+	}
+	b, err = s.circuitBreakBuilder(params.BeaconConfig().SlotsPerEpoch + 1)
+	require.NoError(t, err)
+	require.Equal(t, false, b)
+}
+
+func createState(
+	slot types.Slot,
+	blockRoot [32]byte,
+	parentRoot [32]byte,
+	payloadHash [32]byte,
+	justified *ethpb.Checkpoint,
+	finalized *ethpb.Checkpoint,
+) (state.BeaconState, [32]byte, error) {
+
+	base := &ethpb.BeaconStateBellatrix{
+		Slot:                       slot,
+		RandaoMixes:                make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+		BlockRoots:                 make([][]byte, 1),
+		CurrentJustifiedCheckpoint: justified,
+		FinalizedCheckpoint:        finalized,
+		LatestExecutionPayloadHeader: &enginev1.ExecutionPayloadHeader{
+			BlockHash: payloadHash[:],
+		},
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{
+			ParentRoot: parentRoot[:],
+		},
+	}
+
+	base.BlockRoots[0] = append(base.BlockRoots[0], blockRoot[:]...)
+	st, err := v3.InitializeFromProto(base)
+	return st, blockRoot, err
+}

cmd/beacon-chain/flags/base.go

@@ -16,6 +16,16 @@ var (
 		Usage: "A MEV builder relay string http endpoint, this wil be used to interact MEV builder network using API defined in: https://ethereum.github.io/builder-specs/#/Builder",
 		Value: "",
 	}
+	MaxBuilderConsecutiveMissedSlots = &cli.IntFlag{
+		Name:  "max-builder-consecutive-missed-slots",
+		Usage: "Number of consecutive skip slot to fallback from using relay/builder to local execution engine for block construction",
+		Value: 3,
+	}
+	MaxBuilderEpochMissedSlots = &cli.IntFlag{
+		Name:  "max-builder-epoch-missed-slots",
+		Usage: "Number of total skip slot to fallback from using relay/builder to local execution engine for block construction in last epoch rolling window",
+		Value: 8,
+	}
 	// ExecutionEngineEndpoint provides an HTTP access endpoint to connect to an execution client on the execution layer
 	ExecutionEngineEndpoint = &cli.StringFlag{
 		Name:  "execution-endpoint",

config/params/config.go

@@ -199,6 +199,10 @@ type BeaconChainConfig struct {
 	DefaultFeeRecipient              common.Address // DefaultFeeRecipient where the transaction fee goes to.
 	EthBurnAddressHex                string         // EthBurnAddressHex is the constant eth address written in hex format to burn fees in that network. the default is 0x0
 	DefaultBuilderGasLimit           uint64         // DefaultBuilderGasLimit is the default used to set the gaslimit for the Builder APIs, typically at around 30M wei.
+
+	// Mev-boost circuit breaker
+	MaxBuilderConsecutiveMissedSlots types.Slot // MaxBuilderConsecutiveMissedSlots defines the number of consecutive skip slot to fallback from using relay/builder to local execution engine for block construction.
+	MaxBuilderEpochMissedSlots       types.Slot // MaxBuilderEpochMissedSlots is defines the number of total skip slot (per epoch rolling windows) to fallback from using relay/builder to local execution engine for block construction.
 }
 
 // InitializeForkSchedule initializes the schedules forks baked into the config.

config/params/mainnet_config.go

@@ -251,6 +251,10 @@ var mainnetBeaconConfig = &BeaconChainConfig{
 	TerminalTotalDifficulty:          "115792089237316195423570985008687907853269984665640564039457584007913129638912",
 	EthBurnAddressHex:                "0x0000000000000000000000000000000000000000",
 	DefaultBuilderGasLimit:           uint64(30000000),
+
+	// Mevboost circuit breaker
+	MaxBuilderConsecutiveMissedSlots: 4,
+	MaxBuilderEpochMissedSlots:       6,
 }
 
 // MainnetTestConfig provides a version of the mainnet config that has a different name