Added merkle proof and fixed bad handling of new validators (#9233)

cl/cltypes/beacon_block.go

@@ -293,3 +293,7 @@ func (*BeaconBody) Static() bool {
 func (*BeaconBlock) Static() bool {
 	return false
 }
+
+func (b *BeaconBody) ExecutionPayloadMerkleProof() ([][32]byte, error) {
+	return merkle_tree.MerkleProof(4, 9, b.getSchema(false)...)
+}

cl/merkle_tree/merkle_root.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/common/length"
 	"github.com/ledgerwatch/erigon-lib/types/ssz"
+	"github.com/ledgerwatch/erigon/cl/utils"
 	"github.com/prysmaticlabs/gohashtree"
 )
 
@@ -116,3 +117,44 @@ func MerkleRootFromFlatLeaves(leaves []byte, out []byte) (err error) {
 func MerkleRootFromFlatLeavesWithLimit(leaves []byte, out []byte, limit uint64) (err error) {
 	return globalHasher.merkleizeTrieLeavesFlat(leaves, out, limit)
 }
+
+// Merkle Proof computes the merkle proof for a given schema of objects.
+func MerkleProof(depth, proofIndex int, schema ...interface{}) ([][32]byte, error) {
+	// Calculate the total number of leaves needed based on the schema length
+	maxDepth := GetDepth(uint64(len(schema)))
+	if utils.PowerOf2(uint64(maxDepth)) != uint64(len(schema)) {
+		maxDepth++
+	}
+
+	if depth != int(maxDepth) { // TODO: Add support for lower depths
+		return nil, fmt.Errorf("depth is different than maximum depth, have %d, want %d", depth, maxDepth)
+	}
+	var err error
+	proof := make([][32]byte, maxDepth)
+	currentSizeDepth := utils.PowerOf2(uint64(maxDepth))
+	for len(schema) != int(currentSizeDepth) { // Augment the schema to be a power of 2
+		schema = append(schema, make([]byte, 32))
+	}
+
+	for i := 0; i < depth; i++ {
+		// Hash the left branch
+		if proofIndex >= int(currentSizeDepth)/2 {
+			proof[depth-i-1], err = HashTreeRoot(schema[0 : currentSizeDepth/2]...)
+			if err != nil {
+				return nil, err
+			}
+			schema = schema[currentSizeDepth/2:] // explore the right branch
+			proofIndex -= int(currentSizeDepth) / 2
+			currentSizeDepth /= 2
+			continue
+		}
+		// Hash the right branch
+		proof[depth-i-1], err = HashTreeRoot(schema[currentSizeDepth/2:]...)
+		if err != nil {
+			return nil, err
+		}
+		schema = schema[0 : currentSizeDepth/2] // explore the left branch
+		currentSizeDepth /= 2
+	}
+	return proof, nil
+}

cl/phase1/core/state/raw/hashing.go

@@ -23,6 +23,45 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) {
 	return
 }
 
+func (b *BeaconState) CurrentSyncCommitteeBranch() ([][32]byte, error) {
+	if err := b.computeDirtyLeaves(); err != nil {
+		return nil, err
+	}
+	schema := []interface{}{}
+	for i := 0; i < len(b.leaves); i += 32 {
+		schema = append(schema, b.leaves[i:i+32])
+	}
+	return merkle_tree.MerkleProof(5, 22, schema...)
+}
+
+func (b *BeaconState) NextSyncCommitteeBranch() ([][32]byte, error) {
+	if err := b.computeDirtyLeaves(); err != nil {
+		return nil, err
+	}
+	schema := []interface{}{}
+	for i := 0; i < len(b.leaves); i += 32 {
+		schema = append(schema, b.leaves[i:i+32])
+	}
+	return merkle_tree.MerkleProof(5, 23, schema...)
+}
+
+func (b *BeaconState) FinalityRootBranch() ([][32]byte, error) {
+	if err := b.computeDirtyLeaves(); err != nil {
+		return nil, err
+	}
+	schema := []interface{}{}
+	for i := 0; i < len(b.leaves); i += 32 {
+		schema = append(schema, b.leaves[i:i+32])
+	}
+	proof, err := merkle_tree.MerkleProof(5, 20, schema...)
+	if err != nil {
+		return nil, err
+	}
+
+	proof = append([][32]byte{merkle_tree.Uint64Root(b.finalizedCheckpoint.Epoch())}, proof...)
+	return proof, nil
+}
+
 func preparateRootsForHashing(roots []common.Hash) [][32]byte {
 	ret := make([][32]byte, len(roots))
 	for i := range roots {

cl/phase1/forkchoice/checkpoint_state.go

@@ -69,7 +69,7 @@ func newCheckpointState(beaconConfig *clparams.BeaconChainConfig, anchorPublicKe
 	// Add the post-anchor public keys as surplus
 	for i := len(anchorPublicKeys) / length.Bytes48; i < len(validatorSet); i++ {
 		pos := i - len(anchorPublicKeys)/length.Bytes48
-		copy(publicKeys[pos*length.Bytes48:], validatorSet[i].PublicKeyBytes())
+		copy(publicKeys[pos*length.Bytes48:(pos+1)*length.Bytes48], validatorSet[i].PublicKeyBytes())
 	}
 
 	mixes := solid.NewHashVector(randaoMixesLength)
@@ -170,7 +170,7 @@ func (c *checkpointState) isValidIndexedAttestation(att *cltypes.IndexedAttestat
 			pks = append(pks, c.anchorPublicKeys[v*length.Bytes48:(v+1)*length.Bytes48])
 		} else {
 			offset := uint64(len(c.anchorPublicKeys) / length.Bytes48)
-			pks = append(pks, c.publicKeys[(v-offset)*length.Bytes48:])
+			pks = append(pks, c.publicKeys[(v-offset)*length.Bytes48:(v-offset+1)*length.Bytes48])
 		}
 		return true
 	})

cl/phase1/forkchoice/forkchoice.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sort"
 	"sync"
+	"sync/atomic"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes/solid"
@@ -115,6 +116,8 @@ type ForkChoiceStore struct {
 	// operations pool
 	operationsPool pool.OperationsPool
 	beaconCfg      *clparams.BeaconChainConfig
+
+	synced atomic.Bool
 }
 
 type LatestMessage struct {
@@ -469,3 +472,15 @@ func (f *ForkChoiceStore) ForkNodes() []ForkNode {
 	})
 	return forkNodes
 }
+
+func (f *ForkChoiceStore) Synced() bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.synced.Load()
+}
+
+func (f *ForkChoiceStore) SetSynced(s bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.synced.Store(s)
+}

cl/phase1/forkchoice/forkchoice_mock.go

@@ -225,3 +225,11 @@ func (f *ForkChoiceStorageMock) OnAggregateAndProof(aggregateAndProof *cltypes.S
 	f.Pool.AttestationsPool.Insert(aggregateAndProof.Message.Aggregate.Signature(), aggregateAndProof.Message.Aggregate)
 	return nil
 }
+
+func (f *ForkChoiceStorageMock) Synced() bool {
+	return true
+}
+
+func (f *ForkChoiceStorageMock) SetSynced(synced bool) {
+	panic("implement me")
+}

cl/phase1/forkchoice/interface.go

@@ -41,6 +41,7 @@ type ForkChoiceStorageReader interface {
 	GetStateAtSlot(slot uint64, alwaysCopy bool) (*state.CachingBeaconState, error)
 	GetStateAtStateRoot(root libcommon.Hash, alwaysCopy bool) (*state.CachingBeaconState, error)
 	ForkNodes() []ForkNode
+	Synced() bool
 }
 
 type ForkChoiceStorageWriter interface {
@@ -52,4 +53,5 @@ type ForkChoiceStorageWriter interface {
 	OnBlsToExecutionChange(signedChange *cltypes.SignedBLSToExecutionChange, test bool) error
 	OnBlock(block *cltypes.SignedBeaconBlock, newPayload bool, fullValidation bool) error
 	OnTick(time uint64)
+	SetSynced(synced bool)
 }

cl/phase1/forkchoice/on_attestation.go

@@ -15,6 +15,9 @@ import (
 
 // OnAttestation processes incoming attestations.
 func (f *ForkChoiceStore) OnAttestation(attestation *solid.Attestation, fromBlock bool, insert bool) error {
+	if !f.synced.Load() {
+		return nil
+	}
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.headHash = libcommon.Hash{}
@@ -70,6 +73,9 @@ func (f *ForkChoiceStore) OnAttestation(attestation *solid.Attestation, fromBloc
 }
 
 func (f *ForkChoiceStore) OnAggregateAndProof(aggregateAndProof *cltypes.SignedAggregateAndProof, test bool) error {
+	if !f.synced.Load() {
+		return nil
+	}
 	slot := aggregateAndProof.Message.Aggregate.AttestantionData().Slot()
 	selectionProof := aggregateAndProof.Message.SelectionProof
 	committeeIndex := aggregateAndProof.Message.Aggregate.AttestantionData().ValidatorIndex()

cl/phase1/stages/clstages.go

@@ -507,6 +507,7 @@ func ConsensusClStages(ctx context.Context,
 					if err != nil {
 						return err
 					}
+					cfg.forkChoice.SetSynced(true)
 					if err := cfg.syncedData.OnHeadState(headState); err != nil {
 						return err
 					}

cl/spectest/consensus_tests/appendix.go

@@ -11,6 +11,7 @@ import (
 var TestFormats = spectest.Appendix{}
 
 func init() {
+
 	TestFormats.Add("bls").
 		With("aggregate_verify", &BlsAggregateVerify{}).
 		With("aggregate", spectest.UnimplementedHandler).
@@ -47,7 +48,7 @@ func init() {
 	TestFormats.Add("kzg").
 		With("", spectest.UnimplementedHandler)
 	TestFormats.Add("light_client").
-		With("", spectest.UnimplementedHandler)
+		WithFn("single_merkle_proof", LightClientBeaconBlockBodyExecutionMerkleProof)
 	TestFormats.Add("operations").
 		WithFn("attestation", operationAttestationHandler).
 		WithFn("attester_slashing", operationAttesterSlashingHandler).

cl/spectest/consensus_tests/fork_choice.go

@@ -158,6 +158,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err
 
 	forkStore, err := forkchoice.NewForkChoiceStore(context.Background(), anchorState, nil, nil, pool.NewOperationsPool(&clparams.MainnetBeaconConfig), fork_graph.NewForkGraphDisk(anchorState, afero.NewMemMapFs()))
 	require.NoError(t, err)
+	forkStore.SetSynced(true)
 
 	var steps []ForkChoiceStep
 	err = spectest.ReadYml(root, "steps.yaml", &steps)

cl/spectest/consensus_tests/light_client.go

@@ -0,0 +1,59 @@
+package consensus_tests
+
+import (
+	"io/fs"
+	"testing"
+
+	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cl/phase1/core/state"
+	"github.com/ledgerwatch/erigon/spectest"
+	"github.com/stretchr/testify/require"
+)
+
+type LcBranch struct {
+	Branch []string `yaml:"branch"`
+}
+
+var LightClientBeaconBlockBodyExecutionMerkleProof = spectest.HandlerFunc(func(t *testing.T, root fs.FS, c spectest.TestCase) (err error) {
+	var proof [][32]byte
+	switch c.CaseName {
+	case "execution_merkle_proof":
+		beaconBody := cltypes.NewBeaconBody(&clparams.MainnetBeaconConfig)
+		require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, beaconBody))
+		proof, err = beaconBody.ExecutionPayloadMerkleProof()
+		require.NoError(t, err)
+	case "current_sync_committee_merkle_proof":
+		state := state.New(&clparams.MainnetBeaconConfig)
+		require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state))
+		proof, err = state.CurrentSyncCommitteeBranch()
+		require.NoError(t, err)
+	case "next_sync_committee_merkle_proof":
+		state := state.New(&clparams.MainnetBeaconConfig)
+		require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state))
+		proof, err = state.NextSyncCommitteeBranch()
+		require.NoError(t, err)
+	case "finality_root_merkle_proof":
+		state := state.New(&clparams.MainnetBeaconConfig)
+		require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state))
+
+		proof, err = state.FinalityRootBranch()
+		require.NoError(t, err)
+	default:
+		t.Skip("skipping: ", c.CaseName)
+	}
+
+	// read proof.yaml
+	proofYaml := LcBranch{}
+	err = spectest.ReadYml(root, "proof.yaml", &proofYaml)
+	require.NoError(t, err)
+
+	branch := make([][32]byte, len(proofYaml.Branch))
+	for i, b := range proofYaml.Branch {
+		branch[i] = libcommon.HexToHash(b)
+	}
+
+	require.Equal(t, branch, proof)
+	return nil
+})

spectest/consts.go

@@ -1,7 +1,8 @@
 package spectest
 
 const (
-	PreSsz   = "pre.ssz_snappy"
-	PostSsz  = "post.ssz_snappy"
-	MetaYaml = "meta.yaml"
+	PreSsz    = "pre.ssz_snappy"
+	PostSsz   = "post.ssz_snappy"
+	MetaYaml  = "meta.yaml"
+	ObjectSSZ = "object.ssz_snappy"
 )

spectest/util.go

@@ -2,12 +2,13 @@ package spectest
 
 import (
 	"fmt"
+	"io/fs"
+	"os"
+
 	clparams2 "github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
 	"github.com/ledgerwatch/erigon/cl/phase1/core/state"
 	"github.com/ledgerwatch/erigon/cl/utils"
-	"io/fs"
-	"os"
 
 	"gopkg.in/yaml.v3"
 
@@ -80,6 +81,25 @@ func ReadBlock(root fs.FS, version clparams2.StateVersion, index int) (*cltypes.
 
 	return blk, nil
 }
+
+func ReadBlockByPath(root fs.FS, version clparams2.StateVersion, path string) (*cltypes.SignedBeaconBlock, error) {
+	var blockBytes []byte
+	var err error
+	blockBytes, err = fs.ReadFile(root, path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	blk := cltypes.NewSignedBeaconBlock(&clparams2.MainnetBeaconConfig)
+	if err = utils.DecodeSSZSnappy(blk, blockBytes, int(version)); err != nil {
+		return nil, err
+	}
+
+	return blk, nil
+}
+
 func ReadAnchorBlock(root fs.FS, version clparams2.StateVersion, name string) (*cltypes.BeaconBlock, error) {
 	var blockBytes []byte
 	var err error