From f03d2665ff9f62b1071bbda80063b0511f322105 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 15 Jan 2024 15:01:33 +0100 Subject: [PATCH] Added merkle proof and fixed bad handling of new validators (#9233) --- cl/cltypes/beacon_block.go | 4 ++ cl/merkle_tree/merkle_root.go | 42 +++++++++++++++ cl/phase1/core/state/raw/hashing.go | 39 ++++++++++++++ cl/phase1/forkchoice/checkpoint_state.go | 4 +- cl/phase1/forkchoice/forkchoice.go | 15 ++++++ cl/phase1/forkchoice/forkchoice_mock.go | 8 +++ cl/phase1/forkchoice/interface.go | 2 + cl/phase1/forkchoice/on_attestation.go | 6 +++ cl/phase1/stages/clstages.go | 1 + cl/spectest/consensus_tests/appendix.go | 3 +- cl/spectest/consensus_tests/fork_choice.go | 1 + cl/spectest/consensus_tests/light_client.go | 59 +++++++++++++++++++++ spectest/consts.go | 7 +-- spectest/util.go | 24 ++++++++- 14 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 cl/spectest/consensus_tests/light_client.go diff --git a/cl/cltypes/beacon_block.go b/cl/cltypes/beacon_block.go index c2592f452..d97638037 100644 --- a/cl/cltypes/beacon_block.go +++ b/cl/cltypes/beacon_block.go @@ -293,3 +293,7 @@ func (*BeaconBody) Static() bool { func (*BeaconBlock) Static() bool { return false } + +func (b *BeaconBody) ExecutionPayloadMerkleProof() ([][32]byte, error) { + return merkle_tree.MerkleProof(4, 9, b.getSchema(false)...) +} diff --git a/cl/merkle_tree/merkle_root.go b/cl/merkle_tree/merkle_root.go index ba231d485..a4b9791f9 100644 --- a/cl/merkle_tree/merkle_root.go +++ b/cl/merkle_tree/merkle_root.go @@ -10,6 +10,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/types/ssz" + "github.com/ledgerwatch/erigon/cl/utils" "github.com/prysmaticlabs/gohashtree" ) @@ -116,3 +117,44 @@ func MerkleRootFromFlatLeaves(leaves []byte, out []byte) (err error) { func MerkleRootFromFlatLeavesWithLimit(leaves []byte, out []byte, limit uint64) (err error) { return globalHasher.merkleizeTrieLeavesFlat(leaves, out, limit) } + +// Merkle Proof computes the merkle proof for a given schema of objects. +func MerkleProof(depth, proofIndex int, schema ...interface{}) ([][32]byte, error) { + // Calculate the total number of leaves needed based on the schema length + maxDepth := GetDepth(uint64(len(schema))) + if utils.PowerOf2(uint64(maxDepth)) != uint64(len(schema)) { + maxDepth++ + } + + if depth != int(maxDepth) { // TODO: Add support for lower depths + return nil, fmt.Errorf("depth is different than maximum depth, have %d, want %d", depth, maxDepth) + } + var err error + proof := make([][32]byte, maxDepth) + currentSizeDepth := utils.PowerOf2(uint64(maxDepth)) + for len(schema) != int(currentSizeDepth) { // Augment the schema to be a power of 2 + schema = append(schema, make([]byte, 32)) + } + + for i := 0; i < depth; i++ { + // Hash the left branch + if proofIndex >= int(currentSizeDepth)/2 { + proof[depth-i-1], err = HashTreeRoot(schema[0 : currentSizeDepth/2]...) + if err != nil { + return nil, err + } + schema = schema[currentSizeDepth/2:] // explore the right branch + proofIndex -= int(currentSizeDepth) / 2 + currentSizeDepth /= 2 + continue + } + // Hash the right branch + proof[depth-i-1], err = HashTreeRoot(schema[currentSizeDepth/2:]...) + if err != nil { + return nil, err + } + schema = schema[0 : currentSizeDepth/2] // explore the left branch + currentSizeDepth /= 2 + } + return proof, nil +} diff --git a/cl/phase1/core/state/raw/hashing.go b/cl/phase1/core/state/raw/hashing.go index 72c840fbf..7be41d678 100644 --- a/cl/phase1/core/state/raw/hashing.go +++ b/cl/phase1/core/state/raw/hashing.go @@ -23,6 +23,45 @@ func (b *BeaconState) HashSSZ() (out [32]byte, err error) { return } +func (b *BeaconState) CurrentSyncCommitteeBranch() ([][32]byte, error) { + if err := b.computeDirtyLeaves(); err != nil { + return nil, err + } + schema := []interface{}{} + for i := 0; i < len(b.leaves); i += 32 { + schema = append(schema, b.leaves[i:i+32]) + } + return merkle_tree.MerkleProof(5, 22, schema...) +} + +func (b *BeaconState) NextSyncCommitteeBranch() ([][32]byte, error) { + if err := b.computeDirtyLeaves(); err != nil { + return nil, err + } + schema := []interface{}{} + for i := 0; i < len(b.leaves); i += 32 { + schema = append(schema, b.leaves[i:i+32]) + } + return merkle_tree.MerkleProof(5, 23, schema...) +} + +func (b *BeaconState) FinalityRootBranch() ([][32]byte, error) { + if err := b.computeDirtyLeaves(); err != nil { + return nil, err + } + schema := []interface{}{} + for i := 0; i < len(b.leaves); i += 32 { + schema = append(schema, b.leaves[i:i+32]) + } + proof, err := merkle_tree.MerkleProof(5, 20, schema...) + if err != nil { + return nil, err + } + + proof = append([][32]byte{merkle_tree.Uint64Root(b.finalizedCheckpoint.Epoch())}, proof...) + return proof, nil +} + func preparateRootsForHashing(roots []common.Hash) [][32]byte { ret := make([][32]byte, len(roots)) for i := range roots { diff --git a/cl/phase1/forkchoice/checkpoint_state.go b/cl/phase1/forkchoice/checkpoint_state.go index ba992c6d1..612068945 100644 --- a/cl/phase1/forkchoice/checkpoint_state.go +++ b/cl/phase1/forkchoice/checkpoint_state.go @@ -69,7 +69,7 @@ func newCheckpointState(beaconConfig *clparams.BeaconChainConfig, anchorPublicKe // Add the post-anchor public keys as surplus for i := len(anchorPublicKeys) / length.Bytes48; i < len(validatorSet); i++ { pos := i - len(anchorPublicKeys)/length.Bytes48 - copy(publicKeys[pos*length.Bytes48:], validatorSet[i].PublicKeyBytes()) + copy(publicKeys[pos*length.Bytes48:(pos+1)*length.Bytes48], validatorSet[i].PublicKeyBytes()) } mixes := solid.NewHashVector(randaoMixesLength) @@ -170,7 +170,7 @@ func (c *checkpointState) isValidIndexedAttestation(att *cltypes.IndexedAttestat pks = append(pks, c.anchorPublicKeys[v*length.Bytes48:(v+1)*length.Bytes48]) } else { offset := uint64(len(c.anchorPublicKeys) / length.Bytes48) - pks = append(pks, c.publicKeys[(v-offset)*length.Bytes48:]) + pks = append(pks, c.publicKeys[(v-offset)*length.Bytes48:(v-offset+1)*length.Bytes48]) } return true }) diff --git a/cl/phase1/forkchoice/forkchoice.go b/cl/phase1/forkchoice/forkchoice.go index bde4d322f..1e8793954 100644 --- a/cl/phase1/forkchoice/forkchoice.go +++ b/cl/phase1/forkchoice/forkchoice.go @@ -4,6 +4,7 @@ import ( "context" "sort" "sync" + "sync/atomic" "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes/solid" @@ -115,6 +116,8 @@ type ForkChoiceStore struct { // operations pool operationsPool pool.OperationsPool beaconCfg *clparams.BeaconChainConfig + + synced atomic.Bool } type LatestMessage struct { @@ -469,3 +472,15 @@ func (f *ForkChoiceStore) ForkNodes() []ForkNode { }) return forkNodes } + +func (f *ForkChoiceStore) Synced() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.synced.Load() +} + +func (f *ForkChoiceStore) SetSynced(s bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.synced.Store(s) +} diff --git a/cl/phase1/forkchoice/forkchoice_mock.go b/cl/phase1/forkchoice/forkchoice_mock.go index 3683e5cc3..665b0f126 100644 --- a/cl/phase1/forkchoice/forkchoice_mock.go +++ b/cl/phase1/forkchoice/forkchoice_mock.go @@ -225,3 +225,11 @@ func (f *ForkChoiceStorageMock) OnAggregateAndProof(aggregateAndProof *cltypes.S f.Pool.AttestationsPool.Insert(aggregateAndProof.Message.Aggregate.Signature(), aggregateAndProof.Message.Aggregate) return nil } + +func (f *ForkChoiceStorageMock) Synced() bool { + return true +} + +func (f *ForkChoiceStorageMock) SetSynced(synced bool) { + panic("implement me") +} diff --git a/cl/phase1/forkchoice/interface.go b/cl/phase1/forkchoice/interface.go index 0b62ff8f3..955bd188d 100644 --- a/cl/phase1/forkchoice/interface.go +++ b/cl/phase1/forkchoice/interface.go @@ -41,6 +41,7 @@ type ForkChoiceStorageReader interface { GetStateAtSlot(slot uint64, alwaysCopy bool) (*state.CachingBeaconState, error) GetStateAtStateRoot(root libcommon.Hash, alwaysCopy bool) (*state.CachingBeaconState, error) ForkNodes() []ForkNode + Synced() bool } type ForkChoiceStorageWriter interface { @@ -52,4 +53,5 @@ type ForkChoiceStorageWriter interface { OnBlsToExecutionChange(signedChange *cltypes.SignedBLSToExecutionChange, test bool) error OnBlock(block *cltypes.SignedBeaconBlock, newPayload bool, fullValidation bool) error OnTick(time uint64) + SetSynced(synced bool) } diff --git a/cl/phase1/forkchoice/on_attestation.go b/cl/phase1/forkchoice/on_attestation.go index 5dd9326b0..3e80efac4 100644 --- a/cl/phase1/forkchoice/on_attestation.go +++ b/cl/phase1/forkchoice/on_attestation.go @@ -15,6 +15,9 @@ import ( // OnAttestation processes incoming attestations. func (f *ForkChoiceStore) OnAttestation(attestation *solid.Attestation, fromBlock bool, insert bool) error { + if !f.synced.Load() { + return nil + } f.mu.Lock() defer f.mu.Unlock() f.headHash = libcommon.Hash{} @@ -70,6 +73,9 @@ func (f *ForkChoiceStore) OnAttestation(attestation *solid.Attestation, fromBloc } func (f *ForkChoiceStore) OnAggregateAndProof(aggregateAndProof *cltypes.SignedAggregateAndProof, test bool) error { + if !f.synced.Load() { + return nil + } slot := aggregateAndProof.Message.Aggregate.AttestantionData().Slot() selectionProof := aggregateAndProof.Message.SelectionProof committeeIndex := aggregateAndProof.Message.Aggregate.AttestantionData().ValidatorIndex() diff --git a/cl/phase1/stages/clstages.go b/cl/phase1/stages/clstages.go index 8c5710bfe..5f5e18c80 100644 --- a/cl/phase1/stages/clstages.go +++ b/cl/phase1/stages/clstages.go @@ -507,6 +507,7 @@ func ConsensusClStages(ctx context.Context, if err != nil { return err } + cfg.forkChoice.SetSynced(true) if err := cfg.syncedData.OnHeadState(headState); err != nil { return err } diff --git a/cl/spectest/consensus_tests/appendix.go b/cl/spectest/consensus_tests/appendix.go index 5a7342cb6..b547533ed 100644 --- a/cl/spectest/consensus_tests/appendix.go +++ b/cl/spectest/consensus_tests/appendix.go @@ -11,6 +11,7 @@ import ( var TestFormats = spectest.Appendix{} func init() { + TestFormats.Add("bls"). With("aggregate_verify", &BlsAggregateVerify{}). With("aggregate", spectest.UnimplementedHandler). @@ -47,7 +48,7 @@ func init() { TestFormats.Add("kzg"). With("", spectest.UnimplementedHandler) TestFormats.Add("light_client"). - With("", spectest.UnimplementedHandler) + WithFn("single_merkle_proof", LightClientBeaconBlockBodyExecutionMerkleProof) TestFormats.Add("operations"). WithFn("attestation", operationAttestationHandler). WithFn("attester_slashing", operationAttesterSlashingHandler). diff --git a/cl/spectest/consensus_tests/fork_choice.go b/cl/spectest/consensus_tests/fork_choice.go index ac33534e0..4eff9f50c 100644 --- a/cl/spectest/consensus_tests/fork_choice.go +++ b/cl/spectest/consensus_tests/fork_choice.go @@ -158,6 +158,7 @@ func (b *ForkChoice) Run(t *testing.T, root fs.FS, c spectest.TestCase) (err err forkStore, err := forkchoice.NewForkChoiceStore(context.Background(), anchorState, nil, nil, pool.NewOperationsPool(&clparams.MainnetBeaconConfig), fork_graph.NewForkGraphDisk(anchorState, afero.NewMemMapFs())) require.NoError(t, err) + forkStore.SetSynced(true) var steps []ForkChoiceStep err = spectest.ReadYml(root, "steps.yaml", &steps) diff --git a/cl/spectest/consensus_tests/light_client.go b/cl/spectest/consensus_tests/light_client.go new file mode 100644 index 000000000..370d7fe79 --- /dev/null +++ b/cl/spectest/consensus_tests/light_client.go @@ -0,0 +1,59 @@ +package consensus_tests + +import ( + "io/fs" + "testing" + + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon/cl/clparams" + "github.com/ledgerwatch/erigon/cl/cltypes" + "github.com/ledgerwatch/erigon/cl/phase1/core/state" + "github.com/ledgerwatch/erigon/spectest" + "github.com/stretchr/testify/require" +) + +type LcBranch struct { + Branch []string `yaml:"branch"` +} + +var LightClientBeaconBlockBodyExecutionMerkleProof = spectest.HandlerFunc(func(t *testing.T, root fs.FS, c spectest.TestCase) (err error) { + var proof [][32]byte + switch c.CaseName { + case "execution_merkle_proof": + beaconBody := cltypes.NewBeaconBody(&clparams.MainnetBeaconConfig) + require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, beaconBody)) + proof, err = beaconBody.ExecutionPayloadMerkleProof() + require.NoError(t, err) + case "current_sync_committee_merkle_proof": + state := state.New(&clparams.MainnetBeaconConfig) + require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state)) + proof, err = state.CurrentSyncCommitteeBranch() + require.NoError(t, err) + case "next_sync_committee_merkle_proof": + state := state.New(&clparams.MainnetBeaconConfig) + require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state)) + proof, err = state.NextSyncCommitteeBranch() + require.NoError(t, err) + case "finality_root_merkle_proof": + state := state.New(&clparams.MainnetBeaconConfig) + require.NoError(t, spectest.ReadSsz(root, c.Version(), spectest.ObjectSSZ, state)) + + proof, err = state.FinalityRootBranch() + require.NoError(t, err) + default: + t.Skip("skipping: ", c.CaseName) + } + + // read proof.yaml + proofYaml := LcBranch{} + err = spectest.ReadYml(root, "proof.yaml", &proofYaml) + require.NoError(t, err) + + branch := make([][32]byte, len(proofYaml.Branch)) + for i, b := range proofYaml.Branch { + branch[i] = libcommon.HexToHash(b) + } + + require.Equal(t, branch, proof) + return nil +}) diff --git a/spectest/consts.go b/spectest/consts.go index 0de7e99f0..36be88ac6 100644 --- a/spectest/consts.go +++ b/spectest/consts.go @@ -1,7 +1,8 @@ package spectest const ( - PreSsz = "pre.ssz_snappy" - PostSsz = "post.ssz_snappy" - MetaYaml = "meta.yaml" + PreSsz = "pre.ssz_snappy" + PostSsz = "post.ssz_snappy" + MetaYaml = "meta.yaml" + ObjectSSZ = "object.ssz_snappy" ) diff --git a/spectest/util.go b/spectest/util.go index b34ae6ced..220a9a2ee 100644 --- a/spectest/util.go +++ b/spectest/util.go @@ -2,12 +2,13 @@ package spectest import ( "fmt" + "io/fs" + "os" + clparams2 "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/erigon/cl/cltypes" "github.com/ledgerwatch/erigon/cl/phase1/core/state" "github.com/ledgerwatch/erigon/cl/utils" - "io/fs" - "os" "gopkg.in/yaml.v3" @@ -80,6 +81,25 @@ func ReadBlock(root fs.FS, version clparams2.StateVersion, index int) (*cltypes. return blk, nil } + +func ReadBlockByPath(root fs.FS, version clparams2.StateVersion, path string) (*cltypes.SignedBeaconBlock, error) { + var blockBytes []byte + var err error + blockBytes, err = fs.ReadFile(root, path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + blk := cltypes.NewSignedBeaconBlock(&clparams2.MainnetBeaconConfig) + if err = utils.DecodeSSZSnappy(blk, blockBytes, int(version)); err != nil { + return nil, err + } + + return blk, nil +} + func ReadAnchorBlock(root fs.FS, version clparams2.StateVersion, name string) (*cltypes.BeaconBlock, error) { var blockBytes []byte var err error