From daa6d2e741e8aa6622c8bd00fcea4629242003eb Mon Sep 17 00:00:00 2001 From: Potuz Date: Thu, 2 Nov 2023 20:08:16 -0300 Subject: [PATCH] Implement Merkle proof spectests (#13146) Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> --- container/trie/BUILD.bazel | 1 - container/trie/sparse_merkle.go | 8 +- .../mainnet/deneb/merkle_proof/BUILD.bazel | 11 +++ .../deneb/merkle_proof/merkle_proof_test.go | 11 +++ .../minimal/deneb/merkle_proof/BUILD.bazel | 16 ++++ .../deneb/merkle_proof/merkle_proof_test.go | 11 +++ .../shared/common/merkle_proof/BUILD.bazel | 19 +++++ .../merkle_proof/single_merkle_proof.go | 80 +++++++++++++++++++ .../shared/deneb/merkle_proof/BUILD.bazel | 13 +++ .../shared/deneb/merkle_proof/merkle_proof.go | 12 +++ .../shared/deneb/ssz_static/ssz_static.go | 6 +- 11 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 testing/spectest/mainnet/deneb/merkle_proof/BUILD.bazel create mode 100644 testing/spectest/mainnet/deneb/merkle_proof/merkle_proof_test.go create mode 100644 testing/spectest/minimal/deneb/merkle_proof/BUILD.bazel create mode 100644 testing/spectest/minimal/deneb/merkle_proof/merkle_proof_test.go create mode 100644 testing/spectest/shared/common/merkle_proof/BUILD.bazel create mode 100644 testing/spectest/shared/common/merkle_proof/single_merkle_proof.go create mode 100644 testing/spectest/shared/deneb/merkle_proof/BUILD.bazel create mode 100644 testing/spectest/shared/deneb/merkle_proof/merkle_proof.go diff --git a/container/trie/BUILD.bazel b/container/trie/BUILD.bazel index bf8e3d55d..bc6900cbc 100644 --- a/container/trie/BUILD.bazel +++ b/container/trie/BUILD.bazel @@ -11,7 +11,6 @@ go_library( deps = [ "//crypto/hash:go_default_library", "//encoding/bytesutil:go_default_library", - "//math:go_default_library", "//proto/prysm/v1alpha1:go_default_library", "@com_github_pkg_errors//:go_default_library", ], diff --git a/container/trie/sparse_merkle.go b/container/trie/sparse_merkle.go index 55991ad33..0f2dc4cfc 100644 --- a/container/trie/sparse_merkle.go +++ b/container/trie/sparse_merkle.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" "github.com/prysmaticlabs/prysm/v4/crypto/hash" "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil" - "github.com/prysmaticlabs/prysm/v4/math" protodb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1" ) @@ -214,18 +213,15 @@ func VerifyMerkleProofWithDepth(root, item []byte, merkleIndex uint64, proof [][ if uint64(len(proof)) != depth+1 { return false } - if depth >= 64 { - return false // PowerOf2 would overflow. - } node := bytesutil.ToBytes32(item) for i := uint64(0); i <= depth; i++ { - if (merkleIndex / math.PowerOf2(i) % 2) != 0 { + if (merkleIndex & 1) == 1 { node = hash.Hash(append(proof[i], node[:]...)) } else { node = hash.Hash(append(node[:], proof[i]...)) } + merkleIndex /= 2 } - return bytes.Equal(root, node[:]) } diff --git a/testing/spectest/mainnet/deneb/merkle_proof/BUILD.bazel b/testing/spectest/mainnet/deneb/merkle_proof/BUILD.bazel new file mode 100644 index 000000000..b984d12dc --- /dev/null +++ b/testing/spectest/mainnet/deneb/merkle_proof/BUILD.bazel @@ -0,0 +1,11 @@ +load("@prysm//tools/go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = ["merkle_proof_test.go"], + data = glob(["*.yaml"]) + [ + "@consensus_spec_tests_mainnet//:test_data", + ], + tags = ["spectest"], + deps = ["//testing/spectest/shared/deneb/merkle_proof:go_default_library"], +) diff --git a/testing/spectest/mainnet/deneb/merkle_proof/merkle_proof_test.go b/testing/spectest/mainnet/deneb/merkle_proof/merkle_proof_test.go new file mode 100644 index 000000000..1180148e7 --- /dev/null +++ b/testing/spectest/mainnet/deneb/merkle_proof/merkle_proof_test.go @@ -0,0 +1,11 @@ +package merkle_proof + +import ( + "testing" + + "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/deneb/merkle_proof" +) + +func TestMainnet_Deneb_MerkleProof(t *testing.T) { + merkle_proof.RunMerkleProofTests(t, "mainnet") +} diff --git a/testing/spectest/minimal/deneb/merkle_proof/BUILD.bazel b/testing/spectest/minimal/deneb/merkle_proof/BUILD.bazel new file mode 100644 index 000000000..aa0fcbd4d --- /dev/null +++ b/testing/spectest/minimal/deneb/merkle_proof/BUILD.bazel @@ -0,0 +1,16 @@ +load("@prysm//tools/go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + size = "small", + srcs = ["merkle_proof_test.go"], + data = glob(["*.yaml"]) + [ + "@consensus_spec_tests_minimal//:test_data", + ], + eth_network = "minimal", + tags = [ + "minimal", + "spectest", + ], + deps = ["//testing/spectest/shared/deneb/merkle_proof:go_default_library"], +) diff --git a/testing/spectest/minimal/deneb/merkle_proof/merkle_proof_test.go b/testing/spectest/minimal/deneb/merkle_proof/merkle_proof_test.go new file mode 100644 index 000000000..05fe0c1c4 --- /dev/null +++ b/testing/spectest/minimal/deneb/merkle_proof/merkle_proof_test.go @@ -0,0 +1,11 @@ +package merkle_proof + +import ( + "testing" + + "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/deneb/merkle_proof" +) + +func TestMainnet_Deneb_MerkleProof(t *testing.T) { + merkle_proof.RunMerkleProofTests(t, "minimal") +} diff --git a/testing/spectest/shared/common/merkle_proof/BUILD.bazel b/testing/spectest/shared/common/merkle_proof/BUILD.bazel new file mode 100644 index 000000000..de6e35bc5 --- /dev/null +++ b/testing/spectest/shared/common/merkle_proof/BUILD.bazel @@ -0,0 +1,19 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + testonly = True, + srcs = ["single_merkle_proof.go"], + importpath = "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/common/merkle_proof", + visibility = ["//visibility:public"], + deps = [ + "//container/trie:go_default_library", + "//testing/require:go_default_library", + "//testing/spectest/shared/common/ssz_static:go_default_library", + "//testing/spectest/utils:go_default_library", + "//testing/util:go_default_library", + "@com_github_golang_snappy//:go_default_library", + "@com_github_prysmaticlabs_fastssz//:go_default_library", + "@io_bazel_rules_go//go/tools/bazel:go_default_library", + ], +) diff --git a/testing/spectest/shared/common/merkle_proof/single_merkle_proof.go b/testing/spectest/shared/common/merkle_proof/single_merkle_proof.go new file mode 100644 index 000000000..7e6e18cdd --- /dev/null +++ b/testing/spectest/shared/common/merkle_proof/single_merkle_proof.go @@ -0,0 +1,80 @@ +package merkle_proof + +import ( + "encoding/hex" + "os" + "path" + "testing" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/golang/snappy" + fssz "github.com/prysmaticlabs/fastssz" + "github.com/prysmaticlabs/prysm/v4/container/trie" + "github.com/prysmaticlabs/prysm/v4/testing/require" + "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/common/ssz_static" + "github.com/prysmaticlabs/prysm/v4/testing/spectest/utils" + "github.com/prysmaticlabs/prysm/v4/testing/util" +) + +// SingleMerkleProof is the format used to read spectest Merkle Proof test data. +type SingleMerkleProof struct { + Leaf string `json:"leaf"` + LeafIndex uint64 `json:"leaf_index"` + Branch []string `json:"branch"` +} + +func RunMerkleProofTests(t *testing.T, config, forkOrPhase string, unmarshaller ssz_static.Unmarshaller) { + t.Skip("testvectors are not available yet") + runSingleMerkleProofTests(t, config, forkOrPhase, unmarshaller) +} + +func runSingleMerkleProofTests(t *testing.T, config, forkOrPhase string, unmarshaller ssz_static.Unmarshaller) { + require.NoError(t, utils.SetConfig(t, config)) + + testFolders, basePath := utils.TestFolders(t, config, forkOrPhase, "merkle_proof/single_merkle_proof") + + if len(testFolders) == 0 { + t.Fatalf("No test folders found for %s/%s/merkle_proof", config, forkOrPhase) + } + + for _, folder := range testFolders { + typeFolderBase := path.Join(basePath, folder.Name()) + typeFolder, err := bazel.Runfile(typeFolderBase) + require.NoError(t, err) + modeFolders, err := os.ReadDir(typeFolder) + require.NoError(t, err) + + if len(modeFolders) == 0 { + t.Fatalf("No test folders found for %s", typeFolder) + } + + for _, modeFolder := range modeFolders { + t.Run(path.Join(folder.Name(), modeFolder.Name()), func(t *testing.T) { + serializedBytes, err := util.BazelFileBytes(typeFolder, modeFolder.Name(), "object.ssz_snappy") + require.NoError(t, err) + serializedSSZ, err := snappy.Decode(nil /* dst */, serializedBytes) + require.NoError(t, err, "Failed to decompress") + object, err := unmarshaller(t, serializedSSZ, folder.Name()) + require.NoError(t, err, "Could not unmarshall serialized SSZ") + sszObj, ok := object.(fssz.HashRoot) + require.Equal(t, true, ok) + root, err := sszObj.HashTreeRoot() + require.NoError(t, err) + + proofYamlFile, err := util.BazelFileBytes(typeFolder, modeFolder.Name(), "proof.yaml") + require.NoError(t, err) + proof := &SingleMerkleProof{} + require.NoError(t, utils.UnmarshalYaml(proofYamlFile, proof), "Failed to Unmarshal single Merkle proof") + branch := make([][]byte, len(proof.Branch)) + for i, proofRoot := range proof.Branch { + branch[i], err = hex.DecodeString(proofRoot[2:]) + require.NoError(t, err) + } + leaf, err := hex.DecodeString(proof.Leaf[2:]) + require.NoError(t, err) + + require.Equal(t, true, trie.VerifyMerkleProof(root[:], leaf, proof.LeafIndex, branch)) + }) + } + } +} diff --git a/testing/spectest/shared/deneb/merkle_proof/BUILD.bazel b/testing/spectest/shared/deneb/merkle_proof/BUILD.bazel new file mode 100644 index 000000000..c338b0df5 --- /dev/null +++ b/testing/spectest/shared/deneb/merkle_proof/BUILD.bazel @@ -0,0 +1,13 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + testonly = True, + srcs = ["merkle_proof.go"], + importpath = "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/deneb/merkle_proof", + visibility = ["//visibility:public"], + deps = [ + "//testing/spectest/shared/common/merkle_proof:go_default_library", + "//testing/spectest/shared/deneb/ssz_static:go_default_library", + ], +) diff --git a/testing/spectest/shared/deneb/merkle_proof/merkle_proof.go b/testing/spectest/shared/deneb/merkle_proof/merkle_proof.go new file mode 100644 index 000000000..90edfeb2e --- /dev/null +++ b/testing/spectest/shared/deneb/merkle_proof/merkle_proof.go @@ -0,0 +1,12 @@ +package merkle_proof + +import ( + "testing" + + common "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/common/merkle_proof" + "github.com/prysmaticlabs/prysm/v4/testing/spectest/shared/deneb/ssz_static" +) + +func RunMerkleProofTests(t *testing.T, config string) { + common.RunMerkleProofTests(t, config, "deneb", ssz_static.UnmarshalledSSZ) +} diff --git a/testing/spectest/shared/deneb/ssz_static/ssz_static.go b/testing/spectest/shared/deneb/ssz_static/ssz_static.go index b97b1016c..e1f5fe2ac 100644 --- a/testing/spectest/shared/deneb/ssz_static/ssz_static.go +++ b/testing/spectest/shared/deneb/ssz_static/ssz_static.go @@ -15,7 +15,7 @@ import ( // RunSSZStaticTests executes "ssz_static" tests. func RunSSZStaticTests(t *testing.T, config string) { - common.RunSSZStaticTests(t, config, "deneb", unmarshalledSSZ, customHtr) + common.RunSSZStaticTests(t, config, "deneb", UnmarshalledSSZ, customHtr) } func customHtr(t *testing.T, htrs []common.HTR, object interface{}) []common.HTR { @@ -32,8 +32,8 @@ func customHtr(t *testing.T, htrs []common.HTR, object interface{}) []common.HTR return htrs } -// unmarshalledSSZ unmarshalls serialized input. -func unmarshalledSSZ(t *testing.T, serializedBytes []byte, folderName string) (interface{}, error) { +// UnmarshalledSSZ unmarshalls serialized input. +func UnmarshalledSSZ(t *testing.T, serializedBytes []byte, folderName string) (interface{}, error) { var obj interface{} switch folderName { case "ExecutionPayload":