Build herumi's BLS from source (#5055)

* Build herumi from source. Working so far on linux_amd64 for compile, but tests fail to initialize the curve appropriately * Add copts to go_default_library * llvm toolchain, still WIP * Fixes, make llvm a config flag * fix gazelle resolution * comment * comment * update herumi to the v0.9.4 version * Apply @nisdas patch from https://github.com/herumi/bls-eth-go-binary/pull/5
2024-12-22 03:30:35 +00:00 · 2020-03-09 19:22:41 -07:00 · 2020-03-09 19:22:41 -07:00 · edb6590764
commit edb6590764
parent e77cf724b8
10 changed files with 356 additions and 9 deletions
--- a/.bazelrc
+++ b/.bazelrc
@ -35,6 +35,10 @@ build:release --workspace_status_command=./scripts/workspace_status.sh
 build:release --stamp
 build:release --compilation_mode=opt

+# LLVM compiler for building C/C++ dependencies.
+build:llvm --crosstool_top=@llvm_toolchain//:toolchain
+build:llvm --define compiler=llvm
+
 # multi-arch cross-compiling toolchain configs:
 -----------------------------------------------
 build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
--- a/33
+++ b/33
@ -13,6 +13,28 @@ http_archive(
    ],
 )

+http_archive(
+    name = "com_grail_bazel_toolchain",
+    sha256 = "0bec89e35d8a141c87f28cfc506d6d344785c8eb2ff3a453140a1fe972ada79d",
+    strip_prefix = "bazel-toolchain-77a87103145f86f03f90475d19c2c8854398a444",
+    urls = ["https://github.com/grailbio/bazel-toolchain/archive/77a87103145f86f03f90475d19c2c8854398a444.tar.gz"],
+)
+
+load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+
+bazel_toolchain_dependencies()
+
+load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+
+llvm_toolchain(
+    name = "llvm_toolchain",
+    llvm_version = "9.0.0",
+)
+
+load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
+
+llvm_register_toolchains()
+
 load("@prysm//tools/cross-toolchain:prysm_toolchains.bzl", "configure_prysm_toolchains")

 configure_prysm_toolchains()
@ -95,6 +117,10 @@ load(

 container_repositories()

+load("@prysm//third_party/herumi:herumi.bzl", "bls_dependencies")
+
+bls_dependencies()
+
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")

 go_rules_dependencies()
@ -202,13 +228,6 @@ http_archive(
    url = "https://github.com/bazelbuild/buildtools/archive/bf564b4925ab5876a3f64d8b90fab7f769013d42.zip",
 )

-http_archive(
-    name = "com_github_herumi_bls_eth_go_binary",
-    sha256 = "b5628a95bd1e6ff84f73d87c134bb1e7e9c1a5a2a10b831867d9dad7d8defc3e",
-    strip_prefix = "bls-go-binary-8ee33d1a2e8ba8dcf0c3d0b459d75d42d163339d",
-    url = "https://github.com/nisdas/bls-go-binary/archive/8ee33d1a2e8ba8dcf0c3d0b459d75d42d163339d.zip",
-)
-
 load("@com_github_bazelbuild_buildtools//buildifier:deps.bzl", "buildifier_dependencies")

 buildifier_dependencies()
--- a/shared/bls/BUILD.bazel
+++ b/shared/bls/BUILD.bazel
@ -1,5 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

+# gazelle:resolve go github.com/herumi/bls-eth-go-binary/bls @herumi_bls_eth_go_binary//:go_default_library
+
 go_library(
    name = "go_default_library",
    srcs = ["bls.go"],
@ -11,8 +13,8 @@ go_library(
        "//shared/hashutil:go_default_library",
        "//shared/params:go_default_library",
        "@com_github_dgraph_io_ristretto//:go_default_library",
-        "@com_github_herumi_bls_eth_go_binary//bls:go_default_library",
        "@com_github_pkg_errors//:go_default_library",
+        "@herumi_bls_eth_go_binary//:go_default_library",
    ],
 )

--- a/shared/bls/spectest/BUILD.bazel
+++ b/shared/bls/spectest/BUILD.bazel
@ -36,6 +36,6 @@ go_test(
        "//shared/bytesutil:go_default_library",
        "//shared/testutil:go_default_library",
        "@com_github_ghodss_yaml//:go_default_library",
-        "@com_github_herumi_bls_eth_go_binary//bls:go_default_library",
+        "@herumi_bls_eth_go_binary//:go_default_library",
    ],
 )
--- a/third_party/herumi/BUILD.bazel
+++ b/third_party/herumi/BUILD.bazel
--- a/third_party/herumi/bls.BUILD
+++ b/third_party/herumi/bls.BUILD
@ -0,0 +1,24 @@
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "bls_c384_256",
+    srcs = [
+        "src/bls_c384_256.cpp",
+    ],
+    deps = [
+        "@herumi_mcl//:bn",
+    ],
+    hdrs = [
+        "include/bls/bls.h",
+        "src/bls_c_impl.hpp",
+        "src/qcoeff-bn254.hpp",
+    ],
+    includes = [
+        "include",
+    ],
+    copts = [
+        "-DBLS_SWAP_G",
+        "-DBLS_ETH",
+        "-std=c++03",
+    ],
+)
--- a/third_party/herumi/bls_eth_go_binary.BUILD
+++ b/third_party/herumi/bls_eth_go_binary.BUILD
@ -0,0 +1,119 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+OPTS = [
+    "-DMCL_USE_VINT",
+    "-DMCL_DONT_USE_OPENSSL",
+    "-DMCL_LLVM_BMI2=0",
+    "-DMCL_USE_LLVM=1",
+    "-DMCL_VINT_FIXED_BUFFER",
+    "-DMCL_SIZEOF_UNIT=8",
+    "-DMCL_MAX_BIT_SIZE=384",
+    "-DCYBOZU_DONT_USE_EXCEPTION",
+    "-DCYBOZU_DONT_USE_STRING",
+    "-DBLS_SWAP_G",
+    "-DBLS_ETH",
+]
+
+genrule(
+    name = "base64_ll",
+    outs = ["src/base64.ll"],  # llvm assembly language file.
+    tools = [
+        "@herumi_mcl//:src_gen",
+    ],
+    cmd = "touch func.list && $(location @herumi_mcl//:src_gen) -u 64 -f func.list > $@",
+)
+
+genrule(
+    name = "base64_o",
+    srcs = [
+        "src/base64.ll",
+    ],
+    outs = ["base64.o"],
+    cmd = "external/llvm_toolchain/bin/clang++ -c -o $@ $(location src/base64.ll)",
+    tools = ["@llvm_toolchain//:clang"],
+)
+
+cc_library(
+    name = "lib",
+    srcs = [
+        "@herumi_mcl//:src/fp.cpp",
+        "@herumi_bls//:src/bls_c384_256.cpp",
+        "@herumi_bls//:src/bls_c_impl.hpp",
+        ":base64_o",
+    ],
+    deps = ["@herumi_mcl//:bn"],
+    includes = [
+        "bls/include",
+    ],
+    hdrs = [
+        "bls/include/bls/bls.h",
+        "bls/include/bls/bls384_256.h",
+        "bls/include/mcl/bn.h",
+        "bls/include/mcl/bn_c384_256.h",
+        "@herumi_mcl//:include/mcl/curve_type.h",
+    ],
+    copts = OPTS + [
+        "-std=c++03",
+    ],
+)
+
+cc_library(
+    name = "precompiled",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:android_arm": [
+            "bls/lib/android/armeabi-v7a/libbls384_256.a",
+        ],
+        "@io_bazel_rules_go//go/platform:linux_arm64": [
+            "bls/lib/android/arm64-v8a/libbls384_256.a",
+        ],
+        "@io_bazel_rules_go//go/platform:android_arm64": [
+            "bls/lib/android/arm64-v8a/libbls384_256.a",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin_amd64": [
+            "bls/lib/darwin/amd64/libbls384_256.a",
+        ],
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "bls/lib/linux/amd64/libbls384_256.a",
+        ],
+        "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "bls/lib/windows/amd64/libbls384_256.a",
+        ],
+        "//conditions:default": [],
+    }),
+    hdrs = [
+        "bls/include/bls/bls.h",
+        "bls/include/bls/bls384_256.h",
+        "bls/include/mcl/bn.h",
+        "bls/include/mcl/bn_c384_256.h",
+        "bls/include/mcl/curve_type.h",
+    ],
+    includes = [
+        "bls/include",
+    ],
+    deprecation = "Using precompiled BLS archives. To build BLS from source with llvm, use --config=llvm.",
+)
+
+config_setting(
+    name = "llvm_compiler_enabled",
+    define_values = {
+        "compiler": "llvm",
+    },
+)
+
+go_library(
+    name = "go_default_library",
+    importpath = "github.com/herumi/bls-eth-go-binary/bls",
+    srcs = [
+        "bls/bls.go",
+        "bls/callback.go",
+        "bls/cast.go",
+        "bls/mcl.go",
+    ],
+    cdeps = select({
+        ":llvm_compiler_enabled": [":lib"],
+        "//conditions:default": [":precompiled"],
+    }),
+    copts = OPTS,
+    cgo = True,
+    visibility = ["//visibility:public"],
+)
--- a/third_party/herumi/bls_eth_go_binary_serialization_alloc_fix.patch
+++ b/third_party/herumi/bls_eth_go_binary_serialization_alloc_fix.patch
@ -0,0 +1,49 @@
+diff --git a/bls/bls.go b/bls/bls.go
+index bc3b607..f6fa95f 100644
+--- a/bls/bls.go
+++ b/bls/bls.go
+@@ -157,7 +157,7 @@ type SecretKey struct {
+ 
+ // Serialize --
+ func (sec *SecretKey) Serialize() []byte {
+-	buf := make([]byte, 2048)
+	buf := make([]byte, 32)
+ 	// #nosec
+ 	n := C.blsSecretKeySerialize(unsafe.Pointer(&buf[0]), C.mclSize(len(buf)), &sec.v)
+ 	if n == 0 {
+@@ -354,7 +354,7 @@ func (keys PublicKeys) JSON() string {
+ 
+ // Serialize --
+ func (pub *PublicKey) Serialize() []byte {
+-	buf := make([]byte, 2048)
+	buf := make([]byte, 48)
+ 	// #nosec
+ 	n := C.blsPublicKeySerialize(unsafe.Pointer(&buf[0]), C.mclSize(len(buf)), &pub.v)
+ 	if n == 0 {
+@@ -452,7 +452,7 @@ type Sign struct {
+ 
+ // Serialize --
+ func (sig *Sign) Serialize() []byte {
+-	buf := make([]byte, 2048)
+	buf := make([]byte, 96)
+ 	// #nosec
+ 	n := C.blsSignatureSerialize(unsafe.Pointer(&buf[0]), C.mclSize(len(buf)), &sig.v)
+ 	if n == 0 {
+@@ -665,7 +665,7 @@ func (sig *Sign) VerifyHashWithDomain(pub *PublicKey, hashWithDomain []byte) boo
+ 
+ // SerializeUncompressed --
+ func (pub *PublicKey) SerializeUncompressed() []byte {
+-	buf := make([]byte, 2048)
+	buf := make([]byte, 96)
+ 	// #nosec
+ 	n := C.blsPublicKeySerializeUncompressed(unsafe.Pointer(&buf[0]), C.mclSize(len(buf)), &pub.v)
+ 	if n == 0 {
+@@ -676,7 +676,7 @@ func (pub *PublicKey) SerializeUncompressed() []byte {
+ 
+ // SerializeUncompressed --
+ func (sig *Sign) SerializeUncompressed() []byte {
+-	buf := make([]byte, 2048)
+	buf := make([]byte, 192)
+ 	// #nosec
+ 	n := C.blsSignatureSerializeUncompressed(unsafe.Pointer(&buf[0]), C.mclSize(len(buf)), &sig.v)
+ 	if n == 0 {
--- a/third_party/herumi/herumi.bzl
+++ b/third_party/herumi/herumi.bzl
@ -0,0 +1,50 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+"""
+Herumi's BLS library for go depends on
+- herumi/mcl
+- herumi/bls
+- herumi/bls-eth-go-binary
+"""
+
+def bls_dependencies():
+    # TODO(4804): Update herumi_bls_eth_go_binary and herumi_bls to latest supporting v0.10.0.
+    _maybe(
+        http_archive,
+        name = "herumi_bls_eth_go_binary",
+        strip_prefix = "bls-eth-go-binary-147ed25f233ed0b211e711ed8271606540c58064",
+        urls = [
+            "https://github.com/herumi/bls-eth-go-binary/archive/147ed25f233ed0b211e711ed8271606540c58064.tar.gz",
+        ],
+        sha256 = "bbd04f3354f12982e4ef32c62eb13ceb183303ada1ee69e2869553ed35134321",
+        build_file = "@prysm//third_party/herumi:bls_eth_go_binary.BUILD",
+        # TODO(4804): Delete this patch after updating this archive to commit 381c62473c28af84f424cfb1521c97e48289174a or later.
+        patches = [
+            "@prysm//third_party/herumi:bls_eth_go_binary_serialization_alloc_fix.patch",  # Integrates changes from PR #5.
+        ],
+        patch_args = ["-p1"],
+    )
+    _maybe(
+        http_archive,
+        name = "herumi_mcl",
+        strip_prefix = "mcl-1b043ade54bf7e30b8edc29eb01410746ba92d3d",
+        urls = [
+            "https://github.com/herumi/mcl/archive/1b043ade54bf7e30b8edc29eb01410746ba92d3d.tar.gz",
+        ],
+        sha256 = "306bf22b747db174390bbe43de503131b0b5b75bbe586d44f3465c16bda8d28a",
+        build_file = "@prysm//third_party/herumi:mcl.BUILD",
+    )
+    _maybe(
+        http_archive,
+        name = "herumi_bls",
+        strip_prefix = "bls-b0e010004293a7ffd2a626edc2062950abd09938",
+        urls = [
+            "https://github.com/herumi/bls/archive/b0e010004293a7ffd2a626edc2062950abd09938.tar.gz",
+        ],
+        sha256 = "c7300970c8a639cbbe7465d10f412d6c6ab162b15f2e184b191c9763c2241da4",
+        build_file = "@prysm//third_party/herumi:bls.BUILD",
+    )
+
+def _maybe(repo_rule, name, **kwargs):
+    if name not in native.existing_rules():
+        repo_rule(name = name, **kwargs)
--- a/third_party/herumi/mcl.BUILD
+++ b/third_party/herumi/mcl.BUILD
@ -0,0 +1,80 @@
+package(default_visibility = ["//visibility:public"])
+
+MCL_OPTS = [
+    "-DMCL_USE_VINT",
+    "-DMCL_DONT_USE_OPENSSL",
+    "-DMCL_LLVM_BMI2=0",
+    "-DMCL_USE_LLVM=1",
+    "-DMCL_VINT_FIXED_BUFFER",
+    "-DMCL_SIZEOF_UNIT=8",
+    "-DMCL_MAX_BIT_SIZE=384",
+    "-DCYBOZU_DONT_USE_EXCEPTION",
+    "-DCYBOZU_DONT_USE_STRING",
+    "-std=c++03 ",
+]
+
+cc_library(
+    name = "fp",
+    srcs = [
+        "src/fp.cpp",
+        "src/asm/x86-64.s",
+    ],
+    includes = [
+        "include",
+    ],
+    hdrs = glob([
+        "src/xbyak/*.h",
+        "include/cybozu/*.hpp",
+    ]) + [
+        "include/mcl/op.hpp",
+        "include/mcl/gmp_util.hpp",
+        "include/mcl/vint.hpp",
+        "include/mcl/randgen.hpp",
+        "include/mcl/array.hpp",
+        "include/mcl/util.hpp",
+        "include/mcl/fp_tower.hpp",
+        "include/mcl/fp.hpp",
+        "include/mcl/conversion.hpp",
+        "src/low_func.hpp",
+        "src/fp_generator.hpp",
+        "src/proto.hpp",
+        "src/low_func_llvm.hpp",
+    ],
+    copts = MCL_OPTS,
+)
+
+cc_library(
+    name = "bn",
+    srcs = [
+        "src/bn_c384_256.cpp",
+    ],
+    deps = [":fp"],
+    hdrs = [
+        "include/mcl/bn.h",
+        "include/mcl/curve_type.h",
+        "include/mcl/impl/bn_c_impl.hpp",
+        "include/mcl/bls12_381.hpp",
+        "include/mcl/bn_c384_256.h",
+        "include/mcl/ec.hpp",
+        "include/mcl/mapto_wb19.hpp",
+        "include/mcl/ecparam.hpp",
+        "include/mcl/lagrange.hpp",
+        "include/mcl/bn.hpp",
+        "include/mcl/operator.hpp",
+    ],
+    includes = ["include"],
+    copts = MCL_OPTS,
+)
+
+# src_gen is a tool to generate some llvm assembly language file.
+cc_binary(
+    name = "src_gen",
+    srcs = [
+        "src/gen.cpp",
+        "src/llvm_gen.hpp",
+    ] + glob([
+        "include/cybozu/*.hpp",
+        "include/mcl/*.hpp",
+    ]),
+    includes = ["include"],
+)