Add hermetic_cc_toolchain for a hermetic cc toolchain (#12135)

* Add bazel-zig-cc for a hermetic cc toolchain

* gazelle

* Remove llvm

* remove wl

* Add new URLs for renamed repo

* gazelle

* Update to v2.0.0-rc1

* bump to rc2

* Some PR feedback

* use v2.0.0 from rc2

* Disable hermetic builds for mac and windows.

* bump bazel version, add darwin hack

* fix

* Add the no-op emtpy cc toolchain code

* typo and additional copy

* update protobuf and fix vaticle warning

* Revert "update protobuf and fix vaticle warning"

This reverts commit 7bb4b6b564d2d588567d08dfdd3b9837c26d3c05.

---------

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

.bazelrc

@@ -3,6 +3,7 @@ import %workspace%/build/bazelrc/convenience.bazelrc
 import %workspace%/build/bazelrc/correctness.bazelrc
 import %workspace%/build/bazelrc/cross.bazelrc
 import %workspace%/build/bazelrc/debug.bazelrc
+import %workspace%/build/bazelrc/hermetic-cc.bazelrc
 import %workspace%/build/bazelrc/performance.bazelrc
 
 # E2E run with debug gotag
@@ -27,30 +28,7 @@ build:minimal --@io_bazel_rules_go//go/config:tags=minimal
 build:release --compilation_mode=opt
 build:release --stamp
 
-# LLVM compiler for building C/C++ dependencies.
-build:llvm --define compiler=llvm
-build:llvm --copt -fno-sanitize=vptr,function
-build:llvm --linkopt -fno-sanitize=vptr,function
-# --incompatible_enable_cc_toolchain_resolution not needed after this issue is closed https://github.com/bazelbuild/bazel/issues/7260
-build:llvm --incompatible_enable_cc_toolchain_resolution
-
-build:asan --copt -fsanitize=address,undefined
-build:asan --copt -fno-omit-frame-pointer
-build:asan --linkopt -fsanitize=address,undefined
-build:asan --copt -fno-sanitize=vptr,function
-build:asan --linkopt -fno-sanitize=vptr,function
-build:asan --copt -DADDRESS_SANITIZER=1
-build:asan --copt -D__SANITIZE_ADDRESS__
-build:asan --linkopt -ldl
-
-build:llvm-asan --config=llvm
-build:llvm-asan --config=asan
-build:llvm-asan --linkopt -fuse-ld=ld.lld
-
-build:fuzz --@io_bazel_rules_go//go/config:tags=fuzz
-
 # Build binary with cgo symbolizer for debugging / profiling.
-build:cgo_symbolizer --config=llvm
 build:cgo_symbolizer --copt=-g
 build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
 build:cgo_symbolizer -c dbg
@@ -59,7 +37,7 @@ build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
 # toolchain build debug configs
 #------------------------------
 build:debug --sandbox_debug
-build:debug --toolchain_resolution_debug
+build:debug --toolchain_resolution_debug=".*"
 build:debug --verbose_failures
 build:debug -s
 

.bazelversion

@@ -1 +1 @@
-6.1.0
+6.2.1

WORKSPACE

@@ -16,27 +16,37 @@ load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 
 rules_pkg_dependencies()
 
+HERMETIC_CC_TOOLCHAIN_VERSION = "v2.0.0"
+
 http_archive(
-    name = "com_grail_bazel_toolchain",
+    name = "hermetic_cc_toolchain",
-    sha256 = "b210fc8e58782ef171f428bfc850ed7179bdd805543ebd1aa144b9c93489134f",
+    sha256 = "57f03a6c29793e8add7bd64186fc8066d23b5ffd06fe9cc6b0b8c499914d3a65",
-    strip_prefix = "bazel-toolchain-83e69ba9e4b4fdad0d1d057fcb87addf77c281c9",
+    urls = [
-    urls = ["https://github.com/grailbio/bazel-toolchain/archive/83e69ba9e4b4fdad0d1d057fcb87addf77c281c9.tar.gz"],
+        "https://mirror.bazel.build/github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(HERMETIC_CC_TOOLCHAIN_VERSION),
+        "https://github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(HERMETIC_CC_TOOLCHAIN_VERSION),
+    ],
 )
 
-load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains")
 
-bazel_toolchain_dependencies()
+zig_toolchains()
 
-load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+# Register zig sdk toolchains with support for Ubuntu 20.04 (Focal Fossa) which has an EOL date of April, 2025.
-
+# For ubuntu glibc support, see https://launchpad.net/ubuntu/+source/glibc
-llvm_toolchain(
+register_toolchains(
-    name = "llvm_toolchain",
+    "@zig_sdk//toolchain:linux_amd64_gnu.2.31",
-    llvm_version = "13.0.1",
+    "@zig_sdk//toolchain:linux_arm64_gnu.2.31",
+    # Hermetic cc toolchain is not yet supported on darwin. Sysroot needs to be provided.
+    # See https://github.com/uber/hermetic_cc_toolchain#osx-sysroot
+    #    "@zig_sdk//toolchain:darwin_amd64",
+    #    "@zig_sdk//toolchain:darwin_arm64",
+    # Windows builds are not supported yet.
+    #    "@zig_sdk//toolchain:windows_amd64",
 )
 
-load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
+load("@prysm//tools/cross-toolchain:darwin_cc_hack.bzl", "configure_nonhermetic_darwin")
 
-llvm_register_toolchains()
+configure_nonhermetic_darwin()
 
 load("@prysm//tools/cross-toolchain:prysm_toolchains.bzl", "configure_prysm_toolchains")
 

build/bazelrc/hermetic-cc.bazelrc

@@ -0,0 +1,19 @@
+# From Bazel's perspective, this is almost equivalent to always specifying
+# --extra_toolchains on every bazel <...> command-line invocation. It also
+# means there is no way to disable the toolchain with the command line. This is
+# useful if you find bazel-hermetic-cc useful enough to compile for all of your
+# targets and tools.
+#
+# With BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 Bazel stops detecting the default
+# host toolchain. Configuring toolchains is complicated enough, and the
+# auto-detection (read: fallback to non-hermetic toolchain) is a footgun best
+# avoided. This option is not documented in bazel, so may break. If you intend
+# to use the hermetic toolchain exclusively, it won't hurt.
+build --action_env BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+# This snippet instructs Bazel to use the registered "new kinds of toolchains".
+# This flag not needed after this issue is closed https://github.com/bazelbuild/bazel/issues/7260
+build --incompatible_enable_cc_toolchain_resolution
+
+# Add a no-op warning for users still using --config=llvm
+build:llvm --unconditional_warning="llvm config is no longer used as clang is now the default compiler"

tools/cross-toolchain/darwin_cc_hack.bzl

@@ -0,0 +1,114 @@
+load("@bazel_tools//tools/cpp:unix_cc_configure.bzl", "configure_unix_toolchain")
+load(
+    "@bazel_tools//tools/cpp:lib_cc_configure.bzl",
+    "get_cpu_value",
+    "resolve_labels",
+)
+
+"""
+This file is a copy of https://github.com/bazelbuild/bazel/blob/master/tools/cpp/cc_configure.bzl
+with some minor changes. The original file is licensed under Apache 2.0 license. The gist of this
+is that we want darwin to register the local toolchain and disregard the environment variable of
+BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN. We must support a local toolchain for darwin until
+hermetic_cc_toolchain supports darwin's sysroot in a hermetic way.
+"""
+
+def cc_autoconf_toolchains_impl(repository_ctx):
+    """Generate BUILD file with 'toolchain' targets for the local host C++ toolchain.
+
+    Args:
+      repository_ctx: repository context
+    """
+
+    cpu_value = get_cpu_value(repository_ctx)
+
+    if cpu_value.startswith("darwin"):
+        paths = resolve_labels(repository_ctx, [
+            "@bazel_tools//tools/cpp:BUILD.toolchains.tpl",
+        ])
+        repository_ctx.template(
+            "BUILD",
+            paths["@bazel_tools//tools/cpp:BUILD.toolchains.tpl"],
+            {"%{name}": get_cpu_value(repository_ctx)},
+        )
+    else:
+        repository_ctx.file("BUILD", "# C++ toolchain autoconfiguration was disabled by BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN env variable.")
+
+
+def cc_autoconf_impl(repository_ctx, overriden_tools = dict()):
+    """Generate BUILD file with 'cc_toolchain' targets for the local host C++ toolchain.
+
+    Args:
+       repository_ctx: repository context
+       overriden_tools: dict of tool paths to use instead of autoconfigured tools
+    """
+    cpu_value = get_cpu_value(repository_ctx)
+
+    if cpu_value.startswith("darwin"):
+        print("Configuring local C++ toolchain for Darwin. This is non-hermetic and builds may " + 
+        "not be reproducible. Consider building on linux for a hermetic build.")
+        configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools)
+    else: 
+        paths = resolve_labels(repository_ctx, [
+            "@bazel_tools//tools/cpp:BUILD.empty.tpl",
+            "@bazel_tools//tools/cpp:empty_cc_toolchain_config.bzl",
+        ])
+        repository_ctx.symlink(paths["@bazel_tools//tools/cpp:empty_cc_toolchain_config.bzl"], "cc_toolchain_config.bzl")
+        repository_ctx.template("BUILD", paths["@bazel_tools//tools/cpp:BUILD.empty.tpl"], {
+            "%{cpu}": get_cpu_value(repository_ctx),
+        })
+
+cc_autoconf_toolchains = repository_rule(
+    environ = [
+        "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN",
+    ],
+    implementation = cc_autoconf_toolchains_impl,
+    configure = True,
+)
+
+cc_autoconf = repository_rule(
+    environ = [
+        "ABI_LIBC_VERSION",
+        "ABI_VERSION",
+        "BAZEL_COMPILER",
+        "BAZEL_HOST_SYSTEM",
+        "BAZEL_CONLYOPTS",
+        "BAZEL_CXXOPTS",
+        "BAZEL_LINKOPTS",
+        "BAZEL_LINKLIBS",
+        "BAZEL_LLVM_COV",
+        "BAZEL_LLVM_PROFDATA",
+        "BAZEL_PYTHON",
+        "BAZEL_SH",
+        "BAZEL_TARGET_CPU",
+        "BAZEL_TARGET_LIBC",
+        "BAZEL_TARGET_SYSTEM",
+        "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN",
+        "BAZEL_USE_LLVM_NATIVE_COVERAGE",
+        "BAZEL_LLVM",
+        "BAZEL_IGNORE_SYSTEM_HEADERS_VERSIONS",
+        "USE_CLANG_CL",
+        "CC",
+        "CC_CONFIGURE_DEBUG",
+        "CC_TOOLCHAIN_NAME",
+        "CPLUS_INCLUDE_PATH",
+        "DEVELOPER_DIR",
+        "GCOV",
+        "LIBTOOL",
+        "HOMEBREW_RUBY_PATH",
+        "SYSTEMROOT",
+        "USER",
+    ],
+    implementation = cc_autoconf_impl,
+    configure = True,
+)
+
+def configure_nonhermetic_darwin():
+    """A C++ configuration rules that generate the crosstool file."""
+    cc_autoconf_toolchains(name = "local_config_cc_toolchains")
+    cc_autoconf(name = "local_config_cc")
+    native.bind(name = "cc_toolchain", actual = "@local_config_cc//:toolchain")
+    native.register_toolchains(
+        # Use register_toolchain's target pattern expansion to register all toolchains in the package.
+        "@local_config_cc_toolchains//:all",
+    )