From 88390153122b33ccc1aac3bede4646cbc574a000 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sat, 3 Feb 2024 13:21:21 -0600 Subject: [PATCH] docker: Add coreutils to docker images (#13564) * Add coreutils to docker images * add coreutils dependencies * Add a prysmaticlabs.com/uploads backup of the deb files * Run gazelle and fix issues * Remove broken tar, change http_archive deps to debian_archive, remove http mirrors in favor of snapshot * Add comments about which deps are required by other deps --- WORKSPACE | 11 ++++ distroless_deps.bzl | 17 +++++ tools/BUILD.bazel | 25 -------- tools/image_deps.bzl | 145 ++++++++++++++++++++++++++++++++++-------- tools/prysm_image.bzl | 22 ++++++- 5 files changed, 168 insertions(+), 52 deletions(-) create mode 100644 distroless_deps.bzl diff --git a/WORKSPACE b/WORKSPACE index e4ccc24bb..19b4ea669 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -106,6 +106,13 @@ load("@rules_distroless//distroless:dependencies.bzl", "rules_distroless_depende rules_distroless_dependencies() +http_archive( + name = "distroless", + integrity = "sha256-Cf00kUp1NyXA3LzbdyYy4Kda27wbkB8+A9MliTxq4jE=", + strip_prefix = "distroless-9dc924b9fe812eec2fa0061824dcad39eb09d0d6", + url = "https://github.com/GoogleContainerTools/distroless/archive/9dc924b9fe812eec2fa0061824dcad39eb09d0d6.tar.gz", # 2024-01-24 +) + load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains") aspect_bazel_lib_dependencies() @@ -144,6 +151,10 @@ http_archive( ], ) +load("//:distroless_deps.bzl", "distroless_deps") + +distroless_deps() + # Override default import in rules_go with special patch until # https://github.com/gogo/protobuf/pull/582 is merged. git_repository( diff --git a/distroless_deps.bzl b/distroless_deps.bzl new file mode 100644 index 000000000..1fb6297a9 --- /dev/null +++ b/distroless_deps.bzl @@ -0,0 +1,17 @@ +load("@prysm//tools/go:def.bzl", "go_repository") # gazelle:keep + +def distroless_deps(): + go_repository( + name = "com_github_ulikunitz_xz", + importpath = "github.com/ulikunitz/xz", + sum = "h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=", + version = "v0.5.11", + ) + + go_repository( + name = "com_github_spdx_tools_golang", + importpath = "github.com/spdx/tools-golang", + sum = "h1:9B623Cfs+mclYK6dsae7gLSwuIBHvlgmEup87qpqsAQ=", + version = "v0.3.1-0.20230104082527-d6f58551be3f", + ) + diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 2180de892..17d9e256c 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -32,28 +32,3 @@ pkg_tar( tags = ["manual"], visibility = ["//visibility:public"], ) - -# Create a bash tar layer for docker images. This allows docker images to have access to the "bash" -# command and improves debugging abilities on the image. -genrule( - name = "bash_tar", - srcs = select({ - "@platforms//cpu:x86_64": ["@bash_amd64//file"], - "@platforms//cpu:arm64": ["@bash_arm64//file"], - }), - outs = ["bash.tar"], - cmd = "ar x $< && xz -d data.tar.xz -c >> $@", - visibility = ["//visibility:public"], -) - -# libtinfo6 is required for terminal activity and contains terminfo library. -genrule( - name = "libtinfo6_tar", - srcs = select({ - "@platforms//cpu:x86_64": ["@libtinfo6_amd64//file"], - "@platforms//cpu:arm64": ["@libtinfo6_arm64//file"], - }), - outs = ["libtinfo6.tar"], - cmd = "ar x $< && xz -d data.tar.xz -c >> $@", - visibility = ["//visibility:public"], -) diff --git a/tools/image_deps.bzl b/tools/image_deps.bzl index 1c7b11b1a..621f39fed 100644 --- a/tools/image_deps.bzl +++ b/tools/image_deps.bzl @@ -1,50 +1,145 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@distroless//private/remote:debian_archive.bzl", "debian_archive") def prysm_image_deps(): - http_file( - name = "bash_amd64", + """ + These dependencies are pulled from https://debian.pkgs.org and support Debian 11. + """ + debian_archive( + name = "amd64_debian11_bash", + package_name = "bash", sha256 = "f702ef058e762d7208a9c83f6f6bbf02645533bfd615c54e8cdcce842cd57377", urls = [ - "http://ftp.us.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_amd64.deb", - "http://http.us.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_amd64.deb", - "http://ftp.uk.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_amd64.deb", - "http://ftp.au.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_amd64.deb", + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/b/bash/bash_5.1-2+deb11u1_amd64.deb", "https://prysmaticlabs.com/uploads/bash_5.1-2+deb11u1_amd64.deb", ], ) - http_file( - name = "bash_arm64", + debian_archive( + name = "arm64_debian11_bash", + package_name = "bash", sha256 = "d7c7af5d86f43a885069408a89788f67f248e8124c682bb73936f33874e0611b", urls = [ - "http://ftp.us.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_arm64.deb", - "http://http.us.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_arm64.deb", - "http://ftp.uk.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_arm64.deb", - "http://ftp.au.debian.org/debian/pool/main/b/bash/bash_5.1-2+deb11u1_arm64.deb", + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/b/bash/bash_5.1-2+deb11u1_arm64.deb", "https://prysmaticlabs.com/uploads/bash_5.1-2+deb11u1_arm64.deb", ], ) - http_file( - name = "libtinfo6_amd64", + debian_archive( + name = "amd64_debian11_libtinfo6", + package_name = "libtinfo6", # Required by: bash sha256 = "96ed58b8fd656521e08549c763cd18da6cff1b7801a3a22f29678701a95d7e7b", urls = [ - "http://ftp.us.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", - "http://http.us.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", - "http://ftp.uk.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", - "http://ftp.au.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", "https://prysmaticlabs.com/uploads/libtinfo6_6.2+20201114-2+deb11u2_amd64.deb", ], ) - http_file( - name = "libtinfo6_arm64", + debian_archive( + name = "arm64_debian11_libtinfo6", + package_name = "libtinfo6", # Required by: bash sha256 = "58027c991756930a2abb2f87a829393d3fdbfb76f4eca9795ef38ea2b0510e27", urls = [ - "http://ftp.us.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u1_arm64.deb", - "http://http.us.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u1_arm64.deb", - "http://ftp.uk.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u1_arm64.deb", - "http://ftp.au.debian.org/debian/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u1_arm64.deb", + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/n/ncurses/libtinfo6_6.2+20201114-2+deb11u1_arm64.deb", "https://prysmaticlabs.com/uploads/libtinfo6_6.2+20201114-2+deb11u2_arm64.deb", ], ) + + debian_archive( + name = "amd64_debian11_coreutils", + package_name = "coreutils", + sha256 = "3558a412ab51eee4b60641327cb145bb91415f127769823b68f9335585b308d4", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/c/coreutils/coreutils_8.32-4+b1_amd64.deb", + "https://prysmaticlabs.com/uploads/coreutils_8.32-4+b1_amd64.deb", + ], + ) + + debian_archive( + name = "arm64_debian11_coreutils", + package_name = "coreutils", + sha256 = "6210c84d6ff84b867dc430f661f22f536e1704c27bdb79de38e26f75b853d9c0", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/c/coreutils/coreutils_8.32-4_arm64.deb", + "https://prysmaticlabs.com/uploads/coreutils_8.32-4_arm64.deb", + ], + ) + + debian_archive( + name = "amd64_debian11_libselinux", + package_name = "libselinux", # Required by: coreutils + sha256 = "339f5ede10500c16dd7192d73169c31c4b27ab12130347275f23044ec8c7d897", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/libs/libselinux/libselinux1_3.1-3_amd64.deb", + "https://prysmaticlabs.com/uploads/libselinux1_3.1-3_amd64.deb", + ], + ) + + debian_archive( + name = "arm64_debian11_libselinux", + package_name = "libselinux", # Required by: coreutils + sha256 = "da98279a47dabaa46a83514142f5c691c6a71fa7e582661a3a3db6887ad3e9d1", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/libs/libselinux/libselinux1_3.1-3_arm64.deb", + "https://prysmaticlabs.com/uploads/libselinux1_3.1-3_arm64.deb", + ], + ) + + debian_archive( + name = "amd64_debian11_libpcre2", + package_name = "libpcre2", # Required by: coreutils + sha256 = "ee192c8d22624eb9d0a2ae95056bad7fb371e5abc17e23e16b1de3ddb17a1064", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/p/pcre2/libpcre2-8-0_10.36-2+deb11u1_amd64.deb", + "https://prysmaticlabs.com/uploads/libpcre2-8-0_10.36-2+deb11u1_amd64.deb", + ], + ) + + debian_archive( + name = "arm64_debian11_libpcre2", + package_name = "libpcre2", # Required by: coreutils + sha256 = "27a4362a4793cb67a8ae571bd8c3f7e8654dc2e56d99088391b87af1793cca9c", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/p/pcre2/libpcre2-8-0_10.36-2+deb11u1_arm64.deb", + "https://prysmaticlabs.com/uploads/libpcre2-8-0_10.36-2+deb11u1_arm64.deb", + ], + ) + + debian_archive( + name = "amd64_debian11_libattr1", + package_name = "libattr1", # Required by: coreutils + sha256 = "af3c3562eb2802481a2b9558df1b389f3c6d9b1bf3b4219e000e05131372ebaf", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/a/attr/libattr1_2.4.48-6_amd64.deb", + "https://prysmaticlabs.com/uploads/libattr1_2.4.48-6_amd64.deb", + ], + ) + + debian_archive( + name = "arm64_debian11_libattr1", + package_name = "libattr1", # Required by: coreutils + sha256 = "cb9b59be719a6fdbaabaa60e22aa6158b2de7a68c88ccd7c3fb7f41a25fb43d0", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/a/attr/libattr1_2.4.48-6_arm64.deb", + "https://prysmaticlabs.com/uploads/libattr1_2.4.48-6_arm64.deb", + ], + ) + + debian_archive( + name = "amd64_debian11_libacl1", + package_name = "libacl1", # Required by: coreutils + sha256 = "aa18d721be8aea50fbdb32cd9a319cb18a3f111ea6ad17399aa4ba9324c8e26a", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/a/acl/libacl1_2.2.53-10_amd64.deb", + "https://prysmaticlabs.com/uploads/libacl1_2.2.53-10_amd64.deb", + ], + ) + + debian_archive( + name = "arm64_debian11_libacl1", + package_name = "libacl1", # Required by: coreutils + sha256 = "f164c48192cb47746101de6c59afa3f97777c8fc821e5a30bb890df1f4cb4cfd", + urls = [ + "https://snapshot.debian.org/archive/debian/20231214T085654Z/pool/main/a/acl/libacl1_2.2.53-10_arm64.deb", + "https://prysmaticlabs.com/uploads/libacl1_2.2.53-10_arm64.deb", + ], + ) diff --git a/tools/prysm_image.bzl b/tools/prysm_image.bzl index e53611240..504c7539a 100644 --- a/tools/prysm_image.bzl +++ b/tools/prysm_image.bzl @@ -22,8 +22,26 @@ def prysm_image_upload( entrypoint = entrypoint, tars = [ "//tools:passwd_tar", - "//tools:libtinfo6_tar", - "//tools:bash_tar", + ] + select({ + "@platforms//cpu:x86_64": [ + "@amd64_debian11_bash", + "@amd64_debian11_libtinfo6", + "@amd64_debian11_coreutils", + "@amd64_debian11_libacl1", + "@amd64_debian11_libattr1", + "@amd64_debian11_libselinux", + "@amd64_debian11_libpcre2", + ], + "@platforms//cpu:arm64": [ + "@arm64_debian11_bash", + "@arm64_debian11_libtinfo6", + "@arm64_debian11_coreutils", + "@arm64_debian11_libacl1", + "@arm64_debian11_libattr1", + "@arm64_debian11_libselinux", + "@arm64_debian11_libpcre2", + ], + }) + [ ":binary_tar", ], labels = {