From cca294cba1c129eb88893d22a5502f81ad3c6428 Mon Sep 17 00:00:00 2001 From: Shane Bammel Date: Thu, 14 Mar 2024 13:42:01 -0500 Subject: [PATCH] Update bazel docker builds - Adds oci_tarball build for local docker images. - Adds bazel-in-docker script & dockerfile. - Removes static image tags. - Updates docker repository. --- .gitignore | 3 ++ Dockerfile.builder | 47 +++++++++++++++++++++++++++++ README.md | 11 +++++++ build-in-docker.sh | 38 +++++++++++++++++++++++ cmd/beacon-chain/BUILD.bazel | 3 +- cmd/prysmctl/BUILD.bazel | 3 +- cmd/validator/BUILD.bazel | 3 +- prysm.bat | 3 ++ prysm.sh | 3 ++ tools/bootnode/BUILD.bazel | 3 +- tools/enr-calculator/BUILD.bazel | 3 +- tools/eth1exporter/BUILD.bazel | 3 +- tools/http-request-sink/BUILD.bazel | 3 +- tools/pcli/BUILD.bazel | 3 +- tools/prysm_image.bzl | 23 +++++++------- 15 files changed, 124 insertions(+), 28 deletions(-) create mode 100644 Dockerfile.builder create mode 100755 build-in-docker.sh diff --git a/.gitignore b/.gitignore index 40aebe42b..9828f55ea 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ tmp # spectest coverage reports report.txt + +# bazel-in-docker +.bazel-in-docker* diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 000000000..8927fda32 --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,47 @@ +# bazel-in-docker builder, for use with the build-in-docker.sh script and as a base image in pipelines +FROM ubuntu:22.04 as builder + +# install build dependencies +RUN apt update && apt install -y cmake git libssl-dev libgmp-dev libtinfo5 unzip curl g++ python3 ca-certificates xz-utils + +# install protoc at specific required version +RUN PROTOC_ZIP=protoc-3.14.0-linux-x86_64.zip && \ + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP && \ + unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ + unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ + rm -f $PROTOC_ZIP + +# install bazelisk +RUN curl -o bazelisk -L https://github.com/bazelbuild/bazelisk/releases/download/v1.14.0/bazelisk-linux-amd64 && \ + chmod +x bazelisk && \ + mv bazelisk /usr/local/bin && \ + ln -s /usr/local/bin/bazelisk /usr/local/bin/bazel && \ + bazelisk --version + +# add the official docker certificates (for installing docker-cli) +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc + +# add the official docker repository to apt sources +RUN echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + | tee /etc/apt/sources.list.d/docker.list > /dev/null + +# install docker-cli for use in pipeline builds +RUN apt update && apt install -y docker-ce-cli + +# create app placeholder +RUN mkdir /app +WORKDIR /app + +# prevent git warnings throughout build process +RUN git config --global --add safe.directory /app + +# this command is only run in local builds when using the build-in-docker script +# perform the build then print output directory mappings +CMD bazel build --config=release //cmd/beacon-chain:oci_tarball && \ + bazel build --config=release //cmd/validator:oci_tarball && \ + bazel build --config=release //cmd/prysmctl:oci_tarball && \ + ls -l | grep -o bazel-.* | sed 's/\/root\/.cache/.bazel-in-docker-cache/g' | awk '{print "ln -sf " $3 " " $1}' > .bazel-in-docker-outputs && \ + chmod 666 .bazel-in-docker-outputs diff --git a/README.md b/README.md index ae3baaedc..fb31fecef 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,14 @@ Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.ne ## Legal Disclaimer [Terms of Use](/TERMS_OF_SERVICE.md) + +# Bazel-in-Docker Build + +To get started quickly, a bazel-in-docker script is included that allows for producing local docker image builds without setting up bazel, protoc, and the various build dependencies. + +This [Dockerfile](Dockerfile.builder) and [shell script](build-in-docker.sh) will produce consistent builds using Bazel-in-Docker while still taking advantage of bazel caching for fast repeat builds. + +```sh +# run the bazel build in docker +./build-in-docker.sh +``` diff --git a/build-in-docker.sh b/build-in-docker.sh new file mode 100755 index 000000000..6fbfd7fcc --- /dev/null +++ b/build-in-docker.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +################################################################ +# This is a convenience script for devs which runs bazel within +# docker to build and load local docker images. +# - Removes dependency requirements (other than docker) +# - Uses mounted directories for bazel caching (fast builds) +# - Does NOT push images to remote repository +################################################################ + +# exit on failures +set -e + +# build the prysm-builder image +docker build --platform linux/amd64 -f Dockerfile.builder -t prysm-builder . + +# run the build +docker run --platform linux/amd64 -it -v $(pwd):/app -v $(pwd)/.bazel-in-docker-cache:/root/.cache prysm-builder + +# source the outputs file to build the correct symlink directories: bazel-* +source .bazel-in-docker-outputs +rm .bazel-in-docker-outputs + +# load new images into docker +docker load -i bazel-bin/cmd/beacon-chain/oci_tarball/tarball.tar +docker load -i bazel-bin/cmd/validator/oci_tarball/tarball.tar +docker load -i bazel-bin/cmd/prysmctl/oci_tarball/tarball.tar + +# finish up +echo "" +echo "Build Complete! Bazel outputs are available in the following directories:" +ls | grep bazel- | tr '\n' '\t' +echo "" + +# print new docker images +echo "" +echo "Docker images loaded:" +docker image ls | grep --color=never -E "REPOSITORY|prysm-pulse.* local" diff --git a/cmd/beacon-chain/BUILD.bazel b/cmd/beacon-chain/BUILD.bazel index 92a0b7d04..0155f15f6 100644 --- a/cmd/beacon-chain/BUILD.bazel +++ b/cmd/beacon-chain/BUILD.bazel @@ -73,10 +73,9 @@ prysm_image_upload( name = "push_images", binary = ":beacon-chain", entrypoint = ["/beacon-chain"], - repository = "gcr.io/prysmaticlabs/prysm/beacon-chain", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/beacon-chain", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/cmd/beacon-chain/beacon-chain": "/beacon-chain", }, - tags = ["manual"], ) diff --git a/cmd/prysmctl/BUILD.bazel b/cmd/prysmctl/BUILD.bazel index fdcb9f008..db05a1517 100644 --- a/cmd/prysmctl/BUILD.bazel +++ b/cmd/prysmctl/BUILD.bazel @@ -30,10 +30,9 @@ prysm_image_upload( name = "push_images", binary = ":prysmctl", entrypoint = ["/prysmctl"], - repository = "gcr.io/prysmaticlabs/prysm/cmd/prysmctl", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/prysmctl", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/cmd/prysmctl/prysmctl": "/prysmctl", }, - tags = ["manual"], ) diff --git a/cmd/validator/BUILD.bazel b/cmd/validator/BUILD.bazel index 8fe1251bf..698bb6bd4 100644 --- a/cmd/validator/BUILD.bazel +++ b/cmd/validator/BUILD.bazel @@ -60,10 +60,9 @@ prysm_image_upload( name = "push_images", binary = ":validator", entrypoint = ["/validator"], - repository = "gcr.io/prysmaticlabs/prysm/validator", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/validator", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/cmd/validator/validator": "/validator", }, - tags = ["manual"], ) diff --git a/prysm.bat b/prysm.bat index 370a1d9a2..b3c5ea4b0 100644 --- a/prysm.bat +++ b/prysm.bat @@ -1,5 +1,8 @@ @echo off +echo "Sorry, but prysm.bat cannot be used for PulseChain at this time." +exit /b 1 + SetLocal EnableDelayedExpansion & REM All variables are set local to this run & expanded at execution time rather than at parse time (tip: echo !output!) set PRYLABS_SIGNING_KEY=0AE0051D647BA3C1A917AF4072E33E4DF1A5036E diff --git a/prysm.sh b/prysm.sh index 8ab5d8af0..b69f35a97 100755 --- a/prysm.sh +++ b/prysm.sh @@ -1,5 +1,8 @@ #!/bin/bash +echo "Sorry, but prysm.sh cannot be used for PulseChain at this time." +exit 1 + set -eu # Use this script to download the latest Prysm release binary. diff --git a/tools/bootnode/BUILD.bazel b/tools/bootnode/BUILD.bazel index 4c20b4095..e6b9d5b58 100644 --- a/tools/bootnode/BUILD.bazel +++ b/tools/bootnode/BUILD.bazel @@ -60,10 +60,9 @@ prysm_image_upload( name = "push_images", binary = ":bootnode", entrypoint = ["/bootnode"], - repository = "gcr.io/prysmaticlabs/prysm/bootnode", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/bootnode", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/tools/bootnode/bootnode": "/bootnode", }, - tags = ["manual"], ) diff --git a/tools/enr-calculator/BUILD.bazel b/tools/enr-calculator/BUILD.bazel index 674d69514..613334628 100644 --- a/tools/enr-calculator/BUILD.bazel +++ b/tools/enr-calculator/BUILD.bazel @@ -28,10 +28,9 @@ prysm_image_upload( name = "push_images", binary = ":enr-calculator", entrypoint = ["/enr-calculator"], - repository = "gcr.io/prysmaticlabs/prysm/enr-calculator", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/enr-calculator", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/tools/enr-calculator/enr-calculator": "/enr-calculator", }, - tags = ["manual"], ) diff --git a/tools/eth1exporter/BUILD.bazel b/tools/eth1exporter/BUILD.bazel index 80bfe0989..ddb24cb80 100644 --- a/tools/eth1exporter/BUILD.bazel +++ b/tools/eth1exporter/BUILD.bazel @@ -26,10 +26,9 @@ prysm_image_upload( name = "push_images", binary = ":eth1exporter", entrypoint = ["/eth1exporter"], - repository = "gcr.io/prysmaticlabs/prysm/eth1monitor", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/eth1monitor", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/tools/eth1exporter/eth1exporter": "/eth1exporter", }, - tags = ["manual"], ) diff --git a/tools/http-request-sink/BUILD.bazel b/tools/http-request-sink/BUILD.bazel index 362b9b445..d1e6ffd42 100644 --- a/tools/http-request-sink/BUILD.bazel +++ b/tools/http-request-sink/BUILD.bazel @@ -30,10 +30,9 @@ prysm_image_upload( name = "push_images", binary = ":http-request-sink", entrypoint = ["/http-request-sink"], - repository = "gcr.io/prysmaticlabs/prysm/http-request-sink", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/http-request-sink", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/tools/http-request-sink/http-request-sink": "/http-request-sink", }, - tags = ["manual"], ) diff --git a/tools/pcli/BUILD.bazel b/tools/pcli/BUILD.bazel index a3386f17f..eb99e682f 100644 --- a/tools/pcli/BUILD.bazel +++ b/tools/pcli/BUILD.bazel @@ -38,10 +38,9 @@ prysm_image_upload( name = "push_images", binary = ":pcli", entrypoint = ["/pcli"], - repository = "gcr.io/prysmaticlabs/prysm/pcli", + repository = "registry.gitlab.com/pulsechaincom/prysm-pulse/pcli", symlinks = { # Backwards compatibility for images that depended on the old filepath. "/app/tools/pcli/pcli": "/pcli", }, - tags = ["manual"], ) diff --git a/tools/prysm_image.bzl b/tools/prysm_image.bzl index fa98bb6b6..ccfe7b55f 100644 --- a/tools/prysm_image.bzl +++ b/tools/prysm_image.bzl @@ -7,13 +7,11 @@ def prysm_image_upload( binary, entrypoint, symlinks, - repository, - tags): + repository): pkg_tar( name = "binary_tar", srcs = [binary], symlinks = symlinks, - tags = tags, ) oci_image( @@ -45,9 +43,8 @@ def prysm_image_upload( ":binary_tar", ], labels = { - "org.opencontainers.image.source": "https://github.com/prysmaticlabs/prysm", + "org.opencontainers.image.source": "https://gitlab.com/pulsechaincom/prysm-pulse", }, - tags = tags, ) multi_arch( @@ -57,7 +54,6 @@ def prysm_image_upload( "@io_bazel_rules_go//go/toolchain:linux_amd64_cgo", "@io_bazel_rules_go//go/toolchain:linux_arm64_cgo", ], - tags = tags, ) oci_image_index( @@ -65,18 +61,21 @@ def prysm_image_upload( images = [ ":oci_multiarch", ], - tags = tags, ) - + + # oci_push rule is used to push the multi-platform image to the registry + # must pass a docker image --tag oci_push( name = name, image = ":oci_image_index", repository = repository, - tags = tags, ) - oci_tarball( - name = "oci_image_tarball", + # oci_tarball rule is used to create an image that can be loaded by the local docker runtime + oci_tarball ( + name = "oci_tarball", image = ":oci_image", - repo_tags = [repository+":latest"], + repo_tags = [ + repository + ":local" + ], )