From b68a4e12aa3d3049c1bfa2508834213108b8ee92 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Fri, 1 Dec 2023 16:20:54 -0600 Subject: [PATCH] Update bazel and other CI improvements (#13246) * Update bazel to 6.4.0, review flags * Remove problematic/slow targets --------- Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> --- .bazelversion | 2 +- .buildkite-bazelrc | 10 ++- beacon-chain/blockchain/BUILD.bazel | 100 +++++++-------------- beacon-chain/core/helpers/BUILD.bazel | 2 +- beacon-chain/sync/initial-sync/BUILD.bazel | 47 +--------- config/params/BUILD.bazel | 2 +- testing/README.md | 5 ++ 7 files changed, 48 insertions(+), 120 deletions(-) create mode 100644 testing/README.md diff --git a/.bazelversion b/.bazelversion index 91e4a9f26..19b860c18 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.3.2 +6.4.0 diff --git a/.buildkite-bazelrc b/.buildkite-bazelrc index 3a824554f..6aacfaa4d 100644 --- a/.buildkite-bazelrc +++ b/.buildkite-bazelrc @@ -9,8 +9,8 @@ #build:remote-cache --strategy=Genrule=standalone # Prysm specific remote-cache properties. -#build:remote-cache --disk_cache= -build:remote-cache --remote_download_toplevel +build:remote-cache --remote_download_minimal +build:remote-cache --experimental_remote_build_event_upload=minimal build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092 # Does not work with rules_oci. See https://github.com/bazel-contrib/rules_oci/issues/292 #build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092 @@ -29,7 +29,10 @@ build --experimental_use_hermetic_linux_sandbox # Import workspace options. import %workspace%/.bazelrc -startup --host_jvm_args=-Xmx4g --host_jvm_args=-Xms2g +# Enable blake3 once it is supported in remote cache. See: https://github.com/buchgr/bazel-remote/issues/710 +# startup --digest_function=blake3 + +startup --host_jvm_args=-Xmx8g --host_jvm_args=-Xms4g build --experimental_strict_action_env build --sandbox_tmpfs_path=/tmp build --verbose_failures @@ -39,6 +42,7 @@ build --curses=no --color=no build --keep_going build --test_output=errors build --flaky_test_attempts=5 +build --build_runfile_links=false # Only build runfile symlink forest when required by local action, test, or run command. # Disabled race detection due to unstable test results under constrained environment build kite # build --features=race diff --git a/beacon-chain/blockchain/BUILD.bazel b/beacon-chain/blockchain/BUILD.bazel index a0339a97b..858158c4f 100644 --- a/beacon-chain/blockchain/BUILD.bazel +++ b/beacon-chain/blockchain/BUILD.bazel @@ -99,26 +99,22 @@ go_library( ], ) -test_suite( - name = "go_default_test", - tests = [ - ":go_raceoff_test", - ":go_raceon_test", - ], -) - go_test( - name = "go_raceoff_test", + name = "go_default_test", size = "medium", srcs = [ "blockchain_test.go", + "chain_info_norace_test.go", "chain_info_test.go", "checktags_test.go", + "error_test.go", "execution_engine_test.go", "forkchoice_update_execution_test.go", "head_sync_committee_info_test.go", "head_test.go", + "init_sync_process_block_test.go", "init_test.go", + "lightclient_test.go", "log_test.go", "metrics_test.go", "mock_test.go", @@ -127,98 +123,66 @@ go_test( "process_block_test.go", "receive_attestation_test.go", "receive_block_test.go", + "service_norace_test.go", "service_test.go", "setup_test.go", "weak_subjectivity_checks_test.go", ], embed = [":go_default_library"], gotags = ["develop"], + tags = ["CI_race_detection"], deps = [ "//async/event:go_default_library", "//beacon-chain/blockchain/testing:go_default_library", + "//beacon-chain/cache:go_default_library", "//beacon-chain/cache/depositcache:go_default_library", "//beacon-chain/core/blocks:go_default_library", + "//beacon-chain/core/feed/state:go_default_library", "//beacon-chain/core/helpers:go_default_library", + "//beacon-chain/core/signing:go_default_library", "//beacon-chain/core/transition:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/db/filesystem:go_default_library", "//beacon-chain/db/testing:go_default_library", "//beacon-chain/execution:go_default_library", "//beacon-chain/execution/testing:go_default_library", + "//beacon-chain/forkchoice:go_default_library", + "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library", "//beacon-chain/forkchoice/types:go_default_library", + "//beacon-chain/operations/attestations:go_default_library", + "//beacon-chain/operations/blstoexec:go_default_library", + "//beacon-chain/operations/slashings:go_default_library", + "//beacon-chain/operations/voluntaryexits:go_default_library", "//beacon-chain/p2p:go_default_library", + "//beacon-chain/startup:go_default_library", + "//beacon-chain/state:go_default_library", "//beacon-chain/state/state-native:go_default_library", - "//beacon-chain/state/stateutil:go_default_library", + "//beacon-chain/state/stategen:go_default_library", + "//config/features:go_default_library", "//config/fieldparams:go_default_library", "//config/params:go_default_library", "//consensus-types/blocks:go_default_library", - "//consensus-types/blocks/testing:go_default_library", + "//consensus-types/interfaces:go_default_library", + "//consensus-types/primitives:go_default_library", "//container/trie:go_default_library", + "//crypto/bls:go_default_library", "//encoding/bytesutil:go_default_library", + "//proto/engine/v1:go_default_library", + "//proto/eth/v1:go_default_library", + "//proto/eth/v2:go_default_library", "//proto/prysm/v1alpha1:go_default_library", + "//runtime/version:go_default_library", "//testing/assert:go_default_library", "//testing/require:go_default_library", "//testing/util:go_default_library", - "@com_github_ethereum_go_ethereum//:go_default_library", - "@com_github_ethereum_go_ethereum//common:go_default_library", - "@com_github_ethereum_go_ethereum//core/types:go_default_library", - "@com_github_sirupsen_logrus//:go_default_library", - "@com_github_sirupsen_logrus//hooks/test:go_default_library", - "@in_gopkg_d4l3k_messagediff_v1//:go_default_library", - "@org_golang_google_protobuf//proto:go_default_library", - "@org_golang_x_net//context:go_default_library", - ], -) - -go_test( - name = "go_raceon_test", - srcs = [ - "chain_info_norace_test.go", - "checktags_test.go", - "init_test.go", - "mock_test.go", - "receive_block_test.go", - "service_norace_test.go", - "setup_test.go", - ], - embed = [":go_default_library"], - gc_goopts = [ - # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer - # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once - # the project is migrated to etcd's version of boltdb and the issue has been fixed. - # See: https://github.com/etcd-io/bbolt/issues/187. - "-d=checkptr=0", - ], - gotags = ["develop"], - race = "on", - tags = ["race_on"], - deps = [ - "//async/event:go_default_library", - "//beacon-chain/blockchain/testing:go_default_library", - "//beacon-chain/cache/depositcache:go_default_library", - "//beacon-chain/core/blocks:go_default_library", - "//beacon-chain/core/helpers:go_default_library", - "//beacon-chain/core/transition:go_default_library", - "//beacon-chain/db:go_default_library", - "//beacon-chain/db/testing:go_default_library", - "//beacon-chain/execution:go_default_library", - "//beacon-chain/execution/testing:go_default_library", - "//beacon-chain/forkchoice/types:go_default_library", - "//beacon-chain/p2p:go_default_library", - "//config/params:go_default_library", - "//consensus-types/blocks:go_default_library", - "//consensus-types/blocks/testing:go_default_library", - "//container/trie:go_default_library", - "//encoding/bytesutil:go_default_library", - "//proto/prysm/v1alpha1:go_default_library", - "//testing/assert:go_default_library", - "//testing/require:go_default_library", - "//testing/util:go_default_library", - "@com_github_ethereum_go_ethereum//:go_default_library", + "//time:go_default_library", + "//time/slots:go_default_library", "@com_github_ethereum_go_ethereum//common:go_default_library", "@com_github_ethereum_go_ethereum//core/types:go_default_library", + "@com_github_holiman_uint256//:go_default_library", + "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", "@com_github_sirupsen_logrus//hooks/test:go_default_library", "@org_golang_google_protobuf//proto:go_default_library", - "@org_golang_x_net//context:go_default_library", ], ) diff --git a/beacon-chain/core/helpers/BUILD.bazel b/beacon-chain/core/helpers/BUILD.bazel index 17dc975cf..704d17e38 100644 --- a/beacon-chain/core/helpers/BUILD.bazel +++ b/beacon-chain/core/helpers/BUILD.bazel @@ -58,8 +58,8 @@ go_test( "weak_subjectivity_test.go", ], embed = [":go_default_library"], - race = "on", shard_count = 2, + tags = ["CI_race_detection"], deps = [ "//beacon-chain/cache:go_default_library", "//beacon-chain/core/time:go_default_library", diff --git a/beacon-chain/sync/initial-sync/BUILD.bazel b/beacon-chain/sync/initial-sync/BUILD.bazel index d2078248a..193edd52e 100644 --- a/beacon-chain/sync/initial-sync/BUILD.bazel +++ b/beacon-chain/sync/initial-sync/BUILD.bazel @@ -51,52 +51,6 @@ go_library( ], ) -go_test( - name = "go_raceon_test", - srcs = [ - "blocks_fetcher_test.go", - "blocks_fetcher_utils_test.go", - "blocks_queue_test.go", - "fsm_test.go", - "initial_sync_test.go", - "round_robin_test.go", - ], - embed = [":go_default_library"], - race = "on", - tags = ["race_on"], - deps = [ - "//beacon-chain/blockchain/testing:go_default_library", - "//beacon-chain/core/helpers:go_default_library", - "//beacon-chain/db:go_default_library", - "//beacon-chain/db/testing:go_default_library", - "//beacon-chain/p2p:go_default_library", - "//beacon-chain/p2p/peers:go_default_library", - "//beacon-chain/p2p/testing:go_default_library", - "//beacon-chain/p2p/types:go_default_library", - "//beacon-chain/state/stateutil:go_default_library", - "//beacon-chain/sync:go_default_library", - "//cmd/beacon-chain/flags:go_default_library", - "//config/features:go_default_library", - "//config/params:go_default_library", - "//consensus-types/blocks:go_default_library", - "//container/queue:go_default_library", - "//container/slice:go_default_library", - "//crypto/hash:go_default_library", - "//encoding/bytesutil:go_default_library", - "//proto/prysm/v1alpha1:go_default_library", - "//testing/assert:go_default_library", - "//testing/require:go_default_library", - "//testing/util:go_default_library", - "//time:go_default_library", - "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library", - "@com_github_libp2p_go_libp2p//core:go_default_library", - "@com_github_libp2p_go_libp2p//core/network:go_default_library", - "@com_github_libp2p_go_libp2p//core/peer:go_default_library", - "@com_github_sirupsen_logrus//:go_default_library", - "@com_github_sirupsen_logrus//hooks/test:go_default_library", - ], -) - go_test( name = "go_default_test", srcs = [ @@ -111,6 +65,7 @@ go_test( "service_test.go", ], embed = [":go_default_library"], + tags = ["CI_race_detection"], deps = [ "//async/abool:go_default_library", "//beacon-chain/blockchain/testing:go_default_library", diff --git a/config/params/BUILD.bazel b/config/params/BUILD.bazel index 2746013f1..08103085a 100644 --- a/config/params/BUILD.bazel +++ b/config/params/BUILD.bazel @@ -62,7 +62,7 @@ go_test( ], embed = [":go_default_library"], gotags = ["develop"], - race = "on", + tags = ["CI_race_detection"], deps = [ "//consensus-types/primitives:go_default_library", "//encoding/bytesutil:go_default_library", diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 000000000..3801e92ff --- /dev/null +++ b/testing/README.md @@ -0,0 +1,5 @@ +# Prysm Testing + +Special bazel tags: + +- `"CI_race_detection"`: Apply this tag to test targets that want to run with race detection in CI.