diff --git a/.gitignore b/.gitignore index 4756ea561..40aebe42b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ jwt.hex # manual testing tmp + +# spectest coverage reports +report.txt diff --git a/bazel.sh b/bazel.sh index b89ac678a..d0239499e 100755 --- a/bazel.sh +++ b/bazel.sh @@ -2,7 +2,7 @@ # This script serves as a wrapper around bazel to limit the scope of environment variables that # may change the action output. Using this script should result in a higher cache hit ratio for -# cached actions with a more heremtic build. +# cached actions with a more hermetic build. env -i \ PATH=/usr/bin:/bin \ diff --git a/hack/spectest-report.sh b/hack/spectest-report.sh new file mode 100755 index 000000000..d9fa0a700 --- /dev/null +++ b/hack/spectest-report.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -xe + +# Constants +PROJECT_ROOT=$(pwd) +PRYSM_DIR="${PROJECT_ROOT%/hack}/testing/spectest" +EXCLUSION_LIST="$PRYSM_DIR/exclusions.txt" +BAZEL_DIR="/tmp/spectest_report" +SPEC_REPO="git@github.com:ethereum/consensus-spec-tests.git" +SPEC_DIR="/tmp/consensus-spec" + +# Create directory if it doesn't already exist +mkdir -p "$BAZEL_DIR" + +# Add any passed flags to BAZEL_FLAGS +BAZEL_FLAGS="" +for flag in "$@" +do + BAZEL_FLAGS="$BAZEL_FLAGS $flag" +done + +# Run spectests +bazel test //testing/spectest/... --test_env=SPEC_TEST_REPORT_OUTPUT_DIR="$BAZEL_DIR" $BAZEL_FLAGS + +# Ensure the SPEC_DIR exists and is a git repository +if [ -d "$SPEC_DIR/.git" ]; then + echo "Repository already exists. Pulling latest changes." + (cd "$SPEC_DIR" && git pull) || exit 1 +else + echo "Cloning the GitHub repository." + git clone "$SPEC_REPO" "$SPEC_DIR" || exit 1 +fi + +# Finding all *_tests.txt files in BAZEL_DIR and concatenating them into tests.txt +find "$BAZEL_DIR" -type f -name '*_tests.txt' -exec cat {} + > "$PRYSM_DIR/tests.txt" + +# Generating spec.txt +(cd "$SPEC_DIR" && find tests -maxdepth 4 -mindepth 4 -type d > "$PRYSM_DIR/spec.txt") || exit 1 + +# Comparing spec.txt with tests.txt and generating report.txt +while IFS= read -r line; do + if grep -Fxq "$line" "$EXCLUSION_LIST"; then + # If it's excluded and we have a test for it flag as an error + if grep -q "$line" "$PRYSM_DIR/tests.txt"; then + echo "Error: Excluded item found in tests.txt: $line" + exit 1 # Exit with an error status + else + echo "Skipping excluded item: $line" + fi + continue + fi + if grep -q "$line" "$PRYSM_DIR/tests.txt"; then + echo "found $line" + else + echo "missing $line" + fi +done < "$PRYSM_DIR/spec.txt" > "$PRYSM_DIR/report.txt" + +# Formatting report.txt +{ + echo "Prysm Spectest Report" + echo "" + echo "Tests Missing" + grep '^missing' "$PRYSM_DIR/report.txt" + echo "" + echo "Tests Found" + grep '^found' "$PRYSM_DIR/report.txt" +} > "$PRYSM_DIR/report_temp.txt" && mv "$PRYSM_DIR/report_temp.txt" "$PRYSM_DIR/report.txt" + +# Check for the word "missing" in the report and exit with an error if present +if grep -q '^missing' "$PRYSM_DIR/report.txt"; then + echo "Error: 'missing' tests found in report: $PRYSM_DIR/report.txt" + exit 1 +fi + +# Clean up +rm -f "$PRYSM_DIR/tests.txt" "$PRYSM_DIR/spec.txt" diff --git a/testing/spectest/exclusions.txt b/testing/spectest/exclusions.txt new file mode 100644 index 000000000..8e12f2dbc --- /dev/null +++ b/testing/spectest/exclusions.txt @@ -0,0 +1,296 @@ +# LightClient +tests/mainnet/altair/light_client/single_merkle_proof +tests/mainnet/bellatrix/light_client/single_merkle_proof +tests/mainnet/capella/light_client/single_merkle_proof +tests/mainnet/deneb/light_client/single_merkle_proof +tests/minimal/altair/light_client/single_merkle_proof +tests/minimal/altair/light_client/sync +tests/minimal/altair/light_client/update_ranking +tests/minimal/bellatrix/light_client/single_merkle_proof +tests/minimal/bellatrix/light_client/sync +tests/minimal/bellatrix/light_client/update_ranking +tests/minimal/capella/light_client/single_merkle_proof +tests/minimal/capella/light_client/sync +tests/minimal/capella/light_client/update_ranking +tests/minimal/deneb/light_client/single_merkle_proof +tests/minimal/deneb/light_client/sync +tests/minimal/deneb/light_client/update_ranking + +# SSZ Generic +tests/general/phase0/ssz_generic/basic_vector +tests/general/phase0/ssz_generic/bitlist +tests/general/phase0/ssz_generic/bitvector +tests/general/phase0/ssz_generic/boolean +tests/general/phase0/ssz_generic/containers +tests/general/phase0/ssz_generic/uints + +# EIP6110 +tests/mainnet/eip6110/epoch_processing/effective_balance_updates +tests/mainnet/eip6110/epoch_processing/eth1_data_reset +tests/mainnet/eip6110/epoch_processing/historical_summaries_update +tests/mainnet/eip6110/epoch_processing/inactivity_updates +tests/mainnet/eip6110/epoch_processing/justification_and_finalization +tests/mainnet/eip6110/epoch_processing/participation_flag_updates +tests/mainnet/eip6110/epoch_processing/randao_mixes_reset +tests/mainnet/eip6110/epoch_processing/registry_updates +tests/mainnet/eip6110/epoch_processing/rewards_and_penalties +tests/mainnet/eip6110/epoch_processing/slashings +tests/mainnet/eip6110/epoch_processing/slashings_reset +tests/mainnet/eip6110/finality/finality +tests/mainnet/eip6110/fork_choice/ex_ante +tests/mainnet/eip6110/fork_choice/get_head +tests/mainnet/eip6110/fork_choice/get_proposer_head +tests/mainnet/eip6110/fork_choice/on_block +tests/mainnet/eip6110/fork_choice/should_override_forkchoice_update +tests/mainnet/eip6110/operations/attestation +tests/mainnet/eip6110/operations/attester_slashing +tests/mainnet/eip6110/operations/block_header +tests/mainnet/eip6110/operations/bls_to_execution_change +tests/mainnet/eip6110/operations/deposit +tests/mainnet/eip6110/operations/deposit_receipt +tests/mainnet/eip6110/operations/execution_payload +tests/mainnet/eip6110/operations/proposer_slashing +tests/mainnet/eip6110/operations/sync_aggregate +tests/mainnet/eip6110/operations/voluntary_exit +tests/mainnet/eip6110/operations/withdrawals +tests/mainnet/eip6110/rewards/basic +tests/mainnet/eip6110/rewards/leak +tests/mainnet/eip6110/rewards/random +tests/mainnet/eip6110/sanity/blocks +tests/mainnet/eip6110/sanity/slots +tests/mainnet/eip6110/ssz_static/AggregateAndProof +tests/mainnet/eip6110/ssz_static/Attestation +tests/mainnet/eip6110/ssz_static/AttestationData +tests/mainnet/eip6110/ssz_static/AttesterSlashing +tests/mainnet/eip6110/ssz_static/BLSToExecutionChange +tests/mainnet/eip6110/ssz_static/BeaconBlock +tests/mainnet/eip6110/ssz_static/BeaconBlockBody +tests/mainnet/eip6110/ssz_static/BeaconBlockHeader +tests/mainnet/eip6110/ssz_static/BeaconState +tests/mainnet/eip6110/ssz_static/BlobIdentifier +tests/mainnet/eip6110/ssz_static/BlobSidecar +tests/mainnet/eip6110/ssz_static/Checkpoint +tests/mainnet/eip6110/ssz_static/ContributionAndProof +tests/mainnet/eip6110/ssz_static/Deposit +tests/mainnet/eip6110/ssz_static/DepositData +tests/mainnet/eip6110/ssz_static/DepositMessage +tests/mainnet/eip6110/ssz_static/DepositReceipt +tests/mainnet/eip6110/ssz_static/Eth1Block +tests/mainnet/eip6110/ssz_static/Eth1Data +tests/mainnet/eip6110/ssz_static/ExecutionPayload +tests/mainnet/eip6110/ssz_static/ExecutionPayloadHeader +tests/mainnet/eip6110/ssz_static/Fork +tests/mainnet/eip6110/ssz_static/ForkData +tests/mainnet/eip6110/ssz_static/HistoricalBatch +tests/mainnet/eip6110/ssz_static/HistoricalSummary +tests/mainnet/eip6110/ssz_static/IndexedAttestation +tests/mainnet/eip6110/ssz_static/LightClientBootstrap +tests/mainnet/eip6110/ssz_static/LightClientFinalityUpdate +tests/mainnet/eip6110/ssz_static/LightClientHeader +tests/mainnet/eip6110/ssz_static/LightClientOptimisticUpdate +tests/mainnet/eip6110/ssz_static/LightClientUpdate +tests/mainnet/eip6110/ssz_static/PendingAttestation +tests/mainnet/eip6110/ssz_static/PowBlock +tests/mainnet/eip6110/ssz_static/ProposerSlashing +tests/mainnet/eip6110/ssz_static/SignedAggregateAndProof +tests/mainnet/eip6110/ssz_static/SignedBLSToExecutionChange +tests/mainnet/eip6110/ssz_static/SignedBeaconBlock +tests/mainnet/eip6110/ssz_static/SignedBeaconBlockHeader +tests/mainnet/eip6110/ssz_static/SignedContributionAndProof +tests/mainnet/eip6110/ssz_static/SignedVoluntaryExit +tests/mainnet/eip6110/ssz_static/SigningData +tests/mainnet/eip6110/ssz_static/SyncAggregate +tests/mainnet/eip6110/ssz_static/SyncAggregatorSelectionData +tests/mainnet/eip6110/ssz_static/SyncCommittee +tests/mainnet/eip6110/ssz_static/SyncCommitteeContribution +tests/mainnet/eip6110/ssz_static/SyncCommitteeMessage +tests/mainnet/eip6110/ssz_static/Validator +tests/mainnet/eip6110/ssz_static/VoluntaryExit +tests/mainnet/eip6110/ssz_static/Withdrawal +tests/mainnet/eip6110/sync/optimistic +tests/mainnet/eip6110/transition/core +tests/minimal/eip6110/epoch_processing/effective_balance_updates +tests/minimal/eip6110/epoch_processing/eth1_data_reset +tests/minimal/eip6110/epoch_processing/historical_summaries_update +tests/minimal/eip6110/epoch_processing/inactivity_updates +tests/minimal/eip6110/epoch_processing/justification_and_finalization +tests/minimal/eip6110/epoch_processing/participation_flag_updates +tests/minimal/eip6110/epoch_processing/randao_mixes_reset +tests/minimal/eip6110/epoch_processing/registry_updates +tests/minimal/eip6110/epoch_processing/rewards_and_penalties +tests/minimal/eip6110/epoch_processing/slashings +tests/minimal/eip6110/epoch_processing/slashings_reset +tests/minimal/eip6110/epoch_processing/sync_committee_updates +tests/minimal/eip6110/finality/finality +tests/minimal/eip6110/fork_choice/ex_ante +tests/minimal/eip6110/fork_choice/get_head +tests/minimal/eip6110/fork_choice/get_proposer_head +tests/minimal/eip6110/fork_choice/on_block +tests/minimal/eip6110/fork_choice/reorg +tests/minimal/eip6110/fork_choice/should_override_forkchoice_update +tests/minimal/eip6110/fork_choice/withholding +tests/minimal/eip6110/genesis/initialization +tests/minimal/eip6110/genesis/validity +tests/minimal/eip6110/operations/attestation +tests/minimal/eip6110/operations/attester_slashing +tests/minimal/eip6110/operations/block_header +tests/minimal/eip6110/operations/bls_to_execution_change +tests/minimal/eip6110/operations/deposit +tests/minimal/eip6110/operations/deposit_receipt +tests/minimal/eip6110/operations/execution_payload +tests/minimal/eip6110/operations/proposer_slashing +tests/minimal/eip6110/operations/sync_aggregate +tests/minimal/eip6110/operations/voluntary_exit +tests/minimal/eip6110/operations/withdrawals +tests/minimal/eip6110/rewards/basic +tests/minimal/eip6110/rewards/leak +tests/minimal/eip6110/rewards/random +tests/minimal/eip6110/sanity/blocks +tests/minimal/eip6110/sanity/slots +tests/minimal/eip6110/ssz_static/AggregateAndProof +tests/minimal/eip6110/ssz_static/Attestation +tests/minimal/eip6110/ssz_static/AttestationData +tests/minimal/eip6110/ssz_static/AttesterSlashing +tests/minimal/eip6110/ssz_static/BLSToExecutionChange +tests/minimal/eip6110/ssz_static/BeaconBlock +tests/minimal/eip6110/ssz_static/BeaconBlockBody +tests/minimal/eip6110/ssz_static/BeaconBlockHeader +tests/minimal/eip6110/ssz_static/BeaconState +tests/minimal/eip6110/ssz_static/BlobIdentifier +tests/minimal/eip6110/ssz_static/BlobSidecar +tests/minimal/eip6110/ssz_static/Checkpoint +tests/minimal/eip6110/ssz_static/ContributionAndProof +tests/minimal/eip6110/ssz_static/Deposit +tests/minimal/eip6110/ssz_static/DepositData +tests/minimal/eip6110/ssz_static/DepositMessage +tests/minimal/eip6110/ssz_static/DepositReceipt +tests/minimal/eip6110/ssz_static/Eth1Block +tests/minimal/eip6110/ssz_static/Eth1Data +tests/minimal/eip6110/ssz_static/ExecutionPayload +tests/minimal/eip6110/ssz_static/ExecutionPayloadHeader +tests/minimal/eip6110/ssz_static/Fork +tests/minimal/eip6110/ssz_static/ForkData +tests/minimal/eip6110/ssz_static/HistoricalBatch +tests/minimal/eip6110/ssz_static/HistoricalSummary +tests/minimal/eip6110/ssz_static/IndexedAttestation +tests/minimal/eip6110/ssz_static/LightClientBootstrap +tests/minimal/eip6110/ssz_static/LightClientFinalityUpdate +tests/minimal/eip6110/ssz_static/LightClientHeader +tests/minimal/eip6110/ssz_static/LightClientOptimisticUpdate +tests/minimal/eip6110/ssz_static/LightClientUpdate +tests/minimal/eip6110/ssz_static/PendingAttestation +tests/minimal/eip6110/ssz_static/PowBlock +tests/minimal/eip6110/ssz_static/ProposerSlashing +tests/minimal/eip6110/ssz_static/SignedAggregateAndProof +tests/minimal/eip6110/ssz_static/SignedBLSToExecutionChange +tests/minimal/eip6110/ssz_static/SignedBeaconBlock +tests/minimal/eip6110/ssz_static/SignedBeaconBlockHeader +tests/minimal/eip6110/ssz_static/SignedContributionAndProof +tests/minimal/eip6110/ssz_static/SignedVoluntaryExit +tests/minimal/eip6110/ssz_static/SigningData +tests/minimal/eip6110/ssz_static/SyncAggregate +tests/minimal/eip6110/ssz_static/SyncAggregatorSelectionData +tests/minimal/eip6110/ssz_static/SyncCommittee +tests/minimal/eip6110/ssz_static/SyncCommitteeContribution +tests/minimal/eip6110/ssz_static/SyncCommitteeMessage +tests/minimal/eip6110/ssz_static/Validator +tests/minimal/eip6110/ssz_static/VoluntaryExit +tests/minimal/eip6110/ssz_static/Withdrawal +tests/minimal/eip6110/sync/optimistic +tests/minimal/eip6110/transition/core + +# Whisk +tests/mainnet/whisk/ssz_static/AggregateAndProof +tests/mainnet/whisk/ssz_static/Attestation +tests/mainnet/whisk/ssz_static/AttestationData +tests/mainnet/whisk/ssz_static/AttesterSlashing +tests/mainnet/whisk/ssz_static/BLSToExecutionChange +tests/mainnet/whisk/ssz_static/BeaconBlock +tests/mainnet/whisk/ssz_static/BeaconBlockBody +tests/mainnet/whisk/ssz_static/BeaconBlockHeader +tests/mainnet/whisk/ssz_static/BeaconState +tests/mainnet/whisk/ssz_static/Checkpoint +tests/mainnet/whisk/ssz_static/ContributionAndProof +tests/mainnet/whisk/ssz_static/Deposit +tests/mainnet/whisk/ssz_static/DepositData +tests/mainnet/whisk/ssz_static/DepositMessage +tests/mainnet/whisk/ssz_static/Eth1Block +tests/mainnet/whisk/ssz_static/Eth1Data +tests/mainnet/whisk/ssz_static/ExecutionPayload +tests/mainnet/whisk/ssz_static/ExecutionPayloadHeader +tests/mainnet/whisk/ssz_static/Fork +tests/mainnet/whisk/ssz_static/ForkData +tests/mainnet/whisk/ssz_static/HistoricalBatch +tests/mainnet/whisk/ssz_static/HistoricalSummary +tests/mainnet/whisk/ssz_static/IndexedAttestation +tests/mainnet/whisk/ssz_static/LightClientBootstrap +tests/mainnet/whisk/ssz_static/LightClientFinalityUpdate +tests/mainnet/whisk/ssz_static/LightClientHeader +tests/mainnet/whisk/ssz_static/LightClientOptimisticUpdate +tests/mainnet/whisk/ssz_static/LightClientUpdate +tests/mainnet/whisk/ssz_static/PendingAttestation +tests/mainnet/whisk/ssz_static/PowBlock +tests/mainnet/whisk/ssz_static/ProposerSlashing +tests/mainnet/whisk/ssz_static/SignedAggregateAndProof +tests/mainnet/whisk/ssz_static/SignedBLSToExecutionChange +tests/mainnet/whisk/ssz_static/SignedBeaconBlock +tests/mainnet/whisk/ssz_static/SignedBeaconBlockHeader +tests/mainnet/whisk/ssz_static/SignedContributionAndProof +tests/mainnet/whisk/ssz_static/SignedVoluntaryExit +tests/mainnet/whisk/ssz_static/SigningData +tests/mainnet/whisk/ssz_static/SyncAggregate +tests/mainnet/whisk/ssz_static/SyncAggregatorSelectionData +tests/mainnet/whisk/ssz_static/SyncCommittee +tests/mainnet/whisk/ssz_static/SyncCommitteeContribution +tests/mainnet/whisk/ssz_static/SyncCommitteeMessage +tests/mainnet/whisk/ssz_static/Validator +tests/mainnet/whisk/ssz_static/VoluntaryExit +tests/mainnet/whisk/ssz_static/WhiskTracker +tests/mainnet/whisk/ssz_static/Withdrawal +tests/minimal/whisk/ssz_static/AggregateAndProof +tests/minimal/whisk/ssz_static/Attestation +tests/minimal/whisk/ssz_static/AttestationData +tests/minimal/whisk/ssz_static/AttesterSlashing +tests/minimal/whisk/ssz_static/BLSToExecutionChange +tests/minimal/whisk/ssz_static/BeaconBlock +tests/minimal/whisk/ssz_static/BeaconBlockBody +tests/minimal/whisk/ssz_static/BeaconBlockHeader +tests/minimal/whisk/ssz_static/BeaconState +tests/minimal/whisk/ssz_static/Checkpoint +tests/minimal/whisk/ssz_static/ContributionAndProof +tests/minimal/whisk/ssz_static/Deposit +tests/minimal/whisk/ssz_static/DepositData +tests/minimal/whisk/ssz_static/DepositMessage +tests/minimal/whisk/ssz_static/Eth1Block +tests/minimal/whisk/ssz_static/Eth1Data +tests/minimal/whisk/ssz_static/ExecutionPayload +tests/minimal/whisk/ssz_static/ExecutionPayloadHeader +tests/minimal/whisk/ssz_static/Fork +tests/minimal/whisk/ssz_static/ForkData +tests/minimal/whisk/ssz_static/HistoricalBatch +tests/minimal/whisk/ssz_static/HistoricalSummary +tests/minimal/whisk/ssz_static/IndexedAttestation +tests/minimal/whisk/ssz_static/LightClientBootstrap +tests/minimal/whisk/ssz_static/LightClientFinalityUpdate +tests/minimal/whisk/ssz_static/LightClientHeader +tests/minimal/whisk/ssz_static/LightClientOptimisticUpdate +tests/minimal/whisk/ssz_static/LightClientUpdate +tests/minimal/whisk/ssz_static/PendingAttestation +tests/minimal/whisk/ssz_static/PowBlock +tests/minimal/whisk/ssz_static/ProposerSlashing +tests/minimal/whisk/ssz_static/SignedAggregateAndProof +tests/minimal/whisk/ssz_static/SignedBLSToExecutionChange +tests/minimal/whisk/ssz_static/SignedBeaconBlock +tests/minimal/whisk/ssz_static/SignedBeaconBlockHeader +tests/minimal/whisk/ssz_static/SignedContributionAndProof +tests/minimal/whisk/ssz_static/SignedVoluntaryExit +tests/minimal/whisk/ssz_static/SigningData +tests/minimal/whisk/ssz_static/SyncAggregate +tests/minimal/whisk/ssz_static/SyncAggregatorSelectionData +tests/minimal/whisk/ssz_static/SyncCommittee +tests/minimal/whisk/ssz_static/SyncCommitteeContribution +tests/minimal/whisk/ssz_static/SyncCommitteeMessage +tests/minimal/whisk/ssz_static/Validator +tests/minimal/whisk/ssz_static/VoluntaryExit +tests/minimal/whisk/ssz_static/WhiskTracker +tests/minimal/whisk/ssz_static/Withdrawal \ No newline at end of file diff --git a/testing/spectest/utils/BUILD.bazel b/testing/spectest/utils/BUILD.bazel index 455b92823..2919cb90d 100644 --- a/testing/spectest/utils/BUILD.bazel +++ b/testing/spectest/utils/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//testing/spectest:__subpackages__"], deps = [ "//config/params:go_default_library", + "//io/file:go_default_library", "//testing/require:go_default_library", "@com_github_ghodss_yaml//:go_default_library", "@com_github_json_iterator_go//:go_default_library", diff --git a/testing/spectest/utils/utils.go b/testing/spectest/utils/utils.go index 72603fe86..2dc31fdc9 100644 --- a/testing/spectest/utils/utils.go +++ b/testing/spectest/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "os" "path" "testing" @@ -8,6 +9,7 @@ import ( "github.com/bazelbuild/rules_go/go/tools/bazel" "github.com/ghodss/yaml" jsoniter "github.com/json-iterator/go" + "github.com/prysmaticlabs/prysm/v5/io/file" "github.com/prysmaticlabs/prysm/v5/testing/require" ) @@ -42,6 +44,20 @@ func TestFolders(t testing.TB, config, forkOrPhase, folderPath string) ([]os.Dir if len(testFolders) == 0 { t.Fatalf("No test folders found at %s", testsFolderPath) } - + err = saveSpecTest(testsFolderPath) + require.NoError(t, err) return testFolders, testsFolderPath } + +func saveSpecTest(testFolder string) error { + baseDir := os.Getenv("SPEC_TEST_REPORT_OUTPUT_DIR") + if baseDir == "" { + return nil // Do nothing if spec test report not requested. + } + fullPath := path.Join(baseDir, fmt.Sprintf("%x_tests.txt", testFolder)) + err := file.WriteFile(fullPath, []byte(testFolder)) + if err != nil { + return err + } + return nil +}