mirror/prysm-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/prysm-pulse.git synced 2024-12-21 11:10:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Spec test coverage report hack (#13718)

Browse Source 
* Spec test report hack

* no export

* fix shell complaint

* shell fix?

* shell again?

* chmod +x ./hack/spectest-report.sh

* Review + improvements

* Remove unwanted change

* Add exclusion list

* Fix path + add eip6110 to exclusion

* Fix bazel path nonsense

* Add extra detail about specific test

* Cleanup exclusion list

* Add fail conditions

* Add mkdir

* Shorten filename + mkdir only if new

* Fix names

* Add to exclusion list

* Add report to .gitignore

* Back to stupid names

* Add Bazel flags option

---------

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>
This commit is contained in:
Sammy Rosso 2024-03-25 17:10:32 +01:00 committed by GitHub
parent 6782df917a
commit 14d7416c16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 396 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -41,3 +41,6 @@ jwt.hex
# manual testing
tmp
# spectest coverage reports
report.txt

2
bazel.sh
View File

@ -2,7 +2,7 @@
# This script serves as a wrapper around bazel to limit the scope of environment variables that
# may change the action output. Using this script should result in a higher cache hit ratio for
# cached actions with a more heremtic build.
# cached actions with a more hermetic build.
env -i \
PATH=/usr/bin:/bin \

78
hack/spectest-report.sh Executable file
View File

@ -0,0 +1,78 @@
#!/bin/bash
set -xe
# Constants
PROJECT_ROOT=$(pwd)
PRYSM_DIR="${PROJECT_ROOT%/hack}/testing/spectest"
EXCLUSION_LIST="$PRYSM_DIR/exclusions.txt"
BAZEL_DIR="/tmp/spectest_report"
SPEC_REPO="git@github.com:ethereum/consensus-spec-tests.git"
SPEC_DIR="/tmp/consensus-spec"
# Create directory if it doesn't already exist
mkdir -p "$BAZEL_DIR"
# Add any passed flags to BAZEL_FLAGS
BAZEL_FLAGS=""
for flag in "$@"
do
BAZEL_FLAGS="$BAZEL_FLAGS $flag"
done
# Run spectests
bazel test //testing/spectest/... --test_env=SPEC_TEST_REPORT_OUTPUT_DIR="$BAZEL_DIR" $BAZEL_FLAGS
# Ensure the SPEC_DIR exists and is a git repository
if [ -d "$SPEC_DIR/.git" ]; then
echo "Repository already exists. Pulling latest changes."
(cd "$SPEC_DIR" && git pull) || exit 1
else
echo "Cloning the GitHub repository."
git clone "$SPEC_REPO" "$SPEC_DIR" || exit 1
fi
# Finding all *_tests.txt files in BAZEL_DIR and concatenating them into tests.txt
find "$BAZEL_DIR" -type f -name '*_tests.txt' -exec cat {} + > "$PRYSM_DIR/tests.txt"
# Generating spec.txt
(cd "$SPEC_DIR" && find tests -maxdepth 4 -mindepth 4 -type d > "$PRYSM_DIR/spec.txt") || exit 1
# Comparing spec.txt with tests.txt and generating report.txt
while IFS= read -r line; do
if grep -Fxq "$line" "$EXCLUSION_LIST"; then
# If it's excluded and we have a test for it flag as an error
if grep -q "$line" "$PRYSM_DIR/tests.txt"; then
echo "Error: Excluded item found in tests.txt: $line"
exit 1 # Exit with an error status
else
echo "Skipping excluded item: $line"
fi
continue
fi
if grep -q "$line" "$PRYSM_DIR/tests.txt"; then
echo "found $line"
else
echo "missing $line"
fi
done < "$PRYSM_DIR/spec.txt" > "$PRYSM_DIR/report.txt"
# Formatting report.txt
{
echo "Prysm Spectest Report"
echo ""
echo "Tests Missing"
grep '^missing' "$PRYSM_DIR/report.txt"
echo ""
echo "Tests Found"
grep '^found' "$PRYSM_DIR/report.txt"
} > "$PRYSM_DIR/report_temp.txt" && mv "$PRYSM_DIR/report_temp.txt" "$PRYSM_DIR/report.txt"
# Check for the word "missing" in the report and exit with an error if present
if grep -q '^missing' "$PRYSM_DIR/report.txt"; then
echo "Error: 'missing' tests found in report: $PRYSM_DIR/report.txt"
exit 1
fi
# Clean up
rm -f "$PRYSM_DIR/tests.txt" "$PRYSM_DIR/spec.txt"

296
testing/spectest/exclusions.txt Normal file
View File

@ -0,0 +1,296 @@
# LightClient
tests/mainnet/altair/light_client/single_merkle_proof
tests/mainnet/bellatrix/light_client/single_merkle_proof
tests/mainnet/capella/light_client/single_merkle_proof
tests/mainnet/deneb/light_client/single_merkle_proof
tests/minimal/altair/light_client/single_merkle_proof
tests/minimal/altair/light_client/sync
tests/minimal/altair/light_client/update_ranking
tests/minimal/bellatrix/light_client/single_merkle_proof
tests/minimal/bellatrix/light_client/sync
tests/minimal/bellatrix/light_client/update_ranking
tests/minimal/capella/light_client/single_merkle_proof
tests/minimal/capella/light_client/sync
tests/minimal/capella/light_client/update_ranking
tests/minimal/deneb/light_client/single_merkle_proof
tests/minimal/deneb/light_client/sync
tests/minimal/deneb/light_client/update_ranking
# SSZ Generic
tests/general/phase0/ssz_generic/basic_vector
tests/general/phase0/ssz_generic/bitlist
tests/general/phase0/ssz_generic/bitvector
tests/general/phase0/ssz_generic/boolean
tests/general/phase0/ssz_generic/containers
tests/general/phase0/ssz_generic/uints
# EIP6110
tests/mainnet/eip6110/epoch_processing/effective_balance_updates
tests/mainnet/eip6110/epoch_processing/eth1_data_reset
tests/mainnet/eip6110/epoch_processing/historical_summaries_update
tests/mainnet/eip6110/epoch_processing/inactivity_updates
tests/mainnet/eip6110/epoch_processing/justification_and_finalization
tests/mainnet/eip6110/epoch_processing/participation_flag_updates
tests/mainnet/eip6110/epoch_processing/randao_mixes_reset
tests/mainnet/eip6110/epoch_processing/registry_updates
tests/mainnet/eip6110/epoch_processing/rewards_and_penalties
tests/mainnet/eip6110/epoch_processing/slashings
tests/mainnet/eip6110/epoch_processing/slashings_reset
tests/mainnet/eip6110/finality/finality
tests/mainnet/eip6110/fork_choice/ex_ante
tests/mainnet/eip6110/fork_choice/get_head
tests/mainnet/eip6110/fork_choice/get_proposer_head
tests/mainnet/eip6110/fork_choice/on_block
tests/mainnet/eip6110/fork_choice/should_override_forkchoice_update
tests/mainnet/eip6110/operations/attestation
tests/mainnet/eip6110/operations/attester_slashing
tests/mainnet/eip6110/operations/block_header
tests/mainnet/eip6110/operations/bls_to_execution_change
tests/mainnet/eip6110/operations/deposit
tests/mainnet/eip6110/operations/deposit_receipt
tests/mainnet/eip6110/operations/execution_payload
tests/mainnet/eip6110/operations/proposer_slashing
tests/mainnet/eip6110/operations/sync_aggregate
tests/mainnet/eip6110/operations/voluntary_exit
tests/mainnet/eip6110/operations/withdrawals
tests/mainnet/eip6110/rewards/basic
tests/mainnet/eip6110/rewards/leak
tests/mainnet/eip6110/rewards/random
tests/mainnet/eip6110/sanity/blocks
tests/mainnet/eip6110/sanity/slots
tests/mainnet/eip6110/ssz_static/AggregateAndProof
tests/mainnet/eip6110/ssz_static/Attestation
tests/mainnet/eip6110/ssz_static/AttestationData
tests/mainnet/eip6110/ssz_static/AttesterSlashing
tests/mainnet/eip6110/ssz_static/BLSToExecutionChange
tests/mainnet/eip6110/ssz_static/BeaconBlock
tests/mainnet/eip6110/ssz_static/BeaconBlockBody
tests/mainnet/eip6110/ssz_static/BeaconBlockHeader
tests/mainnet/eip6110/ssz_static/BeaconState
tests/mainnet/eip6110/ssz_static/BlobIdentifier
tests/mainnet/eip6110/ssz_static/BlobSidecar
tests/mainnet/eip6110/ssz_static/Checkpoint
tests/mainnet/eip6110/ssz_static/ContributionAndProof
tests/mainnet/eip6110/ssz_static/Deposit
tests/mainnet/eip6110/ssz_static/DepositData
tests/mainnet/eip6110/ssz_static/DepositMessage
tests/mainnet/eip6110/ssz_static/DepositReceipt
tests/mainnet/eip6110/ssz_static/Eth1Block
tests/mainnet/eip6110/ssz_static/Eth1Data
tests/mainnet/eip6110/ssz_static/ExecutionPayload
tests/mainnet/eip6110/ssz_static/ExecutionPayloadHeader
tests/mainnet/eip6110/ssz_static/Fork
tests/mainnet/eip6110/ssz_static/ForkData
tests/mainnet/eip6110/ssz_static/HistoricalBatch
tests/mainnet/eip6110/ssz_static/HistoricalSummary
tests/mainnet/eip6110/ssz_static/IndexedAttestation
tests/mainnet/eip6110/ssz_static/LightClientBootstrap
tests/mainnet/eip6110/ssz_static/LightClientFinalityUpdate
tests/mainnet/eip6110/ssz_static/LightClientHeader
tests/mainnet/eip6110/ssz_static/LightClientOptimisticUpdate
tests/mainnet/eip6110/ssz_static/LightClientUpdate
tests/mainnet/eip6110/ssz_static/PendingAttestation
tests/mainnet/eip6110/ssz_static/PowBlock
tests/mainnet/eip6110/ssz_static/ProposerSlashing
tests/mainnet/eip6110/ssz_static/SignedAggregateAndProof
tests/mainnet/eip6110/ssz_static/SignedBLSToExecutionChange
tests/mainnet/eip6110/ssz_static/SignedBeaconBlock
tests/mainnet/eip6110/ssz_static/SignedBeaconBlockHeader
tests/mainnet/eip6110/ssz_static/SignedContributionAndProof
tests/mainnet/eip6110/ssz_static/SignedVoluntaryExit
tests/mainnet/eip6110/ssz_static/SigningData
tests/mainnet/eip6110/ssz_static/SyncAggregate
tests/mainnet/eip6110/ssz_static/SyncAggregatorSelectionData
tests/mainnet/eip6110/ssz_static/SyncCommittee
tests/mainnet/eip6110/ssz_static/SyncCommitteeContribution
tests/mainnet/eip6110/ssz_static/SyncCommitteeMessage
tests/mainnet/eip6110/ssz_static/Validator
tests/mainnet/eip6110/ssz_static/VoluntaryExit
tests/mainnet/eip6110/ssz_static/Withdrawal
tests/mainnet/eip6110/sync/optimistic
tests/mainnet/eip6110/transition/core
tests/minimal/eip6110/epoch_processing/effective_balance_updates
tests/minimal/eip6110/epoch_processing/eth1_data_reset
tests/minimal/eip6110/epoch_processing/historical_summaries_update
tests/minimal/eip6110/epoch_processing/inactivity_updates
tests/minimal/eip6110/epoch_processing/justification_and_finalization
tests/minimal/eip6110/epoch_processing/participation_flag_updates
tests/minimal/eip6110/epoch_processing/randao_mixes_reset
tests/minimal/eip6110/epoch_processing/registry_updates
tests/minimal/eip6110/epoch_processing/rewards_and_penalties
tests/minimal/eip6110/epoch_processing/slashings
tests/minimal/eip6110/epoch_processing/slashings_reset
tests/minimal/eip6110/epoch_processing/sync_committee_updates
tests/minimal/eip6110/finality/finality
tests/minimal/eip6110/fork_choice/ex_ante
tests/minimal/eip6110/fork_choice/get_head
tests/minimal/eip6110/fork_choice/get_proposer_head
tests/minimal/eip6110/fork_choice/on_block
tests/minimal/eip6110/fork_choice/reorg
tests/minimal/eip6110/fork_choice/should_override_forkchoice_update
tests/minimal/eip6110/fork_choice/withholding
tests/minimal/eip6110/genesis/initialization
tests/minimal/eip6110/genesis/validity
tests/minimal/eip6110/operations/attestation
tests/minimal/eip6110/operations/attester_slashing
tests/minimal/eip6110/operations/block_header
tests/minimal/eip6110/operations/bls_to_execution_change
tests/minimal/eip6110/operations/deposit
tests/minimal/eip6110/operations/deposit_receipt
tests/minimal/eip6110/operations/execution_payload
tests/minimal/eip6110/operations/proposer_slashing
tests/minimal/eip6110/operations/sync_aggregate
tests/minimal/eip6110/operations/voluntary_exit
tests/minimal/eip6110/operations/withdrawals
tests/minimal/eip6110/rewards/basic
tests/minimal/eip6110/rewards/leak
tests/minimal/eip6110/rewards/random
tests/minimal/eip6110/sanity/blocks
tests/minimal/eip6110/sanity/slots
tests/minimal/eip6110/ssz_static/AggregateAndProof
tests/minimal/eip6110/ssz_static/Attestation
tests/minimal/eip6110/ssz_static/AttestationData
tests/minimal/eip6110/ssz_static/AttesterSlashing
tests/minimal/eip6110/ssz_static/BLSToExecutionChange
tests/minimal/eip6110/ssz_static/BeaconBlock
tests/minimal/eip6110/ssz_static/BeaconBlockBody
tests/minimal/eip6110/ssz_static/BeaconBlockHeader
tests/minimal/eip6110/ssz_static/BeaconState
tests/minimal/eip6110/ssz_static/BlobIdentifier
tests/minimal/eip6110/ssz_static/BlobSidecar
tests/minimal/eip6110/ssz_static/Checkpoint
tests/minimal/eip6110/ssz_static/ContributionAndProof
tests/minimal/eip6110/ssz_static/Deposit
tests/minimal/eip6110/ssz_static/DepositData
tests/minimal/eip6110/ssz_static/DepositMessage
tests/minimal/eip6110/ssz_static/DepositReceipt
tests/minimal/eip6110/ssz_static/Eth1Block
tests/minimal/eip6110/ssz_static/Eth1Data
tests/minimal/eip6110/ssz_static/ExecutionPayload
tests/minimal/eip6110/ssz_static/ExecutionPayloadHeader
tests/minimal/eip6110/ssz_static/Fork
tests/minimal/eip6110/ssz_static/ForkData
tests/minimal/eip6110/ssz_static/HistoricalBatch
tests/minimal/eip6110/ssz_static/HistoricalSummary
tests/minimal/eip6110/ssz_static/IndexedAttestation
tests/minimal/eip6110/ssz_static/LightClientBootstrap
tests/minimal/eip6110/ssz_static/LightClientFinalityUpdate
tests/minimal/eip6110/ssz_static/LightClientHeader
tests/minimal/eip6110/ssz_static/LightClientOptimisticUpdate
tests/minimal/eip6110/ssz_static/LightClientUpdate
tests/minimal/eip6110/ssz_static/PendingAttestation
tests/minimal/eip6110/ssz_static/PowBlock
tests/minimal/eip6110/ssz_static/ProposerSlashing
tests/minimal/eip6110/ssz_static/SignedAggregateAndProof
tests/minimal/eip6110/ssz_static/SignedBLSToExecutionChange
tests/minimal/eip6110/ssz_static/SignedBeaconBlock
tests/minimal/eip6110/ssz_static/SignedBeaconBlockHeader
tests/minimal/eip6110/ssz_static/SignedContributionAndProof
tests/minimal/eip6110/ssz_static/SignedVoluntaryExit
tests/minimal/eip6110/ssz_static/SigningData
tests/minimal/eip6110/ssz_static/SyncAggregate
tests/minimal/eip6110/ssz_static/SyncAggregatorSelectionData
tests/minimal/eip6110/ssz_static/SyncCommittee
tests/minimal/eip6110/ssz_static/SyncCommitteeContribution
tests/minimal/eip6110/ssz_static/SyncCommitteeMessage
tests/minimal/eip6110/ssz_static/Validator
tests/minimal/eip6110/ssz_static/VoluntaryExit
tests/minimal/eip6110/ssz_static/Withdrawal
tests/minimal/eip6110/sync/optimistic
tests/minimal/eip6110/transition/core
# Whisk
tests/mainnet/whisk/ssz_static/AggregateAndProof
tests/mainnet/whisk/ssz_static/Attestation
tests/mainnet/whisk/ssz_static/AttestationData
tests/mainnet/whisk/ssz_static/AttesterSlashing
tests/mainnet/whisk/ssz_static/BLSToExecutionChange
tests/mainnet/whisk/ssz_static/BeaconBlock
tests/mainnet/whisk/ssz_static/BeaconBlockBody
tests/mainnet/whisk/ssz_static/BeaconBlockHeader
tests/mainnet/whisk/ssz_static/BeaconState
tests/mainnet/whisk/ssz_static/Checkpoint
tests/mainnet/whisk/ssz_static/ContributionAndProof
tests/mainnet/whisk/ssz_static/Deposit
tests/mainnet/whisk/ssz_static/DepositData
tests/mainnet/whisk/ssz_static/DepositMessage
tests/mainnet/whisk/ssz_static/Eth1Block
tests/mainnet/whisk/ssz_static/Eth1Data
tests/mainnet/whisk/ssz_static/ExecutionPayload
tests/mainnet/whisk/ssz_static/ExecutionPayloadHeader
tests/mainnet/whisk/ssz_static/Fork
tests/mainnet/whisk/ssz_static/ForkData
tests/mainnet/whisk/ssz_static/HistoricalBatch
tests/mainnet/whisk/ssz_static/HistoricalSummary
tests/mainnet/whisk/ssz_static/IndexedAttestation
tests/mainnet/whisk/ssz_static/LightClientBootstrap
tests/mainnet/whisk/ssz_static/LightClientFinalityUpdate
tests/mainnet/whisk/ssz_static/LightClientHeader
tests/mainnet/whisk/ssz_static/LightClientOptimisticUpdate
tests/mainnet/whisk/ssz_static/LightClientUpdate
tests/mainnet/whisk/ssz_static/PendingAttestation
tests/mainnet/whisk/ssz_static/PowBlock
tests/mainnet/whisk/ssz_static/ProposerSlashing
tests/mainnet/whisk/ssz_static/SignedAggregateAndProof
tests/mainnet/whisk/ssz_static/SignedBLSToExecutionChange
tests/mainnet/whisk/ssz_static/SignedBeaconBlock
tests/mainnet/whisk/ssz_static/SignedBeaconBlockHeader
tests/mainnet/whisk/ssz_static/SignedContributionAndProof
tests/mainnet/whisk/ssz_static/SignedVoluntaryExit
tests/mainnet/whisk/ssz_static/SigningData
tests/mainnet/whisk/ssz_static/SyncAggregate
tests/mainnet/whisk/ssz_static/SyncAggregatorSelectionData
tests/mainnet/whisk/ssz_static/SyncCommittee
tests/mainnet/whisk/ssz_static/SyncCommitteeContribution
tests/mainnet/whisk/ssz_static/SyncCommitteeMessage
tests/mainnet/whisk/ssz_static/Validator
tests/mainnet/whisk/ssz_static/VoluntaryExit
tests/mainnet/whisk/ssz_static/WhiskTracker
tests/mainnet/whisk/ssz_static/Withdrawal
tests/minimal/whisk/ssz_static/AggregateAndProof
tests/minimal/whisk/ssz_static/Attestation
tests/minimal/whisk/ssz_static/AttestationData
tests/minimal/whisk/ssz_static/AttesterSlashing
tests/minimal/whisk/ssz_static/BLSToExecutionChange
tests/minimal/whisk/ssz_static/BeaconBlock
tests/minimal/whisk/ssz_static/BeaconBlockBody
tests/minimal/whisk/ssz_static/BeaconBlockHeader
tests/minimal/whisk/ssz_static/BeaconState
tests/minimal/whisk/ssz_static/Checkpoint
tests/minimal/whisk/ssz_static/ContributionAndProof
tests/minimal/whisk/ssz_static/Deposit
tests/minimal/whisk/ssz_static/DepositData
tests/minimal/whisk/ssz_static/DepositMessage
tests/minimal/whisk/ssz_static/Eth1Block
tests/minimal/whisk/ssz_static/Eth1Data
tests/minimal/whisk/ssz_static/ExecutionPayload
tests/minimal/whisk/ssz_static/ExecutionPayloadHeader
tests/minimal/whisk/ssz_static/Fork
tests/minimal/whisk/ssz_static/ForkData
tests/minimal/whisk/ssz_static/HistoricalBatch
tests/minimal/whisk/ssz_static/HistoricalSummary
tests/minimal/whisk/ssz_static/IndexedAttestation
tests/minimal/whisk/ssz_static/LightClientBootstrap
tests/minimal/whisk/ssz_static/LightClientFinalityUpdate
tests/minimal/whisk/ssz_static/LightClientHeader
tests/minimal/whisk/ssz_static/LightClientOptimisticUpdate
tests/minimal/whisk/ssz_static/LightClientUpdate
tests/minimal/whisk/ssz_static/PendingAttestation
tests/minimal/whisk/ssz_static/PowBlock
tests/minimal/whisk/ssz_static/ProposerSlashing
tests/minimal/whisk/ssz_static/SignedAggregateAndProof
tests/minimal/whisk/ssz_static/SignedBLSToExecutionChange
tests/minimal/whisk/ssz_static/SignedBeaconBlock
tests/minimal/whisk/ssz_static/SignedBeaconBlockHeader
tests/minimal/whisk/ssz_static/SignedContributionAndProof
tests/minimal/whisk/ssz_static/SignedVoluntaryExit
tests/minimal/whisk/ssz_static/SigningData
tests/minimal/whisk/ssz_static/SyncAggregate
tests/minimal/whisk/ssz_static/SyncAggregatorSelectionData
tests/minimal/whisk/ssz_static/SyncCommittee
tests/minimal/whisk/ssz_static/SyncCommitteeContribution
tests/minimal/whisk/ssz_static/SyncCommitteeMessage
tests/minimal/whisk/ssz_static/Validator
tests/minimal/whisk/ssz_static/VoluntaryExit
tests/minimal/whisk/ssz_static/WhiskTracker
tests/minimal/whisk/ssz_static/Withdrawal

1
testing/spectest/utils/BUILD.bazel
View File

@ -11,6 +11,7 @@ go_library(
visibility = ["//testing/spectest:__subpackages__"],
deps = [
"//config/params:go_default_library",
"//io/file:go_default_library",
"//testing/require:go_default_library",
"@com_github_ghodss_yaml//:go_default_library",
"@com_github_json_iterator_go//:go_default_library",

18
testing/spectest/utils/utils.go
View File

@ -1,6 +1,7 @@
package utils
import (
"fmt"
"os"
"path"
"testing"
@ -8,6 +9,7 @@ import (
"github.com/bazelbuild/rules_go/go/tools/bazel"
"github.com/ghodss/yaml"
jsoniter "github.com/json-iterator/go"
"github.com/prysmaticlabs/prysm/v5/io/file"
"github.com/prysmaticlabs/prysm/v5/testing/require"
)
@ -42,6 +44,20 @@ func TestFolders(t testing.TB, config, forkOrPhase, folderPath string) ([]os.Dir
if len(testFolders) == 0 {
t.Fatalf("No test folders found at %s", testsFolderPath)
}
err = saveSpecTest(testsFolderPath)
require.NoError(t, err)
return testFolders, testsFolderPath
}
func saveSpecTest(testFolder string) error {
baseDir := os.Getenv("SPEC_TEST_REPORT_OUTPUT_DIR")
if baseDir == "" {
return nil // Do nothing if spec test report not requested.
}
fullPath := path.Join(baseDir, fmt.Sprintf("%x_tests.txt", testFolder))
err := file.WriteFile(fullPath, []byte(testFolder))
if err != nil {
return err
}
return nil
}