mirror of
https://gitlab.com/pulsechaincom/lighthouse-pulse.git
synced 2025-01-10 04:51:22 +00:00
931daa40d7
## Issue Addressed Resolves #2545 ## Proposed Changes Adds the long-overdue EF tests for fork choice. Although we had pretty good coverage via other implementations that closely followed our approach, it is nonetheless important for us to implement these tests too. During testing I found that we were using a hard-coded `SAFE_SLOTS_TO_UPDATE_JUSTIFIED` value rather than one from the `ChainSpec`. This caused a failure during a minimal preset test. This doesn't represent a risk to mainnet or testnets, since the hard-coded value matched the mainnet preset. ## Failing Cases There is one failing case which is presently marked as `SkippedKnownFailure`: ``` case 4 ("new_finalized_slot_is_justified_checkpoint_ancestor") from /home/paul/development/lighthouse/testing/ef_tests/consensus-spec-tests/tests/minimal/phase0/fork_choice/on_block/pyspec_tests/new_finalized_slot_is_justified_checkpoint_ancestor failed with NotEqual: head check failed: Got Head { slot: Slot(40), root: 0x9183dbaed4191a862bd307d476e687277fc08469fc38618699863333487703e7 } | Expected Head { slot: Slot(24), root: 0x105b49b51bf7103c182aa58860b039550a89c05a4675992e2af703bd02c84570 } ``` This failure is due to #2741. It's not a particularly high priority issue at the moment, so we fix it after merging this PR.
82 lines
3.0 KiB
Python
Executable File
82 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# The purpose of this script is to compare a list of file names that were accessed during testing
|
|
# against all the file names in the consensus-spec-tests repository. It then checks to see which files
|
|
# were not accessed and returns an error if any non-intentionally-ignored files are detected.
|
|
#
|
|
# The ultimate goal is to detect any accidentally-missed spec tests.
|
|
|
|
import os
|
|
import sys
|
|
|
|
# First argument should the path to a file which contains a list of accessed file names.
|
|
accessed_files_filename = sys.argv[1]
|
|
|
|
# Second argument should be the path to the consensus-spec-tests directory.
|
|
tests_dir_filename = sys.argv[2]
|
|
|
|
# If any of the file names found in the consensus-spec-tests directory *starts with* one of the
|
|
# following strings, we will assume they are to be ignored (i.e., we are purposefully *not* running
|
|
# the spec tests).
|
|
excluded_paths = [
|
|
# Merge tests
|
|
"tests/minimal/merge",
|
|
"tests/mainnet/merge",
|
|
# Eth1Block
|
|
#
|
|
# Intentionally omitted, as per https://github.com/sigp/lighthouse/issues/1835
|
|
"tests/minimal/phase0/ssz_static/Eth1Block/",
|
|
"tests/mainnet/phase0/ssz_static/Eth1Block/",
|
|
"tests/minimal/altair/ssz_static/Eth1Block/",
|
|
"tests/mainnet/altair/ssz_static/Eth1Block/",
|
|
# LightClientStore
|
|
"tests/minimal/altair/ssz_static/LightClientStore",
|
|
"tests/mainnet/altair/ssz_static/LightClientStore",
|
|
# LightClientUpdate
|
|
"tests/minimal/altair/ssz_static/LightClientUpdate",
|
|
"tests/mainnet/altair/ssz_static/LightClientUpdate",
|
|
# LightClientSnapshot
|
|
"tests/minimal/altair/ssz_static/LightClientSnapshot",
|
|
"tests/mainnet/altair/ssz_static/LightClientSnapshot",
|
|
# Merkle-proof tests for light clients
|
|
"tests/mainnet/altair/merkle/single_proof/pyspec_tests/",
|
|
"tests/minimal/altair/merkle/single_proof/pyspec_tests/"
|
|
]
|
|
|
|
def normalize_path(path):
|
|
return path.split("consensus-spec-tests/", )[1]
|
|
|
|
# Determine the list of filenames which were accessed during tests.
|
|
passed = set()
|
|
for line in open(accessed_files_filename, 'r').readlines():
|
|
file = normalize_path(line.strip().strip('"'))
|
|
passed.add(file)
|
|
|
|
missed = set()
|
|
accessed_files = 0
|
|
excluded_files = 0
|
|
|
|
# Iterate all files in the tests directory, ensure that all files were either accessed
|
|
# or intentionally missed.
|
|
for root, dirs, files in os.walk(tests_dir_filename):
|
|
for name in files:
|
|
name = normalize_path(os.path.join(root, name))
|
|
if name not in passed:
|
|
excluded = False
|
|
for excluded_path in excluded_paths:
|
|
if name.startswith(excluded_path):
|
|
excluded = True
|
|
break
|
|
if excluded:
|
|
excluded_files += 1
|
|
else:
|
|
print(name)
|
|
missed.add(name)
|
|
else:
|
|
accessed_files += 1
|
|
|
|
# Exit with an error if there were any files missed.
|
|
assert len(missed) == 0, "{} missed files".format(len(missed))
|
|
|
|
print("Accessed {} files ({} intentionally excluded)".format(accessed_files, excluded_files))
|