Refactor feature testing for spec tests (#6737)

* Refactor spec testing for features and simplify usage.

* Fix `SszStatic` tests for PeerDAS: exclude eip7594 test vectors when testing Electra types.

* Merge branch 'unstable' into refactor-ef-tests-features

Author: jimmygchen

testing/ef_tests/src/cases.rs

@@ -74,11 +74,37 @@ pub use ssz_generic::*;
 pub use ssz_static::*;
 pub use transition::TransitionTest;
 
-#[derive(Debug, PartialEq)]
+/// Used for running feature tests for future forks that have not yet been added to `ForkName`.
+/// This runs tests in the directory named by the feature instead of the fork name. This has been
+/// the pattern used in the `consensus-spec-tests` repository:
+/// `consensus-spec-tests/tests/general/[feature_name]/[runner_name].`
+/// e.g. consensus-spec-tests/tests/general/peerdas/ssz_static
+///
+/// The feature tests can be run with one of the following methods:
+/// 1. `handler.run_for_feature(feature_name)` for new tests that are not on existing fork, i.e. a
+///     new handler. This will be temporary and the test will need to be updated to use
+///     `handle.run()` once the feature is incorporated into a fork.
+/// 2. `handler.run()` for tests that are already on existing forks, but with new test vectors for
+///     the feature. In this case the `handler.is_enabled_for_feature` will need to be implemented
+///     to return `true` for the feature in order for the feature test vector to be tested.
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum FeatureName {
     Eip7594,
 }
 
+impl FeatureName {
+    pub fn list_all() -> Vec<FeatureName> {
+        vec![FeatureName::Eip7594]
+    }
+
+    /// `ForkName` to use when running the feature tests.
+    pub fn fork_name(&self) -> ForkName {
+        match self {
+            FeatureName::Eip7594 => ForkName::Deneb,
+        }
+    }
+}
+
 impl Display for FeatureName {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -107,11 +133,13 @@ pub trait Case: Debug + Sync {
         true
     }
 
-    /// Whether or not this test exists for the given `feature_name`.
+    /// Whether or not this test exists for the given `feature_name`. This is intended to be used
+    /// for features that have not been added to a fork yet, and there is usually a separate folder
+    /// for the feature in the `consensus-spec-tests` repository.
     ///
-    /// Returns `true` by default.
+    /// Returns `false` by default.
     fn is_enabled_for_feature(_feature_name: FeatureName) -> bool {
-        true
+        false
     }
 
     /// Execute a test and return the result.

testing/ef_tests/src/cases/get_custody_columns.rs

@@ -21,6 +21,14 @@ impl<E: EthSpec> LoadCase for GetCustodyColumns<E> {
 }
 
 impl<E: EthSpec> Case for GetCustodyColumns<E> {
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        false
+    }
+
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let spec = E::default_spec();
         let node_id = U256::from_str_radix(&self.node_id, 10)
@@ -33,6 +41,7 @@ impl<E: EthSpec> Case for GetCustodyColumns<E> {
         )
         .expect("should compute custody columns")
         .collect::<Vec<_>>();
+
         let expected = &self.result;
         if computed == *expected {
             Ok(())

testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs

@@ -31,10 +31,6 @@ impl<E: EthSpec> Case for KZGBlobToKZGCommitment<E> {
         fork_name == ForkName::Deneb
     }
 
-    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name != FeatureName::Eip7594
-    }
-
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let kzg = get_kzg();
         let commitment = parse_blob::<E>(&self.input.blob).and_then(|blob| {

testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs

@@ -32,10 +32,6 @@ impl<E: EthSpec> Case for KZGComputeBlobKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
-    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name != FeatureName::Eip7594
-    }
-
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGComputeBlobKZGProofInput| -> Result<_, Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs

@@ -26,8 +26,12 @@ impl<E: EthSpec> LoadCase for KZGComputeCellsAndKZGProofs<E> {
 }
 
 impl<E: EthSpec> Case for KZGComputeCellsAndKZGProofs<E> {
-    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
-        fork_name == ForkName::Deneb
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        false
+    }
+
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Eip7594
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs

@@ -39,10 +39,6 @@ impl<E: EthSpec> Case for KZGComputeKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
-    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name != FeatureName::Eip7594
-    }
-
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGComputeKZGProofInput| -> Result<_, Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs

@@ -27,8 +27,12 @@ impl<E: EthSpec> LoadCase for KZGRecoverCellsAndKZGProofs<E> {
 }
 
 impl<E: EthSpec> Case for KZGRecoverCellsAndKZGProofs<E> {
-    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
-        fork_name == ForkName::Deneb
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        false
+    }
+
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Eip7594
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs

@@ -116,10 +116,6 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
-    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name != FeatureName::Eip7594
-    }
-
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGVerifyBlobKZGProofInput| -> Result<(Blob<E>, KzgCommitment, KzgProof), Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs

@@ -33,10 +33,6 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProofBatch<E> {
         fork_name == ForkName::Deneb
     }
 
-    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name != FeatureName::Eip7594
-    }
-
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGVerifyBlobKZGProofBatchInput| -> Result<_, Error> {
             let blobs = input

testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs

@@ -29,8 +29,12 @@ impl<E: EthSpec> LoadCase for KZGVerifyCellKZGProofBatch<E> {
 }
 
 impl<E: EthSpec> Case for KZGVerifyCellKZGProofBatch<E> {
-    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
-        fork_name == ForkName::Deneb
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        false
+    }
+
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Eip7594
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {