Progressive data structures and tests EIP-7916, EIP-8016

Cargo.toml

@@ -302,3 +302,9 @@ debug = true
 
 [patch.crates-io]
 quick-protobuf = { git = "https://github.com/sigp/quick-protobuf.git", rev = "681f413312404ab6e51f0b46f39b0075c6f4ebfd" }
+# FIXME(sproul): REMOVE patch
+milhouse = { git = "https://github.com/sigp/milhouse", branch = "progressive-list" }
+ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", branch = "progressive" }
+tree_hash = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }
+tree_hash_derive = { git = "https://github.com/sigp/tree_hash", branch = "progressive" }

consensus/state_processing/src/per_block_processing/tests.rs

@@ -495,7 +495,9 @@ async fn invalid_attestation_bad_aggregation_bitfield_len() {
         .next()
         .unwrap()
         .aggregation_bits_base_mut()
-        .unwrap() = Bitfield::with_capacity(spec.target_committee_size).unwrap();
+        .unwrap() =
+        Bitfield::<ssz::Variable<<MainnetEthSpec as EthSpec>::MaxValidatorsPerCommittee>>::with_capacity(spec.target_committee_size)
+            .unwrap();
 
     let mut ctxt = ConsensusContext::new(state.slot());
     let result = process_operations::process_attestations(

testing/ef_tests/Cargo.toml

@@ -19,6 +19,7 @@ compare_fields = { workspace = true }
 context_deserialize = { workspace = true }
 educe = { workspace = true }
 eth2_network_config = { workspace = true }
+ethereum_hashing = { workspace = true } # FIXME(sproul): remove
 ethereum_ssz = { workspace = true }
 ethereum_ssz_derive = { workspace = true }
 execution_layer = { workspace = true }

testing/ef_tests/check_all_files_accessed.py

@@ -66,13 +66,6 @@
     "tests/.*/gloas/ssz_static/ExecutionPayloadHeader/.*",
     # ForkChoiceNode is internal to fork choice and probably doesn't need SSZ tests.
     "tests/.*/gloas/ssz_static/ForkChoiceNode/.*",
-    # EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
-    "tests/general/phase0/ssz_generic/progressive_bitlist",
-    "tests/general/phase0/ssz_generic/basic_progressive_list",
-    "tests/general/phase0/ssz_generic/containers/.*/ProgressiveBitsStruct.*",
-    "tests/general/phase0/ssz_generic/containers/.*/ProgressiveTestStruct.*",
-    "tests/general/phase0/ssz_generic/progressive_containers/.*",
-    "tests/general/phase0/ssz_generic/compatible_unions/.*",
     # Ignore full epoch tests for now (just test the sub-transitions).
     "tests/.*/.*/epoch_processing/.*/pre_epoch.ssz_snappy",
     "tests/.*/.*/epoch_processing/.*/post_epoch.ssz_snappy",

testing/ef_tests/src/cases/ssz_generic.rs

@@ -5,15 +5,57 @@ use crate::cases::common::{DecimalU128, DecimalU256, SszStaticType};
 use crate::cases::ssz_static::{check_serialization, check_tree_hash};
 use crate::decode::{context_yaml_decode_file, log_file_access, snappy_decode_file};
 use context_deserialize::{ContextDeserialize, context_deserialize};
-use milhouse::Vector;
+use milhouse::{List, ProgressiveList, Vector};
 use serde::{Deserialize, Deserializer, de::Error as SerdeError};
+use serde_json::Value as JsonValue;
+use ssz::ProgressiveBitList;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{BitList, BitVector, FixedVector, VariableList};
 use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
 use typenum::*;
 use types::ForkName;
 
+/// Helper struct for deserializing compatible unions from `{selector, data}` YAML format.
+#[derive(Deserialize)]
+struct CompatibleUnionYaml {
+    selector: u8,
+    data: JsonValue,
+}
+
+/// Implements `Deserialize` for compatible union types to handle the EF test YAML format.
+///
+/// Deserialize into `CompatibleUnionYaml` which captures `selector` (`u8`) and
+///    `data` (`JsonValue`).
+/// Match on the selector to determine which variant to construct.
+/// Deserialize the `data` field into the appropriate inner type.
+macro_rules! impl_compatible_union_deserialize {
+    ($type:ty, { $($selector:literal => $variant:ident($inner:ty)),+ $(,)? }) => {
+        impl<'de> Deserialize<'de> for $type {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                let yaml = CompatibleUnionYaml::deserialize(deserializer)?;
+                match yaml.selector {
+                    $(
+                        $selector => {
+                            let inner: $inner = serde_json::from_value(yaml.data).map_err(D::Error::custom)?;
+                            Ok(<$type>::$variant(inner))
+                        }
+                    )+
+                    s => Err(D::Error::custom(format!(
+                        "unknown selector {s} for {}", stringify!($type)
+                    ))),
+                }
+            }
+        }
+    };
+}
+
+type U1280 = op!(U128 * U10);
+type U1281 = op!(U1280 + U1);
+
 #[derive(Debug, Clone, Deserialize)]
 #[context_deserialize(ForkName)]
 struct Metadata {
@@ -113,8 +155,15 @@ macro_rules! type_dispatch {
             "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*),
             "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*),
             "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*),
-            // EIP-7916 is still in draft and hasn't been implemented yet https://eips.ethereum.org/EIPS/eip-7916
-            "ProgressiveTestStruct" | "ProgressiveBitsStruct" => Err(Error::SkippedKnownFailure),
+            "ProgressiveBitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveBitsStruct>, $($rest)*),
+            "ProgressiveSingleFieldContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleFieldContainerTestStruct>, $($rest)*),
+            "ProgressiveSingleListContainerTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveSingleListContainerTestStruct>, $($rest)*),
+            "ProgressiveVarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveVarTestStruct>, $($rest)*),
+            "ProgressiveComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveComplexTestStruct>, $($rest)*),
+            "ProgressiveTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ProgressiveTestStruct>, $($rest)*),
+            "CompatibleUnionA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionA>, $($rest)*),
+            "CompatibleUnionBC" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionBC>, $($rest)*),
+            "CompatibleUnionABCA" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* CompatibleUnionABCA>, $($rest)*),
             _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))),
         }
     };
@@ -159,6 +208,17 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "basic_progressive_list" => {
+                let elem_ty = parts[1];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveList,
+                    <>,
+                    [elem_ty => primitive_type]
+                )?;
+            }
             "bitlist" => {
                 let mut limit = parts[1];
 
@@ -186,6 +246,14 @@ impl Case for SszGeneric {
                     [length => typenum]
                 )?;
             }
+            "progressive_bitlist" => {
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    ProgressiveBitList,
+                    <>,
+                )?;
+            }
             "boolean" => {
                 ssz_generic_test::<bool>(&self.path, fork_name)?;
             }
@@ -211,6 +279,28 @@ impl Case for SszGeneric {
                     [type_name => test_container]
                 )?;
             }
+            "progressive_containers" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
+            "compatible_unions" => {
+                let type_name = parts[0];
+
+                type_dispatch!(
+                    ssz_generic_test,
+                    (&self.path, fork_name),
+                    _,
+                    <>,
+                    [type_name => test_container]
+                )?;
+            }
             _ => panic!("unsupported handler: {}", self.handler_name),
         }
         Ok(())
@@ -302,6 +392,15 @@ struct ComplexTestStruct {
     G: Vector<VarTestStruct, U2>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveTestStruct {
+    A: ProgressiveList<u8>,
+    B: ProgressiveList<u64>,
+    C: ProgressiveList<SmallTestStruct>,
+    D: ProgressiveList<ProgressiveList<VarTestStruct>>,
+}
+
 #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
 #[context_deserialize(ForkName)]
 struct BitsStruct {
@@ -312,6 +411,120 @@ struct BitsStruct {
     E: BitVector<U8>,
 }
 
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[context_deserialize(ForkName)]
+struct ProgressiveBitsStruct {
+    A: BitVector<U256>,
+    B: BitList<U256>,
+    C: ProgressiveBitList,
+    D: BitVector<U257>,
+    E: BitList<U257>,
+    F: ProgressiveBitList,
+    G: BitVector<U1280>,
+    H: BitList<U1280>,
+    I: ProgressiveBitList,
+    J: BitVector<U1281>,
+    K: BitList<U1281>,
+    L: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(struct_behaviour = "progressive_container", active_fields(1))]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleFieldContainerTestStruct {
+    A: u8,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(0, 0, 0, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveSingleListContainerTestStruct {
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveVarTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)]
+#[tree_hash(
+    struct_behaviour = "progressive_container",
+    active_fields(1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1)
+)]
+#[context_deserialize(ForkName)]
+struct ProgressiveComplexTestStruct {
+    A: u8,
+    B: List<u16, U123>,
+    C: ProgressiveBitList,
+    D: ProgressiveList<u64>,
+    E: ProgressiveList<SmallTestStruct>,
+    F: ProgressiveList<ProgressiveList<VarTestStruct>>,
+    G: List<ProgressiveSingleFieldContainerTestStruct, U10>,
+    H: ProgressiveList<ProgressiveVarTestStruct>,
+}
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionA {
+    #[ssz(selector = "1")]
+    ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionA, {
+    1 => ProgressiveSingleFieldContainerTestStruct(ProgressiveSingleFieldContainerTestStruct),
+});
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionBC {
+    #[ssz(selector = "2")]
+    ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct),
+    #[ssz(selector = "3")]
+    ProgressiveVarTestStruct(ProgressiveVarTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionBC, {
+    2 => ProgressiveSingleListContainerTestStruct(ProgressiveSingleListContainerTestStruct),
+    3 => ProgressiveVarTestStruct(ProgressiveVarTestStruct),
+});
+
+#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash)]
+#[ssz(enum_behaviour = "compatible_union")]
+#[tree_hash(enum_behaviour = "compatible_union")]
+#[context_deserialize(ForkName)]
+enum CompatibleUnionABCA {
+    #[ssz(selector = "1")]
+    A1(ProgressiveSingleFieldContainerTestStruct),
+    #[ssz(selector = "2")]
+    B1(ProgressiveSingleListContainerTestStruct),
+    #[ssz(selector = "3")]
+    C1(ProgressiveVarTestStruct),
+    #[ssz(selector = "4")]
+    A2(ProgressiveSingleFieldContainerTestStruct),
+}
+
+impl_compatible_union_deserialize!(CompatibleUnionABCA, {
+    1 => A1(ProgressiveSingleFieldContainerTestStruct),
+    2 => B1(ProgressiveSingleListContainerTestStruct),
+    3 => C1(ProgressiveVarTestStruct),
+    4 => A2(ProgressiveSingleFieldContainerTestStruct),
+});
+
 fn byte_list_from_hex_str<'de, D, N: Unsigned>(
     deserializer: D,
 ) -> Result<VariableList<u8, N>, D::Error>

testing/ef_tests/src/handler.rs

@@ -1182,13 +1182,21 @@ impl<H: TypeName> Handler for SszGenericHandler<H> {
 // Supported SSZ generic handlers
 pub struct BasicVector;
 type_name!(BasicVector, "basic_vector");
+pub struct BasicProgressiveList;
+type_name!(BasicProgressiveList, "basic_progressive_list");
 pub struct Bitlist;
 type_name!(Bitlist, "bitlist");
 pub struct Bitvector;
 type_name!(Bitvector, "bitvector");
+pub struct ProgressiveBitlist;
+type_name!(ProgressiveBitlist, "progressive_bitlist");
 pub struct Boolean;
 type_name!(Boolean, "boolean");
 pub struct Uints;
 type_name!(Uints, "uints");
 pub struct Containers;
 type_name!(Containers, "containers");
+pub struct ProgressiveContainers;
+type_name!(ProgressiveContainers, "progressive_containers");
+pub struct CompatibleUnions;
+type_name!(CompatibleUnions, "compatible_unions");