Spec tests Framework

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "anchor/spec_tests/src/ssv-spec"]
+	path = anchor/spec_tests/src/ssv-spec
+	url = https://github.com/ssvlabs/ssv-spec.git

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "anchor/processor",
     "anchor/qbft_manager",
     "anchor/signature_collector",
+    "anchor/spec_tests",
     "anchor/subnet_tracker",
     "anchor/validator_store",
 ]

anchor/spec_tests/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "spec_tests"
+version = "0.1.0"
+edition = { workspace = true }
+authors = ["Sigma Prime <contact@sigmaprime.io>"]
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+walkdir = "2.5.0"

anchor/spec_tests/src/lib.rs

@@ -0,0 +1,121 @@
+#![allow(dead_code)]
+
+mod qbft;
+use std::{collections::HashMap, fs, path::Path, sync::LazyLock};
+
+use qbft::QbftSpecTestType;
+use serde::de::DeserializeOwned;
+use walkdir::WalkDir;
+
+use crate::qbft::*;
+
+// All Spec Test Variants. Maps to an inner variant type that describes specific tests
+#[derive(Eq, PartialEq, Hash)]
+enum SpecTestType {
+    Qbft(QbftSpecTestType),
+}
+
+// Impl display for path construction. Do not change
+impl SpecTestType {
+    fn path(&self) -> &'static Path {
+        match self {
+            SpecTestType::Qbft(_) => Path::new("src/ssv-spec/qbft/spectest/generate/tests"),
+        }
+    }
+}
+
+// Core trait to orchestrate setting up and running spec tests. The spec tests are broken up into
+// different categories with different file strucutres. For each file structure, implementing the
+// required functions allows for a smooth testing process
+trait SpecTest {
+    // Retrieve the name of the test
+    fn name(&self) -> &str;
+
+    // Setup a runner for the test. This will configure and construct eveything required to
+    // execute the test
+    fn setup(&mut self);
+
+    // Run the test and verify that the output is what we were expecting.
+    fn run(&self) -> bool;
+
+    // Return the type of this test. Used as a Key for the loaders and path construction
+    fn test_type() -> SpecTestType
+    where
+        Self: Sized;
+}
+
+// Abstract away repeated logic for registering a test type with the loader
+macro_rules! register_test_loaders {
+    ($($test_type:ty),* $(,)?) => {
+        LazyLock::new(|| {
+            let mut loaders = HashMap::new();
+            $(
+                register_test::<$test_type>(&mut loaders);
+            )*
+            loaders
+        })
+    };
+}
+
+type Loaders = HashMap<SpecTestType, fn(&str) -> Box<dyn SpecTest>>;
+static TEST_LOADERS: LazyLock<Loaders> = register_test_loaders!(TimeoutTest);
+
+// Register a test in the loader. This inserts a mapping from SpecTestType -> loading closure
+// into a map for later access. This is needed to that we can parse from an arbitrary test file to a
+// specific test type T
+fn register_test<T: SpecTest + DeserializeOwned + 'static>(map: &mut Loaders) {
+    map.insert(T::test_type(), |path| {
+        let contents = fs::read_to_string(path)
+            .unwrap_or_else(|_| panic!("Failed to read test file: {}", path));
+        let test: T = serde_json::from_str(&contents)
+            .unwrap_or_else(|e| panic!("Failed to parse test {}: {}", path, e));
+        Box::new(test)
+    });
+}
+
+// Core function to run the tests. Given a SpecTestType, it will navigate to the proper directory,
+// read in all of the tests, make sure they are all setup, and then run each one
+fn run_tests(test_type: SpecTestType) -> bool {
+    let tests: Vec<Box<dyn SpecTest>> = WalkDir::new(test_type.path())
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter_map(|entry| {
+            let path = entry.path();
+
+            // Get the inner variant string to check in filenames
+            let variant = match &test_type {
+                SpecTestType::Qbft(inner) => inner.to_string(),
+            };
+
+            if path.is_file()
+                && path
+                    .file_name()
+                    .map(|name| name.to_string_lossy().contains(&variant))
+                    .unwrap_or(false)
+            {
+                let loader = TEST_LOADERS.get(&test_type).unwrap_or_else(|| {
+                    panic!("No loader registered for:{}", path.to_string_lossy())
+                });
+                Some(loader(&path.to_string_lossy()))
+            } else {
+                None
+            }
+        })
+        .collect();
+    // todo!() do the setup
+    let mut result = true;
+    for test in tests {
+        result &= test.run();
+    }
+    result
+}
+
+#[cfg(test)]
+mod spec_tests {
+    use super::*;
+
+    #[test]
+    fn test_qbft_timeout() {
+        assert!(run_tests(SpecTestType::Qbft(QbftSpecTestType::Timeout)))
+    }
+}

anchor/spec_tests/src/qbft/controller.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for ControllerTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::Controller)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ControllerTest {
+    pub name: String,
+}

anchor/spec_tests/src/qbft/create_message.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for CreateMessageTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::CreateMessage)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CreateMessageTest {
+    pub name: String,
+}

anchor/spec_tests/src/qbft/message_processing.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for MessageProcessingTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::MessageProcessing)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MessageProcessingTest {
+    pub name: String,
+}

anchor/spec_tests/src/qbft/mod.rs

@@ -0,0 +1,34 @@
+// QBFT test variants
+// Modules contain parsing logic
+mod controller;
+mod create_message;
+mod message_processing;
+mod qbft_message;
+mod round_robin;
+mod timeout;
+
+pub use timeout::TimeoutTest;
+
+#[derive(Eq, PartialEq, Hash)]
+pub(crate) enum QbftSpecTestType {
+    Timeout,
+    QbftMessage,
+    MessageProcessing,
+    CreateMessage,
+    Controller,
+    RoundRobin,
+}
+
+// Contains specific identifier for the test file
+impl std::fmt::Display for QbftSpecTestType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            QbftSpecTestType::Timeout => write!(f, "timeout"),
+            QbftSpecTestType::QbftMessage => write!(f, "MsgSpecTest"),
+            QbftSpecTestType::MessageProcessing => write!(f, "MsgProcessingSpecTest"),
+            QbftSpecTestType::CreateMessage => write!(f, "CreateMsgSpecTest"),
+            QbftSpecTestType::Controller => write!(f, "ControllerSpecTest"),
+            QbftSpecTestType::RoundRobin => write!(f, "RoundRobinSpecTest"),
+        }
+    }
+}

anchor/spec_tests/src/qbft/qbft_message.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for QbftMessageTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::QbftMessage)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct QbftMessageTest {
+    name: String,
+}

anchor/spec_tests/src/qbft/round_robin.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for RoundRobinTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::RoundRobin)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RoundRobinTest {
+    name: String,
+}

anchor/spec_tests/src/qbft/timeout.rs

@@ -0,0 +1,24 @@
+use serde::{Deserialize, Serialize};
+
+use crate::{QbftSpecTestType, SpecTest, SpecTestType};
+
+impl SpecTest for TimeoutTest {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn run(&self) -> bool {
+        true
+    }
+
+    fn setup(&mut self) {}
+
+    fn test_type() -> SpecTestType {
+        SpecTestType::Qbft(QbftSpecTestType::Timeout)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TimeoutTest {
+    name: String,
+}