test(e2e): Add end-to-end tests simulating neural-network workloads on a Spider cluster.

tests/huntsman/e2e/Cargo.toml

@@ -4,8 +4,16 @@ version = { workspace = true }
 edition = { workspace = true }
 publish = false
 
+[[test]]
+name = "nn"
+test = false
+
 [dependencies]
 anyhow = { workspace = true }
+huntsman-nn-core = { path = "../../../examples/huntsman/nn/core" }
+rand = { workspace = true }
+rmp-serde = { workspace = true }
+serde = { workspace = true }
 spider-client = { workspace = true }
 spider-core = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }

tests/huntsman/e2e/src/lib.rs

@@ -1,7 +1,10 @@
 //! End-to-end integration-test harness for the huntsman suites.
 
+pub mod nn;
+pub mod payload_serde;
 pub mod test_driver;
 mod types;
 
+pub use payload_serde::*;
 pub use test_driver::SpiderTestDriver;
 pub use types::*;

tests/huntsman/e2e/src/nn/mod.rs

@@ -0,0 +1,11 @@
+//! Self-contained neural-network model for the end-to-end test.
+//!
+//! [`NeuralNetwork`] builds a layered `neuron::dense_*` task graph and reproduces it in-process via
+//! [`NeuralNetwork::simulate`].
+
+mod network;
+mod neuron;
+mod wiring;
+
+pub use network::NeuralNetwork;
+pub use neuron::Neuron;

tests/huntsman/e2e/src/nn/network.rs

@@ -0,0 +1,179 @@
+//! The neural-network model: a layered topology of `neuron::dense_*` neurons whose Spider
+//! [`TaskGraph`] and in-process simulation describe the same DAG.
+
+use huntsman_nn_core::NUM_INPUTS;
+use rand::SeedableRng;
+use rand::rngs::StdRng;
+use spider_core::task::DataTypeDescriptor;
+use spider_core::task::TaskDescriptor;
+use spider_core::task::TaskGraph;
+use spider_core::task::TaskIndex;
+use spider_core::task::TaskInputOutputIndex;
+use spider_core::task::TdlContext;
+use spider_core::task::ValueTypeDescriptor;
+
+use crate::nn::Neuron;
+use crate::nn::wiring;
+
+/// A randomly-wired, layered neural network of `neuron::dense_*` neurons.
+pub struct NeuralNetwork {
+    /// The layers in layer order.
+    layers: Vec<Layer>,
+}
+
+impl NeuralNetwork {
+    /// Factory function.
+    ///
+    /// Validates `layer_specs` via [`wiring::validate`] and generates the inner-layer fan-in
+    /// wiring deterministically from `seed`.
+    ///
+    /// # Returns
+    ///
+    /// The newly created [`NeuralNetwork`] on success.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    ///
+    /// * Forwards [`wiring::validate`]'s return values on failure.
+    pub fn new(layer_specs: Vec<(usize, Neuron)>, seed: u64) -> anyhow::Result<Self> {
+        let sizes: Vec<usize> = layer_specs.iter().map(|(size, _)| *size).collect();
+        wiring::validate(&sizes)?;
+        let mut rng = StdRng::seed_from_u64(seed);
+        let fan_ins = wiring::build_wiring(&sizes, &mut rng);
+        let layers = layer_specs
+            .into_iter()
+            .zip(fan_ins)
+            .map(|((neuron_count, activation), fan_in)| Layer {
+                neuron_count,
+                activation,
+                fan_in,
+            })
+            .collect();
+        Ok(Self { layers })
+    }
+
+    /// # Returns
+    ///
+    /// The number of graph inputs.
+    #[must_use]
+    pub fn num_graph_inputs(&self) -> usize {
+        self.layers[0].neuron_count * NUM_INPUTS
+    }
+
+    /// Builds the Spider [`TaskGraph`] for this network.
+    ///
+    /// # Returns
+    ///
+    /// The [`TaskGraph`] for this network on success.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    ///
+    /// * Forwards [`TaskGraph::new`]'s return values on failure.
+    /// * Forwards [`TaskGraph::insert_task`]'s return values on failure.
+    pub fn to_task_graph(&self) -> anyhow::Result<TaskGraph> {
+        let float64 = DataTypeDescriptor::Value(ValueTypeDescriptor::float64());
+        let mut graph = TaskGraph::new(None, None)?;
+        let first_layer = &self.layers[0];
+        let mut prev_layer: Vec<TaskIndex> = Vec::with_capacity(first_layer.neuron_count);
+
+        for _ in 0..first_layer.neuron_count {
+            let task_idx = graph.insert_task(TaskDescriptor {
+                tdl_context: TdlContext {
+                    package: PACKAGE.to_owned(),
+                    task_func: first_layer.activation.task_name().to_owned(),
+                },
+                execution_policy: None,
+                inputs: vec![float64.clone(); NUM_INPUTS],
+                outputs: vec![float64.clone()],
+                input_sources: None,
+            })?;
+            prev_layer.push(task_idx);
+        }
+
+        for layer in self.layers.iter().skip(1) {
+            let mut curr_layer = Vec::with_capacity(layer.neuron_count);
+            for j in 0..layer.neuron_count {
+                let input_sources: Vec<TaskInputOutputIndex> = layer.fan_in[j]
+                    .iter()
+                    .map(|&src| TaskInputOutputIndex {
+                        task_idx: prev_layer[src],
+                        position: 0,
+                    })
+                    .collect();
+                let task_idx = graph.insert_task(TaskDescriptor {
+                    tdl_context: TdlContext {
+                        package: PACKAGE.to_owned(),
+                        task_func: layer.activation.task_name().to_owned(),
+                    },
+                    execution_policy: None,
+                    inputs: vec![float64.clone(); NUM_INPUTS],
+                    outputs: vec![float64.clone()],
+                    input_sources: Some(input_sources),
+                })?;
+                curr_layer.push(task_idx);
+            }
+            prev_layer = curr_layer;
+        }
+
+        Ok(graph)
+    }
+
+    /// Computes the network's outputs from graph inputs.
+    ///
+    /// # Returns
+    ///
+    /// The network's outputs on success.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    ///
+    /// * [`anyhow::Error`] if `inputs` length is not [`Self::num_graph_inputs`].
+    pub fn simulate(&self, inputs: &[f64]) -> anyhow::Result<Vec<f64>> {
+        let expected = self.num_graph_inputs();
+        anyhow::ensure!(
+            inputs.len() == expected,
+            "expected {expected} graph inputs, got {}",
+            inputs.len(),
+        );
+
+        let first_layer = &self.layers[0];
+        let mut layer_outputs: Vec<f64> = (0..first_layer.neuron_count)
+            .map(|i| {
+                let start = i * NUM_INPUTS;
+                let mut neuron_inputs = [0.0_f64; NUM_INPUTS];
+                neuron_inputs.copy_from_slice(&inputs[start..start + NUM_INPUTS]);
+                first_layer.activation.evaluate_func()(&neuron_inputs)
+            })
+            .collect();
+
+        for layer in self.layers.iter().skip(1) {
+            layer_outputs = (0..layer.neuron_count)
+                .map(|i| {
+                    let neuron_inputs: [f64; NUM_INPUTS] =
+                        std::array::from_fn(|j| layer_outputs[layer.fan_in[i][j]]);
+                    layer.activation.evaluate_func()(&neuron_inputs)
+                })
+                .collect();
+        }
+
+        Ok(layer_outputs)
+    }
+}
+
+/// Name of the TDL package supplying the `neuron::dense_*` tasks.
+const PACKAGE: &str = "nn";
+
+/// One layer of the network: its neuron count, activation, and per-neuron fan-in.
+struct Layer {
+    /// Number of neurons in this layer.
+    neuron_count: usize,
+    /// Activation applied by every neuron in this layer.
+    activation: Neuron,
+    /// Per-neuron fan-in, listing previous-layer output indices feeding each neuron.
+    /// Empty for layer 0, which reads graph inputs directly.
+    fan_in: Vec<Vec<usize>>,
+}

tests/huntsman/e2e/src/nn/neuron.rs

@@ -0,0 +1,46 @@
+//! Activation functions for the end-to-end neural-network test workload.
+//!
+//! Each [`Neuron`] pairs a Spider `neuron::dense_*` task with the in-process
+//! `huntsman_nn_core::dense_*` evaluation function so the task graph and [`super::NeuralNetwork`]'s
+//! simulation share one source of truth.
+
+use huntsman_nn_core::NUM_INPUTS;
+
+/// A dense-layer neuron activation.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum Neuron {
+    /// Rectified-linear activation.
+    Relu,
+
+    /// Logistic-sigmoid activation.
+    Sigmoid,
+
+    /// Identity (no-op) activation.
+    Identity,
+}
+
+impl Neuron {
+    /// # Returns
+    ///
+    /// The `neuron::dense_*` task function name that evaluates this activation.
+    #[must_use]
+    pub const fn task_name(self) -> &'static str {
+        match self {
+            Self::Relu => "neuron::dense_relu",
+            Self::Sigmoid => "neuron::dense_sigmoid",
+            Self::Identity => "neuron::dense_identity",
+        }
+    }
+
+    /// # Returns
+    ///
+    /// The `huntsman_nn_core::dense_*` function that evaluates this activation.
+    #[must_use]
+    pub fn evaluate_func(self) -> fn(&[f64; NUM_INPUTS]) -> f64 {
+        match self {
+            Self::Relu => huntsman_nn_core::dense_relu,
+            Self::Sigmoid => huntsman_nn_core::dense_sigmoid,
+            Self::Identity => huntsman_nn_core::dense_identity,
+        }
+    }
+}