testsys: Add support for k8s workloads

TESTING.md

@@ -381,6 +381,26 @@ cargo make -e TESTSYS_TEST=migration test
 
 To see the state of the tests as they run use `cargo make watch-test`.
 
+### Testing Workloads
+
+Workload tests are tests designed to run as an orchestrated container.
+A workload test is defined in `Test.toml` with a map named `workloads`.
+
+```toml
+[aws-nvidia]
+workloads = { <WORKLOAD-NAME> = "<WORKLOAD-IMAGE-URI>" }
+```
+
+To run the workload test set `TESTSYS_TEST=workload` in the `cargo make test` call.
+
+```shell
+cargo make -e TESTSYS_TEST=workload test
+```
+
+To see the state of the tests as they run use `cargo make watch-test`.
+
+For more information can be found in the [TestSys workload documentation](https://github.com/bottlerocket-os/bottlerocket-test-system/tree/develop/bottlerocket/tests/workload).
+
 ### Custom Test Types
 
 Custom tests can be run with TestSys by calling `cargo make -e TESTSYS_TEST=<CUSTOM-TEST-NAME> test -f <PATH-TO-TEMPLATED-YAML>`.

tools/testsys-config/src/lib.rs

@@ -261,6 +261,9 @@ pub struct GenericVariantConfig {
     pub control_plane_endpoint: Option<String>,
     /// The path to userdata that should be used for Bottlerocket launch
     pub userdata: Option<String>,
+    /// The workload tests that should be run
+    #[serde(default)]
+    pub workloads: BTreeMap<String, String>,
     #[serde(default)]
     pub dev: DeveloperConfig,
 }
@@ -280,6 +283,12 @@ impl GenericVariantConfig {
             self.secrets
         };
 
+        let workloads = if self.workloads.is_empty() {
+            other.workloads
+        } else {
+            self.workloads
+        };
+
         Self {
             cluster_names,
             instance_type: self.instance_type.or(other.instance_type),
@@ -289,6 +298,7 @@ impl GenericVariantConfig {
             conformance_registry: self.conformance_registry.or(other.conformance_registry),
             control_plane_endpoint: self.control_plane_endpoint.or(other.control_plane_endpoint),
             userdata: self.userdata.or(other.userdata),
+            workloads,
             dev: self.dev.merge(other.dev),
         }
     }
@@ -353,6 +363,7 @@ pub struct TestsysImages {
     pub sonobuoy_test_agent_image: Option<String>,
     pub ecs_test_agent_image: Option<String>,
     pub migration_test_agent_image: Option<String>,
+    pub k8s_workload_agent_image: Option<String>,
     pub controller_image: Option<String>,
     pub testsys_agent_pull_secret: Option<String>,
 }
@@ -380,6 +391,7 @@ impl TestsysImages {
             sonobuoy_test_agent_image: Some(format!("{}/sonobuoy-test-agent:{tag}", registry)),
             ecs_test_agent_image: Some(format!("{}/ecs-test-agent:{tag}", registry)),
             migration_test_agent_image: Some(format!("{}/migration-test-agent:{tag}", registry)),
+            k8s_workload_agent_image: Some(format!("{}/k8s-workload-agent:{tag}", registry)),
             controller_image: Some(format!("{}/controller:{tag}", registry)),
             testsys_agent_pull_secret: None,
         }
@@ -409,6 +421,9 @@ impl TestsysImages {
             migration_test_agent_image: self
                 .migration_test_agent_image
                 .or(other.migration_test_agent_image),
+            k8s_workload_agent_image: self
+                .k8s_workload_agent_image
+                .or(other.k8s_workload_agent_image),
             controller_image: self.controller_image.or(other.controller_image),
             testsys_agent_pull_secret: self
                 .testsys_agent_pull_secret

tools/testsys/src/aws_ecs.rs

@@ -193,6 +193,12 @@ impl CrdCreator for AwsEcsCreator {
         Ok(CreateCrdOutput::NewCrd(Box::new(Crd::Test(test_crd))))
     }
 
+    async fn workload_crd<'a>(&self, _test_input: TestInput<'a>) -> Result<CreateCrdOutput> {
+        Err(error::Error::Invalid {
+            what: "Workload testing is not supported for non-k8s variants".to_string(),
+        })
+    }
+
     fn additional_fields(&self, _test_type: &str) -> BTreeMap<String, String> {
         btreemap! {"region".to_string() => self.region.clone()}
     }

tools/testsys/src/aws_k8s.rs

@@ -5,7 +5,7 @@ use crate::crds::{
 };
 use crate::error::{self, Result};
 use crate::migration::migration_crd;
-use crate::sonobuoy::sonobuoy_crd;
+use crate::sonobuoy::{sonobuoy_crd, workload_crd};
 use bottlerocket_types::agent_config::{
     ClusterType, CreationPolicy, EksClusterConfig, EksctlConfig, K8sVersion,
 };
@@ -169,6 +169,12 @@ impl CrdCreator for AwsK8sCreator {
         )?))))
     }
 
+    async fn workload_crd<'a>(&self, test_input: TestInput<'a>) -> Result<CreateCrdOutput> {
+        Ok(CreateCrdOutput::NewCrd(Box::new(Crd::Test(workload_crd(
+            test_input,
+        )?))))
+    }
+
     fn additional_fields(&self, _test_type: &str) -> BTreeMap<String, String> {
         btreemap! {"region".to_string() => self.region.clone()}
     }

tools/testsys/src/crds.rs

@@ -352,6 +352,9 @@ pub(crate) trait CrdCreator: Sync {
     /// Create a testing CRD for this variant of Bottlerocket.
     async fn test_crd<'a>(&self, test_input: TestInput<'a>) -> Result<CreateCrdOutput>;
 
+    /// Create a workload testing CRD for this variant of Bottlerocket.
+    async fn workload_crd<'a>(&self, test_input: TestInput<'a>) -> Result<CreateCrdOutput>;
+
     /// Create a set of additional fields that may be used by an externally defined agent on top of
     /// the ones in `CrdInput`
     fn additional_fields(&self, _test_type: &str) -> BTreeMap<String, String> {
@@ -411,6 +414,34 @@ pub(crate) trait CrdCreator: Sync {
                         crds.push(crd)
                     }
                 }
+                KnownTestType::Workload => {
+                    let bottlerocket_output = self
+                        .bottlerocket_crd(BottlerocketInput {
+                            cluster_crd_name: &cluster_crd_name,
+                            image_id: self.image_id(crd_input)?,
+                            test_type,
+                            crd_input,
+                        })
+                        .await?;
+                    let bottlerocket_crd_name = bottlerocket_output.crd_name();
+                    if let Some(crd) = bottlerocket_output.crd() {
+                        debug!("Bottlerocket crd was created for '{}'", cluster_name);
+                        crds.push(crd)
+                    }
+                    let test_output = self
+                        .workload_crd(TestInput {
+                            cluster_crd_name: &cluster_crd_name,
+                            bottlerocket_crd_name: &bottlerocket_crd_name,
+                            test_type,
+                            crd_input,
+                            prev_tests: Default::default(),
+                            name_suffix: None,
+                        })
+                        .await?;
+                    if let Some(crd) = test_output.crd() {
+                        crds.push(crd)
+                    }
+                }
                 KnownTestType::Migration => {
                     let image_id = if let Some(image_id) = &crd_input.starting_image_id {
                         debug!(

tools/testsys/src/run.rs

@@ -151,6 +151,10 @@ struct CliConfig {
     /// Specify the path to the userdata that should be added for Bottlerocket launch
     #[clap(long, env = "TESTSYS_USERDATA")]
     pub userdata: Option<String>,
+
+    /// A set of workloads that should be run for a workload test (--workload my-workload=<WORKLOAD-IMAGE>)
+    #[clap(long = "workload", parse(try_from_str = parse_workloads), number_of_values = 1)]
+    pub workloads: Vec<(String, String)>,
 }
 
 impl From<CliConfig> for GenericVariantConfig {
@@ -165,6 +169,7 @@ impl From<CliConfig> for GenericVariantConfig {
             control_plane_endpoint: val.control_plane_endpoint,
             userdata: val.userdata,
             dev: Default::default(),
+            workloads: val.workloads.into_iter().collect(),
         }
     }
 }
@@ -400,6 +405,17 @@ fn parse_key_val(s: &str) -> Result<(String, SecretName)> {
     ))
 }
 
+fn parse_workloads(s: &str) -> Result<(String, String)> {
+    let mut iter = s.splitn(2, '=');
+    let key = iter.next().context(error::InvalidSnafu {
+        what: "Key is missing",
+    })?;
+    let value = iter.next().context(error::InvalidSnafu {
+        what: "Value is missing",
+    })?;
+    Ok((key.to_string(), value.to_string()))
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum KnownTestType {
@@ -415,6 +431,8 @@ pub enum KnownTestType {
     /// be created at the starting version, migrated to the target version and back to the starting
     /// version with validation testing.
     Migration,
+    /// Workload testing is used to test specific workloads on a set of Bottlerocket nodes.
+    Workload,
 }
 
 /// If a test type is one that is supported by TestSys it will be created as `Known(KnownTestType)`.
@@ -486,6 +504,13 @@ pub(crate) struct TestsysImages {
     )]
     pub(crate) migration_test: Option<String>,
 
+    /// K8s workload agent URI. If not provided the latest released test agent will be used.
+    #[clap(
+        long = "k8s-workload-agent-image",
+        env = "TESTSYS_K8S_WORKLOAD_AGENT_IMAGE"
+    )]
+    pub(crate) k8s_workload: Option<String>,
+
     /// TestSys controller URI. If not provided the latest released controller will be used.
     #[clap(long = "controller-image", env = "TESTSYS_CONTROLLER_IMAGE")]
     pub(crate) controller_uri: Option<String>,
@@ -509,6 +534,7 @@ impl From<TestsysImages> for testsys_config::TestsysImages {
             sonobuoy_test_agent_image: val.sonobuoy_test,
             ecs_test_agent_image: val.ecs_test,
             migration_test_agent_image: val.migration_test,
+            k8s_workload_agent_image: val.k8s_workload,
             controller_image: val.controller_uri,
             testsys_agent_pull_secret: val.secret,
         }

tools/testsys/src/sonobuoy.rs

@@ -1,7 +1,9 @@
 use crate::crds::TestInput;
 use crate::error::{self, Result};
 use crate::run::KnownTestType;
-use bottlerocket_types::agent_config::{SonobuoyConfig, SonobuoyMode};
+use bottlerocket_types::agent_config::{
+    SonobuoyConfig, SonobuoyMode, WorkloadConfig, WorkloadTest,
+};
 use maplit::btreemap;
 use model::Test;
 use snafu::ResultExt;
@@ -19,7 +21,9 @@ pub(crate) fn sonobuoy_crd(test_input: TestInput) -> Result<Test> {
         .expect("A cluster name is required for migrations");
     let sonobuoy_mode = match test_input.test_type {
         KnownTestType::Conformance => SonobuoyMode::CertifiedConformance,
-        KnownTestType::Quick | KnownTestType::Migration => SonobuoyMode::Quick,
+        KnownTestType::Quick | KnownTestType::Migration | KnownTestType::Workload => {
+            SonobuoyMode::Quick
+        }
     };
 
     let labels = test_input.crd_input.labels(btreemap! {
@@ -83,6 +87,74 @@ pub(crate) fn sonobuoy_crd(test_input: TestInput) -> Result<Test> {
         })
 }
 
+/// Create a workload CRD for K8s testing.
+pub(crate) fn workload_crd(test_input: TestInput) -> Result<Test> {
+    let cluster_resource_name = test_input
+        .cluster_crd_name
+        .as_ref()
+        .expect("A cluster name is required for migrations");
+    let bottlerocket_resource_name = test_input
+        .bottlerocket_crd_name
+        .as_ref()
+        .expect("A cluster name is required for migrations");
+
+    let labels = test_input.crd_input.labels(btreemap! {
+        "testsys/type".to_string() => test_input.test_type.to_string(),
+        "testsys/cluster".to_string() => cluster_resource_name.to_string(),
+    });
+    let plugins: Vec<_> = test_input
+        .crd_input
+        .config
+        .workloads
+        .iter()
+        .map(|(name, image)| WorkloadTest {
+            name: name.to_string(),
+            image: image.to_string(),
+        })
+        .collect();
+    if plugins.is_empty() {
+        return Err(error::Error::Invalid {
+            what: "There were no plugins specified in the workload test.
+            Workloads can be specified in `Test.toml` or via the command line."
+                .to_string(),
+        });
+    }
+
+    WorkloadConfig::builder()
+        .resources(bottlerocket_resource_name)
+        .resources(cluster_resource_name)
+        .set_depends_on(Some(test_input.prev_tests))
+        .set_retries(Some(5))
+        .image(
+            test_input
+                .crd_input
+                .images
+                .k8s_workload_agent_image
+                .to_owned()
+                .expect("The default K8s workload testing image is missing"),
+        )
+        .set_image_pull_secret(
+            test_input
+                .crd_input
+                .images
+                .testsys_agent_pull_secret
+                .to_owned(),
+        )
+        .keep_running(true)
+        .kubeconfig_base64_template(cluster_resource_name, "encodedKubeconfig")
+        .plugins(plugins)
+        .set_secrets(Some(test_input.crd_input.config.secrets.to_owned()))
+        .set_labels(Some(labels))
+        .build(format!(
+            "{}{}",
+            cluster_resource_name,
+            test_input.name_suffix.unwrap_or("-test")
+        ))
+        .context(error::BuildSnafu {
+            what: "Workload CRD",
+        })
+}
+
 fn e2e_repo_config_base64<S>(e2e_registry: S) -> String
 where
     S: Display,

tools/testsys/src/vmware_k8s.rs

@@ -4,7 +4,7 @@ use crate::crds::{
 };
 use crate::error::{self, Result};
 use crate::migration::migration_crd;
-use crate::sonobuoy::sonobuoy_crd;
+use crate::sonobuoy::{sonobuoy_crd, workload_crd};
 use bottlerocket_types::agent_config::{
     CreationPolicy, CustomUserData, K8sVersion, VSphereK8sClusterConfig, VSphereK8sClusterInfo,
     VSphereVmConfig,
@@ -279,6 +279,12 @@ impl CrdCreator for VmwareK8sCreator {
         )?))))
     }
 
+    async fn workload_crd<'a>(&self, test_input: TestInput<'a>) -> Result<CreateCrdOutput> {
+        Ok(CreateCrdOutput::NewCrd(Box::new(Crd::Test(workload_crd(
+            test_input,
+        )?))))
+    }
+
     fn additional_fields(&self, _test_type: &str) -> BTreeMap<String, String> {
         btreemap! {"region".to_string() => self.region.clone()}
     }