pubsys: added SSM parameter validation

Added a validate-ssm command to validate SSM parameters, given
a JSON config file with regions and paths to files containing
the expected parameters.

tools/pubsys/Cargo.toml

@@ -41,6 +41,7 @@ serde_json = "1"
 simplelog = "0.12"
 snafu = "0.7"
 structopt = { version = "0.3", default-features = false }
+tabled = "0.10"
 tempfile = "3"
 tinytemplate = "1"
 tokio = { version = "1", features = ["full"] }  # LTS

tools/pubsys/src/aws/mod.rs

@@ -8,6 +8,7 @@ pub(crate) mod ami;
 pub(crate) mod promote_ssm;
 pub(crate) mod publish_ami;
 pub(crate) mod ssm;
+pub(crate) mod validate_ssm;
 
 /// Builds a Region from the given region name.
 fn region_from_string(name: &str) -> Region {

tools/pubsys/src/aws/ssm/mod.rs

@@ -297,7 +297,7 @@ pub(crate) struct BuildContext<'a> {
 }
 
 /// A map of SsmKey to its value
-type SsmParameters = HashMap<SsmKey, String>;
+pub(crate) type SsmParameters = HashMap<SsmKey, String>;
 
 /// Parse the AMI input file
 fn parse_ami_input(regions: &[String], ssm_args: &SsmArgs) -> Result<HashMap<Region, Image>> {

tools/pubsys/src/aws/ssm/ssm.rs

@@ -7,8 +7,8 @@ use aws_sdk_ssm::output::{GetParametersOutput, PutParameterOutput};
 use aws_sdk_ssm::types::SdkError;
 use aws_sdk_ssm::{Client as SsmClient, Region};
 use futures::future::{join, ready};
-use futures::stream::{self, StreamExt};
-use log::{debug, error, trace, warn};
+use futures::stream::{self, FuturesUnordered, StreamExt};
+use log::{debug, error, info, trace, warn};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
@@ -135,6 +135,88 @@ where
     Ok(parameters)
 }
 
+/// Fetches all SSM parameters under a given prefix using the given clients
+pub(crate) async fn get_parameters_by_prefix<'a>(
+    clients: &'a HashMap<Region, SsmClient>,
+    ssm_prefix: &str,
+) -> HashMap<&'a Region, Result<SsmParameters>> {
+    // Build requests for parameters; we have to request with a regional client so we split them by
+    // region
+    let mut requests = Vec::with_capacity(clients.len());
+    for region in clients.keys() {
+        trace!("Requesting parameters in {}", region);
+        let ssm_client: &SsmClient = &clients[region];
+        let get_future = get_parameters_by_prefix_in_region(region, ssm_client, ssm_prefix);
+
+        requests.push(join(ready(region), get_future));
+    }
+
+    // Send requests in parallel and wait for responses, collecting results into a list.
+    requests
+        .into_iter()
+        .collect::<FuturesUnordered<_>>()
+        .collect()
+        .await
+}
+
+/// Fetches all SSM parameters under a given prefix in a single region
+pub(crate) async fn get_parameters_by_prefix_in_region(
+    region: &Region,
+    client: &SsmClient,
+    ssm_prefix: &str,
+) -> Result<SsmParameters> {
+    info!("Retrieving SSM parameters in {}", region.to_string());
+    let mut parameters = HashMap::new();
+
+    // Send the request
+    let mut get_future = client
+        .get_parameters_by_path()
+        .path(ssm_prefix)
+        .recursive(true)
+        .into_paginator()
+        .send();
+
+    // Iterate over the retrieved parameters
+    while let Some(page) = get_future.next().await {
+        let retrieved_parameters = page
+            .context(error::GetParametersByPathSnafu {
+                path: ssm_prefix,
+                region: region.to_string(),
+            })?
+            .parameters()
+            .unwrap_or_default()
+            .to_owned();
+        for parameter in retrieved_parameters {
+            // Insert a new key-value pair into the map, with the key containing region and parameter name
+            // and the value containing the parameter value
+            parameters.insert(
+                SsmKey::new(
+                    region.to_owned(),
+                    parameter
+                        .name()
+                        .ok_or(error::Error::MissingField {
+                            region: region.to_string(),
+                            field: "name".to_string(),
+                        })?
+                        .to_owned(),
+                ),
+                parameter
+                    .value()
+                    .ok_or(error::Error::MissingField {
+                        region: region.to_string(),
+                        field: "value".to_string(),
+                    })?
+                    .to_owned(),
+            );
+        }
+    }
+    info!(
+        "SSM parameters in {} have been retrieved",
+        region.to_string()
+    );
+    Ok(parameters)
+}
+
 /// Sets the values of the given SSM keys using the given clients
 pub(crate) async fn set_parameters(
     parameters_to_set: &SsmParameters,
@@ -324,8 +406,8 @@ pub(crate) async fn validate_parameters(
     Ok(())
 }
 
-mod error {
-    use aws_sdk_ssm::error::GetParametersError;
+pub(crate) mod error {
+    use aws_sdk_ssm::error::{GetParametersByPathError, GetParametersError};
     use aws_sdk_ssm::types::SdkError;
     use snafu::Snafu;
     use std::error::Error as _;
@@ -334,13 +416,28 @@ mod error {
     #[derive(Debug, Snafu)]
     #[snafu(visibility(pub(super)))]
     #[allow(clippy::large_enum_variant)]
-    pub(crate) enum Error {
+    pub enum Error {
         #[snafu(display("Failed to fetch SSM parameters in {}: {}", region, source.source().map(|x| x.to_string()).unwrap_or("unknown".to_string())))]
         GetParameters {
             region: String,
             source: SdkError<GetParametersError>,
         },
 
+        #[snafu(display(
+            "Failed to fetch SSM parameters by path {} in {}: {}",
+            path,
+            region,
+            source
+        ))]
+        GetParametersByPath {
+            path: String,
+            region: String,
+            source: SdkError<GetParametersByPathError>,
+        },
+
+        #[snafu(display("Missing field in parameter in {}: {}", region, field))]
+        MissingField { region: String, field: String },
+
         #[snafu(display("Response to {} was missing {}", request_type, missing))]
         MissingInResponse {
             region: String,
@@ -369,4 +466,4 @@ mod error {
     }
 }
 pub(crate) use error::Error;
-type Result<T> = std::result::Result<T, error::Error>;
+pub(crate) type Result<T> = std::result::Result<T, error::Error>;