update to async versions of tough and tuftool

deny.toml

@@ -69,18 +69,12 @@ skip = [
     { name = "base64" },
     # several dependencies are using an old version of bitflags
     { name = "bitflags", version = "=1.3" },
-    # tuftool is using an older version of clap
-    { name = "clap", version = "3" },
-    { name = "clap_derive", version = "3" },
-    { name = "clap_lex", version = "0.2" },
     # several dependencies are using an old version of serde_yaml
     { name = "serde_yaml", version = "=0.8" },
     # aws-sdk-rust is using an old version of fastrand
     { name = "fastrand", version = "=1.9" },
     # multiple deps are using an older version of hashbrown
     { name = "hashbrown", version = "=0.12" },
-    # tuftool is using an old clap (v3) which is using old hermit-abi
-    { name = "hermit-abi", version = "0.1" },
     # multiple deps are using an older version of indexmap
     { name = "indexmap", version = "1" },
     # kube-client uses an old version of redox_syscall
@@ -102,11 +96,18 @@ skip-tree = [
     # dependency tree because windows-sys has many sub-crates
     # that differ in major version.
     { name = "windows-sys" },
+
+    # TODO - remove
+    { name = "tough" }
 ]
 
 [sources]
 allow-git = [
     "https://github.com/bottlerocket-os/bottlerocket-test-system",
+
+    # TODO - remove
+    "https://github.com/webern/tough",
+    "https://github.com/phu-cinemo/tough",
 ]
 # Deny crates from unknown registries or git repositories.
 unknown-registry = "deny"

tools/pubsys/Cargo.toml

@@ -17,6 +17,7 @@ aws-sdk-sts = "0.28"
 aws-smithy-types = "0.55"
 aws-types = "0.55"
 buildsys = { path = "../buildsys", version = "0.1" }
+bytes = "1"
 chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
 clap = { version = "4", features = ["derive"] }
 coldsnap = { version = "0.6", default-features = false, features = ["aws-sdk-rust-rustls"] }
@@ -30,7 +31,6 @@ nonzero_ext = "0.3"
 num_cpus = "1"
 parse-datetime = { path = "../parse-datetime", version = "0.1" }
 pubsys-config = { path = "../pubsys-config/", version = "0.1" }
-rayon = "1"
 # Need to bring in reqwest with a TLS feature so tough can support TLS repos.
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "blocking"] }
 semver = "1"
@@ -42,11 +42,11 @@ snafu = "0.7"
 tabled = "0.10"
 tempfile = "3"
 tinytemplate = "1"
-tokio = { version = "1", features = ["full"] }  # LTS
+tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["time"] }
 toml = "0.8"
-tough = { version = "0.14", features = ["http"] }
-tough-kms = "0.6"
-tough-ssm = "0.9"
+tough = { version = "0.14", git = "https://github.com/phu-cinemo/tough.git", rev = "4704b1d26e48cc161e7e5d79a9c095ac35b54723", features = ["http"] }
+tough-kms = { version = "0.6", git = "https://github.com/webern/tough.git", rev = "3f4e35845bb15a4c3c4b5d7df0ed905c6d2cab8f" }
+tough-ssm = { version = "0.9", git = "https://github.com/webern/tough.git", rev = "3f4e35845bb15a4c3c4b5d7df0ed905c6d2cab8f" }
 update-metadata = { path = "../update-metadata/", version = "0.1" }
 url = { version = "2", features = ["serde"] }

tools/pubsys/src/aws/ami/mod.rs

@@ -85,7 +85,9 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
         Ok(amis) => {
             // Write the AMI IDs to file if requested
             if let Some(ref path) = ami_args.ami_output {
-                write_amis(path, &amis).context(error::WriteAmisSnafu { path })?;
+                write_amis(path, &amis)
+                    .await
+                    .context(error::WriteAmisSnafu { path })?;
             }
             Ok(())
         }

tools/pubsys/src/aws/ami/register.rs

@@ -8,6 +8,7 @@ use buildsys::manifest::{self, ImageFeature};
 use coldsnap::{SnapshotUploader, SnapshotWaiter};
 use log::{debug, info, warn};
 use snafu::{ensure, OptionExt, ResultExt};
+use tokio::fs;
 
 const ROOT_DEVICE_NAME: &str = "/dev/xvda";
 const DATA_DEVICE_NAME: &str = "/dev/xvdb";
@@ -48,9 +49,11 @@ async fn _register_image(
     let (os_volume_size, data_volume_size) = image_layout.publish_image_sizes_gib();
 
     let uefi_data =
-        std::fs::read_to_string(&ami_args.uefi_data).context(error::LoadUefiDataSnafu {
-            path: &ami_args.uefi_data,
-        })?;
+        fs::read_to_string(&ami_args.uefi_data)
+            .await
+            .context(error::LoadUefiDataSnafu {
+                path: &ami_args.uefi_data,
+            })?;
 
     debug!("Uploading images into EBS snapshots in {}", region);
     let uploader = SnapshotUploader::new(ebs_client);

tools/pubsys/src/aws/promote_ssm/mod.rs

@@ -114,6 +114,7 @@ pub(crate) async fn run(args: &Args, promote_args: &PromoteArgs) -> Result<()> {
     // in their naming
     let template_parameters =
         template::get_parameters(&promote_args.template_path, &source_build_context)
+            .await
             .context(error::FindTemplatesSnafu)?;
 
     if template_parameters.parameters.is_empty() {

tools/pubsys/src/aws/publish_ami/mod.rs

@@ -23,9 +23,10 @@ use log::{debug, error, info, trace};
 use pubsys_config::InfraConfig;
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, HashSet};
-use std::fs::File;
 use std::iter::FromIterator;
 use std::path::PathBuf;
+use tokio::fs;
+use tokio::fs::File;
 
 #[derive(Debug, Parser)]
 #[group(id = "who", required = true, multiple = true)]
@@ -82,12 +83,14 @@ pub(crate) async fn run(args: &Args, publish_args: &Who) -> Result<()> {
         "Using AMI data from path: {}",
         publish_args.ami_input.display()
     );
-    let file = File::open(&publish_args.ami_input).context(error::FileSnafu {
-        op: "open",
-        path: &publish_args.ami_input,
-    })?;
-    let mut ami_input: HashMap<String, Image> =
-        serde_json::from_reader(file).context(error::DeserializeSnafu {
+    let file = File::open(&publish_args.ami_input)
+        .await
+        .context(error::FileSnafu {
+            op: "open",
+            path: &publish_args.ami_input,
+        })?;
+    let mut ami_input: HashMap<String, Image> = serde_json::from_reader(file.into_std().await)
+        .context(error::DeserializeSnafu {
             path: &publish_args.ami_input,
         })?;
     trace!("Parsed AMI input: {:?}", ami_input);
@@ -218,19 +221,19 @@ pub(crate) async fn run(args: &Args, publish_args: &Who) -> Result<()> {
             .into_iter()
             .map(|(region, image)| (region.to_string(), image))
             .collect::<HashMap<String, Image>>(),
-    )?;
+    )
+    .await?;
 
     Ok(())
 }
 
-pub(crate) fn write_amis(path: &PathBuf, amis: &HashMap<String, Image>) -> Result<()> {
-    let file = File::create(path).context(error::FileSnafu {
+pub(crate) async fn write_amis(path: &PathBuf, amis: &HashMap<String, Image>) -> Result<()> {
+    let json = serde_json::to_string_pretty(&amis).context(error::SerializeSnafu { path })?;
+    fs::write(path, &json).await.context(error::FileSnafu {
         op: "write AMIs to file",
         path,
     })?;
-    serde_json::to_writer_pretty(file, &amis).context(error::SerializeSnafu { path })?;
     info!("Wrote AMI data to {}", path.display());
-
     Ok(())
 }
 

tools/pubsys/src/aws/ssm/mod.rs

@@ -119,6 +119,7 @@ pub(crate) async fn run(args: &Args, ssm_args: &SsmArgs) -> Result<()> {
         ssm_args.template_path.display()
     );
     let template_parameters = template::get_parameters(&ssm_args.template_path, &build_context)
+        .await
         .context(error::FindTemplatesSnafu)?;
 
     if template_parameters.parameters.is_empty() {

tools/pubsys/src/aws/ssm/template.rs

@@ -8,9 +8,9 @@ use log::trace;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, ResultExt};
 use std::collections::HashMap;
-use std::fs;
 use std::path::Path;
 use tinytemplate::TinyTemplate;
+use tokio::fs;
 
 /// Represents a single SSM parameter
 #[derive(Debug, Deserialize)]
@@ -36,14 +36,16 @@ pub(crate) struct TemplateParameters {
 
 /// Deserializes template parameters from the template file, taking into account conditional
 /// parameters that may or may not apply based on our build context.
-pub(crate) fn get_parameters(
+pub(crate) async fn get_parameters(
     template_path: &Path,
     build_context: &BuildContext<'_>,
 ) -> Result<TemplateParameters> {
-    let templates_str = fs::read_to_string(template_path).context(error::FileSnafu {
-        op: "read",
-        path: &template_path,
-    })?;
+    let templates_str = fs::read_to_string(template_path)
+        .await
+        .context(error::FileSnafu {
+            op: "read",
+            path: &template_path,
+        })?;
     let mut template_parameters: TemplateParameters =
         toml::from_str(&templates_str).context(error::InvalidTomlSnafu {
             path: &template_path,

tools/pubsys/src/aws/validate_ami/mod.rs

@@ -15,8 +15,8 @@ use log::{error, info, trace};
 use pubsys_config::InfraConfig;
 use snafu::ResultExt;
 use std::collections::{HashMap, HashSet};
-use std::fs::File;
 use std::path::PathBuf;
+use tokio::fs;
 
 /// Validates EC2 images by calling `describe-images` on all images in the file given by
 /// `expected-amis-path` and ensuring that the returned `public`, `ena-support`,
@@ -134,13 +134,13 @@ pub(crate) async fn validate(
         };
 
         // Write the results as JSON
-        serde_json::to_writer_pretty(
-            &File::create(write_results_path).context(error::WriteValidationResultsSnafu {
+        let json = serde_json::to_string_pretty(&results)
+            .context(error::SerializeValidationResultsSnafu)?;
+        fs::write(&write_results_path, &json).await.context(
+            error::WriteValidationResultsSnafu {
                 path: write_results_path,
-            })?,
-            &results,
-        )
-        .context(error::SerializeValidationResultsSnafu)?;
+            },
+        )?;
     }
 
     Ok(validation_results)
@@ -199,12 +199,14 @@ pub(crate) async fn parse_expected_amis(
     expected_amis_path: &PathBuf,
 ) -> Result<HashMap<Region, Vec<ImageDef>>> {
     // Parse the JSON file as a `HashMap` of region_name, mapped to an `ImageData` struct
-    let expected_amis: HashMap<RegionName, ImageData> = serde_json::from_reader(
-        &File::open(expected_amis_path.clone()).context(error::ReadExpectedImagesFileSnafu {
-            path: expected_amis_path,
-        })?,
-    )
-    .context(error::ParseExpectedImagesFileSnafu)?;
+    let file_bytes =
+        fs::read(&expected_amis_path)
+            .await
+            .context(error::ReadExpectedImagesFileSnafu {
+                path: expected_amis_path,
+            })?;
+    let expected_amis: HashMap<RegionName, ImageData> =
+        serde_json::from_slice(&file_bytes).context(error::ParseExpectedImagesFileSnafu)?;
 
     // Extract the `Vec<ImageDef>` from the `ImageData` structs
     let vectored_images = expected_amis

tools/pubsys/src/aws/validate_ami/results.rs

@@ -210,7 +210,7 @@ mod test {
             (Region::new("us-west-2"), HashSet::from([])),
             (Region::new("us-east-1"), HashSet::from([])),
         ]));
-        let results_filtered = results.get_results_for_status(&vec![
+        let results_filtered = results.get_results_for_status(&[
             AmiValidationResultStatus::Correct,
             AmiValidationResultStatus::Incorrect,
             AmiValidationResultStatus::Missing,
@@ -355,7 +355,7 @@ mod test {
             ),
         ]));
         let results_filtered =
-            results.get_results_for_status(&vec![AmiValidationResultStatus::Correct]);
+            results.get_results_for_status(&[AmiValidationResultStatus::Correct]);
 
         assert_eq!(
             results_filtered,
@@ -525,7 +525,7 @@ mod test {
                 ]),
             ),
         ]));
-        let results_filtered = results.get_results_for_status(&vec![
+        let results_filtered = results.get_results_for_status(&[
             AmiValidationResultStatus::Correct,
             AmiValidationResultStatus::Incorrect,
         ]);
@@ -756,7 +756,7 @@ mod test {
                 )]),
             ),
         ]));
-        let results_filtered = results.get_results_for_status(&vec![
+        let results_filtered = results.get_results_for_status(&[
             AmiValidationResultStatus::Correct,
             AmiValidationResultStatus::Incorrect,
             AmiValidationResultStatus::Missing,
@@ -1027,7 +1027,7 @@ mod test {
             ),
         ]));
         let results_filtered =
-            results.get_results_for_status(&vec![AmiValidationResultStatus::Missing]);
+            results.get_results_for_status(&[AmiValidationResultStatus::Missing]);
 
         assert_eq!(results_filtered, HashSet::new());
     }

tools/pubsys/src/aws/validate_ssm/mod.rs

@@ -14,8 +14,8 @@ use log::{error, info, trace};
 use pubsys_config::InfraConfig;
 use snafu::ResultExt;
 use std::collections::{HashMap, HashSet};
-use std::fs::File;
 use std::path::PathBuf;
+use tokio::fs;
 
 /// Validates SSM parameters and AMIs
 #[derive(Debug, Parser)]
@@ -128,13 +128,13 @@ pub async fn validate(
         };
 
         // Write the results as JSON
-        serde_json::to_writer_pretty(
-            &File::create(write_results_path).context(error::WriteValidationResultsSnafu {
+        let json = serde_json::to_string_pretty(&results)
+            .context(error::SerializeValidationResultsSnafu)?;
+        fs::write(write_results_path, &json)
+            .await
+            .context(error::WriteValidationResultsSnafu {
                 path: write_results_path,
-            })?,
-            &results,
-        )
-        .context(error::SerializeValidationResultsSnafu)?;
+            })?;
     }
 
     Ok(validation_results)
@@ -206,15 +206,15 @@ type ParameterValue = String;
 pub(crate) async fn parse_parameters(
     expected_parameters_file: &PathBuf,
 ) -> Result<HashMap<Region, HashMap<SsmKey, String>>> {
+    let file_bytes = fs::read(expected_parameters_file.clone()).await.context(
+        error::ReadExpectedParameterFileSnafu {
+            path: expected_parameters_file,
+        },
+    )?;
     // Parse the JSON file as a HashMap of region_name, mapped to a HashMap of parameter_name and
     // parameter_value
     let expected_parameters: HashMap<RegionName, HashMap<ParameterName, ParameterValue>> =
-        serde_json::from_reader(&File::open(expected_parameters_file.clone()).context(
-            error::ReadExpectedParameterFileSnafu {
-                path: expected_parameters_file,
-            },
-        )?)
-        .context(error::ParseExpectedParameterFileSnafu)?;
+        serde_json::from_slice(&file_bytes).context(error::ParseExpectedParameterFileSnafu)?;
 
     // Iterate over the parsed HashMap, converting the nested HashMap into a HashMap of Region
     // mapped to a HashMap of SsmKey, String