diff --git a/tools/ci-cdk/canary-lambda/src/canary.rs b/tools/ci-cdk/canary-lambda/src/canary.rs index 9158b4751f3..a7d497b52e3 100644 --- a/tools/ci-cdk/canary-lambda/src/canary.rs +++ b/tools/ci-cdk/canary-lambda/src/canary.rs @@ -11,8 +11,8 @@ use std::pin::Pin; use aws_config::SdkConfig; use tracing::{info_span, Instrument}; -use crate::paginator_canary; -use crate::{s3_canary, transcribe_canary}; +use crate::current_canary::paginator_canary; +use crate::current_canary::{s3_canary, transcribe_canary}; #[macro_export] macro_rules! mk_canary { diff --git a/tools/ci-cdk/canary-lambda/src/latest.rs b/tools/ci-cdk/canary-lambda/src/latest.rs new file mode 100644 index 00000000000..238c3611665 --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/latest.rs @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +pub(crate) mod paginator_canary; +pub(crate) mod s3_canary; +pub(crate) mod transcribe_canary; diff --git a/tools/ci-cdk/canary-lambda/src/latest/paginator_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/paginator_canary.rs new file mode 100644 index 00000000000..291f7179864 --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/latest/paginator_canary.rs @@ -0,0 +1,71 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::mk_canary; +use anyhow::bail; + +use aws_sdk_ec2 as ec2; +use aws_sdk_ec2::types::InstanceType; + +use crate::CanaryEnv; +use tokio_stream::StreamExt; + +mk_canary!( + "ec2_paginator", + |sdk_config: &aws_config::SdkConfig, env: &CanaryEnv| { + paginator_canary(ec2::Client::new(sdk_config), env.page_size) + } +); + +pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::Result<()> { + let mut history = client + .describe_spot_price_history() + .instance_types(InstanceType::M1Medium) + .into_paginator() + .page_size(page_size as i32) + .send(); + + let mut num_pages = 0; + while let Some(page) = history.try_next().await? { + let items_in_page = page.spot_price_history.unwrap_or_default().len(); + if items_in_page > page_size as usize { + bail!( + "failed to retrieve results of correct page size (expected {}, got {})", + page_size, + items_in_page + ) + } + num_pages += 1; + } + if dbg!(num_pages) < 2 { + bail!( + "expected 3+ pages containing ~60 results but got {} pages", + num_pages + ) + } + + // https://github.com/awslabs/aws-sdk-rust/issues/405 + let _ = client + .describe_vpcs() + .into_paginator() + .items() + .send() + .collect::, _>>() + .await?; + + Ok(()) +} + +#[cfg(test)] +mod test { + use crate::paginator_canary::paginator_canary; + + #[tokio::test] + async fn test_paginator() { + let conf = aws_config::load_from_env().await; + let client = aws_sdk_ec2::Client::new(&conf); + paginator_canary(client, 20).await.unwrap() + } +} diff --git a/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs new file mode 100644 index 00000000000..fbcba976d86 --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs @@ -0,0 +1,140 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::canary::CanaryError; +use crate::{mk_canary, CanaryEnv}; +use anyhow::Context; +use aws_config::SdkConfig; +use aws_sdk_s3 as s3; +use s3::presigning::PresigningConfig; +use s3::primitives::ByteStream; +use std::time::Duration; +use uuid::Uuid; + +const METADATA_TEST_VALUE: &str = "some value"; + +mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| s3_canary( + s3::Client::new(sdk_config), + env.s3_bucket_name.clone() +)); + +pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Result<()> { + let test_key = Uuid::new_v4().as_u128().to_string(); + + // Look for the test object and expect that it doesn't exist + match client + .get_object() + .bucket(&s3_bucket_name) + .key(&test_key) + .send() + .await + { + Ok(_) => { + return Err( + CanaryError(format!("Expected object {} to not exist in S3", test_key)).into(), + ); + } + Err(err) => { + let err = err.into_service_error(); + // If we get anything other than "No such key", we have a problem + if !err.is_no_such_key() { + return Err(err).context("unexpected s3::GetObject failure"); + } + } + } + + // Put the test object + client + .put_object() + .bucket(&s3_bucket_name) + .key(&test_key) + .body(ByteStream::from_static(b"test")) + .metadata("something", METADATA_TEST_VALUE) + .send() + .await + .context("s3::PutObject")?; + + // Get the test object and verify it looks correct + let output = client + .get_object() + .bucket(&s3_bucket_name) + .key(&test_key) + .send() + .await + .context("s3::GetObject[2]")?; + + // repeat the test with a presigned url + let uri = client + .get_object() + .bucket(&s3_bucket_name) + .key(&test_key) + .presigned(PresigningConfig::expires_in(Duration::from_secs(120)).unwrap()) + .await + .unwrap(); + let response = reqwest::get(uri.uri().to_string()) + .await + .context("s3::presigned")? + .text() + .await?; + if response != "test" { + return Err(CanaryError(format!("presigned URL returned bad data: {:?}", response)).into()); + } + + let mut result = Ok(()); + match output.metadata() { + Some(map) => { + // Option::as_deref doesn't work here since the deref of &String is String + let value = map.get("something").map(|s| s.as_str()).unwrap_or(""); + if value != METADATA_TEST_VALUE { + result = Err(CanaryError(format!( + "S3 metadata was incorrect. Expected `{}` but got `{}`.", + METADATA_TEST_VALUE, value + )) + .into()); + } + } + None => { + result = Err(CanaryError("S3 metadata was missing".into()).into()); + } + } + + let payload = output + .body + .collect() + .await + .context("download s3::GetObject[2] body")? + .into_bytes(); + if std::str::from_utf8(payload.as_ref()).context("s3 payload")? != "test" { + result = Err(CanaryError("S3 object body didn't match what was put there".into()).into()); + } + + // Delete the test object + client + .delete_object() + .bucket(&s3_bucket_name) + .key(&test_key) + .send() + .await + .context("s3::DeleteObject")?; + + result +} + +// This test runs against an actual AWS account. Comment out the `ignore` to run it. +// Be sure to set the `TEST_S3_BUCKET` environment variable to the S3 bucket to use, +// and also make sure the credential profile sets the region (or set `AWS_DEFAULT_PROFILE`). +#[ignore] +#[cfg(test)] +#[tokio::test] +async fn test_s3_canary() { + let config = aws_config::load_from_env().await; + let client = s3::Client::new(&config); + s3_canary( + client, + std::env::var("TEST_S3_BUCKET").expect("TEST_S3_BUCKET must be set"), + ) + .await + .expect("success"); +} diff --git a/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs new file mode 100644 index 00000000000..8f6420fc1bd --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/latest/transcribe_canary.rs @@ -0,0 +1,92 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::canary::CanaryError; +use crate::mk_canary; +use async_stream::stream; +use aws_config::SdkConfig; +use aws_sdk_transcribestreaming as transcribe; +use bytes::BufMut; +use transcribe::primitives::Blob; +use transcribe::types::{ + AudioEvent, AudioStream, LanguageCode, MediaEncoding, TranscriptResultStream, +}; + +const CHUNK_SIZE: usize = 8192; +use crate::canary::CanaryEnv; + +mk_canary!( + "transcribe_canary", + |sdk_config: &SdkConfig, env: &CanaryEnv| { + transcribe_canary( + transcribe::Client::new(sdk_config), + env.expected_transcribe_result.clone(), + ) + } +); + +pub async fn transcribe_canary( + client: transcribe::Client, + expected_transcribe_result: String, +) -> anyhow::Result<()> { + let input_stream = stream! { + let pcm = pcm_data(); + for chunk in pcm.chunks(CHUNK_SIZE) { + yield Ok(AudioStream::AudioEvent(AudioEvent::builder().audio_chunk(Blob::new(chunk)).build())); + } + }; + + let mut output = client + .start_stream_transcription() + .language_code(LanguageCode::EnGb) + .media_sample_rate_hertz(8000) + .media_encoding(MediaEncoding::Pcm) + .audio_stream(input_stream.into()) + .send() + .await?; + + let mut full_message = String::new(); + while let Some(event) = output.transcript_result_stream.recv().await? { + match event { + TranscriptResultStream::TranscriptEvent(transcript_event) => { + let transcript = transcript_event.transcript.unwrap(); + for result in transcript.results.unwrap_or_default() { + if !result.is_partial { + let first_alternative = &result.alternatives.as_ref().unwrap()[0]; + full_message += first_alternative.transcript.as_ref().unwrap(); + full_message.push(' '); + } + } + } + otherwise => panic!("received unexpected event type: {:?}", otherwise), + } + } + + if expected_transcribe_result != full_message.trim() { + Err(CanaryError(format!( + "Transcription from Transcribe doesn't look right:\n\ + Expected: `{}`\n\ + Actual: `{}`\n", + expected_transcribe_result, + full_message.trim() + )) + .into()) + } else { + Ok(()) + } +} + +fn pcm_data() -> Vec { + let reader = + hound::WavReader::new(&include_bytes!("../../audio/hello-transcribe-8000.wav")[..]) + .expect("valid wav data"); + let samples_result: hound::Result> = reader.into_samples::().collect(); + + let mut pcm: Vec = Vec::new(); + for sample in samples_result.unwrap() { + pcm.put_i16_le(sample); + } + pcm +} diff --git a/tools/ci-cdk/canary-lambda/src/main.rs b/tools/ci-cdk/canary-lambda/src/main.rs index 3bb723c8024..688462031db 100644 --- a/tools/ci-cdk/canary-lambda/src/main.rs +++ b/tools/ci-cdk/canary-lambda/src/main.rs @@ -19,35 +19,18 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::EnvFilter; use tracing_texray::TeXRayLayer; -/// Conditionally include the module based on the $version feature gate -/// -/// When the module is not included, an `mk_canary` function will be generated that returns `None`. -macro_rules! canary_module { - ($name: ident, since: $version: expr) => { - #[cfg(feature = $version)] - mod $name; - - #[cfg(not(feature = $version))] - mod $name { - pub(crate) fn mk_canary( - _clients: &aws_config::SdkConfig, - _env: &crate::canary::CanaryEnv, - ) -> Option<(&'static str, crate::canary::CanaryFuture)> { - tracing::warn!(concat!( - stringify!($name), - " is disabled because it is not supported by this version of the SDK." - )); - None - } - } - }; -} - mod canary; -mod s3_canary; -canary_module!(paginator_canary, since: "v0.4.1"); -mod transcribe_canary; +#[cfg(feature = "latest")] +mod latest; +#[cfg(feature = "latest")] +pub(crate) use latest as current_canary; + +// NOTE: This module can be deleted 3 releases after release-2023-01-26 +#[cfg(feature = "release-2023-01-26")] +mod release_2023_01_26; +#[cfg(feature = "release-2023-01-26")] +pub(crate) use release_2023_01_26 as current_canary; #[tokio::main] async fn main() -> Result<(), Error> { diff --git a/tools/ci-cdk/canary-lambda/src/release_2023_01_26.rs b/tools/ci-cdk/canary-lambda/src/release_2023_01_26.rs new file mode 100644 index 00000000000..238c3611665 --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/release_2023_01_26.rs @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +pub(crate) mod paginator_canary; +pub(crate) mod s3_canary; +pub(crate) mod transcribe_canary; diff --git a/tools/ci-cdk/canary-lambda/src/paginator_canary.rs b/tools/ci-cdk/canary-lambda/src/release_2023_01_26/paginator_canary.rs similarity index 100% rename from tools/ci-cdk/canary-lambda/src/paginator_canary.rs rename to tools/ci-cdk/canary-lambda/src/release_2023_01_26/paginator_canary.rs diff --git a/tools/ci-cdk/canary-lambda/src/s3_canary.rs b/tools/ci-cdk/canary-lambda/src/release_2023_01_26/s3_canary.rs similarity index 100% rename from tools/ci-cdk/canary-lambda/src/s3_canary.rs rename to tools/ci-cdk/canary-lambda/src/release_2023_01_26/s3_canary.rs diff --git a/tools/ci-cdk/canary-lambda/src/transcribe_canary.rs b/tools/ci-cdk/canary-lambda/src/release_2023_01_26/transcribe_canary.rs similarity index 94% rename from tools/ci-cdk/canary-lambda/src/transcribe_canary.rs rename to tools/ci-cdk/canary-lambda/src/release_2023_01_26/transcribe_canary.rs index c0e2db2fdbf..554f4c3ddf1 100644 --- a/tools/ci-cdk/canary-lambda/src/transcribe_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/release_2023_01_26/transcribe_canary.rs @@ -79,8 +79,9 @@ pub async fn transcribe_canary( } fn pcm_data() -> Vec { - let reader = hound::WavReader::new(&include_bytes!("../audio/hello-transcribe-8000.wav")[..]) - .expect("valid wav data"); + let reader = + hound::WavReader::new(&include_bytes!("../../audio/hello-transcribe-8000.wav")[..]) + .expect("valid wav data"); let samples_result: hound::Result> = reader.into_samples::().collect(); let mut pcm: Vec = Vec::new(); diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index 1f0c4a3f982..0d214f4d897 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -65,7 +65,7 @@ const REQUIRED_SDK_CRATES: &[&str] = &[ lazy_static! { static ref NOTABLE_SDK_RELEASE_TAGS: Vec = vec![ - ReleaseTag::from_str("v0.4.1").unwrap(), // first version to add support for paginators + ReleaseTag::from_str("release-2023-01-26").unwrap(), // last version before the crate reorg ]; } @@ -116,11 +116,14 @@ fn enabled_features(crate_source: &CrateSource) -> Vec { let mut enabled = Vec::new(); if let CrateSource::VersionsManifest { release_tag, .. } = crate_source { for notable in NOTABLE_SDK_RELEASE_TAGS.iter() { - if notable < release_tag { + if notable <= release_tag { enabled.push(notable.as_str().into()); } } } + if enabled.is_empty() { + enabled.push("latest".into()); + } enabled } @@ -158,11 +161,12 @@ fn generate_crate_manifest(crate_source: CrateSource) -> Result { } } write!(output, "\n[features]\n").unwrap(); + writeln!(output, "latest = []").unwrap(); for release_tag in NOTABLE_SDK_RELEASE_TAGS.iter() { writeln!( output, "\"{release_tag}\" = []", - release_tag = release_tag.as_str().replace('-', "_") + release_tag = release_tag.as_str() ) .unwrap(); } @@ -436,8 +440,9 @@ aws-sdk-ec2 = { path = "some/sdk/path/ec2" } aws-sdk-transcribestreaming = { path = "some/sdk/path/transcribestreaming" } [features] -"v0.4.1" = [] -default = [] +latest = [] +"release-2023-01-26" = [] +default = ["latest"] "#, generate_crate_manifest(CrateSource::Path("some/sdk/path".into())).expect("success") ); @@ -499,8 +504,9 @@ aws-sdk-ec2 = "0.19.0" aws-sdk-transcribestreaming = "0.16.0" [features] -"v0.4.1" = [] -default = ["v0.4.1"] +latest = [] +"release-2023-01-26" = [] +default = ["release-2023-01-26"] "#, generate_crate_manifest(CrateSource::VersionsManifest { versions: VersionsManifest { @@ -517,7 +523,7 @@ default = ["v0.4.1"] .collect(), release: None, }, - release_tag: ReleaseTag::from_str("release-2022-07-26").unwrap(), + release_tag: ReleaseTag::from_str("release-2023-05-26").unwrap(), }) .expect("success") );