From 39be6ce656368d662c8542dc5a4e95662e9b7560 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 9 Feb 2024 22:42:00 -0600 Subject: [PATCH 01/16] Add placeholder types for S3 Express and enable control flow to be redirected for S3 Express use case (#3386) ## Motivation and Context This PR is the first in the series to support the S3 Express feature in the Rust SDK. The work will be done in the feature branch, and once it is code complete, the branch will be merged to main. ## Description This PR adds placeholder types for S3 Express and enables control flow to be redirected for S3 Express use case. For instance, if we run the following example code against a generated SDK from this PR: ```rust let shared_config = aws_config::from_env().region(aws_sdk_s3::config::Region::new("us-east-1")).load().await; let client = aws_sdk_s3::Client::new(&shared_config); client.list_objects_v2().bucket("testbucket--use1-az4--x-s3";).send().await.unwrap(); ``` it will end up ``` thread 's3_express' panicked at 'not yet implemented', /Users/awsaito/src/smithy-rs/aws/sdk/build/aws-sdk/sdk/s3/src/s3_express.rs:104:13 ``` which points to ``` impl ProvideCredentials for DefaultS3ExpressIdentityProvider { fn provide_credentials<'a>(&'a self) -> aws_credential_types::provider::future::ProvideCredentials<'a> where Self: 'a, { todo!() <--- } } ``` ### Implementation decisions - `DefaultS3ExpressIdentityProvider` has an accompanying identity cache. That identity cache cannot be configured by customers so it makes sense for the provider itself to internally own it. In that case, we do NOT want to use the identity cache stored in `RuntimeComponents`, since it interferes with the S3 Express's caching policy. To that end, I added an enum `CacheLocation` to `SharedIdentityResolver` (it already had the `cache_partition` field so it was kind of aware of caching). - Two reasons why `CacheLocation` is added to `SharedIdentityResolver`, but not to individual, concrete `IdentityResolver`s. One is `SharedIdentityResolver` was already cache aware, as mentioned above. The other is that it is more flexible that way; The cache location is not tied to a type of identity resolver, but we can select it when creating a `SharedIdentityResolver`. - I considered but did not add a field `cacheable` to `Identity` since I wanted to keep `Identity` as plain data, keeping the concept of "caching" somewhere outside. - I've added a separate `Config` method, `set_express_credentials_provider`, to override credentials provider for S3 Express. There are other SDKs (e.g. [Ruby](https://www.rubydoc.info/gems/aws-sdk-s3/Aws/S3/Client)) that follow this style and it makes it clear to the customers that this is the method to use when overriding the express credentials provider. The existing `set_credentials_provider`, given its input type, cannot tell whether a passed-in credentials provider is for a regular `sigv4` or for S3 Express. ## Testing Only verified that control flow could be altered for an S3 Express use case, as shown above. Further testing will be added in subsequent PRs. ## Checklist I am planning to include in `CHANGELOG.next.toml` a user guide for S3 Express once the feature branch `ysaito/s3express` is ready to be merged to main. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti Co-authored-by: AWS SDK Rust Bot Co-authored-by: AWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com> Co-authored-by: Zelda Hessler --- .github/workflows/release.yml | 28 +- CHANGELOG.md | 3 + CHANGELOG.next.toml | 52 ++++ aws/SDK_CHANGELOG.next.json | 36 +-- aws/rust-runtime/aws-inlineable/src/lib.rs | 5 + .../aws-inlineable/src/s3_express.rs | 125 +++++++++ .../aws-runtime/src/retries/classifiers.rs | 2 +- aws/rust-runtime/aws-sigv4/Cargo.toml | 7 +- .../smithy/rustsdk/AwsCodegenDecorator.kt | 2 + .../rustsdk/IntegrationTestDependencies.kt | 6 + .../customize/s3/S3ExpressDecorator.kt | 184 ++++++++++++ .../endpoints/OperationInputTestGenerator.kt | 8 + aws/sdk/integration-tests/Cargo.toml | 1 + aws/sdk/integration-tests/dynamodb/Cargo.toml | 7 +- .../tests/retry-classifier-customization.rs | 139 +++++++++ .../ServiceRuntimePluginGenerator.kt | 8 + .../rest-json-extras.smithy | 24 ++ .../protocol/ServerProtocolTestGenerator.kt | 56 +--- examples/pokemon-service/Cargo.toml | 5 +- gradle.properties | 6 +- .../src/client/identity.rs | 28 ++ .../src/client/retries/classifiers.rs | 263 ++++++++++++++++-- .../src/client/runtime_components.rs | 60 +++- .../src/client/identity/cache/lazy.rs | 10 +- .../src/client/orchestrator/auth.rs | 12 +- rust-runtime/aws-smithy-types/Cargo.toml | 7 +- rust-runtime/aws-smithy-types/src/blob.rs | 2 +- tools/ci-scripts/check-semver-hazards | 41 +++ 28 files changed, 988 insertions(+), 139 deletions(-) create mode 100644 aws/rust-runtime/aws-inlineable/src/s3_express.rs create mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt create mode 100755 tools/ci-scripts/check-semver-hazards diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37fcdb81fa..9dc4dea2cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,11 +99,37 @@ jobs: run_sdk_examples: false git_ref: ${{ inputs.commit_sha }} + check-semver-hazards: + name: Check for semver hazards + needs: + - acquire-base-image + # We need `always` here otherwise this job won't run if the previous job has been skipped + # See https://samanpavel.medium.com/github-actions-conditional-job-execution-e6aa363d2867 + if: always() + runs-on: smithy_ubuntu-latest_8-core + steps: + - uses: actions/checkout@v3 + with: + path: smithy-rs + ref: ${{ inputs.commit_sha }} + fetch-depth: 0 + - uses: actions/checkout@v3 + with: + repository: awslabs/aws-sdk-rust + path: aws-sdk-rust + fetch-depth: 0 + - name: Run check-semver-hazards + uses: ./smithy-rs/.github/actions/docker-build + with: + action: check-semver-hazards + action-arguments: ${{ inputs.stable_semantic_version }} ${{ inputs.unstable_semantic_version }} + get-or-create-release-branch: name: Get or create a release branch needs: - - release-ci - acquire-base-image + - check-semver-hazards + - release-ci # We need `always` here otherwise this job won't run if the previous job has been skipped # See https://samanpavel.medium.com/github-actions-conditional-job-execution-e6aa363d2867 if: | diff --git a/CHANGELOG.md b/CHANGELOG.md index b1b26f704c..52ac7d010a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ +February 8th, 2024 +================== + January 24th, 2024 ================== diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index d7d27b7e25..86f909f4d2 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -46,3 +46,55 @@ message = "Added impl `Display` to Enums." references = ["smithy-rs#3336", "smithy-rs#3391"] meta = { "breaking" = false, "tada" = false, "bug" = false} author = "iampkmone" + +[[aws-sdk-rust]] +message = """ +Retry classifiers will now be sorted by priority. This change only affects requests +that are retried. Some requests that were previously been classified as transient +errors may now be classified as throttling errors. + +If you were + +- configuring multiple custom retry classifiers +- that would disagree on how to classify a response +- that have differing priorities + +you may see a behavior change in that classification for the same response is now +dependent on the classifier priority instead of the order in which the classifier +was added. +""" +references = ["smithy-rs#3322"] +meta = { "breaking" = false, "bug" = true, "tada" = false } +author = "Velfi" + +[[smithy-rs]] +message = """ +Retry classifiers will now be sorted by priority. This change only affects requests +that are retried. Some requests that were previously been classified as transient +errors may now be classified as throttling errors. + +If you were + +- configuring multiple custom retry classifiers +- that would disagree on how to classify a response +- that have differing priorities + +you may see a behavior change in that classification for the same response is now +dependent on the classifier priority instead of the order in which the classifier +was added. +""" +references = ["smithy-rs#3322"] +meta = { "breaking" = false, "bug" = true, "tada" = false } +author = "Velfi" + +[[smithy-rs]] +message = "Cap the maximum jitter fraction for identity cache refresh buffer time to 0.5. It was previously 1.0, and if the fraction was randomly set to 1.0, it was equivalent to disregarding the buffer time for cache refresh." +references = ["smithy-rs#3402"] +meta = { "breaking" = false, "tada" = false, "bug" = true, "target" = "client" } +author = "ysaito1001" + +[[aws-sdk-rust]] +message = "Cap the maximum jitter fraction for credentials cache refresh buffer time to 0.5. It was previously 1.0, and if the fraction was randomly set to 1.0, it was equivalent to disregarding the buffer time for cache refresh." +references = ["smithy-rs#3402"] +meta = { "breaking" = false, "tada" = false, "bug" = true } +author = "ysaito1001" diff --git a/aws/SDK_CHANGELOG.next.json b/aws/SDK_CHANGELOG.next.json index 6b8f55bba9..35ba2f2c58 100644 --- a/aws/SDK_CHANGELOG.next.json +++ b/aws/SDK_CHANGELOG.next.json @@ -5,20 +5,6 @@ { "smithy-rs": [], "aws-sdk-rust": [ - { - "message": "`crate::event_receiver::EventReceiver` is now re-exported as `crate::primitives::event_stream::EventReceiver` when a service supports event stream operations.", - "meta": { - "bug": true, - "breaking": false, - "tada": false - }, - "author": "ysaito1001", - "references": [ - "smithy-rs#3305" - ], - "since-commit": "9f0ba850e03241f657e2e40ca185780e0a5878cb", - "age": 5 - }, { "message": "Add support for constructing [`SdkBody`] and [`ByteStream`] from `http-body` 1.0 bodies. Note that this is initial support and works via a backwards compatibility shim to http-body 0.4. Hyper 1.0 is not supported.", "meta": { @@ -32,7 +18,7 @@ "aws-sdk-rust#977" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": " Add `PaginationStreamExt` extension trait to `aws-smithy-types-convert` behind the `convert-streams` feature. This makes it possible to treat a paginator as a [`futures_core::Stream`](https://docs.rs/futures-core/latest/futures_core/stream/trait.Stream.html), allowing customers to use stream combinators like [`map`](https://docs.rs/tokio-stream/latest/tokio_stream/trait.StreamExt.html#method.map) and [`filter`](https://docs.rs/tokio-stream/latest/tokio_stream/trait.StreamExt.html#method.filter).\n\nExample:\n\n```rust\nuse aws_smithy_types_convert::stream::PaginationStreamExt\nlet stream = s3_client.list_objects_v2().bucket(\"...\").into_paginator().send().into_stream_03x();\n```\n", @@ -46,7 +32,7 @@ "smithy-rs#3299" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": "Serialize 0/false in query parameters, and ignore actual default value during serialization instead of just 0/false. See [changelog discussion](https://github.com/smithy-lang/smithy-rs/discussions/3312) for details.", @@ -61,7 +47,7 @@ "smithy-rs#3312" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": "Add `as_service_err()` to `SdkError` to allow checking the type of an error is without taking ownership.", @@ -77,7 +63,7 @@ "aws-sdk-rust#1010" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": "Fix bug in `CredentialsProcess` provider where `expiry` was incorrectly treated as a required field.", @@ -92,7 +78,7 @@ "aws-sdk-rust#1021" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": "~/.aws/config and ~/.aws/credentials now parse keys in a case-insensitive way. This means the `AWS_SECRET_ACCESS_KEY` is supported in addition to `aws_secret_access_key`.", @@ -108,7 +94,7 @@ "smithy-rs#3344" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 3 + "age": 4 }, { "message": "`EndpointPrefix` and `apply_endpoint` moved from aws-smithy-http to aws-smithy-runtime-api so that is in a stable (1.x) crate. A deprecated type alias was left in place with a note showing the new location.", @@ -122,7 +108,7 @@ "smithy-rs#3318" ], "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", - "age": 2 + "age": 3 }, { "message": "Fix bug where overriding the credentials at the operation level failed if credentials were already set.", @@ -137,7 +123,7 @@ "smithy-rs#3363" ], "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", - "age": 2 + "age": 3 }, { "message": "Add `apply_to_request_http1x` to `aws-sigv4` to enable signing http = 1.0 requests.", @@ -152,7 +138,7 @@ "smithy-rs#3366" ], "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", - "age": 2 + "age": 3 }, { "message": "The types in the aws-http crate were moved into aws-runtime. Deprecated type aliases were put in place to point to the new locations.", @@ -166,7 +152,7 @@ "smithy-rs#3355" ], "since-commit": "a781be3cd8d22f4ebb5c06a758ddd5f1d6824ded", - "age": 1 + "age": 2 }, { "message": "Add support for `[sso-session]` in AWS config file for AWS Identity Center SSO credentials. Note that this does not include support for AWS Builder ID SSO sessions for services such as Code Catalyst (these lack the `sso_account_id` and `sso_role_name` fields in the profile config). Support for AWS Builder IDs is still being tracked in https://github.com/awslabs/aws-sdk-rust/issues/703.", @@ -181,7 +167,7 @@ "smithy-rs#3379" ], "since-commit": "a781be3cd8d22f4ebb5c06a758ddd5f1d6824ded", - "age": 1 + "age": 2 } ], "aws-sdk-model": [] diff --git a/aws/rust-runtime/aws-inlineable/src/lib.rs b/aws/rust-runtime/aws-inlineable/src/lib.rs index 0ae9627703..88459ef03d 100644 --- a/aws/rust-runtime/aws-inlineable/src/lib.rs +++ b/aws/rust-runtime/aws-inlineable/src/lib.rs @@ -31,6 +31,11 @@ pub mod presigning; /// Presigning interceptors pub mod presigning_interceptors; +// This module uses module paths that assume the target crate to which it is copied, e.g. +// `crate::config::endpoint::Params`. If included into `aws-inlineable`, this module would +// fail to compile. +// pub mod s3_express; + /// Special logic for extracting request IDs from S3's responses. #[allow(dead_code)] pub mod s3_request_id; diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs new file mode 100644 index 0000000000..76b8a95093 --- /dev/null +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -0,0 +1,125 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/// Supporting code for S3 Express auth +pub(crate) mod auth { + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::auth::{ + AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, Sign, + }; + use aws_smithy_runtime_api::client::identity::{Identity, SharedIdentityResolver}; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_runtime_api::client::runtime_components::{ + GetIdentityResolver, RuntimeComponents, + }; + use aws_smithy_types::config_bag::ConfigBag; + + /// Auth scheme ID for S3 Express. + pub(crate) const SCHEME_ID: AuthSchemeId = AuthSchemeId::new("sigv4-s3express"); + + /// S3 Express auth scheme. + #[derive(Debug, Default)] + pub(crate) struct S3ExpressAuthScheme { + signer: S3ExpressSigner, + } + + impl S3ExpressAuthScheme { + /// Creates a new `S3ExpressAuthScheme`. + pub(crate) fn new() -> Self { + Default::default() + } + } + + impl AuthScheme for S3ExpressAuthScheme { + fn scheme_id(&self) -> AuthSchemeId { + SCHEME_ID + } + + fn identity_resolver( + &self, + identity_resolvers: &dyn GetIdentityResolver, + ) -> Option { + identity_resolvers.identity_resolver(self.scheme_id()) + } + + fn signer(&self) -> &dyn Sign { + &self.signer + } + } + + /// S3 Express signer. + #[derive(Debug, Default)] + pub(crate) struct S3ExpressSigner; + + impl Sign for S3ExpressSigner { + fn sign_http_request( + &self, + _request: &mut HttpRequest, + _identity: &Identity, + _auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + _runtime_components: &RuntimeComponents, + _config_bag: &ConfigBag, + ) -> Result<(), BoxError> { + todo!() + } + } +} + +/// Supporting code for S3 Express identity cache +pub(crate) mod identity_cache { + /// The caching implementation for S3 Express identity. + /// + /// While customers can either disable S3 Express itself or provide a custom S3 Express identity + /// provider, configuring S3 Express identity cache is not supported. Thus, this is _the_ + /// implementation of S3 Express identity cache. + #[derive(Debug)] + pub(crate) struct S3ExpressIdentityCache; +} + +/// Supporting code for S3 Express identity provider +pub(crate) mod identity_provider { + use crate::s3_express::identity_cache::S3ExpressIdentityCache; + use aws_smithy_runtime_api::client::identity::{ + IdentityCacheLocation, IdentityFuture, ResolveIdentity, + }; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; + use aws_smithy_types::config_bag::ConfigBag; + + #[derive(Debug)] + pub(crate) struct DefaultS3ExpressIdentityProvider { + _cache: S3ExpressIdentityCache, + } + + #[derive(Default)] + pub(crate) struct Builder; + + impl DefaultS3ExpressIdentityProvider { + pub(crate) fn builder() -> Builder { + Builder + } + } + + impl Builder { + pub(crate) fn build(self) -> DefaultS3ExpressIdentityProvider { + DefaultS3ExpressIdentityProvider { + _cache: S3ExpressIdentityCache, + } + } + } + + impl ResolveIdentity for DefaultS3ExpressIdentityProvider { + fn resolve_identity<'a>( + &'a self, + _runtime_components: &'a RuntimeComponents, + _config_bag: &'a ConfigBag, + ) -> IdentityFuture<'a> { + todo!() + } + + fn cache_location(&self) -> IdentityCacheLocation { + IdentityCacheLocation::IdentityResolver + } + } +} diff --git a/aws/rust-runtime/aws-runtime/src/retries/classifiers.rs b/aws/rust-runtime/aws-runtime/src/retries/classifiers.rs index 9655bec82b..f051234b4a 100644 --- a/aws/rust-runtime/aws-runtime/src/retries/classifiers.rs +++ b/aws/rust-runtime/aws-runtime/src/retries/classifiers.rs @@ -100,7 +100,7 @@ where } fn priority(&self) -> RetryClassifierPriority { - RetryClassifierPriority::with_lower_priority_than( + RetryClassifierPriority::run_before( RetryClassifierPriority::modeled_as_retryable_classifier(), ) } diff --git a/aws/rust-runtime/aws-sigv4/Cargo.toml b/aws/rust-runtime/aws-sigv4/Cargo.toml index f29a2ba01c..21d109472b 100644 --- a/aws/rust-runtime/aws-sigv4/Cargo.toml +++ b/aws/rust-runtime/aws-sigv4/Cargo.toml @@ -44,7 +44,6 @@ zeroize = { version = "^1", optional = true } aws-credential-types = { path = "../aws-credential-types", features = ["test-util", "hardcoded-credentials"] } aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client", "test-util"] } bytes = "1" -criterion = "0.5" hex-literal = "0.4.1" httparse = "1.8" libfuzzer-sys = "0.4.6" @@ -55,6 +54,12 @@ serde_derive = "1.0.180" serde_json = "1.0.104" time = { version = "0.3.5", features = ["parsing"] } +# TODO(https://github.com/smithy-lang/smithy-rs/issues/3398): Remove clap dependency once the SDK MSRV is 1.74. +# Clap was added and pinned to 4.4.x because it is pulled in by criterion, and 4.5.x requires an MSRV of Rust 1.74. +# Since the SDK MSRV is 1.72, this causes it to fail to compile. +clap = "~4.4" +criterion = "0.5" + [target.'cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))'.dev-dependencies] ring = "0.17.5" diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 3bc616a9b8..2c13e4b2b7 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -18,6 +18,7 @@ import software.amazon.smithy.rustsdk.customize.glacier.GlacierDecorator import software.amazon.smithy.rustsdk.customize.onlyApplyTo import software.amazon.smithy.rustsdk.customize.route53.Route53Decorator import software.amazon.smithy.rustsdk.customize.s3.S3Decorator +import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator import software.amazon.smithy.rustsdk.customize.sso.SSODecorator @@ -64,6 +65,7 @@ val DECORATORS: List = Route53Decorator().onlyApplyTo("com.amazonaws.route53#AWSDnsV20130401"), "com.amazonaws.s3#AmazonS3".applyDecorators( S3Decorator(), + S3ExpressDecorator(), S3ExtendedRequestIdDecorator(), ), S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt index a51fde33bc..26fd7cab44 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt @@ -30,6 +30,7 @@ import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Compani import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.smithyProtocolTestHelpers import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.smithyRuntime import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.smithyRuntimeApiTestUtil +import software.amazon.smithy.rust.codegen.core.rustlang.CratesIo import software.amazon.smithy.rust.codegen.core.rustlang.DependencyScope import software.amazon.smithy.rust.codegen.core.rustlang.Writable import software.amazon.smithy.rust.codegen.core.rustlang.writable @@ -106,6 +107,11 @@ class IntegrationTestDependencies( addDependency(TracingSubscriber) } if (hasBenches) { + // TODO(https://github.com/smithy-lang/smithy-rs/issues/3398): Remove clap dependency once the + // SDK MSRV is 1.74. Clap was added and pinned to 4.4.x because it is pulled in by criterion, + // and 4.5.x requires an MSRV of Rust 1.74. Since the SDK MSRV is 1.72, this causes it to fail + // to compile. + addDependency(CargoDependency("clap", CratesIo("~4.4"), DependencyScope.Dev)) addDependency(Criterion) } for (serviceSpecific in serviceSpecificCustomizations()) { diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt new file mode 100644 index 0000000000..8529fdfc47 --- /dev/null +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -0,0 +1,184 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk.customize.s3 + +import software.amazon.smithy.aws.traits.auth.SigV4Trait +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport +import software.amazon.smithy.rust.codegen.client.smithy.customize.AuthSchemeOption +import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rustsdk.AwsRuntimeType +import software.amazon.smithy.rustsdk.InlineAwsDependency + +class S3ExpressDecorator : ClientCodegenDecorator { + override val name: String = "S3ExpressDecorator" + override val order: Byte = 0 + + private fun sigv4S3Express() = + writable { + rust( + "#T", + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + ) + } + + override fun authOptions( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, + baseAuthSchemeOptions: List, + ): List = + baseAuthSchemeOptions + + AuthSchemeOption.StaticAuthSchemeOption( + SigV4Trait.ID, + listOf(sigv4S3Express()), + ) + + override fun serviceRuntimePluginCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = + baseCustomizations + listOf(S3ExpressServiceRuntimePluginCustomization(codegenContext)) + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + listOf(S3ExpressIdentityProviderConfig(codegenContext)) +} + +private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientCodegenContext) : + ServiceRuntimePluginCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope by lazy { + arrayOf( + "DefaultS3ExpressIdentityProvider" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("identity_provider::DefaultS3ExpressIdentityProvider"), + "IdentityCacheLocation" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::IdentityCacheLocation"), + "S3ExpressAuthScheme" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::S3ExpressAuthScheme"), + "S3_EXPRESS_SCHEME_ID" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + "SharedAuthScheme" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::auth::SharedAuthScheme"), + "SharedCredentialsProvider" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::SharedCredentialsProvider"), + ), + "SharedIdentityResolver" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::SharedIdentityResolver"), + ) + } + + override fun section(section: ServiceRuntimePluginSection): Writable = + writable { + when (section) { + is ServiceRuntimePluginSection.RegisterRuntimeComponents -> { + section.registerAuthScheme(this) { + rustTemplate( + "#{SharedAuthScheme}::new(#{S3ExpressAuthScheme}::new())", + *codegenScope, + ) + } + + section.registerIdentityResolver( + this, + writable { + rustTemplate("#{S3_EXPRESS_SCHEME_ID}", *codegenScope) + }, + writable { + rustTemplate("#{DefaultS3ExpressIdentityProvider}::builder().build()", *codegenScope) + }, + ) + } + + else -> {} + } + } +} + +class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "IdentityCacheLocation" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::IdentityCacheLocation"), + "ProvideCredentials" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::ProvideCredentials"), + ), + "SharedCredentialsProvider" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::SharedCredentialsProvider"), + ), + "SharedIdentityResolver" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::SharedIdentityResolver"), + "S3_EXPRESS_SCHEME_ID" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + ) + + override fun section(section: ServiceConfig) = + writable { + when (section) { + ServiceConfig.BuilderImpl -> { + rustTemplate( + """ + /// Sets the credentials provider for S3 Express One Zone + pub fn express_credentials_provider(mut self, credentials_provider: impl #{ProvideCredentials} + 'static) -> Self { + self.set_express_credentials_provider(#{Some}(#{SharedCredentialsProvider}::new(credentials_provider))); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Sets the credentials provider for S3 Express One Zone + pub fn set_express_credentials_provider(&mut self, credentials_provider: #{Option}<#{SharedCredentialsProvider}>) -> &mut Self { + if let #{Some}(credentials_provider) = credentials_provider { + self.runtime_components.set_identity_resolver(#{S3_EXPRESS_SCHEME_ID}, credentials_provider); + } + self + } + """, + *codegenScope, + ) + } + + else -> emptySection + } + } +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt index 5bcc870ae5..544d30ea8f 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt @@ -126,9 +126,17 @@ class OperationInputTestGenerator(_ctx: ClientCodegenContext, private val test: private val model = ctx.model private val instantiator = ClientInstantiator(ctx) + /** tests using S3 Express bucket names need to be disabled until the implementation is in place **/ + private fun EndpointTestCase.isSigV4S3Express() = + expect.endpoint.orNull()?.properties?.get("authSchemes")?.asArrayNode()?.orNull() + ?.map { it.expectObjectNode().expectStringMember("name").value }?.contains("sigv4-s3express") == true + fun generateInput(testOperationInput: EndpointTestOperationInput) = writable { val operationName = testOperationInput.operationName.toSnakeCase() + if (test.isSigV4S3Express()) { + Attribute.shouldPanic("not yet implemented").render(this) + } tokioTest(safeName("operation_input_test_$operationName")) { rustTemplate( """ diff --git a/aws/sdk/integration-tests/Cargo.toml b/aws/sdk/integration-tests/Cargo.toml index f18a443839..546c557175 100644 --- a/aws/sdk/integration-tests/Cargo.toml +++ b/aws/sdk/integration-tests/Cargo.toml @@ -1,6 +1,7 @@ # Note: This workspace exists so that these tests can be run without having to build the entire SDK. When you run # `./gradlew -Paws.fullsdk=true :aws:sdk:assemble` these tests are copied into their respective Service crates. [workspace] +resolver = "2" members = [ "dynamodb", "ec2", diff --git a/aws/sdk/integration-tests/dynamodb/Cargo.toml b/aws/sdk/integration-tests/dynamodb/Cargo.toml index c6b3837748..9ff0738fb4 100644 --- a/aws/sdk/integration-tests/dynamodb/Cargo.toml +++ b/aws/sdk/integration-tests/dynamodb/Cargo.toml @@ -23,7 +23,6 @@ aws-smithy-runtime-api = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime-ap aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types", features = ["test-util"]} aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } bytes = "1.0.0" -criterion = { version = "0.5.0" } futures-util = { version = "0.3.29", default-features = false } http = "0.2.0" serde_json = "1.0.0" @@ -31,6 +30,12 @@ tokio = { version = "1.23.1", features = ["full", "test-util"] } tokio-stream = "0.1.5" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } +# TODO(https://github.com/smithy-lang/smithy-rs/issues/3398): Remove clap dependency once the SDK MSRV is 1.74. +# Clap was added and pinned to 4.4.x because it is pulled in by criterion, and 4.5.x requires an MSRV of Rust 1.74. +# Since the SDK MSRV is 1.72, this causes it to fail to compile. +clap = "~4.4" +criterion = "0.5" + [[bench]] name = "deserialization_bench" harness = false diff --git a/aws/sdk/integration-tests/s3/tests/retry-classifier-customization.rs b/aws/sdk/integration-tests/s3/tests/retry-classifier-customization.rs index 3035c7091e..ecac15f75c 100644 --- a/aws/sdk/integration-tests/s3/tests/retry-classifier-customization.rs +++ b/aws/sdk/integration-tests/s3/tests/retry-classifier-customization.rs @@ -8,6 +8,7 @@ use aws_sdk_s3::config::retry::{ClassifyRetry, RetryAction, RetryConfig}; use aws_sdk_s3::config::SharedAsyncSleep; use aws_smithy_async::rt::sleep::TokioSleep; use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; +use aws_smithy_runtime_api::client::retries::classifiers::RetryClassifierPriority; use aws_smithy_types::body::SdkBody; use std::sync::{Arc, Mutex}; @@ -133,3 +134,141 @@ async fn test_retry_classifier_customization_for_operation() { // ensure our custom retry classifier was called at least once. assert_ne!(customization_test_classifier.counter(), 0); } + +#[derive(Debug, Clone)] +struct OrderingTestClassifier { + counter: Arc>, + name: &'static str, + priority: RetryClassifierPriority, +} + +impl OrderingTestClassifier { + pub fn new(name: &'static str, priority: RetryClassifierPriority) -> Self { + Self { + counter: Arc::new(Mutex::new(0u8)), + name, + priority, + } + } + + pub fn counter(&self) -> u8 { + *self.counter.lock().unwrap() + } +} + +impl ClassifyRetry for OrderingTestClassifier { + fn classify_retry(&self, _ctx: &InterceptorContext) -> RetryAction { + tracing::debug!("Running classifier {}", self.name); + *self.counter.lock().unwrap() += 1; + RetryAction::NoActionIndicated + } + + fn name(&self) -> &'static str { + "Ordering Test Retry Classifier" + } + + fn priority(&self) -> RetryClassifierPriority { + self.priority.clone() + } +} + +#[tracing_test::traced_test] +#[tokio::test] +async fn test_retry_classifier_customization_ordering() { + let http_client = StaticReplayClient::new(vec![ + ReplayEvent::new(req(), err()), + ReplayEvent::new(req(), ok()), + ]); + + let classifier_a = OrderingTestClassifier::new("6", RetryClassifierPriority::default()); + let classifier_b = OrderingTestClassifier::new( + "5", + RetryClassifierPriority::run_before(classifier_a.priority()), + ); + let classifier_c = OrderingTestClassifier::new( + "4", + RetryClassifierPriority::run_before(classifier_b.priority()), + ); + let classifier_d = OrderingTestClassifier::new( + "3", + RetryClassifierPriority::run_before(classifier_c.priority()), + ); + let classifier_e = OrderingTestClassifier::new( + "2", + RetryClassifierPriority::run_before(classifier_d.priority()), + ); + let classifier_f = OrderingTestClassifier::new( + "1", + RetryClassifierPriority::run_before(classifier_e.priority()), + ); + + let config = aws_sdk_s3::Config::builder() + .with_test_defaults() + .sleep_impl(SharedAsyncSleep::new(TokioSleep::new())) + .http_client(http_client) + .retry_config(RetryConfig::standard()) + .retry_classifier(classifier_d.clone()) + .retry_classifier(classifier_b.clone()) + .retry_classifier(classifier_f.clone()) + .build(); + + let client = aws_sdk_s3::Client::from_conf(config); + let _ = client + .get_object() + .bucket("bucket") + .key("key") + .customize() + .config_override( + aws_sdk_s3::config::Config::builder() + .retry_classifier(classifier_c.clone()) + .retry_classifier(classifier_a.clone()) + .retry_classifier(classifier_e.clone()), + ) + .send() + .await + .expect_err("fails without attempting a retry"); + + // ensure our classifiers were each called at least once. + assert_ne!(classifier_a.counter(), 0, "classifier_a was never called"); + assert_ne!(classifier_b.counter(), 0, "classifier_b was never called"); + assert_ne!(classifier_c.counter(), 0, "classifier_c was never called"); + assert_ne!(classifier_d.counter(), 0, "classifier_d was never called"); + assert_ne!(classifier_e.counter(), 0, "classifier_e was never called"); + assert_ne!(classifier_f.counter(), 0, "classifier_f was never called"); + + // ensure the classifiers were called in the correct order. + logs_assert(|lines: &[&str]| { + let mut found_log_a = false; + let mut line_iter = lines.iter(); + + while found_log_a == false { + match line_iter.next() { + Some(&line) => { + if line.contains("Running classifier 1") { + found_log_a = true; + } + } + None => { + return Err("Couldn't find log line for classifier 1".to_owned()); + } + } + } + + for i in 2..=6 { + match line_iter.next() { + Some(&line) => { + if line.contains(&format!("Running classifier {i}")) { + // pass + } else { + return Err(format!("Expected to find log line for classifier {i} after {} but found '{line}'", i - 1)); + } + } + None => { + return Err(format!("Logs ended earlier than expected ({i})")); + } + } + } + + Ok(()) + }); +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt index 66095c1555..699e5bff19 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt @@ -69,6 +69,14 @@ sealed class ServiceRuntimePluginSection(name: String) : Section(name) { ) { writer.rust("runtime_components.push_retry_classifier(#T);", classifier) } + + fun registerIdentityResolver( + writer: RustWriter, + schemeId: Writable, + identityResolver: Writable, + ) { + writer.rust("runtime_components.set_identity_resolver(#T, #T);", schemeId, identityResolver) + } } } typealias ServiceRuntimePluginCustomization = NamedCustomization diff --git a/codegen-core/common-test-models/rest-json-extras.smithy b/codegen-core/common-test-models/rest-json-extras.smithy index a4ff54af96..2667cad6c4 100644 --- a/codegen-core/common-test-models/rest-json-extras.smithy +++ b/codegen-core/common-test-models/rest-json-extras.smithy @@ -8,6 +8,30 @@ use smithy.test#httpRequestTests use smithy.test#httpResponseTests use smithy.framework#ValidationException +// TODO(https://github.com/smithy-lang/smithy/pull/2132): Remove this test once it's fixed in Smithy +apply AllQueryStringTypes @httpRequestTests([ + { + id: "RestJsonZeroAndFalseQueryValuesFixed" + documentation: "Query values of 0 and false are serialized" + protocol: restJson1 + method: "GET" + uri: "/AllQueryStringTypesInput" + body: "" + queryParams: [ + "Integer=0" + "Boolean=false" + ] + params: { + queryInteger: 0 + queryBoolean: false + queryParamsMapOfStringList: { + queryInteger: ["0"] + queryBoolean: ["false"] + } + } + } +]) + apply QueryPrecedence @httpRequestTests([ { id: "UrlParamsKeyEncoding", diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocolTestGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocolTestGenerator.kt index b9cb2f533e..e72ff691dd 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocolTestGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocolTestGenerator.kt @@ -812,6 +812,8 @@ class ServerProtocolTestGenerator( FailingTest(RestJson, "RestJsonEndpointTrait", TestType.Request), FailingTest(RestJson, "RestJsonEndpointTraitWithHostLabel", TestType.Request), FailingTest(RestJson, "RestJsonOmitsEmptyListQueryValues", TestType.Request), + // TODO(https://github.com/smithy-lang/smithy/pull/2132): Remove this failing test once it's fixed in Smithy + FailingTest(RestJson, "RestJsonZeroAndFalseQueryValues", TestType.Request), // Tests involving `@range` on floats. // Pending resolution from the Smithy team, see https://github.com/smithy-lang/smithy-rs/issues/2007. FailingTest(RestJsonValidation, "RestJsonMalformedRangeFloat_case0", TestType.MalformedRequest), @@ -912,60 +914,6 @@ class ServerProtocolTestGenerator( "S3EscapePathObjectKeyInUriLabel", ) - private fun fixRestJsonAllQueryStringTypes( - testCase: HttpRequestTestCase, - @Suppress("UNUSED_PARAMETER") - operationShape: OperationShape, - ): HttpRequestTestCase = - testCase.toBuilder().params( - Node.parse( - """ - { - "queryString": "Hello there", - "queryStringList": ["a", "b", "c"], - "queryStringSet": ["a", "b", "c"], - "queryByte": 1, - "queryShort": 2, - "queryInteger": 3, - "queryIntegerList": [1, 2, 3], - "queryIntegerSet": [1, 2, 3], - "queryLong": 4, - "queryFloat": 1.1, - "queryDouble": 1.1, - "queryDoubleList": [1.1, 2.1, 3.1], - "queryBoolean": true, - "queryBooleanList": [true, false, true], - "queryTimestamp": 1, - "queryTimestampList": [1, 2, 3], - "queryEnum": "Foo", - "queryIntegerEnum": 1, - "queryIntegerEnumList": [1,2,3], - "queryEnumList": ["Foo", "Baz", "Bar"], - "queryParamsMapOfStringList": { - "String": ["Hello there"], - "StringList": ["a", "b", "c"], - "StringSet": ["a", "b", "c"], - "Byte": ["1"], - "Short": ["2"], - "Integer": ["3"], - "IntegerList": ["1", "2", "3"], - "IntegerSet": ["1", "2", "3"], - "Long": ["4"], - "Float": ["1.1"], - "Double": ["1.1"], - "DoubleList": ["1.1", "2.1", "3.1"], - "Boolean": ["true"], - "BooleanList": ["true", "false", "true"], - "Timestamp": ["1970-01-01T00:00:01Z"], - "TimestampList": ["1970-01-01T00:00:01Z", "1970-01-01T00:00:02Z", "1970-01-01T00:00:03Z"], - "Enum": ["Foo"], - "EnumList": ["Foo", "Baz", "Bar"] - } - } - """.trimMargin(), - ).asObjectNode().get(), - ).build() - // TODO(https://github.com/awslabs/smithy/issues/1506) private fun fixRestJsonMalformedPatternReDOSString( testCase: HttpMalformedRequestTestCase, diff --git a/examples/pokemon-service/Cargo.toml b/examples/pokemon-service/Cargo.toml index 8af48fa141..d8218be623 100644 --- a/examples/pokemon-service/Cargo.toml +++ b/examples/pokemon-service/Cargo.toml @@ -7,7 +7,10 @@ authors = ["Smithy-rs Server Team "] description = "A smithy Rust service to retrieve information about Pokémon." [dependencies] -clap = { version = "4.1.11", features = ["derive"] } +# TODO(https://github.com/smithy-lang/smithy-rs/issues/3398): Unpin the clap dependency once the SDK MSRV is Rust 1.74. +# Clap was added and pinned to 4.4.x because 4.5.x requires an MSRV of Rust 1.74. +# Since the SDK MSRV is 1.72, this causes it to fail to compile. +clap = { version = "~4.4", features = ["derive"] } http = "0.2" hyper = { version = "0.14.26", features = ["server"] } tokio = "1.26.0" diff --git a/gradle.properties b/gradle.properties index 71f5254c82..939ed15c6d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,16 +12,16 @@ rust.msrv=1.72.1 org.gradle.jvmargs=-Xmx1024M # Version number to use for the generated stable runtime crates -smithy.rs.runtime.crate.stable.version=1.1.4 +smithy.rs.runtime.crate.stable.version=1.1.5 # Version number to use for the generated unstable runtime crates -smithy.rs.runtime.crate.unstable.version=0.60.4 +smithy.rs.runtime.crate.unstable.version=0.60.5 kotlin.code.style=official # codegen smithyGradlePluginVersion=0.9.0 -smithyVersion=1.43.0 +smithyVersion=1.44.0 # kotlin kotlinVersion=1.9.20 diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs b/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs index f0e972675a..2671a883a0 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs @@ -159,6 +159,30 @@ pub trait ResolveIdentity: Send + Sync + Debug { fn fallback_on_interrupt(&self) -> Option { None } + + /// Returns the location of an identity cache associated with this identity resolver. + /// + /// By default, identity resolvers will use the identity cache stored in runtime components. + /// Implementing types can change the cache location if they want to. Refer to [`IdentityCacheLocation`] + /// explaining why a concrete identity resolver might want to change the cache location. + fn cache_location(&self) -> IdentityCacheLocation { + IdentityCacheLocation::RuntimeComponents + } +} + +/// Cache location for identity caching. +/// +/// Identities are usually cached in the identity cache owned by [`RuntimeComponents`]. However, +/// we do have identities whose caching mechanism is internally managed by their identity resolver, +/// in which case we want to avoid the `RuntimeComponents`-owned identity cache interfering with +/// the internal caching policy. +#[non_exhaustive] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum IdentityCacheLocation { + /// Indicates the identity cache is owned by [`RuntimeComponents`]. + RuntimeComponents, + /// Indicates the identity cache is internally managed by the identity resolver. + IdentityResolver, } /// Container for a shared identity resolver. @@ -194,6 +218,10 @@ impl ResolveIdentity for SharedIdentityResolver { ) -> IdentityFuture<'a> { self.inner.resolve_identity(runtime_components, config_bag) } + + fn cache_location(&self) -> IdentityCacheLocation { + self.inner.cache_location() + } } impl_shared_conversions!(convert SharedIdentityResolver from ResolveIdentity using SharedIdentityResolver::new); diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/retries/classifiers.rs b/rust-runtime/aws-smithy-runtime-api/src/client/retries/classifiers.rs index fc8544c7cb..ee66f7fdb2 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/retries/classifiers.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/retries/classifiers.rs @@ -3,11 +3,36 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Classifier for determining if a retry is necessary and related code. - +//! Classifiers for determining if a retry is necessary and related code. +//! +//! When a request fails, a retry strategy should inspect the result with retry +//! classifiers to understand if and how the request should be retried. +//! +//! Because multiple classifiers are often used, and because some are more +//! specific than others in what they identify as retryable, classifiers are +//! run in a sequence that is determined by their priority. +//! +//! Classifiers that are higher priority are run **after** classifiers +//! with a lower priority. The intention is that: +//! +//! 1. Generic classifiers that look at things like the HTTP error code run +//! first. +//! 2. More specific classifiers such as ones that check for certain error +//! messages are run **after** the generic classifiers. This gives them the +//! ability to override the actions set by the generic retry classifiers. +//! +//! Put another way: +//! +//! | large nets target common failures with basic behavior | run before | small nets target specific failures with special behavior| +//! |-------------------------------------------------------|-----------------------|----------------------------------------------------------| +//! | low priority classifiers | results overridden by | high priority classifiers | + +use crate::box_error::BoxError; use crate::client::interceptors::context::InterceptorContext; use crate::client::runtime_components::sealed::ValidateConfig; +use crate::client::runtime_components::RuntimeComponents; use crate::impl_shared_conversions; +use aws_smithy_types::config_bag::ConfigBag; use aws_smithy_types::retry::ErrorKind; use std::fmt; use std::sync::Arc; @@ -88,7 +113,7 @@ pub enum RetryReason { RetryableError { /// The kind of error. kind: ErrorKind, - /// A server may tells us to retry only after a specific time has elapsed. + /// A server may tell us to retry only after a specific time has elapsed. retry_after: Option, }, } @@ -106,9 +131,10 @@ impl fmt::Display for RetryReason { } } -/// The priority of a retry classifier. Classifiers with a higher priority will run before -/// classifiers with a lower priority. Classifiers with equal priorities make no guarantees -/// about which will run first. +/// The priority of a retry classifier. Classifiers with a higher priority will +/// run **after** classifiers with a lower priority and may override their +/// result. Classifiers with equal priorities make no guarantees about which +/// will run first. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RetryClassifierPriority { inner: Inner, @@ -116,25 +142,25 @@ pub struct RetryClassifierPriority { #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Inner { - // The default priority for the `HttpStatusCodeClassifier`. + /// The default priority for the `HttpStatusCodeClassifier`. HttpStatusCodeClassifier, - // The default priority for the `ModeledAsRetryableClassifier`. + /// The default priority for the `ModeledAsRetryableClassifier`. ModeledAsRetryableClassifier, - // The default priority for the `TransientErrorClassifier`. + /// The default priority for the `TransientErrorClassifier`. TransientErrorClassifier, - // The priority of some other classifier. + /// The priority of some other classifier. Other(i8), } impl PartialOrd for RetryClassifierPriority { fn partial_cmp(&self, other: &Self) -> Option { - Some(other.as_i8().cmp(&self.as_i8())) + Some(self.as_i8().cmp(&other.as_i8())) } } impl Ord for RetryClassifierPriority { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - other.as_i8().cmp(&self.as_i8()) + self.as_i8().cmp(&other.as_i8()) } } @@ -160,17 +186,29 @@ impl RetryClassifierPriority { } } + #[deprecated = "use the less-confusingly-named `RetryClassifierPriority::run_before` instead"] /// Create a new `RetryClassifierPriority` with lower priority than the given priority. pub fn with_lower_priority_than(other: Self) -> Self { + Self::run_before(other) + } + + /// Create a new `RetryClassifierPriority` that can be overridden by the given priority. + pub fn run_before(other: Self) -> Self { Self { - inner: Inner::Other(other.as_i8() + 1), + inner: Inner::Other(other.as_i8() - 1), } } + #[deprecated = "use the less-confusingly-named `RetryClassifierPriority::run_after` instead"] /// Create a new `RetryClassifierPriority` with higher priority than the given priority. pub fn with_higher_priority_than(other: Self) -> Self { + Self::run_after(other) + } + + /// Create a new `RetryClassifierPriority` that can override the given priority. + pub fn run_after(other: Self) -> Self { Self { - inner: Inner::Other(other.as_i8() - 1), + inner: Inner::Other(other.as_i8() + 1), } } @@ -238,33 +276,200 @@ impl ClassifyRetry for SharedRetryClassifier { } } -impl ValidateConfig for SharedRetryClassifier {} +impl ValidateConfig for SharedRetryClassifier { + fn validate_final_config( + &self, + runtime_components: &RuntimeComponents, + _cfg: &ConfigBag, + ) -> Result<(), BoxError> { + #[cfg(debug_assertions)] + { + // Because this is validating that the implementation is correct rather + // than validating user input, we only want to run this in debug builds. + let retry_classifiers = runtime_components.retry_classifiers_slice(); + let out_of_order: Vec<_> = retry_classifiers + .windows(2) + .filter(|&w| w[0].value().priority() > w[1].value().priority()) + .collect(); + + if !out_of_order.is_empty() { + return Err("retry classifiers are mis-ordered; this is a bug".into()); + } + } + Ok(()) + } +} #[cfg(test)] mod tests { - use super::RetryClassifierPriority; + use super::{ClassifyRetry, RetryAction, RetryClassifierPriority, SharedRetryClassifier}; + use crate::client::interceptors::context::InterceptorContext; #[test] - fn test_classifier_lower_priority_than() { - let classifier_a = RetryClassifierPriority::default(); - let classifier_b = RetryClassifierPriority::with_lower_priority_than(classifier_a); - let classifier_c = RetryClassifierPriority::with_lower_priority_than(classifier_b); - - let mut list = vec![classifier_b, classifier_a, classifier_c]; + fn test_preset_priorities() { + let before_modeled_as_retryable = RetryClassifierPriority::run_before( + RetryClassifierPriority::modeled_as_retryable_classifier(), + ); + let mut list = vec![ + RetryClassifierPriority::modeled_as_retryable_classifier(), + RetryClassifierPriority::http_status_code_classifier(), + RetryClassifierPriority::transient_error_classifier(), + before_modeled_as_retryable, + ]; list.sort(); - assert_eq!(vec![classifier_c, classifier_b, classifier_a], list); + assert_eq!( + vec![ + RetryClassifierPriority::http_status_code_classifier(), + before_modeled_as_retryable, + RetryClassifierPriority::modeled_as_retryable_classifier(), + RetryClassifierPriority::transient_error_classifier(), + ], + list + ); } #[test] - fn test_classifier_higher_priority_than() { - let classifier_c = RetryClassifierPriority::default(); - let classifier_b = RetryClassifierPriority::with_higher_priority_than(classifier_c); - let classifier_a = RetryClassifierPriority::with_higher_priority_than(classifier_b); + fn test_classifier_run_before() { + // Ensure low-priority classifiers run *before* high-priority classifiers. + let high_priority_classifier = RetryClassifierPriority::default(); + let mid_priority_classifier = RetryClassifierPriority::run_before(high_priority_classifier); + let low_priority_classifier = RetryClassifierPriority::run_before(mid_priority_classifier); + + let mut list = vec![ + mid_priority_classifier, + high_priority_classifier, + low_priority_classifier, + ]; + list.sort(); - let mut list = vec![classifier_b, classifier_c, classifier_a]; + assert_eq!( + vec![ + low_priority_classifier, + mid_priority_classifier, + high_priority_classifier + ], + list + ); + } + + #[test] + fn test_classifier_run_after() { + // Ensure high-priority classifiers run *after* low-priority classifiers. + let low_priority_classifier = RetryClassifierPriority::default(); + let mid_priority_classifier = RetryClassifierPriority::run_after(low_priority_classifier); + let high_priority_classifier = RetryClassifierPriority::run_after(mid_priority_classifier); + + let mut list = vec![ + mid_priority_classifier, + low_priority_classifier, + high_priority_classifier, + ]; list.sort(); - assert_eq!(vec![classifier_c, classifier_b, classifier_a], list); + assert_eq!( + vec![ + low_priority_classifier, + mid_priority_classifier, + high_priority_classifier + ], + list + ); + } + + #[derive(Debug)] + struct ClassifierStub { + name: &'static str, + priority: RetryClassifierPriority, + } + + impl ClassifyRetry for ClassifierStub { + fn classify_retry(&self, _ctx: &InterceptorContext) -> RetryAction { + todo!() + } + + fn name(&self) -> &'static str { + self.name + } + + fn priority(&self) -> RetryClassifierPriority { + self.priority + } + } + + fn wrap(name: &'static str, priority: RetryClassifierPriority) -> SharedRetryClassifier { + SharedRetryClassifier::new(ClassifierStub { name, priority }) + } + + #[test] + fn test_shared_classifier_run_before() { + // Ensure low-priority classifiers run *before* high-priority classifiers, + // even after wrapping. + let high_priority_classifier = RetryClassifierPriority::default(); + let mid_priority_classifier = RetryClassifierPriority::run_before(high_priority_classifier); + let low_priority_classifier = RetryClassifierPriority::run_before(mid_priority_classifier); + + let mut list = vec![ + wrap("mid", mid_priority_classifier), + wrap("high", high_priority_classifier), + wrap("low", low_priority_classifier), + ]; + list.sort_by_key(|rc| rc.priority()); + + let actual: Vec<_> = list.iter().map(|it| it.name()).collect(); + assert_eq!(vec!["low", "mid", "high"], actual); + } + + #[test] + fn test_shared_classifier_run_after() { + // Ensure high-priority classifiers run *after* low-priority classifiers, + // even after wrapping. + let low_priority_classifier = RetryClassifierPriority::default(); + let mid_priority_classifier = RetryClassifierPriority::run_after(low_priority_classifier); + let high_priority_classifier = RetryClassifierPriority::run_after(mid_priority_classifier); + + let mut list = vec![ + wrap("mid", mid_priority_classifier), + wrap("high", high_priority_classifier), + wrap("low", low_priority_classifier), + ]; + list.sort_by_key(|rc| rc.priority()); + + let actual: Vec<_> = list.iter().map(|it| it.name()).collect(); + assert_eq!(vec!["low", "mid", "high"], actual); + } + + #[test] + fn test_shared_preset_priorities() { + let before_modeled_as_retryable = RetryClassifierPriority::run_before( + RetryClassifierPriority::modeled_as_retryable_classifier(), + ); + let mut list = vec![ + wrap( + "modeled as retryable", + RetryClassifierPriority::modeled_as_retryable_classifier(), + ), + wrap( + "http status code", + RetryClassifierPriority::http_status_code_classifier(), + ), + wrap( + "transient error", + RetryClassifierPriority::transient_error_classifier(), + ), + wrap("before 'modeled as retryable'", before_modeled_as_retryable), + ]; + list.sort_by_key(|rc| rc.priority()); + + let actual: Vec<_> = list.iter().map(|it| it.name()).collect(); + assert_eq!( + vec![ + "http status code", + "before 'modeled as retryable'", + "modeled as retryable", + "transient error" + ], + actual + ); } } diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs index dc9f65d56a..4480779a00 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs @@ -142,7 +142,7 @@ impl ValidateConfig for SharedConfigValidator { cfg: &ConfigBag, ) -> Result<(), BoxError> { match &self.inner { - ValidatorInner::BaseConfigStaticFn(validator) => (validator)(runtime_components, cfg), + ValidatorInner::BaseConfigStaticFn(validator) => validator(runtime_components, cfg), ValidatorInner::Shared(validator) => { validator.validate_base_client_config(runtime_components, cfg) } @@ -334,9 +334,12 @@ macro_rules! declare_runtime_components { /// Builds [`RuntimeComponents`] from this builder. pub fn build(self) -> Result<$rc_name, BuildError> { - Ok($rc_name { + let mut rcs = $rc_name { $($field_name: builder_field_value!($outer_type self.$field_name $($option)?),)+ - }) + }; + rcs.sort(); + + Ok(rcs) } } }; @@ -421,6 +424,11 @@ impl RuntimeComponents { self.retry_classifiers.iter().map(|s| s.value.clone()) } + // Needed for `impl ValidateConfig for SharedRetryClassifier {` + pub(crate) fn retry_classifiers_slice(&self) -> &[Tracked] { + self.retry_classifiers.as_slice() + } + /// Returns the retry strategy. pub fn retry_strategy(&self) -> SharedRetryStrategy { self.retry_strategy.value.clone() @@ -470,6 +478,7 @@ impl RuntimeComponents { for validator in self.config_validators() { validator.validate_final_config(self, cfg)?; } + validate!(Option: self.http_client); validate!(Required: self.endpoint_resolver); validate!(Vec: &self.auth_schemes); @@ -477,8 +486,14 @@ impl RuntimeComponents { validate!(Map: self.identity_resolvers); validate!(Vec: &self.interceptors); validate!(Required: self.retry_strategy); + validate!(Vec: &self.retry_classifiers); + Ok(()) } + + fn sort(&mut self) { + self.retry_classifiers.sort_by_key(|rc| rc.value.priority()); + } } impl RuntimeComponentsBuilder { @@ -695,7 +710,7 @@ impl RuntimeComponentsBuilder { self } - /// Adds an retry_classifier. + /// Adds a retry_classifier. pub fn push_retry_classifier( &mut self, retry_classifier: impl ClassifyRetry + 'static, @@ -707,7 +722,7 @@ impl RuntimeComponentsBuilder { self } - /// Adds an retry_classifier. + /// Adds a retry_classifier. pub fn with_retry_classifier(mut self, retry_classifier: impl ClassifyRetry + 'static) -> Self { self.push_retry_classifier(retry_classifier); self @@ -855,7 +870,7 @@ impl RuntimeComponentsBuilder { #[derive(Clone, Debug)] #[cfg_attr(test, derive(Eq, PartialEq))] -struct Tracked { +pub(crate) struct Tracked { _origin: &'static str, value: T, } @@ -867,6 +882,10 @@ impl Tracked { value, } } + + pub(crate) fn value(&self) -> &T { + &self.value + } } impl RuntimeComponentsBuilder { @@ -882,8 +901,7 @@ impl RuntimeComponentsBuilder { fn resolve_auth_scheme_options( &self, _: &crate::client::auth::AuthSchemeOptionResolverParams, - ) -> Result, crate::box_error::BoxError> - { + ) -> Result, BoxError> { unreachable!("fake auth scheme option resolver must be overridden for this test") } } @@ -946,8 +964,7 @@ impl RuntimeComponentsBuilder { &self, _: &RuntimeComponents, _: &ConfigBag, - ) -> Result - { + ) -> Result { unreachable!("fake retry strategy must be overridden for this test") } @@ -956,8 +973,7 @@ impl RuntimeComponentsBuilder { _: &crate::client::interceptors::context::InterceptorContext, _: &RuntimeComponents, _: &ConfigBag, - ) -> Result - { + ) -> Result { unreachable!("fake retry strategy must be overridden for this test") } } @@ -1067,6 +1083,10 @@ mod tests { } } + impl TestRc { + fn sort(&mut self) {} + } + let builder1 = TestRcBuilder { builder_name: "builder1", some_required_component: Some(Tracked::new("builder1", "override_me".into())), @@ -1138,6 +1158,10 @@ mod tests { } } + impl TestRc { + fn sort(&mut self) {} + } + let rc = TestRcBuilder::new("test").build().unwrap(); // Ensure the correct types were used @@ -1156,6 +1180,10 @@ mod tests { } } + impl TestRc { + fn sort(&mut self) {} + } + let rc = TestRcBuilder::new("test").build().unwrap(); // Ensure the correct types were used @@ -1173,6 +1201,10 @@ mod tests { } } + impl TestRc { + fn sort(&mut self) {} + } + let rc = TestRcBuilder::new("test").build().unwrap(); // Ensure the correct types were used @@ -1201,7 +1233,7 @@ mod tests { _: &'a RuntimeComponents, _: &'a ConfigBag, ) -> IdentityFuture<'a> { - IdentityFuture::ready(Ok(Identity::new("doesntmatter", None))) + IdentityFuture::ready(Ok(Identity::new("doesn't matter", None))) } } @@ -1223,6 +1255,6 @@ mod tests { .expect("identity should be resolved") }); - assert_eq!(Some(&"doesntmatter"), identity.data::<&str>()); + assert_eq!(Some(&"doesn't matter"), identity.data::<&str>()); } } diff --git a/rust-runtime/aws-smithy-runtime/src/client/identity/cache/lazy.rs b/rust-runtime/aws-smithy-runtime/src/client/identity/cache/lazy.rs index 8581d7c12c..2461754cff 100644 --- a/rust-runtime/aws-smithy-runtime/src/client/identity/cache/lazy.rs +++ b/rust-runtime/aws-smithy-runtime/src/client/identity/cache/lazy.rs @@ -24,7 +24,7 @@ use tracing::Instrument; const DEFAULT_LOAD_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_EXPIRATION: Duration = Duration::from_secs(15 * 60); const DEFAULT_BUFFER_TIME: Duration = Duration::from_secs(10); -const DEFAULT_BUFFER_TIME_JITTER_FRACTION: fn() -> f64 = fastrand::f64; +const DEFAULT_BUFFER_TIME_JITTER_FRACTION: fn() -> f64 = || fastrand::f64() * 0.5; /// Builder for lazy identity caching. #[derive(Default, Debug)] @@ -86,7 +86,7 @@ impl LazyCacheBuilder { /// For example, if the identity are expiring in 15 minutes, and the buffer time is 10 seconds, /// then any requests made after 14 minutes and 50 seconds will load a new identity. /// - /// Note: random jitter value between [0.0, 1.0] is multiplied to this buffer time. + /// Note: random jitter value between [0.0, 0.5] is multiplied to this buffer time. /// /// Defaults to 10 seconds. pub fn buffer_time(mut self, buffer_time: Duration) -> Self { @@ -99,7 +99,7 @@ impl LazyCacheBuilder { /// For example, if the identity are expiring in 15 minutes, and the buffer time is 10 seconds, /// then any requests made after 14 minutes and 50 seconds will load a new identity. /// - /// Note: random jitter value between [0.0, 1.0] is multiplied to this buffer time. + /// Note: random jitter value between [0.0, 0.5] is multiplied to this buffer time. /// /// Defaults to 10 seconds. pub fn set_buffer_time(&mut self, buffer_time: Option) -> &mut Self { @@ -113,7 +113,7 @@ impl LazyCacheBuilder { /// and buffer time jitter fraction is 0.2, then buffer time is adjusted to 8 seconds. /// Therefore, any requests made after 14 minutes and 52 seconds will load a new identity. /// - /// Defaults to a randomly generated value between 0.0 and 1.0. This setter is for testing only. + /// Defaults to a randomly generated value between 0.0 and 0.5. This setter is for testing only. #[allow(unused)] #[cfg(test)] fn buffer_time_jitter_fraction(mut self, buffer_time_jitter_fraction: fn() -> f64) -> Self { @@ -127,7 +127,7 @@ impl LazyCacheBuilder { /// and buffer time jitter fraction is 0.2, then buffer time is adjusted to 8 seconds. /// Therefore, any requests made after 14 minutes and 52 seconds will load a new identity. /// - /// Defaults to a randomly generated value between 0.0 and 1.0. This setter is for testing only. + /// Defaults to a randomly generated value between 0.0 and 0.5. This setter is for testing only. #[allow(unused)] #[cfg(test)] fn set_buffer_time_jitter_fraction( diff --git a/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs b/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs index 98a39a9a43..0bf5ecf1a7 100644 --- a/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs +++ b/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs @@ -4,12 +4,14 @@ */ use crate::client::auth::no_auth::NO_AUTH_SCHEME_ID; +use crate::client::identity::IdentityCache; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::auth::{ AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, AuthSchemeOptionResolverParams, ResolveAuthSchemeOptions, }; -use aws_smithy_runtime_api::client::identity::ResolveCachedIdentity; +use aws_smithy_runtime_api::client::identity::ResolveIdentity; +use aws_smithy_runtime_api::client::identity::{IdentityCacheLocation, ResolveCachedIdentity}; use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_types::config_bag::ConfigBag; @@ -135,7 +137,13 @@ pub(super) async fn orchestrate_auth( if let Some(auth_scheme) = runtime_components.auth_scheme(scheme_id) { // Use the resolved auth scheme to resolve an identity if let Some(identity_resolver) = auth_scheme.identity_resolver(runtime_components) { - let identity_cache = runtime_components.identity_cache(); + let identity_cache = if identity_resolver.cache_location() + == IdentityCacheLocation::RuntimeComponents + { + runtime_components.identity_cache() + } else { + IdentityCache::no_cache() + }; let signer = auth_scheme.signer(); trace!( auth_scheme = ?auth_scheme, diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 4d50bff226..b083e1e1c3 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -53,7 +53,6 @@ tokio-util = { version = "0.7", optional = true } [dev-dependencies] base64 = "0.13.0" ciborium = { version = "0.2.1" } -criterion = "0.5" lazy_static = "1.4" proptest = "1" rand = "0.8.4" @@ -69,6 +68,12 @@ tokio = { version = "1.23.1", features = [ tokio-stream = "0.1.5" tempfile = "3.2.0" +# TODO(https://github.com/smithy-lang/smithy-rs/issues/3398): Remove clap dependency once the SDK MSRV is 1.74. +# Clap was added and pinned to 4.4.x because it is pulled in by criterion, and 4.5.x requires an MSRV of Rust 1.74. +# Since the SDK MSRV is 1.72, this causes it to fail to compile. +clap = "~4.4" +criterion = "0.5" + [package.metadata.docs.rs] all-features = true targets = ["x86_64-unknown-linux-gnu"] diff --git a/rust-runtime/aws-smithy-types/src/blob.rs b/rust-runtime/aws-smithy-types/src/blob.rs index eebdd60ca2..7f76a45df3 100644 --- a/rust-runtime/aws-smithy-types/src/blob.rs +++ b/rust-runtime/aws-smithy-types/src/blob.rs @@ -6,7 +6,7 @@ /// Binary Blob Type /// /// Blobs represent protocol-agnostic binary content. -#[derive(Debug, PartialEq, Eq, Hash, Clone)] +#[derive(Debug, Default, PartialEq, Eq, Hash, Clone)] pub struct Blob { inner: Vec, } diff --git a/tools/ci-scripts/check-semver-hazards b/tools/ci-scripts/check-semver-hazards new file mode 100755 index 0000000000..460fb5a46c --- /dev/null +++ b/tools/ci-scripts/check-semver-hazards @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +# This script patches the new runtime crates into an old AWS SDK and runs tests +# to check for semver hazards, such as a `Storable` being in an unstable runtime crate. + +C_YELLOW='\033[1;33m' +C_RESET='\033[0m' + +if [ "$#" -ne 2 ]; then + echo "Usage: check-semver-hazards " + exit 1 +fi +STABLE_CRATE_VERSION="$1" +UNSTABLE_CRATE_VERSION="$2" +echo "Stable crate version: ${STABLE_CRATE_VERSION}" +echo "Unstable crate version: ${UNSTABLE_CRATE_VERSION}" + +# Need to allow warnings since there may be deprecations that the old SDK uses +unset RUSTFLAGS + +set -eux + +echo -e "${C_YELLOW}# Patching SDK...${C_RESET}" +runtime-versioner patch-runtime \ + --sdk-path "$(pwd)/aws-sdk-rust" \ + --smithy-rs-path "$(pwd)/smithy-rs" \ + --stable-crate-version "${STABLE_CRATE_VERSION}" \ + --unstable-crate-version "${UNSTABLE_CRATE_VERSION}" + +# Testing just a small subset of the full SDK to check for semver hazards +echo -e "${C_YELLOW}# Testing SDK...${C_RESET}" +for sdk in dynamodb s3; do + echo -e "${C_YELLOW}# Testing ${sdk}...${C_RESET}" + pushd "aws-sdk-rust/sdk/${sdk}" &>/dev/null + cargo test --all-features + popd &>/dev/null +done From 486b91de07ebcab72d47d55e334d344dd64497de Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 16 Feb 2024 20:33:01 -0600 Subject: [PATCH 02/16] Allow list-objects-v2 to run against an S3 Express bucket (#3388) ## Motivation and Context Adds an implementation spike to allow `list-objects-v2` (possibly others, haven't tested yet) to run against an S3 Express bucket. ## Description This PR implements two ingredients, `S3ExpressIdentityProvider` and `S3ExpressSigner`. `S3ExpressIdentityProvider` uses an internal S3 client to obtain an S3 Express session token that is passed to `S3ExpressSigner`. `S3ExpressSigner` then signs a request with that token, using effectively sigv4 but with session token omitted and an extra header added instead, `x-amz-s3session-token`. In addition, this PR supports presigning for S3 Express. Similarly to signing headers, presigning for S3 Express excludes a query param `X-Amz-Security-Token` and instead uses `X-Amz-S3session-Token` for the signing query params. The following screeshot shows that a presigned URL from `get_object` works for an S3 Express bucket:

chain-provider-ext-timeout-2

Some implementation details: - Since `S3ExpressIdentityProvider` passes an S3 Express bucket name for S3's `create_session` API to obtain an S3 Express session token, it needs to obtain the bucket name from somewhere. `S3ExpressIdentityProvider::ProvideCredentials` I put previously did not have enough arguments for us to figure this out, so I switched to `S3ExpressIdentityProvider::ResolveIdentity` that takes enough arguments. - `SigV4Signer::sign_http_request` did not allow calling code to pass a configured `SigningSettings`; The signer needs to exclude a header `x-amz-security-token` and include `x-amz-s3session-token`. To make this happen, I made `sigv4::extract_operation_config` and `sigv4::settings` public APIs (previously private). - One area I haven't quite figured out yet is how to configure the inner S3 client to call `create_session`. The changes in this PR inherits runtime components & config bag from the "outer" S3 client, but customers may want to configure the inner S3 client in a more flexible manner (e.g. operation timeout). ## Testing To lock the behavior at this time, I added a connection recording test for `list-objects-v2`. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti Co-authored-by: AWS SDK Rust Bot Co-authored-by: AWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com> Co-authored-by: Zelda Hessler --- .../aws-inlineable/src/s3_express.rs | 139 ++++++++++++- .../aws-runtime/src/auth/sigv4.rs | 64 ++++-- .../src/http_request/canonical_request.rs | 27 ++- .../aws-sigv4/src/http_request/settings.rs | 5 + .../aws-sigv4/src/http_request/sign.rs | 10 +- .../rustsdk/EndpointBuiltInsDecorator.kt | 1 - .../amazon/smithy/rustsdk/RegionDecorator.kt | 4 + .../smithy/rustsdk/UserAgentDecorator.kt | 5 + .../rustsdk/customize/s3/S3Decorator.kt | 2 + .../endpoints/OperationInputTestGenerator.kt | 8 - .../tests/data/express/list-objects-v2.json | 193 ++++++++++++++++++ aws/sdk/integration-tests/s3/tests/express.rs | 131 ++++++++++++ .../ResiliencyConfigCustomization.kt | 15 ++ .../ClientContextConfigCustomization.kt | 2 +- .../config/ServiceConfigGenerator.kt | 57 ++++-- ...lledStreamProtectionConfigCustomization.kt | 8 + .../config/ServiceConfigGeneratorTest.kt | 48 +++++ .../src/client/runtime_components.rs | 28 +++ 18 files changed, 689 insertions(+), 58 deletions(-) create mode 100644 aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json create mode 100644 aws/sdk/integration-tests/s3/tests/express.rs diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 76b8a95093..3388145112 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -5,6 +5,8 @@ /// Supporting code for S3 Express auth pub(crate) mod auth { + use aws_runtime::auth::sigv4::SigV4Signer; + use aws_sigv4::http_request::{SignatureLocation, SigningSettings}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::auth::{ AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, Sign, @@ -56,15 +58,37 @@ pub(crate) mod auth { impl Sign for S3ExpressSigner { fn sign_http_request( &self, - _request: &mut HttpRequest, - _identity: &Identity, - _auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, - _runtime_components: &RuntimeComponents, - _config_bag: &ConfigBag, + request: &mut HttpRequest, + identity: &Identity, + auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + runtime_components: &RuntimeComponents, + config_bag: &ConfigBag, ) -> Result<(), BoxError> { - todo!() + let operation_config = + SigV4Signer::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; + let mut settings = SigV4Signer::signing_settings(&operation_config); + override_session_token_name(&mut settings)?; + + SigV4Signer.sign_http_request( + request, + identity, + settings, + &operation_config, + runtime_components, + config_bag, + ) } } + + fn override_session_token_name(settings: &mut SigningSettings) -> Result<(), BoxError> { + let session_token_name_override = match settings.signature_location { + SignatureLocation::Headers => Some("x-amz-s3session-token"), + SignatureLocation::QueryParams => Some("X-Amz-S3session-Token"), + _ => { return Err(BoxError::from("`SignatureLocation` adds a new variant, which needs to be handled in a separate match arm")) }, + }; + settings.session_token_name_override = session_token_name_override; + Ok(()) + } } /// Supporting code for S3 Express identity cache @@ -80,11 +104,21 @@ pub(crate) mod identity_cache { /// Supporting code for S3 Express identity provider pub(crate) mod identity_provider { + use std::time::SystemTime; + use crate::s3_express::identity_cache::S3ExpressIdentityCache; + use crate::types::SessionCredentials; + use aws_credential_types::provider::error::CredentialsError; + use aws_credential_types::Credentials; + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::endpoint::EndpointResolverParams; use aws_smithy_runtime_api::client::identity::{ - IdentityCacheLocation, IdentityFuture, ResolveIdentity, + Identity, IdentityCacheLocation, IdentityFuture, ResolveCachedIdentity, ResolveIdentity, + }; + use aws_smithy_runtime_api::client::interceptors::SharedInterceptor; + use aws_smithy_runtime_api::client::runtime_components::{ + GetIdentityResolver, RuntimeComponents, }; - use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_types::config_bag::ConfigBag; #[derive(Debug)] @@ -95,10 +129,93 @@ pub(crate) mod identity_provider { #[derive(Default)] pub(crate) struct Builder; + impl TryFrom for Credentials { + type Error = BoxError; + + fn try_from(session_creds: SessionCredentials) -> Result { + Ok(Credentials::new( + session_creds.access_key_id, + session_creds.secret_access_key, + Some(session_creds.session_token), + Some(SystemTime::try_from(session_creds.expiration).map_err(|_| { + CredentialsError::unhandled( + "credential expiration time cannot be represented by a SystemTime", + ) + })?), + "s3express", + )) + } + } + impl DefaultS3ExpressIdentityProvider { pub(crate) fn builder() -> Builder { Builder } + + async fn identity<'a>( + &'a self, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, + ) -> Result { + let bucket_name = self.bucket_name(config_bag)?; + + let sigv4_identity_resolver = runtime_components + .identity_resolver(aws_runtime::auth::sigv4::SCHEME_ID) + .ok_or("identity resolver for sigv4 should be set for S3")?; + let _aws_identity = runtime_components + .identity_cache() + .resolve_cached_identity(sigv4_identity_resolver, runtime_components, config_bag) + .await?; + + // TODO(S3Express): use both `bucket_name` and `aws_identity` as part of `S3ExpressIdentityCache` implementation + + let express_session_credentials = self + .express_session_credentials(bucket_name, runtime_components, config_bag) + .await?; + + let data = Credentials::try_from(express_session_credentials)?; + + Ok(Identity::new(data.clone(), data.expiry())) + } + + fn bucket_name<'a>(&'a self, config_bag: &'a ConfigBag) -> Result<&'a str, BoxError> { + let params = config_bag + .load::() + .expect("endpoint resolver params must be set"); + let params = params + .get::() + .expect("`Params` should be wrapped in `EndpointResolverParams`"); + params + .bucket() + .ok_or("A bucket was not set in endpoint params".into()) + } + + async fn express_session_credentials<'a>( + &'a self, + bucket_name: &'a str, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, + ) -> Result { + let mut config_builder = crate::config::Builder::from_config_bag(config_bag); + + // inherits all runtime components from a current S3 operation but clears out + // out interceptors configured for that operation + let mut rc_builder = runtime_components.to_builder(); + rc_builder.set_interceptors(std::iter::empty::()); + config_builder.runtime_components = rc_builder; + + let client = crate::Client::from_conf(config_builder.build()); + let response = client + .create_session() + .bucket(bucket_name) + .session_mode(crate::types::SessionMode::ReadWrite) + .send() + .await?; + + response + .credentials + .ok_or("no session credentials in response".into()) + } } impl Builder { @@ -112,10 +229,10 @@ pub(crate) mod identity_provider { impl ResolveIdentity for DefaultS3ExpressIdentityProvider { fn resolve_identity<'a>( &'a self, - _runtime_components: &'a RuntimeComponents, - _config_bag: &'a ConfigBag, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, ) -> IdentityFuture<'a> { - todo!() + IdentityFuture::new(async move { self.identity(runtime_components, config_bag).await }) } fn cache_location(&self) -> IdentityCacheLocation { diff --git a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs index 1e3dc5ad39..4330c72dfa 100644 --- a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs +++ b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs @@ -72,7 +72,8 @@ impl SigV4Signer { Self } - fn settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { + /// Creates a [`SigningSettings`] from the given `operation_config`. + pub fn signing_settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { super::settings(operation_config) } @@ -117,10 +118,11 @@ impl SigV4Signer { .expect("all required fields set")) } - fn extract_operation_config<'a>( + /// Extracts a [`SigV4OperationSigningConfig`]. + pub fn extract_operation_config<'a>( auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'a>, config_bag: &'a ConfigBag, - ) -> Result, SigV4SigningError> { + ) -> Result, BoxError> { let operation_config = config_bag .load::() .ok_or(SigV4SigningError::MissingOperationSigningConfig)?; @@ -141,28 +143,27 @@ impl SigV4Signer { } } } -} -impl Sign for SigV4Signer { - fn sign_http_request( + /// Signs the given `request`. + /// + /// This is a helper used by [`Sign::sign_http_request`] and will be useful if calling code + /// needs to pass a configured `settings`. + /// + /// TODO(S3Express): Make this method more user friendly, possibly returning a builder + /// instead of taking these input parameters. The builder will have a `sign` method that + /// does what this method body currently does. + pub fn sign_http_request( &self, request: &mut HttpRequest, identity: &Identity, - auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + settings: SigningSettings, + operation_config: &SigV4OperationSigningConfig, runtime_components: &RuntimeComponents, - config_bag: &ConfigBag, + #[allow(unused_variables)] config_bag: &ConfigBag, ) -> Result<(), BoxError> { - let operation_config = - Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; let request_time = runtime_components.time_source().unwrap_or_default().now(); - - if identity.data::().is_none() { - return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); - }; - - let settings = Self::settings(&operation_config); let signing_params = - Self::signing_params(settings, identity, &operation_config, request_time)?; + Self::signing_params(settings, identity, operation_config, request_time)?; let (signing_instructions, _signature) = { // A body that is already in memory can be signed directly. A body that is not in memory @@ -218,6 +219,35 @@ impl Sign for SigV4Signer { } } +impl Sign for SigV4Signer { + fn sign_http_request( + &self, + request: &mut HttpRequest, + identity: &Identity, + auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + runtime_components: &RuntimeComponents, + config_bag: &ConfigBag, + ) -> Result<(), BoxError> { + if identity.data::().is_none() { + return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); + }; + + let operation_config = + Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; + + let settings = Self::signing_settings(&operation_config); + + self.sign_http_request( + request, + identity, + settings, + &operation_config, + runtime_components, + config_bag, + ) + } +} + #[cfg(feature = "event-stream")] mod event_stream { use aws_sigv4::event_stream::{sign_empty_message, sign_message}; diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 5fae247dd2..5bc088ecf5 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -10,8 +10,8 @@ use crate::http_request::settings::UriPathNormalizationMode; use crate::http_request::sign::SignableRequest; use crate::http_request::uri_path_normalization::normalize_uri_path; use crate::http_request::url_escape::percent_encode_path; -use crate::http_request::PercentEncodingMode; use crate::http_request::{PayloadChecksumKind, SignableBody, SignatureLocation, SigningParams}; +use crate::http_request::{PercentEncodingMode, SigningSettings}; use crate::sign::v4::sha256_hex_string; use crate::SignatureVersion; use aws_smithy_http::query_writer::QueryWriter; @@ -218,7 +218,7 @@ impl<'a> CanonicalRequest<'a> { let creq = CanonicalRequest { method: req.method(), path, - params: Self::params(req.uri(), &values), + params: Self::params(req.uri(), &values, params.settings()), headers: canonical_headers, values, }; @@ -250,6 +250,11 @@ impl<'a> CanonicalRequest<'a> { Self::insert_host_header(&mut canonical_headers, req.uri()); + let token_header_name = params + .settings() + .session_token_name_override + .unwrap_or(header::X_AMZ_SECURITY_TOKEN); + if params.settings().signature_location == SignatureLocation::Headers { let creds = params .credentials() @@ -259,7 +264,7 @@ impl<'a> CanonicalRequest<'a> { if let Some(security_token) = creds.session_token() { let mut sec_header = HeaderValue::from_str(security_token)?; sec_header.set_sensitive(true); - canonical_headers.insert(header::X_AMZ_SECURITY_TOKEN, sec_header); + canonical_headers.insert(token_header_name, sec_header); } if params.settings().payload_checksum_kind == PayloadChecksumKind::XAmzSha256 { @@ -283,7 +288,7 @@ impl<'a> CanonicalRequest<'a> { } if params.settings().session_token_mode == SessionTokenMode::Exclude - && name == HeaderName::from_static(header::X_AMZ_SECURITY_TOKEN) + && name == HeaderName::from_static(token_header_name) { continue; } @@ -320,7 +325,11 @@ impl<'a> CanonicalRequest<'a> { } } - fn params(uri: &Uri, values: &SignatureValues<'_>) -> Option { + fn params( + uri: &Uri, + values: &SignatureValues<'_>, + settings: &SigningSettings, + ) -> Option { let mut params: Vec<(Cow<'_, str>, Cow<'_, str>)> = form_urlencoded::parse(uri.query().unwrap_or_default().as_bytes()).collect(); fn add_param<'a>(params: &mut Vec<(Cow<'a, str>, Cow<'a, str>)>, k: &'a str, v: &'a str) { @@ -345,7 +354,13 @@ impl<'a> CanonicalRequest<'a> { ); if let Some(security_token) = values.security_token { - add_param(&mut params, param::X_AMZ_SECURITY_TOKEN, security_token); + add_param( + &mut params, + settings + .session_token_name_override + .unwrap_or(param::X_AMZ_SECURITY_TOKEN), + security_token, + ); } } // Sort by param name, and then by param value diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs index 787619fc36..9b51345866 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs @@ -37,6 +37,10 @@ pub struct SigningSettings { /// canonical request. Other services require only it to be added after /// calculating the signature. pub session_token_mode: SessionTokenMode, + + /// Some services require an alternative session token header or query param instead of + /// `x-amz-security-token` or `X-Amz-Security-Token`. + pub session_token_name_override: Option<&'static str>, } /// HTTP payload checksum type @@ -133,6 +137,7 @@ impl Default for SigningSettings { excluded_headers, uri_path_normalization_mode: UriPathNormalizationMode::Enabled, session_token_mode: SessionTokenMode::Include, + session_token_name_override: None, } } } diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs index 2f27b3c4b7..dc8308dfda 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs @@ -297,7 +297,10 @@ fn calculate_signing_params<'a>( if let Some(security_token) = creds.session_token() { signing_params.push(( - param::X_AMZ_SECURITY_TOKEN, + params + .settings() + .session_token_name_override + .unwrap_or(param::X_AMZ_SECURITY_TOKEN), Cow::Owned(security_token.to_string()), )); } @@ -368,7 +371,10 @@ fn calculate_signing_headers<'a>( if let Some(security_token) = creds.session_token() { add_header( &mut headers, - header::X_AMZ_SECURITY_TOKEN, + params + .settings + .session_token_name_override + .unwrap_or(header::X_AMZ_SECURITY_TOKEN), security_token, true, ); diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt index 3a9b581549..7900b1ca56 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt @@ -148,7 +148,6 @@ fun decoratorForBuiltIn( standardConfigParam( clientParamBuilder?.toConfigParam(builtIn, codegenContext.runtimeConfig) ?: ConfigParam.Builder() .toConfigParam(builtIn, codegenContext.runtimeConfig), - codegenContext, ) } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt index c877d4aac5..37b9a59b40 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt @@ -211,6 +211,10 @@ class RegionProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomi ) } + is ServiceConfig.BuilderFromConfigBag -> { + rustTemplate("${section.builder}.set_region(${section.config_bag}.load::<#{Region}>().cloned());", *codegenScope) + } + else -> emptySection } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt index 927a4c8dd2..4325aee01b 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt @@ -145,6 +145,11 @@ class UserAgentDecorator : ClientCodegenDecorator { ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate("${section.builder}.set_app_name(${section.config_bag}.load::<#{AppName}>().cloned());", *codegenScope) + } + is ServiceConfig.BuilderBuild -> writable { rust("layer.store_put(#T.clone());", ClientRustModule.Meta.toType().resolve("API_METADATA")) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt index 5c91ead09a..928a80f50c 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt @@ -52,6 +52,8 @@ class S3Decorator : ClientCodegenDecorator { private val logger: Logger = Logger.getLogger(javaClass.name) private val invalidXmlRootAllowList = setOf( + // To work around https://github.com/awslabs/aws-sdk-rust/issues/991 + ShapeId.from("com.amazonaws.s3#CreateSessionOutput"), // API returns GetObjectAttributes_Response_ instead of Output ShapeId.from("com.amazonaws.s3#GetObjectAttributesOutput"), ) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt index 544d30ea8f..5bcc870ae5 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt @@ -126,17 +126,9 @@ class OperationInputTestGenerator(_ctx: ClientCodegenContext, private val test: private val model = ctx.model private val instantiator = ClientInstantiator(ctx) - /** tests using S3 Express bucket names need to be disabled until the implementation is in place **/ - private fun EndpointTestCase.isSigV4S3Express() = - expect.endpoint.orNull()?.properties?.get("authSchemes")?.asArrayNode()?.orNull() - ?.map { it.expectObjectNode().expectStringMember("name").value }?.contains("sigv4-s3express") == true - fun generateInput(testOperationInput: EndpointTestOperationInput) = writable { val operationName = testOperationInput.operationName.toSnakeCase() - if (test.isSigV4S3Express()) { - Attribute.shouldPanic("not yet implemented").render(this) - } tokioTest(safeName("operation_input_test_$operationName")) { rustTemplate( """ diff --git a/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json b/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json new file mode 100644 index 0000000000..9bbc52fe05 --- /dev/null +++ b/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json @@ -0,0 +1,193 @@ +{ + "events": [ + { + "connection_id": 0, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session", + "headers": { + "amz-sdk-request": [ + "attempt=1; max=1" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-create-session-mode": [ + "ReadWrite" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=1f63e649e5433837f97c037489613e036b98bff8cdf8d7bcf24b770d041ac0b9" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 0, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "x-amz-id-2": [ + "TQk2NSay" + ], + "x-amz-request-id": [ + "0033eada6b00018d568cbe9f0509499a7de17df8" + ], + "date": [ + "Mon, 29 Jan 2024 18:48:00 GMT" + ], + "content-type": [ + "application/xml" + ], + "content-length": [ + "333" + ], + "server": [ + "AmazonS3" + ] + } + } + } + } + } + }, + { + "connection_id": 0, + "action": { + "Data": { + "data": { + "Utf8": "\nTESTSESSIONTOKENTESTSECRETKEYASIARTESTID2024-01-29T18:53:01Z" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=d1765aa7ec005607ba94fdda08c6739228d5ee14eb8316e80264c35649661a19" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-s3session-token": [ + "TESTSESSIONTOKEN" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 1, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "content-type": [ + "application/xml" + ], + "date": [ + "Mon, 29 Jan 2024 18:48:00 GMT" + ], + "x-amz-request-id": [ + "0033eada6b00018d568cbf350509f775295f94b5" + ], + "x-amz-id-2": [ + "IltTpRJVF1U" + ], + "server": [ + "AmazonS3" + ], + "content-length": [ + "520" + ], + "x-amz-bucket-region": [ + "us-west-2" + ] + } + } + } + } + } + }, + { + "connection_id": 1, + "action": { + "Data": { + "data": { + "Utf8": "\ns3express-test-bucket--usw2-az1--x-s311000falseCRC32"b357dc928b454965a8dd11716a37dab8"hello-world.txt2024-01-29T18:32:24.000Z14EXPRESS_ONEZONE" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + } + ], + "docs": "traffic recording of executing list-objects-v2 against an S3 Express One Zone bucket", + "version": "V0" +} diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs new file mode 100644 index 0000000000..cc0d99aa58 --- /dev/null +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -0,0 +1,131 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use std::time::{Duration, SystemTime}; + +use aws_config::Region; +use aws_sdk_s3::presigning::PresigningConfig; +use aws_sdk_s3::primitives::SdkBody; +use aws_sdk_s3::{Client, Config}; +use aws_smithy_runtime::client::http::test_util::dvr::ReplayingClient; +use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; +use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs; +use http::Uri; + +#[tokio::test] +async fn list_objects_v2() { + let _logs = capture_test_logs(); + + let http_client = + ReplayingClient::from_file("tests/data/express/list-objects-v2.json").unwrap(); + let config = aws_config::from_env() + .http_client(http_client.clone()) + .no_credentials() + .region("us-west-2") + .load() + .await; + let config = Config::from(&config) + .to_builder() + .with_test_defaults() + .build(); + let client = aws_sdk_s3::Client::from_conf(config); + + let result = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + dbg!(result).expect("success"); + + http_client + .validate_body_and_headers(Some(&["x-amz-s3session-token"]), "application/xml") + .await + .unwrap(); +} + +fn create_session_request() -> http::Request { + http::Request::builder() + .uri("https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session") + .header("x-amz-create-session-mode", "ReadWrite") + .method("GET") + .body(SdkBody::empty()) + .unwrap() +} + +fn create_session_response() -> http::Response { + http::Response::builder() + .status(200) + .body(SdkBody::from( + r#" + + + TESTSESSIONTOKEN + TESTSECRETKEY + ASIARTESTID + 2024-01-29T18:53:01Z + + + "#, + )) + .unwrap() +} + +#[tokio::test] +async fn presigning() { + let http_client = StaticReplayClient::new(vec![ReplayEvent::new( + create_session_request(), + create_session_response(), + )]); + + let config = aws_sdk_s3::Config::builder() + .http_client(http_client) + .region(Region::new("us-west-2")) + .with_test_defaults() + .build(); + let client = Client::from_conf(config); + + let presigning_config = PresigningConfig::builder() + .start_time(SystemTime::UNIX_EPOCH + Duration::from_secs(1234567891)) + .expires_in(Duration::from_secs(30)) + .build() + .unwrap(); + + let presigned = client + .get_object() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .key("ferris.png") + .presigned(presigning_config) + .await + .unwrap(); + + let uri = presigned.uri().parse::().unwrap(); + + let pq = uri.path_and_query().unwrap(); + let path = pq.path(); + let query = pq.query().unwrap(); + let mut query_params: Vec<&str> = query.split('&').collect(); + query_params.sort(); + + pretty_assertions::assert_eq!( + "s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com", + uri.authority().unwrap() + ); + assert_eq!("GET", presigned.method()); + assert_eq!("/ferris.png", path); + pretty_assertions::assert_eq!( + &[ + "X-Amz-Algorithm=AWS4-HMAC-SHA256", + "X-Amz-Credential=ASIARTESTID%2F20090213%2Fus-west-2%2Fs3express%2Faws4_request", + "X-Amz-Date=20090213T233131Z", + "X-Amz-Expires=30", + "X-Amz-S3session-Token=TESTSESSIONTOKEN", + "X-Amz-Signature=c09c93c7878184492cb960d59e148af932dff6b19609e63e3484599903d97e44", + "X-Amz-SignedHeaders=host", + "x-id=GetObject" + ][..], + &query_params + ); + assert_eq!(presigned.headers().count(), 0); +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt index 8b6153b9a6..a0a6202358 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt @@ -274,6 +274,21 @@ class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : Conf ) } + is ServiceConfig.BuilderFromConfigBag -> { + rustTemplate( + "${section.builder}.set_retry_config(${section.config_bag}.load::<#{RetryConfig}>().cloned());", + *codegenScope, + ) + rustTemplate( + "${section.builder}.set_timeout_config(${section.config_bag}.load::<#{TimeoutConfig}>().cloned());", + *codegenScope, + ) + rustTemplate( + "${section.builder}.set_retry_partition(${section.config_bag}.load::<#{RetryPartition}>().cloned());", + *codegenScope, + ) + } + else -> emptySection } } diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt index 96ddb2083c..adf7bfddb5 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt @@ -40,7 +40,7 @@ class ClientContextConfigCustomization(ctx: ClientCodegenContext) : ConfigCustom private val configParams = ctx.serviceShape.getTrait()?.parameters.orEmpty().toList() .map { (key, value) -> fromClientParam(key, value, ctx.symbolProvider, runtimeConfig) } - private val decorators = configParams.map { standardConfigParam(it, ctx) } + private val decorators = configParams.map { standardConfigParam(it) } companion object { fun toSymbol( diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt index 38fecb7e48..40b89549b3 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt @@ -80,19 +80,26 @@ sealed class ServiceConfig(name: String) : Section(name) { /** impl block of `ConfigBuilder` **/ object BuilderImpl : ServiceConfig("BuilderImpl") + // It is important to ensure through type system that each field added to config implements this injection, + // tracked by smithy-rs#3419 + /** - * Convert from a field in the builder to the final field in config + * Load a value from a config bag and store it in ConfigBuilder * e.g. * ```kotlin - * rust("""my_field: my_field.unwrap_or_else(||"default")""") + * rust("""builder.set_field(config_bag.load::().cloned())""") * ``` */ - object BuilderBuild : ServiceConfig("BuilderBuild") + data class BuilderFromConfigBag(val builder: String, val config_bag: String) : ServiceConfig("BuilderFromConfigBag") /** - * A section for customizing individual fields in the initializer of Config + * Convert from a field in the builder to the final field in config + * e.g. + * ```kotlin + * rust("""my_field: my_field.unwrap_or_else(||"default")""") + * ``` */ - object BuilderBuildExtras : ServiceConfig("BuilderBuildExtras") + object BuilderBuild : ServiceConfig("BuilderBuild") /** * A section for setting up a field to be used by ConfigOverrideRuntimePlugin @@ -203,10 +210,7 @@ fun loadFromConfigBag( * 2. convenience setter (non-optional) * 3. standard setter (&mut self) */ -fun standardConfigParam( - param: ConfigParam, - codegenContext: ClientCodegenContext, -): ConfigCustomization = +fun standardConfigParam(param: ConfigParam): ConfigCustomization = object : ConfigCustomization() { override fun section(section: ServiceConfig): Writable { return when (section) { @@ -235,6 +239,16 @@ fun standardConfigParam( ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + """ + ${section.builder}.set_${param.name}(${section.config_bag}.#{load_from_config_bag}); + """, + "load_from_config_bag" to loadFromConfigBag(param.type.name, param.newtype!!), + ) + } + else -> emptySection } } @@ -305,6 +319,26 @@ class ServiceConfigGenerator( ), ) + private fun builderFromConfigBag() = + writable { + val builderVar = "builder" + val configBagVar = "config_bag" + + docs("Constructs a config builder from the given `$configBagVar`, setting only fields stored in the config bag,") + docs("but not those in runtime components.") + Attribute.AllowUnused.render(this) + rustBlockTemplate( + "pub(crate) fn from_config_bag($configBagVar: &#{ConfigBag}) -> Self", + *codegenScope, + ) { + rust("let mut $builderVar = Self::new();") + customizations.forEach { + it.section(ServiceConfig.BuilderFromConfigBag(builderVar, configBagVar))(this) + } + rust("$builderVar") + } + } + private fun behaviorMv() = writable { val docs = """ @@ -451,6 +485,8 @@ class ServiceConfigGenerator( writer.rustBlock("impl Builder") { writer.docs("Constructs a config builder.") writer.rust("pub fn new() -> Self { Self::default() }") + + builderFromConfigBag()(this) customizations.forEach { it.section(ServiceConfig.BuilderImpl)(this) } @@ -522,9 +558,6 @@ class ServiceConfigGenerator( it.section(ServiceConfig.BuilderBuild)(this) } rustBlock("Config") { - customizations.forEach { - it.section(ServiceConfig.BuilderBuildExtras)(this) - } rustTemplate( """ config: #{Layer}::from(layer.clone()).with_name("$moduleUseName::config::Config").freeze(), diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt index 0c67acbf43..83c3b6dd6b 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt @@ -96,6 +96,14 @@ class StalledStreamProtectionConfigCustomization(codegenContext: ClientCodegenCo ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_stalled_stream_protection(${section.config_bag}.load::<#{StalledStreamProtectionConfig}>().cloned());", + *codegenScope, + ) + } + else -> emptySection } } diff --git a/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt b/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt index 4e9e88f414..e80d29efea 100644 --- a/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt +++ b/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt @@ -15,6 +15,7 @@ import software.amazon.smithy.rust.codegen.client.testutil.clientIntegrationTest import software.amazon.smithy.rust.codegen.core.rustlang.Writable import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization import software.amazon.smithy.rust.codegen.core.testutil.BasicTestModels @@ -24,6 +25,11 @@ import software.amazon.smithy.rust.codegen.core.util.lookup import software.amazon.smithy.rust.codegen.core.util.toPascalCase internal class ServiceConfigGeneratorTest { + private fun codegenScope(rc: RuntimeConfig): Array> = + arrayOf( + "ConfigBag" to RuntimeType.configBag(rc), + ) + @Test fun `idempotency token when used`() { fun model(trait: String) = @@ -124,6 +130,20 @@ internal class ServiceConfigGeneratorTest { ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + """ + ${section.builder} = ${section.builder}.config_field(${section.config_bag}.load::<#{T}>().map(|u| u.0).unwrap()); + """, + "T" to + configParamNewtype( + "config_field".toPascalCase(), RuntimeType.U64.toSymbol(), + codegenContext.runtimeConfig, + ), + ) + } + else -> emptySection } } @@ -174,6 +194,34 @@ internal class ServiceConfigGeneratorTest { assert_eq!(config.runtime_plugins.len(), 1); """, ) + + unitTest( + "builder_from_config_bag", + """ + use aws_smithy_runtime::client::retries::RetryPartition; + use aws_smithy_types::config_bag::ConfigBag; + use aws_smithy_types::config_bag::Layer; + use aws_smithy_types::retry::RetryConfig; + use aws_smithy_types::timeout::TimeoutConfig; + + let mut layer = Layer::new("test"); + layer.store_put(crate::config::ConfigField(0)); + layer.store_put(RetryConfig::disabled()); + layer.store_put(crate::config::StalledStreamProtectionConfig::disabled()); + layer.store_put(TimeoutConfig::builder().build()); + layer.store_put(RetryPartition::new("test")); + + let config_bag = ConfigBag::of_layers(vec![layer]); + let builder = crate::config::Builder::from_config_bag(&config_bag); + let config = builder.build(); + + assert_eq!(config.config_field(), 0); + assert!(config.retry_config().is_some()); + assert!(config.stalled_stream_protection().is_some()); + assert!(config.timeout_config().is_some()); + assert!(config.retry_partition().is_some()); + """, + ) } } } diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs index 4480779a00..5d86dd5405 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs @@ -386,6 +386,14 @@ impl RuntimeComponents { RuntimeComponentsBuilder::new(name) } + /// Clones and converts this [`RuntimeComponents`] into a [`RuntimeComponentsBuilder`]. + pub fn to_builder(&self) -> RuntimeComponentsBuilder { + RuntimeComponentsBuilder::from_runtime_components( + self.clone(), + "RuntimeComponentsBuilder::from_runtime_components", + ) + } + /// Returns the auth scheme option resolver. pub fn auth_scheme_option_resolver(&self) -> SharedAuthSchemeOptionResolver { self.auth_scheme_option_resolver.value.clone() @@ -497,6 +505,26 @@ impl RuntimeComponents { } impl RuntimeComponentsBuilder { + /// Creates a new [`RuntimeComponentsBuilder`], inheriting all fields from the given + /// [`RuntimeComponents`]. + pub fn from_runtime_components(rc: RuntimeComponents, builder_name: &'static str) -> Self { + Self { + builder_name, + auth_scheme_option_resolver: Some(rc.auth_scheme_option_resolver), + http_client: rc.http_client, + endpoint_resolver: Some(rc.endpoint_resolver), + auth_schemes: rc.auth_schemes, + identity_cache: Some(rc.identity_cache), + identity_resolvers: Some(rc.identity_resolvers), + interceptors: rc.interceptors, + retry_classifiers: rc.retry_classifiers, + retry_strategy: Some(rc.retry_strategy), + time_source: rc.time_source, + sleep_impl: rc.sleep_impl, + config_validators: rc.config_validators, + } + } + /// Returns the auth scheme option resolver. pub fn auth_scheme_option_resolver(&self) -> Option { self.auth_scheme_option_resolver From 0a75b4183c8a6c4ac11d57efe75b6f8da1110e2f Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Tue, 20 Feb 2024 13:47:18 -0600 Subject: [PATCH 03/16] Add S3 Express identity cache (#3390) ## Motivation and Context Adds a default implementation for S3 Express identity cache. ## Description This PR adds the said cache for S3 Express. This cache is not configurable from outside and solely owned by the default S3 Express identity provider. It is implemented in terms of an LRU cache keyed on a string generated by `sha256hmac(random 64-byte key, access_key_id + secret_key) + bucket_name` (note: `access_key_id` and `secret_key` are for a customer's credentials but not for a retrieved `create_session` API token). Cache values are of type `ExpiringCache` that contains a session token retrieved by S3's `create_session` API. When a customer is trying to use a cached session token but if it has expired, `ExpiringCache` calls the S3's `create_session` API, stores in it a new session token, and returns it to the customer. ## Testing Added unit tests for `S3IdentityCache` and a connection recording test for `list-objects-v2` running against both express and regular buckets to exercise a use case where a customer is switching between those buckets. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti Co-authored-by: Russell Cohen Co-authored-by: AWS SDK Rust Bot Co-authored-by: AWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com> Co-authored-by: Zelda Hessler --- aws/rust-runtime/aws-inlineable/Cargo.toml | 4 + .../aws-inlineable/src/s3_express.rs | 461 +++++++++++++- .../customize/s3/S3ExpressDecorator.kt | 59 +- .../s3/tests/data/express/mixed-auths.json | 585 ++++++++++++++++++ aws/sdk/integration-tests/s3/tests/express.rs | 58 ++ .../codegen/core/rustlang/CargoDependency.kt | 3 + 6 files changed, 1132 insertions(+), 38 deletions(-) create mode 100644 aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json diff --git a/aws/rust-runtime/aws-inlineable/Cargo.toml b/aws/rust-runtime/aws-inlineable/Cargo.toml index 85b6c440cb..401bf4cca6 100644 --- a/aws/rust-runtime/aws-inlineable/Cargo.toml +++ b/aws/rust-runtime/aws-inlineable/Cargo.toml @@ -22,10 +22,14 @@ aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", featur aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] } aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["http-body-0-4-x"] } bytes = "1" +fastrand = "2.0.0" hex = "0.4.3" http = "0.2.9" http-body = "0.4.5" +hmac = "0.12" +lru = "0.12.2" ring = "0.17.5" +sha2 = "0.10" tokio = "1.23.1" tracing = "0.1" diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 3388145112..6608707d2e 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -93,23 +93,408 @@ pub(crate) mod auth { /// Supporting code for S3 Express identity cache pub(crate) mod identity_cache { + use aws_credential_types::Credentials; + use aws_smithy_async::time::SharedTimeSource; + use aws_smithy_runtime::expiring_cache::ExpiringCache; + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::identity::Identity; + use aws_smithy_types::DateTime; + use fastrand::Rng; + use hmac::{digest::FixedOutput, Hmac, Mac}; + use lru::LruCache; + use sha2::Sha256; + use std::fmt; + use std::future::Future; + use std::hash::Hash; + use std::num::NonZeroUsize; + use std::sync::Mutex; + use std::time::{Duration, SystemTime}; + + pub(crate) const DEFAULT_MAX_CACHE_CAPACITY: usize = 100; + pub(crate) const DEFAULT_BUFFER_TIME: Duration = Duration::from_secs(10); + + #[derive(Clone, Eq, PartialEq, Hash)] + pub(crate) struct CacheKey(String); + /// The caching implementation for S3 Express identity. /// /// While customers can either disable S3 Express itself or provide a custom S3 Express identity /// provider, configuring S3 Express identity cache is not supported. Thus, this is _the_ /// implementation of S3 Express identity cache. - #[derive(Debug)] - pub(crate) struct S3ExpressIdentityCache; -} + pub(crate) struct S3ExpressIdentityCache { + inner: Mutex>>, + time_source: SharedTimeSource, + buffer_time: Duration, + random_bytes: [u8; 64], + } + + impl fmt::Debug for S3ExpressIdentityCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (size, capacity) = { + let cache = self.inner.lock().unwrap(); + (cache.len(), cache.cap()) + }; + write!( + f, + "S3ExpressIdentityCache {{ time_source: {:?}, buffer_time: {:?} }}, with size/capacity: {}/{}", + self.time_source, &self.buffer_time, size, capacity, + ) + } + } + + impl S3ExpressIdentityCache { + pub(crate) fn new( + capacity: usize, + time_source: SharedTimeSource, + buffer_time: Duration, + ) -> Self { + // It'd be nice to use a cryptographically secure random generator but not necessary. + // The cache is memory only and randomization here is mostly to obfuscate the key and + // make it reasonable length. + let mut rng = Rng::default(); + let mut random_bytes = [0u8; 64]; + rng.fill(&mut random_bytes); + Self { + inner: Mutex::new(LruCache::new(NonZeroUsize::new(capacity).unwrap())), + time_source, + buffer_time, + random_bytes, + } + } + + pub(crate) fn key(&self, bucket_name: &str, creds: &Credentials) -> CacheKey { + CacheKey({ + let mut mac = Hmac::::new_from_slice(self.random_bytes.as_slice()) + .expect("should be created from random 64 bytes"); + let input = format!("{}{}", creds.access_key_id(), creds.secret_access_key()); + mac.update(input.as_ref()); + let mut inner = hex::encode(mac.finalize_fixed()); + inner.push_str(bucket_name); + inner + }) + } + + pub(crate) async fn get_or_load( + &self, + key: CacheKey, + loader: F, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let expiring_cache = { + let mut inner = self.inner.lock().unwrap(); + inner + .get_or_insert_mut(key, || ExpiringCache::new(self.buffer_time)) + .clone() + }; + + let now = self.time_source.now(); + + match expiring_cache.yield_or_clear_if_expired(now).await { + Some(identity) => { + tracing::debug!( + buffer_time=?self.buffer_time, + cached_expiration=?identity.expiration(), + now=?now, + "loaded identity from cache" + ); + Ok(identity) + } + None => { + let start_time = self.time_source.now(); + let identity = expiring_cache.get_or_load(loader).await?; + let expiration = identity + .expiration() + .ok_or("SessionCredentials` always has expiration")?; + let printable = DateTime::from(expiration); + tracing::info!( + new_expiration=%printable, + valid_for=?expiration.duration_since(self.time_source.now()).unwrap_or_default(), + "identity cache miss occurred; added new identity (took {:?})", + self.time_source.now().duration_since(start_time).unwrap_or_default() + ); + Ok(identity) + } + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + use aws_smithy_async::rt::sleep::TokioSleep; + use aws_smithy_async::test_util::ManualTimeSource; + use aws_smithy_runtime_api::client::identity::http::Token; + use aws_smithy_runtime_api::client::identity::{ + IdentityFuture, ResolveIdentity, SharedIdentityResolver, + }; + use aws_smithy_runtime_api::client::runtime_components::{ + RuntimeComponents, RuntimeComponentsBuilder, + }; + use aws_smithy_runtime_api::shared::IntoShared; + use aws_smithy_types::config_bag::ConfigBag; + use futures_util::stream::FuturesUnordered; + use std::sync::Arc; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use tracing::info; + + fn epoch_secs(secs: u64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(secs) + } + + fn identity_expiring_in(expired_secs: u64) -> Identity { + let expiration = Some(epoch_secs(expired_secs)); + Identity::new(Token::new("test", expiration), expiration) + } + + fn test_identity_resolver( + load_list: Vec>, + ) -> SharedIdentityResolver { + #[derive(Debug)] + struct Resolver(Mutex>>); + impl ResolveIdentity for Resolver { + fn resolve_identity<'a>( + &'a self, + _: &'a RuntimeComponents, + _config_bag: &'a ConfigBag, + ) -> IdentityFuture<'a> { + let mut list = self.0.lock().unwrap(); + if list.len() > 0 { + let next = list.remove(0); + info!("refreshing the identity to {:?}", next); + IdentityFuture::ready(next) + } else { + drop(list); + panic!("no more identities") + } + } + } + + SharedIdentityResolver::new(Resolver(Mutex::new(load_list))) + } + + async fn load( + identity_resolver: SharedIdentityResolver, + runtime_components: &RuntimeComponents, + ) -> Result<(Identity, SystemTime), BoxError> { + let identity = identity_resolver + .resolve_identity(&runtime_components, &ConfigBag::base()) + .await + .unwrap(); + Ok((identity.clone(), identity.expiration().unwrap())) + } + + async fn expect_identity( + expired_secs: u64, + sut: &S3ExpressIdentityCache, + key: CacheKey, + loader: F, + ) where + F: FnOnce() -> Fut, + Fut: Future>, + { + let identity = sut.get_or_load(key, loader).await.unwrap(); + assert_eq!(Some(epoch_secs(expired_secs)), identity.expiration()); + } + + #[tokio::test] + async fn reload_expired_test_identity() { + let time = ManualTimeSource::new(UNIX_EPOCH); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(time.clone())) + .with_sleep_impl(Some(TokioSleep::new())) + .build() + .unwrap(); + + let sut = + S3ExpressIdentityCache::new(1, time.clone().into_shared(), DEFAULT_BUFFER_TIME); + + let identity_resolver = test_identity_resolver(vec![ + Ok(identity_expiring_in(1000)), + Ok(identity_expiring_in(2000)), + ]); + + let key = sut.key( + "test-bucket--usw2-az1--x-s3", + &Credentials::for_tests_with_session_token(), + ); + + // First call to the cache, populating a cache entry. + expect_identity(1000, &sut, key.clone(), || { + let identity_resolver = identity_resolver.clone(); + let runtime_components = runtime_components.clone(); + async move { load(identity_resolver, &runtime_components).await } + }) + .await; + + // Testing for a cache hit by advancing time such that the updated time is before the expiration of the first identity + // i.e. 500 < 1000. + time.set_time(epoch_secs(500)); + + expect_identity(1000, &sut, key.clone(), || async move { + panic!("new identity should not be loaded") + }) + .await; + + // Testing for a cache miss by advancing time such that the updated time is now after the expiration of the first identity + // and before the expiration of the second identity i.e. 1000 < 1500 && 1500 < 2000. + time.set_time(epoch_secs(1500)); + + expect_identity(2000, &sut, key, || async move { + load(identity_resolver, &runtime_components).await + }) + .await; + } + #[test] + fn load_contention() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_time() + .worker_threads(16) + .build() + .unwrap(); + + let time = ManualTimeSource::new(epoch_secs(0)); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(time.clone())) + .with_sleep_impl(Some(TokioSleep::new())) + .build() + .unwrap(); + + let number_of_buckets = 4; + let sut = Arc::new(S3ExpressIdentityCache::new( + number_of_buckets, + time.clone().into_shared(), + DEFAULT_BUFFER_TIME, + )); + + // Nested for loops below advance time by 200 in total, and each identity has the expiration + // such that no matter what order async tasks are executed, it never expires. + let safe_expiration = number_of_buckets as u64 * 50 + DEFAULT_BUFFER_TIME.as_secs() + 1; + let identity_resolver = test_identity_resolver(vec![ + Ok(identity_expiring_in(safe_expiration)), + Ok(identity_expiring_in(safe_expiration)), + Ok(identity_expiring_in(safe_expiration)), + Ok(identity_expiring_in(safe_expiration)), + ]); + + let mut tasks = Vec::new(); + for i in 0..number_of_buckets { + let key = sut.key( + &format!("test-bucket-{i}-usw2-az1--x-s3"), + &Credentials::for_tests_with_session_token(), + ); + for _ in 0..50 { + let sut = sut.clone(); + let key = key.clone(); + let identity_resolver = identity_resolver.clone(); + let time = time.clone(); + let runtime_components = runtime_components.clone(); + tasks.push(rt.spawn(async move { + let now = time.advance(Duration::from_secs(1)); + let identity: Identity = sut + .get_or_load(key, || async move { + load(identity_resolver, &runtime_components).await + }) + .await + .unwrap(); + + assert!( + identity.expiration().unwrap() >= now, + "{:?} >= {:?}", + identity.expiration(), + now + ); + })); + } + } + let tasks = tasks.into_iter().collect::>(); + for task in tasks { + rt.block_on(task).unwrap(); + } + } + + #[tokio::test] + async fn identity_fetch_triggered_by_lru_eviction() { + let time = ManualTimeSource::new(UNIX_EPOCH); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(time.clone())) + .with_sleep_impl(Some(TokioSleep::new())) + .build() + .unwrap(); + + // Create a cache of size 2. + let sut = S3ExpressIdentityCache::new(2, time.into_shared(), DEFAULT_BUFFER_TIME); + + let identity_resolver = test_identity_resolver(vec![ + Ok(identity_expiring_in(1000)), + Ok(identity_expiring_in(2000)), + Ok(identity_expiring_in(3000)), + Ok(identity_expiring_in(4000)), + ]); + + let [key1, key2, key3] = [1, 2, 3].map(|i| { + sut.key( + &format!("test-bucket-{i}--usw2-az1--x-s3"), + &Credentials::for_tests_with_session_token(), + ) + }); + + // This should pupulate a cache entry for `key1`. + expect_identity(1000, &sut, key1.clone(), || { + let identity_resolver = identity_resolver.clone(); + let runtime_components = runtime_components.clone(); + async move { load(identity_resolver, &runtime_components).await } + }) + .await; + // This immediate next call for `key1` should be a cache hit. + expect_identity(1000, &sut, key1.clone(), || async move { + panic!("new identity should not be loaded") + }) + .await; + + // This should pupulate a cache entry for `key2`. + expect_identity(2000, &sut, key2, || { + let identity_resolver = identity_resolver.clone(); + let runtime_components = runtime_components.clone(); + async move { load(identity_resolver, &runtime_components).await } + }) + .await; + + // This should pupulate a cache entry for `key3`, but evicting a cache entry for `key1` because the cache is full. + expect_identity(3000, &sut, key3.clone(), || { + let identity_resolver = identity_resolver.clone(); + let runtime_components = runtime_components.clone(); + async move { load(identity_resolver, &runtime_components).await } + }) + .await; + + // Attempt to get an identity for `key1` should end up fetching a new one since its cache entry has been evicted. + // This fetch should now evict a cache entry for `key2`. + expect_identity(4000, &sut, key1, || async move { + load(identity_resolver, &runtime_components).await + }) + .await; + + // A cache entry for `key3` should still exist in the cache. + expect_identity(3000, &sut, key3, || async move { + panic!("new identity should not be loaded") + }) + .await; + } + } +} /// Supporting code for S3 Express identity provider pub(crate) mod identity_provider { - use std::time::SystemTime; + use std::time::{Duration, SystemTime}; use crate::s3_express::identity_cache::S3ExpressIdentityCache; use crate::types::SessionCredentials; use aws_credential_types::provider::error::CredentialsError; use aws_credential_types::Credentials; + use aws_smithy_async::time::{SharedTimeSource, TimeSource}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::endpoint::EndpointResolverParams; use aws_smithy_runtime_api::client::identity::{ @@ -119,16 +504,16 @@ pub(crate) mod identity_provider { use aws_smithy_runtime_api::client::runtime_components::{ GetIdentityResolver, RuntimeComponents, }; + use aws_smithy_runtime_api::shared::IntoShared; use aws_smithy_types::config_bag::ConfigBag; + use super::identity_cache::{DEFAULT_BUFFER_TIME, DEFAULT_MAX_CACHE_CAPACITY}; + #[derive(Debug)] pub(crate) struct DefaultS3ExpressIdentityProvider { - _cache: S3ExpressIdentityCache, + cache: S3ExpressIdentityCache, } - #[derive(Default)] - pub(crate) struct Builder; - impl TryFrom for Credentials { type Error = BoxError; @@ -149,7 +534,7 @@ pub(crate) mod identity_provider { impl DefaultS3ExpressIdentityProvider { pub(crate) fn builder() -> Builder { - Builder + Builder::default() } async fn identity<'a>( @@ -162,20 +547,28 @@ pub(crate) mod identity_provider { let sigv4_identity_resolver = runtime_components .identity_resolver(aws_runtime::auth::sigv4::SCHEME_ID) .ok_or("identity resolver for sigv4 should be set for S3")?; - let _aws_identity = runtime_components + let aws_identity = runtime_components .identity_cache() .resolve_cached_identity(sigv4_identity_resolver, runtime_components, config_bag) .await?; - // TODO(S3Express): use both `bucket_name` and `aws_identity` as part of `S3ExpressIdentityCache` implementation - - let express_session_credentials = self - .express_session_credentials(bucket_name, runtime_components, config_bag) - .await?; - - let data = Credentials::try_from(express_session_credentials)?; - - Ok(Identity::new(data.clone(), data.expiry())) + let credentials = aws_identity.data::().ok_or( + "wrong identity type for SigV4. Expected AWS credentials but got `{identity:?}", + )?; + + let key = self.cache.key(bucket_name, credentials); + self.cache + .get_or_load(key, || async move { + let creds = self + .express_session_credentials(bucket_name, runtime_components, config_bag) + .await?; + let data = Credentials::try_from(creds)?; + Ok(( + Identity::new(data.clone(), data.expiry()), + data.expiry().unwrap(), + )) + }) + .await } fn bucket_name<'a>(&'a self, config_bag: &'a ConfigBag) -> Result<&'a str, BoxError> { @@ -218,10 +611,38 @@ pub(crate) mod identity_provider { } } + #[derive(Default)] + pub(crate) struct Builder { + time_source: Option, + buffer_time: Option, + } + impl Builder { + pub(crate) fn time_source(mut self, time_source: impl TimeSource + 'static) -> Self { + self.set_time_source(time_source.into_shared()); + self + } + pub(crate) fn set_time_source(&mut self, time_source: SharedTimeSource) -> &mut Self { + self.time_source = Some(time_source.into_shared()); + self + } + #[allow(dead_code)] + pub(crate) fn buffer_time(mut self, buffer_time: Duration) -> Self { + self.set_buffer_time(Some(buffer_time)); + self + } + #[allow(dead_code)] + pub(crate) fn set_buffer_time(&mut self, buffer_time: Option) -> &mut Self { + self.buffer_time = buffer_time; + self + } pub(crate) fn build(self) -> DefaultS3ExpressIdentityProvider { DefaultS3ExpressIdentityProvider { - _cache: S3ExpressIdentityCache, + cache: S3ExpressIdentityCache::new( + DEFAULT_MAX_CACHE_CAPACITY, + self.time_source.unwrap_or_default(), + self.buffer_time.unwrap_or(DEFAULT_BUFFER_TIME), + ), } } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 8529fdfc47..cda1971795 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -15,12 +15,15 @@ import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRunti import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Writable import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rustsdk.AwsCargoDependency import software.amazon.smithy.rustsdk.AwsRuntimeType import software.amazon.smithy.rustsdk.InlineAwsDependency @@ -28,13 +31,11 @@ class S3ExpressDecorator : ClientCodegenDecorator { override val name: String = "S3ExpressDecorator" override val order: Byte = 0 - private fun sigv4S3Express() = + private fun sigv4S3Express(runtimeConfig: RuntimeConfig) = writable { rust( "#T", - RuntimeType.forInlineDependency( - InlineAwsDependency.forRustFile("s3_express"), - ).resolve("auth::SCHEME_ID"), + s3ExpressModule(runtimeConfig).resolve("auth::SCHEME_ID"), ) } @@ -46,7 +47,7 @@ class S3ExpressDecorator : ClientCodegenDecorator { baseAuthSchemeOptions + AuthSchemeOption.StaticAuthSchemeOption( SigV4Trait.ID, - listOf(sigv4S3Express()), + listOf(sigv4S3Express(codegenContext.runtimeConfig)), ) override fun serviceRuntimePluginCustomizations( @@ -67,20 +68,14 @@ private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientC private val codegenScope by lazy { arrayOf( "DefaultS3ExpressIdentityProvider" to - RuntimeType.forInlineDependency( - InlineAwsDependency.forRustFile("s3_express"), - ).resolve("identity_provider::DefaultS3ExpressIdentityProvider"), + s3ExpressModule(runtimeConfig).resolve("identity_provider::DefaultS3ExpressIdentityProvider"), "IdentityCacheLocation" to RuntimeType.smithyRuntimeApiClient(runtimeConfig) .resolve("client::identity::IdentityCacheLocation"), "S3ExpressAuthScheme" to - RuntimeType.forInlineDependency( - InlineAwsDependency.forRustFile("s3_express"), - ).resolve("auth::S3ExpressAuthScheme"), + s3ExpressModule(runtimeConfig).resolve("auth::S3ExpressAuthScheme"), "S3_EXPRESS_SCHEME_ID" to - RuntimeType.forInlineDependency( - InlineAwsDependency.forRustFile("s3_express"), - ).resolve("auth::SCHEME_ID"), + s3ExpressModule(runtimeConfig).resolve("auth::SCHEME_ID"), "SharedAuthScheme" to RuntimeType.smithyRuntimeApiClient(runtimeConfig) .resolve("client::auth::SharedAuthScheme"), @@ -112,7 +107,14 @@ private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientC rustTemplate("#{S3_EXPRESS_SCHEME_ID}", *codegenScope) }, writable { - rustTemplate("#{DefaultS3ExpressIdentityProvider}::builder().build()", *codegenScope) + rustTemplate( + """ + #{DefaultS3ExpressIdentityProvider}::builder() + .time_source(${section.serviceConfigName}.time_source().unwrap_or_default()) + .build() + """, + *codegenScope, + ) }, ) } @@ -144,9 +146,7 @@ class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : Co RuntimeType.smithyRuntimeApiClient(runtimeConfig) .resolve("client::identity::SharedIdentityResolver"), "S3_EXPRESS_SCHEME_ID" to - RuntimeType.forInlineDependency( - InlineAwsDependency.forRustFile("s3_express"), - ).resolve("auth::SCHEME_ID"), + s3ExpressModule(runtimeConfig).resolve("auth::SCHEME_ID"), ) override fun section(section: ServiceConfig) = @@ -182,3 +182,26 @@ class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : Co } } } + +private fun s3ExpressModule(runtimeConfig: RuntimeConfig) = + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile( + "s3_express", + additionalDependency = s3ExpressDependencies(runtimeConfig), + ), + ) + +private fun s3ExpressDependencies(runtimeConfig: RuntimeConfig) = + arrayOf( + AwsCargoDependency.awsCredentialTypes(runtimeConfig), + AwsCargoDependency.awsRuntime(runtimeConfig), + AwsCargoDependency.awsSigv4(runtimeConfig), + CargoDependency.FastRand, + CargoDependency.Hex, + CargoDependency.Hmac, + CargoDependency.Lru, + CargoDependency.Sha2, + CargoDependency.smithyAsync(runtimeConfig), + CargoDependency.smithyRuntimeApiClient(runtimeConfig), + CargoDependency.smithyTypes(runtimeConfig), + ) diff --git a/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json b/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json new file mode 100644 index 0000000000..631cb3bf01 --- /dev/null +++ b/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json @@ -0,0 +1,585 @@ +{ + "events": [ + { + "connection_id": 0, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session", + "headers": { + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=4d44fb95628114b17a0676e2da758dee74f8a0337a3f0183e5b91e4fea6d9303" + ], + "x-amz-create-session-mode": [ + "ReadWrite" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-date": [ + "20090213T233130Z" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 0, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "content-type": [ + "application/xml" + ], + "server": [ + "AmazonS3" + ], + "content-length": [ + "1035" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ], + "x-amz-request-id": [ + "0033eada6b00018d62cbbce90509fbf343a84844" + ], + "x-amz-id-2": [ + "ucekwm0" + ] + } + } + } + } + } + }, + { + "connection_id": 0, + "action": { + "Data": { + "data": { + "Utf8": "\nAgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9PLqL36OqPDlwPN6koQgVrhr7R0QEuFbzsH4Bvz4btNZIQUUAEYD2XZOSJYC274XBMVU2024-02-01T03:57:15Z" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-s3session-token": [ + "AgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9P" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=NZIQUUAEYD2XZOSJYC274XBMVU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=63ec8429a44ae20698b21a810030589d2369a13c352d32970931de7b2cede687" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 1, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "server": [ + "AmazonS3" + ], + "x-amz-bucket-region": [ + "us-west-2" + ], + "x-amz-request-id": [ + "0033eada6b00018d62cbbd9c0509eacc646e551d" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ], + "content-length": [ + "520" + ], + "content-type": [ + "application/xml" + ], + "x-amz-id-2": [ + "VLW4GcfH" + ] + } + } + } + } + } + }, + { + "connection_id": 1, + "action": { + "Data": { + "data": { + "Utf8": "\ns3express-test-bucket--usw2-az1--x-s311000falseCRC32"b357dc928b454965a8dd11716a37dab8"hello-world.txt2024-01-29T18:32:24.000Z14EXPRESS_ONEZONE" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 2, + "action": { + "Request": { + "request": { + "uri": "https://regular-test-bucket.s3.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-user-agent, Signature=28c4fed28f2d3ee6780b5ce36a052becdb4257e910a991467eb947aad3d7ea81" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 2, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 2, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "x-amz-request-id": [ + "SCS4NK66GX0KHW8Q" + ], + "content-type": [ + "application/xml" + ], + "server": [ + "AmazonS3" + ], + "x-amz-bucket-region": [ + "us-west-2" + ], + "transfer-encoding": [ + "chunked" + ], + "x-amz-id-2": [ + "1TqTeHEvPfLuR0LGMqatChbIYSJm6p0VsCbdOG0HG5q3BsVhYg5RMIOzAGYSF0xBVn+SLpmTkU6m1ARguYLRnA==" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ] + } + } + } + } + } + }, + { + "connection_id": 2, + "action": { + "Data": { + "data": { + "Utf8": "\nregular-test-bucket11000falseferris.png2024-02-01T03:48:28.000Z"1316cc7c39e43c50c160f0aa8168db41"58413STANDARD" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 2, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 3, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket-2--usw2-az3--x-s3.s3express-usw2-az3.us-west-2.amazonaws.com/?session", + "headers": { + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=4147d1c9c1c54efea9729be3816ec31e25496b7b1a6f116c479e8391444daa7c" + ], + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "x-amz-create-session-mode": [ + "ReadWrite" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 3, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 3, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "server": [ + "AmazonS3" + ], + "content-type": [ + "application/xml" + ], + "x-amz-request-id": [ + "0033eada6b00018d62cbc0e3050ad61d5828bce1" + ], + "content-length": [ + "1043" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ], + "x-amz-id-2": [ + "bL6uF0sLOL" + ] + } + } + } + } + } + }, + { + "connection_id": 3, + "action": { + "Data": { + "data": { + "Utf8": "\nAgAAANCuNFCuOMGL3GiOJQLuVjgOSnFrPq+7P4foPPYEAAAAAAAAAA6gXF7NAQAAAAAAAL6C3/74KCW+AXc/l3wx7EZdxgJtYloTtkuUXCdptWfMT+fTaAkgNGIg2f7Iw+reIMDGN+CfwZGXbnL+rbL6rDf7WFJ5EVb+nIAdeJepa8ZWAYxY1eBq2PLmXcAd/QX8dKfJijmcFk4eDi40J1TPoiL7uao6pxUkj8lK9P9MUTl+IYtvLMcEGGa5/k+Xx+Fclrj+HNA/NoEDfOub9yQ4ei/rfX7efkVm/YWcexLwSmiZ32KaoQRxa2C1DIHozypgQmA+M2K1KTDzRd9Ks2YSTlqsmKzaHnqvxrbqpQec2Z6Tb7t9z1WimidZw9hh10v57kSdnNWGfAWtaOcsopHs0QqVIxTc2XiO32pojj6NtPgoGb3jSmbbDg9Q3905CsPt4KFhaLM3oIHBbQzVNAqewWuBm0Z3olpmHOBF3CBQ7HB2QQ4g8lYouUOyQQqJTlF8ls12VCI8rprnlqcS2iqn/neWwcFbc4/cdRvu0AeOmktCnq8izc6eXiaIBso6Tv7KO4vVfCjVeRG7UkzCyTd3SsnEULENOxWTToo8BpUyxtO6ULaoI7GSQz64jHUMAguZ82zO3gX2TA2qF0ALG0JnKaA+pnri5SyjObMeV9XHzs6gDjtqiCOafa7fd/Najg/Tz2531bSDoGsQX4OGYD3AZC32QIUZXYZQULJ6J7EPNU2024-02-01T03:57:16Z" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 3, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 4, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket-2--usw2-az3--x-s3.s3express-usw2-az3.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "authorization": [ + "AWS4-HMAC-SHA256 Credential=YD3AZC32QIUZXYZQULJ6J7EPNU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=7e6b04135e77aa126142365335cbeb14f98c031d6abc3ce9c281d528034c9874" + ], + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "x-amz-s3session-token": [ + "AgAAANCuNFCuOMGL3GiOJQLuVjgOSnFrPq+7P4foPPYEAAAAAAAAAA6gXF7NAQAAAAAAAL6C3/74KCW+AXc/l3wx7EZdxgJtYloTtkuUXCdptWfMT+fTaAkgNGIg2f7Iw+reIMDGN+CfwZGXbnL+rbL6rDf7WFJ5EVb+nIAdeJepa8ZWAYxY1eBq2PLmXcAd/QX8dKfJijmcFk4eDi40J1TPoiL7uao6pxUkj8lK9P9MUTl+IYtvLMcEGGa5/k+Xx+Fclrj+HNA/NoEDfOub9yQ4ei/rfX7efkVm/YWcexLwSmiZ32KaoQRxa2C1DIHozypgQmA+M2K1KTDzRd9Ks2YSTlqsmKzaHnqvxrbqpQec2Z6Tb7t9z1WimidZw9hh10v57kSdnNWGfAWtaOcsopHs0QqVIxTc2XiO32pojj6NtPgoGb3jSmbbDg9Q3905CsPt4KFhaLM3oIHBbQzVNAqewWuBm0Z3olpmHOBF3CBQ7HB2QQ4g8lYouUOyQQqJTlF8ls12VCI8rprnlqcS2iqn/neWwcFbc4/cdRvu0AeOmktCnq8izc6eXiaIBso6Tv7KO4vVfCjVeRG7UkzCyTd3SsnEULENOxWTToo8BpUyxtO6ULaoI7GSQz64jHUMAguZ82zO3gX2TA2qF0ALG0JnKaA+pnri5SyjObMeV9XH" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 4, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 4, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "x-amz-request-id": [ + "0033eada6b00018d62cbc175050a70195c6f02fc" + ], + "content-length": [ + "517" + ], + "x-amz-id-2": [ + "PrnjEZSu97xW" + ], + "server": [ + "AmazonS3" + ], + "x-amz-bucket-region": [ + "us-west-2" + ], + "content-type": [ + "application/xml" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ] + } + } + } + } + } + }, + { + "connection_id": 4, + "action": { + "Data": { + "data": { + "Utf8": "\ns3express-test-bucket-2--usw2-az3--x-s311000falseCRC32"278b621d9c444360b3614a78cf1a64d1"foobar.txt2024-02-01T03:32:07.000Z10EXPRESS_ONEZONE" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 4, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 5, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "x-amz-content-sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-s3session-token": [ + "AgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9P" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=NZIQUUAEYD2XZOSJYC274XBMVU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=63ec8429a44ae20698b21a810030589d2369a13c352d32970931de7b2cede687" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 5, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 5, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "x-amz-id-2": [ + "LX0peO7j" + ], + "x-amz-request-id": [ + "0033eada6b00018d62cbc1cd0509f0c8b6f330e7" + ], + "server": [ + "AmazonS3" + ], + "x-amz-bucket-region": [ + "us-west-2" + ], + "date": [ + "Fri, 13 Feb 2009 23:31:30 GMT" + ], + "content-type": [ + "application/xml" + ], + "content-length": [ + "520" + ] + } + } + } + } + } + }, + { + "connection_id": 5, + "action": { + "Data": { + "data": { + "Utf8": "\ns3express-test-bucket--usw2-az1--x-s311000falseCRC32"b357dc928b454965a8dd11716a37dab8"hello-world.txt2024-01-29T18:32:24.000Z14EXPRESS_ONEZONE" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 5, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + } + ], + "docs": "traffic recording of listing objects in both an S3 Express One Zone bucket and a regular S3 bucket", + "version": "V0" +} diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index cc0d99aa58..b361397332 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -45,6 +45,64 @@ async fn list_objects_v2() { .unwrap(); } +#[tokio::test] +async fn mixed_auths() { + let _logs = capture_test_logs(); + + let http_client = ReplayingClient::from_file("tests/data/express/mixed-auths.json").unwrap(); + let config = aws_config::from_env() + .http_client(http_client.clone()) + .no_credentials() + .region("us-west-2") + .load() + .await; + let config = Config::from(&config) + .to_builder() + .with_test_defaults() + .build(); + let client = aws_sdk_s3::Client::from_conf(config); + + // A call to an S3 Express bucket where we should see two request/response pairs, + // one for the `create_session` API and the other for `list_objects_v2` in S3 Express bucket. + let result = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + dbg!(result).expect("success"); + + // A call to a regular bucket, and request headers should not contain `x-amz-s3session-token`. + let result = client + .list_objects_v2() + .bucket("regular-test-bucket") + .send() + .await; + dbg!(result).expect("success"); + + // A call to another S3 Express bucket where we should again see two request/response pairs, + // one for the `create_session` API and the other for `list_objects_v2` in S3 Express bucket. + let result = client + .list_objects_v2() + .bucket("s3express-test-bucket-2--usw2-az3--x-s3") + .send() + .await; + dbg!(result).expect("success"); + + // This call should be an identity cache hit for the first S3 Express bucket, + // thus no HTTP request should be sent to the `create_session` API. + let result = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + dbg!(result).expect("success"); + + http_client + .validate_body_and_headers(Some(&["x-amz-s3session-token"]), "application/xml") + .await + .unwrap(); +} + fn create_session_request() -> http::Request { http::Request::builder() .uri("https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session") diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt index 23ce4428af..378cbed860 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt @@ -248,16 +248,19 @@ data class CargoDependency( val BytesUtils: CargoDependency = CargoDependency("bytes-utils", CratesIo("0.1.0")) val FastRand: CargoDependency = CargoDependency("fastrand", CratesIo("2.0.0")) val Hex: CargoDependency = CargoDependency("hex", CratesIo("0.4.3")) + val Hmac: CargoDependency = CargoDependency("hmac", CratesIo("0.12")) val Http: CargoDependency = CargoDependency("http", CratesIo("0.2.9")) val HttpBody: CargoDependency = CargoDependency("http-body", CratesIo("0.4.4")) val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14.26")) val HyperWithStream: CargoDependency = Hyper.withFeature("stream") val LazyStatic: CargoDependency = CargoDependency("lazy_static", CratesIo("1.4.0")) + val Lru: CargoDependency = CargoDependency("lru", CratesIo("0.12.2")) val Md5: CargoDependency = CargoDependency("md-5", CratesIo("0.10.0"), rustName = "md5") val PercentEncoding: CargoDependency = CargoDependency("percent-encoding", CratesIo("2.0.0")) val Regex: CargoDependency = CargoDependency("regex", CratesIo("1.5.5")) val RegexLite: CargoDependency = CargoDependency("regex-lite", CratesIo("0.1.5")) val Ring: CargoDependency = CargoDependency("ring", CratesIo("0.17.5")) + val Sha2: CargoDependency = CargoDependency("sha2", CratesIo("0.10")) val TokioStream: CargoDependency = CargoDependency("tokio-stream", CratesIo("0.1.7")) val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4")) val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) From dc8626f5c234d5c1ecd068526afe3c9d3efb7980 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Tue, 20 Feb 2024 16:20:11 -0600 Subject: [PATCH 04/16] Anonymize test session token --- .../s3/tests/data/express/mixed-auths.json | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json b/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json index 631cb3bf01..b896ff3c98 100644 --- a/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json +++ b/aws/sdk/integration-tests/s3/tests/data/express/mixed-auths.json @@ -8,7 +8,7 @@ "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session", "headers": { "authorization": [ - "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=4d44fb95628114b17a0676e2da758dee74f8a0337a3f0183e5b91e4fea6d9303" + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=38991fe77e65f7e9ffba9ddaeb0ba457d8673bb5c11068eaeb53a4f5fa7c4136" ], "x-amz-create-session-mode": [ "ReadWrite" @@ -58,7 +58,7 @@ "AmazonS3" ], "content-length": [ - "1035" + "333" ], "date": [ "Fri, 13 Feb 2009 23:31:30 GMT" @@ -80,7 +80,7 @@ "action": { "Data": { "data": { - "Utf8": "\nAgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9PLqL36OqPDlwPN6koQgVrhr7R0QEuFbzsH4Bvz4btNZIQUUAEYD2XZOSJYC274XBMVU2024-02-01T03:57:15Z" + "Utf8": "\nTESTSESSIONTOKENTESTSECRETKEYASIARTESTID2024-01-29T18:53:01Z" }, "direction": "Response" } @@ -106,10 +106,10 @@ "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" ], "x-amz-s3session-token": [ - "AgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9P" + "TESTSESSIONTOKEN" ], "authorization": [ - "AWS4-HMAC-SHA256 Credential=NZIQUUAEYD2XZOSJYC274XBMVU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=63ec8429a44ae20698b21a810030589d2369a13c352d32970931de7b2cede687" + "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=d1765aa7ec005607ba94fdda08c6739228d5ee14eb8316e80264c35649661a19" ], "amz-sdk-request": [ "attempt=1; max=3" @@ -308,7 +308,7 @@ "20090213T233130Z" ], "authorization": [ - "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=4147d1c9c1c54efea9729be3816ec31e25496b7b1a6f116c479e8391444daa7c" + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=9e664e775f458662091e9df58f1f37f66cb74e21ac19fd9a618a9189285ca152" ], "x-amz-content-sha256": [ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -349,7 +349,7 @@ "0033eada6b00018d62cbc0e3050ad61d5828bce1" ], "content-length": [ - "1043" + "333" ], "date": [ "Fri, 13 Feb 2009 23:31:30 GMT" @@ -368,7 +368,7 @@ "action": { "Data": { "data": { - "Utf8": "\nAgAAANCuNFCuOMGL3GiOJQLuVjgOSnFrPq+7P4foPPYEAAAAAAAAAA6gXF7NAQAAAAAAAL6C3/74KCW+AXc/l3wx7EZdxgJtYloTtkuUXCdptWfMT+fTaAkgNGIg2f7Iw+reIMDGN+CfwZGXbnL+rbL6rDf7WFJ5EVb+nIAdeJepa8ZWAYxY1eBq2PLmXcAd/QX8dKfJijmcFk4eDi40J1TPoiL7uao6pxUkj8lK9P9MUTl+IYtvLMcEGGa5/k+Xx+Fclrj+HNA/NoEDfOub9yQ4ei/rfX7efkVm/YWcexLwSmiZ32KaoQRxa2C1DIHozypgQmA+M2K1KTDzRd9Ks2YSTlqsmKzaHnqvxrbqpQec2Z6Tb7t9z1WimidZw9hh10v57kSdnNWGfAWtaOcsopHs0QqVIxTc2XiO32pojj6NtPgoGb3jSmbbDg9Q3905CsPt4KFhaLM3oIHBbQzVNAqewWuBm0Z3olpmHOBF3CBQ7HB2QQ4g8lYouUOyQQqJTlF8ls12VCI8rprnlqcS2iqn/neWwcFbc4/cdRvu0AeOmktCnq8izc6eXiaIBso6Tv7KO4vVfCjVeRG7UkzCyTd3SsnEULENOxWTToo8BpUyxtO6ULaoI7GSQz64jHUMAguZ82zO3gX2TA2qF0ALG0JnKaA+pnri5SyjObMeV9XHzs6gDjtqiCOafa7fd/Najg/Tz2531bSDoGsQX4OGYD3AZC32QIUZXYZQULJ6J7EPNU2024-02-01T03:57:16Z" + "Utf8": "\nTESTSESSIONTOKENTESTSECRETKEYASIARTESTID2024-01-29T18:53:01Z" }, "direction": "Response" } @@ -391,7 +391,7 @@ "uri": "https://s3express-test-bucket-2--usw2-az3--x-s3.s3express-usw2-az3.us-west-2.amazonaws.com/?list-type=2", "headers": { "authorization": [ - "AWS4-HMAC-SHA256 Credential=YD3AZC32QIUZXYZQULJ6J7EPNU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=7e6b04135e77aa126142365335cbeb14f98c031d6abc3ce9c281d528034c9874" + "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=53848e132e259d7dbc06eb3063d5a32a243525f4fbf2cd015d45ff03f354d6f1" ], "x-amz-content-sha256": [ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -409,7 +409,7 @@ "attempt=1; max=3" ], "x-amz-s3session-token": [ - "AgAAANCuNFCuOMGL3GiOJQLuVjgOSnFrPq+7P4foPPYEAAAAAAAAAA6gXF7NAQAAAAAAAL6C3/74KCW+AXc/l3wx7EZdxgJtYloTtkuUXCdptWfMT+fTaAkgNGIg2f7Iw+reIMDGN+CfwZGXbnL+rbL6rDf7WFJ5EVb+nIAdeJepa8ZWAYxY1eBq2PLmXcAd/QX8dKfJijmcFk4eDi40J1TPoiL7uao6pxUkj8lK9P9MUTl+IYtvLMcEGGa5/k+Xx+Fclrj+HNA/NoEDfOub9yQ4ei/rfX7efkVm/YWcexLwSmiZ32KaoQRxa2C1DIHozypgQmA+M2K1KTDzRd9Ks2YSTlqsmKzaHnqvxrbqpQec2Z6Tb7t9z1WimidZw9hh10v57kSdnNWGfAWtaOcsopHs0QqVIxTc2XiO32pojj6NtPgoGb3jSmbbDg9Q3905CsPt4KFhaLM3oIHBbQzVNAqewWuBm0Z3olpmHOBF3CBQ7HB2QQ4g8lYouUOyQQqJTlF8ls12VCI8rprnlqcS2iqn/neWwcFbc4/cdRvu0AeOmktCnq8izc6eXiaIBso6Tv7KO4vVfCjVeRG7UkzCyTd3SsnEULENOxWTToo8BpUyxtO6ULaoI7GSQz64jHUMAguZ82zO3gX2TA2qF0ALG0JnKaA+pnri5SyjObMeV9XH" + "TESTSESSIONTOKEN" ] }, "method": "GET" @@ -501,13 +501,13 @@ "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" ], "x-amz-s3session-token": [ - "AgAAAC4zxhfvZg/nsnQ+E5m+e2qfGfH8yDMk/595OyIEAAAAAAAAAILVdpnHAQAAAAAAAASwftXYM0Sz7TWEtmiOj3JZDHuvfme1Vb8RNXfiSKsRZoHc6F86O9pg4lgaffF4slyOwcUDv2PYnd9TSU+u40nqMCym+Joog46Q3YOTJXk0K18j9dWcNQaDrN5mZpfPtIFTtFM/Gm2PpGvFCzssffMFWMwWt5gHzHtlMHocG7uX16JsttVRiAe14iDiHcFy+ka6RcX8vgXoFnM8tVhE2jby/zYQ0GdbnzVqJoZ/DjFXoEEQ/Rp16Fq0x5dHAPB0AIPH49fYP0rHLn3I8KcTxu8NUNmWjt5HNCXCWrKIFowAtGQNFsuyUDOCDxbMsoKYgYdIRDclqgoSKdCy6E0TUUr+L+KgytQ+Br20imMRo5rFvxa6sXgynILnOGOISBOzNreKJ0JdfVLWv1F9SnauEVfBYfnn8k7aH2n/52EnD8e3gJ0AacMX4oEadZQrVfNyqhcnctQ1a+wFo0Orw8YOM4TGB3nUFwLfS7sUBOv5BXbHeKzl6pvMcFV+caquOE0jdzNPkVVhmS8OCKbicIul8KHmMYElmPy5+riXya+5cv/KymqzxZ4FuMGkEeYXHjntF2LmWVOkF8pGBIpcrt9BlLLxyTxlye9P" + "TESTSESSIONTOKEN" ], "amz-sdk-request": [ "attempt=1; max=3" ], "authorization": [ - "AWS4-HMAC-SHA256 Credential=NZIQUUAEYD2XZOSJYC274XBMVU/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=63ec8429a44ae20698b21a810030589d2369a13c352d32970931de7b2cede687" + "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=d1765aa7ec005607ba94fdda08c6739228d5ee14eb8316e80264c35649661a19" ] }, "method": "GET" From 9d354526bebfe74a717b389f675573ced64e9306 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Mon, 26 Feb 2024 10:46:23 -0600 Subject: [PATCH 05/16] Allow S3 Express to use CRC32 for default request checksum (#3432) ## Motivation and Context Changes the default request checksum to CRC32 for S3 Express. ## Description This PR modifies the default request checksum to CRC32 for S3 Express. Note that if a customer specifies a custom checksum algorithm via such as [checksum_algorithm](https://docs.rs/aws-sdk-s3/latest/aws_sdk_s3/operation/put_object/struct.PutObjectInput.html#structfield.checksum_algorithm), then the PR does not affect anything and respects the customer-specified checksum algorithm. Otherwise, if an operation is annotated with a `requestChecksumRequired` trait and set to true, the default checksum becomes relevant. The default algorithm is currently `MD5` and S3 Express requires it to be `CRC32`. The code changes in this PR reflect that requirement. Implementation details include - `RequestChecksumInterceptor` is a single-sourced place to handle request checksum, but since it is a common place for all services we cannot add S3-Express-specific logic in there to configure the default checksum. Instead, we have introduced a slight more abstract concept of overriding the default checksum via `DefaultRequestChecksumOverride`. `RequestChecksumInterceptor` grabs it from a `ConfigBag` and uses it (if found) to configure the default checksum. - We deferred a point of adding a checksum to a request body to `RequestChecksumInterceptor::modify_before_signing` from `RequestChecksumInterceptor::modify_before_retry_loop`. This is because we won't know whether S3 Express checksum default actually kicks until an endpoint is fully resolved. `modify_before_retry_loop` occurs before `client::orchestrator::endpoints::orchestrate_endpoint` so we need to choose a different interceptor method that runs after it. - I abandoned having a separate S3-specific `RequestChecksumInterceptor` as it was more complicated than this approach. The wrinkle is that if we have an S3-specific `RequestChecksumInterceptor`, then it needs to either a) disable [the general RequestChecksumInterceptor](https://github.com/smithy-lang/smithy-rs/blob/f7be5f801504e7edf5cb6b731f3400c37b298abc/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs#L79-L100) entirely or b) only change the checksum default and delegate the rest to the general `RequestChecksumInterceptor`. In both cases the general `RequestChecksumInterceptor` must be disabled in runtime components, otherwise a request checksum related header will incorrectly be added twice. To disable the general `RequestChecksumInterceptor` using [disable_interceptor](https://github.com/smithy-lang/smithy-rs/blob/main/rust-runtime/aws-smithy-runtime-api/src/client/interceptors.rs#L839), we need to remove a generic parameter `AP` from `RequestChecksumInterceptor` since we cannot enumerate all possible `AP` when specifying it in `disable_interceptor`. ## Testing Added integration tests for S3 Express default checksum to `integration-tests/s3/tests/express.rs`. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../src/http_request_checksum.rs | 54 +++++- .../rustsdk/HttpRequestChecksumDecorator.kt | 2 +- .../customize/s3/S3ExpressDecorator.kt | 78 +++++++++ aws/sdk/integration-tests/s3/tests/express.rs | 165 +++++++++++++++--- 4 files changed, 267 insertions(+), 32 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 2ccfc5d28e..48fa609639 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -60,6 +60,45 @@ impl Storable for RequestChecksumInterceptorState { type Storer = StoreReplace; } +pub(crate) type CustomDefaultFn = Box< + dyn Fn(Option, &ConfigBag) -> Option + + Send + + Sync + + 'static, +>; + +pub(crate) struct DefaultRequestChecksumOverride { + custom_default: CustomDefaultFn, +} +impl fmt::Debug for DefaultRequestChecksumOverride { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DefaultRequestChecksumOverride").finish() + } +} +impl Storable for DefaultRequestChecksumOverride { + type Storer = StoreReplace; +} +impl DefaultRequestChecksumOverride { + pub(crate) fn new(custom_default: F) -> Self + where + F: Fn(Option, &ConfigBag) -> Option + + Send + + Sync + + 'static, + { + Self { + custom_default: Box::new(custom_default), + } + } + pub(crate) fn custom_default( + &self, + original: Option, + config_bag: &ConfigBag, + ) -> Option { + (self.custom_default)(original, config_bag) + } +} + pub(crate) struct RequestChecksumInterceptor { algorithm_provider: AP, } @@ -102,7 +141,7 @@ where /// Calculate a checksum and modify the request to include the checksum as a header /// (for in-memory request bodies) or a trailer (for streaming request bodies). /// Streaming bodies must be sized or this will return an error. - fn modify_before_retry_loop( + fn modify_before_signing( &self, context: &mut BeforeTransmitInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, @@ -112,7 +151,8 @@ where .load::() .expect("set in `read_before_serialization`"); - if let Some(checksum_algorithm) = state.checksum_algorithm { + let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg); + if let Some(checksum_algorithm) = checksum_algorithm { let request = context.request_mut(); add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } @@ -121,6 +161,16 @@ where } } +fn incorporate_custom_default( + checksum: Option, + cfg: &ConfigBag, +) -> Option { + match cfg.load::() { + Some(checksum_override) => checksum_override.custom_default(checksum, cfg), + None => checksum, + } +} + fn add_checksum_for_request_body( request: &mut HttpRequest, checksum_algorithm: ChecksumAlgorithm, diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 2500712e8c..d3d46916ab 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -26,7 +26,7 @@ import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull -private fun RuntimeConfig.awsInlineableHttpRequestChecksum() = +internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = RuntimeType.forInlineDependency( InlineAwsDependency.forRustFile( "http_request_checksum", visibility = Visibility.PUBCRATE, diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index cda1971795..4e7f83ceff 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -7,10 +7,13 @@ package software.amazon.smithy.rustsdk.customize.s3 import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.AuthSchemeOption import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization @@ -26,6 +29,7 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.pre import software.amazon.smithy.rustsdk.AwsCargoDependency import software.amazon.smithy.rustsdk.AwsRuntimeType import software.amazon.smithy.rustsdk.InlineAwsDependency +import software.amazon.smithy.rustsdk.awsInlineableHttpRequestChecksum class S3ExpressDecorator : ClientCodegenDecorator { override val name: String = "S3ExpressDecorator" @@ -60,6 +64,16 @@ class S3ExpressDecorator : ClientCodegenDecorator { codegenContext: ClientCodegenContext, baseCustomizations: List, ): List = baseCustomizations + listOf(S3ExpressIdentityProviderConfig(codegenContext)) + + override fun operationCustomizations( + codegenContext: ClientCodegenContext, + operation: OperationShape, + baseCustomizations: List, + ): List = + baseCustomizations + + S3ExpressRequestChecksumCustomization( + codegenContext, operation, + ) } private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientCodegenContext) : @@ -183,6 +197,69 @@ class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : Co } } +class S3ExpressRequestChecksumCustomization( + private val codegenContext: ClientCodegenContext, + private val operationShape: OperationShape, +) : OperationCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + + private val codegenScope = + arrayOf( + *preludeScope, + "ChecksumAlgorithm" to RuntimeType.smithyChecksums(runtimeConfig).resolve("ChecksumAlgorithm"), + "ConfigBag" to RuntimeType.configBag(runtimeConfig), + "DefaultRequestChecksumOverride" to + runtimeConfig.awsInlineableHttpRequestChecksum() + .resolve("DefaultRequestChecksumOverride"), + "Document" to RuntimeType.smithyTypes(runtimeConfig).resolve("Document"), + ) + + override fun section(section: OperationSection): Writable = + writable { + when (section) { + is OperationSection.AdditionalRuntimePluginConfig -> { + rustTemplate( + """ + ${section.newLayerName}.store_put(#{DefaultRequestChecksumOverride}::new( + |original: #{Option}<#{ChecksumAlgorithm}>, + cfg: &#{ConfigBag}| { + // S3 does not have the `ChecksumAlgorithm::Md5`, therefore customers cannot set it + // from outside. + if original != #{Some}(#{ChecksumAlgorithm}::Md5) { + return original; + } + + let endpoint = cfg + .load::() + .expect("endpoint added to config bag by endpoint orchestrator"); + + match endpoint.properties().get("backend") { + Some(#{Document}::String(backend)) if backend.as_str() == "S3Express" => { + #{customDefault:W} + } + _ => original + } + } + )); + """, + *codegenScope, + "customDefault" to + writable { + if (operationShape.id == ShapeId.from("com.amazonaws.s3#UploadPart")) { + rustTemplate("#{None}", *codegenScope) + } else { + rustTemplate("#{Some}(#{ChecksumAlgorithm}::Crc32)", *codegenScope) + } + }, + ) + } + + else -> { } + } + } +} + private fun s3ExpressModule(runtimeConfig: RuntimeConfig) = RuntimeType.forInlineDependency( InlineAwsDependency.forRustFile( @@ -202,6 +279,7 @@ private fun s3ExpressDependencies(runtimeConfig: RuntimeConfig) = CargoDependency.Lru, CargoDependency.Sha2, CargoDependency.smithyAsync(runtimeConfig), + CargoDependency.smithyChecksums(runtimeConfig), CargoDependency.smithyRuntimeApiClient(runtimeConfig), CargoDependency.smithyTypes(runtimeConfig), ) diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index b361397332..a82fd38fce 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -5,32 +5,36 @@ use std::time::{Duration, SystemTime}; -use aws_config::Region; +use aws_sdk_s3::config::Builder; use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::SdkBody; +use aws_sdk_s3::types::ChecksumAlgorithm; use aws_sdk_s3::{Client, Config}; use aws_smithy_runtime::client::http::test_util::dvr::ReplayingClient; use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs; use http::Uri; +async fn test_client(update_builder: F) -> Client +where + F: Fn(Builder) -> Builder, +{ + let sdk_config = aws_config::from_env() + .no_credentials() + .region("us-west-2") + .load() + .await; + let config = Config::from(&sdk_config).to_builder().with_test_defaults(); + aws_sdk_s3::Client::from_conf(update_builder(config).build()) +} + #[tokio::test] async fn list_objects_v2() { let _logs = capture_test_logs(); let http_client = ReplayingClient::from_file("tests/data/express/list-objects-v2.json").unwrap(); - let config = aws_config::from_env() - .http_client(http_client.clone()) - .no_credentials() - .region("us-west-2") - .load() - .await; - let config = Config::from(&config) - .to_builder() - .with_test_defaults() - .build(); - let client = aws_sdk_s3::Client::from_conf(config); + let client = test_client(|b| b.http_client(http_client.clone())).await; let result = client .list_objects_v2() @@ -50,17 +54,7 @@ async fn mixed_auths() { let _logs = capture_test_logs(); let http_client = ReplayingClient::from_file("tests/data/express/mixed-auths.json").unwrap(); - let config = aws_config::from_env() - .http_client(http_client.clone()) - .no_credentials() - .region("us-west-2") - .load() - .await; - let config = Config::from(&config) - .to_builder() - .with_test_defaults() - .build(); - let client = aws_sdk_s3::Client::from_conf(config); + let client = test_client(|b| b.http_client(http_client.clone())).await; // A call to an S3 Express bucket where we should see two request/response pairs, // one for the `create_session` API and the other for `list_objects_v2` in S3 Express bucket. @@ -137,12 +131,7 @@ async fn presigning() { create_session_response(), )]); - let config = aws_sdk_s3::Config::builder() - .http_client(http_client) - .region(Region::new("us-west-2")) - .with_test_defaults() - .build(); - let client = Client::from_conf(config); + let client = test_client(|b| b.http_client(http_client.clone())).await; let presigning_config = PresigningConfig::builder() .start_time(SystemTime::UNIX_EPOCH + Duration::from_secs(1234567891)) @@ -187,3 +176,121 @@ async fn presigning() { ); assert_eq!(presigned.headers().count(), 0); } + +fn operation_request_with_checksum( + query: &str, + kv: Option<(&str, &str)>, +) -> http::Request { + let mut b = http::Request::builder() + .uri(&format!("https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/{query}")) + .method("GET"); + if let Some((key, value)) = kv { + b = b.header(key, value); + } + b.body(SdkBody::empty()).unwrap() +} + +fn response_ok() -> http::Response { + http::Response::builder() + .status(200) + .body(SdkBody::empty()) + .unwrap() +} + +#[tokio::test] +async fn user_specified_checksum_should_be_respected() { + async fn runner(checksum: ChecksumAlgorithm, value: &str) { + let http_client = StaticReplayClient::new(vec![ + ReplayEvent::new(create_session_request(), create_session_response()), + ReplayEvent::new( + operation_request_with_checksum( + "test?x-id=PutObject", + Some(( + &format!("x-amz-checksum-{}", checksum.as_str().to_lowercase()), + &format!("{value}"), + )), + ), + response_ok(), + ), + ]); + let client = test_client(|b| b.http_client(http_client.clone())).await; + + let _ = client + .put_object() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .key("test") + .body(SdkBody::empty().into()) + .checksum_algorithm(checksum) + .send() + .await; + + http_client.assert_requests_match(&[""]); + } + + let checksum_value_pairs = &[ + (ChecksumAlgorithm::Crc32, "AAAAAA=="), + (ChecksumAlgorithm::Crc32C, "AAAAAA=="), + (ChecksumAlgorithm::Sha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="), + ( + ChecksumAlgorithm::Sha256, + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", + ), + ]; + for (checksum, value) in checksum_value_pairs { + runner(checksum.clone(), *value).await; + } +} + +#[tokio::test] +async fn default_checksum_should_be_crc32_for_operation_requiring_checksum() { + let http_client = StaticReplayClient::new(vec![ + ReplayEvent::new(create_session_request(), create_session_response()), + ReplayEvent::new( + operation_request_with_checksum( + "?delete&x-id=DeleteObjects", + Some(("x-amz-checksum-crc32", "AAAAAA==")), + ), + response_ok(), + ), + ]); + let client = test_client(|b| b.http_client(http_client.clone())).await; + + let _ = client + .delete_objects() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + + http_client.assert_requests_match(&[""]); +} + +#[tokio::test] +async fn default_checksum_should_be_none() { + let http_client = StaticReplayClient::new(vec![ + ReplayEvent::new(create_session_request(), create_session_response()), + ReplayEvent::new( + operation_request_with_checksum("test?x-id=PutObject", None), + response_ok(), + ), + ]); + let client = test_client(|b| b.http_client(http_client.clone())).await; + + let _ = client + .put_object() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .key("test") + .body(SdkBody::empty().into()) + .send() + .await; + + http_client.assert_requests_match(&[""]); + + let mut all_checksums = ChecksumAlgorithm::values() + .iter() + .map(|checksum| format!("amz-checksum-{}", checksum.to_lowercase())) + .chain(std::iter::once("content-md5".to_string())); + + assert!(!all_checksums.any(|checksum| http_client + .actual_requests() + .any(|req| req.headers().iter().any(|(key, _)| key == checksum)))); +} From 721fadd4748c89a06b5cae09312717a9b908ebdb Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Tue, 27 Feb 2024 15:42:09 -0600 Subject: [PATCH 06/16] Add support to disable S3 Express session auth (#3433) ## Motivation and Context Adds the ability to disable S3 Express session auth (causing it to use a regular sigv4 session auth instead). ## Description S3 Express One Zone is an opt out feature, and there are three ways to disable it (with the order of precedence as listed): - through the `disable_s3_express_session_auth` method on an S3 client - through an environment variable `AWS_S3_DISABLE_EXPRESS_SESSION_AUTH` - through a profile file with a key `s3_disable_express_session_auth` (won't be supported until https://github.com/awslabs/aws-sdk-rust/issues/1073 is addressed) If one of the places is set to true/false, then the subsequent places with lower precedence will not be considered. Something to be aware of when setting the disable option through the environment variable. The environment variable is only checked during a client construction, meaning that if a customer sets it after the client has been created the SDK will not take the environment variable value into account, i.e. the following snippet will NOT disable the S3 Express session auth: ``` let config = aws_config::load_from_env().await; let client = aws_sdk_s3::Client::new(&config); // Set the env variable to true after an S3 client has been created std::env::set_var("AWS_S3_DISABLE_EXPRESS_SESSION_AUTH", "true"); let _ = client .list_objects_v2() .bucket("s3express-test-bucket--usw2-az1--x-s3") .send() .await; ``` ## Testing Added unit tests for `S3ExpressRuntimePlugin` and integration tests for verifying disabling S3 Express session auth. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../aws-inlineable/src/s3_express.rs | 131 ++++++++++++++++++ .../rustsdk/AwsFluentClientDecorator.kt | 7 +- .../customize/s3/S3ExpressDecorator.kt | 31 ++++- aws/sdk/integration-tests/s3/tests/express.rs | 61 +++++++- .../client/FluentClientDecorator.kt | 4 + .../client/FluentClientGenerator.kt | 18 ++- 6 files changed, 246 insertions(+), 6 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 6608707d2e..0f78ca56f7 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -661,3 +661,134 @@ pub(crate) mod identity_provider { } } } + +/// Supporting code for S3 Express runtime plugin +pub(crate) mod runtime_plugin { + use aws_smithy_runtime_api::client::runtime_plugin::RuntimePlugin; + use aws_smithy_types::config_bag::{FrozenLayer, Layer}; + use aws_types::os_shim_internal::Env; + + mod env { + pub(super) const S3_DISABLE_EXPRESS_SESSION_AUTH: &str = + "AWS_S3_DISABLE_EXPRESS_SESSION_AUTH"; + } + + #[derive(Debug)] + pub(crate) struct S3ExpressRuntimePlugin { + config: FrozenLayer, + } + + impl S3ExpressRuntimePlugin { + pub(crate) fn new(config: FrozenLayer) -> Self { + Self::new_with(config, Env::real()) + } + + fn new_with(config: FrozenLayer, env: Env) -> Self { + let mut layer = Layer::new("S3ExpressRuntimePlugin"); + if config + .load::() + .is_none() + { + match env.get(env::S3_DISABLE_EXPRESS_SESSION_AUTH) { + Ok(value) + if value.eq_ignore_ascii_case("true") + || value.eq_ignore_ascii_case("false") => + { + let value = value + .to_lowercase() + .parse::() + .expect("just checked to be a bool-valued string"); + layer.store_or_unset(Some(crate::config::DisableS3ExpressSessionAuth( + value, + ))); + } + Ok(value) => { + tracing::warn!("environment variable `{}` ignored since it only accepts either `true` or `false` (case-insensitive), but got `{}`.", + env::S3_DISABLE_EXPRESS_SESSION_AUTH, + value) + } + _ => { + // TODO(aws-sdk-rust#1073): Transfer a value of + // `s3_disable_express_session_auth` from a profile file to `layer` + } + } + } + + Self { + config: layer.freeze(), + } + } + } + + impl RuntimePlugin for S3ExpressRuntimePlugin { + fn config(&self) -> Option { + Some(self.config.clone()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn disable_option_set_from_service_client_should_take_the_highest_precedence() { + // Disable option is set from service client. + let mut layer = Layer::new("test"); + layer.store_put(crate::config::DisableS3ExpressSessionAuth(true)); + + // An environment variable says the session auth is _not_ disabled, but it will be + // overruled by what is in `layer`. + let sut = S3ExpressRuntimePlugin::new_with( + layer.freeze(), + Env::from_slice(&[(super::env::S3_DISABLE_EXPRESS_SESSION_AUTH, "false")]), + ); + + // While this runtime plugin does not contain the config value, `ServiceRuntimePlugin` + // will eventually provide it when a config bag is fully set up in the orchestrator. + assert!(sut.config().is_some_and(|cfg| cfg + .load::() + .is_none())); + } + + #[test] + fn disable_option_set_from_env_should_take_the_second_highest_precedence() { + // No disable option is set from service client. + let layer = Layer::new("test"); + + // An environment variable says session auth is disabled + let sut = S3ExpressRuntimePlugin::new_with( + layer.freeze(), + Env::from_slice(&[(super::env::S3_DISABLE_EXPRESS_SESSION_AUTH, "true")]), + ); + + let cfg = sut.config().unwrap(); + assert!( + cfg.load::() + .unwrap() + .0 + ); + } + + #[should_panic] + #[test] + fn disable_option_set_from_profile_file_should_take_the_lowest_precedence() { + // TODO(aws-sdk-rust#1073): Implement a test that mimics only setting + // `s3_disable_express_session_auth` in a profile file + todo!() + } + + #[test] + fn disable_option_should_be_unspecified_if_unset() { + // No disable option is set from service client. + let layer = Layer::new("test"); + + // An environment variable says session auth is disabled + let sut = S3ExpressRuntimePlugin::new_with(layer.freeze(), Env::from_slice(&[])); + + let cfg = sut.config().unwrap(); + assert!(cfg + .load::() + .is_none()); + } + } +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt index 257542c509..d2437a05aa 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt @@ -5,6 +5,7 @@ package software.amazon.smithy.rustsdk +import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.ClientRustModule import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator @@ -27,7 +28,9 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RustCrate import software.amazon.smithy.rust.codegen.core.smithy.generators.LibRsCustomization import software.amazon.smithy.rust.codegen.core.smithy.generators.LibRsSection +import software.amazon.smithy.rust.codegen.core.util.letIf import software.amazon.smithy.rust.codegen.core.util.serviceNameOrDefault +import software.amazon.smithy.rustsdk.customize.s3.S3ExpressFluentClientCustomization private class Types(runtimeConfig: RuntimeConfig) { private val smithyTypes = RuntimeType.smithyTypes(runtimeConfig) @@ -55,7 +58,9 @@ class AwsFluentClientDecorator : ClientCodegenDecorator { listOf( AwsPresignedFluentBuilderMethod(codegenContext), AwsFluentClientDocs(codegenContext), - ), + ).letIf(codegenContext.serviceShape.id == ShapeId.from("com.amazonaws.s3#AmazonS3")) { + it + S3ExpressFluentClientCustomization(codegenContext) + }, ).render(rustCrate) rustCrate.withModule(ClientRustModule.client) { AwsFluentClientExtensions(codegenContext, types).render(this) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 4e7f83ceff..052d6afd23 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -16,6 +16,8 @@ import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCus import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.client.FluentClientCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.client.FluentClientSection import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency @@ -138,7 +140,7 @@ private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientC } } -class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomization() { +private class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomization() { private val runtimeConfig = codegenContext.runtimeConfig private val codegenScope = arrayOf( @@ -197,6 +199,33 @@ class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : Co } } +class S3ExpressFluentClientCustomization( + codegenContext: ClientCodegenContext, +) : FluentClientCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "S3ExpressRuntimePlugin" to s3ExpressModule(runtimeConfig).resolve("runtime_plugin::S3ExpressRuntimePlugin"), + ) + + override fun section(section: FluentClientSection): Writable = + writable { + when (section) { + is FluentClientSection.AdditionalBaseClientPlugins -> { + rustTemplate( + """ + ${section.plugins} = ${section.plugins}.with_client_plugin(#{S3ExpressRuntimePlugin}::new(${section.config}.config.clone())); + """, + *codegenScope, + ) + } + + else -> emptySection + } + } +} + class S3ExpressRequestChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index a82fd38fce..a53f01e233 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -5,13 +5,16 @@ use std::time::{Duration, SystemTime}; +use aws_config::Region; use aws_sdk_s3::config::Builder; use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::SdkBody; use aws_sdk_s3::types::ChecksumAlgorithm; use aws_sdk_s3::{Client, Config}; use aws_smithy_runtime::client::http::test_util::dvr::ReplayingClient; -use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; +use aws_smithy_runtime::client::http::test_util::{ + capture_request, ReplayEvent, StaticReplayClient, +}; use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs; use http::Uri; @@ -28,6 +31,7 @@ where aws_sdk_s3::Client::from_conf(update_builder(config).build()) } +// TODO(S3Express): Convert this test to the S3 express section in canary #[tokio::test] async fn list_objects_v2() { let _logs = capture_test_logs(); @@ -294,3 +298,58 @@ async fn default_checksum_should_be_none() { .actual_requests() .any(|req| req.headers().iter().any(|(key, _)| key == checksum)))); } + +#[tokio::test] +async fn disable_s3_express_session_auth_at_service_client_level() { + let (http_client, request) = capture_request(None); + let conf = Config::builder() + .http_client(http_client) + .region(Region::new("us-west-2")) + .with_test_defaults() + .disable_s3_express_session_auth(true) + .build(); + let client = Client::from_conf(conf); + + let _ = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + + let req = request.expect_request(); + assert!( + !req.headers() + .get("authorization") + .unwrap() + .contains("x-amz-create-session-mode"), + "x-amz-create-session-mode should not appear in headers when S3 Express session auth is disabled" + ); +} + +#[tokio::test] +async fn disable_s3_express_session_auth_at_operation_level() { + let (http_client, request) = capture_request(None); + let conf = Config::builder() + .http_client(http_client) + .region(Region::new("us-west-2")) + .with_test_defaults() + .build(); + let client = Client::from_conf(conf); + + let _ = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .customize() + .config_override(Config::builder().disable_s3_express_session_auth(true)) + .send() + .await; + + let req = request.expect_request(); + assert!( + !req.headers() + .get("authorization") + .unwrap() + .contains("x-amz-create-session-mode"), + "x-amz-create-session-mode should not appear in headers when S3 Express session auth is disabled" + ); +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientDecorator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientDecorator.kt index 76ed95ed1c..2a2c0c1353 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientDecorator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientDecorator.kt @@ -76,6 +76,10 @@ sealed class FluentClientSection(name: String) : Section(name) { /** Write custom code into the docs */ data class FluentClientDocs(val serviceShape: ServiceShape) : FluentClientSection("FluentClientDocs") + + /** Write custom code for adding additional client plugins to base_client_runtime_plugins */ + data class AdditionalBaseClientPlugins(val plugins: String, val config: String) : + FluentClientSection("AdditionalBaseClientPlugins") } abstract class FluentClientCustomization : NamedCustomization() diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientGenerator.kt index 059c61be78..b31f98d1ce 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/FluentClientGenerator.kt @@ -162,7 +162,7 @@ class FluentClientGenerator( """, *preludeScope, "Arc" to RuntimeType.Arc, - "base_client_runtime_plugins" to baseClientRuntimePluginsFn(codegenContext), + "base_client_runtime_plugins" to baseClientRuntimePluginsFn(codegenContext, customizations), "BoxError" to RuntimeType.boxError(runtimeConfig), "client_docs" to writable { @@ -467,7 +467,10 @@ class FluentClientGenerator( } } -private fun baseClientRuntimePluginsFn(codegenContext: ClientCodegenContext): RuntimeType = +private fun baseClientRuntimePluginsFn( + codegenContext: ClientCodegenContext, + customizations: List, +): RuntimeType = codegenContext.runtimeConfig.let { rc -> RuntimeType.forInlineFun("base_client_runtime_plugins", ClientRustModule.config) { val api = RuntimeType.smithyRuntimeApiClient(rc) @@ -501,9 +504,11 @@ private fun baseClientRuntimePluginsFn(codegenContext: ClientCodegenContext): Ru .with_runtime_components(config.runtime_components.clone()) ) // codegen config - .with_client_plugin(crate::config::ServiceRuntimePlugin::new(config)) + .with_client_plugin(crate::config::ServiceRuntimePlugin::new(config.clone())) .with_client_plugin(#{NoAuthRuntimePlugin}::new()); + #{additional_client_plugins:W}; + for plugin in configured_plugins { plugins = plugins.with_client_plugin(plugin); } @@ -511,6 +516,13 @@ private fun baseClientRuntimePluginsFn(codegenContext: ClientCodegenContext): Ru } """, *preludeScope, + "additional_client_plugins" to + writable { + writeCustomizations( + customizations, + FluentClientSection.AdditionalBaseClientPlugins("plugins", "config"), + ) + }, "DefaultPluginParams" to rt.resolve("client::defaults::DefaultPluginParams"), "default_plugins" to rt.resolve("client::defaults::default_plugins"), "NoAuthRuntimePlugin" to rt.resolve("client::auth::no_auth::NoAuthRuntimePlugin"), From 01a0838b39f29e42a1d7878726b22acaabd115d2 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Tue, 5 Mar 2024 13:58:33 -0600 Subject: [PATCH 07/16] Add test for overriding S3 Express credentials provider (#3459) ## Description Add a test overriding S3 Express credentials provider. When a customer passes a `CredentialsProvider` to `.express_credentials_provider()` that produces a credential with a session token, the request header should contain `x-amz-s3session-token` with that session token. ## Testing Added a new integration test `support_customer_overriding_express_credentials_provider`. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- aws/sdk/integration-tests/s3/tests/express.rs | 69 +++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index a53f01e233..c2bfe51c32 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -6,7 +6,7 @@ use std::time::{Duration, SystemTime}; use aws_config::Region; -use aws_sdk_s3::config::Builder; +use aws_sdk_s3::config::{Builder, Credentials}; use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::SdkBody; use aws_sdk_s3::types::ChecksumAlgorithm; @@ -22,11 +22,7 @@ async fn test_client(update_builder: F) -> Client where F: Fn(Builder) -> Builder, { - let sdk_config = aws_config::from_env() - .no_credentials() - .region("us-west-2") - .load() - .await; + let sdk_config = aws_config::from_env().region("us-west-2").load().await; let config = Config::from(&sdk_config).to_builder().with_test_defaults(); aws_sdk_s3::Client::from_conf(update_builder(config).build()) } @@ -353,3 +349,64 @@ async fn disable_s3_express_session_auth_at_operation_level() { "x-amz-create-session-mode should not appear in headers when S3 Express session auth is disabled" ); } + +#[tokio::test] +async fn support_customer_overriding_express_credentials_provider() { + let expected_session_token = "testsessiontoken"; + let client_overriding_express_credentials_provider = || async move { + let (http_client, rx) = capture_request(None); + let client = test_client(|b| { + let credentials = Credentials::new( + "testaccess", + "testsecret", + Some(expected_session_token.to_owned()), + None, + "test", + ); + b.http_client(http_client.clone()) + // Pass a credential with a session token so that + // `x-amz-s3session-token` should appear in the request header + // when s3 session auth is enabled. + .express_credentials_provider(credentials.clone()) + // Pass a credential with a session token so that + // `x-amz-security-token` should appear in the request header + // when s3 session auth is disabled. + .credentials_provider(credentials) + }) + .await; + (client, rx) + }; + + // Test `x-amz-s3session-token` should be present with `expected_session_token`. + let (client, rx) = client_overriding_express_credentials_provider().await; + let _ = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + + let req = rx.expect_request(); + let actual_session_token = req + .headers() + .get("x-amz-s3session-token") + .expect("x-amz-s3session-token should be present"); + assert_eq!(expected_session_token, actual_session_token); + assert!(req.headers().get("x-amz-security-token").is_none()); + + // With a regular S3 bucket, test `x-amz-security-token` should be present with `expected_session_token`, + // instead of `x-amz-s3session-token`. + let (client, rx) = client_overriding_express_credentials_provider().await; + let _ = client + .list_objects_v2() + .bucket("regular-test-bucket") + .send() + .await; + + let req = rx.expect_request(); + let actual_session_token = req + .headers() + .get("x-amz-security-token") + .expect("x-amz-security-token should be present"); + assert_eq!(expected_session_token, actual_session_token); + assert!(req.headers().get("x-amz-s3session-token").is_none()); +} From 8d056bdcae88867268f26adddc2ee7c3277e2bdb Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Tue, 5 Mar 2024 19:38:51 -0600 Subject: [PATCH 08/16] Drop `S3ExpressSigner` and override session token name via `RuntimePlugin` (#3457) ## Motivation and Context In response to https://github.com/smithy-lang/smithy-rs/pull/3455#issuecomment-1973771991, it's clear that we need to iterate more on the potential signing API. This PR takes a different approach where `S3ExpressSigner` is dropped and a session token name override, if any, is placed into and retrieved from `ConfigBag`, which is used within `SigV4Signer::sign_http_request`. ## Testing Existing tests in CI A semver failure in CI can be ignored; A public API SigV4Signer::sign_http_request has been removed that only existed in the branch. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../aws-inlineable/src/s3_express.rs | 98 +++++++++---------- aws/rust-runtime/aws-runtime/src/auth.rs | 48 ++++++++- .../aws-runtime/src/auth/sigv4.rs | 71 +++++--------- .../customize/s3/S3ExpressDecorator.kt | 17 ++-- 4 files changed, 124 insertions(+), 110 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 0f78ca56f7..6a4b39a641 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -6,17 +6,9 @@ /// Supporting code for S3 Express auth pub(crate) mod auth { use aws_runtime::auth::sigv4::SigV4Signer; - use aws_sigv4::http_request::{SignatureLocation, SigningSettings}; - use aws_smithy_runtime_api::box_error::BoxError; - use aws_smithy_runtime_api::client::auth::{ - AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, Sign, - }; - use aws_smithy_runtime_api::client::identity::{Identity, SharedIdentityResolver}; - use aws_smithy_runtime_api::client::orchestrator::HttpRequest; - use aws_smithy_runtime_api::client::runtime_components::{ - GetIdentityResolver, RuntimeComponents, - }; - use aws_smithy_types::config_bag::ConfigBag; + use aws_smithy_runtime_api::client::auth::{AuthScheme, AuthSchemeId, Sign}; + use aws_smithy_runtime_api::client::identity::SharedIdentityResolver; + use aws_smithy_runtime_api::client::runtime_components::GetIdentityResolver; /// Auth scheme ID for S3 Express. pub(crate) const SCHEME_ID: AuthSchemeId = AuthSchemeId::new("sigv4-s3express"); @@ -24,7 +16,7 @@ pub(crate) mod auth { /// S3 Express auth scheme. #[derive(Debug, Default)] pub(crate) struct S3ExpressAuthScheme { - signer: S3ExpressSigner, + signer: SigV4Signer, } impl S3ExpressAuthScheme { @@ -50,45 +42,6 @@ pub(crate) mod auth { &self.signer } } - - /// S3 Express signer. - #[derive(Debug, Default)] - pub(crate) struct S3ExpressSigner; - - impl Sign for S3ExpressSigner { - fn sign_http_request( - &self, - request: &mut HttpRequest, - identity: &Identity, - auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, - runtime_components: &RuntimeComponents, - config_bag: &ConfigBag, - ) -> Result<(), BoxError> { - let operation_config = - SigV4Signer::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; - let mut settings = SigV4Signer::signing_settings(&operation_config); - override_session_token_name(&mut settings)?; - - SigV4Signer.sign_http_request( - request, - identity, - settings, - &operation_config, - runtime_components, - config_bag, - ) - } - } - - fn override_session_token_name(settings: &mut SigningSettings) -> Result<(), BoxError> { - let session_token_name_override = match settings.signature_location { - SignatureLocation::Headers => Some("x-amz-s3session-token"), - SignatureLocation::QueryParams => Some("X-Amz-S3session-Token"), - _ => { return Err(BoxError::from("`SignatureLocation` adds a new variant, which needs to be handled in a separate match arm")) }, - }; - settings.session_token_name_override = session_token_name_override; - Ok(()) - } } /// Supporting code for S3 Express identity cache @@ -664,8 +617,10 @@ pub(crate) mod identity_provider { /// Supporting code for S3 Express runtime plugin pub(crate) mod runtime_plugin { - use aws_smithy_runtime_api::client::runtime_plugin::RuntimePlugin; - use aws_smithy_types::config_bag::{FrozenLayer, Layer}; + use aws_runtime::auth::SigV4SessionTokenNameOverride; + use aws_sigv4::http_request::{SignatureLocation, SigningSettings}; + use aws_smithy_runtime_api::{box_error::BoxError, client::runtime_plugin::RuntimePlugin}; + use aws_smithy_types::config_bag::{ConfigBag, FrozenLayer, Layer}; use aws_types::os_shim_internal::Env; mod env { @@ -714,6 +669,27 @@ pub(crate) mod runtime_plugin { } } + let session_token_name_override = SigV4SessionTokenNameOverride::new( + |settings: &SigningSettings, cfg: &ConfigBag| { + // Not configured for S3 express, use the original session token name override + if !crate::s3_express::utils::for_s3_express(cfg) { + return Ok(settings.session_token_name_override); + } + + let session_token_name_override = Some(match settings.signature_location { + SignatureLocation::Headers => "x-amz-s3session-token", + SignatureLocation::QueryParams => "X-Amz-S3session-Token", + _ => { + return Err(BoxError::from( + "`SignatureLocation` adds a new variant, which needs to be handled in a separate match arm", + )) + } + }); + Ok(session_token_name_override) + }, + ); + layer.store_or_unset(Some(session_token_name_override)); + Self { config: layer.freeze(), } @@ -792,3 +768,19 @@ pub(crate) mod runtime_plugin { } } } + +pub(crate) mod utils { + use aws_smithy_types::{config_bag::ConfigBag, Document}; + + pub(crate) fn for_s3_express(cfg: &ConfigBag) -> bool { + let endpoint = cfg + .load::() + .expect("endpoint added to config bag by endpoint orchestrator"); + + if let Some(Document::String(backend)) = endpoint.properties().get("backend") { + backend.as_str() == "S3Express" + } else { + false + } + } +} diff --git a/aws/rust-runtime/aws-runtime/src/auth.rs b/aws/rust-runtime/aws-runtime/src/auth.rs index 6b1ecef2bf..9a46d73ed3 100644 --- a/aws/rust-runtime/aws-runtime/src/auth.rs +++ b/aws/rust-runtime/aws-runtime/src/auth.rs @@ -11,7 +11,7 @@ use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::auth::AuthSchemeEndpointConfig; use aws_smithy_runtime_api::client::identity::Identity; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; -use aws_smithy_types::config_bag::{Storable, StoreReplace}; +use aws_smithy_types::config_bag::{ConfigBag, Storable, StoreReplace}; use aws_smithy_types::Document; use aws_types::region::{Region, SigningRegion, SigningRegionSet}; use aws_types::SigningName; @@ -75,6 +75,52 @@ impl Default for SigningOptions { } } +pub(crate) type SessionTokenNameOverrideFn = Box< + dyn Fn(&SigningSettings, &ConfigBag) -> Result, BoxError> + + Send + + Sync + + 'static, +>; + +/// Custom config that provides the alternative session token name for [`SigningSettings`] +pub struct SigV4SessionTokenNameOverride { + name_override: SessionTokenNameOverrideFn, +} + +impl SigV4SessionTokenNameOverride { + /// Creates a new `SigV4SessionTokenNameOverride` + pub fn new(name_override: F) -> Self + where + F: Fn(&SigningSettings, &ConfigBag) -> Result, BoxError> + + Send + + Sync + + 'static, + { + Self { + name_override: Box::new(name_override), + } + } + + /// Provides a session token name override + pub fn name_override( + &self, + settings: &SigningSettings, + config_bag: &ConfigBag, + ) -> Result, BoxError> { + (self.name_override)(settings, config_bag) + } +} + +impl fmt::Debug for SigV4SessionTokenNameOverride { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SessionTokenNameOverride").finish() + } +} + +impl Storable for SigV4SessionTokenNameOverride { + type Storer = StoreReplace; +} + /// SigV4 signing configuration for an operation /// /// Although these fields MAY be customized on a per request basis, they are generally static diff --git a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs index 4330c72dfa..c3d2a78984 100644 --- a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs +++ b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs @@ -6,7 +6,7 @@ use crate::auth; use crate::auth::{ extract_endpoint_auth_scheme_signing_name, extract_endpoint_auth_scheme_signing_region, - SigV4OperationSigningConfig, SigV4SigningError, + SigV4OperationSigningConfig, SigV4SessionTokenNameOverride, SigV4SigningError, }; use aws_credential_types::Credentials; use aws_sigv4::http_request::{ @@ -72,8 +72,7 @@ impl SigV4Signer { Self } - /// Creates a [`SigningSettings`] from the given `operation_config`. - pub fn signing_settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { + fn settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { super::settings(operation_config) } @@ -143,27 +142,38 @@ impl SigV4Signer { } } } +} - /// Signs the given `request`. - /// - /// This is a helper used by [`Sign::sign_http_request`] and will be useful if calling code - /// needs to pass a configured `settings`. - /// - /// TODO(S3Express): Make this method more user friendly, possibly returning a builder - /// instead of taking these input parameters. The builder will have a `sign` method that - /// does what this method body currently does. - pub fn sign_http_request( +impl Sign for SigV4Signer { + fn sign_http_request( &self, request: &mut HttpRequest, identity: &Identity, - settings: SigningSettings, - operation_config: &SigV4OperationSigningConfig, + auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, runtime_components: &RuntimeComponents, - #[allow(unused_variables)] config_bag: &ConfigBag, + config_bag: &ConfigBag, ) -> Result<(), BoxError> { + if identity.data::().is_none() { + return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); + }; + + let operation_config = + Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; let request_time = runtime_components.time_source().unwrap_or_default().now(); + + let settings = if let Some(session_token_name_override) = + config_bag.load::() + { + let mut settings = Self::settings(&operation_config); + let name_override = session_token_name_override.name_override(&settings, config_bag)?; + settings.session_token_name_override = name_override; + settings + } else { + Self::settings(&operation_config) + }; + let signing_params = - Self::signing_params(settings, identity, operation_config, request_time)?; + Self::signing_params(settings, identity, &operation_config, request_time)?; let (signing_instructions, _signature) = { // A body that is already in memory can be signed directly. A body that is not in memory @@ -219,35 +229,6 @@ impl SigV4Signer { } } -impl Sign for SigV4Signer { - fn sign_http_request( - &self, - request: &mut HttpRequest, - identity: &Identity, - auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, - runtime_components: &RuntimeComponents, - config_bag: &ConfigBag, - ) -> Result<(), BoxError> { - if identity.data::().is_none() { - return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); - }; - - let operation_config = - Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; - - let settings = Self::signing_settings(&operation_config); - - self.sign_http_request( - request, - identity, - settings, - &operation_config, - runtime_components, - config_bag, - ) - } -} - #[cfg(feature = "event-stream")] mod event_stream { use aws_sigv4::event_stream::{sign_empty_message, sign_message}; diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 052d6afd23..9a4f7298a6 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -227,11 +227,10 @@ class S3ExpressFluentClientCustomization( } class S3ExpressRequestChecksumCustomization( - private val codegenContext: ClientCodegenContext, + codegenContext: ClientCodegenContext, private val operationShape: OperationShape, ) : OperationCustomization() { private val runtimeConfig = codegenContext.runtimeConfig - private val inputShape = codegenContext.model.expectShape(operationShape.inputShape) private val codegenScope = arrayOf( @@ -242,6 +241,7 @@ class S3ExpressRequestChecksumCustomization( runtimeConfig.awsInlineableHttpRequestChecksum() .resolve("DefaultRequestChecksumOverride"), "Document" to RuntimeType.smithyTypes(runtimeConfig).resolve("Document"), + "for_s3_express" to s3ExpressModule(runtimeConfig).resolve("utils::for_s3_express"), ) override fun section(section: OperationSection): Writable = @@ -259,15 +259,10 @@ class S3ExpressRequestChecksumCustomization( return original; } - let endpoint = cfg - .load::() - .expect("endpoint added to config bag by endpoint orchestrator"); - - match endpoint.properties().get("backend") { - Some(#{Document}::String(backend)) if backend.as_str() == "S3Express" => { - #{customDefault:W} - } - _ => original + if #{for_s3_express}(cfg) { + #{customDefault:W} + } else { + original } } )); From 7f8c28b7038372927ec6196eff88384452f908dd Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 6 Mar 2024 12:06:03 -0600 Subject: [PATCH 09/16] Update CHANGELOG.next.toml --- CHANGELOG.next.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 29a298d7a1..8a5e733353 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -40,3 +40,9 @@ message = "Add support for Lambda's `InvokeWithResponseStreaming` and Bedrock Ag references = ["aws-sdk-rust#1075", "aws-sdk-rust#1080", "smithy-rs#3451"] meta = { "breaking" = false, "bug" = false, "tada" = true } author = "jdisanti" + +[[aws-sdk-rust]] +message = "Add support for S3 Express One Zone. See [the user guide](https://github.com/awslabs/aws-sdk-rust/discussions/1091) for more details." +references = ["aws-sdk-rust#992", "smithy-rs#3465"] +meta = { "breaking" = false, "bug" = false, "tada" = true } +author = "ysaito1001" From a4e42efb605cb57c7e0e274bcc7277631f43b983 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 6 Mar 2024 12:17:37 -0600 Subject: [PATCH 10/16] Fix incorrect merge --- aws/rust-runtime/aws-runtime/src/auth/sigv4.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs index c3d2a78984..6d81b86d45 100644 --- a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs +++ b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs @@ -117,11 +117,10 @@ impl SigV4Signer { .expect("all required fields set")) } - /// Extracts a [`SigV4OperationSigningConfig`]. - pub fn extract_operation_config<'a>( + fn extract_operation_config<'a>( auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'a>, config_bag: &'a ConfigBag, - ) -> Result, BoxError> { + ) -> Result, SigV4SigningError> { let operation_config = config_bag .load::() .ok_or(SigV4SigningError::MissingOperationSigningConfig)?; From 75ed6fd5d3b738b1967e935c4d9b10396814a60c Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Thu, 7 Mar 2024 21:54:58 -0600 Subject: [PATCH 11/16] Add canary for S3 Express (disabled by default until it's been released) (#3462) ## Motivation and Context Adds canary for S3 Express ## Description This new canary tests the following APIs - `ListDirectoryBuckets` (a new API for S3 Express) - `ListObjectsV2` (converted from a connection recording test in `integration-tests`) - `Put/Get/DeleteObject` (the same set of APIs tested for regular S3 bucket and for MRAP in s3 canary) - `CreateMultipartUpload/UploadPart/CompleteMultipartUpload` (operations whose default checksum is `None` for S3 Express) - `DeleteObjects` (an operation that requires request checksum and the default checksum is CRC32) ## Testing - verified s3 express canary passed in our release pipeline - verified setting an environment variable `ENABLE_S3_EXPRESS_CANARY` correctly skipped the s3 express canary in our release pipeline ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- aws/rust-runtime/aws-inlineable/Cargo.toml | 4 + .../aws-inlineable/src/s3_express.rs | 4 +- .../rustsdk/customize/s3/S3Decorator.kt | 2 + .../customize/s3/S3ExpressDecorator.kt | 4 + .../tests/data/express/list-objects-v2.json | 193 ------ aws/sdk/integration-tests/s3/tests/express.rs | 22 - .../codegen/core/rustlang/CargoDependency.kt | 3 + tools/ci-cdk/canary-lambda/src/canary.rs | 7 + .../canary-lambda/src/latest/s3_canary.rs | 251 +++++++- tools/ci-cdk/canary-runner/src/run.rs | 32 +- tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts | 27 + tools/ci-cdk/package-lock.json | 607 ++++++++++-------- 12 files changed, 642 insertions(+), 514 deletions(-) delete mode 100644 aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json diff --git a/aws/rust-runtime/aws-inlineable/Cargo.toml b/aws/rust-runtime/aws-inlineable/Cargo.toml index b13dc268c5..66e8f7ff0b 100644 --- a/aws/rust-runtime/aws-inlineable/Cargo.toml +++ b/aws/rust-runtime/aws-inlineable/Cargo.toml @@ -12,6 +12,10 @@ publish = false repository = "https://github.com/smithy-lang/smithy-rs" [dependencies] +# Used by lru, and this forces it to be a later version that avoids +# https://github.com/tkaitchuck/aHash/issues/200 +# when built with `cargo update -Z minimal-versions` +ahash = "0.8.11" aws-credential-types = { path = "../aws-credential-types" } aws-runtime = { path = "../aws-runtime", features = ["http-02x"] } aws-sigv4 = { path = "../aws-sigv4" } diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 6a4b39a641..e96fd0c7dc 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -542,7 +542,9 @@ pub(crate) mod identity_provider { runtime_components: &'a RuntimeComponents, config_bag: &'a ConfigBag, ) -> Result { - let mut config_builder = crate::config::Builder::from_config_bag(config_bag); + // TODO(Post S3Express release): Thread through `BehaviorVersion` from the outer S3 client + let mut config_builder = crate::config::Builder::from_config_bag(config_bag) + .behavior_version(crate::config::BehaviorVersion::latest()); // inherits all runtime components from a current S3 operation but clears out // out interceptors configured for that operation diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt index 928a80f50c..8af54cfe6e 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt @@ -56,6 +56,8 @@ class S3Decorator : ClientCodegenDecorator { ShapeId.from("com.amazonaws.s3#CreateSessionOutput"), // API returns GetObjectAttributes_Response_ instead of Output ShapeId.from("com.amazonaws.s3#GetObjectAttributesOutput"), + // API returns ListAllMyDirectoryBucketsResult instead of ListDirectoryBucketsOutput + ShapeId.from("com.amazonaws.s3#ListDirectoryBucketsOutput"), ) override fun protocols( diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 9a4f7298a6..361b58cbf5 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -294,6 +294,10 @@ private fun s3ExpressModule(runtimeConfig: RuntimeConfig) = private fun s3ExpressDependencies(runtimeConfig: RuntimeConfig) = arrayOf( + // Used by lru, and this forces it to be a later version that avoids + // https://github.com/tkaitchuck/aHash/issues/200 + // when built with `cargo update -Z minimal-versions` + CargoDependency.AHash, AwsCargoDependency.awsCredentialTypes(runtimeConfig), AwsCargoDependency.awsRuntime(runtimeConfig), AwsCargoDependency.awsSigv4(runtimeConfig), diff --git a/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json b/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json deleted file mode 100644 index 9bbc52fe05..0000000000 --- a/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "events": [ - { - "connection_id": 0, - "action": { - "Request": { - "request": { - "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session", - "headers": { - "amz-sdk-request": [ - "attempt=1; max=1" - ], - "x-amz-user-agent": [ - "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" - ], - "x-amz-create-session-mode": [ - "ReadWrite" - ], - "x-amz-date": [ - "20090213T233130Z" - ], - "authorization": [ - "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=1f63e649e5433837f97c037489613e036b98bff8cdf8d7bcf24b770d041ac0b9" - ], - "user-agent": [ - "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" - ] - }, - "method": "GET" - } - } - } - }, - { - "connection_id": 0, - "action": { - "Eof": { - "ok": true, - "direction": "Request" - } - } - }, - { - "connection_id": 0, - "action": { - "Response": { - "response": { - "Ok": { - "status": 200, - "headers": { - "x-amz-id-2": [ - "TQk2NSay" - ], - "x-amz-request-id": [ - "0033eada6b00018d568cbe9f0509499a7de17df8" - ], - "date": [ - "Mon, 29 Jan 2024 18:48:00 GMT" - ], - "content-type": [ - "application/xml" - ], - "content-length": [ - "333" - ], - "server": [ - "AmazonS3" - ] - } - } - } - } - } - }, - { - "connection_id": 0, - "action": { - "Data": { - "data": { - "Utf8": "\nTESTSESSIONTOKENTESTSECRETKEYASIARTESTID2024-01-29T18:53:01Z" - }, - "direction": "Response" - } - } - }, - { - "connection_id": 0, - "action": { - "Eof": { - "ok": true, - "direction": "Response" - } - } - }, - { - "connection_id": 1, - "action": { - "Request": { - "request": { - "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2", - "headers": { - "authorization": [ - "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=d1765aa7ec005607ba94fdda08c6739228d5ee14eb8316e80264c35649661a19" - ], - "x-amz-date": [ - "20090213T233130Z" - ], - "amz-sdk-request": [ - "attempt=1; max=3" - ], - "x-amz-user-agent": [ - "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" - ], - "x-amz-s3session-token": [ - "TESTSESSIONTOKEN" - ], - "user-agent": [ - "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" - ] - }, - "method": "GET" - } - } - } - }, - { - "connection_id": 1, - "action": { - "Eof": { - "ok": true, - "direction": "Request" - } - } - }, - { - "connection_id": 1, - "action": { - "Response": { - "response": { - "Ok": { - "status": 200, - "headers": { - "content-type": [ - "application/xml" - ], - "date": [ - "Mon, 29 Jan 2024 18:48:00 GMT" - ], - "x-amz-request-id": [ - "0033eada6b00018d568cbf350509f775295f94b5" - ], - "x-amz-id-2": [ - "IltTpRJVF1U" - ], - "server": [ - "AmazonS3" - ], - "content-length": [ - "520" - ], - "x-amz-bucket-region": [ - "us-west-2" - ] - } - } - } - } - } - }, - { - "connection_id": 1, - "action": { - "Data": { - "data": { - "Utf8": "\ns3express-test-bucket--usw2-az1--x-s311000falseCRC32"b357dc928b454965a8dd11716a37dab8"hello-world.txt2024-01-29T18:32:24.000Z14EXPRESS_ONEZONE" - }, - "direction": "Response" - } - } - }, - { - "connection_id": 1, - "action": { - "Eof": { - "ok": true, - "direction": "Response" - } - } - } - ], - "docs": "traffic recording of executing list-objects-v2 against an S3 Express One Zone bucket", - "version": "V0" -} diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index c2bfe51c32..85bc961b6f 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -27,28 +27,6 @@ where aws_sdk_s3::Client::from_conf(update_builder(config).build()) } -// TODO(S3Express): Convert this test to the S3 express section in canary -#[tokio::test] -async fn list_objects_v2() { - let _logs = capture_test_logs(); - - let http_client = - ReplayingClient::from_file("tests/data/express/list-objects-v2.json").unwrap(); - let client = test_client(|b| b.http_client(http_client.clone())).await; - - let result = client - .list_objects_v2() - .bucket("s3express-test-bucket--usw2-az1--x-s3") - .send() - .await; - dbg!(result).expect("success"); - - http_client - .validate_body_and_headers(Some(&["x-amz-s3session-token"]), "application/xml") - .await - .unwrap(); -} - #[tokio::test] async fn mixed_auths() { let _logs = capture_test_logs(); diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt index ec06ecdfa4..07a6a4c02a 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt @@ -242,6 +242,9 @@ data class CargoDependency( } companion object { + // Forces AHash to be a later version that avoids + // https://github.com/tkaitchuck/aHash/issues/200 + val AHash: CargoDependency = CargoDependency("ahash", CratesIo("0.8.11")) val OnceCell: CargoDependency = CargoDependency("once_cell", CratesIo("1.16")) val Url: CargoDependency = CargoDependency("url", CratesIo("2.3.1")) val Bytes: CargoDependency = CargoDependency("bytes", CratesIo("1.0.0")) diff --git a/tools/ci-cdk/canary-lambda/src/canary.rs b/tools/ci-cdk/canary-lambda/src/canary.rs index 6b60811d4e..b23d92ec76 100644 --- a/tools/ci-cdk/canary-lambda/src/canary.rs +++ b/tools/ci-cdk/canary-lambda/src/canary.rs @@ -50,9 +50,11 @@ pub fn get_canaries_to_run( .collect() } +#[derive(Clone)] pub struct CanaryEnv { pub(crate) s3_bucket_name: String, pub(crate) s3_mrap_bucket_arn: String, + pub(crate) s3_express_bucket_name: String, pub(crate) expected_transcribe_result: String, #[allow(dead_code)] pub(crate) page_size: usize, @@ -63,6 +65,7 @@ impl fmt::Debug for CanaryEnv { f.debug_struct("CanaryEnv") .field("s3_bucket_name", &"*** redacted ***") .field("s3_mrap_bucket_arn", &"*** redacted ***") + .field("s3_express_bucket_name", &"*** redacted ***") .field( "expected_transcribe_result", &self.expected_transcribe_result, @@ -79,6 +82,9 @@ impl CanaryEnv { // S3 MRAP bucket name to test against let s3_mrap_bucket_arn = env::var("CANARY_S3_MRAP_BUCKET_ARN").expect("CANARY_S3_MRAP_BUCKET_ARN must be set"); + // S3 Express bucket name to test against + let s3_express_bucket_name = env::var("CANARY_S3_EXPRESS_BUCKET_NAME") + .expect("CANARY_S3_EXPRESS_BUCKET_NAME must be set"); // Expected transcription from Amazon Transcribe from the embedded audio file. // This is an environment variable so that the code doesn't need to be changed if @@ -97,6 +103,7 @@ impl CanaryEnv { Self { s3_bucket_name, s3_mrap_bucket_arn, + s3_express_bucket_name, expected_transcribe_result, page_size, } diff --git a/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs index ac5d92d82d..4b49bd4fb2 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs @@ -5,9 +5,10 @@ use crate::canary::CanaryError; use crate::{mk_canary, CanaryEnv}; -use anyhow::Context; +use anyhow::{Context, Error}; use aws_config::SdkConfig; use aws_sdk_s3 as s3; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Delete, ObjectIdentifier}; use s3::config::Region; use s3::presigning::PresigningConfig; use s3::primitives::ByteStream; @@ -16,17 +17,19 @@ use uuid::Uuid; const METADATA_TEST_VALUE: &str = "some value"; -mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| s3_canary( - s3::Client::new(sdk_config), - env.s3_bucket_name.clone(), - env.s3_mrap_bucket_arn.clone() -)); +mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| { + let sdk_config = sdk_config.clone(); + let env = env.clone(); + async move { + let client = s3::Client::new(&sdk_config); + s3_canary(client.clone(), env.s3_bucket_name.clone()).await?; + s3_mrap_canary(client.clone(), env.s3_mrap_bucket_arn.clone()).await?; + s3_express_canary(client, env.s3_express_bucket_name.clone()).await + } +}); -pub async fn s3_canary( - client: s3::Client, - s3_bucket_name: String, - s3_mrap_bucket_arn: String, -) -> anyhow::Result<()> { +/// Runs canary exercising S3 APIs against a regular bucket +pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Result<()> { let test_key = Uuid::new_v4().as_u128().to_string(); // Look for the test object and expect that it doesn't exist @@ -126,8 +129,12 @@ pub async fn s3_canary( .await .context("s3::DeleteObject")?; - // Return early if the result is an error - result?; + result +} + +/// Runs canary exercising S3 APIs against an MRAP bucket +pub async fn s3_mrap_canary(client: s3::Client, s3_mrap_bucket_arn: String) -> anyhow::Result<()> { + let test_key = Uuid::new_v4().as_u128().to_string(); // We deliberately use a region that doesn't exist here so that we can // ensure these requests are SigV4a requests. Because the current endpoint @@ -188,16 +195,210 @@ pub async fn s3_canary( .config_override(config_override) .send() .await - .context("s3::DeleteObject")?; + .context("s3::DeleteObject[MRAP]")?; result } +/// Runs canary exercising S3 APIs against an Express One Zone bucket +pub async fn s3_express_canary( + client: s3::Client, + s3_express_bucket_name: String, +) -> anyhow::Result<()> { + // TODO(Post S3Express release): Delete this once S3 Express has been released and its canary is on by default + match std::env::var("ENABLE_S3_EXPRESS_CANARY") { + Ok(value) if value.eq_ignore_ascii_case("true") => { + println!( + "`ENABLE_S3_EXPRESS_CANARY` is set to `true`, proceeding with s3 express canary" + ); + } + _ => { + println!("`ENABLE_S3_EXPRESS_CANARY` is not set to `true`, skipping s3 express canary"); + return Ok(()); + } + } + + let test_key = Uuid::new_v4().as_u128().to_string(); + + // Test a directory bucket exists + let directory_buckets = client + .list_directory_buckets() + .send() + .await + .context("s3::ListDirectoryBuckets[Express]")?; + assert!(directory_buckets + .buckets + .map(|buckets| buckets + .iter() + .any(|b| b.name() == Some(&s3_express_bucket_name))) + .expect("true")); + + // Check test object does not exist in the directory bucket + let list_objects_v2_output = client + .list_objects_v2() + .bucket(&s3_express_bucket_name) + .send() + .await + .context("s3::ListObjectsV2[EXPRESS]")?; + match list_objects_v2_output.contents { + Some(contents) => { + // should the directory bucket contains some leftover object, + // it better not be the test object + assert!(!contents.iter().any(|c| c.key() == Some(&test_key))); + } + _ => { /* No objects in the directory bucket, good to go */ } + } + + // Put the test object + client + .put_object() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .body(ByteStream::from_static(b"test")) + .metadata("something", METADATA_TEST_VALUE) + .send() + .await + .context("s3::PutObject[EXPRESS]")?; + + // Get the test object and verify it looks correct + let output = client + .get_object() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .send() + .await + .context("s3::GetObject[EXPRESS]")?; + + // repeat the test with a presigned url + let uri = client + .get_object() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .presigned(PresigningConfig::expires_in(Duration::from_secs(120)).unwrap()) + .await + .unwrap(); + let response = reqwest::get(uri.uri().to_string()) + .await + .context("s3::presigned")? + .text() + .await?; + if response != "test" { + return Err(CanaryError(format!("presigned URL returned bad data: {:?}", response)).into()); + } + + let metadata_value = output + .metadata() + .and_then(|m| m.get("something")) + .map(String::as_str); + let result: anyhow::Result<()> = match metadata_value { + Some(value) => { + if value == METADATA_TEST_VALUE { + let payload = output + .body + .collect() + .await + .context("download s3::GetObject[EXPRESS] body")? + .into_bytes(); + if std::str::from_utf8(payload.as_ref()).context("s3 payload")? == "test" { + Ok(()) + } else { + Err(CanaryError("S3 object body didn't match what was put there".into()).into()) + } + } else { + Err(CanaryError(format!( + "S3 metadata was incorrect. Expected `{}` but got `{}`.", + METADATA_TEST_VALUE, value + )) + .into()) + } + } + None => Err(CanaryError("S3 metadata was missing".into()).into()), + }; + + result?; + + // Delete the test object + client + .delete_object() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .send() + .await + .context("s3::DeleteObject[EXPRESS]")?; + + // Another key for MultipartUpload (verifying default checksum is None) + let test_key = Uuid::new_v4().as_u128().to_string(); + + // Create multipart upload + let create_mpu_output = client + .create_multipart_upload() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .send() + .await + .unwrap(); + let upload_id = create_mpu_output + .upload_id() + .context("s3::CreateMultipartUpload[EXPRESS]")?; + + // Upload part + let upload_part_output = client + .upload_part() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .part_number(1) + .body(ByteStream::from_static(b"test")) + .upload_id(upload_id) + .send() + .await + .context("s3::UploadPart[EXPRESS]")?; + + // Complete multipart upload + client + .complete_multipart_upload() + .bucket(&s3_express_bucket_name) + .key(&test_key) + .upload_id(upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .set_parts(Some(vec![CompletedPart::builder() + .e_tag(upload_part_output.e_tag.unwrap_or_default()) + .part_number(1) + .build()])) + .build(), + ) + .send() + .await + .context("s3::CompleteMultipartUpload[EXPRESS]")?; + + // Delete test objects using the DeleteObjects operation whose default checksum should be CRC32 + client + .delete_objects() + .bucket(&s3_express_bucket_name) + .delete( + Delete::builder() + .objects( + ObjectIdentifier::builder() + .key(&test_key) + .build() + .context("failed to build `ObjectIdentifier`")?, + ) + .build() + .context("failed to build `Delete`")?, + ) + .send() + .await + .context("s3::DeleteObjects[EXPRESS]")?; + + Ok::<(), Error>(()) +} + // This test runs against an actual AWS account. Comment out the `ignore` to run it. // Be sure the following environment variables are set: // // - `TEST_S3_BUCKET`: The S3 bucket to use // - `TEST_S3_MRAP_BUCKET_ARN`: The MRAP bucket ARN to use +// - `TEST_S3_EXPRESS_BUCKET`: The S3 express bucket to use // // Also, make sure the correct region (likely `us-west-2`) by the credentials or explictly. #[ignore] @@ -206,11 +407,23 @@ pub async fn s3_canary( async fn test_s3_canary() { let config = aws_config::load_from_env().await; let client = s3::Client::new(&config); - s3_canary( - client, + + let mut futures = Vec::new(); + + futures.push(tokio::spawn(s3_canary( + client.clone(), std::env::var("TEST_S3_BUCKET").expect("TEST_S3_BUCKET must be set"), + ))); + futures.push(tokio::spawn(s3_mrap_canary( + client.clone(), std::env::var("TEST_S3_MRAP_BUCKET_ARN").expect("TEST_S3_MRAP_BUCKET_ARN must be set"), - ) - .await - .expect("success"); + ))); + futures.push(tokio::spawn(s3_express_canary( + client, + std::env::var("TEST_S3_EXPRESS_BUCKET").expect("TEST_S3_EXPRESS_BUCKET must be set"), + ))); + + for fut in futures { + fut.await.expect("joined").expect("success"); + } } diff --git a/tools/ci-cdk/canary-runner/src/run.rs b/tools/ci-cdk/canary-runner/src/run.rs index 6d6d4e9f63..b4394c6348 100644 --- a/tools/ci-cdk/canary-runner/src/run.rs +++ b/tools/ci-cdk/canary-runner/src/run.rs @@ -102,10 +102,14 @@ pub struct RunArgs { #[clap(long, required_unless_present = "cdk-output")] lambda_test_s3_bucket_name: Option, - /// The name of the S3 bucket for the canary Lambda to interact with + /// The ARN of the S3 MRAP bucket for the canary Lambda to interact with #[clap(long, required_unless_present = "cdk-output")] lambda_test_s3_mrap_bucket_arn: Option, + /// The name of the S3 Express One Zone bucket for the canary Lambda to interact with + #[clap(long, required_unless_present = "cdk-output")] + lambda_test_s3_express_bucket_name: Option, + /// The ARN of the role that the Lambda will execute as #[clap(long, required_unless_present = "cdk-output")] lambda_execution_role_arn: Option, @@ -121,6 +125,7 @@ struct Options { lambda_code_s3_bucket_name: String, lambda_test_s3_bucket_name: String, lambda_test_s3_mrap_bucket_arn: String, + lambda_test_s3_express_bucket_name: String, lambda_execution_role_arn: String, } @@ -135,6 +140,8 @@ impl Options { lambda_test_s3_bucket_name: String, #[serde(rename = "canarytestmrapbucketarn")] lambda_test_s3_mrap_bucket_arn: String, + #[serde(rename = "canarytestexpressbucketname")] + lambda_test_s3_express_bucket_name: String, #[serde(rename = "lambdaexecutionrolearn")] lambda_execution_role_arn: String, } @@ -157,6 +164,7 @@ impl Options { lambda_code_s3_bucket_name: value.inner.lambda_code_s3_bucket_name, lambda_test_s3_bucket_name: value.inner.lambda_test_s3_bucket_name, lambda_test_s3_mrap_bucket_arn: value.inner.lambda_test_s3_mrap_bucket_arn, + lambda_test_s3_express_bucket_name: value.inner.lambda_test_s3_express_bucket_name, lambda_execution_role_arn: value.inner.lambda_execution_role_arn, }) } else { @@ -171,6 +179,9 @@ impl Options { lambda_test_s3_mrap_bucket_arn: run_opt .lambda_test_s3_mrap_bucket_arn .expect("required"), + lambda_test_s3_express_bucket_name: run_opt + .lambda_test_s3_express_bucket_name + .expect("required"), lambda_execution_role_arn: run_opt.lambda_execution_role_arn.expect("required"), }) } @@ -277,6 +288,7 @@ async fn run_canary(options: &Options, config: &aws_config::SdkConfig) -> Result &options.lambda_code_s3_bucket_name, &options.lambda_test_s3_bucket_name, &options.lambda_test_s3_mrap_bucket_arn, + &options.lambda_test_s3_express_bucket_name, ) .await .context(here!())?; @@ -354,15 +366,17 @@ async fn create_lambda_fn( code_s3_bucket: &str, test_s3_bucket: &str, test_s3_mrap_bucket_arn: &str, + test_s3_express_bucket: &str, ) -> Result<()> { use lambda::types::*; - let env_builder = match expected_speech_text_by_transcribe { + let mut env_builder = match expected_speech_text_by_transcribe { Some(expected_speech_text_by_transcribe) => Environment::builder() .variables("RUST_BACKTRACE", "1") .variables("RUST_LOG", "info") .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket) .variables("CANARY_S3_MRAP_BUCKET_ARN", test_s3_mrap_bucket_arn) + .variables("CANARY_S3_EXPRESS_BUCKET_NAME", test_s3_express_bucket) .variables( "CANARY_EXPECTED_TRANSCRIBE_RESULT", expected_speech_text_by_transcribe, @@ -371,9 +385,15 @@ async fn create_lambda_fn( .variables("RUST_BACKTRACE", "1") .variables("RUST_LOG", "info") .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket) - .variables("CANARY_S3_MRAP_BUCKET_ARN", test_s3_mrap_bucket_arn), + .variables("CANARY_S3_MRAP_BUCKET_ARN", test_s3_mrap_bucket_arn) + .variables("CANARY_S3_EXPRESS_BUCKET_NAME", test_s3_express_bucket), }; + // TODO(Post S3Express release): Delete this once S3 Express has been released and its canary is on by default + if let Ok(value) = env::var("ENABLE_S3_EXPRESS_CANARY") { + env_builder = env_builder.variables("ENABLE_S3_EXPRESS_CANARY", value); + } + lambda_client .create_function() .function_name(bundle_name) @@ -504,7 +524,8 @@ mod tests { lambda_code_s3_bucket_name: None, lambda_test_s3_bucket_name: None, lambda_execution_role_arn: None, - lambda_test_s3_mrap_bucket_arn: None + lambda_test_s3_mrap_bucket_arn: None, + lambda_test_s3_express_bucket_name: None }, RunArgs::try_parse_from([ "run", @@ -535,6 +556,8 @@ mod tests { "arn:aws:lambda::role/exe-role", "--lambda-test-s3-mrap-bucket-arn", "arn:aws:s3::000000000000:accesspoint/example.mrap", + "--lambda-test-s3-express-bucket-name", + "test--usw2-az1--x-s3", ]) .unwrap(); assert_eq!( @@ -549,6 +572,7 @@ mod tests { lambda_execution_role_arn: "arn:aws:lambda::role/exe-role".to_owned(), lambda_test_s3_mrap_bucket_arn: "arn:aws:s3::000000000000:accesspoint/example.mrap" .to_owned(), + lambda_test_s3_express_bucket_name: "test--usw2-az1--x-s3".to_owned(), }, Options::load_from(run_args).unwrap(), ); diff --git a/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts b/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts index 42e7e33fdb..dd64ab0172 100644 --- a/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts +++ b/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts @@ -16,6 +16,10 @@ import { BucketEncryption, CfnMultiRegionAccessPoint, } from "aws-cdk-lib/aws-s3"; +import { aws_s3express as s3express } from 'aws-cdk-lib'; +import { + CfnDirectoryBucket +} from "aws-cdk-lib/aws-s3express"; import { StackProps, Stack, Tags, RemovalPolicy, Duration, CfnOutput } from "aws-cdk-lib"; import { Construct } from "constructs"; import { GitHubOidcRole } from "../constructs/github-oidc-role"; @@ -30,11 +34,13 @@ export class CanaryStack extends Stack { public readonly canaryCodeBucket: Bucket; public readonly canaryTestBucket: Bucket; public readonly canaryTestMrap: CfnMultiRegionAccessPoint; + public readonly canaryTestExpressBucket: CfnDirectoryBucket; public readonly lambdaExecutionRoleArn: CfnOutput; public readonly canaryCodeBucketName: CfnOutput; public readonly canaryTestBucketName: CfnOutput; public readonly canaryTestMrapBucketArn: CfnOutput; + public readonly canaryTestExpressBucketName: CfnOutput; constructor(scope: Construct, id: string, props: Properties) { super(scope, id, props); @@ -143,6 +149,18 @@ export class CanaryStack extends Stack { }); } + this.canaryTestExpressBucket = new CfnDirectoryBucket(this, 'canary-test-express-bucket', { + dataRedundancy: 'SingleAvailabilityZone', + locationName: "usw2-az1", + }); + + // Output the bucket name to make it easier to invoke the canary runner + this.canaryTestExpressBucketName = new CfnOutput(this, "canary-test-express-bucket-name", { + value: this.canaryTestExpressBucket.ref, + description: "Name of the canary express test bucket", + exportName: "canaryExpressTestBucket", + }); + // Create a role for the canary Lambdas to assume this.lambdaExecutionRole = new Role(this, "lambda-execution-role", { roleName: "aws-sdk-rust-canary-lambda-exec-role", @@ -174,6 +192,15 @@ export class CanaryStack extends Stack { resources: [`${canaryTestMrapBucketArn}`, `${canaryTestMrapBucketArn}/object/*`], })); + // Allow canaries to perform operations on test express bucket + this.lambdaExecutionRole.addToPolicy( + new PolicyStatement({ + actions: ['s3express:*'], + effect: Effect.ALLOW, + resources: [`${this.canaryTestExpressBucket.attrArn}`], + }) + ); + // Allow canaries to call Transcribe's StartStreamTranscription this.lambdaExecutionRole.addToPolicy( new PolicyStatement({ diff --git a/tools/ci-cdk/package-lock.json b/tools/ci-cdk/package-lock.json index 8da1026934..abcc2cd24b 100644 --- a/tools/ci-cdk/package-lock.json +++ b/tools/ci-cdk/package-lock.json @@ -38,22 +38,22 @@ } }, "node_modules/@ampproject/remapping": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", - "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", "dev": true, "dependencies": { - "@jridgewell/gen-mapping": "^0.3.0", - "@jridgewell/trace-mapping": "^0.3.9" + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.200", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", - "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==", + "version": "2.2.202", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.202.tgz", + "integrity": "sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==", "dev": true }, "node_modules/@aws-cdk/asset-kubectl-v20": { @@ -69,12 +69,12 @@ "dev": true }, "node_modules/@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", + "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", "dev": true, "dependencies": { - "@babel/highlight": "^7.22.13", + "@babel/highlight": "^7.23.4", "chalk": "^2.4.2" }, "engines": { @@ -153,34 +153,34 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.22.9", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.22.9.tgz", - "integrity": "sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ==", + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.5.tgz", + "integrity": "sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/core": { - "version": "7.22.10", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.22.10.tgz", - "integrity": "sha512-fTmqbbUBAwCcre6zPzNngvsI0aNrPZe77AeqvDxWM9Nm+04RrJ3CAmGHA9f7lJQY6ZMhRztNemy4uslDxTX4Qw==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", + "integrity": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR+K9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", "dev": true, "dependencies": { "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.22.10", - "@babel/generator": "^7.22.10", - "@babel/helper-compilation-targets": "^7.22.10", - "@babel/helper-module-transforms": "^7.22.9", - "@babel/helpers": "^7.22.10", - "@babel/parser": "^7.22.10", - "@babel/template": "^7.22.5", - "@babel/traverse": "^7.22.10", - "@babel/types": "^7.22.10", - "convert-source-map": "^1.7.0", + "@babel/code-frame": "^7.23.5", + "@babel/generator": "^7.23.6", + "@babel/helper-compilation-targets": "^7.23.6", + "@babel/helper-module-transforms": "^7.23.3", + "@babel/helpers": "^7.24.0", + "@babel/parser": "^7.24.0", + "@babel/template": "^7.24.0", + "@babel/traverse": "^7.24.0", + "@babel/types": "^7.24.0", + "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", + "json5": "^2.2.3", "semver": "^6.3.1" }, "engines": { @@ -191,6 +191,12 @@ "url": "https://opencollective.com/babel" } }, + "node_modules/@babel/core/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true + }, "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -201,12 +207,12 @@ } }, "node_modules/@babel/generator": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", - "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.6.tgz", + "integrity": "sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==", "dev": true, "dependencies": { - "@babel/types": "^7.23.0", + "@babel/types": "^7.23.6", "@jridgewell/gen-mapping": "^0.3.2", "@jridgewell/trace-mapping": "^0.3.17", "jsesc": "^2.5.1" @@ -216,14 +222,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.22.10", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.10.tgz", - "integrity": "sha512-JMSwHD4J7SLod0idLq5PKgI+6g/hLD/iuWBq08ZX49xE14VpVEojJ5rHWptpirV2j020MvypRLAXAO50igCJ5Q==", + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.23.6.tgz", + "integrity": "sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==", "dev": true, "dependencies": { - "@babel/compat-data": "^7.22.9", - "@babel/helper-validator-option": "^7.22.5", - "browserslist": "^4.21.9", + "@babel/compat-data": "^7.23.5", + "@babel/helper-validator-option": "^7.23.5", + "browserslist": "^4.22.2", "lru-cache": "^5.1.1", "semver": "^6.3.1" }, @@ -275,28 +281,28 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz", - "integrity": "sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==", + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz", + "integrity": "sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==", "dev": true, "dependencies": { - "@babel/types": "^7.22.5" + "@babel/types": "^7.22.15" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.22.9", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz", - "integrity": "sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==", + "version": "7.23.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz", + "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==", "dev": true, "dependencies": { - "@babel/helper-environment-visitor": "^7.22.5", - "@babel/helper-module-imports": "^7.22.5", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-module-imports": "^7.22.15", "@babel/helper-simple-access": "^7.22.5", "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/helper-validator-identifier": "^7.22.5" + "@babel/helper-validator-identifier": "^7.22.20" }, "engines": { "node": ">=6.9.0" @@ -306,9 +312,9 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz", - "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.0.tgz", + "integrity": "sha512-9cUznXMG0+FxRuJfvL82QlTqIzhVW9sL0KjMPHhAOOvpQGL8QtdxnBKILjBqxlHyliz0yCa1G903ZXI/FuHy2w==", "dev": true, "engines": { "node": ">=6.9.0" @@ -339,9 +345,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz", + "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==", "dev": true, "engines": { "node": ">=6.9.0" @@ -357,32 +363,32 @@ } }, "node_modules/@babel/helper-validator-option": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.5.tgz", - "integrity": "sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==", + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz", + "integrity": "sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helpers": { - "version": "7.22.10", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.22.10.tgz", - "integrity": "sha512-a41J4NW8HyZa1I1vAndrraTlPZ/eZoga2ZgS7fEr0tZJGVU4xqdE80CEm0CcNjha5EZ8fTBYLKHF0kqDUuAwQw==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.0.tgz", + "integrity": "sha512-ulDZdc0Aj5uLc5nETsa7EPx2L7rM0YJM8r7ck7U73AXi7qOV44IHHRAYZHY6iU1rr3C5N4NtTmMRUJP6kwCWeA==", "dev": true, "dependencies": { - "@babel/template": "^7.22.5", - "@babel/traverse": "^7.22.10", - "@babel/types": "^7.22.10" + "@babel/template": "^7.24.0", + "@babel/traverse": "^7.24.0", + "@babel/types": "^7.24.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", + "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", "dev": true, "dependencies": { "@babel/helper-validator-identifier": "^7.22.20", @@ -465,9 +471,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", - "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.0.tgz", + "integrity": "sha512-QuP/FxEAzMSjXygs8v4N9dvdXzEHN4W1oF3PxuWAtPo08UdM17u89RDMgjLn/mlc56iM0HlLmVkO/wgR+rDgHg==", "dev": true, "bin": { "parser": "bin/babel-parser.js" @@ -624,9 +630,9 @@ } }, "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.22.5.tgz", - "integrity": "sha512-1mS2o03i7t1c6VzH6fdQ3OA8tcEIxwG18zIPRp+UY1Ihv6W+XZzBCVxExF9upussPXJ0xE9XRHwMoNs1ep/nRQ==", + "version": "7.23.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.23.3.tgz", + "integrity": "sha512-9EiNjVJOMwCO+43TqoTrgQ8jMwcAd0sWyXi9RPfIsLTj4R2MADDDQXELhffaUx/uJv2AYcxBgPwH6j4TIA4ytQ==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.22.5" @@ -639,34 +645,34 @@ } }, "node_modules/@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.0.tgz", + "integrity": "sha512-Bkf2q8lMB0AFpX0NFEqSbx1OkTHf0f+0j82mkw+ZpzBnkk7e9Ql0891vlfgi+kHwOk8tQjiQHpqh4LaSa0fKEA==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" + "@babel/code-frame": "^7.23.5", + "@babel/parser": "^7.24.0", + "@babel/types": "^7.24.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.0.tgz", + "integrity": "sha512-HfuJlI8qq3dEDmNU5ChzzpZRWq+oxCZQyMzIMEqLho+AQnhMnKQUzH6ydo3RBl/YjPCuk68Y6s0Gx0AeyULiWw==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", + "@babel/code-frame": "^7.23.5", + "@babel/generator": "^7.23.6", "@babel/helper-environment-visitor": "^7.22.20", "@babel/helper-function-name": "^7.23.0", "@babel/helper-hoist-variables": "^7.22.5", "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", + "@babel/parser": "^7.24.0", + "@babel/types": "^7.24.0", + "debug": "^4.3.1", "globals": "^11.1.0" }, "engines": { @@ -683,12 +689,12 @@ } }, "node_modules/@babel/types": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", - "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.0.tgz", + "integrity": "sha512-+j7a5c253RfKh8iABBhywc8NSfP5LURe7Uh4qpsh6jc+aLJguvmIUBdjSdEMQv2bENrCR5MfRdjGo7vzS/ob7w==", "dev": true, "dependencies": { - "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-string-parser": "^7.23.4", "@babel/helper-validator-identifier": "^7.22.20", "to-fast-properties": "^2.0.0" }, @@ -740,18 +746,18 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.7.0.tgz", - "integrity": "sha512-+HencqxU7CFJnQb7IKtuNBqS6Yx3Tz4kOL8BJXo+JyeiBm5MEX6pO8onXDkjrkCRlfYXS1Axro15ZjVFe9YgsA==", + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", "dev": true, "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, "node_modules/@eslint/eslintrc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.2.tgz", - "integrity": "sha512-+wvgpDsrB1YqAMdEUCcnTlpfVBH7Vqn6A/NT3D8WVXFIaKMlErPIZT3oCIAVCOtarRpMtelZLqJeU3t7WY6X6g==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", @@ -772,22 +778,22 @@ } }, "node_modules/@eslint/js": { - "version": "8.47.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.47.0.tgz", - "integrity": "sha512-P6omY1zv5MItm93kLM8s2vr1HICJH8v0dvddDhysbIuZ+vcjOHg5Zbkf1mTkcmi2JA9oBG2anOkRnW8WJTS8Og==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", + "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", - "integrity": "sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==", + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^1.2.1", - "debug": "^4.1.1", + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", "minimatch": "^3.0.5" }, "engines": { @@ -808,9 +814,9 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz", + "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==", "dev": true }, "node_modules/@istanbuljs/load-nyc-config": { @@ -1162,32 +1168,32 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", - "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", "dev": true, "dependencies": { - "@jridgewell/set-array": "^1.0.1", + "@jridgewell/set-array": "^1.2.1", "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" + "@jridgewell/trace-mapping": "^0.3.24" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", - "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", - "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", "dev": true, "engines": { "node": ">=6.0.0" @@ -1200,9 +1206,9 @@ "dev": true }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -1296,9 +1302,9 @@ "dev": true }, "node_modules/@types/babel__core": { - "version": "7.20.1", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.1.tgz", - "integrity": "sha512-aACu/U/omhdk15O4Nfb+fHgH/z3QsfQzpnvRZhYhThms83ZnAOZz7zZAWO7mn2yyNQaA4xTO8GLK3uqFU4bYYw==", + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", "dev": true, "dependencies": { "@babel/parser": "^7.20.7", @@ -1309,18 +1315,18 @@ } }, "node_modules/@types/babel__generator": { - "version": "7.6.4", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.4.tgz", - "integrity": "sha512-tFkciB9j2K755yrTALxD44McOrk+gfpIpvC3sxHjRawj6PfnQxrse4Clq5y/Rq+G3mrBurMax/lG8Qn2t9mSsg==", + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", "dev": true, "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.1.tgz", - "integrity": "sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g==", + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", "dev": true, "dependencies": { "@babel/parser": "^7.1.0", @@ -1328,42 +1334,42 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.20.1", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.1.tgz", - "integrity": "sha512-MitHFXnhtgwsGZWtT68URpOvLN4EREih1u3QtQiN4VdAxWKRVvGCSvw/Qth0M0Qq3pJpnGOu5JaM/ydK7OGbqg==", + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.5.tgz", + "integrity": "sha512-WXCyOcRtH37HAUkpXhUduaxdm82b4GSlyTqajXviN4EfiuPgNYR109xMCKvpl6zPIpua0DGlMEDCq+g8EdoheQ==", "dev": true, "dependencies": { "@babel/types": "^7.20.7" } }, "node_modules/@types/graceful-fs": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.6.tgz", - "integrity": "sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", "dev": true, "dependencies": { "@types/node": "*" } }, "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", - "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", "dev": true }, "node_modules/@types/istanbul-lib-report": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", - "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", "dev": true, "dependencies": { "@types/istanbul-lib-coverage": "*" } }, "node_modules/@types/istanbul-reports": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz", - "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", "dev": true, "dependencies": { "@types/istanbul-lib-report": "*" @@ -1380,9 +1386,9 @@ } }, "node_modules/@types/json-schema": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", - "integrity": "sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==", + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, "node_modules/@types/node": { @@ -1398,30 +1404,30 @@ "dev": true }, "node_modules/@types/semver": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz", - "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==", + "version": "7.5.8", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", + "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, "node_modules/@types/stack-utils": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", - "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, "node_modules/@types/yargs": { - "version": "16.0.5", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.5.tgz", - "integrity": "sha512-AxO/ADJOBFJScHbWhq2xAhlWP24rY4aCEG/NFaMvbT3X2MgRsLjhjQwsn0Zi5zn0LG9jUhCCZMeX9Dkuw6k+vQ==", + "version": "16.0.9", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.9.tgz", + "integrity": "sha512-tHhzvkFXZQeTECenFoRljLBYPZJ7jAVxqqtEI0qTLOmuultnFp4I9yKE17vTuhf7BkhCu7I4XuemPgikDVuYqA==", "dev": true, "dependencies": { "@types/yargs-parser": "*" } }, "node_modules/@types/yargs-parser": { - "version": "21.0.0", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", - "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { @@ -1612,16 +1618,23 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, "node_modules/abab": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", + "deprecated": "Use your platform's native atob() and btoa() methods instead", "dev": true }, "node_modules/acorn": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", - "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -1790,9 +1803,9 @@ "dev": true }, "node_modules/aws-cdk": { - "version": "2.93.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.93.0.tgz", - "integrity": "sha512-C0o7rzlXbQ3othvQ9uZamRwr741MSX/9eZ74zNJvpkX5Eitx/XoQYwUHeD+cbb4lKHMi7m2SwJfx3yOEkpu9OQ==", + "version": "2.131.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.131.0.tgz", + "integrity": "sha512-ji+MwGFGC88HE/EqV6/VARBp5mu3nXIDa/GYwtGycJqu6WqXhNZXWeDH0JsWaY6+BSUdpY6pr6KWpV+MDyVkDg==", "dev": true, "bin": { "cdk": "bin/cdk" @@ -1805,9 +1818,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.93.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.93.0.tgz", - "integrity": "sha512-kKbcKkts272Ju5xjGKI3pXTOpiJxW4OQbDF8Vmw/NIkkuJLo8GlRCFfeOfoN/hilvlYQgENA67GCgSWccbvu7w==", + "version": "2.131.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.131.0.tgz", + "integrity": "sha512-9XLgiTgY+q0S3K93VPeJO0chIN8BZwZ3aSrILvF868Dz+0NTNrD2m5M0xGK5Rw0uoJS+N+DvGaz/2hLAiVqcBw==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -1818,21 +1831,23 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], "dev": true, "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.200", + "@aws-cdk/asset-awscli-v1": "^2.2.202", "@aws-cdk/asset-kubectl-v20": "^2.1.2", "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^11.1.1", - "ignore": "^5.2.4", + "fs-extra": "^11.2.0", + "ignore": "^5.3.1", "jsonschema": "^1.4.1", + "mime-types": "^2.1.35", "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.5.4", + "punycode": "^2.3.1", + "semver": "^7.6.0", "table": "^6.8.1", "yaml": "1.10.2" }, @@ -1960,7 +1975,7 @@ "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.2.0", "dev": true, "inBundle": true, "license": "MIT", @@ -1980,7 +1995,7 @@ "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.1", "dev": true, "inBundle": true, "license": "MIT", @@ -2042,6 +2057,27 @@ "node": ">=10" } }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "dev": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/aws-cdk-lib/node_modules/minimatch": { "version": "3.1.2", "dev": true, @@ -2055,7 +2091,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "dev": true, "inBundle": true, "license": "MIT", @@ -2073,7 +2109,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.5.4", + "version": "7.6.0", "dev": true, "inBundle": true, "license": "ISC", @@ -2147,7 +2183,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "dev": true, "inBundle": true, "license": "MIT", @@ -2306,9 +2342,9 @@ "dev": true }, "node_modules/browserslist": { - "version": "4.21.10", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz", - "integrity": "sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", + "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", "dev": true, "funding": [ { @@ -2325,10 +2361,10 @@ } ], "dependencies": { - "caniuse-lite": "^1.0.30001517", - "electron-to-chromium": "^1.4.477", - "node-releases": "^2.0.13", - "update-browserslist-db": "^1.0.11" + "caniuse-lite": "^1.0.30001587", + "electron-to-chromium": "^1.4.668", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.0.13" }, "bin": { "browserslist": "cli.js" @@ -2383,9 +2419,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001522", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001522.tgz", - "integrity": "sha512-TKiyTVZxJGhsTszLuzb+6vUZSjVOAhClszBr2Ta2k9IwtNBT/4dzmL6aywt0HCgEZlmwJzXJd8yNiob6HgwTRg==", + "version": "1.0.30001593", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001593.tgz", + "integrity": "sha512-UWM1zlo3cZfkpBysd7AS+z+v007q9G1+fLTUU42rQnY6t2axoogPW/xol6T7juU5EUoOhML4WgBIdG+9yYqAjQ==", "dev": true, "funding": [ { @@ -2428,9 +2464,9 @@ } }, "node_modules/ci-info": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", - "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", "dev": true, "funding": [ { @@ -2512,9 +2548,9 @@ "dev": true }, "node_modules/constructs": { - "version": "10.2.69", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.2.69.tgz", - "integrity": "sha512-0AiM/uQe5Uk6JVe/62oolmSN2MjbFQkOlYrM3fFGZLKuT+g7xlAI10EebFhyCcZwI2JAcWuWCmmCAyCothxjuw==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", + "integrity": "sha512-vbK8i3rIb/xwZxSpTjz3SagHn1qq9BChLEfy5Hf6fB3/2eFbrwt2n9kHwQcS0CPTRBesreeAcsJfMq2229FnbQ==", "dev": true, "engines": { "node": ">= 16.14.0" @@ -2692,6 +2728,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "deprecated": "Use your platform's native DOMException instead", "dev": true, "dependencies": { "webidl-conversions": "^5.0.0" @@ -2710,9 +2747,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.500", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.500.tgz", - "integrity": "sha512-P38NO8eOuWOKY1sQk5yE0crNtrjgjJj6r3NrbIKtG18KzCHmHE2Bt+aQA7/y0w3uYsHWxDa6icOohzjLJ4vJ4A==", + "version": "1.4.690", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.690.tgz", + "integrity": "sha512-+2OAGjUx68xElQhydpcbqH50hE8Vs2K6TkAeLhICYfndb67CVH0UsZaijmRUE3rHlIxU1u0jxwhgVe6fK3YANA==", "dev": true }, "node_modules/emittery": { @@ -2743,9 +2780,9 @@ } }, "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", "dev": true, "engines": { "node": ">=6" @@ -2794,18 +2831,19 @@ } }, "node_modules/eslint": { - "version": "8.47.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.47.0.tgz", - "integrity": "sha512-spUQWrdPt+pRVP1TTJLmfRNJJHHZryFmptzcafwSvHsceV81djHOdnEeDmkdotZyLNjDhrOasNK8nikkoG1O8Q==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", + "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "^8.47.0", - "@humanwhocodes/config-array": "^0.11.10", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.0", + "@humanwhocodes/config-array": "^0.11.14", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -3053,9 +3091,9 @@ "dev": true }, "node_modules/fast-glob": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz", - "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -3093,9 +3131,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", - "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -3151,12 +3189,13 @@ } }, "node_modules/flat-cache": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", - "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", "dev": true, "dependencies": { - "flatted": "^3.1.0", + "flatted": "^3.2.9", + "keyv": "^4.5.3", "rimraf": "^3.0.2" }, "engines": { @@ -3164,9 +3203,9 @@ } }, "node_modules/flatted": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", - "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", + "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, "node_modules/form-data": { @@ -3204,10 +3243,13 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/gensync": { "version": "1.0.0-beta.2", @@ -3281,9 +3323,9 @@ } }, "node_modules/globals": { - "version": "13.21.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz", - "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -3327,18 +3369,6 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -3348,6 +3378,18 @@ "node": ">=8" } }, + "node_modules/hasown": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.1.tgz", + "integrity": "sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -3415,9 +3457,9 @@ } }, "node_modules/ignore": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", - "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true, "engines": { "node": ">= 4" @@ -3490,12 +3532,12 @@ "dev": true }, "node_modules/is-core-module": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", - "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "dev": true, "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -3589,9 +3631,9 @@ "dev": true }, "node_modules/istanbul-lib-coverage": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", - "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", "dev": true, "engines": { "node": ">=8" @@ -3651,9 +3693,9 @@ } }, "node_modules/istanbul-reports": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.6.tgz", - "integrity": "sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", "dev": true, "dependencies": { "html-escaper": "^2.0.0", @@ -4352,6 +4394,12 @@ "node": ">=4" } }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -4382,6 +4430,15 @@ "node": ">=6" } }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -4586,9 +4643,9 @@ "dev": true }, "node_modules/node-releases": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz", - "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==", + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", "dev": true }, "node_modules/normalize-path": { @@ -4937,9 +4994,9 @@ "dev": true }, "node_modules/punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, "engines": { "node": ">=6" @@ -4993,9 +5050,9 @@ "dev": true }, "node_modules/resolve": { - "version": "1.22.4", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz", - "integrity": "sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg==", + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "dev": true, "dependencies": { "is-core-module": "^2.13.0", @@ -5115,9 +5172,9 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -5487,9 +5544,9 @@ } }, "node_modules/ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", @@ -5530,9 +5587,9 @@ } }, "node_modules/ts-node/node_modules/acorn-walk": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", - "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz", + "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==", "dev": true, "engines": { "node": ">=0.4.0" @@ -5624,9 +5681,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", - "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", + "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==", "dev": true, "funding": [ { From eebe8afb6b50449fdc9c8702bef9a66fd9787d8a Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 8 Mar 2024 09:08:35 -0600 Subject: [PATCH 12/16] Allow `canary-runner` to specify lambda's memory size --- tools/ci-cdk/canary-runner/src/run.rs | 66 +++++++++++++++++---------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/tools/ci-cdk/canary-runner/src/run.rs b/tools/ci-cdk/canary-runner/src/run.rs index b4394c6348..85fb004816 100644 --- a/tools/ci-cdk/canary-runner/src/run.rs +++ b/tools/ci-cdk/canary-runner/src/run.rs @@ -36,6 +36,8 @@ use aws_sdk_lambda as lambda; use aws_sdk_s3 as s3; use std::collections::HashMap; +const DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB: i32 = 512; + lazy_static::lazy_static! { // Occasionally, a breaking change introduced in smithy-rs will cause the canary to fail // for older versions of the SDK since the canary is in the smithy-rs repository and will @@ -89,6 +91,10 @@ pub struct RunArgs { #[clap(long)] expected_speech_text_by_transcribe: Option, + /// Memory allocated for the function. + #[clap(long)] + lambda_function_memory_size_in_mb: Option, + /// File path to a CDK outputs JSON file. This can be used instead /// of all the --lambda... args. #[clap(long)] @@ -122,6 +128,7 @@ struct Options { sdk_path: Option, musl: bool, expected_speech_text_by_transcribe: Option, + lambda_function_memory_size_in_mb: i32, lambda_code_s3_bucket_name: String, lambda_test_s3_bucket_name: String, lambda_test_s3_mrap_bucket_arn: String, @@ -161,6 +168,9 @@ impl Options { sdk_path: run_opt.sdk_path, musl: run_opt.musl, expected_speech_text_by_transcribe: run_opt.expected_speech_text_by_transcribe, + lambda_function_memory_size_in_mb: run_opt + .lambda_function_memory_size_in_mb + .unwrap_or(DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB), lambda_code_s3_bucket_name: value.inner.lambda_code_s3_bucket_name, lambda_test_s3_bucket_name: value.inner.lambda_test_s3_bucket_name, lambda_test_s3_mrap_bucket_arn: value.inner.lambda_test_s3_mrap_bucket_arn, @@ -174,6 +184,9 @@ impl Options { sdk_path: run_opt.sdk_path, musl: run_opt.musl, expected_speech_text_by_transcribe: run_opt.expected_speech_text_by_transcribe, + lambda_function_memory_size_in_mb: run_opt + .lambda_function_memory_size_in_mb + .unwrap_or(DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB), lambda_code_s3_bucket_name: run_opt.lambda_code_s3_bucket_name.expect("required"), lambda_test_s3_bucket_name: run_opt.lambda_test_s3_bucket_name.expect("required"), lambda_test_s3_mrap_bucket_arn: run_opt @@ -283,12 +296,7 @@ async fn run_canary(options: &Options, config: &aws_config::SdkConfig) -> Result lambda_client.clone(), bundle_name, bundle_file_name, - &options.lambda_execution_role_arn, - options.expected_speech_text_by_transcribe.as_ref(), - &options.lambda_code_s3_bucket_name, - &options.lambda_test_s3_bucket_name, - &options.lambda_test_s3_mrap_bucket_arn, - &options.lambda_test_s3_express_bucket_name, + options, ) .await .context(here!())?; @@ -356,27 +364,27 @@ async fn upload_bundle( Ok(()) } -#[allow(clippy::too_many_arguments)] async fn create_lambda_fn( lambda_client: lambda::Client, bundle_name: &str, bundle_file_name: &str, - execution_role: &str, - expected_speech_text_by_transcribe: Option<&String>, - code_s3_bucket: &str, - test_s3_bucket: &str, - test_s3_mrap_bucket_arn: &str, - test_s3_express_bucket: &str, + options: &Options, ) -> Result<()> { use lambda::types::*; - let mut env_builder = match expected_speech_text_by_transcribe { + let mut env_builder = match &options.expected_speech_text_by_transcribe { Some(expected_speech_text_by_transcribe) => Environment::builder() .variables("RUST_BACKTRACE", "1") .variables("RUST_LOG", "info") - .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket) - .variables("CANARY_S3_MRAP_BUCKET_ARN", test_s3_mrap_bucket_arn) - .variables("CANARY_S3_EXPRESS_BUCKET_NAME", test_s3_express_bucket) + .variables("CANARY_S3_BUCKET_NAME", &options.lambda_test_s3_bucket_name) + .variables( + "CANARY_S3_MRAP_BUCKET_ARN", + &options.lambda_test_s3_mrap_bucket_arn, + ) + .variables( + "CANARY_S3_EXPRESS_BUCKET_NAME", + &options.lambda_test_s3_express_bucket_name, + ) .variables( "CANARY_EXPECTED_TRANSCRIBE_RESULT", expected_speech_text_by_transcribe, @@ -384,9 +392,15 @@ async fn create_lambda_fn( None => Environment::builder() .variables("RUST_BACKTRACE", "1") .variables("RUST_LOG", "info") - .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket) - .variables("CANARY_S3_MRAP_BUCKET_ARN", test_s3_mrap_bucket_arn) - .variables("CANARY_S3_EXPRESS_BUCKET_NAME", test_s3_express_bucket), + .variables("CANARY_S3_BUCKET_NAME", &options.lambda_test_s3_bucket_name) + .variables( + "CANARY_S3_MRAP_BUCKET_ARN", + &options.lambda_test_s3_mrap_bucket_arn, + ) + .variables( + "CANARY_S3_EXPRESS_BUCKET_NAME", + &options.lambda_test_s3_express_bucket_name, + ), }; // TODO(Post S3Express release): Delete this once S3 Express has been released and its canary is on by default @@ -398,17 +412,18 @@ async fn create_lambda_fn( .create_function() .function_name(bundle_name) .runtime(Runtime::Providedal2) - .role(execution_role) + .role(&options.lambda_execution_role_arn) .handler("aws-sdk-rust-lambda-canary") .code( FunctionCode::builder() - .s3_bucket(code_s3_bucket) + .s3_bucket(&options.lambda_code_s3_bucket_name) .s3_key(bundle_file_name) .build(), ) .publish(true) .environment(env_builder.build()) .timeout(180) + .memory_size(options.lambda_function_memory_size_in_mb) .send() .await .context(here!("failed to create canary Lambda function"))?; @@ -507,8 +522,7 @@ async fn delete_lambda_fn(lambda_client: lambda::Client, bundle_name: &str) -> R #[cfg(test)] mod tests { - use crate::run::Options; - use crate::run::RunArgs; + use crate::run::{Options, RunArgs, DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB}; use clap::Parser; #[test] @@ -520,6 +534,7 @@ mod tests { sdk_path: Some("artifact-aws-sdk-rust/sdk".into()), musl: false, expected_speech_text_by_transcribe: Some("Good day to you transcribe.".to_owned()), + lambda_function_memory_size_in_mb: Some(1024), cdk_output: Some("../cdk-outputs.json".into()), lambda_code_s3_bucket_name: None, lambda_test_s3_bucket_name: None, @@ -533,6 +548,8 @@ mod tests { "artifact-aws-sdk-rust/sdk", "--expected-speech-text-by-transcribe", "Good day to you transcribe.", + "--lambda-function-memory-size-in-mb", + "1024", "--cdk-output", "../cdk-outputs.json", ]) @@ -567,6 +584,7 @@ mod tests { sdk_path: Some("artifact-aws-sdk-rust/sdk".into()), musl: false, expected_speech_text_by_transcribe: Some("Good day to you transcribe.".to_owned()), + lambda_function_memory_size_in_mb: DEFAULT_LAMBDA_FUNCTION_MEMORY_SIZE_IN_MB, lambda_code_s3_bucket_name: "bucket-for-code".to_owned(), lambda_test_s3_bucket_name: "bucket-for-test".to_owned(), lambda_execution_role_arn: "arn:aws:lambda::role/exe-role".to_owned(), From 60e6a896d542dd8b737bb74693a99c742e60f2c2 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 8 Mar 2024 12:23:45 -0600 Subject: [PATCH 13/16] Pass what's minimally needed to `S3ExpressRuntimePlugin` This commit addresses https://github.com/smithy-lang/smithy-rs/pull/3465#discussion_r1518033302 --- .../aws-inlineable/src/s3_express.rs | 31 ++++++++----------- .../customize/s3/S3ExpressDecorator.kt | 9 +++++- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index e96fd0c7dc..630e90c946 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -636,16 +636,18 @@ pub(crate) mod runtime_plugin { } impl S3ExpressRuntimePlugin { - pub(crate) fn new(config: FrozenLayer) -> Self { - Self::new_with(config, Env::real()) + pub(crate) fn new( + disable_s3_express_session_token: Option, + ) -> Self { + Self::new_with(disable_s3_express_session_token, Env::real()) } - fn new_with(config: FrozenLayer, env: Env) -> Self { + fn new_with( + disable_s3_express_session_token: Option, + env: Env, + ) -> Self { let mut layer = Layer::new("S3ExpressRuntimePlugin"); - if config - .load::() - .is_none() - { + if disable_s3_express_session_token.is_none() { match env.get(env::S3_DISABLE_EXPRESS_SESSION_AUTH) { Ok(value) if value.eq_ignore_ascii_case("true") @@ -711,13 +713,12 @@ pub(crate) mod runtime_plugin { #[test] fn disable_option_set_from_service_client_should_take_the_highest_precedence() { // Disable option is set from service client. - let mut layer = Layer::new("test"); - layer.store_put(crate::config::DisableS3ExpressSessionAuth(true)); + let disable_s3_express_session_token = crate::config::DisableS3ExpressSessionAuth(true); // An environment variable says the session auth is _not_ disabled, but it will be // overruled by what is in `layer`. let sut = S3ExpressRuntimePlugin::new_with( - layer.freeze(), + Some(disable_s3_express_session_token), Env::from_slice(&[(super::env::S3_DISABLE_EXPRESS_SESSION_AUTH, "false")]), ); @@ -730,12 +731,9 @@ pub(crate) mod runtime_plugin { #[test] fn disable_option_set_from_env_should_take_the_second_highest_precedence() { - // No disable option is set from service client. - let layer = Layer::new("test"); - // An environment variable says session auth is disabled let sut = S3ExpressRuntimePlugin::new_with( - layer.freeze(), + None, Env::from_slice(&[(super::env::S3_DISABLE_EXPRESS_SESSION_AUTH, "true")]), ); @@ -757,11 +755,8 @@ pub(crate) mod runtime_plugin { #[test] fn disable_option_should_be_unspecified_if_unset() { - // No disable option is set from service client. - let layer = Layer::new("test"); - // An environment variable says session auth is disabled - let sut = S3ExpressRuntimePlugin::new_with(layer.freeze(), Env::from_slice(&[])); + let sut = S3ExpressRuntimePlugin::new_with(None, Env::from_slice(&[])); let cfg = sut.config().unwrap(); assert!(cfg diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 361b58cbf5..e48ddfcef6 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -215,7 +215,14 @@ class S3ExpressFluentClientCustomization( is FluentClientSection.AdditionalBaseClientPlugins -> { rustTemplate( """ - ${section.plugins} = ${section.plugins}.with_client_plugin(#{S3ExpressRuntimePlugin}::new(${section.config}.config.clone())); + ${section.plugins} = ${section.plugins}.with_client_plugin( + #{S3ExpressRuntimePlugin}::new( + ${section.config} + .config + .load::() + .cloned() + ) + ); """, *codegenScope, ) From 374ed2a91000ffa9d69a5d4401fd13a01569ee78 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 8 Mar 2024 13:18:41 -0600 Subject: [PATCH 14/16] Define a function to provide default checksum altorithm This commit addresses https://github.com/smithy-lang/smithy-rs/pull/3465#discussion_r1518045160 --- .../aws-inlineable/src/s3_express.rs | 23 +++++++++ .../customize/s3/S3ExpressDecorator.kt | 47 +++++++------------ 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 630e90c946..f917507b98 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -766,6 +766,29 @@ pub(crate) mod runtime_plugin { } } +pub(crate) mod checksum { + use aws_smithy_checksums::ChecksumAlgorithm; + use aws_smithy_types::config_bag::ConfigBag; + + pub(crate) fn provide_default_checksum_algorithm( + original_checksum: Option, + cfg: &ConfigBag, + ) -> Option { + // S3 does not have the `ChecksumAlgorithm::Md5`, therefore customers cannot set it + // from outside. + if original_checksum != Some(ChecksumAlgorithm::Md5) { + return original_checksum; + } + + if crate::s3_express::utils::for_s3_express(cfg) { + // S3 Express requires setting the default checksum algorithm to CRC-32 + Some(ChecksumAlgorithm::Crc32) + } else { + original_checksum + } + } +} + pub(crate) mod utils { use aws_smithy_types::{config_bag::ConfigBag, Document}; diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index e48ddfcef6..7c37af7dec 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -5,9 +5,9 @@ package software.amazon.smithy.rustsdk.customize.s3 +import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.AuthSchemeOption @@ -28,6 +28,7 @@ import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rustsdk.AwsCargoDependency import software.amazon.smithy.rustsdk.AwsRuntimeType import software.amazon.smithy.rustsdk.InlineAwsDependency @@ -249,41 +250,29 @@ class S3ExpressRequestChecksumCustomization( .resolve("DefaultRequestChecksumOverride"), "Document" to RuntimeType.smithyTypes(runtimeConfig).resolve("Document"), "for_s3_express" to s3ExpressModule(runtimeConfig).resolve("utils::for_s3_express"), + "provide_default_checksum_algorithm" to s3ExpressModule(runtimeConfig).resolve("checksum::provide_default_checksum_algorithm"), ) override fun section(section: OperationSection): Writable = writable { + // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one + val checksumTrait = operationShape.getTrait() ?: return@writable when (section) { is OperationSection.AdditionalRuntimePluginConfig -> { - rustTemplate( - """ - ${section.newLayerName}.store_put(#{DefaultRequestChecksumOverride}::new( - |original: #{Option}<#{ChecksumAlgorithm}>, - cfg: &#{ConfigBag}| { - // S3 does not have the `ChecksumAlgorithm::Md5`, therefore customers cannot set it - // from outside. - if original != #{Some}(#{ChecksumAlgorithm}::Md5) { - return original; - } - - if #{for_s3_express}(cfg) { - #{customDefault:W} - } else { - original - } - } - )); - """, - *codegenScope, - "customDefault" to - writable { - if (operationShape.id == ShapeId.from("com.amazonaws.s3#UploadPart")) { - rustTemplate("#{None}", *codegenScope) - } else { + if (checksumTrait.isRequestChecksumRequired) { + rustTemplate( + """ + ${section.newLayerName}.store_put(#{DefaultRequestChecksumOverride}::new( + #{provide_default_checksum_algorithm} + )); + """, + *codegenScope, + "customDefault" to + writable { rustTemplate("#{Some}(#{ChecksumAlgorithm}::Crc32)", *codegenScope) - } - }, - ) + }, + ) + } } else -> { } From 3c3843f158ea680a10c1a16c6f5d81f3bf44d6d3 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 8 Mar 2024 13:57:31 -0600 Subject: [PATCH 15/16] Let default checksum provider return `DefaultRequestChecksumOverride` This commit addresses https://github.com/smithy-lang/smithy-rs/pull/3465#discussion_r1518208643 --- .../src/http_request_checksum.rs | 2 +- .../aws-inlineable/src/s3_express.rs | 33 +++++++++++-------- .../customize/s3/S3ExpressDecorator.kt | 8 +---- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 48fa609639..3e010068c6 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -60,7 +60,7 @@ impl Storable for RequestChecksumInterceptorState { type Storer = StoreReplace; } -pub(crate) type CustomDefaultFn = Box< +type CustomDefaultFn = Box< dyn Fn(Option, &ConfigBag) -> Option + Send + Sync diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index f917507b98..fa6623d3b0 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -767,25 +767,30 @@ pub(crate) mod runtime_plugin { } pub(crate) mod checksum { + use crate::http_request_checksum::DefaultRequestChecksumOverride; use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_types::config_bag::ConfigBag; pub(crate) fn provide_default_checksum_algorithm( - original_checksum: Option, - cfg: &ConfigBag, - ) -> Option { - // S3 does not have the `ChecksumAlgorithm::Md5`, therefore customers cannot set it - // from outside. - if original_checksum != Some(ChecksumAlgorithm::Md5) { - return original_checksum; - } - - if crate::s3_express::utils::for_s3_express(cfg) { - // S3 Express requires setting the default checksum algorithm to CRC-32 - Some(ChecksumAlgorithm::Crc32) - } else { - original_checksum + ) -> crate::http_request_checksum::DefaultRequestChecksumOverride { + fn _provide_default_checksum_algorithm( + original_checksum: Option, + cfg: &ConfigBag, + ) -> Option { + // S3 does not have the `ChecksumAlgorithm::Md5`, therefore customers cannot set it + // from outside. + if original_checksum != Some(ChecksumAlgorithm::Md5) { + return original_checksum; + } + + if crate::s3_express::utils::for_s3_express(cfg) { + // S3 Express requires setting the default checksum algorithm to CRC-32 + Some(ChecksumAlgorithm::Crc32) + } else { + original_checksum + } } + DefaultRequestChecksumOverride::new(_provide_default_checksum_algorithm) } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt index 7c37af7dec..b29ef0693b 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -32,7 +32,6 @@ import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rustsdk.AwsCargoDependency import software.amazon.smithy.rustsdk.AwsRuntimeType import software.amazon.smithy.rustsdk.InlineAwsDependency -import software.amazon.smithy.rustsdk.awsInlineableHttpRequestChecksum class S3ExpressDecorator : ClientCodegenDecorator { override val name: String = "S3ExpressDecorator" @@ -245,9 +244,6 @@ class S3ExpressRequestChecksumCustomization( *preludeScope, "ChecksumAlgorithm" to RuntimeType.smithyChecksums(runtimeConfig).resolve("ChecksumAlgorithm"), "ConfigBag" to RuntimeType.configBag(runtimeConfig), - "DefaultRequestChecksumOverride" to - runtimeConfig.awsInlineableHttpRequestChecksum() - .resolve("DefaultRequestChecksumOverride"), "Document" to RuntimeType.smithyTypes(runtimeConfig).resolve("Document"), "for_s3_express" to s3ExpressModule(runtimeConfig).resolve("utils::for_s3_express"), "provide_default_checksum_algorithm" to s3ExpressModule(runtimeConfig).resolve("checksum::provide_default_checksum_algorithm"), @@ -262,9 +258,7 @@ class S3ExpressRequestChecksumCustomization( if (checksumTrait.isRequestChecksumRequired) { rustTemplate( """ - ${section.newLayerName}.store_put(#{DefaultRequestChecksumOverride}::new( - #{provide_default_checksum_algorithm} - )); + ${section.newLayerName}.store_put(#{provide_default_checksum_algorithm}()); """, *codegenScope, "customDefault" to From 3b8f2f45aa4732df097711d52901928f388d2dd4 Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Mon, 11 Mar 2024 10:32:57 -0500 Subject: [PATCH 16/16] Fix S3 Express bug where SigV4 session token was incorrectly overriden (#3474) ## Description S3 express canary exposed a bug introduced in smithy-rs#3457 where the code overwrote the regular SigV4 session token name with the S3 Expression session token name when it shouldn't. ``` 4: Error { code: "InvalidRequest", message: "CreateSession request should not include \"x-amz-s3session-token\"", aws_request_id: "01c0c8864e00018e20558a130509f37740b906e4", s3_extended_request_id: "DGTJhRqVMbZdAHQ" } ``` For APIs like `ListDirectoryBuckets` or `CreateSession`, we should not overwrite `x-amz-security-token` with `x-amz-s3session-token` in the request header. In the said PR, `aws_sdk_s3::s3_express::utils::for_s3_express` did not take into account auth schemes attached to a resolved endpoint, failing to detect that it should not override the session token name for the said APIs. This PR will resolve that issue. ## Testing - Added an integration test verifying the fix (this test currently fails when run in the destination branch, `ysaito/s3express`) - Verified all canary (including wasm, s3express) passed ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../aws-inlineable/src/s3_express.rs | 17 ++++++--- aws/sdk/integration-tests/s3/tests/express.rs | 38 +++++++++++++++---- tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts | 14 ++++++- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index fa6623d3b0..d87a7895c0 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -798,14 +798,21 @@ pub(crate) mod utils { use aws_smithy_types::{config_bag::ConfigBag, Document}; pub(crate) fn for_s3_express(cfg: &ConfigBag) -> bool { + // logic borrowed from aws_smithy_runtime::client::orchestrator::auth::extract_endpoint_auth_scheme_config let endpoint = cfg .load::() .expect("endpoint added to config bag by endpoint orchestrator"); - if let Some(Document::String(backend)) = endpoint.properties().get("backend") { - backend.as_str() == "S3Express" - } else { - false - } + let auth_schemes = match endpoint.properties().get("authSchemes") { + Some(Document::Array(schemes)) => schemes, + _ => return false, + }; + auth_schemes.iter().any(|doc| { + let config_scheme_id = doc + .as_object() + .and_then(|object| object.get("name")) + .and_then(Document::as_string); + config_scheme_id == Some(crate::s3_express::auth::SCHEME_ID.as_str()) + }) } } diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index 85bc961b6f..0c20884f23 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -27,6 +27,34 @@ where aws_sdk_s3::Client::from_conf(update_builder(config).build()) } +#[tokio::test] +async fn create_session_request_should_not_include_x_amz_s3session_token() { + let (http_client, request) = capture_request(None); + // There was a bug where a regular SigV4 session token was overwritten by an express session token + // even for CreateSession API request. + // To exercise that code path, it is important to include credentials with a session token below. + let conf = Config::builder() + .http_client(http_client) + .region(Region::new("us-west-2")) + .credentials_provider(::aws_credential_types::Credentials::for_tests_with_session_token()) + .build(); + let client = Client::from_conf(conf); + + let _ = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + + let req = request.expect_request(); + assert!( + req.headers().get("x-amz-create-session-mode").is_some(), + "`x-amz-create-session-mode` should appear in headers of the first request when an express bucket is specified" + ); + assert!(req.headers().get("x-amz-security-token").is_some()); + assert!(req.headers().get("x-amz-s3session-token").is_none()); +} + #[tokio::test] async fn mixed_auths() { let _logs = capture_test_logs(); @@ -292,10 +320,7 @@ async fn disable_s3_express_session_auth_at_service_client_level() { let req = request.expect_request(); assert!( - !req.headers() - .get("authorization") - .unwrap() - .contains("x-amz-create-session-mode"), + req.headers().get("x-amz-create-session-mode").is_none(), "x-amz-create-session-mode should not appear in headers when S3 Express session auth is disabled" ); } @@ -320,10 +345,7 @@ async fn disable_s3_express_session_auth_at_operation_level() { let req = request.expect_request(); assert!( - !req.headers() - .get("authorization") - .unwrap() - .contains("x-amz-create-session-mode"), + req.headers().get("x-amz-create-session-mode").is_none(), "x-amz-create-session-mode should not appear in headers when S3 Express session auth is disabled" ); } diff --git a/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts b/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts index dd64ab0172..4237717233 100644 --- a/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts +++ b/tools/ci-cdk/lib/aws-sdk-rust/canary-stack.ts @@ -193,14 +193,26 @@ export class CanaryStack extends Stack { })); // Allow canaries to perform operations on test express bucket + // Unlike S3, no need to grant separate permissions for GetObject, PutObject, and so on because + // the session token enables access instead: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-security-iam.html#s3-express-security-iam-actions this.lambdaExecutionRole.addToPolicy( new PolicyStatement({ - actions: ['s3express:*'], + actions: ['s3express:CreateSession'], effect: Effect.ALLOW, resources: [`${this.canaryTestExpressBucket.attrArn}`], }) ); + // Allow canaries to list directory buckets + this.lambdaExecutionRole.addToPolicy( + new PolicyStatement({ + actions: ['s3express:ListAllMyDirectoryBuckets'], + effect: Effect.ALLOW, + resources: ["*"], + }) + ); + // Allow canaries to call Transcribe's StartStreamTranscription this.lambdaExecutionRole.addToPolicy( new PolicyStatement({