From 486b91de07ebcab72d47d55e334d344dd64497de Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 16 Feb 2024 20:33:01 -0600 Subject: [PATCH] Allow list-objects-v2 to run against an S3 Express bucket (#3388) ## Motivation and Context Adds an implementation spike to allow `list-objects-v2` (possibly others, haven't tested yet) to run against an S3 Express bucket. ## Description This PR implements two ingredients, `S3ExpressIdentityProvider` and `S3ExpressSigner`. `S3ExpressIdentityProvider` uses an internal S3 client to obtain an S3 Express session token that is passed to `S3ExpressSigner`. `S3ExpressSigner` then signs a request with that token, using effectively sigv4 but with session token omitted and an extra header added instead, `x-amz-s3session-token`. In addition, this PR supports presigning for S3 Express. Similarly to signing headers, presigning for S3 Express excludes a query param `X-Amz-Security-Token` and instead uses `X-Amz-S3session-Token` for the signing query params. The following screeshot shows that a presigned URL from `get_object` works for an S3 Express bucket:

chain-provider-ext-timeout-2

Some implementation details: - Since `S3ExpressIdentityProvider` passes an S3 Express bucket name for S3's `create_session` API to obtain an S3 Express session token, it needs to obtain the bucket name from somewhere. `S3ExpressIdentityProvider::ProvideCredentials` I put previously did not have enough arguments for us to figure this out, so I switched to `S3ExpressIdentityProvider::ResolveIdentity` that takes enough arguments. - `SigV4Signer::sign_http_request` did not allow calling code to pass a configured `SigningSettings`; The signer needs to exclude a header `x-amz-security-token` and include `x-amz-s3session-token`. To make this happen, I made `sigv4::extract_operation_config` and `sigv4::settings` public APIs (previously private). - One area I haven't quite figured out yet is how to configure the inner S3 client to call `create_session`. The changes in this PR inherits runtime components & config bag from the "outer" S3 client, but customers may want to configure the inner S3 client in a more flexible manner (e.g. operation timeout). ## Testing To lock the behavior at this time, I added a connection recording test for `list-objects-v2`. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti Co-authored-by: AWS SDK Rust Bot Co-authored-by: AWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com> Co-authored-by: Zelda Hessler --- .../aws-inlineable/src/s3_express.rs | 139 ++++++++++++- .../aws-runtime/src/auth/sigv4.rs | 64 ++++-- .../src/http_request/canonical_request.rs | 27 ++- .../aws-sigv4/src/http_request/settings.rs | 5 + .../aws-sigv4/src/http_request/sign.rs | 10 +- .../rustsdk/EndpointBuiltInsDecorator.kt | 1 - .../amazon/smithy/rustsdk/RegionDecorator.kt | 4 + .../smithy/rustsdk/UserAgentDecorator.kt | 5 + .../rustsdk/customize/s3/S3Decorator.kt | 2 + .../endpoints/OperationInputTestGenerator.kt | 8 - .../tests/data/express/list-objects-v2.json | 193 ++++++++++++++++++ aws/sdk/integration-tests/s3/tests/express.rs | 131 ++++++++++++ .../ResiliencyConfigCustomization.kt | 15 ++ .../ClientContextConfigCustomization.kt | 2 +- .../config/ServiceConfigGenerator.kt | 57 ++++-- ...lledStreamProtectionConfigCustomization.kt | 8 + .../config/ServiceConfigGeneratorTest.kt | 48 +++++ .../src/client/runtime_components.rs | 28 +++ 18 files changed, 689 insertions(+), 58 deletions(-) create mode 100644 aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json create mode 100644 aws/sdk/integration-tests/s3/tests/express.rs diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs index 76b8a95093..3388145112 100644 --- a/aws/rust-runtime/aws-inlineable/src/s3_express.rs +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -5,6 +5,8 @@ /// Supporting code for S3 Express auth pub(crate) mod auth { + use aws_runtime::auth::sigv4::SigV4Signer; + use aws_sigv4::http_request::{SignatureLocation, SigningSettings}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::auth::{ AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, Sign, @@ -56,15 +58,37 @@ pub(crate) mod auth { impl Sign for S3ExpressSigner { fn sign_http_request( &self, - _request: &mut HttpRequest, - _identity: &Identity, - _auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, - _runtime_components: &RuntimeComponents, - _config_bag: &ConfigBag, + request: &mut HttpRequest, + identity: &Identity, + auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + runtime_components: &RuntimeComponents, + config_bag: &ConfigBag, ) -> Result<(), BoxError> { - todo!() + let operation_config = + SigV4Signer::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; + let mut settings = SigV4Signer::signing_settings(&operation_config); + override_session_token_name(&mut settings)?; + + SigV4Signer.sign_http_request( + request, + identity, + settings, + &operation_config, + runtime_components, + config_bag, + ) } } + + fn override_session_token_name(settings: &mut SigningSettings) -> Result<(), BoxError> { + let session_token_name_override = match settings.signature_location { + SignatureLocation::Headers => Some("x-amz-s3session-token"), + SignatureLocation::QueryParams => Some("X-Amz-S3session-Token"), + _ => { return Err(BoxError::from("`SignatureLocation` adds a new variant, which needs to be handled in a separate match arm")) }, + }; + settings.session_token_name_override = session_token_name_override; + Ok(()) + } } /// Supporting code for S3 Express identity cache @@ -80,11 +104,21 @@ pub(crate) mod identity_cache { /// Supporting code for S3 Express identity provider pub(crate) mod identity_provider { + use std::time::SystemTime; + use crate::s3_express::identity_cache::S3ExpressIdentityCache; + use crate::types::SessionCredentials; + use aws_credential_types::provider::error::CredentialsError; + use aws_credential_types::Credentials; + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::endpoint::EndpointResolverParams; use aws_smithy_runtime_api::client::identity::{ - IdentityCacheLocation, IdentityFuture, ResolveIdentity, + Identity, IdentityCacheLocation, IdentityFuture, ResolveCachedIdentity, ResolveIdentity, + }; + use aws_smithy_runtime_api::client::interceptors::SharedInterceptor; + use aws_smithy_runtime_api::client::runtime_components::{ + GetIdentityResolver, RuntimeComponents, }; - use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_types::config_bag::ConfigBag; #[derive(Debug)] @@ -95,10 +129,93 @@ pub(crate) mod identity_provider { #[derive(Default)] pub(crate) struct Builder; + impl TryFrom for Credentials { + type Error = BoxError; + + fn try_from(session_creds: SessionCredentials) -> Result { + Ok(Credentials::new( + session_creds.access_key_id, + session_creds.secret_access_key, + Some(session_creds.session_token), + Some(SystemTime::try_from(session_creds.expiration).map_err(|_| { + CredentialsError::unhandled( + "credential expiration time cannot be represented by a SystemTime", + ) + })?), + "s3express", + )) + } + } + impl DefaultS3ExpressIdentityProvider { pub(crate) fn builder() -> Builder { Builder } + + async fn identity<'a>( + &'a self, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, + ) -> Result { + let bucket_name = self.bucket_name(config_bag)?; + + let sigv4_identity_resolver = runtime_components + .identity_resolver(aws_runtime::auth::sigv4::SCHEME_ID) + .ok_or("identity resolver for sigv4 should be set for S3")?; + let _aws_identity = runtime_components + .identity_cache() + .resolve_cached_identity(sigv4_identity_resolver, runtime_components, config_bag) + .await?; + + // TODO(S3Express): use both `bucket_name` and `aws_identity` as part of `S3ExpressIdentityCache` implementation + + let express_session_credentials = self + .express_session_credentials(bucket_name, runtime_components, config_bag) + .await?; + + let data = Credentials::try_from(express_session_credentials)?; + + Ok(Identity::new(data.clone(), data.expiry())) + } + + fn bucket_name<'a>(&'a self, config_bag: &'a ConfigBag) -> Result<&'a str, BoxError> { + let params = config_bag + .load::() + .expect("endpoint resolver params must be set"); + let params = params + .get::() + .expect("`Params` should be wrapped in `EndpointResolverParams`"); + params + .bucket() + .ok_or("A bucket was not set in endpoint params".into()) + } + + async fn express_session_credentials<'a>( + &'a self, + bucket_name: &'a str, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, + ) -> Result { + let mut config_builder = crate::config::Builder::from_config_bag(config_bag); + + // inherits all runtime components from a current S3 operation but clears out + // out interceptors configured for that operation + let mut rc_builder = runtime_components.to_builder(); + rc_builder.set_interceptors(std::iter::empty::()); + config_builder.runtime_components = rc_builder; + + let client = crate::Client::from_conf(config_builder.build()); + let response = client + .create_session() + .bucket(bucket_name) + .session_mode(crate::types::SessionMode::ReadWrite) + .send() + .await?; + + response + .credentials + .ok_or("no session credentials in response".into()) + } } impl Builder { @@ -112,10 +229,10 @@ pub(crate) mod identity_provider { impl ResolveIdentity for DefaultS3ExpressIdentityProvider { fn resolve_identity<'a>( &'a self, - _runtime_components: &'a RuntimeComponents, - _config_bag: &'a ConfigBag, + runtime_components: &'a RuntimeComponents, + config_bag: &'a ConfigBag, ) -> IdentityFuture<'a> { - todo!() + IdentityFuture::new(async move { self.identity(runtime_components, config_bag).await }) } fn cache_location(&self) -> IdentityCacheLocation { diff --git a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs index 1e3dc5ad39..4330c72dfa 100644 --- a/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs +++ b/aws/rust-runtime/aws-runtime/src/auth/sigv4.rs @@ -72,7 +72,8 @@ impl SigV4Signer { Self } - fn settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { + /// Creates a [`SigningSettings`] from the given `operation_config`. + pub fn signing_settings(operation_config: &SigV4OperationSigningConfig) -> SigningSettings { super::settings(operation_config) } @@ -117,10 +118,11 @@ impl SigV4Signer { .expect("all required fields set")) } - fn extract_operation_config<'a>( + /// Extracts a [`SigV4OperationSigningConfig`]. + pub fn extract_operation_config<'a>( auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'a>, config_bag: &'a ConfigBag, - ) -> Result, SigV4SigningError> { + ) -> Result, BoxError> { let operation_config = config_bag .load::() .ok_or(SigV4SigningError::MissingOperationSigningConfig)?; @@ -141,28 +143,27 @@ impl SigV4Signer { } } } -} -impl Sign for SigV4Signer { - fn sign_http_request( + /// Signs the given `request`. + /// + /// This is a helper used by [`Sign::sign_http_request`] and will be useful if calling code + /// needs to pass a configured `settings`. + /// + /// TODO(S3Express): Make this method more user friendly, possibly returning a builder + /// instead of taking these input parameters. The builder will have a `sign` method that + /// does what this method body currently does. + pub fn sign_http_request( &self, request: &mut HttpRequest, identity: &Identity, - auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + settings: SigningSettings, + operation_config: &SigV4OperationSigningConfig, runtime_components: &RuntimeComponents, - config_bag: &ConfigBag, + #[allow(unused_variables)] config_bag: &ConfigBag, ) -> Result<(), BoxError> { - let operation_config = - Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; let request_time = runtime_components.time_source().unwrap_or_default().now(); - - if identity.data::().is_none() { - return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); - }; - - let settings = Self::settings(&operation_config); let signing_params = - Self::signing_params(settings, identity, &operation_config, request_time)?; + Self::signing_params(settings, identity, operation_config, request_time)?; let (signing_instructions, _signature) = { // A body that is already in memory can be signed directly. A body that is not in memory @@ -218,6 +219,35 @@ impl Sign for SigV4Signer { } } +impl Sign for SigV4Signer { + fn sign_http_request( + &self, + request: &mut HttpRequest, + identity: &Identity, + auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + runtime_components: &RuntimeComponents, + config_bag: &ConfigBag, + ) -> Result<(), BoxError> { + if identity.data::().is_none() { + return Err(SigV4SigningError::WrongIdentityType(identity.clone()).into()); + }; + + let operation_config = + Self::extract_operation_config(auth_scheme_endpoint_config, config_bag)?; + + let settings = Self::signing_settings(&operation_config); + + self.sign_http_request( + request, + identity, + settings, + &operation_config, + runtime_components, + config_bag, + ) + } +} + #[cfg(feature = "event-stream")] mod event_stream { use aws_sigv4::event_stream::{sign_empty_message, sign_message}; diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 5fae247dd2..5bc088ecf5 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -10,8 +10,8 @@ use crate::http_request::settings::UriPathNormalizationMode; use crate::http_request::sign::SignableRequest; use crate::http_request::uri_path_normalization::normalize_uri_path; use crate::http_request::url_escape::percent_encode_path; -use crate::http_request::PercentEncodingMode; use crate::http_request::{PayloadChecksumKind, SignableBody, SignatureLocation, SigningParams}; +use crate::http_request::{PercentEncodingMode, SigningSettings}; use crate::sign::v4::sha256_hex_string; use crate::SignatureVersion; use aws_smithy_http::query_writer::QueryWriter; @@ -218,7 +218,7 @@ impl<'a> CanonicalRequest<'a> { let creq = CanonicalRequest { method: req.method(), path, - params: Self::params(req.uri(), &values), + params: Self::params(req.uri(), &values, params.settings()), headers: canonical_headers, values, }; @@ -250,6 +250,11 @@ impl<'a> CanonicalRequest<'a> { Self::insert_host_header(&mut canonical_headers, req.uri()); + let token_header_name = params + .settings() + .session_token_name_override + .unwrap_or(header::X_AMZ_SECURITY_TOKEN); + if params.settings().signature_location == SignatureLocation::Headers { let creds = params .credentials() @@ -259,7 +264,7 @@ impl<'a> CanonicalRequest<'a> { if let Some(security_token) = creds.session_token() { let mut sec_header = HeaderValue::from_str(security_token)?; sec_header.set_sensitive(true); - canonical_headers.insert(header::X_AMZ_SECURITY_TOKEN, sec_header); + canonical_headers.insert(token_header_name, sec_header); } if params.settings().payload_checksum_kind == PayloadChecksumKind::XAmzSha256 { @@ -283,7 +288,7 @@ impl<'a> CanonicalRequest<'a> { } if params.settings().session_token_mode == SessionTokenMode::Exclude - && name == HeaderName::from_static(header::X_AMZ_SECURITY_TOKEN) + && name == HeaderName::from_static(token_header_name) { continue; } @@ -320,7 +325,11 @@ impl<'a> CanonicalRequest<'a> { } } - fn params(uri: &Uri, values: &SignatureValues<'_>) -> Option { + fn params( + uri: &Uri, + values: &SignatureValues<'_>, + settings: &SigningSettings, + ) -> Option { let mut params: Vec<(Cow<'_, str>, Cow<'_, str>)> = form_urlencoded::parse(uri.query().unwrap_or_default().as_bytes()).collect(); fn add_param<'a>(params: &mut Vec<(Cow<'a, str>, Cow<'a, str>)>, k: &'a str, v: &'a str) { @@ -345,7 +354,13 @@ impl<'a> CanonicalRequest<'a> { ); if let Some(security_token) = values.security_token { - add_param(&mut params, param::X_AMZ_SECURITY_TOKEN, security_token); + add_param( + &mut params, + settings + .session_token_name_override + .unwrap_or(param::X_AMZ_SECURITY_TOKEN), + security_token, + ); } } // Sort by param name, and then by param value diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs index 787619fc36..9b51345866 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs @@ -37,6 +37,10 @@ pub struct SigningSettings { /// canonical request. Other services require only it to be added after /// calculating the signature. pub session_token_mode: SessionTokenMode, + + /// Some services require an alternative session token header or query param instead of + /// `x-amz-security-token` or `X-Amz-Security-Token`. + pub session_token_name_override: Option<&'static str>, } /// HTTP payload checksum type @@ -133,6 +137,7 @@ impl Default for SigningSettings { excluded_headers, uri_path_normalization_mode: UriPathNormalizationMode::Enabled, session_token_mode: SessionTokenMode::Include, + session_token_name_override: None, } } } diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs index 2f27b3c4b7..dc8308dfda 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs @@ -297,7 +297,10 @@ fn calculate_signing_params<'a>( if let Some(security_token) = creds.session_token() { signing_params.push(( - param::X_AMZ_SECURITY_TOKEN, + params + .settings() + .session_token_name_override + .unwrap_or(param::X_AMZ_SECURITY_TOKEN), Cow::Owned(security_token.to_string()), )); } @@ -368,7 +371,10 @@ fn calculate_signing_headers<'a>( if let Some(security_token) = creds.session_token() { add_header( &mut headers, - header::X_AMZ_SECURITY_TOKEN, + params + .settings + .session_token_name_override + .unwrap_or(header::X_AMZ_SECURITY_TOKEN), security_token, true, ); diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt index 3a9b581549..7900b1ca56 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/EndpointBuiltInsDecorator.kt @@ -148,7 +148,6 @@ fun decoratorForBuiltIn( standardConfigParam( clientParamBuilder?.toConfigParam(builtIn, codegenContext.runtimeConfig) ?: ConfigParam.Builder() .toConfigParam(builtIn, codegenContext.runtimeConfig), - codegenContext, ) } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt index c877d4aac5..37b9a59b40 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/RegionDecorator.kt @@ -211,6 +211,10 @@ class RegionProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomi ) } + is ServiceConfig.BuilderFromConfigBag -> { + rustTemplate("${section.builder}.set_region(${section.config_bag}.load::<#{Region}>().cloned());", *codegenScope) + } + else -> emptySection } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt index 927a4c8dd2..4325aee01b 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt @@ -145,6 +145,11 @@ class UserAgentDecorator : ClientCodegenDecorator { ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate("${section.builder}.set_app_name(${section.config_bag}.load::<#{AppName}>().cloned());", *codegenScope) + } + is ServiceConfig.BuilderBuild -> writable { rust("layer.store_put(#T.clone());", ClientRustModule.Meta.toType().resolve("API_METADATA")) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt index 5c91ead09a..928a80f50c 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt @@ -52,6 +52,8 @@ class S3Decorator : ClientCodegenDecorator { private val logger: Logger = Logger.getLogger(javaClass.name) private val invalidXmlRootAllowList = setOf( + // To work around https://github.com/awslabs/aws-sdk-rust/issues/991 + ShapeId.from("com.amazonaws.s3#CreateSessionOutput"), // API returns GetObjectAttributes_Response_ instead of Output ShapeId.from("com.amazonaws.s3#GetObjectAttributesOutput"), ) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt index 544d30ea8f..5bcc870ae5 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt @@ -126,17 +126,9 @@ class OperationInputTestGenerator(_ctx: ClientCodegenContext, private val test: private val model = ctx.model private val instantiator = ClientInstantiator(ctx) - /** tests using S3 Express bucket names need to be disabled until the implementation is in place **/ - private fun EndpointTestCase.isSigV4S3Express() = - expect.endpoint.orNull()?.properties?.get("authSchemes")?.asArrayNode()?.orNull() - ?.map { it.expectObjectNode().expectStringMember("name").value }?.contains("sigv4-s3express") == true - fun generateInput(testOperationInput: EndpointTestOperationInput) = writable { val operationName = testOperationInput.operationName.toSnakeCase() - if (test.isSigV4S3Express()) { - Attribute.shouldPanic("not yet implemented").render(this) - } tokioTest(safeName("operation_input_test_$operationName")) { rustTemplate( """ diff --git a/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json b/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json new file mode 100644 index 0000000000..9bbc52fe05 --- /dev/null +++ b/aws/sdk/integration-tests/s3/tests/data/express/list-objects-v2.json @@ -0,0 +1,193 @@ +{ + "events": [ + { + "connection_id": 0, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session", + "headers": { + "amz-sdk-request": [ + "attempt=1; max=1" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-create-session-mode": [ + "ReadWrite" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ANOTREAL/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-create-session-mode;x-amz-date;x-amz-user-agent, Signature=1f63e649e5433837f97c037489613e036b98bff8cdf8d7bcf24b770d041ac0b9" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 0, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "x-amz-id-2": [ + "TQk2NSay" + ], + "x-amz-request-id": [ + "0033eada6b00018d568cbe9f0509499a7de17df8" + ], + "date": [ + "Mon, 29 Jan 2024 18:48:00 GMT" + ], + "content-type": [ + "application/xml" + ], + "content-length": [ + "333" + ], + "server": [ + "AmazonS3" + ] + } + } + } + } + } + }, + { + "connection_id": 0, + "action": { + "Data": { + "data": { + "Utf8": "\nTESTSESSIONTOKENTESTSECRETKEYASIARTESTID2024-01-29T18:53:01Z" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Request": { + "request": { + "uri": "https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?list-type=2", + "headers": { + "authorization": [ + "AWS4-HMAC-SHA256 Credential=ASIARTESTID/20090213/us-west-2/s3express/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-s3session-token;x-amz-user-agent, Signature=d1765aa7ec005607ba94fdda08c6739228d5ee14eb8316e80264c35649661a19" + ], + "x-amz-date": [ + "20090213T233130Z" + ], + "amz-sdk-request": [ + "attempt=1; max=3" + ], + "x-amz-user-agent": [ + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0" + ], + "x-amz-s3session-token": [ + "TESTSESSIONTOKEN" + ], + "user-agent": [ + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" + ] + }, + "method": "GET" + } + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 1, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "headers": { + "content-type": [ + "application/xml" + ], + "date": [ + "Mon, 29 Jan 2024 18:48:00 GMT" + ], + "x-amz-request-id": [ + "0033eada6b00018d568cbf350509f775295f94b5" + ], + "x-amz-id-2": [ + "IltTpRJVF1U" + ], + "server": [ + "AmazonS3" + ], + "content-length": [ + "520" + ], + "x-amz-bucket-region": [ + "us-west-2" + ] + } + } + } + } + } + }, + { + "connection_id": 1, + "action": { + "Data": { + "data": { + "Utf8": "\ns3express-test-bucket--usw2-az1--x-s311000falseCRC32"b357dc928b454965a8dd11716a37dab8"hello-world.txt2024-01-29T18:32:24.000Z14EXPRESS_ONEZONE" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 1, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + } + ], + "docs": "traffic recording of executing list-objects-v2 against an S3 Express One Zone bucket", + "version": "V0" +} diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs new file mode 100644 index 0000000000..cc0d99aa58 --- /dev/null +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -0,0 +1,131 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use std::time::{Duration, SystemTime}; + +use aws_config::Region; +use aws_sdk_s3::presigning::PresigningConfig; +use aws_sdk_s3::primitives::SdkBody; +use aws_sdk_s3::{Client, Config}; +use aws_smithy_runtime::client::http::test_util::dvr::ReplayingClient; +use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; +use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs; +use http::Uri; + +#[tokio::test] +async fn list_objects_v2() { + let _logs = capture_test_logs(); + + let http_client = + ReplayingClient::from_file("tests/data/express/list-objects-v2.json").unwrap(); + let config = aws_config::from_env() + .http_client(http_client.clone()) + .no_credentials() + .region("us-west-2") + .load() + .await; + let config = Config::from(&config) + .to_builder() + .with_test_defaults() + .build(); + let client = aws_sdk_s3::Client::from_conf(config); + + let result = client + .list_objects_v2() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .send() + .await; + dbg!(result).expect("success"); + + http_client + .validate_body_and_headers(Some(&["x-amz-s3session-token"]), "application/xml") + .await + .unwrap(); +} + +fn create_session_request() -> http::Request { + http::Request::builder() + .uri("https://s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com/?session") + .header("x-amz-create-session-mode", "ReadWrite") + .method("GET") + .body(SdkBody::empty()) + .unwrap() +} + +fn create_session_response() -> http::Response { + http::Response::builder() + .status(200) + .body(SdkBody::from( + r#" + + + TESTSESSIONTOKEN + TESTSECRETKEY + ASIARTESTID + 2024-01-29T18:53:01Z + + + "#, + )) + .unwrap() +} + +#[tokio::test] +async fn presigning() { + let http_client = StaticReplayClient::new(vec![ReplayEvent::new( + create_session_request(), + create_session_response(), + )]); + + let config = aws_sdk_s3::Config::builder() + .http_client(http_client) + .region(Region::new("us-west-2")) + .with_test_defaults() + .build(); + let client = Client::from_conf(config); + + let presigning_config = PresigningConfig::builder() + .start_time(SystemTime::UNIX_EPOCH + Duration::from_secs(1234567891)) + .expires_in(Duration::from_secs(30)) + .build() + .unwrap(); + + let presigned = client + .get_object() + .bucket("s3express-test-bucket--usw2-az1--x-s3") + .key("ferris.png") + .presigned(presigning_config) + .await + .unwrap(); + + let uri = presigned.uri().parse::().unwrap(); + + let pq = uri.path_and_query().unwrap(); + let path = pq.path(); + let query = pq.query().unwrap(); + let mut query_params: Vec<&str> = query.split('&').collect(); + query_params.sort(); + + pretty_assertions::assert_eq!( + "s3express-test-bucket--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com", + uri.authority().unwrap() + ); + assert_eq!("GET", presigned.method()); + assert_eq!("/ferris.png", path); + pretty_assertions::assert_eq!( + &[ + "X-Amz-Algorithm=AWS4-HMAC-SHA256", + "X-Amz-Credential=ASIARTESTID%2F20090213%2Fus-west-2%2Fs3express%2Faws4_request", + "X-Amz-Date=20090213T233131Z", + "X-Amz-Expires=30", + "X-Amz-S3session-Token=TESTSESSIONTOKEN", + "X-Amz-Signature=c09c93c7878184492cb960d59e148af932dff6b19609e63e3484599903d97e44", + "X-Amz-SignedHeaders=host", + "x-id=GetObject" + ][..], + &query_params + ); + assert_eq!(presigned.headers().count(), 0); +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt index 8b6153b9a6..a0a6202358 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt @@ -274,6 +274,21 @@ class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : Conf ) } + is ServiceConfig.BuilderFromConfigBag -> { + rustTemplate( + "${section.builder}.set_retry_config(${section.config_bag}.load::<#{RetryConfig}>().cloned());", + *codegenScope, + ) + rustTemplate( + "${section.builder}.set_timeout_config(${section.config_bag}.load::<#{TimeoutConfig}>().cloned());", + *codegenScope, + ) + rustTemplate( + "${section.builder}.set_retry_partition(${section.config_bag}.load::<#{RetryPartition}>().cloned());", + *codegenScope, + ) + } + else -> emptySection } } diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt index 96ddb2083c..adf7bfddb5 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/endpoint/ClientContextConfigCustomization.kt @@ -40,7 +40,7 @@ class ClientContextConfigCustomization(ctx: ClientCodegenContext) : ConfigCustom private val configParams = ctx.serviceShape.getTrait()?.parameters.orEmpty().toList() .map { (key, value) -> fromClientParam(key, value, ctx.symbolProvider, runtimeConfig) } - private val decorators = configParams.map { standardConfigParam(it, ctx) } + private val decorators = configParams.map { standardConfigParam(it) } companion object { fun toSymbol( diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt index 38fecb7e48..40b89549b3 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt @@ -80,19 +80,26 @@ sealed class ServiceConfig(name: String) : Section(name) { /** impl block of `ConfigBuilder` **/ object BuilderImpl : ServiceConfig("BuilderImpl") + // It is important to ensure through type system that each field added to config implements this injection, + // tracked by smithy-rs#3419 + /** - * Convert from a field in the builder to the final field in config + * Load a value from a config bag and store it in ConfigBuilder * e.g. * ```kotlin - * rust("""my_field: my_field.unwrap_or_else(||"default")""") + * rust("""builder.set_field(config_bag.load::().cloned())""") * ``` */ - object BuilderBuild : ServiceConfig("BuilderBuild") + data class BuilderFromConfigBag(val builder: String, val config_bag: String) : ServiceConfig("BuilderFromConfigBag") /** - * A section for customizing individual fields in the initializer of Config + * Convert from a field in the builder to the final field in config + * e.g. + * ```kotlin + * rust("""my_field: my_field.unwrap_or_else(||"default")""") + * ``` */ - object BuilderBuildExtras : ServiceConfig("BuilderBuildExtras") + object BuilderBuild : ServiceConfig("BuilderBuild") /** * A section for setting up a field to be used by ConfigOverrideRuntimePlugin @@ -203,10 +210,7 @@ fun loadFromConfigBag( * 2. convenience setter (non-optional) * 3. standard setter (&mut self) */ -fun standardConfigParam( - param: ConfigParam, - codegenContext: ClientCodegenContext, -): ConfigCustomization = +fun standardConfigParam(param: ConfigParam): ConfigCustomization = object : ConfigCustomization() { override fun section(section: ServiceConfig): Writable { return when (section) { @@ -235,6 +239,16 @@ fun standardConfigParam( ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + """ + ${section.builder}.set_${param.name}(${section.config_bag}.#{load_from_config_bag}); + """, + "load_from_config_bag" to loadFromConfigBag(param.type.name, param.newtype!!), + ) + } + else -> emptySection } } @@ -305,6 +319,26 @@ class ServiceConfigGenerator( ), ) + private fun builderFromConfigBag() = + writable { + val builderVar = "builder" + val configBagVar = "config_bag" + + docs("Constructs a config builder from the given `$configBagVar`, setting only fields stored in the config bag,") + docs("but not those in runtime components.") + Attribute.AllowUnused.render(this) + rustBlockTemplate( + "pub(crate) fn from_config_bag($configBagVar: &#{ConfigBag}) -> Self", + *codegenScope, + ) { + rust("let mut $builderVar = Self::new();") + customizations.forEach { + it.section(ServiceConfig.BuilderFromConfigBag(builderVar, configBagVar))(this) + } + rust("$builderVar") + } + } + private fun behaviorMv() = writable { val docs = """ @@ -451,6 +485,8 @@ class ServiceConfigGenerator( writer.rustBlock("impl Builder") { writer.docs("Constructs a config builder.") writer.rust("pub fn new() -> Self { Self::default() }") + + builderFromConfigBag()(this) customizations.forEach { it.section(ServiceConfig.BuilderImpl)(this) } @@ -522,9 +558,6 @@ class ServiceConfigGenerator( it.section(ServiceConfig.BuilderBuild)(this) } rustBlock("Config") { - customizations.forEach { - it.section(ServiceConfig.BuilderBuildExtras)(this) - } rustTemplate( """ config: #{Layer}::from(layer.clone()).with_name("$moduleUseName::config::Config").freeze(), diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt index 0c67acbf43..83c3b6dd6b 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/StalledStreamProtectionConfigCustomization.kt @@ -96,6 +96,14 @@ class StalledStreamProtectionConfigCustomization(codegenContext: ClientCodegenCo ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_stalled_stream_protection(${section.config_bag}.load::<#{StalledStreamProtectionConfig}>().cloned());", + *codegenScope, + ) + } + else -> emptySection } } diff --git a/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt b/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt index 4e9e88f414..e80d29efea 100644 --- a/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt +++ b/codegen-client/src/test/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGeneratorTest.kt @@ -15,6 +15,7 @@ import software.amazon.smithy.rust.codegen.client.testutil.clientIntegrationTest import software.amazon.smithy.rust.codegen.core.rustlang.Writable import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization import software.amazon.smithy.rust.codegen.core.testutil.BasicTestModels @@ -24,6 +25,11 @@ import software.amazon.smithy.rust.codegen.core.util.lookup import software.amazon.smithy.rust.codegen.core.util.toPascalCase internal class ServiceConfigGeneratorTest { + private fun codegenScope(rc: RuntimeConfig): Array> = + arrayOf( + "ConfigBag" to RuntimeType.configBag(rc), + ) + @Test fun `idempotency token when used`() { fun model(trait: String) = @@ -124,6 +130,20 @@ internal class ServiceConfigGeneratorTest { ) } + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + """ + ${section.builder} = ${section.builder}.config_field(${section.config_bag}.load::<#{T}>().map(|u| u.0).unwrap()); + """, + "T" to + configParamNewtype( + "config_field".toPascalCase(), RuntimeType.U64.toSymbol(), + codegenContext.runtimeConfig, + ), + ) + } + else -> emptySection } } @@ -174,6 +194,34 @@ internal class ServiceConfigGeneratorTest { assert_eq!(config.runtime_plugins.len(), 1); """, ) + + unitTest( + "builder_from_config_bag", + """ + use aws_smithy_runtime::client::retries::RetryPartition; + use aws_smithy_types::config_bag::ConfigBag; + use aws_smithy_types::config_bag::Layer; + use aws_smithy_types::retry::RetryConfig; + use aws_smithy_types::timeout::TimeoutConfig; + + let mut layer = Layer::new("test"); + layer.store_put(crate::config::ConfigField(0)); + layer.store_put(RetryConfig::disabled()); + layer.store_put(crate::config::StalledStreamProtectionConfig::disabled()); + layer.store_put(TimeoutConfig::builder().build()); + layer.store_put(RetryPartition::new("test")); + + let config_bag = ConfigBag::of_layers(vec![layer]); + let builder = crate::config::Builder::from_config_bag(&config_bag); + let config = builder.build(); + + assert_eq!(config.config_field(), 0); + assert!(config.retry_config().is_some()); + assert!(config.stalled_stream_protection().is_some()); + assert!(config.timeout_config().is_some()); + assert!(config.retry_partition().is_some()); + """, + ) } } } diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs index 4480779a00..5d86dd5405 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs @@ -386,6 +386,14 @@ impl RuntimeComponents { RuntimeComponentsBuilder::new(name) } + /// Clones and converts this [`RuntimeComponents`] into a [`RuntimeComponentsBuilder`]. + pub fn to_builder(&self) -> RuntimeComponentsBuilder { + RuntimeComponentsBuilder::from_runtime_components( + self.clone(), + "RuntimeComponentsBuilder::from_runtime_components", + ) + } + /// Returns the auth scheme option resolver. pub fn auth_scheme_option_resolver(&self) -> SharedAuthSchemeOptionResolver { self.auth_scheme_option_resolver.value.clone() @@ -497,6 +505,26 @@ impl RuntimeComponents { } impl RuntimeComponentsBuilder { + /// Creates a new [`RuntimeComponentsBuilder`], inheriting all fields from the given + /// [`RuntimeComponents`]. + pub fn from_runtime_components(rc: RuntimeComponents, builder_name: &'static str) -> Self { + Self { + builder_name, + auth_scheme_option_resolver: Some(rc.auth_scheme_option_resolver), + http_client: rc.http_client, + endpoint_resolver: Some(rc.endpoint_resolver), + auth_schemes: rc.auth_schemes, + identity_cache: Some(rc.identity_cache), + identity_resolvers: Some(rc.identity_resolvers), + interceptors: rc.interceptors, + retry_classifiers: rc.retry_classifiers, + retry_strategy: Some(rc.retry_strategy), + time_source: rc.time_source, + sleep_impl: rc.sleep_impl, + config_validators: rc.config_validators, + } + } + /// Returns the auth scheme option resolver. pub fn auth_scheme_option_resolver(&self) -> Option { self.auth_scheme_option_resolver