From 3233dbe2b4c37187d81b5972965be6686ab4839f Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Fri, 9 Feb 2024 22:42:00 -0600 Subject: [PATCH] Add placeholder types for S3 Express and enable control flow to be redirected for S3 Express use case (#3386) ## Motivation and Context This PR is the first in the series to support the S3 Express feature in the Rust SDK. The work will be done in the feature branch, and once it is code complete, the branch will be merged to main. ## Description This PR adds placeholder types for S3 Express and enables control flow to be redirected for S3 Express use case. For instance, if we run the following example code against a generated SDK from this PR: ```rust let shared_config = aws_config::from_env().region(aws_sdk_s3::config::Region::new("us-east-1")).load().await; let client = aws_sdk_s3::Client::new(&shared_config); client.list_objects_v2().bucket("testbucket--use1-az4--x-s3";).send().await.unwrap(); ``` it will end up ``` thread 's3_express' panicked at 'not yet implemented', /Users/awsaito/src/smithy-rs/aws/sdk/build/aws-sdk/sdk/s3/src/s3_express.rs:104:13 ``` which points to ``` impl ProvideCredentials for DefaultS3ExpressIdentityProvider { fn provide_credentials<'a>(&'a self) -> aws_credential_types::provider::future::ProvideCredentials<'a> where Self: 'a, { todo!() <--- } } ``` ### Implementation decisions - `DefaultS3ExpressIdentityProvider` has an accompanying identity cache. That identity cache cannot be configured by customers so it makes sense for the provider itself to internally own it. In that case, we do NOT want to use the identity cache stored in `RuntimeComponents`, since it interferes with the S3 Express's caching policy. To that end, I added an enum `CacheLocation` to `SharedIdentityResolver` (it already had the `cache_partition` field so it was kind of aware of caching). - Two reasons why `CacheLocation` is added to `SharedIdentityResolver`, but not to individual, concrete `IdentityResolver`s. One is `SharedIdentityResolver` was already cache aware, as mentioned above. The other is that it is more flexible that way; The cache location is not tied to a type of identity resolver, but we can select it when creating a `SharedIdentityResolver`. - I considered but did not add a field `cacheable` to `Identity` since I wanted to keep `Identity` as plain data, keeping the concept of "caching" somewhere outside. - I've added a separate `Config` method, `set_express_credentials_provider`, to override credentials provider for S3 Express. There are other SDKs (e.g. [Ruby](https://www.rubydoc.info/gems/aws-sdk-s3/Aws/S3/Client)) that follow this style and it makes it clear to the customers that this is the method to use when overriding the express credentials provider. The existing `set_credentials_provider`, given its input type, cannot tell whether a passed-in credentials provider is for a regular `sigv4` or for S3 Express. ## Testing Only verified that control flow could be altered for an S3 Express use case, as shown above. Further testing will be added in subsequent PRs. ## Checklist I am planning to include in `CHANGELOG.next.toml` a user guide for S3 Express once the feature branch `ysaito/s3express` is ready to be merged to main. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti Co-authored-by: AWS SDK Rust Bot Co-authored-by: AWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com> Co-authored-by: Zelda Hessler --- aws/rust-runtime/aws-inlineable/src/lib.rs | 5 + .../aws-inlineable/src/s3_express.rs | 125 ++++++++++++ .../smithy/rustsdk/AwsCodegenDecorator.kt | 2 + .../customize/s3/S3ExpressDecorator.kt | 184 ++++++++++++++++++ .../endpoints/OperationInputTestGenerator.kt | 8 + .../ServiceRuntimePluginGenerator.kt | 8 + .../src/client/identity.rs | 28 +++ .../src/client/orchestrator/auth.rs | 12 +- 8 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 aws/rust-runtime/aws-inlineable/src/s3_express.rs create mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt diff --git a/aws/rust-runtime/aws-inlineable/src/lib.rs b/aws/rust-runtime/aws-inlineable/src/lib.rs index 0ae9627703..88459ef03d 100644 --- a/aws/rust-runtime/aws-inlineable/src/lib.rs +++ b/aws/rust-runtime/aws-inlineable/src/lib.rs @@ -31,6 +31,11 @@ pub mod presigning; /// Presigning interceptors pub mod presigning_interceptors; +// This module uses module paths that assume the target crate to which it is copied, e.g. +// `crate::config::endpoint::Params`. If included into `aws-inlineable`, this module would +// fail to compile. +// pub mod s3_express; + /// Special logic for extracting request IDs from S3's responses. #[allow(dead_code)] pub mod s3_request_id; diff --git a/aws/rust-runtime/aws-inlineable/src/s3_express.rs b/aws/rust-runtime/aws-inlineable/src/s3_express.rs new file mode 100644 index 0000000000..76b8a95093 --- /dev/null +++ b/aws/rust-runtime/aws-inlineable/src/s3_express.rs @@ -0,0 +1,125 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/// Supporting code for S3 Express auth +pub(crate) mod auth { + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::auth::{ + AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, Sign, + }; + use aws_smithy_runtime_api::client::identity::{Identity, SharedIdentityResolver}; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_runtime_api::client::runtime_components::{ + GetIdentityResolver, RuntimeComponents, + }; + use aws_smithy_types::config_bag::ConfigBag; + + /// Auth scheme ID for S3 Express. + pub(crate) const SCHEME_ID: AuthSchemeId = AuthSchemeId::new("sigv4-s3express"); + + /// S3 Express auth scheme. + #[derive(Debug, Default)] + pub(crate) struct S3ExpressAuthScheme { + signer: S3ExpressSigner, + } + + impl S3ExpressAuthScheme { + /// Creates a new `S3ExpressAuthScheme`. + pub(crate) fn new() -> Self { + Default::default() + } + } + + impl AuthScheme for S3ExpressAuthScheme { + fn scheme_id(&self) -> AuthSchemeId { + SCHEME_ID + } + + fn identity_resolver( + &self, + identity_resolvers: &dyn GetIdentityResolver, + ) -> Option { + identity_resolvers.identity_resolver(self.scheme_id()) + } + + fn signer(&self) -> &dyn Sign { + &self.signer + } + } + + /// S3 Express signer. + #[derive(Debug, Default)] + pub(crate) struct S3ExpressSigner; + + impl Sign for S3ExpressSigner { + fn sign_http_request( + &self, + _request: &mut HttpRequest, + _identity: &Identity, + _auth_scheme_endpoint_config: AuthSchemeEndpointConfig<'_>, + _runtime_components: &RuntimeComponents, + _config_bag: &ConfigBag, + ) -> Result<(), BoxError> { + todo!() + } + } +} + +/// Supporting code for S3 Express identity cache +pub(crate) mod identity_cache { + /// The caching implementation for S3 Express identity. + /// + /// While customers can either disable S3 Express itself or provide a custom S3 Express identity + /// provider, configuring S3 Express identity cache is not supported. Thus, this is _the_ + /// implementation of S3 Express identity cache. + #[derive(Debug)] + pub(crate) struct S3ExpressIdentityCache; +} + +/// Supporting code for S3 Express identity provider +pub(crate) mod identity_provider { + use crate::s3_express::identity_cache::S3ExpressIdentityCache; + use aws_smithy_runtime_api::client::identity::{ + IdentityCacheLocation, IdentityFuture, ResolveIdentity, + }; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; + use aws_smithy_types::config_bag::ConfigBag; + + #[derive(Debug)] + pub(crate) struct DefaultS3ExpressIdentityProvider { + _cache: S3ExpressIdentityCache, + } + + #[derive(Default)] + pub(crate) struct Builder; + + impl DefaultS3ExpressIdentityProvider { + pub(crate) fn builder() -> Builder { + Builder + } + } + + impl Builder { + pub(crate) fn build(self) -> DefaultS3ExpressIdentityProvider { + DefaultS3ExpressIdentityProvider { + _cache: S3ExpressIdentityCache, + } + } + } + + impl ResolveIdentity for DefaultS3ExpressIdentityProvider { + fn resolve_identity<'a>( + &'a self, + _runtime_components: &'a RuntimeComponents, + _config_bag: &'a ConfigBag, + ) -> IdentityFuture<'a> { + todo!() + } + + fn cache_location(&self) -> IdentityCacheLocation { + IdentityCacheLocation::IdentityResolver + } + } +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 3bc616a9b8..2c13e4b2b7 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -18,6 +18,7 @@ import software.amazon.smithy.rustsdk.customize.glacier.GlacierDecorator import software.amazon.smithy.rustsdk.customize.onlyApplyTo import software.amazon.smithy.rustsdk.customize.route53.Route53Decorator import software.amazon.smithy.rustsdk.customize.s3.S3Decorator +import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator import software.amazon.smithy.rustsdk.customize.sso.SSODecorator @@ -64,6 +65,7 @@ val DECORATORS: List = Route53Decorator().onlyApplyTo("com.amazonaws.route53#AWSDnsV20130401"), "com.amazonaws.s3#AmazonS3".applyDecorators( S3Decorator(), + S3ExpressDecorator(), S3ExtendedRequestIdDecorator(), ), S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt new file mode 100644 index 0000000000..8529fdfc47 --- /dev/null +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpressDecorator.kt @@ -0,0 +1,184 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk.customize.s3 + +import software.amazon.smithy.aws.traits.auth.SigV4Trait +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport +import software.amazon.smithy.rust.codegen.client.smithy.customize.AuthSchemeOption +import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rustsdk.AwsRuntimeType +import software.amazon.smithy.rustsdk.InlineAwsDependency + +class S3ExpressDecorator : ClientCodegenDecorator { + override val name: String = "S3ExpressDecorator" + override val order: Byte = 0 + + private fun sigv4S3Express() = + writable { + rust( + "#T", + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + ) + } + + override fun authOptions( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, + baseAuthSchemeOptions: List, + ): List = + baseAuthSchemeOptions + + AuthSchemeOption.StaticAuthSchemeOption( + SigV4Trait.ID, + listOf(sigv4S3Express()), + ) + + override fun serviceRuntimePluginCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = + baseCustomizations + listOf(S3ExpressServiceRuntimePluginCustomization(codegenContext)) + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + listOf(S3ExpressIdentityProviderConfig(codegenContext)) +} + +private class S3ExpressServiceRuntimePluginCustomization(codegenContext: ClientCodegenContext) : + ServiceRuntimePluginCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope by lazy { + arrayOf( + "DefaultS3ExpressIdentityProvider" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("identity_provider::DefaultS3ExpressIdentityProvider"), + "IdentityCacheLocation" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::IdentityCacheLocation"), + "S3ExpressAuthScheme" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::S3ExpressAuthScheme"), + "S3_EXPRESS_SCHEME_ID" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + "SharedAuthScheme" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::auth::SharedAuthScheme"), + "SharedCredentialsProvider" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::SharedCredentialsProvider"), + ), + "SharedIdentityResolver" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::SharedIdentityResolver"), + ) + } + + override fun section(section: ServiceRuntimePluginSection): Writable = + writable { + when (section) { + is ServiceRuntimePluginSection.RegisterRuntimeComponents -> { + section.registerAuthScheme(this) { + rustTemplate( + "#{SharedAuthScheme}::new(#{S3ExpressAuthScheme}::new())", + *codegenScope, + ) + } + + section.registerIdentityResolver( + this, + writable { + rustTemplate("#{S3_EXPRESS_SCHEME_ID}", *codegenScope) + }, + writable { + rustTemplate("#{DefaultS3ExpressIdentityProvider}::builder().build()", *codegenScope) + }, + ) + } + + else -> {} + } + } +} + +class S3ExpressIdentityProviderConfig(codegenContext: ClientCodegenContext) : ConfigCustomization() { + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "IdentityCacheLocation" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::IdentityCacheLocation"), + "ProvideCredentials" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::ProvideCredentials"), + ), + "SharedCredentialsProvider" to + configReexport( + AwsRuntimeType.awsCredentialTypes(runtimeConfig) + .resolve("provider::SharedCredentialsProvider"), + ), + "SharedIdentityResolver" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::identity::SharedIdentityResolver"), + "S3_EXPRESS_SCHEME_ID" to + RuntimeType.forInlineDependency( + InlineAwsDependency.forRustFile("s3_express"), + ).resolve("auth::SCHEME_ID"), + ) + + override fun section(section: ServiceConfig) = + writable { + when (section) { + ServiceConfig.BuilderImpl -> { + rustTemplate( + """ + /// Sets the credentials provider for S3 Express One Zone + pub fn express_credentials_provider(mut self, credentials_provider: impl #{ProvideCredentials} + 'static) -> Self { + self.set_express_credentials_provider(#{Some}(#{SharedCredentialsProvider}::new(credentials_provider))); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Sets the credentials provider for S3 Express One Zone + pub fn set_express_credentials_provider(&mut self, credentials_provider: #{Option}<#{SharedCredentialsProvider}>) -> &mut Self { + if let #{Some}(credentials_provider) = credentials_provider { + self.runtime_components.set_identity_resolver(#{S3_EXPRESS_SCHEME_ID}, credentials_provider); + } + self + } + """, + *codegenScope, + ) + } + + else -> emptySection + } + } +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt index 5bcc870ae5..544d30ea8f 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/endpoints/OperationInputTestGenerator.kt @@ -126,9 +126,17 @@ class OperationInputTestGenerator(_ctx: ClientCodegenContext, private val test: private val model = ctx.model private val instantiator = ClientInstantiator(ctx) + /** tests using S3 Express bucket names need to be disabled until the implementation is in place **/ + private fun EndpointTestCase.isSigV4S3Express() = + expect.endpoint.orNull()?.properties?.get("authSchemes")?.asArrayNode()?.orNull() + ?.map { it.expectObjectNode().expectStringMember("name").value }?.contains("sigv4-s3express") == true + fun generateInput(testOperationInput: EndpointTestOperationInput) = writable { val operationName = testOperationInput.operationName.toSnakeCase() + if (test.isSigV4S3Express()) { + Attribute.shouldPanic("not yet implemented").render(this) + } tokioTest(safeName("operation_input_test_$operationName")) { rustTemplate( """ diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt index 66095c1555..699e5bff19 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/ServiceRuntimePluginGenerator.kt @@ -69,6 +69,14 @@ sealed class ServiceRuntimePluginSection(name: String) : Section(name) { ) { writer.rust("runtime_components.push_retry_classifier(#T);", classifier) } + + fun registerIdentityResolver( + writer: RustWriter, + schemeId: Writable, + identityResolver: Writable, + ) { + writer.rust("runtime_components.set_identity_resolver(#T, #T);", schemeId, identityResolver) + } } } typealias ServiceRuntimePluginCustomization = NamedCustomization diff --git a/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs b/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs index f0e972675a..2671a883a0 100644 --- a/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs +++ b/rust-runtime/aws-smithy-runtime-api/src/client/identity.rs @@ -159,6 +159,30 @@ pub trait ResolveIdentity: Send + Sync + Debug { fn fallback_on_interrupt(&self) -> Option { None } + + /// Returns the location of an identity cache associated with this identity resolver. + /// + /// By default, identity resolvers will use the identity cache stored in runtime components. + /// Implementing types can change the cache location if they want to. Refer to [`IdentityCacheLocation`] + /// explaining why a concrete identity resolver might want to change the cache location. + fn cache_location(&self) -> IdentityCacheLocation { + IdentityCacheLocation::RuntimeComponents + } +} + +/// Cache location for identity caching. +/// +/// Identities are usually cached in the identity cache owned by [`RuntimeComponents`]. However, +/// we do have identities whose caching mechanism is internally managed by their identity resolver, +/// in which case we want to avoid the `RuntimeComponents`-owned identity cache interfering with +/// the internal caching policy. +#[non_exhaustive] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum IdentityCacheLocation { + /// Indicates the identity cache is owned by [`RuntimeComponents`]. + RuntimeComponents, + /// Indicates the identity cache is internally managed by the identity resolver. + IdentityResolver, } /// Container for a shared identity resolver. @@ -194,6 +218,10 @@ impl ResolveIdentity for SharedIdentityResolver { ) -> IdentityFuture<'a> { self.inner.resolve_identity(runtime_components, config_bag) } + + fn cache_location(&self) -> IdentityCacheLocation { + self.inner.cache_location() + } } impl_shared_conversions!(convert SharedIdentityResolver from ResolveIdentity using SharedIdentityResolver::new); diff --git a/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs b/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs index 98a39a9a43..0bf5ecf1a7 100644 --- a/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs +++ b/rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs @@ -4,12 +4,14 @@ */ use crate::client::auth::no_auth::NO_AUTH_SCHEME_ID; +use crate::client::identity::IdentityCache; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::auth::{ AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, AuthSchemeOptionResolverParams, ResolveAuthSchemeOptions, }; -use aws_smithy_runtime_api::client::identity::ResolveCachedIdentity; +use aws_smithy_runtime_api::client::identity::ResolveIdentity; +use aws_smithy_runtime_api::client::identity::{IdentityCacheLocation, ResolveCachedIdentity}; use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_types::config_bag::ConfigBag; @@ -135,7 +137,13 @@ pub(super) async fn orchestrate_auth( if let Some(auth_scheme) = runtime_components.auth_scheme(scheme_id) { // Use the resolved auth scheme to resolve an identity if let Some(identity_resolver) = auth_scheme.identity_resolver(runtime_components) { - let identity_cache = runtime_components.identity_cache(); + let identity_cache = if identity_resolver.cache_location() + == IdentityCacheLocation::RuntimeComponents + { + runtime_components.identity_cache() + } else { + IdentityCache::no_cache() + }; let signer = auth_scheme.signer(); trace!( auth_scheme = ?auth_scheme,