Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow list-objects-v2 to run against an S3 Express bucket #3388

Merged
merged 35 commits into from
Feb 17, 2024
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
dbcfe73
Add placeholder types for S3 Express
ysaito1001 Jan 25, 2024
714aaad
Enable control flow to be altered for S3 Express
ysaito1001 Jan 25, 2024
9086db2
Fix lint-related failures in CI
ysaito1001 Jan 25, 2024
1f07f92
Temporarily disable S3 Express endpoint tests
ysaito1001 Jan 25, 2024
13c3c5a
Update aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk…
ysaito1001 Jan 29, 2024
38ce80f
Make s3_express inlineable modules `pub(crate)`
ysaito1001 Jan 29, 2024
ef1e1be
Remove unnecessary `set_shared_identity_resolver`
ysaito1001 Jan 29, 2024
9e4542b
Allow list-objects-v2 to run against S3 Express bucket
ysaito1001 Jan 29, 2024
67223e3
Add recording test for S3 Express list-objects-v2
ysaito1001 Jan 29, 2024
b30abb2
Exclude `s3_express` from aws-inlineable's module tree
ysaito1001 Jan 30, 2024
2fcd2f7
Enable S3 Express endpoint tests
ysaito1001 Jan 30, 2024
21ad106
Update aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk…
ysaito1001 Feb 7, 2024
9acaf29
Support presigning for S3 Express
ysaito1001 Feb 8, 2024
ec08ae9
Add integration test for S3 Express `get_object` presigning
ysaito1001 Feb 8, 2024
c2a2708
Make `cache_location` a method on `ResolveIdentity`
ysaito1001 Feb 8, 2024
15e89c6
Merge branch 'ysaito/s3express' into s3express-add-placeholders
ysaito1001 Feb 8, 2024
c4df9c7
Error instead of panic on missing bucket name
ysaito1001 Feb 8, 2024
0e405e1
Error instead of panic in `add_token_to_request`
ysaito1001 Feb 9, 2024
e47157f
Update comment for `invalidXmlRootAllowList`
ysaito1001 Feb 9, 2024
5a4e3b2
Implement `from_runtime_components` instead of `From` trait
ysaito1001 Feb 9, 2024
51518f9
Allow `config::Builder` to be created from `ConfigBag`
ysaito1001 Feb 9, 2024
573ef36
Add `TODO` to make `sign_http_request` user-friendly
ysaito1001 Feb 9, 2024
457f6b8
Add a reference to docs for `IdentityCacheLocation`
ysaito1001 Feb 9, 2024
f569231
Use rustTemplate instead of rustBlockTemplate for readability
ysaito1001 Feb 10, 2024
c3d754f
Exclude `s3_express` from `aws-inlineable`'s module tree
ysaito1001 Feb 10, 2024
5b27e4d
Merge branch 'main' into s3express-add-placeholders
ysaito1001 Feb 10, 2024
99a9112
Merge branch 's3express-add-placeholders' into s3express-allow-list-o…
ysaito1001 Feb 10, 2024
3233dbe
Add placeholder types for S3 Express and enable control flow to be re…
ysaito1001 Feb 10, 2024
63dcb93
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
daea66b
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
6c795b9
Merge branch 'ysaito/s3express' into s3express-allow-list-objects-v2-…
ysaito1001 Feb 10, 2024
d2354bd
Anonymize tokens in S3 Express recording test
ysaito1001 Feb 12, 2024
027c2cd
Remove `From<RuntimeComponents> for RuntimeComponentsBuilder`
ysaito1001 Feb 14, 2024
830ef02
Add a setting to replace session token name
ysaito1001 Feb 15, 2024
97cecd3
Add a tracking issue for `BuilderFromConfigBag`
ysaito1001 Feb 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Enable control flow to be altered for S3 Express
This commit updates parts of the orchestrator so that when an S3 Express
bucket name is passed, control flow will be directed to placeholder types
added in the previous commit.
ysaito1001 committed Jan 25, 2024
commit 714aaad3a9aaaa4bee4a57ffccfcbd8f77954ec4
4 changes: 4 additions & 0 deletions aws/rust-runtime/aws-inlineable/src/lib.rs
Original file line number Diff line number Diff line change
@@ -31,6 +31,10 @@ pub mod presigning;
/// Presigning interceptors
pub mod presigning_interceptors;

/// Supporting types for S3 Express.
#[allow(dead_code)]
pub mod s3_express;

/// Special logic for extracting request IDs from S3's responses.
#[allow(dead_code)]
pub mod s3_request_id;
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ import software.amazon.smithy.rustsdk.customize.glacier.GlacierDecorator
import software.amazon.smithy.rustsdk.customize.onlyApplyTo
import software.amazon.smithy.rustsdk.customize.route53.Route53Decorator
import software.amazon.smithy.rustsdk.customize.s3.S3Decorator
import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator
import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator
import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator
import software.amazon.smithy.rustsdk.customize.sso.SSODecorator
@@ -64,6 +65,7 @@ val DECORATORS: List<ClientCodegenDecorator> =
Route53Decorator().onlyApplyTo("com.amazonaws.route53#AWSDnsV20130401"),
"com.amazonaws.s3#AmazonS3".applyDecorators(
S3Decorator(),
S3ExpressDecorator(),
S3ExtendedRequestIdDecorator(),
),
S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"),
Original file line number Diff line number Diff line change
@@ -69,6 +69,14 @@ sealed class ServiceRuntimePluginSection(name: String) : Section(name) {
) {
writer.rust("runtime_components.push_retry_classifier(#T);", classifier)
}

fun registerSharedIdentityResolver(
writer: RustWriter,
schemeId: Writable,
identityResolver: Writable,
) {
writer.rust("runtime_components.set_shared_identity_resolver(#T, #T);", schemeId, identityResolver)
}
}
}
typealias ServiceRuntimePluginCustomization = NamedCustomization<ServiceRuntimePluginSection>
31 changes: 31 additions & 0 deletions rust-runtime/aws-smithy-runtime-api/src/client/identity.rs
Original file line number Diff line number Diff line change
@@ -161,19 +161,45 @@ pub trait ResolveIdentity: Send + Sync + Debug {
}
}

/// Cache location for identity caching.
///
/// Identities are usually cached in the identity cache owned by [`RuntimeComponents`]. However,
/// we do have identities whose caching mechanism is internally managed by their identity resolver,
/// in which case we want to avoid the `RuntimeComponents`-owned identity cache interfering with
/// the internal caching policy.
#[non_exhaustive]
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum IdentityCacheLocation {
/// Indicates the identity cache is owned by [`RuntimeComponents`].
RuntimeComponents,
/// Indicates the identity cache is internally managed by the identity resolver.
IdentityResolver,
}

/// Container for a shared identity resolver.
#[derive(Clone, Debug)]
pub struct SharedIdentityResolver {
inner: Arc<dyn ResolveIdentity>,
cache_partition: IdentityCachePartition,
cache_location: IdentityCacheLocation,
}

impl SharedIdentityResolver {
/// Creates a new [`SharedIdentityResolver`] from the given resolver.
pub fn new(resolver: impl ResolveIdentity + 'static) -> Self {
Self::new_with_cache_location(resolver, IdentityCacheLocation::RuntimeComponents)
}

/// Creates a new [`SharedIdentityResolver`] from the given resolver with the additional argument
/// specifying where the identity cache location is.
pub fn new_with_cache_location(
resolver: impl ResolveIdentity + 'static,
cache_location: IdentityCacheLocation,
) -> Self {
Self {
inner: Arc::new(resolver),
cache_partition: IdentityCachePartition::new(),
cache_location,
}
}

@@ -184,6 +210,11 @@ impl SharedIdentityResolver {
pub fn cache_partition(&self) -> IdentityCachePartition {
self.cache_partition
}

/// Returns where identities retrieved by this resolver are cached.
pub fn cache_location(&self) -> IdentityCacheLocation {
self.cache_location
}
}

impl ResolveIdentity for SharedIdentityResolver {
Original file line number Diff line number Diff line change
@@ -612,11 +612,23 @@ impl RuntimeComponentsBuilder {
&mut self,
scheme_id: AuthSchemeId,
identity_resolver: impl ResolveIdentity + 'static,
) -> &mut Self {
self.set_shared_identity_resolver(scheme_id, identity_resolver.into_shared())
}

/// Sets the identity resolver for a given `scheme_id`.
///
/// This is effectively the same as [`Self::set_identity_resolver`] but takes a [`SharedIdentityResolver`]
/// instead.
pub fn set_shared_identity_resolver(
&mut self,
scheme_id: AuthSchemeId,
shared_identity_resolver: SharedIdentityResolver,
) -> &mut Self {
let mut resolvers = self.identity_resolvers.take().unwrap_or_default();
resolvers.insert(
scheme_id,
Tracked::new(self.builder_name, identity_resolver.into_shared()),
Tracked::new(self.builder_name, shared_identity_resolver),
);
self.identity_resolvers = Some(resolvers);
self
11 changes: 9 additions & 2 deletions rust-runtime/aws-smithy-runtime/src/client/orchestrator/auth.rs
Original file line number Diff line number Diff line change
@@ -4,12 +4,13 @@
*/

use crate::client::auth::no_auth::NO_AUTH_SCHEME_ID;
use crate::client::identity::IdentityCache;
use aws_smithy_runtime_api::box_error::BoxError;
use aws_smithy_runtime_api::client::auth::{
AuthScheme, AuthSchemeEndpointConfig, AuthSchemeId, AuthSchemeOptionResolverParams,
ResolveAuthSchemeOptions,
};
use aws_smithy_runtime_api::client::identity::ResolveCachedIdentity;
use aws_smithy_runtime_api::client::identity::{IdentityCacheLocation, ResolveCachedIdentity};
use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
use aws_smithy_types::config_bag::ConfigBag;
@@ -135,7 +136,13 @@ pub(super) async fn orchestrate_auth(
if let Some(auth_scheme) = runtime_components.auth_scheme(scheme_id) {
// Use the resolved auth scheme to resolve an identity
if let Some(identity_resolver) = auth_scheme.identity_resolver(runtime_components) {
let identity_cache = runtime_components.identity_cache();
let identity_cache = if identity_resolver.cache_location()
== IdentityCacheLocation::RuntimeComponents
{
runtime_components.identity_cache()
} else {
IdentityCache::no_cache()
};
let signer = auth_scheme.signer();
trace!(
auth_scheme = ?auth_scheme,