Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for S3 Express One Zone #3465

Merged
merged 22 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
39be6ce
Add placeholder types for S3 Express and enable control flow to be re…
ysaito1001 Feb 10, 2024
beb472b
Merge branch 'main' into ysaito/s3express
ysaito1001 Feb 10, 2024
486b91d
Allow list-objects-v2 to run against an S3 Express bucket (#3388)
ysaito1001 Feb 17, 2024
57c95aa
Merge branch 'main' into ysaito/s3express
ysaito1001 Feb 17, 2024
0a75b41
Add S3 Express identity cache (#3390)
ysaito1001 Feb 20, 2024
301eb9d
Merge branch 'main' into ysaito/s3express
ysaito1001 Feb 20, 2024
dc8626f
Anonymize test session token
ysaito1001 Feb 20, 2024
9d35452
Allow S3 Express to use CRC32 for default request checksum (#3432)
ysaito1001 Feb 26, 2024
721fadd
Add support to disable S3 Express session auth (#3433)
ysaito1001 Feb 27, 2024
01a0838
Add test for overriding S3 Express credentials provider (#3459)
ysaito1001 Mar 5, 2024
8d056bd
Drop `S3ExpressSigner` and override session token name via `RuntimePl…
ysaito1001 Mar 6, 2024
32a93b0
Merge branch 'main' into ysaito/s3express
ysaito1001 Mar 6, 2024
7f8c28b
Update CHANGELOG.next.toml
ysaito1001 Mar 6, 2024
a4e42ef
Fix incorrect merge
ysaito1001 Mar 6, 2024
db23423
Merge branch 'main' into ysaito/s3express
ysaito1001 Mar 7, 2024
75ed6fd
Add canary for S3 Express (disabled by default until it's been releas…
ysaito1001 Mar 8, 2024
eebe8af
Allow `canary-runner` to specify lambda's memory size
ysaito1001 Mar 8, 2024
60e6a89
Pass what's minimally needed to `S3ExpressRuntimePlugin`
ysaito1001 Mar 8, 2024
374ed2a
Define a function to provide default checksum altorithm
ysaito1001 Mar 8, 2024
3c3843f
Let default checksum provider return `DefaultRequestChecksumOverride`
ysaito1001 Mar 8, 2024
3b8f2f4
Fix S3 Express bug where SigV4 session token was incorrectly override…
ysaito1001 Mar 11, 2024
d04304f
Merge branch 'main' into ysaito/s3express
ysaito1001 Mar 11, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.next.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,9 @@ message = "Upgrade Smithy to 1.45."
references = ["smithy-rs#3470"]
meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "all" }
authors = ["jdisanti"]

[[aws-sdk-rust]]
message = "Add support for S3 Express One Zone. See [the user guide](https://github.com/awslabs/aws-sdk-rust/discussions/1091) for more details."
references = ["aws-sdk-rust#992", "smithy-rs#3465"]
meta = { "breaking" = false, "bug" = false, "tada" = true }
author = "ysaito1001"
8 changes: 8 additions & 0 deletions aws/rust-runtime/aws-inlineable/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ publish = false
repository = "https://github.com/smithy-lang/smithy-rs"

[dependencies]
# Used by lru, and this forces it to be a later version that avoids
# https://github.com/tkaitchuck/aHash/issues/200
# when built with `cargo update -Z minimal-versions`
ahash = "0.8.11"
aws-credential-types = { path = "../aws-credential-types" }
aws-runtime = { path = "../aws-runtime", features = ["http-02x"] }
aws-sigv4 = { path = "../aws-sigv4" }
Expand All @@ -22,10 +26,14 @@ aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", featur
aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] }
aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["http-body-0-4-x"] }
bytes = "1"
fastrand = "2.0.0"
hex = "0.4.3"
http = "0.2.9"
http-body = "0.4.5"
hmac = "0.12"
lru = "0.12.2"
ring = "0.17.5"
sha2 = "0.10"
tokio = "1.23.1"
tracing = "0.1"

Expand Down
54 changes: 52 additions & 2 deletions aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,45 @@ impl Storable for RequestChecksumInterceptorState {
type Storer = StoreReplace<Self>;
}

type CustomDefaultFn = Box<
dyn Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>
+ Send
+ Sync
+ 'static,
>;

pub(crate) struct DefaultRequestChecksumOverride {
custom_default: CustomDefaultFn,
}
impl fmt::Debug for DefaultRequestChecksumOverride {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DefaultRequestChecksumOverride").finish()
}
}
impl Storable for DefaultRequestChecksumOverride {
type Storer = StoreReplace<Self>;
}
impl DefaultRequestChecksumOverride {
pub(crate) fn new<F>(custom_default: F) -> Self
where
F: Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>
+ Send
+ Sync
+ 'static,
{
Self {
custom_default: Box::new(custom_default),
}
}
pub(crate) fn custom_default(
&self,
original: Option<ChecksumAlgorithm>,
config_bag: &ConfigBag,
) -> Option<ChecksumAlgorithm> {
(self.custom_default)(original, config_bag)
}
}

pub(crate) struct RequestChecksumInterceptor<AP> {
algorithm_provider: AP,
}
Expand Down Expand Up @@ -102,7 +141,7 @@ where
/// Calculate a checksum and modify the request to include the checksum as a header
/// (for in-memory request bodies) or a trailer (for streaming request bodies).
/// Streaming bodies must be sized or this will return an error.
fn modify_before_retry_loop(
fn modify_before_signing(
&self,
context: &mut BeforeTransmitInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
Expand All @@ -112,7 +151,8 @@ where
.load::<RequestChecksumInterceptorState>()
.expect("set in `read_before_serialization`");

if let Some(checksum_algorithm) = state.checksum_algorithm {
let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg);
if let Some(checksum_algorithm) = checksum_algorithm {
let request = context.request_mut();
add_checksum_for_request_body(request, checksum_algorithm, cfg)?;
}
Expand All @@ -121,6 +161,16 @@ where
}
}

fn incorporate_custom_default(
checksum: Option<ChecksumAlgorithm>,
cfg: &ConfigBag,
) -> Option<ChecksumAlgorithm> {
match cfg.load::<DefaultRequestChecksumOverride>() {
Some(checksum_override) => checksum_override.custom_default(checksum, cfg),
None => checksum,
}
}

fn add_checksum_for_request_body(
request: &mut HttpRequest,
checksum_algorithm: ChecksumAlgorithm,
Expand Down
5 changes: 5 additions & 0 deletions aws/rust-runtime/aws-inlineable/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ pub mod presigning;
/// Presigning interceptors
pub mod presigning_interceptors;

// This module uses module paths that assume the target crate to which it is copied, e.g.
// `crate::config::endpoint::Params`. If included into `aws-inlineable`, this module would
// fail to compile.
// pub mod s3_express;

/// Special logic for extracting request IDs from S3's responses.
#[allow(dead_code)]
pub mod s3_request_id;
Expand Down
Loading
Loading