From e9abec67cf814b52101c2ed1b66a86f4f1f478c3 Mon Sep 17 00:00:00 2001 From: Russell Cohen Date: Wed, 17 Jan 2024 14:17:45 -0500 Subject: [PATCH 1/9] Update sigv4 to allow applying signature to http1x URIs (#3366) ## Motivation and Context - https://github.com/awslabs/aws-sdk-rust/issues/1041 ## Description Enable signing Http 1x requests. ## Testing Unit tests ## Checklist - [ ] I have updated `CHANGELOG.next.toml` if I made changes to the smithy-rs codegen or runtime crates - [ ] I have updated `CHANGELOG.next.toml` if I made changes to the AWS SDK, generated SDK code, or SDK runtime crates ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- CHANGELOG.next.toml | 6 ++ aws/rust-runtime/aws-sigv4/Cargo.toml | 10 ++- .../aws-sigv4/src/http_request.rs | 7 +- .../src/http_request/canonical_request.rs | 16 ++-- .../aws-sigv4/src/http_request/error.rs | 4 +- .../aws-sigv4/src/http_request/settings.rs | 2 +- .../aws-sigv4/src/http_request/sign.rs | 73 ++++++++++++++++--- .../aws-sigv4/src/http_request/test.rs | 10 +-- 8 files changed, 93 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 8a30e72ab44..8703e81c286 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -34,3 +34,9 @@ message = "Fix bug where overriding the credentials at the operation level faile references = ["aws-sdk-rust#901", "smithy-rs#3363"] meta = { "breaking" = false, "bug" = true, "tada" = false } author = "rcoh" + +[[aws-sdk-rust]] +message = "Add `apply_to_request_http1x` to `aws-sigv4` to enable signing http = 1.0 requests." +references = ["aws-sdk-rust#1041", "smithy-rs#3366"] +meta = { "breaking" = false, "bug" = false, "tada" = true } +author = "rcoh" diff --git a/aws/rust-runtime/aws-sigv4/Cargo.toml b/aws/rust-runtime/aws-sigv4/Cargo.toml index fff908eba9d..f29a2ba01ca 100644 --- a/aws/rust-runtime/aws-sigv4/Cargo.toml +++ b/aws/rust-runtime/aws-sigv4/Cargo.toml @@ -9,9 +9,10 @@ license = "Apache-2.0" repository = "https://github.com/smithy-lang/smithy-rs" [features] -default = ["sign-http"] -http0-compat = ["dep:http"] -sign-http = ["dep:http", "dep:percent-encoding", "dep:form_urlencoded"] +default = ["sign-http", "http1"] +http0-compat = ["dep:http0"] +http1 = ["dep:http"] +sign-http = ["dep:http0", "dep:percent-encoding", "dep:form_urlencoded"] sign-eventstream = ["dep:aws-smithy-eventstream"] sigv4a = ["dep:p256", "dep:crypto-bigint", "dep:subtle", "dep:zeroize", "dep:ring"] @@ -25,7 +26,8 @@ bytes = "1" form_urlencoded = { version = "1.0", optional = true } hex = "0.4" hmac = "0.12" -http = { version = "0.2", optional = true } +http0 = { version = "0.2", optional = true, package = "http" } +http = { version = "1", optional = true } num-bigint = { version = "0.4.2", optional = true } once_cell = "1.8" p256 = { version = "0.11", features = ["ecdsa"], optional = true } diff --git a/aws/rust-runtime/aws-sigv4/src/http_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request.rs index 6bb6b9235c2..2bd0c92903d 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request.rs @@ -13,14 +13,15 @@ //! # use aws_credential_types::Credentials; //! use aws_smithy_runtime_api::client::identity::Identity; //! # use aws_sigv4::http_request::SignableBody; -//! #[cfg(feature = "http0-compat")] +//! #[cfg(feature = "http1")] //! fn test() -> Result<(), aws_sigv4::http_request::SigningError> { //! use aws_sigv4::http_request::{sign, SigningSettings, SigningParams, SignableRequest}; //! use aws_sigv4::sign::v4; -//! use http; +//! use http0; //! use std::time::SystemTime; //! //! // Set up information and settings for the signing +//! // You can obtain credentials from `SdkConfig`. //! let identity = Credentials::new( //! "AKIDEXAMPLE", //! "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", @@ -49,7 +50,7 @@ //! let mut my_req = http::Request::new("..."); //! // Sign and then apply the signature to the request //! let (signing_instructions, _signature) = sign(signable_request, &signing_params)?.into_parts(); -//! signing_instructions.apply_to_request_http0x(&mut my_req); +//! signing_instructions.apply_to_request_http1x(&mut my_req); //! # Ok(()) //! # } //! ``` diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 32b6cfe4f3d..5fae247dd2a 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -15,8 +15,8 @@ use crate::http_request::{PayloadChecksumKind, SignableBody, SignatureLocation, use crate::sign::v4::sha256_hex_string; use crate::SignatureVersion; use aws_smithy_http::query_writer::QueryWriter; -use http::header::{AsHeaderName, HeaderName, HOST}; -use http::{HeaderMap, HeaderValue, Uri}; +use http0::header::{AsHeaderName, HeaderName, HOST}; +use http0::{HeaderMap, HeaderValue, Uri}; use std::borrow::Cow; use std::cmp::Ordering; use std::convert::TryFrom; @@ -626,7 +626,7 @@ mod tests { use aws_credential_types::Credentials; use aws_smithy_http::query_writer::QueryWriter; use aws_smithy_runtime_api::client::identity::Identity; - use http::{HeaderValue, Uri}; + use http0::{HeaderValue, Uri}; use pretty_assertions::assert_eq; use proptest::{prelude::*, proptest}; use std::borrow::Cow; @@ -794,7 +794,7 @@ mod tests { #[test] fn test_tilde_in_uri() { - let req = http::Request::builder() + let req = http0::Request::builder() .uri("https://s3.us-east-1.amazonaws.com/my-bucket?list-type=2&prefix=~objprefix&single&k=&unreserved=-_.~").body("").unwrap().into(); let req = SignableRequest::from(&req); let identity = Credentials::for_tests().into(); @@ -815,7 +815,7 @@ mod tests { query_writer.insert("list-type", "2"); query_writer.insert("prefix", &all_printable_ascii_chars); - let req = http::Request::builder() + let req = http0::Request::builder() .uri(query_writer.build_uri()) .body("") .unwrap() @@ -863,7 +863,7 @@ mod tests { // It should exclude authorization, user-agent, x-amzn-trace-id headers from presigning #[test] fn non_presigning_header_exclusion() { - let request = http::Request::builder() + let request = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header("authorization", "test-authorization") .header("content-type", "application/xml") @@ -895,7 +895,7 @@ mod tests { // It should exclude authorization, user-agent, x-amz-user-agent, x-amzn-trace-id headers from presigning #[test] fn presigning_header_exclusion() { - let request = http::Request::builder() + let request = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header("authorization", "test-authorization") .header("content-type", "application/xml") @@ -944,7 +944,7 @@ mod tests { valid_input, ) ) { - let mut request_builder = http::Request::builder() + let mut request_builder = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header("content-type", "application/xml") .header("content-length", "0"); diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/error.rs b/aws/rust-runtime/aws-sigv4/src/http_request/error.rs index 39f57dffa5d..6f53783ff80 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/error.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/error.rs @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -use http::header::{InvalidHeaderName, InvalidHeaderValue}; -use http::uri::InvalidUri; +use http0::header::{InvalidHeaderName, InvalidHeaderValue}; +use http0::uri::InvalidUri; use std::error::Error; use std::fmt; diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs index 3c690e6ddca..787619fc364 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/settings.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use http::header::{AUTHORIZATION, USER_AGENT}; +use http0::header::{AUTHORIZATION, USER_AGENT}; use std::borrow::Cow; use std::time::Duration; diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs index 2cef0dbd3ce..2f27b3c4b71 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/sign.rs @@ -14,7 +14,7 @@ use crate::sign::v4; #[cfg(feature = "sigv4a")] use crate::sign::v4a; use crate::{SignatureVersion, SigningOutput}; -use http::Uri; +use http0::Uri; use std::borrow::Cow; use std::fmt::{Debug, Formatter}; use std::str; @@ -162,10 +162,10 @@ impl SigningInstructions { #[cfg(any(feature = "http0-compat", test))] /// Applies the instructions to the given `request`. - pub fn apply_to_request_http0x(self, request: &mut http::Request) { + pub fn apply_to_request_http0x(self, request: &mut http0::Request) { let (new_headers, new_query) = self.into_parts(); for header in new_headers.into_iter() { - let mut value = http::HeaderValue::from_str(&header.value).unwrap(); + let mut value = http0::HeaderValue::from_str(&header.value).unwrap(); value.set_sensitive(header.sensitive); request.headers_mut().insert(header.key, value); } @@ -178,6 +178,34 @@ impl SigningInstructions { *request.uri_mut() = query.build_uri(); } } + + #[cfg(any(feature = "http1", test))] + /// Applies the instructions to the given `request`. + pub fn apply_to_request_http1x(self, request: &mut http::Request) { + // TODO(https://github.com/smithy-lang/smithy-rs/issues/3367): Update query writer to reduce + // allocations + let (new_headers, new_query) = self.into_parts(); + for header in new_headers.into_iter() { + let mut value = http::HeaderValue::from_str(&header.value).unwrap(); + value.set_sensitive(header.sensitive); + request.headers_mut().insert(header.key, value); + } + + if !new_query.is_empty() { + let mut query = aws_smithy_http::query_writer::QueryWriter::new_from_string( + &request.uri().to_string(), + ) + .expect("unreachable: URI is valid"); + for (name, value) in new_query { + query.insert(name, &value); + } + *request.uri_mut() = query + .build_uri() + .to_string() + .parse() + .expect("unreachable: URI is valid"); + } + } } /// Produces a signature for the given `request` and returns instructions @@ -444,7 +472,7 @@ mod tests { }; use crate::sign::v4; use aws_credential_types::Credentials; - use http::{HeaderValue, Request}; + use http0::{HeaderValue, Request}; use pretty_assertions::assert_eq; use proptest::proptest; use std::borrow::Cow; @@ -830,7 +858,7 @@ mod tests { } .into(); - let original = http::Request::builder() + let original = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header("some-header", HeaderValue::from_str("テスト").unwrap()) .body("") @@ -846,7 +874,7 @@ mod tests { let mut signed = original.as_http_request(); out.output.apply_to_request_http0x(&mut signed); - let expected = http::Request::builder() + let expected = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header("some-header", HeaderValue::from_str("テスト").unwrap()) .header( @@ -884,7 +912,7 @@ mod tests { } .into(); - let original = http::Request::builder() + let original = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .body("") .unwrap() @@ -907,7 +935,7 @@ mod tests { .output .apply_to_request_http0x(&mut signed); - let expected = http::Request::builder() + let expected = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header( "x-amz-date", @@ -945,7 +973,7 @@ mod tests { } .into(); - let original = http::Request::builder() + let original = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header( "some-header", @@ -964,7 +992,7 @@ mod tests { let mut signed = original.as_http_request(); out.output.apply_to_request_http0x(&mut signed); - let expected = http::Request::builder() + let expected = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .header( "some-header", @@ -1027,7 +1055,7 @@ mod tests { add_header(&mut headers, "some-other-header", "bar", false); let instructions = SigningInstructions::new(headers, vec![]); - let mut request = http::Request::builder() + let mut request = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com") .body("") .unwrap(); @@ -1047,7 +1075,7 @@ mod tests { ]; let instructions = SigningInstructions::new(vec![], params); - let mut request = http::Request::builder() + let mut request = http0::Request::builder() .uri("https://some-endpoint.some-region.amazonaws.com/some/path") .body("") .unwrap(); @@ -1059,4 +1087,25 @@ mod tests { request.uri().path_and_query().unwrap().to_string() ); } + + #[test] + fn apply_signing_instructions_query_params_http_1x() { + let params = vec![ + ("some-param", Cow::Borrowed("f&o?o")), + ("some-other-param?", Cow::Borrowed("bar")), + ]; + let instructions = SigningInstructions::new(vec![], params); + + let mut request = http::Request::builder() + .uri("https://some-endpoint.some-region.amazonaws.com/some/path") + .body("") + .unwrap(); + + instructions.apply_to_request_http1x(&mut request); + + assert_eq!( + "/some/path?some-param=f%26o%3Fo&some-other-param%3F=bar", + request.uri().path_and_query().unwrap().to_string() + ); + } } diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/test.rs b/aws/rust-runtime/aws-sigv4/src/http_request/test.rs index 9d5bec6d5d0..be6c4964627 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/test.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/test.rs @@ -6,7 +6,7 @@ //! Functions shared between the tests of several modules. use crate::http_request::{SignableBody, SignableRequest}; -use http::{Method, Uri}; +use http0::{Method, Uri}; use std::error::Error as StdError; pub(crate) mod v4 { @@ -258,8 +258,8 @@ impl TestRequest { self.body = TestSignedBody::Signable(body); } - pub(crate) fn as_http_request(&self) -> http::Request<&'static str> { - let mut builder = http::Request::builder() + pub(crate) fn as_http_request(&self) -> http0::Request<&'static str> { + let mut builder = http0::Request::builder() .uri(&self.uri) .method(Method::from_bytes(self.method.as_bytes()).unwrap()); for (k, v) in &self.headers { @@ -269,8 +269,8 @@ impl TestRequest { } } -impl> From> for TestRequest { - fn from(value: http::Request) -> Self { +impl> From> for TestRequest { + fn from(value: http0::Request) -> Self { let invalid = value .headers() .values() From 2044a4820e06c0eb3a7de684a1bde68c1860fb0c Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Wed, 17 Jan 2024 13:05:18 -0800 Subject: [PATCH 2/9] Set aws-smithy-mocks-experimental to 0.1 (#3372) _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: Russell Cohen --- rust-runtime/aws-smithy-mocks-experimental/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml b/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml index 88319c82a99..4d87de72c1e 100644 --- a/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml +++ b/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml @@ -1,12 +1,11 @@ [package] name = "aws-smithy-mocks-experimental" -version = "0.0.0-smithy-rs-head" +version = "0.1.0" authors = ["AWS Rust SDK Team "] description = "Experimental testing utilities for smithy-rs generated clients" edition = "2021" license = "Apache-2.0" repository = "https://github.com/smithy-lang/smithy-rs" -publish = false [dependencies] aws-smithy-types = "1" From 59c73e44008121d2e2f6bd45f1a0b295c5191a27 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Wed, 17 Jan 2024 19:26:00 -0800 Subject: [PATCH 3/9] Include aws-smithy-mocks-experimental in the release (#3374) We intend to include aws-smithy-mocks-experimental in the release, but it needs to be added to the CrateSet to actually be published. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- buildSrc/src/main/kotlin/CrateSet.kt | 1 + rust-runtime/aws-smithy-mocks-experimental/Cargo.toml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/CrateSet.kt b/buildSrc/src/main/kotlin/CrateSet.kt index d616d619f0d..e3ad6e49ee9 100644 --- a/buildSrc/src/main/kotlin/CrateSet.kt +++ b/buildSrc/src/main/kotlin/CrateSet.kt @@ -63,6 +63,7 @@ object CrateSet { "aws-smithy-http-auth", "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-mocks-experimental", "aws-smithy-protocol-test", "aws-smithy-query", "aws-smithy-runtime", diff --git a/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml b/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml index 4d87de72c1e..ef06256e5d2 100644 --- a/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml +++ b/rust-runtime/aws-smithy-mocks-experimental/Cargo.toml @@ -9,8 +9,7 @@ repository = "https://github.com/smithy-lang/smithy-rs" [dependencies] aws-smithy-types = "1" -aws-smithy-runtime-api = { version = "1", features = ["http-02x"] } -aws-smithy-runtime = { version = "1", features = ["test-util"] } +aws-smithy-runtime-api = { version = "1", features = ["client", "http-02x"] } [dev-dependencies] aws-sdk-s3 = { version = "1", features = ["test-util"] } From edf6e77bfa991aef9afa5acf293a911f7982511a Mon Sep 17 00:00:00 2001 From: AWS SDK Rust Bot Date: Thu, 18 Jan 2024 17:58:31 +0000 Subject: [PATCH 4/9] Upgrade the smithy-rs runtime crates version to 1.1.3 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 1e7ef28b2a6..d0dce9a7814 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,10 +12,10 @@ rust.msrv=1.72.1 org.gradle.jvmargs=-Xmx1024M # Version number to use for the generated stable runtime crates -smithy.rs.runtime.crate.stable.version=1.1.2 +smithy.rs.runtime.crate.stable.version=1.1.3 # Version number to use for the generated unstable runtime crates -smithy.rs.runtime.crate.unstable.version=0.60.2 +smithy.rs.runtime.crate.unstable.version=0.60.3 kotlin.code.style=official From 4d7c84c5dc80f80f1a83191e59941d1c5cda4794 Mon Sep 17 00:00:00 2001 From: AWS SDK Rust Bot Date: Thu, 18 Jan 2024 18:00:40 +0000 Subject: [PATCH 5/9] Update changelog --- CHANGELOG.md | 7 +++ CHANGELOG.next.toml | 32 +----------- aws/SDK_CHANGELOG.next.json | 98 +++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5b2bc139f..0ccafaf7f76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ +January 18th, 2024 +================== +**New this release:** +- (client, [smithy-rs#3318](https://github.com/smithy-lang/smithy-rs/issues/3318)) `EndpointPrefix` and `apply_endpoint` moved from aws-smithy-http to aws-smithy-runtime-api so that is in a stable (1.x) crate. A deprecated type alias was left in place with a note showing the new location. +- (client, [smithy-rs#3325](https://github.com/smithy-lang/smithy-rs/issues/3325)) The `Metadata` storable was moved from aws_smithy_http into aws_smithy_runtime_api. A deprecated type alias was left in place with a note showing where the new location is. + + January 10th, 2024 ================== **New this release:** diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 8703e81c286..fc4c4c2578b 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -9,34 +9,4 @@ # message = "Fix typos in module documentation for generated crates" # references = ["smithy-rs#920"] # meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client | server | all"} -# author = "rcoh" - -[[aws-sdk-rust]] -message = "`EndpointPrefix` and `apply_endpoint` moved from aws-smithy-http to aws-smithy-runtime-api so that is in a stable (1.x) crate. A deprecated type alias was left in place with a note showing the new location." -references = ["smithy-rs#3318"] -meta = { "breaking" = false, "tada" = false, "bug" = false } -author = "jdisanti" - -[[smithy-rs]] -message = "`EndpointPrefix` and `apply_endpoint` moved from aws-smithy-http to aws-smithy-runtime-api so that is in a stable (1.x) crate. A deprecated type alias was left in place with a note showing the new location." -references = ["smithy-rs#3318"] -meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"} -author = "jdisanti" - -[[smithy-rs]] -message = "The `Metadata` storable was moved from aws_smithy_http into aws_smithy_runtime_api. A deprecated type alias was left in place with a note showing where the new location is." -references = ["smithy-rs#3325"] -meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client" } -author = "jdisanti" - -[[aws-sdk-rust]] -message = "Fix bug where overriding the credentials at the operation level failed if credentials were already set." -references = ["aws-sdk-rust#901", "smithy-rs#3363"] -meta = { "breaking" = false, "bug" = true, "tada" = false } -author = "rcoh" - -[[aws-sdk-rust]] -message = "Add `apply_to_request_http1x` to `aws-sigv4` to enable signing http = 1.0 requests." -references = ["aws-sdk-rust#1041", "smithy-rs#3366"] -meta = { "breaking" = false, "bug" = false, "tada" = true } -author = "rcoh" +# author = "rcoh" \ No newline at end of file diff --git a/aws/SDK_CHANGELOG.next.json b/aws/SDK_CHANGELOG.next.json index 455929f32a1..298b0700c63 100644 --- a/aws/SDK_CHANGELOG.next.json +++ b/aws/SDK_CHANGELOG.next.json @@ -5,36 +5,6 @@ { "smithy-rs": [], "aws-sdk-rust": [ - { - "message": "Fix `config::Builder::set_credentials_provider` to override a credentials provider previously set.", - "meta": { - "bug": true, - "breaking": false, - "tada": false - }, - "author": "ysaito1001", - "references": [ - "aws-sdk-rust#973", - "smithy-rs#3278" - ], - "since-commit": "529b3f03e2b945ea2e5e879183ccfd8e74b7377c", - "age": 5 - }, - { - "message": "`config::Config::credentials_provider` has been broken since `release-2023-11-15` and is now marked as `deprecated` explicitly.", - "meta": { - "bug": false, - "breaking": false, - "tada": false - }, - "author": "ysaito1001", - "references": [ - "smithy-rs#3251", - "smithy-rs#3278" - ], - "since-commit": "529b3f03e2b945ea2e5e879183ccfd8e74b7377c", - "age": 5 - }, { "message": "Loading native TLS trusted certs for the default HTTP client now only occurs if the default HTTP client is not overridden in config.", "meta": { @@ -47,7 +17,7 @@ "smithy-rs#3262" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "Client creation now takes microseconds instead of milliseconds.\nPreviously, it would take 2-3 milliseconds for each client instantiation due to time spent compiling regexes.\nFor applications that used several clients, this would increase start-up time in cases where it really matters,\nsuch as for AWS Lambda cold starts. This time was improved by both changing regex implementation and caching the\nresult of the compilation.", @@ -62,7 +32,7 @@ "smithy-rs#3269" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "Add `test_credentials` to `ConfigLoader` in `aws_config`. This allows the following pattern during tests:\n\n```rust\nasync fn main() {\n let conf = aws_config::defaults(BehaviorVersion::latest())\n .test_credentials()\n .await;\n}\n```\n\nThis is designed for unit tests and using local mocks like DynamoDB Local and LocalStack with the SDK.\n", @@ -77,7 +47,7 @@ "aws-sdk-rust#971" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "Improve the error messages for when auth fails to select an auth scheme for a request.", @@ -92,7 +62,7 @@ "smithy-rs#3277" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "Fix documentation and examples on HyperConnector and HyperClientBuilder.", @@ -107,7 +77,7 @@ "smithy-rs#3282" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "All generated docs now include docsrs labels when features are required", @@ -122,7 +92,7 @@ "smithy-rs#3295" ], "since-commit": "fc335cbc87e70aa63895828fca55b51795b94a6c", - "age": 4 + "age": 5 }, { "message": "`crate::event_receiver::EventReceiver` is now re-exported as `crate::primitives::event_stream::EventReceiver` when a service supports event stream operations.", @@ -136,7 +106,7 @@ "smithy-rs#3305" ], "since-commit": "9f0ba850e03241f657e2e40ca185780e0a5878cb", - "age": 3 + "age": 4 }, { "message": "Add support for constructing [`SdkBody`] and [`ByteStream`] from `http-body` 1.0 bodies. Note that this is initial support and works via a backwards compatibility shim to http-body 0.4. Hyper 1.0 is not supported.", @@ -151,7 +121,7 @@ "aws-sdk-rust#977" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 1 + "age": 2 }, { "message": " Add `PaginationStreamExt` extension trait to `aws-smithy-types-convert` behind the `convert-streams` feature. This makes it possible to treat a paginator as a [`futures_core::Stream`](https://docs.rs/futures-core/latest/futures_core/stream/trait.Stream.html), allowing customers to use stream combinators like [`map`](https://docs.rs/tokio-stream/latest/tokio_stream/trait.StreamExt.html#method.map) and [`filter`](https://docs.rs/tokio-stream/latest/tokio_stream/trait.StreamExt.html#method.filter).\n\nExample:\n\n```rust\nuse aws_smithy_types_convert::stream::PaginationStreamExt\nlet stream = s3_client.list_objects_v2().bucket(\"...\").into_paginator().send().into_stream_03x();\n```\n", @@ -165,7 +135,7 @@ "smithy-rs#3299" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 1 + "age": 2 }, { "message": "Serialize 0/false in query parameters, and ignore actual default value during serialization instead of just 0/false. See [changelog discussion](https://github.com/smithy-lang/smithy-rs/discussions/3312) for details.", @@ -180,7 +150,7 @@ "smithy-rs#3312" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 1 + "age": 2 }, { "message": "Add `as_service_err()` to `SdkError` to allow checking the type of an error is without taking ownership.", @@ -196,7 +166,7 @@ "aws-sdk-rust#1010" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 1 + "age": 2 }, { "message": "Fix bug in `CredentialsProcess` provider where `expiry` was incorrectly treated as a required field.", @@ -211,7 +181,7 @@ "aws-sdk-rust#1021" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", - "age": 1 + "age": 2 }, { "message": "~/.aws/config and ~/.aws/credentials now parse keys in a case-insensitive way. This means the `AWS_SECRET_ACCESS_KEY` is supported in addition to `aws_secret_access_key`.", @@ -227,6 +197,50 @@ "smithy-rs#3344" ], "since-commit": "e235a2fd9ec45335a3b2018028c2d3a2ac13ffdf", + "age": 2 + }, + { + "message": "`EndpointPrefix` and `apply_endpoint` moved from aws-smithy-http to aws-smithy-runtime-api so that is in a stable (1.x) crate. A deprecated type alias was left in place with a note showing the new location.", + "meta": { + "bug": false, + "breaking": false, + "tada": false + }, + "author": "jdisanti", + "references": [ + "smithy-rs#3318" + ], + "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", + "age": 1 + }, + { + "message": "Fix bug where overriding the credentials at the operation level failed if credentials were already set.", + "meta": { + "bug": true, + "breaking": false, + "tada": false + }, + "author": "rcoh", + "references": [ + "aws-sdk-rust#901", + "smithy-rs#3363" + ], + "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", + "age": 1 + }, + { + "message": "Add `apply_to_request_http1x` to `aws-sigv4` to enable signing http = 1.0 requests.", + "meta": { + "bug": false, + "breaking": false, + "tada": true + }, + "author": "rcoh", + "references": [ + "aws-sdk-rust#1041", + "smithy-rs#3366" + ], + "since-commit": "edf6e77bfa991aef9afa5acf293a911f7982511a", "age": 1 } ], From 366ccab1bcd1d54183cad01472e182cf80be38b1 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 18 Jan 2024 17:24:53 -0800 Subject: [PATCH 6/9] Fix describe tag issue when finding ancestor tag (#3376) The runtime-versioner's audit subcommand fails when the current commit is the tagged release commit due to a bug in `ancestor_tag` where it was failing to trim the output from `git describe --tags`. I fixed this earlier in #3369 since I was running into it there, but that hasn't been reviewed/merged yet. This issue is causing problems for the latest smithy-rs/SDK release, so I've pulled it out into a separate PR. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- tools/ci-build/runtime-versioner/src/tag.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci-build/runtime-versioner/src/tag.rs b/tools/ci-build/runtime-versioner/src/tag.rs index def5b6d6347..c9a88259102 100644 --- a/tools/ci-build/runtime-versioner/src/tag.rs +++ b/tools/ci-build/runtime-versioner/src/tag.rs @@ -43,10 +43,11 @@ fn ancestor_tag(repo: &Repo) -> Result { let tag = repo .git(["describe", "--tags"]) .expect_success_output("find the current ancestor release tag")?; - let maybe_release_tag = ReleaseTag::from_str(&tag); + let tag = tag.trim(); + let maybe_release_tag = ReleaseTag::from_str(tag); let release_tag = match maybe_release_tag { Ok(tag) => Some(tag), - Err(_) => strip_describe_tags_suffix(&tag) + Err(_) => strip_describe_tags_suffix(tag) .map(ReleaseTag::from_str) .transpose() .context("failed to find ancestor release tag")?, From 81db5c48e99fdcda463d7c80831a4e08f1af08cb Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Tue, 23 Jan 2024 12:06:17 -0800 Subject: [PATCH 7/9] Defer event stream semver hazard fix (#3371) Defer fixing the event stream semver hazard. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- rust-runtime/aws-smithy-eventstream/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-eventstream/Cargo.toml b/rust-runtime/aws-smithy-eventstream/Cargo.toml index b6979af34ee..b97b75e33c3 100644 --- a/rust-runtime/aws-smithy-eventstream/Cargo.toml +++ b/rust-runtime/aws-smithy-eventstream/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "aws-smithy-eventstream" -version = "0.0.0-smithy-rs-head" +# Only patch releases can be made to this runtime crate until https://github.com/smithy-lang/smithy-rs/issues/3370 is resolved +version = "0.60.4" +# authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Event stream logic for smithy-rs." edition = "2021" From 4c256ef0202122b501576e5cfa225cbebf3f009c Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Tue, 23 Jan 2024 14:10:34 -0800 Subject: [PATCH 8/9] Move `aws-http` types into `aws-runtime` (#3355) This issue addresses a semver compatibility problem similar to the one described in https://github.com/smithy-lang/smithy-rs/pull/3318, except for the storable types in the aws-http crate. I opted to move all of aws-http into aws-runtime since there was so little in there. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- CHANGELOG.next.toml | 8 +- aws/rust-runtime/aws-config/Cargo.toml | 1 - .../aws-config/src/imds/client.rs | 3 +- aws/rust-runtime/aws-http/Cargo.toml | 17 +- aws/rust-runtime/aws-http/README.md | 5 +- aws/rust-runtime/aws-http/external-types.toml | 10 +- .../aws-http/src/content_encoding.rs | 625 +----------- aws/rust-runtime/aws-http/src/user_agent.rs | 781 +-------------- aws/rust-runtime/aws-inlineable/Cargo.toml | 3 +- .../src/http_request_checksum.rs | 6 +- aws/rust-runtime/aws-runtime/Cargo.toml | 9 +- .../aws-runtime/external-types.toml | 4 + .../aws-runtime/src/content_encoding.rs | 613 ++++++++++++ aws/rust-runtime/aws-runtime/src/lib.rs | 4 + .../aws-runtime/src/user_agent.rs | 886 +++++++++++++----- .../aws-runtime/src/user_agent/interceptor.rs | 277 ++++++ .../smithy/rustsdk/AwsCargoDependency.kt | 2 - .../amazon/smithy/rustsdk/AwsRuntimeType.kt | 2 - .../rustsdk/HttpRequestChecksumDecorator.kt | 1 + .../smithy/rustsdk/UserAgentDecorator.kt | 4 +- aws/sdk/integration-tests/dynamodb/Cargo.toml | 1 - aws/sdk/integration-tests/glacier/Cargo.toml | 1 - aws/sdk/integration-tests/iam/Cargo.toml | 1 - aws/sdk/integration-tests/kms/Cargo.toml | 1 - aws/sdk/integration-tests/lambda/Cargo.toml | 1 - aws/sdk/integration-tests/polly/Cargo.toml | 1 - .../integration-tests/qldbsession/Cargo.toml | 1 - aws/sdk/integration-tests/s3/Cargo.toml | 1 - .../integration-tests/s3/tests/checksums.rs | 2 +- .../s3/tests/request_information_headers.rs | 2 +- .../integration-tests/s3control/Cargo.toml | 1 - .../transcribestreaming/Cargo.toml | 1 - 32 files changed, 1672 insertions(+), 1603 deletions(-) create mode 100644 aws/rust-runtime/aws-runtime/src/content_encoding.rs create mode 100644 aws/rust-runtime/aws-runtime/src/user_agent/interceptor.rs diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index fc4c4c2578b..03b9741050b 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -9,4 +9,10 @@ # message = "Fix typos in module documentation for generated crates" # references = ["smithy-rs#920"] # meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client | server | all"} -# author = "rcoh" \ No newline at end of file +# author = "rcoh" + +[[aws-sdk-rust]] +message = "The types in the aws-http crate were moved into aws-runtime. Deprecated type aliases were put in place to point to the new locations." +references = ["smithy-rs#3355"] +meta = { "breaking" = false, "tada" = false, "bug" = false } +author = "jdisanti" diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index b9552409687..bd84fc4f9ce 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -23,7 +23,6 @@ allow-compilation = [] [dependencies] aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../sdk/build/aws-sdk/sdk/aws-http" } aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false } aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async" } aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" } diff --git a/aws/rust-runtime/aws-config/src/imds/client.rs b/aws/rust-runtime/aws-config/src/imds/client.rs index 154225b29e8..a660369c5c3 100644 --- a/aws/rust-runtime/aws-config/src/imds/client.rs +++ b/aws/rust-runtime/aws-config/src/imds/client.rs @@ -11,8 +11,7 @@ use crate::imds::client::error::{BuildError, ImdsError, InnerImdsError, InvalidE use crate::imds::client::token::TokenRuntimePlugin; use crate::provider_config::ProviderConfig; use crate::PKG_VERSION; -use aws_http::user_agent::{ApiMetadata, AwsUserAgent}; -use aws_runtime::user_agent::UserAgentInterceptor; +use aws_runtime::user_agent::{ApiMetadata, AwsUserAgent, UserAgentInterceptor}; use aws_smithy_runtime::client::orchestrator::operation::Operation; use aws_smithy_runtime::client::retries::strategy::StandardRetryStrategy; use aws_smithy_runtime_api::box_error::BoxError; diff --git a/aws/rust-runtime/aws-http/Cargo.toml b/aws/rust-runtime/aws-http/Cargo.toml index e33b302d648..e01593ab5c3 100644 --- a/aws/rust-runtime/aws-http/Cargo.toml +++ b/aws/rust-runtime/aws-http/Cargo.toml @@ -1,25 +1,14 @@ [package] name = "aws-http" -version = "0.0.0-smithy-rs-head" +version = "0.60.5" authors = ["AWS Rust SDK Team ", "Russell Cohen "] -description = "HTTP specific AWS SDK behaviors." +description = "This crate is no longer used by the AWS SDK and is deprecated." edition = "2021" license = "Apache-2.0" repository = "https://github.com/smithy-lang/smithy-rs" [dependencies] -aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] } -aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["http-body-0-4-x"] } -aws-types = { path = "../aws-types" } -bytes = "1.1" -http = "0.2.3" -http-body = "0.4.5" -tracing = "0.1" -pin-project-lite = "0.2.9" - -[dev-dependencies] -bytes-utils = "0.1.2" -tokio = { version = "1.23.1", features = ["macros", "rt", "time"] } +aws-runtime = { path = "../aws-runtime", features = ["http-02x"] } [package.metadata.docs.rs] all-features = true diff --git a/aws/rust-runtime/aws-http/README.md b/aws/rust-runtime/aws-http/README.md index b4d264c8230..3e85fe86833 100644 --- a/aws/rust-runtime/aws-http/README.md +++ b/aws/rust-runtime/aws-http/README.md @@ -1,9 +1,6 @@ # aws-http -This crate provides middleware for AWS SDKs using HTTP including: -* Generalized retry policy -* Middleware for setting `User-Agent` headers based on runtime configuration -* Credential loading async middleware +This crate is no longer used by the AWS SDK and is deprecated. This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/smithy-lang/smithy-rs) code generator. In most cases, it should not be used directly. diff --git a/aws/rust-runtime/aws-http/external-types.toml b/aws/rust-runtime/aws-http/external-types.toml index 429bf1a9098..db178fc895d 100644 --- a/aws/rust-runtime/aws-http/external-types.toml +++ b/aws/rust-runtime/aws-http/external-types.toml @@ -1,11 +1,3 @@ allowed_external_types = [ - "aws_smithy_runtime_api::http::headers::Headers", - "aws_smithy_types::body::Error", - "aws_smithy_types::config_bag::storable::Storable", - "aws_smithy_types::config_bag::storable::StoreReplace", - "aws_smithy_types::error::metadata::Builder", - "aws_types::app_name::AppName", - "aws_types::os_shim_internal::Env", - "bytes::bytes::Bytes", - "http_body::Body", + "aws_runtime::*", ] diff --git a/aws/rust-runtime/aws-http/src/content_encoding.rs b/aws/rust-runtime/aws-http/src/content_encoding.rs index 2c00cff9cb9..7e1becf6432 100644 --- a/aws/rust-runtime/aws-http/src/content_encoding.rs +++ b/aws/rust-runtime/aws-http/src/content_encoding.rs @@ -3,611 +3,30 @@ * SPDX-License-Identifier: Apache-2.0 */ -use bytes::{Bytes, BytesMut}; -use http::{HeaderMap, HeaderValue}; -use http_body::{Body, SizeHint}; -use pin_project_lite::pin_project; - -use std::pin::Pin; -use std::task::{Context, Poll}; - -const CRLF: &str = "\r\n"; -const CHUNK_TERMINATOR: &str = "0\r\n"; -const TRAILER_SEPARATOR: &[u8] = b":"; - -/// Content encoding header value constants +/// Use aws_runtime::content_encoding::header_value instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::content_encoding::header_value instead." +)] pub mod header_value { - /// Header value denoting "aws-chunked" encoding + /// Use aws_runtime::content_encoding::header_value::AWS_CHUNKED instead. + #[deprecated( + since = "0.60.2", + note = "Use aws_runtime::content_encoding::header_value::AWS_CHUNKED instead." + )] pub const AWS_CHUNKED: &str = "aws-chunked"; } -/// Options used when constructing an [`AwsChunkedBody`]. -#[derive(Debug, Default)] -#[non_exhaustive] -pub struct AwsChunkedBodyOptions { - /// The total size of the stream. Because we only support unsigned encoding - /// this implies that there will only be a single chunk containing the - /// underlying payload. - stream_length: u64, - /// The length of each trailer sent within an `AwsChunkedBody`. Necessary in - /// order to correctly calculate the total size of the body accurately. - trailer_lengths: Vec, -} - -impl AwsChunkedBodyOptions { - /// Create a new [`AwsChunkedBodyOptions`]. - pub fn new(stream_length: u64, trailer_lengths: Vec) -> Self { - Self { - stream_length, - trailer_lengths, - } - } - - fn total_trailer_length(&self) -> u64 { - self.trailer_lengths.iter().sum::() - // We need to account for a CRLF after each trailer name/value pair - + (self.trailer_lengths.len() * CRLF.len()) as u64 - } - - /// Set a trailer len - pub fn with_trailer_len(mut self, trailer_len: u64) -> Self { - self.trailer_lengths.push(trailer_len); - self - } -} - -#[derive(Debug, PartialEq, Eq)] -enum AwsChunkedBodyState { - /// Write out the size of the chunk that will follow. Then, transition into the - /// `WritingChunk` state. - WritingChunkSize, - /// Write out the next chunk of data. Multiple polls of the inner body may need to occur before - /// all data is written out. Once there is no more data to write, transition into the - /// `WritingTrailers` state. - WritingChunk, - /// Write out all trailers associated with this `AwsChunkedBody` and then transition into the - /// `Closed` state. - WritingTrailers, - /// This is the final state. Write out the body terminator and then remain in this state. - Closed, -} - -pin_project! { - /// A request body compatible with `Content-Encoding: aws-chunked`. This implementation is only - /// capable of writing a single chunk and does not support signed chunks. - /// - /// Chunked-Body grammar is defined in [ABNF] as: - /// - /// ```txt - /// Chunked-Body = *chunk - /// last-chunk - /// chunked-trailer - /// CRLF - /// - /// chunk = chunk-size CRLF chunk-data CRLF - /// chunk-size = 1*HEXDIG - /// last-chunk = 1*("0") CRLF - /// chunked-trailer = *( entity-header CRLF ) - /// entity-header = field-name ":" OWS field-value OWS - /// ``` - /// For more info on what the abbreviations mean, see https://datatracker.ietf.org/doc/html/rfc7230#section-1.2 - /// - /// [ABNF]:https://en.wikipedia.org/wiki/Augmented_Backus%E2%80%93Naur_form - #[derive(Debug)] - pub struct AwsChunkedBody { - #[pin] - inner: InnerBody, - #[pin] - state: AwsChunkedBodyState, - options: AwsChunkedBodyOptions, - inner_body_bytes_read_so_far: usize, - } -} - -impl AwsChunkedBody { - /// Wrap the given body in an outer body compatible with `Content-Encoding: aws-chunked` - pub fn new(body: Inner, options: AwsChunkedBodyOptions) -> Self { - Self { - inner: body, - state: AwsChunkedBodyState::WritingChunkSize, - options, - inner_body_bytes_read_so_far: 0, - } - } - - fn encoded_length(&self) -> u64 { - let mut length = 0; - if self.options.stream_length != 0 { - length += get_unsigned_chunk_bytes_length(self.options.stream_length); - } - - // End chunk - length += CHUNK_TERMINATOR.len() as u64; - - // Trailers - for len in self.options.trailer_lengths.iter() { - length += len + CRLF.len() as u64; - } - - // Encoding terminator - length += CRLF.len() as u64; - - length - } -} - -fn get_unsigned_chunk_bytes_length(payload_length: u64) -> u64 { - let hex_repr_len = int_log16(payload_length); - hex_repr_len + CRLF.len() as u64 + payload_length + CRLF.len() as u64 -} - -/// Writes trailers out into a `string` and then converts that `String` to a `Bytes` before -/// returning. -/// -/// - Trailer names are separated by a single colon only, no space. -/// - Trailer names with multiple values will be written out one line per value, with the name -/// appearing on each line. -fn trailers_as_aws_chunked_bytes( - trailer_map: Option, - estimated_length: u64, -) -> BytesMut { - if let Some(trailer_map) = trailer_map { - let mut current_header_name = None; - let mut trailers = BytesMut::with_capacity(estimated_length.try_into().unwrap_or_default()); - - for (header_name, header_value) in trailer_map.into_iter() { - // When a header has multiple values, the name only comes up in iteration the first time - // we see it. Therefore, we need to keep track of the last name we saw and fall back to - // it when `header_name == None`. - current_header_name = header_name.or(current_header_name); - - // In practice, this will always exist, but `if let` is nicer than unwrap - if let Some(header_name) = current_header_name.as_ref() { - trailers.extend_from_slice(header_name.as_ref()); - trailers.extend_from_slice(TRAILER_SEPARATOR); - trailers.extend_from_slice(header_value.as_bytes()); - trailers.extend_from_slice(CRLF.as_bytes()); - } - } - - trailers - } else { - BytesMut::new() - } -} - -/// Given an optional `HeaderMap`, calculate the total number of bytes required to represent the -/// `HeaderMap`. If no `HeaderMap` is given as input, return 0. -/// -/// - Trailer names are separated by a single colon only, no space. -/// - Trailer names with multiple values will be written out one line per value, with the name -/// appearing on each line. -fn total_rendered_length_of_trailers(trailer_map: Option<&HeaderMap>) -> u64 { - match trailer_map { - Some(trailer_map) => trailer_map - .iter() - .map(|(trailer_name, trailer_value)| { - trailer_name.as_str().len() - + TRAILER_SEPARATOR.len() - + trailer_value.len() - + CRLF.len() - }) - .sum::() as u64, - None => 0, - } -} - -impl Body for AwsChunkedBody -where - Inner: Body, -{ - type Data = Bytes; - type Error = aws_smithy_types::body::Error; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - tracing::trace!(state = ?self.state, "polling AwsChunkedBody"); - let mut this = self.project(); - - match *this.state { - AwsChunkedBodyState::WritingChunkSize => { - if this.options.stream_length == 0 { - // If the stream is empty, we skip to writing trailers after writing the CHUNK_TERMINATOR. - *this.state = AwsChunkedBodyState::WritingTrailers; - tracing::trace!("stream is empty, writing chunk terminator"); - Poll::Ready(Some(Ok(Bytes::from([CHUNK_TERMINATOR].concat())))) - } else { - *this.state = AwsChunkedBodyState::WritingChunk; - // A chunk must be prefixed by chunk size in hexadecimal - let chunk_size = format!("{:X?}{CRLF}", this.options.stream_length); - tracing::trace!(%chunk_size, "writing chunk size"); - let chunk_size = Bytes::from(chunk_size); - Poll::Ready(Some(Ok(chunk_size))) - } - } - AwsChunkedBodyState::WritingChunk => match this.inner.poll_data(cx) { - Poll::Ready(Some(Ok(data))) => { - tracing::trace!(len = data.len(), "writing chunk data"); - *this.inner_body_bytes_read_so_far += data.len(); - Poll::Ready(Some(Ok(data))) - } - Poll::Ready(None) => { - let actual_stream_length = *this.inner_body_bytes_read_so_far as u64; - let expected_stream_length = this.options.stream_length; - if actual_stream_length != expected_stream_length { - let err = Box::new(AwsChunkedBodyError::StreamLengthMismatch { - actual: actual_stream_length, - expected: expected_stream_length, - }); - return Poll::Ready(Some(Err(err))); - }; - - tracing::trace!("no more chunk data, writing CRLF and chunk terminator"); - *this.state = AwsChunkedBodyState::WritingTrailers; - // Since we wrote chunk data, we end it with a CRLF and since we only write - // a single chunk, we write the CHUNK_TERMINATOR immediately after - Poll::Ready(Some(Ok(Bytes::from([CRLF, CHUNK_TERMINATOR].concat())))) - } - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Pending => Poll::Pending, - }, - AwsChunkedBodyState::WritingTrailers => { - return match this.inner.poll_trailers(cx) { - Poll::Ready(Ok(trailers)) => { - *this.state = AwsChunkedBodyState::Closed; - let expected_length = total_rendered_length_of_trailers(trailers.as_ref()); - let actual_length = this.options.total_trailer_length(); - - if expected_length != actual_length { - let err = - Box::new(AwsChunkedBodyError::ReportedTrailerLengthMismatch { - actual: actual_length, - expected: expected_length, - }); - return Poll::Ready(Some(Err(err))); - } - - let mut trailers = - trailers_as_aws_chunked_bytes(trailers, actual_length + 1); - // Insert the final CRLF to close the body - trailers.extend_from_slice(CRLF.as_bytes()); - - Poll::Ready(Some(Ok(trailers.into()))) - } - Poll::Pending => Poll::Pending, - Poll::Ready(Err(e)) => Poll::Ready(Some(Err(e))), - }; - } - AwsChunkedBodyState::Closed => Poll::Ready(None), - } - } - - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll>, Self::Error>> { - // Trailers were already appended to the body because of the content encoding scheme - Poll::Ready(Ok(None)) - } - - fn is_end_stream(&self) -> bool { - self.state == AwsChunkedBodyState::Closed - } - - fn size_hint(&self) -> SizeHint { - SizeHint::with_exact(self.encoded_length()) - } -} - -/// Errors related to `AwsChunkedBody` -#[derive(Debug)] -enum AwsChunkedBodyError { - /// Error that occurs when the sum of `trailer_lengths` set when creating an `AwsChunkedBody` is - /// not equal to the actual length of the trailers returned by the inner `http_body::Body` - /// implementor. These trailer lengths are necessary in order to correctly calculate the total - /// size of the body for setting the content length header. - ReportedTrailerLengthMismatch { actual: u64, expected: u64 }, - /// Error that occurs when the `stream_length` set when creating an `AwsChunkedBody` is not - /// equal to the actual length of the body returned by the inner `http_body::Body` implementor. - /// `stream_length` must be correct in order to set an accurate content length header. - StreamLengthMismatch { actual: u64, expected: u64 }, -} - -impl std::fmt::Display for AwsChunkedBodyError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::ReportedTrailerLengthMismatch { actual, expected } => { - write!(f, "When creating this AwsChunkedBody, length of trailers was reported as {expected}. However, when double checking during trailer encoding, length was found to be {actual} instead.") - } - Self::StreamLengthMismatch { actual, expected } => { - write!(f, "When creating this AwsChunkedBody, stream length was reported as {expected}. However, when double checking during body encoding, length was found to be {actual} instead.") - } - } - } -} - -impl std::error::Error for AwsChunkedBodyError {} - -// Used for finding how many hexadecimal digits it takes to represent a base 10 integer -fn int_log16(mut i: T) -> u64 -where - T: std::ops::DivAssign + PartialOrd + From + Copy, -{ - let mut len = 0; - let zero = T::from(0); - let sixteen = T::from(16); - - while i > zero { - i /= sixteen; - len += 1; - } - - len -} - -#[cfg(test)] -mod tests { - use super::{ - total_rendered_length_of_trailers, trailers_as_aws_chunked_bytes, AwsChunkedBody, - AwsChunkedBodyOptions, CHUNK_TERMINATOR, CRLF, - }; - - use aws_smithy_types::body::SdkBody; - use bytes::{Buf, Bytes}; - use bytes_utils::SegmentedBuf; - use http::{HeaderMap, HeaderValue}; - use http_body::{Body, SizeHint}; - use pin_project_lite::pin_project; - - use std::io::Read; - use std::pin::Pin; - use std::task::{Context, Poll}; - use std::time::Duration; - - pin_project! { - struct SputteringBody { - parts: Vec>, - cursor: usize, - delay_in_millis: u64, - } - } - - impl SputteringBody { - fn len(&self) -> usize { - self.parts.iter().flatten().map(|b| b.len()).sum() - } - } - - impl Body for SputteringBody { - type Data = Bytes; - type Error = aws_smithy_types::body::Error; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - if self.cursor == self.parts.len() { - return Poll::Ready(None); - } - - let this = self.project(); - let delay_in_millis = *this.delay_in_millis; - let next_part = this.parts.get_mut(*this.cursor).unwrap().take(); - - match next_part { - None => { - *this.cursor += 1; - let waker = cx.waker().clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(delay_in_millis)).await; - waker.wake(); - }); - Poll::Pending - } - Some(data) => { - *this.cursor += 1; - Poll::Ready(Some(Ok(data))) - } - } - } - - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll>, Self::Error>> { - Poll::Ready(Ok(None)) - } - - fn is_end_stream(&self) -> bool { - false - } - - fn size_hint(&self) -> SizeHint { - SizeHint::new() - } - } - - #[tokio::test] - async fn test_aws_chunked_encoding() { - let test_fut = async { - let input_str = "Hello world"; - let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, Vec::new()); - let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); - - let mut output = SegmentedBuf::new(); - while let Some(buf) = body.data().await { - output.push(buf.unwrap()); - } - - let mut actual_output = String::new(); - output - .reader() - .read_to_string(&mut actual_output) - .expect("Doesn't cause IO errors"); - - let expected_output = "B\r\nHello world\r\n0\r\n\r\n"; - - assert_eq!(expected_output, actual_output); - assert!( - body.trailers() - .await - .expect("no errors occurred during trailer polling") - .is_none(), - "aws-chunked encoded bodies don't have normal HTTP trailers" - ); - - // You can insert a `tokio::time::sleep` here to verify the timeout works as intended - }; - - let timeout_duration = Duration::from_secs(3); - if tokio::time::timeout(timeout_duration, test_fut) - .await - .is_err() - { - panic!("test_aws_chunked_encoding timed out after {timeout_duration:?}"); - } - } - - #[tokio::test] - async fn test_aws_chunked_encoding_sputtering_body() { - let test_fut = async { - let input = SputteringBody { - parts: vec![ - Some(Bytes::from_static(b"chunk 1, ")), - None, - Some(Bytes::from_static(b"chunk 2, ")), - Some(Bytes::from_static(b"chunk 3, ")), - None, - None, - Some(Bytes::from_static(b"chunk 4, ")), - Some(Bytes::from_static(b"chunk 5, ")), - Some(Bytes::from_static(b"chunk 6")), - ], - cursor: 0, - delay_in_millis: 500, - }; - let opts = AwsChunkedBodyOptions::new(input.len() as u64, Vec::new()); - let mut body = AwsChunkedBody::new(input, opts); +/// Use aws_runtime::content_encoding::AwsChunkedBodyOption instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::content_encoding::AwsChunkedBodyOptions instead." +)] +pub type AwsChunkedBodyOptions = aws_runtime::content_encoding::AwsChunkedBodyOptions; - let mut output = SegmentedBuf::new(); - while let Some(buf) = body.data().await { - output.push(buf.unwrap()); - } - - let mut actual_output = String::new(); - output - .reader() - .read_to_string(&mut actual_output) - .expect("Doesn't cause IO errors"); - - let expected_output = - "34\r\nchunk 1, chunk 2, chunk 3, chunk 4, chunk 5, chunk 6\r\n0\r\n\r\n"; - - assert_eq!(expected_output, actual_output); - assert!( - body.trailers() - .await - .expect("no errors occurred during trailer polling") - .is_none(), - "aws-chunked encoded bodies don't have normal HTTP trailers" - ); - }; - - let timeout_duration = Duration::from_secs(3); - if tokio::time::timeout(timeout_duration, test_fut) - .await - .is_err() - { - panic!( - "test_aws_chunked_encoding_sputtering_body timed out after {timeout_duration:?}" - ); - } - } - - #[tokio::test] - #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 44, expected: 0 }"] - async fn test_aws_chunked_encoding_incorrect_trailer_length_panic() { - let input_str = "Hello world"; - // Test body has no trailers, so this length is incorrect and will trigger an assert panic - // When the panic occurs, it will actually expect a length of 44. This is because, when using - // aws-chunked encoding, each trailer will end with a CRLF which is 2 bytes long. - let wrong_trailer_len = 42; - let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, vec![wrong_trailer_len]); - let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); - - // We don't care about the body contents but we have to read it all before checking for trailers - while let Some(buf) = body.data().await { - drop(buf.unwrap()); - } - - assert!( - body.trailers() - .await - .expect("no errors occurred during trailer polling") - .is_none(), - "aws-chunked encoded bodies don't have normal HTTP trailers" - ); - } - - #[tokio::test] - async fn test_aws_chunked_encoding_empty_body() { - let input_str = ""; - let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, Vec::new()); - let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); - - let mut output = SegmentedBuf::new(); - while let Some(buf) = body.data().await { - output.push(buf.unwrap()); - } - - let mut actual_output = String::new(); - output - .reader() - .read_to_string(&mut actual_output) - .expect("Doesn't cause IO errors"); - - let expected_output = [CHUNK_TERMINATOR, CRLF].concat(); - - assert_eq!(expected_output, actual_output); - assert!( - body.trailers() - .await - .expect("no errors occurred during trailer polling") - .is_none(), - "aws-chunked encoded bodies don't have normal HTTP trailers" - ); - } - - #[tokio::test] - async fn test_total_rendered_length_of_trailers() { - let mut trailers = HeaderMap::new(); - - trailers.insert("empty_value", HeaderValue::from_static("")); - - trailers.insert("single_value", HeaderValue::from_static("value 1")); - - trailers.insert("two_values", HeaderValue::from_static("value 1")); - trailers.append("two_values", HeaderValue::from_static("value 2")); - - trailers.insert("three_values", HeaderValue::from_static("value 1")); - trailers.append("three_values", HeaderValue::from_static("value 2")); - trailers.append("three_values", HeaderValue::from_static("value 3")); - - let trailers = Some(trailers); - let actual_length = total_rendered_length_of_trailers(trailers.as_ref()); - let expected_length = (trailers_as_aws_chunked_bytes(trailers, actual_length).len()) as u64; - - assert_eq!(expected_length, actual_length); - } - - #[tokio::test] - async fn test_total_rendered_length_of_empty_trailers() { - let trailers = Some(HeaderMap::new()); - let actual_length = total_rendered_length_of_trailers(trailers.as_ref()); - let expected_length = (trailers_as_aws_chunked_bytes(trailers, actual_length).len()) as u64; - - assert_eq!(expected_length, actual_length); - } -} +/// Use aws_runtime::content_encoding::AwsChunkedBody instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::content_encoding::AwsChunkedBody instead." +)] +pub type AwsChunkedBody = aws_runtime::content_encoding::AwsChunkedBody; diff --git a/aws/rust-runtime/aws-http/src/user_agent.rs b/aws/rust-runtime/aws-http/src/user_agent.rs index 9d92723720c..f68ecf91bee 100644 --- a/aws/rust-runtime/aws-http/src/user_agent.rs +++ b/aws/rust-runtime/aws-http/src/user_agent.rs @@ -3,736 +3,51 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_smithy_types::config_bag::{Storable, StoreReplace}; -use aws_types::app_name::AppName; -use aws_types::build_metadata::{OsFamily, BUILD_METADATA}; -use aws_types::os_shim_internal::Env; -use std::borrow::Cow; -use std::error::Error; -use std::fmt; - -/// AWS User Agent -/// -/// Ths struct should be inserted into the [`ConfigBag`](aws_smithy_types::config_bag::ConfigBag) -/// during operation construction. The `UserAgentInterceptor` reads `AwsUserAgent` -/// from the config bag and sets the `User-Agent` and `x-amz-user-agent` headers. -#[derive(Clone, Debug)] -pub struct AwsUserAgent { - sdk_metadata: SdkMetadata, - api_metadata: ApiMetadata, - os_metadata: OsMetadata, - language_metadata: LanguageMetadata, - exec_env_metadata: Option, - feature_metadata: Vec, - config_metadata: Vec, - framework_metadata: Vec, - app_name: Option, - build_env_additional_metadata: Option, -} - -impl AwsUserAgent { - /// Load a User Agent configuration from the environment - /// - /// This utilizes [`BUILD_METADATA`](const@aws_types::build_metadata::BUILD_METADATA) from `aws_types` - /// to capture the Rust version & target platform. `ApiMetadata` provides - /// the version & name of the specific service. - pub fn new_from_environment(env: Env, api_metadata: ApiMetadata) -> Self { - let build_metadata = &BUILD_METADATA; - let sdk_metadata = SdkMetadata { - name: "rust", - version: build_metadata.core_pkg_version, - }; - let os_metadata = OsMetadata { - os_family: &build_metadata.os_family, - version: None, - }; - let exec_env_metadata = env - .get("AWS_EXECUTION_ENV") - .ok() - .map(|name| ExecEnvMetadata { name }); - - // Retrieve additional metadata at compile-time from the AWS_SDK_RUST_BUILD_UA_METADATA env var - let build_env_additional_metadata = option_env!("AWS_SDK_RUST_BUILD_UA_METADATA") - .and_then(|value| AdditionalMetadata::new(value).ok()); - - AwsUserAgent { - sdk_metadata, - api_metadata, - os_metadata, - language_metadata: LanguageMetadata { - lang: "rust", - version: BUILD_METADATA.rust_version, - extras: Default::default(), - }, - exec_env_metadata, - feature_metadata: Default::default(), - config_metadata: Default::default(), - framework_metadata: Default::default(), - app_name: Default::default(), - build_env_additional_metadata, - } - } - - /// For test purposes, construct an environment-independent User Agent - /// - /// Without this, running CI on a different platform would produce different user agent strings - pub fn for_tests() -> Self { - Self { - sdk_metadata: SdkMetadata { - name: "rust", - version: "0.123.test", - }, - api_metadata: ApiMetadata { - service_id: "test-service".into(), - version: "0.123", - }, - os_metadata: OsMetadata { - os_family: &OsFamily::Windows, - version: Some("XPSP3".to_string()), - }, - language_metadata: LanguageMetadata { - lang: "rust", - version: "1.50.0", - extras: Default::default(), - }, - exec_env_metadata: None, - feature_metadata: Vec::new(), - config_metadata: Vec::new(), - framework_metadata: Vec::new(), - app_name: None, - build_env_additional_metadata: None, - } - } - - #[doc(hidden)] - /// Adds feature metadata to the user agent. - pub fn with_feature_metadata(mut self, metadata: FeatureMetadata) -> Self { - self.feature_metadata.push(metadata); - self - } - - #[doc(hidden)] - /// Adds feature metadata to the user agent. - pub fn add_feature_metadata(&mut self, metadata: FeatureMetadata) -> &mut Self { - self.feature_metadata.push(metadata); - self - } - - #[doc(hidden)] - /// Adds config metadata to the user agent. - pub fn with_config_metadata(mut self, metadata: ConfigMetadata) -> Self { - self.config_metadata.push(metadata); - self - } - - #[doc(hidden)] - /// Adds config metadata to the user agent. - pub fn add_config_metadata(&mut self, metadata: ConfigMetadata) -> &mut Self { - self.config_metadata.push(metadata); - self - } - - #[doc(hidden)] - /// Adds framework metadata to the user agent. - pub fn with_framework_metadata(mut self, metadata: FrameworkMetadata) -> Self { - self.framework_metadata.push(metadata); - self - } - - #[doc(hidden)] - /// Adds framework metadata to the user agent. - pub fn add_framework_metadata(&mut self, metadata: FrameworkMetadata) -> &mut Self { - self.framework_metadata.push(metadata); - self - } - - /// Sets the app name for the user agent. - pub fn with_app_name(mut self, app_name: AppName) -> Self { - self.app_name = Some(app_name); - self - } - - /// Sets the app name for the user agent. - pub fn set_app_name(&mut self, app_name: AppName) -> &mut Self { - self.app_name = Some(app_name); - self - } - - /// Generate a new-style user agent style header - /// - /// This header should be set at `x-amz-user-agent` - pub fn aws_ua_header(&self) -> String { - /* - ABNF for the user agent (see the bottom of the file for complete ABNF): - ua-string = sdk-metadata RWS - [api-metadata RWS] - os-metadata RWS - language-metadata RWS - [env-metadata RWS] - *(feat-metadata RWS) - *(config-metadata RWS) - *(framework-metadata RWS) - [appId] - */ - let mut ua_value = String::new(); - use std::fmt::Write; - // unwrap calls should never fail because string formatting will always succeed. - write!(ua_value, "{} ", &self.sdk_metadata).unwrap(); - write!(ua_value, "{} ", &self.api_metadata).unwrap(); - write!(ua_value, "{} ", &self.os_metadata).unwrap(); - write!(ua_value, "{} ", &self.language_metadata).unwrap(); - if let Some(ref env_meta) = self.exec_env_metadata { - write!(ua_value, "{} ", env_meta).unwrap(); - } - for feature in &self.feature_metadata { - write!(ua_value, "{} ", feature).unwrap(); - } - for config in &self.config_metadata { - write!(ua_value, "{} ", config).unwrap(); - } - for framework in &self.framework_metadata { - write!(ua_value, "{} ", framework).unwrap(); - } - if let Some(app_name) = &self.app_name { - write!(ua_value, "app/{}", app_name).unwrap(); - } - if let Some(additional_metadata) = &self.build_env_additional_metadata { - write!(ua_value, "{}", additional_metadata).unwrap(); - } - if ua_value.ends_with(' ') { - ua_value.truncate(ua_value.len() - 1); - } - ua_value - } - - /// Generate an old-style User-Agent header for backward compatibility - /// - /// This header is intended to be set at `User-Agent` - pub fn ua_header(&self) -> String { - let mut ua_value = String::new(); - use std::fmt::Write; - write!(ua_value, "{} ", &self.sdk_metadata).unwrap(); - write!(ua_value, "{} ", &self.os_metadata).unwrap(); - write!(ua_value, "{}", &self.language_metadata).unwrap(); - ua_value - } -} - -impl Storable for AwsUserAgent { - type Storer = StoreReplace; -} - -#[derive(Clone, Copy, Debug)] -struct SdkMetadata { - name: &'static str, - version: &'static str, -} - -impl fmt::Display for SdkMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "aws-sdk-{}/{}", self.name, self.version) - } -} - -/// Metadata about the client that's making the call. -#[derive(Clone, Debug)] -pub struct ApiMetadata { - service_id: Cow<'static, str>, - version: &'static str, -} - -impl ApiMetadata { - /// Creates new `ApiMetadata`. - pub const fn new(service_id: &'static str, version: &'static str) -> Self { - Self { - service_id: Cow::Borrowed(service_id), - version, - } - } -} - -impl fmt::Display for ApiMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "api/{}/{}", self.service_id, self.version) - } -} - -impl Storable for ApiMetadata { - type Storer = StoreReplace; -} - -/// Error for when an user agent metadata doesn't meet character requirements. -/// -/// Metadata may only have alphanumeric characters and any of these characters: -/// ```text -/// !#$%&'*+-.^_`|~ -/// ``` -/// Spaces are not allowed. -#[derive(Debug)] -#[non_exhaustive] -pub struct InvalidMetadataValue; - -impl Error for InvalidMetadataValue {} - -impl fmt::Display for InvalidMetadataValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "User agent metadata can only have alphanumeric characters, or any of \ - '!' | '#' | '$' | '%' | '&' | '\\'' | '*' | '+' | '-' | \ - '.' | '^' | '_' | '`' | '|' | '~'" - ) - } -} - -fn validate_metadata(value: Cow<'static, str>) -> Result, InvalidMetadataValue> { - fn valid_character(c: char) -> bool { - match c { - _ if c.is_ascii_alphanumeric() => true, - '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | '-' | '.' | '^' | '_' | '`' | '|' - | '~' => true, - _ => false, - } - } - if !value.chars().all(valid_character) { - return Err(InvalidMetadataValue); - } - Ok(value) -} - -#[doc(hidden)] -/// Additional metadata that can be bundled with framework or feature metadata. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct AdditionalMetadata { - value: Cow<'static, str>, -} - -impl AdditionalMetadata { - /// Creates `AdditionalMetadata`. - /// - /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or - /// has characters other than the following: - /// ```text - /// !#$%&'*+-.^_`|~ - /// ``` - pub fn new(value: impl Into>) -> Result { - Ok(Self { - value: validate_metadata(value.into())?, - }) - } -} - -impl fmt::Display for AdditionalMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // additional-metadata = "md/" ua-pair - write!(f, "md/{}", self.value) - } -} - -#[derive(Clone, Debug, Default)] -struct AdditionalMetadataList(Vec); - -impl AdditionalMetadataList { - fn push(&mut self, metadata: AdditionalMetadata) { - self.0.push(metadata); - } -} - -impl fmt::Display for AdditionalMetadataList { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for metadata in &self.0 { - write!(f, " {}", metadata)?; - } - Ok(()) - } -} - -#[doc(hidden)] -/// Metadata about a feature that is being used in the SDK. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct FeatureMetadata { - name: Cow<'static, str>, - version: Option>, - additional: AdditionalMetadataList, -} - -impl FeatureMetadata { - /// Creates `FeatureMetadata`. - /// - /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or - /// has characters other than the following: - /// ```text - /// !#$%&'*+-.^_`|~ - /// ``` - pub fn new( - name: impl Into>, - version: Option>, - ) -> Result { - Ok(Self { - name: validate_metadata(name.into())?, - version: version.map(validate_metadata).transpose()?, - additional: Default::default(), - }) - } - - /// Bundles additional arbitrary metadata with this feature metadata. - pub fn with_additional(mut self, metadata: AdditionalMetadata) -> Self { - self.additional.push(metadata); - self - } -} - -impl fmt::Display for FeatureMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // feat-metadata = "ft/" name ["/" version] *(RWS additional-metadata) - if let Some(version) = &self.version { - write!(f, "ft/{}/{}{}", self.name, version, self.additional) - } else { - write!(f, "ft/{}{}", self.name, self.additional) - } - } -} - -#[doc(hidden)] -/// Metadata about a config value that is being used in the SDK. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct ConfigMetadata { - config: Cow<'static, str>, - value: Option>, -} - -impl ConfigMetadata { - /// Creates `ConfigMetadata`. - /// - /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or - /// has characters other than the following: - /// ```text - /// !#$%&'*+-.^_`|~ - /// ``` - pub fn new( - config: impl Into>, - value: Option>, - ) -> Result { - Ok(Self { - config: validate_metadata(config.into())?, - value: value.map(validate_metadata).transpose()?, - }) - } -} - -impl fmt::Display for ConfigMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // config-metadata = "cfg/" config ["/" value] - if let Some(value) = &self.value { - write!(f, "cfg/{}/{}", self.config, value) - } else { - write!(f, "cfg/{}", self.config) - } - } -} - -#[doc(hidden)] -/// Metadata about a software framework that is being used with the SDK. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct FrameworkMetadata { - name: Cow<'static, str>, - version: Option>, - additional: AdditionalMetadataList, -} - -impl FrameworkMetadata { - /// Creates `FrameworkMetadata`. - /// - /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or - /// has characters other than the following: - /// ```text - /// !#$%&'*+-.^_`|~ - /// ``` - pub fn new( - name: impl Into>, - version: Option>, - ) -> Result { - Ok(Self { - name: validate_metadata(name.into())?, - version: version.map(validate_metadata).transpose()?, - additional: Default::default(), - }) - } - - /// Bundles additional arbitrary metadata with this framework metadata. - pub fn with_additional(mut self, metadata: AdditionalMetadata) -> Self { - self.additional.push(metadata); - self - } -} - -impl fmt::Display for FrameworkMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // framework-metadata = "lib/" name ["/" version] *(RWS additional-metadata) - if let Some(version) = &self.version { - write!(f, "lib/{}/{}{}", self.name, version, self.additional) - } else { - write!(f, "lib/{}{}", self.name, self.additional) - } - } -} - -#[derive(Clone, Debug)] -struct OsMetadata { - os_family: &'static OsFamily, - version: Option, -} - -impl fmt::Display for OsMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let os_family = match self.os_family { - OsFamily::Windows => "windows", - OsFamily::Linux => "linux", - OsFamily::Macos => "macos", - OsFamily::Android => "android", - OsFamily::Ios => "ios", - OsFamily::Other => "other", - }; - write!(f, "os/{}", os_family)?; - if let Some(ref version) = self.version { - write!(f, "/{}", version)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug)] -struct LanguageMetadata { - lang: &'static str, - version: &'static str, - extras: AdditionalMetadataList, -} -impl fmt::Display for LanguageMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // language-metadata = "lang/" language "/" version *(RWS additional-metadata) - write!(f, "lang/{}/{}{}", self.lang, self.version, self.extras) - } -} - -#[derive(Clone, Debug)] -struct ExecEnvMetadata { - name: String, -} -impl fmt::Display for ExecEnvMetadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "exec-env/{}", &self.name) - } -} - -#[cfg(test)] -mod test { - use super::*; - use aws_types::app_name::AppName; - use aws_types::build_metadata::OsFamily; - use aws_types::os_shim_internal::Env; - use std::borrow::Cow; - - fn make_deterministic(ua: &mut AwsUserAgent) { - // hard code some variable things for a deterministic test - ua.sdk_metadata.version = "0.1"; - ua.language_metadata.version = "1.50.0"; - ua.os_metadata.os_family = &OsFamily::Macos; - ua.os_metadata.version = Some("1.15".to_string()); - } - - #[test] - fn generate_a_valid_ua() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_execution_env() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment( - Env::from_slice(&[("AWS_EXECUTION_ENV", "lambda")]), - api_metadata, - ); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 exec-env/lambda" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_features() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) - .with_feature_metadata( - FeatureMetadata::new("test-feature", Some(Cow::Borrowed("1.0"))).unwrap(), - ) - .with_feature_metadata( - FeatureMetadata::new("other-feature", None) - .unwrap() - .with_additional(AdditionalMetadata::new("asdf").unwrap()), - ); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 ft/test-feature/1.0 ft/other-feature md/asdf" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_config() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) - .with_config_metadata( - ConfigMetadata::new("some-config", Some(Cow::Borrowed("5"))).unwrap(), - ) - .with_config_metadata(ConfigMetadata::new("other-config", None).unwrap()); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 cfg/some-config/5 cfg/other-config" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_frameworks() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) - .with_framework_metadata( - FrameworkMetadata::new("some-framework", Some(Cow::Borrowed("1.3"))) - .unwrap() - .with_additional(AdditionalMetadata::new("something").unwrap()), - ) - .with_framework_metadata(FrameworkMetadata::new("other", None).unwrap()); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 lib/some-framework/1.3 md/something lib/other" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_app_name() { - let api_metadata = ApiMetadata { - service_id: "dynamodb".into(), - version: "123", - }; - let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) - .with_app_name(AppName::new("my_app").unwrap()); - make_deterministic(&mut ua); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 app/my_app" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" - ); - } - - #[test] - fn generate_a_valid_ua_with_build_env_additional_metadata() { - let mut ua = AwsUserAgent::for_tests(); - ua.build_env_additional_metadata = Some(AdditionalMetadata::new("asdf").unwrap()); - assert_eq!( - ua.aws_ua_header(), - "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0 md/asdf" - ); - assert_eq!( - ua.ua_header(), - "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" - ); - } -} - -/* -Appendix: User Agent ABNF -sdk-ua-header = "x-amz-user-agent:" OWS ua-string OWS -ua-pair = ua-name ["/" ua-value] -ua-name = token -ua-value = token -version = token -name = token -service-id = token -sdk-name = java / ruby / php / dotnet / python / cli / kotlin / rust / js / cpp / go / go-v2 -os-family = windows / linux / macos / android / ios / other -config = retry-mode -additional-metadata = "md/" ua-pair -sdk-metadata = "aws-sdk-" sdk-name "/" version -api-metadata = "api/" service-id "/" version -os-metadata = "os/" os-family ["/" version] -language-metadata = "lang/" language "/" version *(RWS additional-metadata) -env-metadata = "exec-env/" name -feat-metadata = "ft/" name ["/" version] *(RWS additional-metadata) -config-metadata = "cfg/" config ["/" value] -framework-metadata = "lib/" name ["/" version] *(RWS additional-metadata) -app-id = "app/" name -build-env-additional-metadata = "md/" value -ua-string = sdk-metadata RWS - [api-metadata RWS] - os-metadata RWS - language-metadata RWS - [env-metadata RWS] - *(feat-metadata RWS) - *(config-metadata RWS) - *(framework-metadata RWS) - [app-id] - [build-env-additional-metadata] - -# New metadata field might be added in the future and they must follow this format -prefix = token -metadata = prefix "/" ua-pair - -# token, RWS and OWS are defined in [RFC 7230](https://tools.ietf.org/html/rfc7230) -OWS = *( SP / HTAB ) - ; optional whitespace -RWS = 1*( SP / HTAB ) - ; required whitespace -token = 1*tchar -tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / - "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA -*/ +/// Use aws_runtime::user_agent::AwsUserAgent instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::AwsUserAgent instead." +)] +pub type AwsUserAgent = aws_runtime::user_agent::AwsUserAgent; + +/// Use aws_runtime::user_agent::ApiMetadata instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::ApiMetadata instead." +)] +pub type ApiMetadata = aws_runtime::user_agent::ApiMetadata; + +/// Use aws_runtime::user_agent::InvalidMetadataValue instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::InvalidMetadataValue instead." +)] +pub type InvalidMetadataValue = aws_runtime::user_agent::InvalidMetadataValue; + +/// Use aws_runtime::user_agent::AdditionalMetadata instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::AdditionalMetadata instead." +)] +pub type AdditionalMetadata = aws_runtime::user_agent::AdditionalMetadata; + +/// Use aws_runtime::user_agent::FeatureMetadata instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::FeatureMetadata instead." +)] +pub type FeatureMetadata = aws_runtime::user_agent::FeatureMetadata; + +/// Use aws_runtime::user_agent::ConfigMetadata instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::ConfigMetadata instead." +)] +pub type ConfigMetadata = aws_runtime::user_agent::ConfigMetadata; + +/// Use aws_runtime::user_agent::FrameworkMetadata instead. +#[deprecated( + since = "0.60.2", + note = "Use aws_runtime::user_agent::FrameworkMetadata instead." +)] +pub type FrameworkMetadata = aws_runtime::user_agent::FrameworkMetadata; diff --git a/aws/rust-runtime/aws-inlineable/Cargo.toml b/aws/rust-runtime/aws-inlineable/Cargo.toml index 514aa0c2d8c..85b6c440cb5 100644 --- a/aws/rust-runtime/aws-inlineable/Cargo.toml +++ b/aws/rust-runtime/aws-inlineable/Cargo.toml @@ -13,8 +13,7 @@ repository = "https://github.com/smithy-lang/smithy-rs" [dependencies] aws-credential-types = { path = "../aws-credential-types" } -aws-http = { path = "../aws-http" } -aws-runtime = { path = "../aws-runtime" } +aws-runtime = { path = "../aws-runtime", features = ["http-02x"] } aws-sigv4 = { path = "../aws-sigv4" } aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async", features = ["rt-tokio"] } aws-smithy-checksums = { path = "../../../rust-runtime/aws-smithy-checksums" } diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index ef07f2c34bb..2ccfc5d28ea 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -7,8 +7,8 @@ //! Interceptor for handling Smithy `@httpChecksum` request checksumming with AWS SigV4 -use aws_http::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; -use aws_runtime::auth::SigV4OperationSigningConfig; +use aws_runtime::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; +use aws_runtime::{auth::SigV4OperationSigningConfig, content_encoding::header_value::AWS_CHUNKED}; use aws_sigv4::http_request::SignableBody; use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; @@ -199,7 +199,7 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( ); headers.insert( http::header::CONTENT_ENCODING, - HeaderValue::from_str(aws_http::content_encoding::header_value::AWS_CHUNKED) + HeaderValue::from_str(AWS_CHUNKED) .map_err(BuildError::other) .expect("\"aws-chunked\" will always be a valid HeaderValue"), ); diff --git a/aws/rust-runtime/aws-runtime/Cargo.toml b/aws/rust-runtime/aws-runtime/Cargo.toml index 4ba2423b9be..a436ab87249 100644 --- a/aws/rust-runtime/aws-runtime/Cargo.toml +++ b/aws/rust-runtime/aws-runtime/Cargo.toml @@ -9,12 +9,12 @@ repository = "https://github.com/smithy-lang/smithy-rs" [features] event-stream = ["dep:aws-smithy-eventstream", "aws-sigv4/sign-eventstream"] +http-02x = [] test-util = [] sigv4a = ["aws-sigv4/sigv4a"] [dependencies] aws-credential-types = { path = "../aws-credential-types" } -aws-http = { path = "../aws-http" } # TODO(httpRefactor): Remove the http0-compat feature aws-sigv4 = { path = "../aws-sigv4", features = ["http0-compat"] } aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async" } @@ -23,21 +23,26 @@ aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" } aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] } aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" } aws-types = { path = "../aws-types" } +bytes = "1.1" fastrand = "2.0.0" http = "0.2.3" +http-body = "0.4.5" percent-encoding = "2.1.0" +pin-project-lite = "0.2.9" tracing = "0.1" uuid = { version = "1" } [dev-dependencies] aws-credential-types = { path = "../aws-credential-types", features = ["test-util"] } aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async", features = ["test-util"] } -aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["test-util"] } aws-smithy-protocol-test = { path = "../../../rust-runtime/aws-smithy-protocol-test" } aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["test-util"] } +aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["test-util"] } +bytes-utils = "0.1.2" proptest = "1.2" serde = { version = "1", features = ["derive"]} serde_json = "1" +tokio = { version = "1.23.1", features = ["macros", "rt", "time"] } tracing-test = "0.2.4" [package.metadata.docs.rs] diff --git a/aws/rust-runtime/aws-runtime/external-types.toml b/aws/rust-runtime/aws-runtime/external-types.toml index 941f09767bf..d42e6a36165 100644 --- a/aws/rust-runtime/aws-runtime/external-types.toml +++ b/aws/rust-runtime/aws-runtime/external-types.toml @@ -3,4 +3,8 @@ allowed_external_types = [ "aws_smithy_types::*", "aws_smithy_runtime_api::*", "aws_types::*", + "bytes::bytes::Bytes", + + # Used by the aws-chunked implementation + "http_body::Body", ] diff --git a/aws/rust-runtime/aws-runtime/src/content_encoding.rs b/aws/rust-runtime/aws-runtime/src/content_encoding.rs new file mode 100644 index 00000000000..2c00cff9cb9 --- /dev/null +++ b/aws/rust-runtime/aws-runtime/src/content_encoding.rs @@ -0,0 +1,613 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use bytes::{Bytes, BytesMut}; +use http::{HeaderMap, HeaderValue}; +use http_body::{Body, SizeHint}; +use pin_project_lite::pin_project; + +use std::pin::Pin; +use std::task::{Context, Poll}; + +const CRLF: &str = "\r\n"; +const CHUNK_TERMINATOR: &str = "0\r\n"; +const TRAILER_SEPARATOR: &[u8] = b":"; + +/// Content encoding header value constants +pub mod header_value { + /// Header value denoting "aws-chunked" encoding + pub const AWS_CHUNKED: &str = "aws-chunked"; +} + +/// Options used when constructing an [`AwsChunkedBody`]. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct AwsChunkedBodyOptions { + /// The total size of the stream. Because we only support unsigned encoding + /// this implies that there will only be a single chunk containing the + /// underlying payload. + stream_length: u64, + /// The length of each trailer sent within an `AwsChunkedBody`. Necessary in + /// order to correctly calculate the total size of the body accurately. + trailer_lengths: Vec, +} + +impl AwsChunkedBodyOptions { + /// Create a new [`AwsChunkedBodyOptions`]. + pub fn new(stream_length: u64, trailer_lengths: Vec) -> Self { + Self { + stream_length, + trailer_lengths, + } + } + + fn total_trailer_length(&self) -> u64 { + self.trailer_lengths.iter().sum::() + // We need to account for a CRLF after each trailer name/value pair + + (self.trailer_lengths.len() * CRLF.len()) as u64 + } + + /// Set a trailer len + pub fn with_trailer_len(mut self, trailer_len: u64) -> Self { + self.trailer_lengths.push(trailer_len); + self + } +} + +#[derive(Debug, PartialEq, Eq)] +enum AwsChunkedBodyState { + /// Write out the size of the chunk that will follow. Then, transition into the + /// `WritingChunk` state. + WritingChunkSize, + /// Write out the next chunk of data. Multiple polls of the inner body may need to occur before + /// all data is written out. Once there is no more data to write, transition into the + /// `WritingTrailers` state. + WritingChunk, + /// Write out all trailers associated with this `AwsChunkedBody` and then transition into the + /// `Closed` state. + WritingTrailers, + /// This is the final state. Write out the body terminator and then remain in this state. + Closed, +} + +pin_project! { + /// A request body compatible with `Content-Encoding: aws-chunked`. This implementation is only + /// capable of writing a single chunk and does not support signed chunks. + /// + /// Chunked-Body grammar is defined in [ABNF] as: + /// + /// ```txt + /// Chunked-Body = *chunk + /// last-chunk + /// chunked-trailer + /// CRLF + /// + /// chunk = chunk-size CRLF chunk-data CRLF + /// chunk-size = 1*HEXDIG + /// last-chunk = 1*("0") CRLF + /// chunked-trailer = *( entity-header CRLF ) + /// entity-header = field-name ":" OWS field-value OWS + /// ``` + /// For more info on what the abbreviations mean, see https://datatracker.ietf.org/doc/html/rfc7230#section-1.2 + /// + /// [ABNF]:https://en.wikipedia.org/wiki/Augmented_Backus%E2%80%93Naur_form + #[derive(Debug)] + pub struct AwsChunkedBody { + #[pin] + inner: InnerBody, + #[pin] + state: AwsChunkedBodyState, + options: AwsChunkedBodyOptions, + inner_body_bytes_read_so_far: usize, + } +} + +impl AwsChunkedBody { + /// Wrap the given body in an outer body compatible with `Content-Encoding: aws-chunked` + pub fn new(body: Inner, options: AwsChunkedBodyOptions) -> Self { + Self { + inner: body, + state: AwsChunkedBodyState::WritingChunkSize, + options, + inner_body_bytes_read_so_far: 0, + } + } + + fn encoded_length(&self) -> u64 { + let mut length = 0; + if self.options.stream_length != 0 { + length += get_unsigned_chunk_bytes_length(self.options.stream_length); + } + + // End chunk + length += CHUNK_TERMINATOR.len() as u64; + + // Trailers + for len in self.options.trailer_lengths.iter() { + length += len + CRLF.len() as u64; + } + + // Encoding terminator + length += CRLF.len() as u64; + + length + } +} + +fn get_unsigned_chunk_bytes_length(payload_length: u64) -> u64 { + let hex_repr_len = int_log16(payload_length); + hex_repr_len + CRLF.len() as u64 + payload_length + CRLF.len() as u64 +} + +/// Writes trailers out into a `string` and then converts that `String` to a `Bytes` before +/// returning. +/// +/// - Trailer names are separated by a single colon only, no space. +/// - Trailer names with multiple values will be written out one line per value, with the name +/// appearing on each line. +fn trailers_as_aws_chunked_bytes( + trailer_map: Option, + estimated_length: u64, +) -> BytesMut { + if let Some(trailer_map) = trailer_map { + let mut current_header_name = None; + let mut trailers = BytesMut::with_capacity(estimated_length.try_into().unwrap_or_default()); + + for (header_name, header_value) in trailer_map.into_iter() { + // When a header has multiple values, the name only comes up in iteration the first time + // we see it. Therefore, we need to keep track of the last name we saw and fall back to + // it when `header_name == None`. + current_header_name = header_name.or(current_header_name); + + // In practice, this will always exist, but `if let` is nicer than unwrap + if let Some(header_name) = current_header_name.as_ref() { + trailers.extend_from_slice(header_name.as_ref()); + trailers.extend_from_slice(TRAILER_SEPARATOR); + trailers.extend_from_slice(header_value.as_bytes()); + trailers.extend_from_slice(CRLF.as_bytes()); + } + } + + trailers + } else { + BytesMut::new() + } +} + +/// Given an optional `HeaderMap`, calculate the total number of bytes required to represent the +/// `HeaderMap`. If no `HeaderMap` is given as input, return 0. +/// +/// - Trailer names are separated by a single colon only, no space. +/// - Trailer names with multiple values will be written out one line per value, with the name +/// appearing on each line. +fn total_rendered_length_of_trailers(trailer_map: Option<&HeaderMap>) -> u64 { + match trailer_map { + Some(trailer_map) => trailer_map + .iter() + .map(|(trailer_name, trailer_value)| { + trailer_name.as_str().len() + + TRAILER_SEPARATOR.len() + + trailer_value.len() + + CRLF.len() + }) + .sum::() as u64, + None => 0, + } +} + +impl Body for AwsChunkedBody +where + Inner: Body, +{ + type Data = Bytes; + type Error = aws_smithy_types::body::Error; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + tracing::trace!(state = ?self.state, "polling AwsChunkedBody"); + let mut this = self.project(); + + match *this.state { + AwsChunkedBodyState::WritingChunkSize => { + if this.options.stream_length == 0 { + // If the stream is empty, we skip to writing trailers after writing the CHUNK_TERMINATOR. + *this.state = AwsChunkedBodyState::WritingTrailers; + tracing::trace!("stream is empty, writing chunk terminator"); + Poll::Ready(Some(Ok(Bytes::from([CHUNK_TERMINATOR].concat())))) + } else { + *this.state = AwsChunkedBodyState::WritingChunk; + // A chunk must be prefixed by chunk size in hexadecimal + let chunk_size = format!("{:X?}{CRLF}", this.options.stream_length); + tracing::trace!(%chunk_size, "writing chunk size"); + let chunk_size = Bytes::from(chunk_size); + Poll::Ready(Some(Ok(chunk_size))) + } + } + AwsChunkedBodyState::WritingChunk => match this.inner.poll_data(cx) { + Poll::Ready(Some(Ok(data))) => { + tracing::trace!(len = data.len(), "writing chunk data"); + *this.inner_body_bytes_read_so_far += data.len(); + Poll::Ready(Some(Ok(data))) + } + Poll::Ready(None) => { + let actual_stream_length = *this.inner_body_bytes_read_so_far as u64; + let expected_stream_length = this.options.stream_length; + if actual_stream_length != expected_stream_length { + let err = Box::new(AwsChunkedBodyError::StreamLengthMismatch { + actual: actual_stream_length, + expected: expected_stream_length, + }); + return Poll::Ready(Some(Err(err))); + }; + + tracing::trace!("no more chunk data, writing CRLF and chunk terminator"); + *this.state = AwsChunkedBodyState::WritingTrailers; + // Since we wrote chunk data, we end it with a CRLF and since we only write + // a single chunk, we write the CHUNK_TERMINATOR immediately after + Poll::Ready(Some(Ok(Bytes::from([CRLF, CHUNK_TERMINATOR].concat())))) + } + Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), + Poll::Pending => Poll::Pending, + }, + AwsChunkedBodyState::WritingTrailers => { + return match this.inner.poll_trailers(cx) { + Poll::Ready(Ok(trailers)) => { + *this.state = AwsChunkedBodyState::Closed; + let expected_length = total_rendered_length_of_trailers(trailers.as_ref()); + let actual_length = this.options.total_trailer_length(); + + if expected_length != actual_length { + let err = + Box::new(AwsChunkedBodyError::ReportedTrailerLengthMismatch { + actual: actual_length, + expected: expected_length, + }); + return Poll::Ready(Some(Err(err))); + } + + let mut trailers = + trailers_as_aws_chunked_bytes(trailers, actual_length + 1); + // Insert the final CRLF to close the body + trailers.extend_from_slice(CRLF.as_bytes()); + + Poll::Ready(Some(Ok(trailers.into()))) + } + Poll::Pending => Poll::Pending, + Poll::Ready(Err(e)) => Poll::Ready(Some(Err(e))), + }; + } + AwsChunkedBodyState::Closed => Poll::Ready(None), + } + } + + fn poll_trailers( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll>, Self::Error>> { + // Trailers were already appended to the body because of the content encoding scheme + Poll::Ready(Ok(None)) + } + + fn is_end_stream(&self) -> bool { + self.state == AwsChunkedBodyState::Closed + } + + fn size_hint(&self) -> SizeHint { + SizeHint::with_exact(self.encoded_length()) + } +} + +/// Errors related to `AwsChunkedBody` +#[derive(Debug)] +enum AwsChunkedBodyError { + /// Error that occurs when the sum of `trailer_lengths` set when creating an `AwsChunkedBody` is + /// not equal to the actual length of the trailers returned by the inner `http_body::Body` + /// implementor. These trailer lengths are necessary in order to correctly calculate the total + /// size of the body for setting the content length header. + ReportedTrailerLengthMismatch { actual: u64, expected: u64 }, + /// Error that occurs when the `stream_length` set when creating an `AwsChunkedBody` is not + /// equal to the actual length of the body returned by the inner `http_body::Body` implementor. + /// `stream_length` must be correct in order to set an accurate content length header. + StreamLengthMismatch { actual: u64, expected: u64 }, +} + +impl std::fmt::Display for AwsChunkedBodyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ReportedTrailerLengthMismatch { actual, expected } => { + write!(f, "When creating this AwsChunkedBody, length of trailers was reported as {expected}. However, when double checking during trailer encoding, length was found to be {actual} instead.") + } + Self::StreamLengthMismatch { actual, expected } => { + write!(f, "When creating this AwsChunkedBody, stream length was reported as {expected}. However, when double checking during body encoding, length was found to be {actual} instead.") + } + } + } +} + +impl std::error::Error for AwsChunkedBodyError {} + +// Used for finding how many hexadecimal digits it takes to represent a base 10 integer +fn int_log16(mut i: T) -> u64 +where + T: std::ops::DivAssign + PartialOrd + From + Copy, +{ + let mut len = 0; + let zero = T::from(0); + let sixteen = T::from(16); + + while i > zero { + i /= sixteen; + len += 1; + } + + len +} + +#[cfg(test)] +mod tests { + use super::{ + total_rendered_length_of_trailers, trailers_as_aws_chunked_bytes, AwsChunkedBody, + AwsChunkedBodyOptions, CHUNK_TERMINATOR, CRLF, + }; + + use aws_smithy_types::body::SdkBody; + use bytes::{Buf, Bytes}; + use bytes_utils::SegmentedBuf; + use http::{HeaderMap, HeaderValue}; + use http_body::{Body, SizeHint}; + use pin_project_lite::pin_project; + + use std::io::Read; + use std::pin::Pin; + use std::task::{Context, Poll}; + use std::time::Duration; + + pin_project! { + struct SputteringBody { + parts: Vec>, + cursor: usize, + delay_in_millis: u64, + } + } + + impl SputteringBody { + fn len(&self) -> usize { + self.parts.iter().flatten().map(|b| b.len()).sum() + } + } + + impl Body for SputteringBody { + type Data = Bytes; + type Error = aws_smithy_types::body::Error; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + if self.cursor == self.parts.len() { + return Poll::Ready(None); + } + + let this = self.project(); + let delay_in_millis = *this.delay_in_millis; + let next_part = this.parts.get_mut(*this.cursor).unwrap().take(); + + match next_part { + None => { + *this.cursor += 1; + let waker = cx.waker().clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(delay_in_millis)).await; + waker.wake(); + }); + Poll::Pending + } + Some(data) => { + *this.cursor += 1; + Poll::Ready(Some(Ok(data))) + } + } + } + + fn poll_trailers( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll>, Self::Error>> { + Poll::Ready(Ok(None)) + } + + fn is_end_stream(&self) -> bool { + false + } + + fn size_hint(&self) -> SizeHint { + SizeHint::new() + } + } + + #[tokio::test] + async fn test_aws_chunked_encoding() { + let test_fut = async { + let input_str = "Hello world"; + let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, Vec::new()); + let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); + + let mut output = SegmentedBuf::new(); + while let Some(buf) = body.data().await { + output.push(buf.unwrap()); + } + + let mut actual_output = String::new(); + output + .reader() + .read_to_string(&mut actual_output) + .expect("Doesn't cause IO errors"); + + let expected_output = "B\r\nHello world\r\n0\r\n\r\n"; + + assert_eq!(expected_output, actual_output); + assert!( + body.trailers() + .await + .expect("no errors occurred during trailer polling") + .is_none(), + "aws-chunked encoded bodies don't have normal HTTP trailers" + ); + + // You can insert a `tokio::time::sleep` here to verify the timeout works as intended + }; + + let timeout_duration = Duration::from_secs(3); + if tokio::time::timeout(timeout_duration, test_fut) + .await + .is_err() + { + panic!("test_aws_chunked_encoding timed out after {timeout_duration:?}"); + } + } + + #[tokio::test] + async fn test_aws_chunked_encoding_sputtering_body() { + let test_fut = async { + let input = SputteringBody { + parts: vec![ + Some(Bytes::from_static(b"chunk 1, ")), + None, + Some(Bytes::from_static(b"chunk 2, ")), + Some(Bytes::from_static(b"chunk 3, ")), + None, + None, + Some(Bytes::from_static(b"chunk 4, ")), + Some(Bytes::from_static(b"chunk 5, ")), + Some(Bytes::from_static(b"chunk 6")), + ], + cursor: 0, + delay_in_millis: 500, + }; + let opts = AwsChunkedBodyOptions::new(input.len() as u64, Vec::new()); + let mut body = AwsChunkedBody::new(input, opts); + + let mut output = SegmentedBuf::new(); + while let Some(buf) = body.data().await { + output.push(buf.unwrap()); + } + + let mut actual_output = String::new(); + output + .reader() + .read_to_string(&mut actual_output) + .expect("Doesn't cause IO errors"); + + let expected_output = + "34\r\nchunk 1, chunk 2, chunk 3, chunk 4, chunk 5, chunk 6\r\n0\r\n\r\n"; + + assert_eq!(expected_output, actual_output); + assert!( + body.trailers() + .await + .expect("no errors occurred during trailer polling") + .is_none(), + "aws-chunked encoded bodies don't have normal HTTP trailers" + ); + }; + + let timeout_duration = Duration::from_secs(3); + if tokio::time::timeout(timeout_duration, test_fut) + .await + .is_err() + { + panic!( + "test_aws_chunked_encoding_sputtering_body timed out after {timeout_duration:?}" + ); + } + } + + #[tokio::test] + #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 44, expected: 0 }"] + async fn test_aws_chunked_encoding_incorrect_trailer_length_panic() { + let input_str = "Hello world"; + // Test body has no trailers, so this length is incorrect and will trigger an assert panic + // When the panic occurs, it will actually expect a length of 44. This is because, when using + // aws-chunked encoding, each trailer will end with a CRLF which is 2 bytes long. + let wrong_trailer_len = 42; + let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, vec![wrong_trailer_len]); + let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); + + // We don't care about the body contents but we have to read it all before checking for trailers + while let Some(buf) = body.data().await { + drop(buf.unwrap()); + } + + assert!( + body.trailers() + .await + .expect("no errors occurred during trailer polling") + .is_none(), + "aws-chunked encoded bodies don't have normal HTTP trailers" + ); + } + + #[tokio::test] + async fn test_aws_chunked_encoding_empty_body() { + let input_str = ""; + let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, Vec::new()); + let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts); + + let mut output = SegmentedBuf::new(); + while let Some(buf) = body.data().await { + output.push(buf.unwrap()); + } + + let mut actual_output = String::new(); + output + .reader() + .read_to_string(&mut actual_output) + .expect("Doesn't cause IO errors"); + + let expected_output = [CHUNK_TERMINATOR, CRLF].concat(); + + assert_eq!(expected_output, actual_output); + assert!( + body.trailers() + .await + .expect("no errors occurred during trailer polling") + .is_none(), + "aws-chunked encoded bodies don't have normal HTTP trailers" + ); + } + + #[tokio::test] + async fn test_total_rendered_length_of_trailers() { + let mut trailers = HeaderMap::new(); + + trailers.insert("empty_value", HeaderValue::from_static("")); + + trailers.insert("single_value", HeaderValue::from_static("value 1")); + + trailers.insert("two_values", HeaderValue::from_static("value 1")); + trailers.append("two_values", HeaderValue::from_static("value 2")); + + trailers.insert("three_values", HeaderValue::from_static("value 1")); + trailers.append("three_values", HeaderValue::from_static("value 2")); + trailers.append("three_values", HeaderValue::from_static("value 3")); + + let trailers = Some(trailers); + let actual_length = total_rendered_length_of_trailers(trailers.as_ref()); + let expected_length = (trailers_as_aws_chunked_bytes(trailers, actual_length).len()) as u64; + + assert_eq!(expected_length, actual_length); + } + + #[tokio::test] + async fn test_total_rendered_length_of_empty_trailers() { + let trailers = Some(HeaderMap::new()); + let actual_length = total_rendered_length_of_trailers(trailers.as_ref()); + let expected_length = (trailers_as_aws_chunked_bytes(trailers, actual_length).len()) as u64; + + assert_eq!(expected_length, actual_length); + } +} diff --git a/aws/rust-runtime/aws-runtime/src/lib.rs b/aws/rust-runtime/aws-runtime/src/lib.rs index df8415f20e8..8487640adb5 100644 --- a/aws/rust-runtime/aws-runtime/src/lib.rs +++ b/aws/rust-runtime/aws-runtime/src/lib.rs @@ -19,6 +19,10 @@ /// Supporting code for authentication in the AWS SDK. pub mod auth; +/// AWS-specific content-encoding tools +#[cfg(feature = "http-02x")] +pub mod content_encoding; + /// Supporting code for recursion detection in the AWS SDK. pub mod recursion_detection; diff --git a/aws/rust-runtime/aws-runtime/src/user_agent.rs b/aws/rust-runtime/aws-runtime/src/user_agent.rs index bf4564acecb..7c580d93784 100644 --- a/aws/rust-runtime/aws-runtime/src/user_agent.rs +++ b/aws/rust-runtime/aws-runtime/src/user_agent.rs @@ -3,275 +3,739 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_http::user_agent::{ApiMetadata, AwsUserAgent}; -use aws_smithy_runtime_api::box_error::BoxError; -use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut; -use aws_smithy_runtime_api::client::interceptors::Intercept; -use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; -use aws_smithy_types::config_bag::ConfigBag; +use aws_smithy_types::config_bag::{Storable, StoreReplace}; use aws_types::app_name::AppName; +use aws_types::build_metadata::{OsFamily, BUILD_METADATA}; use aws_types::os_shim_internal::Env; -use http::header::{InvalidHeaderValue, USER_AGENT}; -use http::{HeaderName, HeaderValue}; use std::borrow::Cow; +use std::error::Error; use std::fmt; -#[allow(clippy::declare_interior_mutable_const)] // we will never mutate this -const X_AMZ_USER_AGENT: HeaderName = HeaderName::from_static("x-amz-user-agent"); +mod interceptor; +pub use interceptor::UserAgentInterceptor; + +/// AWS User Agent +/// +/// Ths struct should be inserted into the [`ConfigBag`](aws_smithy_types::config_bag::ConfigBag) +/// during operation construction. The `UserAgentInterceptor` reads `AwsUserAgent` +/// from the config bag and sets the `User-Agent` and `x-amz-user-agent` headers. +#[derive(Clone, Debug)] +pub struct AwsUserAgent { + sdk_metadata: SdkMetadata, + api_metadata: ApiMetadata, + os_metadata: OsMetadata, + language_metadata: LanguageMetadata, + exec_env_metadata: Option, + feature_metadata: Vec, + config_metadata: Vec, + framework_metadata: Vec, + app_name: Option, + build_env_additional_metadata: Option, +} + +impl AwsUserAgent { + /// Load a User Agent configuration from the environment + /// + /// This utilizes [`BUILD_METADATA`](const@aws_types::build_metadata::BUILD_METADATA) from `aws_types` + /// to capture the Rust version & target platform. `ApiMetadata` provides + /// the version & name of the specific service. + pub fn new_from_environment(env: Env, api_metadata: ApiMetadata) -> Self { + let build_metadata = &BUILD_METADATA; + let sdk_metadata = SdkMetadata { + name: "rust", + version: build_metadata.core_pkg_version, + }; + let os_metadata = OsMetadata { + os_family: &build_metadata.os_family, + version: None, + }; + let exec_env_metadata = env + .get("AWS_EXECUTION_ENV") + .ok() + .map(|name| ExecEnvMetadata { name }); + + // Retrieve additional metadata at compile-time from the AWS_SDK_RUST_BUILD_UA_METADATA env var + let build_env_additional_metadata = option_env!("AWS_SDK_RUST_BUILD_UA_METADATA") + .and_then(|value| AdditionalMetadata::new(value).ok()); + + AwsUserAgent { + sdk_metadata, + api_metadata, + os_metadata, + language_metadata: LanguageMetadata { + lang: "rust", + version: BUILD_METADATA.rust_version, + extras: Default::default(), + }, + exec_env_metadata, + feature_metadata: Default::default(), + config_metadata: Default::default(), + framework_metadata: Default::default(), + app_name: Default::default(), + build_env_additional_metadata, + } + } + + /// For test purposes, construct an environment-independent User Agent + /// + /// Without this, running CI on a different platform would produce different user agent strings + pub fn for_tests() -> Self { + Self { + sdk_metadata: SdkMetadata { + name: "rust", + version: "0.123.test", + }, + api_metadata: ApiMetadata { + service_id: "test-service".into(), + version: "0.123", + }, + os_metadata: OsMetadata { + os_family: &OsFamily::Windows, + version: Some("XPSP3".to_string()), + }, + language_metadata: LanguageMetadata { + lang: "rust", + version: "1.50.0", + extras: Default::default(), + }, + exec_env_metadata: None, + feature_metadata: Vec::new(), + config_metadata: Vec::new(), + framework_metadata: Vec::new(), + app_name: None, + build_env_additional_metadata: None, + } + } + + #[doc(hidden)] + /// Adds feature metadata to the user agent. + pub fn with_feature_metadata(mut self, metadata: FeatureMetadata) -> Self { + self.feature_metadata.push(metadata); + self + } + + #[doc(hidden)] + /// Adds feature metadata to the user agent. + pub fn add_feature_metadata(&mut self, metadata: FeatureMetadata) -> &mut Self { + self.feature_metadata.push(metadata); + self + } + + #[doc(hidden)] + /// Adds config metadata to the user agent. + pub fn with_config_metadata(mut self, metadata: ConfigMetadata) -> Self { + self.config_metadata.push(metadata); + self + } + + #[doc(hidden)] + /// Adds config metadata to the user agent. + pub fn add_config_metadata(&mut self, metadata: ConfigMetadata) -> &mut Self { + self.config_metadata.push(metadata); + self + } + + #[doc(hidden)] + /// Adds framework metadata to the user agent. + pub fn with_framework_metadata(mut self, metadata: FrameworkMetadata) -> Self { + self.framework_metadata.push(metadata); + self + } + + #[doc(hidden)] + /// Adds framework metadata to the user agent. + pub fn add_framework_metadata(&mut self, metadata: FrameworkMetadata) -> &mut Self { + self.framework_metadata.push(metadata); + self + } + + /// Sets the app name for the user agent. + pub fn with_app_name(mut self, app_name: AppName) -> Self { + self.app_name = Some(app_name); + self + } + + /// Sets the app name for the user agent. + pub fn set_app_name(&mut self, app_name: AppName) -> &mut Self { + self.app_name = Some(app_name); + self + } + + /// Generate a new-style user agent style header + /// + /// This header should be set at `x-amz-user-agent` + pub fn aws_ua_header(&self) -> String { + /* + ABNF for the user agent (see the bottom of the file for complete ABNF): + ua-string = sdk-metadata RWS + [api-metadata RWS] + os-metadata RWS + language-metadata RWS + [env-metadata RWS] + *(feat-metadata RWS) + *(config-metadata RWS) + *(framework-metadata RWS) + [appId] + */ + let mut ua_value = String::new(); + use std::fmt::Write; + // unwrap calls should never fail because string formatting will always succeed. + write!(ua_value, "{} ", &self.sdk_metadata).unwrap(); + write!(ua_value, "{} ", &self.api_metadata).unwrap(); + write!(ua_value, "{} ", &self.os_metadata).unwrap(); + write!(ua_value, "{} ", &self.language_metadata).unwrap(); + if let Some(ref env_meta) = self.exec_env_metadata { + write!(ua_value, "{} ", env_meta).unwrap(); + } + for feature in &self.feature_metadata { + write!(ua_value, "{} ", feature).unwrap(); + } + for config in &self.config_metadata { + write!(ua_value, "{} ", config).unwrap(); + } + for framework in &self.framework_metadata { + write!(ua_value, "{} ", framework).unwrap(); + } + if let Some(app_name) = &self.app_name { + write!(ua_value, "app/{}", app_name).unwrap(); + } + if let Some(additional_metadata) = &self.build_env_additional_metadata { + write!(ua_value, "{}", additional_metadata).unwrap(); + } + if ua_value.ends_with(' ') { + ua_value.truncate(ua_value.len() - 1); + } + ua_value + } + + /// Generate an old-style User-Agent header for backward compatibility + /// + /// This header is intended to be set at `User-Agent` + pub fn ua_header(&self) -> String { + let mut ua_value = String::new(); + use std::fmt::Write; + write!(ua_value, "{} ", &self.sdk_metadata).unwrap(); + write!(ua_value, "{} ", &self.os_metadata).unwrap(); + write!(ua_value, "{}", &self.language_metadata).unwrap(); + ua_value + } +} + +impl Storable for AwsUserAgent { + type Storer = StoreReplace; +} + +#[derive(Clone, Copy, Debug)] +struct SdkMetadata { + name: &'static str, + version: &'static str, +} + +impl fmt::Display for SdkMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "aws-sdk-{}/{}", self.name, self.version) + } +} + +/// Metadata about the client that's making the call. +#[derive(Clone, Debug)] +pub struct ApiMetadata { + service_id: Cow<'static, str>, + version: &'static str, +} + +impl ApiMetadata { + /// Creates new `ApiMetadata`. + pub const fn new(service_id: &'static str, version: &'static str) -> Self { + Self { + service_id: Cow::Borrowed(service_id), + version, + } + } +} + +impl fmt::Display for ApiMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "api/{}/{}", self.service_id, self.version) + } +} + +impl Storable for ApiMetadata { + type Storer = StoreReplace; +} +/// Error for when an user agent metadata doesn't meet character requirements. +/// +/// Metadata may only have alphanumeric characters and any of these characters: +/// ```text +/// !#$%&'*+-.^_`|~ +/// ``` +/// Spaces are not allowed. #[derive(Debug)] -enum UserAgentInterceptorError { - MissingApiMetadata, - InvalidHeaderValue(InvalidHeaderValue), +#[non_exhaustive] +pub struct InvalidMetadataValue; + +impl Error for InvalidMetadataValue {} + +impl fmt::Display for InvalidMetadataValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "User agent metadata can only have alphanumeric characters, or any of \ + '!' | '#' | '$' | '%' | '&' | '\\'' | '*' | '+' | '-' | \ + '.' | '^' | '_' | '`' | '|' | '~'" + ) + } } -impl std::error::Error for UserAgentInterceptorError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::InvalidHeaderValue(source) => Some(source), - Self::MissingApiMetadata => None, +fn validate_metadata(value: Cow<'static, str>) -> Result, InvalidMetadataValue> { + fn valid_character(c: char) -> bool { + match c { + _ if c.is_ascii_alphanumeric() => true, + '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | '-' | '.' | '^' | '_' | '`' | '|' + | '~' => true, + _ => false, } } + if !value.chars().all(valid_character) { + return Err(InvalidMetadataValue); + } + Ok(value) +} + +#[doc(hidden)] +/// Additional metadata that can be bundled with framework or feature metadata. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct AdditionalMetadata { + value: Cow<'static, str>, +} + +impl AdditionalMetadata { + /// Creates `AdditionalMetadata`. + /// + /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or + /// has characters other than the following: + /// ```text + /// !#$%&'*+-.^_`|~ + /// ``` + pub fn new(value: impl Into>) -> Result { + Ok(Self { + value: validate_metadata(value.into())?, + }) + } +} + +impl fmt::Display for AdditionalMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // additional-metadata = "md/" ua-pair + write!(f, "md/{}", self.value) + } +} + +#[derive(Clone, Debug, Default)] +struct AdditionalMetadataList(Vec); + +impl AdditionalMetadataList { + fn push(&mut self, metadata: AdditionalMetadata) { + self.0.push(metadata); + } } -impl fmt::Display for UserAgentInterceptorError { +impl fmt::Display for AdditionalMetadataList { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::InvalidHeaderValue(_) => "AwsUserAgent generated an invalid HTTP header value. This is a bug. Please file an issue.", - Self::MissingApiMetadata => "The UserAgentInterceptor requires ApiMetadata to be set before the request is made. This is a bug. Please file an issue.", + for metadata in &self.0 { + write!(f, " {}", metadata)?; + } + Ok(()) + } +} + +#[doc(hidden)] +/// Metadata about a feature that is being used in the SDK. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct FeatureMetadata { + name: Cow<'static, str>, + version: Option>, + additional: AdditionalMetadataList, +} + +impl FeatureMetadata { + /// Creates `FeatureMetadata`. + /// + /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or + /// has characters other than the following: + /// ```text + /// !#$%&'*+-.^_`|~ + /// ``` + pub fn new( + name: impl Into>, + version: Option>, + ) -> Result { + Ok(Self { + name: validate_metadata(name.into())?, + version: version.map(validate_metadata).transpose()?, + additional: Default::default(), }) } + + /// Bundles additional arbitrary metadata with this feature metadata. + pub fn with_additional(mut self, metadata: AdditionalMetadata) -> Self { + self.additional.push(metadata); + self + } } -impl From for UserAgentInterceptorError { - fn from(err: InvalidHeaderValue) -> Self { - UserAgentInterceptorError::InvalidHeaderValue(err) +impl fmt::Display for FeatureMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // feat-metadata = "ft/" name ["/" version] *(RWS additional-metadata) + if let Some(version) = &self.version { + write!(f, "ft/{}/{}{}", self.name, version, self.additional) + } else { + write!(f, "ft/{}{}", self.name, self.additional) + } } } -/// Generates and attaches the AWS SDK's user agent to a HTTP request +#[doc(hidden)] +/// Metadata about a config value that is being used in the SDK. +#[derive(Clone, Debug)] #[non_exhaustive] -#[derive(Debug, Default)] -pub struct UserAgentInterceptor; - -impl UserAgentInterceptor { - /// Creates a new `UserAgentInterceptor` - pub fn new() -> Self { - UserAgentInterceptor - } -} - -fn header_values( - ua: &AwsUserAgent, -) -> Result<(HeaderValue, HeaderValue), UserAgentInterceptorError> { - // Pay attention to the extremely subtle difference between ua_header and aws_ua_header below... - Ok(( - HeaderValue::try_from(ua.ua_header())?, - HeaderValue::try_from(ua.aws_ua_header())?, - )) -} - -impl Intercept for UserAgentInterceptor { - fn name(&self) -> &'static str { - "UserAgentInterceptor" - } - - fn modify_before_signing( - &self, - context: &mut BeforeTransmitInterceptorContextMut<'_>, - _runtime_components: &RuntimeComponents, - cfg: &mut ConfigBag, - ) -> Result<(), BoxError> { - // Allow for overriding the user agent by an earlier interceptor (so, for example, - // tests can use `AwsUserAgent::for_tests()`) by attempting to grab one out of the - // config bag before creating one. - let ua: Cow<'_, AwsUserAgent> = cfg - .load::() - .map(Cow::Borrowed) - .map(Result::<_, UserAgentInterceptorError>::Ok) - .unwrap_or_else(|| { - let api_metadata = cfg - .load::() - .ok_or(UserAgentInterceptorError::MissingApiMetadata)?; - let mut ua = AwsUserAgent::new_from_environment(Env::real(), api_metadata.clone()); - - let maybe_app_name = cfg.load::(); - if let Some(app_name) = maybe_app_name { - ua.set_app_name(app_name.clone()); - } - Ok(Cow::Owned(ua)) - })?; - - let headers = context.request_mut().headers_mut(); - let (user_agent, x_amz_user_agent) = header_values(&ua)?; - headers.append(USER_AGENT, user_agent); - headers.append(X_AMZ_USER_AGENT, x_amz_user_agent); - Ok(()) +pub struct ConfigMetadata { + config: Cow<'static, str>, + value: Option>, +} + +impl ConfigMetadata { + /// Creates `ConfigMetadata`. + /// + /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or + /// has characters other than the following: + /// ```text + /// !#$%&'*+-.^_`|~ + /// ``` + pub fn new( + config: impl Into>, + value: Option>, + ) -> Result { + Ok(Self { + config: validate_metadata(config.into())?, + value: value.map(validate_metadata).transpose()?, + }) } } -#[cfg(test)] -mod tests { - use super::*; - use aws_smithy_runtime_api::client::interceptors::context::{Input, InterceptorContext}; - use aws_smithy_runtime_api::client::interceptors::Intercept; - use aws_smithy_runtime_api::client::orchestrator::HttpRequest; - use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; - use aws_smithy_types::config_bag::{ConfigBag, Layer}; - use aws_smithy_types::error::display::DisplayErrorContext; - - fn expect_header<'a>(context: &'a InterceptorContext, header_name: &str) -> &'a str { - context - .request() - .expect("request is set") - .headers() - .get(header_name) - .unwrap() - } - - fn context() -> InterceptorContext { - let mut context = InterceptorContext::new(Input::doesnt_matter()); - context.enter_serialization_phase(); - context.set_request(HttpRequest::empty()); - let _ = context.take_input(); - context.enter_before_transmit_phase(); - context +impl fmt::Display for ConfigMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // config-metadata = "cfg/" config ["/" value] + if let Some(value) = &self.value { + write!(f, "cfg/{}/{}", self.config, value) + } else { + write!(f, "cfg/{}", self.config) + } } +} - #[test] - fn test_overridden_ua() { - let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); - let mut context = context(); +#[doc(hidden)] +/// Metadata about a software framework that is being used with the SDK. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct FrameworkMetadata { + name: Cow<'static, str>, + version: Option>, + additional: AdditionalMetadataList, +} - let mut layer = Layer::new("test"); - layer.store_put(AwsUserAgent::for_tests()); - layer.store_put(ApiMetadata::new("unused", "unused")); - let mut cfg = ConfigBag::of_layers(vec![layer]); +impl FrameworkMetadata { + /// Creates `FrameworkMetadata`. + /// + /// This will result in `InvalidMetadataValue` if the given value isn't alphanumeric or + /// has characters other than the following: + /// ```text + /// !#$%&'*+-.^_`|~ + /// ``` + pub fn new( + name: impl Into>, + version: Option>, + ) -> Result { + Ok(Self { + name: validate_metadata(name.into())?, + version: version.map(validate_metadata).transpose()?, + additional: Default::default(), + }) + } + + /// Bundles additional arbitrary metadata with this framework metadata. + pub fn with_additional(mut self, metadata: AdditionalMetadata) -> Self { + self.additional.push(metadata); + self + } +} + +impl fmt::Display for FrameworkMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // framework-metadata = "lib/" name ["/" version] *(RWS additional-metadata) + if let Some(version) = &self.version { + write!(f, "lib/{}/{}{}", self.name, version, self.additional) + } else { + write!(f, "lib/{}{}", self.name, self.additional) + } + } +} - let interceptor = UserAgentInterceptor::new(); - let mut ctx = Into::into(&mut context); - interceptor - .modify_before_signing(&mut ctx, &rc, &mut cfg) - .unwrap(); +#[derive(Clone, Debug)] +struct OsMetadata { + os_family: &'static OsFamily, + version: Option, +} - let header = expect_header(&context, "user-agent"); - assert_eq!(AwsUserAgent::for_tests().ua_header(), header); - assert!(!header.contains("unused")); +impl fmt::Display for OsMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let os_family = match self.os_family { + OsFamily::Windows => "windows", + OsFamily::Linux => "linux", + OsFamily::Macos => "macos", + OsFamily::Android => "android", + OsFamily::Ios => "ios", + OsFamily::Other => "other", + }; + write!(f, "os/{}", os_family)?; + if let Some(ref version) = self.version { + write!(f, "/{}", version)?; + } + Ok(()) + } +} +#[derive(Clone, Debug)] +struct LanguageMetadata { + lang: &'static str, + version: &'static str, + extras: AdditionalMetadataList, +} +impl fmt::Display for LanguageMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // language-metadata = "lang/" language "/" version *(RWS additional-metadata) + write!(f, "lang/{}/{}{}", self.lang, self.version, self.extras) + } +} + +#[derive(Clone, Debug)] +struct ExecEnvMetadata { + name: String, +} +impl fmt::Display for ExecEnvMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "exec-env/{}", &self.name) + } +} + +#[cfg(test)] +mod test { + use super::*; + use aws_types::app_name::AppName; + use aws_types::build_metadata::OsFamily; + use aws_types::os_shim_internal::Env; + use std::borrow::Cow; + + fn make_deterministic(ua: &mut AwsUserAgent) { + // hard code some variable things for a deterministic test + ua.sdk_metadata.version = "0.1"; + ua.language_metadata.version = "1.50.0"; + ua.os_metadata.os_family = &OsFamily::Macos; + ua.os_metadata.version = Some("1.15".to_string()); + } + + #[test] + fn generate_a_valid_ua() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata); + make_deterministic(&mut ua); + assert_eq!( + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0" + ); assert_eq!( - AwsUserAgent::for_tests().aws_ua_header(), - expect_header(&context, "x-amz-user-agent") + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" ); } #[test] - fn test_default_ua() { - let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); - let mut context = context(); - - let api_metadata = ApiMetadata::new("some-service", "some-version"); - let mut layer = Layer::new("test"); - layer.store_put(api_metadata.clone()); - let mut config = ConfigBag::of_layers(vec![layer]); - - let interceptor = UserAgentInterceptor::new(); - let mut ctx = Into::into(&mut context); - interceptor - .modify_before_signing(&mut ctx, &rc, &mut config) - .unwrap(); - - let expected_ua = AwsUserAgent::new_from_environment(Env::real(), api_metadata); - assert!( - expected_ua.aws_ua_header().contains("some-service"), - "precondition" + fn generate_a_valid_ua_with_execution_env() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment( + Env::from_slice(&[("AWS_EXECUTION_ENV", "lambda")]), + api_metadata, ); + make_deterministic(&mut ua); assert_eq!( - expected_ua.ua_header(), - expect_header(&context, "user-agent") + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 exec-env/lambda" ); assert_eq!( - expected_ua.aws_ua_header(), - expect_header(&context, "x-amz-user-agent") + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" ); } #[test] - fn test_app_name() { - let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); - let mut context = context(); - - let api_metadata = ApiMetadata::new("some-service", "some-version"); - let mut layer = Layer::new("test"); - layer.store_put(api_metadata); - layer.store_put(AppName::new("my_awesome_app").unwrap()); - let mut config = ConfigBag::of_layers(vec![layer]); - - let interceptor = UserAgentInterceptor::new(); - let mut ctx = Into::into(&mut context); - interceptor - .modify_before_signing(&mut ctx, &rc, &mut config) - .unwrap(); - - let app_value = "app/my_awesome_app"; - let header = expect_header(&context, "user-agent"); - assert!( - !header.contains(app_value), - "expected `{header}` to not contain `{app_value}`" + fn generate_a_valid_ua_with_features() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) + .with_feature_metadata( + FeatureMetadata::new("test-feature", Some(Cow::Borrowed("1.0"))).unwrap(), + ) + .with_feature_metadata( + FeatureMetadata::new("other-feature", None) + .unwrap() + .with_additional(AdditionalMetadata::new("asdf").unwrap()), + ); + make_deterministic(&mut ua); + assert_eq!( + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 ft/test-feature/1.0 ft/other-feature md/asdf" ); - - let header = expect_header(&context, "x-amz-user-agent"); - assert!( - header.contains(app_value), - "expected `{header}` to contain `{app_value}`" + assert_eq!( + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" ); } #[test] - fn test_api_metadata_missing() { - let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); - let mut context = context(); - let mut config = ConfigBag::base(); - - let interceptor = UserAgentInterceptor::new(); - let mut ctx = Into::into(&mut context); - - let error = format!( - "{}", - DisplayErrorContext( - &*interceptor - .modify_before_signing(&mut ctx, &rc, &mut config) - .expect_err("it should error") + fn generate_a_valid_ua_with_config() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) + .with_config_metadata( + ConfigMetadata::new("some-config", Some(Cow::Borrowed("5"))).unwrap(), ) + .with_config_metadata(ConfigMetadata::new("other-config", None).unwrap()); + make_deterministic(&mut ua); + assert_eq!( + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 cfg/some-config/5 cfg/other-config" ); - assert!( - error.contains("This is a bug"), - "`{error}` should contain message `This is a bug`" + assert_eq!( + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" ); } #[test] - fn test_api_metadata_missing_with_ua_override() { - let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); - let mut context = context(); - - let mut layer = Layer::new("test"); - layer.store_put(AwsUserAgent::for_tests()); - let mut config = ConfigBag::of_layers(vec![layer]); - - let interceptor = UserAgentInterceptor::new(); - let mut ctx = Into::into(&mut context); - - interceptor - .modify_before_signing(&mut ctx, &rc, &mut config) - .expect("it should succeed"); + fn generate_a_valid_ua_with_frameworks() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) + .with_framework_metadata( + FrameworkMetadata::new("some-framework", Some(Cow::Borrowed("1.3"))) + .unwrap() + .with_additional(AdditionalMetadata::new("something").unwrap()), + ) + .with_framework_metadata(FrameworkMetadata::new("other", None).unwrap()); + make_deterministic(&mut ua); + assert_eq!( + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 lib/some-framework/1.3 md/something lib/other" + ); + assert_eq!( + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" + ); + } - let header = expect_header(&context, "user-agent"); - assert_eq!(AwsUserAgent::for_tests().ua_header(), header); - assert!(!header.contains("unused")); + #[test] + fn generate_a_valid_ua_with_app_name() { + let api_metadata = ApiMetadata { + service_id: "dynamodb".into(), + version: "123", + }; + let mut ua = AwsUserAgent::new_from_environment(Env::from_slice(&[]), api_metadata) + .with_app_name(AppName::new("my_app").unwrap()); + make_deterministic(&mut ua); + assert_eq!( + ua.aws_ua_header(), + "aws-sdk-rust/0.1 api/dynamodb/123 os/macos/1.15 lang/rust/1.50.0 app/my_app" + ); + assert_eq!( + ua.ua_header(), + "aws-sdk-rust/0.1 os/macos/1.15 lang/rust/1.50.0" + ); + } + #[test] + fn generate_a_valid_ua_with_build_env_additional_metadata() { + let mut ua = AwsUserAgent::for_tests(); + ua.build_env_additional_metadata = Some(AdditionalMetadata::new("asdf").unwrap()); assert_eq!( - AwsUserAgent::for_tests().aws_ua_header(), - expect_header(&context, "x-amz-user-agent") + ua.aws_ua_header(), + "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0 md/asdf" + ); + assert_eq!( + ua.ua_header(), + "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0" ); } } + +/* +Appendix: User Agent ABNF +sdk-ua-header = "x-amz-user-agent:" OWS ua-string OWS +ua-pair = ua-name ["/" ua-value] +ua-name = token +ua-value = token +version = token +name = token +service-id = token +sdk-name = java / ruby / php / dotnet / python / cli / kotlin / rust / js / cpp / go / go-v2 +os-family = windows / linux / macos / android / ios / other +config = retry-mode +additional-metadata = "md/" ua-pair +sdk-metadata = "aws-sdk-" sdk-name "/" version +api-metadata = "api/" service-id "/" version +os-metadata = "os/" os-family ["/" version] +language-metadata = "lang/" language "/" version *(RWS additional-metadata) +env-metadata = "exec-env/" name +feat-metadata = "ft/" name ["/" version] *(RWS additional-metadata) +config-metadata = "cfg/" config ["/" value] +framework-metadata = "lib/" name ["/" version] *(RWS additional-metadata) +app-id = "app/" name +build-env-additional-metadata = "md/" value +ua-string = sdk-metadata RWS + [api-metadata RWS] + os-metadata RWS + language-metadata RWS + [env-metadata RWS] + *(feat-metadata RWS) + *(config-metadata RWS) + *(framework-metadata RWS) + [app-id] + [build-env-additional-metadata] + +# New metadata field might be added in the future and they must follow this format +prefix = token +metadata = prefix "/" ua-pair + +# token, RWS and OWS are defined in [RFC 7230](https://tools.ietf.org/html/rfc7230) +OWS = *( SP / HTAB ) + ; optional whitespace +RWS = 1*( SP / HTAB ) + ; required whitespace +token = 1*tchar +tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / + "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +*/ diff --git a/aws/rust-runtime/aws-runtime/src/user_agent/interceptor.rs b/aws/rust-runtime/aws-runtime/src/user_agent/interceptor.rs new file mode 100644 index 00000000000..dadf5ec28e7 --- /dev/null +++ b/aws/rust-runtime/aws-runtime/src/user_agent/interceptor.rs @@ -0,0 +1,277 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::user_agent::{ApiMetadata, AwsUserAgent}; +use aws_smithy_runtime_api::box_error::BoxError; +use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut; +use aws_smithy_runtime_api::client::interceptors::Intercept; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_types::config_bag::ConfigBag; +use aws_types::app_name::AppName; +use aws_types::os_shim_internal::Env; +use http::header::{InvalidHeaderValue, USER_AGENT}; +use http::{HeaderName, HeaderValue}; +use std::borrow::Cow; +use std::fmt; + +#[allow(clippy::declare_interior_mutable_const)] // we will never mutate this +const X_AMZ_USER_AGENT: HeaderName = HeaderName::from_static("x-amz-user-agent"); + +#[derive(Debug)] +enum UserAgentInterceptorError { + MissingApiMetadata, + InvalidHeaderValue(InvalidHeaderValue), +} + +impl std::error::Error for UserAgentInterceptorError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::InvalidHeaderValue(source) => Some(source), + Self::MissingApiMetadata => None, + } + } +} + +impl fmt::Display for UserAgentInterceptorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::InvalidHeaderValue(_) => "AwsUserAgent generated an invalid HTTP header value. This is a bug. Please file an issue.", + Self::MissingApiMetadata => "The UserAgentInterceptor requires ApiMetadata to be set before the request is made. This is a bug. Please file an issue.", + }) + } +} + +impl From for UserAgentInterceptorError { + fn from(err: InvalidHeaderValue) -> Self { + UserAgentInterceptorError::InvalidHeaderValue(err) + } +} + +/// Generates and attaches the AWS SDK's user agent to a HTTP request +#[non_exhaustive] +#[derive(Debug, Default)] +pub struct UserAgentInterceptor; + +impl UserAgentInterceptor { + /// Creates a new `UserAgentInterceptor` + pub fn new() -> Self { + UserAgentInterceptor + } +} + +fn header_values( + ua: &AwsUserAgent, +) -> Result<(HeaderValue, HeaderValue), UserAgentInterceptorError> { + // Pay attention to the extremely subtle difference between ua_header and aws_ua_header below... + Ok(( + HeaderValue::try_from(ua.ua_header())?, + HeaderValue::try_from(ua.aws_ua_header())?, + )) +} + +impl Intercept for UserAgentInterceptor { + fn name(&self) -> &'static str { + "UserAgentInterceptor" + } + + fn modify_before_signing( + &self, + context: &mut BeforeTransmitInterceptorContextMut<'_>, + _runtime_components: &RuntimeComponents, + cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + // Allow for overriding the user agent by an earlier interceptor (so, for example, + // tests can use `AwsUserAgent::for_tests()`) by attempting to grab one out of the + // config bag before creating one. + let ua: Cow<'_, AwsUserAgent> = cfg + .load::() + .map(Cow::Borrowed) + .map(Result::<_, UserAgentInterceptorError>::Ok) + .unwrap_or_else(|| { + let api_metadata = cfg + .load::() + .ok_or(UserAgentInterceptorError::MissingApiMetadata)?; + let mut ua = AwsUserAgent::new_from_environment(Env::real(), api_metadata.clone()); + + let maybe_app_name = cfg.load::(); + if let Some(app_name) = maybe_app_name { + ua.set_app_name(app_name.clone()); + } + Ok(Cow::Owned(ua)) + })?; + + let headers = context.request_mut().headers_mut(); + let (user_agent, x_amz_user_agent) = header_values(&ua)?; + headers.append(USER_AGENT, user_agent); + headers.append(X_AMZ_USER_AGENT, x_amz_user_agent); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aws_smithy_runtime_api::client::interceptors::context::{Input, InterceptorContext}; + use aws_smithy_runtime_api::client::interceptors::Intercept; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; + use aws_smithy_types::config_bag::{ConfigBag, Layer}; + use aws_smithy_types::error::display::DisplayErrorContext; + + fn expect_header<'a>(context: &'a InterceptorContext, header_name: &str) -> &'a str { + context + .request() + .expect("request is set") + .headers() + .get(header_name) + .unwrap() + } + + fn context() -> InterceptorContext { + let mut context = InterceptorContext::new(Input::doesnt_matter()); + context.enter_serialization_phase(); + context.set_request(HttpRequest::empty()); + let _ = context.take_input(); + context.enter_before_transmit_phase(); + context + } + + #[test] + fn test_overridden_ua() { + let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); + let mut context = context(); + + let mut layer = Layer::new("test"); + layer.store_put(AwsUserAgent::for_tests()); + layer.store_put(ApiMetadata::new("unused", "unused")); + let mut cfg = ConfigBag::of_layers(vec![layer]); + + let interceptor = UserAgentInterceptor::new(); + let mut ctx = Into::into(&mut context); + interceptor + .modify_before_signing(&mut ctx, &rc, &mut cfg) + .unwrap(); + + let header = expect_header(&context, "user-agent"); + assert_eq!(AwsUserAgent::for_tests().ua_header(), header); + assert!(!header.contains("unused")); + + assert_eq!( + AwsUserAgent::for_tests().aws_ua_header(), + expect_header(&context, "x-amz-user-agent") + ); + } + + #[test] + fn test_default_ua() { + let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); + let mut context = context(); + + let api_metadata = ApiMetadata::new("some-service", "some-version"); + let mut layer = Layer::new("test"); + layer.store_put(api_metadata.clone()); + let mut config = ConfigBag::of_layers(vec![layer]); + + let interceptor = UserAgentInterceptor::new(); + let mut ctx = Into::into(&mut context); + interceptor + .modify_before_signing(&mut ctx, &rc, &mut config) + .unwrap(); + + let expected_ua = AwsUserAgent::new_from_environment(Env::real(), api_metadata); + assert!( + expected_ua.aws_ua_header().contains("some-service"), + "precondition" + ); + assert_eq!( + expected_ua.ua_header(), + expect_header(&context, "user-agent") + ); + assert_eq!( + expected_ua.aws_ua_header(), + expect_header(&context, "x-amz-user-agent") + ); + } + + #[test] + fn test_app_name() { + let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); + let mut context = context(); + + let api_metadata = ApiMetadata::new("some-service", "some-version"); + let mut layer = Layer::new("test"); + layer.store_put(api_metadata); + layer.store_put(AppName::new("my_awesome_app").unwrap()); + let mut config = ConfigBag::of_layers(vec![layer]); + + let interceptor = UserAgentInterceptor::new(); + let mut ctx = Into::into(&mut context); + interceptor + .modify_before_signing(&mut ctx, &rc, &mut config) + .unwrap(); + + let app_value = "app/my_awesome_app"; + let header = expect_header(&context, "user-agent"); + assert!( + !header.contains(app_value), + "expected `{header}` to not contain `{app_value}`" + ); + + let header = expect_header(&context, "x-amz-user-agent"); + assert!( + header.contains(app_value), + "expected `{header}` to contain `{app_value}`" + ); + } + + #[test] + fn test_api_metadata_missing() { + let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); + let mut context = context(); + let mut config = ConfigBag::base(); + + let interceptor = UserAgentInterceptor::new(); + let mut ctx = Into::into(&mut context); + + let error = format!( + "{}", + DisplayErrorContext( + &*interceptor + .modify_before_signing(&mut ctx, &rc, &mut config) + .expect_err("it should error") + ) + ); + assert!( + error.contains("This is a bug"), + "`{error}` should contain message `This is a bug`" + ); + } + + #[test] + fn test_api_metadata_missing_with_ua_override() { + let rc = RuntimeComponentsBuilder::for_tests().build().unwrap(); + let mut context = context(); + + let mut layer = Layer::new("test"); + layer.store_put(AwsUserAgent::for_tests()); + let mut config = ConfigBag::of_layers(vec![layer]); + + let interceptor = UserAgentInterceptor::new(); + let mut ctx = Into::into(&mut context); + + interceptor + .modify_before_signing(&mut ctx, &rc, &mut config) + .expect("it should succeed"); + + let header = expect_header(&context, "user-agent"); + assert_eq!(AwsUserAgent::for_tests().ua_header(), header); + assert!(!header.contains("unused")); + + assert_eq!( + AwsUserAgent::for_tests().aws_ua_header(), + expect_header(&context, "x-amz-user-agent") + ); + } +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCargoDependency.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCargoDependency.kt index 001f17984e1..ae93a61e5af 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCargoDependency.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCargoDependency.kt @@ -19,8 +19,6 @@ object AwsCargoDependency { fun awsCredentialTypes(runtimeConfig: RuntimeConfig) = runtimeConfig.awsRuntimeCrate("aws-credential-types") - fun awsHttp(runtimeConfig: RuntimeConfig) = runtimeConfig.awsRuntimeCrate("aws-http") - fun awsRuntime(runtimeConfig: RuntimeConfig) = runtimeConfig.awsRuntimeCrate("aws-runtime") fun awsRuntimeApi(runtimeConfig: RuntimeConfig) = runtimeConfig.awsRuntimeCrate("aws-runtime-api") diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsRuntimeType.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsRuntimeType.kt index b7042a90ee4..3ab7f571d10 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsRuntimeType.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsRuntimeType.kt @@ -50,8 +50,6 @@ object AwsRuntimeType { fun awsCredentialTypesTestUtil(runtimeConfig: RuntimeConfig) = AwsCargoDependency.awsCredentialTypes(runtimeConfig).toDevDependency().withFeature("test-util").toType() - fun awsHttp(runtimeConfig: RuntimeConfig) = AwsCargoDependency.awsHttp(runtimeConfig).toType() - fun awsSigv4(runtimeConfig: RuntimeConfig) = AwsCargoDependency.awsSigv4(runtimeConfig).toType() fun awsTypes(runtimeConfig: RuntimeConfig) = AwsCargoDependency.awsTypes(runtimeConfig).toType() diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 37bff546c94..2500712e8ca 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -34,6 +34,7 @@ private fun RuntimeConfig.awsInlineableHttpRequestChecksum() = CargoDependency.Http, CargoDependency.HttpBody, CargoDependency.Tracing, + AwsCargoDependency.awsRuntime(this).withFeature("http-02x"), CargoDependency.smithyChecksums(this), CargoDependency.smithyHttp(this), CargoDependency.smithyRuntimeApiClient(this), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt index b8b69f275cf..927a4c8dd2c 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/UserAgentDecorator.kt @@ -71,7 +71,7 @@ class UserAgentDecorator : ClientCodegenDecorator { pub(crate) static API_METADATA: #{user_agent}::ApiMetadata = #{user_agent}::ApiMetadata::new(${serviceId.dq()}, #{PKG_VERSION}); """, - "user_agent" to AwsRuntimeType.awsHttp(runtimeConfig).resolve("user_agent"), + "user_agent" to AwsRuntimeType.awsRuntime(runtimeConfig).resolve("user_agent"), "PKG_VERSION" to CrateVersionCustomization.pkgVersion(ClientRustModule.Meta), ) } @@ -109,7 +109,7 @@ class UserAgentDecorator : ClientCodegenDecorator { arrayOf( *preludeScope, "AppName" to AwsRuntimeType.awsTypes(runtimeConfig).resolve("app_name::AppName"), - "AwsUserAgent" to AwsRuntimeType.awsHttp(runtimeConfig).resolve("user_agent::AwsUserAgent"), + "AwsUserAgent" to AwsRuntimeType.awsRuntime(runtimeConfig).resolve("user_agent::AwsUserAgent"), ) override fun section(section: ServiceConfig): Writable = diff --git a/aws/sdk/integration-tests/dynamodb/Cargo.toml b/aws/sdk/integration-tests/dynamodb/Cargo.toml index 57eadecc005..c6b38377483 100644 --- a/aws/sdk/integration-tests/dynamodb/Cargo.toml +++ b/aws/sdk/integration-tests/dynamodb/Cargo.toml @@ -14,7 +14,6 @@ publish = false approx = "0.5.1" aws-config = { path = "../../build/aws-sdk/sdk/aws-config" } aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-sdk-dynamodb = { path = "../../build/aws-sdk/sdk/dynamodb", features = ["behavior-version-latest"] } aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", features = ["test-util"] } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } diff --git a/aws/sdk/integration-tests/glacier/Cargo.toml b/aws/sdk/integration-tests/glacier/Cargo.toml index 418cbb6bae8..6674d35d472 100644 --- a/aws/sdk/integration-tests/glacier/Cargo.toml +++ b/aws/sdk/integration-tests/glacier/Cargo.toml @@ -12,7 +12,6 @@ publish = false [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http"} aws-sdk-glacier = { path = "../../build/aws-sdk/sdk/glacier", features = ["behavior-version-latest"] } aws-smithy-protocol-test = { path = "../../build/aws-sdk/sdk/aws-smithy-protocol-test"} aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "test-util"] } diff --git a/aws/sdk/integration-tests/iam/Cargo.toml b/aws/sdk/integration-tests/iam/Cargo.toml index 45235fa71aa..0c1a02e8b8d 100644 --- a/aws/sdk/integration-tests/iam/Cargo.toml +++ b/aws/sdk/integration-tests/iam/Cargo.toml @@ -12,7 +12,6 @@ publish = false [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http"} aws-sdk-iam = { path = "../../build/aws-sdk/sdk/iam", features = ["behavior-version-latest"] } aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "test-util"] } aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } diff --git a/aws/sdk/integration-tests/kms/Cargo.toml b/aws/sdk/integration-tests/kms/Cargo.toml index d29e5b2158e..86e314597df 100644 --- a/aws/sdk/integration-tests/kms/Cargo.toml +++ b/aws/sdk/integration-tests/kms/Cargo.toml @@ -16,7 +16,6 @@ test-util = [] [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-runtime = { path = "../../build/aws-sdk/sdk/aws-runtime" } aws-sdk-kms = { path = "../../build/aws-sdk/sdk/kms", features = ["test-util", "behavior-version-latest"] } aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", features = ["test-util"] } diff --git a/aws/sdk/integration-tests/lambda/Cargo.toml b/aws/sdk/integration-tests/lambda/Cargo.toml index ec891d0100a..45ff53adffc 100644 --- a/aws/sdk/integration-tests/lambda/Cargo.toml +++ b/aws/sdk/integration-tests/lambda/Cargo.toml @@ -10,7 +10,6 @@ publish = false [dev-dependencies] async-stream = "0.3.0" aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-sdk-lambda = { path = "../../build/aws-sdk/sdk/lambda", features = ["behavior-version-latest"] } aws-smithy-eventstream = { path = "../../build/aws-sdk/sdk/aws-smithy-eventstream" } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } diff --git a/aws/sdk/integration-tests/polly/Cargo.toml b/aws/sdk/integration-tests/polly/Cargo.toml index 9e7e83a51ce..c68b4e69ce8 100644 --- a/aws/sdk/integration-tests/polly/Cargo.toml +++ b/aws/sdk/integration-tests/polly/Cargo.toml @@ -12,7 +12,6 @@ publish = false [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http"} aws-sdk-polly = { path = "../../build/aws-sdk/sdk/polly", features = ["behavior-version-latest"] } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } bytes = "1.0.0" diff --git a/aws/sdk/integration-tests/qldbsession/Cargo.toml b/aws/sdk/integration-tests/qldbsession/Cargo.toml index 06673d84338..d235881dd4a 100644 --- a/aws/sdk/integration-tests/qldbsession/Cargo.toml +++ b/aws/sdk/integration-tests/qldbsession/Cargo.toml @@ -16,7 +16,6 @@ test-util = [] [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-sdk-qldbsession = { path = "../../build/aws-sdk/sdk/qldbsession", features = ["test-util", "behavior-version-latest"] } aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async" } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } diff --git a/aws/sdk/integration-tests/s3/Cargo.toml b/aws/sdk/integration-tests/s3/Cargo.toml index cdb22592ba8..8ba7542e603 100644 --- a/aws/sdk/integration-tests/s3/Cargo.toml +++ b/aws/sdk/integration-tests/s3/Cargo.toml @@ -18,7 +18,6 @@ test-util = [] async-std = "1.12.0" aws-config = { path = "../../build/aws-sdk/sdk/aws-config", features = ["behavior-version-latest"] } aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-runtime = { path = "../../build/aws-sdk/sdk/aws-runtime", features = ["test-util"] } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", features = ["test-util", "behavior-version-latest"] } aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", features = ["test-util", "rt-tokio"] } diff --git a/aws/sdk/integration-tests/s3/tests/checksums.rs b/aws/sdk/integration-tests/s3/tests/checksums.rs index a61f9026e56..175f7c5f0fd 100644 --- a/aws/sdk/integration-tests/s3/tests/checksums.rs +++ b/aws/sdk/integration-tests/s3/tests/checksums.rs @@ -226,7 +226,7 @@ async fn test_checksum_on_streaming_request<'a>( "x-amz-trailer is incorrect" ); assert_eq!( - HeaderValue::from_static(aws_http::content_encoding::header_value::AWS_CHUNKED), + HeaderValue::from_static(aws_runtime::content_encoding::header_value::AWS_CHUNKED), content_encoding, "content-encoding wasn't set to aws-chunked" ); diff --git a/aws/sdk/integration-tests/s3/tests/request_information_headers.rs b/aws/sdk/integration-tests/s3/tests/request_information_headers.rs index e12a4317a95..47de27c437e 100644 --- a/aws/sdk/integration-tests/s3/tests/request_information_headers.rs +++ b/aws/sdk/integration-tests/s3/tests/request_information_headers.rs @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_http::user_agent::AwsUserAgent; use aws_runtime::invocation_id::{InvocationId, PredefinedInvocationIdGenerator}; +use aws_runtime::user_agent::AwsUserAgent; use aws_sdk_s3::config::interceptors::BeforeSerializationInterceptorContextMut; use aws_sdk_s3::config::interceptors::FinalizerInterceptorContextRef; use aws_sdk_s3::config::retry::RetryConfig; diff --git a/aws/sdk/integration-tests/s3control/Cargo.toml b/aws/sdk/integration-tests/s3control/Cargo.toml index 9cad807f122..fc059b28375 100644 --- a/aws/sdk/integration-tests/s3control/Cargo.toml +++ b/aws/sdk/integration-tests/s3control/Cargo.toml @@ -16,7 +16,6 @@ test-util = [] [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-sdk-s3control = { path = "../../build/aws-sdk/sdk/s3control", features = ["test-util", "behavior-version-latest"] } aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async" } aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "test-util"] } diff --git a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml index 0624ef56785..221d7b4b78a 100644 --- a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml +++ b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml @@ -11,7 +11,6 @@ publish = false [dev-dependencies] async-stream = "0.3.0" aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-http = { path = "../../build/aws-sdk/sdk/aws-http" } aws-sdk-transcribestreaming = { path = "../../build/aws-sdk/sdk/transcribestreaming", features = ["behavior-version-latest"] } aws-smithy-eventstream = { path = "../../build/aws-sdk/sdk/aws-smithy-eventstream" } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } From 5e20575e27769665bbf176958ce90fc63b09d3a2 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Tue, 23 Jan 2024 17:55:43 -0800 Subject: [PATCH 9/9] Make SDK runtime patch tool that supports SDK runtime crates (#3369) A tool to patch in new versions of smithy-rs runtime crates was introduced in #3327 so that semver breaks could better be detected for upcoming releases. However, this tool didn't support patching in SDK runtime crates, which is necessary for testing #3355. This PR moves the tool into the runtime-versioner tool, and adjusts it to support both smithy-rs and SDK runtime crates. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../runtime-release-dryrun/Cargo.lock | 353 ------------------ .../runtime-release-dryrun/Cargo.toml | 11 - .../runtime-release-dryrun/src/main.rs | 182 --------- tools/ci-build/runtime-versioner/Cargo.lock | 60 +++ tools/ci-build/runtime-versioner/Cargo.toml | 1 + .../src/{ => command}/audit.rs | 0 .../runtime-versioner/src/command/patch.rs | 187 ++++++++++ tools/ci-build/runtime-versioner/src/main.rs | 59 ++- tools/ci-build/runtime-versioner/src/repo.rs | 11 +- 9 files changed, 315 insertions(+), 549 deletions(-) delete mode 100644 tools/ci-build/runtime-release-dryrun/Cargo.lock delete mode 100644 tools/ci-build/runtime-release-dryrun/Cargo.toml delete mode 100644 tools/ci-build/runtime-release-dryrun/src/main.rs rename tools/ci-build/runtime-versioner/src/{ => command}/audit.rs (100%) create mode 100644 tools/ci-build/runtime-versioner/src/command/patch.rs diff --git a/tools/ci-build/runtime-release-dryrun/Cargo.lock b/tools/ci-build/runtime-release-dryrun/Cargo.lock deleted file mode 100644 index 5d86df86e9b..00000000000 --- a/tools/ci-build/runtime-release-dryrun/Cargo.lock +++ /dev/null @@ -1,353 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "anyhow" -version = "1.0.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clap" -version = "4.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5840cd9093aabeabf7fd932754c435b7674520fc3ddc935c397837050f0f1e4b" -dependencies = [ - "atty", - "bitflags", - "clap_derive", - "clap_lex", - "once_cell", - "strsim", - "termcolor", -] - -[[package]] -name = "clap_derive" -version = "4.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92289ffc6fb4a85d85c246ddb874c05a87a2e540fb6ad52f7ca07c8c1e1840b1" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "console" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys", -] - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "indicatif" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "unicode-width", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "runtime-release-dryrun" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "indicatif", -] - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "termcolor" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff --git a/tools/ci-build/runtime-release-dryrun/Cargo.toml b/tools/ci-build/runtime-release-dryrun/Cargo.toml deleted file mode 100644 index cc4d5e35b5a..00000000000 --- a/tools/ci-build/runtime-release-dryrun/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "runtime-release-dryrun" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -clap = { version = "4", features = ["derive"] } -anyhow = "1.0.75" -indicatif = "0.17.7" diff --git a/tools/ci-build/runtime-release-dryrun/src/main.rs b/tools/ci-build/runtime-release-dryrun/src/main.rs deleted file mode 100644 index d10d77b9df7..00000000000 --- a/tools/ci-build/runtime-release-dryrun/src/main.rs +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -// this tool is simple and works at what it does -// potential improvements: -// - support the release of rust-runtime crates -// - support patching the users system-wide ~/.cargo/config.toml - -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time::Duration; - -use anyhow::{bail, Context, Result}; -use clap::Parser; -use indicatif::{ProgressBar, ProgressStyle}; - -#[derive(Parser, Debug)] -struct DryRunSdk { - /// Path to the aws-sdk-rust repo - #[clap(long)] - pub sdk_path: PathBuf, - - /// Tag of the aws-sdk-rust repo to dry-run against - #[clap(long)] - pub rust_sdk_tag: String, - - /// Path to the artifact produced by the release-dry run - #[clap(long)] - pub smithy_rs_release: PathBuf, -} - -#[derive(Parser, Debug)] -#[clap( - name = "runtime-release-dryrun", - about = "CLI tool to prepare the aws-sdk-rust to test the result of releasing a new set of runtime crates.", - version -)] -#[allow(clippy::enum_variant_names)] // Want the "use" prefix in the CLI subcommand names for clarity -enum Args { - /// Dry run a smithy-rs release against a rust SDK release - /// - /// You will need: - /// 1. An `aws-sdk-rust` repo with a clean working tree - /// 2. A directory containing the artifacts from a smithy-rs release dry run. This is an artifact - /// named `artifacts-generate-smithy-rs-release` from the GH action (e.g. https://github.com/smithy-lang/smithy-rs/actions/runs/7200898068) - /// 3. An `aws-sdk-rust` release tag you want to test against - /// - /// After running the tool, you might want to do something like `cargo test` in `s3`. Make sure - /// to run `cargo update` to pull in the new dependencies. Use `cargo tree` to confirm you're - /// actually consuming the new versions. - #[clap(verbatim_doc_comment)] - DryRunSdk(DryRunSdk), -} - -fn main() -> Result<()> { - let args = Args::parse(); - match args { - Args::DryRunSdk(args) => dry_run_sdk(args).map_err(|err| { - // workaround an indicatif (bug?) where one character is stripped from output on the error message - eprintln!(" "); - err - })?, - } - Ok(()) -} - -fn step(message: &'static str, step: impl FnOnce() -> Result) -> Result { - let spinner = ProgressBar::new_spinner() - .with_message(message) - .with_style(ProgressStyle::with_template("{spinner} {msg} {elapsed}").unwrap()); - spinner.enable_steady_tick(Duration::from_millis(100)); - let result = step(); - let check = match &result { - Ok(_) => "✅", - Err(_) => "❌", - }; - spinner.set_style(ProgressStyle::with_template("{msg} {elapsed}").unwrap()); - spinner.finish_with_message(format!("{check} {message}")); - result -} - -fn run(command: &mut Command) -> anyhow::Result<()> { - let status = command.output()?; - if !status.status.success() { - bail!( - "command `{:?}` failed:\n{}{}", - command, - String::from_utf8_lossy(&status.stdout), - String::from_utf8_lossy(&status.stderr) - ); - } - Ok(()) -} - -fn dry_run_sdk(args: DryRunSdk) -> Result<()> { - step("Checking out SDK tag", || { - run(Command::new("git") - .arg("checkout") - .arg(&args.rust_sdk_tag) - .current_dir(&args.sdk_path)) - .context("failed to checkout aws-sdk-rust revision")?; - Ok(()) - })?; - - // By default the SDK dependencies also include a path component. This prevents - // patching from working - step("Applying version-only dependencies", || { - run(Command::new("sdk-versioner") - .args([ - "use-version-dependencies", - "--versions-toml", - "versions.toml", - "sdk", - ]) - .current_dir(&args.sdk_path))?; - - run(Command::new("git") - .args(["checkout", "-B", "smithy-release-dryrun"]) - .current_dir(&args.sdk_path))?; - run(Command::new("git") - .args([ - "commit", - "-am", - "removing path dependencies to allow patching", - ]) - .current_dir(&args.sdk_path))?; - Ok(()) - })?; - - let patches = step("computing patches", || { - let path = Path::new(&args.smithy_rs_release).join("crates-to-publish"); - let crates_to_patch = std::fs::read_dir(&path) - .context(format!("could list crates in directory {:?}", path))? - .map(|dir| dir.unwrap().file_name()) - .map(|osstr| osstr.into_string().expect("invalid utf-8 directory")) - .collect::>(); - - let patch_sections = crates_to_patch - .iter() - .map(|crte| { - let path = Path::new(&args.smithy_rs_release) - .join("crates-to-publish") - .join(crte); - assert!( - path.exists(), - "tried to reference a crate that did not exist!" - ); - format!( - "{crte} = {{ path = '{}' }}", - path.canonicalize().unwrap().to_str().unwrap() - ) - }) - .collect::>() - .join("\n"); - Ok(format!("[patch.crates-io]\n{patch_sections}")) - })?; - - // Note: in the future we could also automatically apply this to the system wide ~/.cargo/config.toml - step("apply patches to workspace Cargo.toml", || { - let workspace_cargo_toml = Path::new(&args.sdk_path).join("Cargo.toml"); - if !workspace_cargo_toml.exists() { - bail!( - "Could not find the workspace Cargo.toml to patch {:?}", - workspace_cargo_toml - ); - } - let current_contents = std::fs::read_to_string(&workspace_cargo_toml) - .context("could not read workspace cargo.toml")?; - std::fs::write( - workspace_cargo_toml, - format!("{current_contents}\n{patches}"), - )?; - run(Command::new("git") - .args(["commit", "-am", "patching workspace Cargo.toml"]) - .current_dir(&args.sdk_path))?; - Ok(()) - })?; - println!("{:?} has been updated to build against patches. Use `cargo update` to recompute the dependencies.", &args.sdk_path); - Ok(()) -} diff --git a/tools/ci-build/runtime-versioner/Cargo.lock b/tools/ci-build/runtime-versioner/Cargo.lock index f73ff75a995..35ea1e2fdee 100644 --- a/tools/ci-build/runtime-versioner/Cargo.lock +++ b/tools/ci-build/runtime-versioner/Cargo.lock @@ -209,6 +209,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -245,6 +258,12 @@ dependencies = [ "toml 0.8.8", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -522,6 +541,28 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -646,6 +687,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -735,6 +782,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "proc-macro2" version = "1.0.76" @@ -852,6 +905,7 @@ dependencies = [ "camino", "clap", "crates-index", + "indicatif", "reqwest", "smithy-rs-tool-common", "tempfile", @@ -1341,6 +1395,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "url" version = "2.5.0" diff --git a/tools/ci-build/runtime-versioner/Cargo.toml b/tools/ci-build/runtime-versioner/Cargo.toml index 15a356eb0b0..fa6834bac61 100644 --- a/tools/ci-build/runtime-versioner/Cargo.toml +++ b/tools/ci-build/runtime-versioner/Cargo.toml @@ -18,6 +18,7 @@ anyhow = "1.0.75" camino = "1.1.6" clap = { version = "4.4.11", features = ["derive"] } crates-index = "2.3.0" +indicatif = "0.17.7" reqwest = { version = "0.11.22", features = ["blocking"] } smithy-rs-tool-common = { version = "0.1", path = "../smithy-rs-tool-common" } tempfile = "3.9.0" diff --git a/tools/ci-build/runtime-versioner/src/audit.rs b/tools/ci-build/runtime-versioner/src/command/audit.rs similarity index 100% rename from tools/ci-build/runtime-versioner/src/audit.rs rename to tools/ci-build/runtime-versioner/src/command/audit.rs diff --git a/tools/ci-build/runtime-versioner/src/command/patch.rs b/tools/ci-build/runtime-versioner/src/command/patch.rs new file mode 100644 index 00000000000..db6753c127c --- /dev/null +++ b/tools/ci-build/runtime-versioner/src/command/patch.rs @@ -0,0 +1,187 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::{ + repo::Repo, + tag::{previous_release_tag, release_tags}, + PatchRuntime, PatchRuntimeWith, +}; +use anyhow::{bail, Context, Result}; +use camino::Utf8Path; +use indicatif::{ProgressBar, ProgressStyle}; +use smithy_rs_tool_common::command::sync::CommandExt; +use std::{fs, time::Duration}; + +pub fn patch(args: PatchRuntime) -> Result<()> { + let smithy_rs = step("Resolving smithy-rs", || { + Repo::new(args.smithy_rs_path.as_deref()) + })?; + if is_dirty(&smithy_rs)? { + bail!("smithy-rs has a dirty working tree. Aborting."); + } + + step( + "Patching smithy-rs/gradle.properties with given crate version numbers", + || patch_gradle_properties(&smithy_rs, &args), + )?; + + // Use aws:sdk:assemble to generate both the smithy-rs runtime and AWS SDK + // runtime crates with the correct version numbers. + step("Generating an AWS SDK", || { + smithy_rs + .cmd( + "./gradlew", + // limit services down to minimum required to reduce generation time + ["-Paws.services=+sts,+sso,+ssooidc", "aws:sdk:assemble"], + ) + .expect_success_output("assemble SDK") + })?; + + patch_with(PatchRuntimeWith { + sdk_path: args.sdk_path, + runtime_crate_path: smithy_rs.root.join("aws/sdk/build/aws-sdk/sdk"), + previous_release_tag: args.previous_release_tag, + })?; + + Ok(()) +} + +pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { + let aws_sdk_rust = step("Resolving aws-sdk-rust", || Repo::new(Some(&args.sdk_path)))?; + if is_dirty(&aws_sdk_rust)? { + bail!("aws-sdk-rust has a dirty working tree. Aborting."); + } + + // Make sure the aws-sdk-rust repo is on the correct release tag + let release_tags = step("Resolving aws-sdk-rust release tags", || { + release_tags(&aws_sdk_rust) + })?; + let previous_release_tag = step("Resolving release tag", || { + previous_release_tag( + &aws_sdk_rust, + &release_tags, + args.previous_release_tag.as_deref(), + ) + })?; + step("Checking out release tag", || { + aws_sdk_rust + .git(["checkout", previous_release_tag.as_str()]) + .expect_success_output("check out release tag in aws-sdk-rust") + })?; + + // Patch the new runtime crates into the old SDK + step("Applying version-only dependencies", || { + apply_version_only_dependencies(&aws_sdk_rust) + })?; + step("Patching aws-sdk-rust root Cargo.toml", || { + patch_workspace_cargo_toml(&aws_sdk_rust, &args.runtime_crate_path) + })?; + step("Running cargo update", || { + aws_sdk_rust + .cmd("cargo", ["update"]) + .expect_success_output("cargo update") + })?; + + Ok(()) +} + +fn patch_gradle_properties(smithy_rs: &Repo, args: &PatchRuntime) -> Result<()> { + let props_path = smithy_rs.root.join("gradle.properties"); + let props = + fs::read_to_string(&props_path).context("failed to read smithy-rs/gradle.properties")?; + let mut new_props = String::with_capacity(props.len()); + for line in props.lines() { + if line.starts_with("smithy.rs.runtime.crate.stable.version=") { + new_props.push_str(&format!( + "smithy.rs.runtime.crate.stable.version={}", + args.stable_crate_version + )); + } else if line.starts_with("smithy.rs.runtime.crate.unstable.version=") { + new_props.push_str(&format!( + "smithy.rs.runtime.crate.unstable.version={}", + args.unstable_crate_version + )); + } else { + new_props.push_str(line); + } + new_props.push('\n'); + } + fs::write(&props_path, new_props).context("failed to write smithy-rs/gradle.properties")?; + Ok(()) +} + +fn apply_version_only_dependencies(aws_sdk_rust: &Repo) -> Result<()> { + aws_sdk_rust + .cmd( + "sdk-versioner", + [ + "use-version-dependencies", + "--versions-toml", + "versions.toml", + "sdk", + ], + ) + .expect_success_output("run sdk-versioner")?; + Ok(()) +} + +fn patch_workspace_cargo_toml(aws_sdk_rust: &Repo, runtime_crate_path: &Utf8Path) -> Result<()> { + let crates_to_patch = fs::read_dir(runtime_crate_path) + .context(format!( + "could list crates in directory {:?}", + runtime_crate_path + ))? + .map(|dir| dir.unwrap().file_name()) + .map(|osstr| osstr.into_string().expect("invalid utf-8 directory")) + .filter(|name| name.starts_with("aws-")) + .collect::>(); + + let patch_sections = crates_to_patch + .iter() + .map(|crte| { + let path = runtime_crate_path.join(crte); + assert!( + path.exists(), + "tried to reference a crate that did not exist!" + ); + format!( + "{crte} = {{ path = '{}' }}", + path.canonicalize_utf8().unwrap() + ) + }) + .collect::>() + .join("\n"); + let patch_section = format!("\n[patch.crates-io]\n{patch_sections}"); + + let manifest_path = aws_sdk_rust.root.join("Cargo.toml"); + let mut manifest_content = + fs::read_to_string(&manifest_path).context("failed to read aws-sdk-rust/Cargo.toml")?; + manifest_content.push_str(&patch_section); + fs::write(&manifest_path, &manifest_content) + .context("failed to write aws-sdk-rust/Cargo.toml")?; + Ok(()) +} + +fn is_dirty(repo: &Repo) -> Result { + let result = repo + .git(["status", "--porcelain"]) + .expect_success_output("git status")?; + Ok(!result.trim().is_empty()) +} + +fn step(message: &'static str, step: impl FnOnce() -> Result) -> Result { + let spinner = ProgressBar::new_spinner() + .with_message(message) + .with_style(ProgressStyle::with_template("{spinner} {msg} {elapsed}").unwrap()); + spinner.enable_steady_tick(Duration::from_millis(100)); + let result = step(); + let check = match &result { + Ok(_) => "✅", + Err(_) => "❌", + }; + spinner.set_style(ProgressStyle::with_template("{msg} {elapsed}").unwrap()); + spinner.finish_with_message(format!("{check} {message}")); + result +} diff --git a/tools/ci-build/runtime-versioner/src/main.rs b/tools/ci-build/runtime-versioner/src/main.rs index e09951f043f..6a7dafa7bc2 100644 --- a/tools/ci-build/runtime-versioner/src/main.rs +++ b/tools/ci-build/runtime-versioner/src/main.rs @@ -12,7 +12,14 @@ use camino::Utf8PathBuf; use clap::Parser; use tracing_subscriber::{filter::LevelFilter, EnvFilter}; -mod audit; +mod command { + mod audit; + pub use audit::audit; + + mod patch; + pub use patch::{patch, patch_with}; +} + mod index; mod repo; mod tag; @@ -41,6 +48,41 @@ pub struct PreviousReleaseTag { smithy_rs_path: Option, } +#[derive(clap::Args, Clone)] +pub struct PatchRuntime { + /// Path to aws-sdk-rust. + #[arg(long)] + sdk_path: Utf8PathBuf, + /// Path to smithy-rs. Defaults to current working directory. + #[arg(long)] + smithy_rs_path: Option, + /// Explicitly state the previous release's tag. Discovers it if not provided. + #[arg(long)] + previous_release_tag: Option, + /// Version number for stable crates. + #[arg(long)] + stable_crate_version: String, + /// Version number for unstable crates. + #[arg(long)] + unstable_crate_version: String, +} + +#[derive(clap::Args, Clone)] +pub struct PatchRuntimeWith { + /// Path to aws-sdk-rust. + #[arg(long)] + sdk_path: Utf8PathBuf, + /// Path to runtime crates to patch in. + /// + /// Note: this doesn't need to be a complete set of runtime crates. It will + /// only patch the crates included in the provided path. + #[arg(long)] + runtime_crate_path: Utf8PathBuf, + /// Explicitly state the previous release's tag. Discovers it if not provided. + #[arg(long)] + previous_release_tag: Option, +} + #[derive(clap::Parser, Clone)] #[clap(author, version, about)] enum Command { @@ -56,6 +98,17 @@ enum Command { /// Outputs the previous release tag for the revision at HEAD. PreviousReleaseTag(PreviousReleaseTag), + + /// Patch a previous SDK release with the latest to-be-released runtime crates. + /// + /// This will generate a runtime with the given smithy-rs repo. + PatchRuntime(PatchRuntime), + + /// Patch a previous SDK release with a given runtime. + /// + /// This will use an existing runtime at the path provided. For example, + /// if you want to try a runtime from a GitHub Actions workflow. + PatchRuntimeWith(PatchRuntimeWith), } fn main() -> Result<()> { @@ -70,7 +123,7 @@ fn main() -> Result<()> { let command = Command::parse(); match command { - Command::Audit(args) => audit::audit(args), + Command::Audit(args) => command::audit(args), Command::PreviousReleaseTag(args) => { let repo = Repo::new(args.smithy_rs_path.as_deref())?; let tags = release_tags(&repo)?; @@ -78,5 +131,7 @@ fn main() -> Result<()> { println!("{tag}"); Ok(()) } + Command::PatchRuntime(args) => command::patch(args), + Command::PatchRuntimeWith(args) => command::patch_with(args), } } diff --git a/tools/ci-build/runtime-versioner/src/repo.rs b/tools/ci-build/runtime-versioner/src/repo.rs index 45290368a3d..f51ae070f84 100644 --- a/tools/ci-build/runtime-versioner/src/repo.rs +++ b/tools/ci-build/runtime-versioner/src/repo.rs @@ -27,7 +27,16 @@ impl Repo { I: IntoIterator, S: AsRef, { - let mut cmd = Command::new("git"); + self.cmd("git", args) + } + + /// Returns a `std::process::Command` set to run a shell command in this repo with the given args + pub fn cmd(&self, cmd: &str, args: I) -> Command + where + I: IntoIterator, + S: AsRef, + { + let mut cmd = Command::new(cmd); cmd.current_dir(&self.root); cmd.args(args); cmd