From ce9e38defdf457f31a97a11b052f2461602c8d50 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 20 Jan 2025 20:34:09 +0000 Subject: [PATCH 1/5] Add ability to add session token to aws get request --- .../core/src/clp/aws/AwsAuthenticationSigner.cpp | 11 ++++++++++- .../core/src/clp/aws/AwsAuthenticationSigner.hpp | 11 +++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 0b2d6fdb90..0f4a2c7184 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -248,8 +248,16 @@ auto AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_view timestamp) const -> string { auto const uri = fmt::format("{}/{}", m_access_key_id, scope); + std::string session_token_parameter; + if (m_session_token.has_value()) { + session_token_parameter = fmt::format( + "&{}={}", + "X-Amz-Security-Token", + encode_uri(m_session_token.value(), false) + ); + } return fmt::format( - "{}={}&{}={}&{}={}&{}={}&{}={}", + "{}={}&{}={}&{}={}&{}={}{}&{}={}", cXAmzAlgorithm, cAws4HmacSha256, cXAmzCredential, @@ -258,6 +266,7 @@ AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_vi timestamp, cXAmzExpires, cDefaultExpireTime.count(), + session_token_parameter, cXAmzSignedHeaders, cDefaultSignedHeaders ); diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp index 293ffcd4cd..c665d6e46e 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.hpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.hpp @@ -2,6 +2,7 @@ #define CLP_AWS_AWSAUTHENTICATIONSIGNER_HPP #include +#include #include #include #include @@ -69,9 +70,14 @@ class AwsAuthenticationSigner { static constexpr std::string_view cHttpGetMethod{"GET"}; // Constructors - AwsAuthenticationSigner(std::string access_key_id, std::string secret_access_key) + AwsAuthenticationSigner( + std::string access_key_id, + std::string secret_access_key, + std::optional session_token + ) : m_access_key_id{std::move(access_key_id)}, - m_secret_access_key{std::move(secret_access_key)} {} + m_secret_access_key{std::move(secret_access_key)}, + m_session_token{std::move(session_token)} {} // Methods /** @@ -129,6 +135,7 @@ class AwsAuthenticationSigner { // Variables std::string m_access_key_id; std::string m_secret_access_key; + std::optional m_session_token; }; } // namespace clp::aws From 0637d46de014453b5dd410a59d48766c2123ca31 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 20 Jan 2025 20:34:25 +0000 Subject: [PATCH 2/5] Add support for AWS_SESSION_TOKEN environment variable --- components/core/src/clp_s/CommandLineArguments.cpp | 6 +++--- components/core/src/clp_s/InputConfig.hpp | 1 + components/core/src/clp_s/ReaderUtils.cpp | 14 +++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index ece270a03e..7fcee9a705 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -274,7 +274,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ->default_value(auth), "Type of authentication required for network requests (s3 | none). Authentication" " with s3 requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment" - " variables." + " variables, and optionally the AWS_SESSION_TOKEN environment variable." ); // clang-format on @@ -428,7 +428,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ->default_value(auth), "Type of authentication required for network requests (s3 | none). Authentication" " with s3 requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment" - " variables." + " variables, and optionally the AWS_SESSION_TOKEN environment variable." ); // clang-format on extraction_options.add(decompression_options); @@ -582,7 +582,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ->default_value(auth), "Type of authentication required for network requests (s3 | none). Authentication" " with s3 requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment" - " variables." + " variables, and optionally the AWS_SESSION_TOKEN environment variable." ); // clang-format on search_options.add(match_options); diff --git a/components/core/src/clp_s/InputConfig.hpp b/components/core/src/clp_s/InputConfig.hpp index 8c6afc2ea8..1d0eae35dd 100644 --- a/components/core/src/clp_s/InputConfig.hpp +++ b/components/core/src/clp_s/InputConfig.hpp @@ -11,6 +11,7 @@ namespace clp_s { // Constants used for input configuration constexpr char cAwsAccessKeyIdEnvVar[] = "AWS_ACCESS_KEY_ID"; constexpr char cAwsSecretAccessKeyEnvVar[] = "AWS_SECRET_ACCESS_KEY"; +constexpr char cAwsSessionTokenEnvVar[] = "AWS_SESSION_TOKEN"; /** * Enum class defining the source of a resource. diff --git a/components/core/src/clp_s/ReaderUtils.cpp b/components/core/src/clp_s/ReaderUtils.cpp index 6286d367fc..2a149bb6fa 100644 --- a/components/core/src/clp_s/ReaderUtils.cpp +++ b/components/core/src/clp_s/ReaderUtils.cpp @@ -1,11 +1,14 @@ #include "ReaderUtils.hpp" #include +#include #include +#include #include #include "../clp/aws/AwsAuthenticationSigner.hpp" +#include "../clp/CurlDownloadHandler.hpp" #include "../clp/FileReader.hpp" #include "../clp/NetworkReader.hpp" #include "../clp/ReaderInterface.hpp" @@ -164,8 +167,17 @@ bool try_sign_url(std::string& url) { ); return false; } + std::optional optional_aws_session_token{std::nullopt}; + auto const aws_session_token = std::getenv(cAwsSessionTokenEnvVar); + if (nullptr != aws_session_token) { + optional_aws_session_token = std::string{aws_session_token}; + } - clp::aws::AwsAuthenticationSigner signer{aws_access_key, aws_secret_access_key}; + clp::aws::AwsAuthenticationSigner signer{ + aws_access_key, + aws_secret_access_key, + optional_aws_session_token + }; try { clp::aws::S3Url s3_url{url}; From 14157894e31d7cca3cb88cdbf908a32032a420ee Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 20 Jan 2025 21:18:26 +0000 Subject: [PATCH 3/5] Make X-Amz-Security-Token a constant --- components/core/src/clp/aws/AwsAuthenticationSigner.cpp | 2 +- components/core/src/clp/aws/constants.hpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index 0f4a2c7184..be95527c41 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -252,7 +252,7 @@ AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_vi if (m_session_token.has_value()) { session_token_parameter = fmt::format( "&{}={}", - "X-Amz-Security-Token", + cAmzSecurityToken, encode_uri(m_session_token.value(), false) ); } diff --git a/components/core/src/clp/aws/constants.hpp b/components/core/src/clp/aws/constants.hpp index caebe92a1a..94ee36465b 100644 --- a/components/core/src/clp/aws/constants.hpp +++ b/components/core/src/clp/aws/constants.hpp @@ -12,6 +12,7 @@ constexpr std::string_view cXAmzAlgorithm{"X-Amz-Algorithm"}; constexpr std::string_view cXAmzCredential{"X-Amz-Credential"}; constexpr std::string_view cXAmzDate{"X-Amz-Date"}; constexpr std::string_view cXAmzExpires{"X-Amz-Expires"}; +constexpr std::string_view cAmzSecurityToken{"X-Amz-Security-Token"}; constexpr std::string_view cXAmzSignature{"X-Amz-Signature"}; constexpr std::string_view cXAmzSignedHeaders{"X-Amz-SignedHeaders"}; From c274780a5c9c34cc1904f22c1247a06db979acaa Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 27 Jan 2025 17:11:40 +0000 Subject: [PATCH 4/5] Address review comments --- components/core/src/clp/aws/AwsAuthenticationSigner.cpp | 2 +- components/core/src/clp/aws/constants.hpp | 2 +- components/core/src/clp_s/ReaderUtils.cpp | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index be95527c41..aab55701d1 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -252,7 +252,7 @@ AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_vi if (m_session_token.has_value()) { session_token_parameter = fmt::format( "&{}={}", - cAmzSecurityToken, + cXAmzSecurityToken, encode_uri(m_session_token.value(), false) ); } diff --git a/components/core/src/clp/aws/constants.hpp b/components/core/src/clp/aws/constants.hpp index 94ee36465b..c85cfba531 100644 --- a/components/core/src/clp/aws/constants.hpp +++ b/components/core/src/clp/aws/constants.hpp @@ -12,7 +12,7 @@ constexpr std::string_view cXAmzAlgorithm{"X-Amz-Algorithm"}; constexpr std::string_view cXAmzCredential{"X-Amz-Credential"}; constexpr std::string_view cXAmzDate{"X-Amz-Date"}; constexpr std::string_view cXAmzExpires{"X-Amz-Expires"}; -constexpr std::string_view cAmzSecurityToken{"X-Amz-Security-Token"}; +constexpr std::string_view cXAmzSecurityToken{"X-Amz-Security-Token"}; constexpr std::string_view cXAmzSignature{"X-Amz-Signature"}; constexpr std::string_view cXAmzSignedHeaders{"X-Amz-SignedHeaders"}; diff --git a/components/core/src/clp_s/ReaderUtils.cpp b/components/core/src/clp_s/ReaderUtils.cpp index 2a149bb6fa..b0f3e31820 100644 --- a/components/core/src/clp_s/ReaderUtils.cpp +++ b/components/core/src/clp_s/ReaderUtils.cpp @@ -3,12 +3,10 @@ #include #include #include -#include #include #include "../clp/aws/AwsAuthenticationSigner.hpp" -#include "../clp/CurlDownloadHandler.hpp" #include "../clp/FileReader.hpp" #include "../clp/NetworkReader.hpp" #include "../clp/ReaderInterface.hpp" From a46fd092b8a62a0385e032224d5fbd867cf900bd Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 27 Jan 2025 17:29:15 +0000 Subject: [PATCH 5/5] Address more review comments --- .../src/clp/aws/AwsAuthenticationSigner.cpp | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp index aab55701d1..b058843b24 100644 --- a/components/core/src/clp/aws/AwsAuthenticationSigner.cpp +++ b/components/core/src/clp/aws/AwsAuthenticationSigner.cpp @@ -248,16 +248,8 @@ auto AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_view timestamp) const -> string { auto const uri = fmt::format("{}/{}", m_access_key_id, scope); - std::string session_token_parameter; - if (m_session_token.has_value()) { - session_token_parameter = fmt::format( - "&{}={}", - cXAmzSecurityToken, - encode_uri(m_session_token.value(), false) - ); - } - return fmt::format( - "{}={}&{}={}&{}={}&{}={}{}&{}={}", + auto canonical_query_string = fmt::format( + "{}={}&{}={}&{}={}&{}={}", cXAmzAlgorithm, cAws4HmacSha256, cXAmzCredential, @@ -265,11 +257,17 @@ AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_vi cXAmzDate, timestamp, cXAmzExpires, - cDefaultExpireTime.count(), - session_token_parameter, - cXAmzSignedHeaders, - cDefaultSignedHeaders + cDefaultExpireTime.count() ); + if (m_session_token.has_value()) { + canonical_query_string.append(fmt::format( + "&{}={}", + cXAmzSecurityToken, + encode_uri(m_session_token.value(), false) + )); + } + canonical_query_string.append(fmt::format("&{}={}", cXAmzSignedHeaders, cDefaultSignedHeaders)); + return canonical_query_string; } auto AwsAuthenticationSigner::get_signing_key(