Crt http integration #2293 rebased

.cspell.json

@@ -57,7 +57,9 @@
         // Dependencies
         "openssl", "libcurl",
         // Aws common runtime libraries
-        "mqtt"
+        "mqtt",
+        // Aws common runtime API
+        "Mtls"
     ],
     "ignoreWords": [
         // variables defined in ConcurrentStreamBuf

CMakeLists.txt

@@ -33,6 +33,7 @@ if (LEGACY_BUILD)
     option(BUILD_SHARED_LIBS "If enabled, all aws sdk libraries will be build as shared objects; otherwise all Aws libraries will be built as static objects" ON)
     option(FORCE_SHARED_CRT "If enabled, will unconditionally link the standard libraries in dynamically, otherwise the standard library will be linked in based on the BUILD_SHARED_LIBS setting" ON)
     option(SIMPLE_INSTALL "If enabled, removes all the additional indirection (platform/cpu/config) in the bin and lib directories on the install step" ON)
+    option(USE_CRT_HTTP_CLIENT "If enabled, the common runtime HTTP client will be used, and the legacy systems such as WinHttp and libcurl will not be built or included" OFF)
     option(NO_HTTP_CLIENT "If enabled, no platform-default http client will be included in the library.  For the library to be used you will need to provide your own platform-specific implementation" OFF)
     option(NO_ENCRYPTION "If enabled, no platform-default encryption will be included in the library.  For the library to be used you will need to provide your own platform-specific implementations" OFF)
     option(USE_IXML_HTTP_REQUEST_2 "If enabled on windows, the com object IXmlHttpRequest2 will be used for the http stack" OFF)

cmake/external_dependencies.cmake

@@ -61,7 +61,7 @@ elseif(ENABLE_INJECTED_ENCRYPTION)
 endif()
 
 # Http client control
-if(NOT NO_HTTP_CLIENT)
+if(NOT NO_HTTP_CLIENT AND NOT USE_CRT_HTTP_CLIENT)
     if(PLATFORM_WINDOWS)
         if(FORCE_CURL)
             set(ENABLE_CURL_CLIENT 1)
@@ -120,6 +120,8 @@ if(NOT NO_HTTP_CLIENT)
     else()
         message(FATAL_ERROR "No http client available for target platform and client injection not enabled (-DNO_HTTP_CLIENT=ON)")
     endif()
+elseif(USE_CRT_HTTP_CLIENT)
+    add_definitions("-DAWS_SDK_USE_CRT_HTTP -DHAVE_H2_CLIENT")
 else()
     message(STATUS "You will need to inject an http client implementation before making any http requests!")
 endif()

src/aws-cpp-sdk-core/CMakeLists.txt

@@ -195,8 +195,12 @@ elseif(ENABLE_WINDOWS_CLIENT)
         unset(CMAKE_REQUIRED_LIBRARIES)
 
     endif()
+elseif(USE_CRT_HTTP_CLIENT)
+    file(GLOB CRT_HTTP_HEADERS "include/aws/core/http/crt/*.h")
+    file(GLOB CRT_HTTP_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/source/http/crt/*.cpp")
 endif()
 
+
 if (PLATFORM_WINDOWS)
     file(GLOB NET_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/source/net/windows/*.cpp")
 elseif(PLATFORM_LINUX OR PLATFORM_APPLE OR PLATFORM_ANDROID)
@@ -227,6 +231,7 @@ file(GLOB AWS_NATIVE_SDK_COMMON_SRC
     ${AWS_CLIENT_SOURCE}
     ${HTTP_STANDARD_SOURCE}
     ${HTTP_CLIENT_SOURCE}
+    ${CRT_HTTP_SOURCE}
     ${CONFIG_SOURCE}
     ${CONFIG_DEFAULTS_SOURCE}
     ${ENDPOINT_SOURCE}
@@ -399,6 +404,8 @@ if(MSVC)
     elseif(ENABLE_WINDOWS_CLIENT)
         source_group("Header Files\\aws\\core\\http\\windows" FILES ${HTTP_WINDOWS_CLIENT_HEADERS})
     endif()
+        source_group("Header Files\\aws\\core\\http\\crt" FILES ${CRT_HTTP_HEADERS})
+
 
     # encryption conditional headers
     if(ENABLE_BCRYPT_ENCRYPTION)
@@ -446,8 +453,11 @@ if(MSVC)
         source_group("Source Files\\http\\curl" FILES ${HTTP_CURL_CLIENT_SOURCE})
     elseif(ENABLE_WINDOWS_CLIENT)
         source_group("Source Files\\http\\windows" FILES ${HTTP_WINDOWS_CLIENT_SOURCE})
+    elseif(USE_CRT_HTTP_CLIENT)
+        source_group("Source Files\\http\\crt" FILES ${CRT_HTTP_SOURCE})
     endif()
 
+
     # encryption conditional source
     if(ENABLE_BCRYPT_ENCRYPTION)
         source_group("Source Files\\utils\\crypto\\bcrypt" FILES ${UTILS_CRYPTO_BCRYPT_SOURCE})
@@ -629,8 +639,11 @@ if(ENABLE_CURL_CLIENT)
     install (FILES ${HTTP_CURL_CLIENT_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/core/http/curl)
 elseif(ENABLE_WINDOWS_CLIENT)
     install (FILES ${HTTP_WINDOWS_CLIENT_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/core/http/windows)
+elseif(USE_CRT_HTTP_CLIENT)
+    install (FILES ${CRT_HTTP_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/core/http/crt)
 endif()
 
+
 # encryption headers
 if(ENABLE_BCRYPT_ENCRYPTION)
   install (FILES ${UTILS_CRYPTO_BCRYPT_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/core/utils/crypto/bcrypt)

src/aws-cpp-sdk-core/include/aws/core/http/HttpClient.h

@@ -67,10 +67,17 @@ namespace Aws
 
             bool ContinueRequest(const Aws::Http::HttpRequest&) const;
 
+            explicit operator bool() const
+            {
+               return !m_bad;
+            }
+
+        protected:
+            bool m_bad;
+
         private:
 
             std::atomic< bool > m_disableRequestProcessing;
-
             std::mutex m_requestProcessingSignalLock;
             std::condition_variable m_requestProcessingSignal;
         };

src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h

@@ -552,7 +552,7 @@ namespace Aws
             {
                 m_requestHash = std::make_pair(algorithmName, hash);
             }
-            const std::pair<Aws::String, std::shared_ptr<Aws::Utils::Crypto::Hash>>& GetRequestHash() { return m_requestHash; }
+            const std::pair<Aws::String, std::shared_ptr<Aws::Utils::Crypto::Hash>>& GetRequestHash() const { return m_requestHash; }
 
             void AddResponseValidationHash(const Aws::String& algorithmName, const std::shared_ptr<Aws::Utils::Crypto::Hash>& hash)
             {

src/aws-cpp-sdk-core/include/aws/core/http/HttpResponse.h

@@ -195,6 +195,14 @@ namespace Aws
              * Adds a header to the http response object.
              */
             virtual void AddHeader(const Aws::String&, const Aws::String&) = 0;
+            /**
+              * Add a header to the http response object, and move the value.
+              * The name can't be moved as it is converted to lower-case.
+              *
+              * It isn't pure virtual for backwards compatiblity reasons, but the StandardHttpResponse used by default in the SDK
+              * implements the move.
+              */
+            virtual void AddHeader(const Aws::String& headerName, Aws::String&& headerValue) { AddHeader(headerName, headerValue); };
             /**
              * Sets the content type header on the http response object.
              */

src/aws-cpp-sdk-core/include/aws/core/http/crt/CRTHttpClient.h

@@ -0,0 +1,75 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/core/Core_EXPORTS.h>
+
+#include <aws/core/http/HttpClient.h>
+#include <aws/core/http/standard/StandardHttpResponse.h>
+#include <aws/core/client/ClientConfiguration.h>
+
+#include <aws/crt/io/TlsOptions.h>
+#include <aws/crt/http/HttpConnection.h>
+
+namespace Aws
+{
+    namespace Crt
+    {
+        namespace Http
+        {
+            class HttpClientConnectionManager;
+            class HttpClientConnectionOptions;
+        }
+
+        namespace Io
+        {
+            class ClientBootstrap;
+        }
+    }
+
+    namespace Client
+    {
+        struct ClientConfiguration;
+    } // namespace Client
+
+    namespace Http
+    {
+        /**
+         *  Common Runtime implementation of AWS SDK for C++ HttpClient interface.
+         */
+        class AWS_CORE_API CRTHttpClient : public HttpClient {
+        public:
+            using Base = HttpClient;
+
+            /**
+             * Initializes the client with relevant parameters from clientConfig.
+             */
+            CRTHttpClient(const Aws::Client::ClientConfiguration& clientConfig, Crt::Io::ClientBootstrap& bootstrap);
+            ~CRTHttpClient() override;
+
+            std::shared_ptr<HttpResponse> MakeRequest(const std::shared_ptr<HttpRequest>& request,
+                Aws::Utils::RateLimits::RateLimiterInterface* readLimiter,
+                Aws::Utils::RateLimits::RateLimiterInterface* writeLimiter) const override;
+
+        private:
+            // Yeah, I know, but someone made MakeRequest() const and didn't think about the fact that
+            // making an HTTP request most certainly mutates state. It was me. I'm the person that did that, and
+            // now we're stuck with it. Thanks me.
+            mutable std::unordered_map<Aws::String, const std::shared_ptr<Crt::Http::HttpClientConnectionManager>> m_connectionPools;
+            mutable std::mutex m_connectionPoolLock;
+
+            Crt::Optional<Crt::Io::TlsContext> m_context;
+            Crt::Optional<Crt::Http::HttpClientConnectionProxyOptions> m_proxyOptions;
+
+            Crt::Io::ClientBootstrap& m_bootstrap;
+            Client::ClientConfiguration m_configuration;
+
+            std::shared_ptr<Crt::Http::HttpClientConnectionManager> GetWithCreateConnectionManagerForRequest(const std::shared_ptr<HttpRequest>& request, const Crt::Http::HttpClientConnectionOptions& connectionOptions) const;
+            Crt::Http::HttpClientConnectionOptions CreateConnectionOptionsForRequest(const std::shared_ptr<HttpRequest>& request) const;
+            void CheckAndInitializeProxySettings(const Aws::Client::ClientConfiguration& clientConfig);
+
+            static Aws::String ResolveConnectionPoolKey(const URI& uri);
+        };
+    }
+}

src/aws-cpp-sdk-core/include/aws/core/http/standard/StandardHttpResponse.h

@@ -37,28 +37,33 @@ namespace Aws
                 /**
                  * Get the headers from this response
                  */
-                HeaderValueCollection GetHeaders() const;
+                HeaderValueCollection GetHeaders() const override;
                 /**
                  * Returns true if the response contains a header by headerName
                  */
-                bool HasHeader(const char* headerName) const;
+                bool HasHeader(const char* headerName) const override;
                 /**
                  * Returns the value for a header at headerName if it exists.
                  */
-                const Aws::String& GetHeader(const Aws::String&) const;
+                const Aws::String& GetHeader(const Aws::String&) const override;
                 /**
                  * Gets the response body of the response.
                  */
-                inline Aws::IOStream& GetResponseBody() const { return bodyStream.GetUnderlyingStream(); }
+                inline Aws::IOStream& GetResponseBody() const override { return bodyStream.GetUnderlyingStream(); }
                 /**
                  * Gives full control of the memory of the ResponseBody over to the caller. At this point, it is the caller's
                  * responsibility to clean up this object.
                  */
-                inline Utils::Stream::ResponseStream&& SwapResponseStreamOwnership() { return std::move(bodyStream); }
+                inline Utils::Stream::ResponseStream&& SwapResponseStreamOwnership() override { return std::move(bodyStream); }
                 /**
                  * Adds a header to the http response object.
                  */
-                void AddHeader(const Aws::String&, const Aws::String&);
+                void AddHeader(const Aws::String&, const Aws::String&) override;
+                /**
+                 * Add a header to the http response object, and move the value.
+                 * The name can't be moved as it is converted to lower-case.
+                 */
+                void AddHeader(const Aws::String& headerName, Aws::String&& headerValue) override;
 
             private:
                 StandardHttpResponse(const StandardHttpResponse&);

src/aws-cpp-sdk-core/source/Aws.cpp

@@ -16,6 +16,10 @@
 #include <aws/core/config/AWSProfileConfigLoader.h>
 #include <aws/core/internal/AWSHttpResourceClient.h>
 
+#if AWS_SDK_USE_CRT_HTTP
+#include <aws/crt/Api.h>
+#endif
+
 namespace Aws
 {
     static const char* ALLOCATION_TAG = "Aws_Init_Cleanup";

src/aws-cpp-sdk-core/source/http/HttpClient.cpp

@@ -10,6 +10,7 @@ using namespace Aws;
 using namespace Aws::Http;
 
 HttpClient::HttpClient() :
+    m_bad(false),
     m_disableRequestProcessing( false ),
     m_requestProcessingSignalLock(),
     m_requestProcessingSignal()

src/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp

@@ -5,6 +5,10 @@
 
 #include <aws/core/http/HttpClientFactory.h>
 
+#if AWS_SDK_USE_CRT_HTTP
+#include <aws/core/http/crt/CRTHttpClient.h>
+#include <aws/core/Globals.h>
+#endif
 #if ENABLE_CURL_CLIENT
 #include <aws/core/http/curl/CurlHttpClient.h>
 #include <signal.h>
@@ -62,10 +66,12 @@ namespace Aws
         {
             std::shared_ptr<HttpClient> CreateHttpClient(const ClientConfiguration& clientConfiguration) const override
             {
+#if AWS_SDK_USE_CRT_HTTP
+                return Aws::MakeShared<CRTHttpClient>(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, clientConfiguration, *GetDefaultClientBootstrap());
                 // Figure out whether the selected option is available but fail gracefully and return a default of some type if not
                 // Windows clients:  Http and Inet are always options, Curl MIGHT be an option if USE_CURL_CLIENT is on, and http is "default"
                 // Other clients: Curl is your default
-#if ENABLE_WINDOWS_CLIENT
+#elif ENABLE_WINDOWS_CLIENT
 #if ENABLE_WINDOWS_IXML_HTTP_REQUEST_2_CLIENT
 #if BYPASS_DEFAULT_PROXY
                 switch (clientConfiguration.httpLibOverride)
@@ -189,7 +195,16 @@ namespace Aws
         std::shared_ptr<HttpClient> CreateHttpClient(const Aws::Client::ClientConfiguration& clientConfiguration)
         {
             assert(GetHttpClientFactory());
-            return GetHttpClientFactory()->CreateHttpClient(clientConfiguration);
+            auto client = GetHttpClientFactory()->CreateHttpClient(clientConfiguration);
+
+            if (!client)
+            {
+                AWS_LOGSTREAM_FATAL(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, "Initializing Http Client failed!");
+                // assert just in case this is a misconfiguration at development time to make the dev's job easier.
+                assert(false && "Http client initialization failed. Some client configuration parameters are probably invalid");
+            }
+
+            return client;
         }
 
         std::shared_ptr<HttpRequest> CreateHttpRequest(const Aws::String& uri, HttpMethod method, const Aws::IOStreamFactory& streamFactory)