From a73b00dc03ca9b3900e5a517a429786274a8fe5b Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 17 Jun 2022 16:56:37 -0700 Subject: [PATCH 001/149] Initial WebSocket implementation --- sdk/core/azure-core/CMakeLists.txt | 14 +- sdk/core/azure-core/inc/azure/core/base64.hpp | 15 + .../inc/azure/core/http/curl_transport.hpp | 7 +- .../websockets/curl_websockets_transport.hpp | 115 +++++ .../azure/core/http/websockets/websockets.hpp | 182 ++++++++ .../http/websockets/websockets_transport.hpp | 153 +++++++ .../core/internal/cryptography/sha_hash.hpp | 51 +++ sdk/core/azure-core/src/base64.cpp | 11 +- .../azure-core/src/cryptography/sha_hash.cpp | 18 + sdk/core/azure-core/src/http/curl/curl.cpp | 33 +- .../src/http/curl/curl_session_private.hpp | 15 +- .../src/http/curl/curl_websockets.cpp | 84 ++++ .../src/http/websockets/websocket_frame.hpp | 296 +++++++++++++ .../src/http/websockets/websockets.cpp | 102 +++++ .../src/http/websockets/websocketsimpl.cpp | 417 ++++++++++++++++++ .../src/http/websockets/websocketsimpl.hpp | 55 +++ sdk/core/azure-core/test/ut/CMakeLists.txt | 2 +- .../azure-core/test/ut/websocket_test.cpp | 278 ++++++++++++ 18 files changed, 1836 insertions(+), 12 deletions(-) create mode 100644 sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp create mode 100644 sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp create mode 100644 sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp create mode 100644 sdk/core/azure-core/src/http/curl/curl_websockets.cpp create mode 100644 sdk/core/azure-core/src/http/websockets/websocket_frame.hpp create mode 100644 sdk/core/azure-core/src/http/websockets/websockets.cpp create mode 100644 sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp create mode 100644 sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp create mode 100644 sdk/core/azure-core/test/ut/websocket_test.cpp diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index 931dace33a..8b5eb5dd54 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -42,9 +42,11 @@ if(BUILD_TRANSPORT_CURL) src/http/curl/curl_connection_pool_private.hpp src/http/curl/curl_connection_private.hpp src/http/curl/curl_session_private.hpp - ) + src/http/curl/curl_websockets.cpp + ) SET(CURL_TRANSPORT_ADAPTER_INC inc/azure/core/http/curl_transport.hpp + inc/azure/core/http/websockets/curl_websockets_transport.hpp ) endif() if(BUILD_TRANSPORT_WINHTTP) @@ -74,6 +76,8 @@ set( inc/azure/core/http/policies/policy.hpp inc/azure/core/http/raw_response.hpp inc/azure/core/http/transport.hpp + inc/azure/core/http/websockets/websockets.hpp + inc/azure/core/http/websockets/websockets_transport.hpp inc/azure/core/internal/client_options.hpp inc/azure/core/internal/contract.hpp inc/azure/core/internal/cryptography/sha_hash.hpp @@ -82,13 +86,13 @@ set( inc/azure/core/internal/extendable_enumeration.hpp inc/azure/core/internal/http/pipeline.hpp inc/azure/core/internal/http/user_agent.hpp + inc/azure/core/internal/input_sanitizer.hpp inc/azure/core/internal/io/null_body_stream.hpp inc/azure/core/internal/json/json.hpp inc/azure/core/internal/json/json_optional.hpp inc/azure/core/internal/json/json_serializable.hpp inc/azure/core/internal/strings.hpp inc/azure/core/internal/tracing/service_tracing.hpp - inc/azure/core/internal/input_sanitizer.hpp inc/azure/core/io/body_stream.hpp inc/azure/core/match_conditions.hpp inc/azure/core/modified_conditions.hpp @@ -130,13 +134,15 @@ set( src/http/transport_policy.cpp src/http/url.cpp src/http/user_agent.cpp - src/io/body_stream.cpp + src/http/websockets/websockets.cpp + src/http/websockets/websocketsimpl.cpp + src/io/body_stream.cpp src/io/random_access_file_body_stream.cpp src/logger.cpp src/operation_status.cpp src/private/environment_log_level_listener.hpp - src/private/package_version.hpp src/private/input_sanitizer.cpp + src/private/package_version.hpp src/strings.cpp src/tracing/tracing.cpp src/uuid.cpp diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index 689736ab53..a16678421c 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace Azure { namespace Core { @@ -38,6 +39,20 @@ namespace Azure { namespace Core { */ static std::string Base64Encode(const std::vector& data); + /** + * @brief Encodes the vector of binary data into UTF-8 encoded text represented as Base64. + * + * @param data The binary data to be encoded. + * @param length The length of the binaryData parameter. + * @return The UTF-8 encoded text in Base64. + */ + static std::string Base64Encode(uint8_t const* const data, size_t length); + + template static std::string Base64Encode(std::array const& data) + { + return Base64Encode(data.data(), data.size()); + } + /** * @brief Decodes the UTF-8 encoded text represented as Base64 into binary data. * diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index a06fac3499..a6f1c795bb 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -13,6 +13,7 @@ #include "azure/core/http/transport.hpp" namespace Azure { namespace Core { namespace Http { + class CurlNetworkConnection; namespace _detail { /** @@ -123,13 +124,17 @@ namespace Azure { namespace Core { namespace Http { std::chrono::milliseconds ConnectionTimeout = _detail::DefaultConnectionTimeout; }; + /** * @brief Concrete implementation of an HTTP Transport that uses libcurl. */ - class CurlTransport final : public HttpTransport { + class CurlTransport : public HttpTransport { private: CurlTransportOptions m_options; + protected: + virtual void OnUpgradedConnection(std::unique_ptr&){}; + public: /** * @brief Construct a new CurlTransport object. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp new file mode 100644 index 0000000000..c93f511640 --- /dev/null +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief #Azure::Core::Http::HttpTransport implementation via CURL. + */ + +#pragma once + +#include "azure/core/context.hpp" +#include "azure/core/http/curl_transport.hpp" +#include "azure/core/http/http.hpp" +#include "azure/core/http/transport.hpp" +#include "azure/core/http/websockets/websockets_transport.hpp" +#include + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + /** + * @brief Concrete implementation of a WebSocket Transport that uses libcurl. + */ + class CurlWebSocketTransport : public WebSocketTransport, public CurlTransport { + // std::unique_ptr cannot be constructed on an incomplete type (CurlNetworkConnection), but + // std::shared_ptr can be. + std::shared_ptr m_upgradedConnection; + void OnUpgradedConnection( + std::unique_ptr& upgradedConnection) override; + + public: + /** + * @brief Construct a new CurlTransport object. + * + * @param options Optional parameter to override the default options. + */ + CurlWebSocketTransport(CurlTransportOptions const& options = CurlTransportOptions()) + : CurlTransport(options) + { + } + /** + * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse + * + * @param request an HTTP Request to be send. + * @param context A context to control the request lifetime. + * + * @return unique ptr to an HTTP RawResponse. + */ + virtual std::unique_ptr Send(Request& request, Context const& context) override; + + /** + * @brief Indicates if the transports supports native websockets or not. + * + * @detail For the CURL websocket transport, the transport does NOT support native websockets - + * it is the responsibility of the client of the WebSocketTransport to format WebSocket protocol + * elements. + */ + virtual bool NativeWebsocketSupport() override { return false; } + + virtual void CompleteUpgrade() override; + + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + */ + virtual void Close() override; + + // Native WebSocket support methods. + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + * @param status Status value to be sent to the remote node. Application defined. + * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. + * @param context Context for the operation. + * + */ + virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override + { + throw std::runtime_error("Not implemented."); + } + + /** + * @brief Send a frame of data to the remote node. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + * @brief frameType Frame type sent to the server, Text or Binary. + * @brief frameData Frame data to be sent to the server. + */ + virtual void SendFrame(WebSocketFrameType, std::vector, Azure::Core::Context const&) + override + { + throw std::runtime_error("Not implemented."); + } + + // Non-Native WebSocket support. + /** + * @brief This function is used when working with streams to pull more data from the wire. + * Function will try to keep pulling data from socket until the buffer is all written or until + * there is no more data to get from the socket. + * + */ + virtual size_t ReadFromSocket(uint8_t* buffer, size_t bufferSize, Context const& context) + override; + + /** + * @brief This method will use libcurl socket to write all the bytes from buffer. + * + */ + virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) + override; + }; + +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp new file mode 100644 index 0000000000..f0a57c5246 --- /dev/null +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief Utilities to be used by HTTP transport implementations. + */ + +#pragma once + +#include "azure/core/context.hpp" +#include "azure/core/http/http.hpp" +#include "azure/core/http/transport.hpp" +#include "azure/core/internal/client_options.hpp" +#include +#include + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + namespace _detail { + class WebSocketImplementation; + } + + enum class WebSocketResultType : int + { + Unknown, + TextFrameReceived, + BinaryFrameReceived, + ContinuationReceived, + PeerClosed, + }; + + enum class WebSocketErrorCode : uint16_t + { + OK = 1000, + EndpointDisappearing = 1001, + ProtocolError = 1002, + UnknownDataType = 1003, + Reserved1 = 1004, + NoStatusCodePresent = 1005, + ConnectionClosedWithoutCloseFrame = 1006, + InvalidMessageData = 1007, + PolicyViolation = 1008, + MessageTooLarge = 1009, + ExtensionNotFound = 1010, + UnexpectedError = 1011, + TlsHandshakeFailure = 1015, + }; + + class WebSocketTextFrame; + class WebSocketBinaryFrame; + class WebSocketContinuationFrame; + class WebSocketPeerCloseFrame; + + struct WebSocketResult + { + WebSocketResultType ResultType; + std::shared_ptr AsTextFrame(); + std::shared_ptr AsBinaryFrame(); + std::shared_ptr AsPeerCloseFrame(); + std::shared_ptr AsContinuationFrame(); + }; + + class WebSocketTextFrame : public WebSocketResult, + public std::enable_shared_from_this { + private: + public: + WebSocketTextFrame() = default; + WebSocketTextFrame(bool isFinalFrame, unsigned char const* body, size_t size) + : WebSocketResult{WebSocketResultType::TextFrameReceived}, Text(body, body + size), + IsFinalFrame(isFinalFrame) + { + } + std::string Text; + bool IsFinalFrame; + }; + class WebSocketBinaryFrame : public WebSocketResult, + public std::enable_shared_from_this { + private: + public: + WebSocketBinaryFrame() = default; + WebSocketBinaryFrame(bool isFinal, unsigned char const* body, size_t size) + : WebSocketResult{WebSocketResultType::BinaryFrameReceived}, Data(body, body + size), + IsFinalFrame(isFinal) + { + } + std::vector Data; + bool IsFinalFrame; + }; + + class WebSocketContinuationFrame : public WebSocketResult, + public std::enable_shared_from_this { + public: + WebSocketContinuationFrame() = default; + WebSocketContinuationFrame(bool isFinal, unsigned char const* body, size_t size) + : WebSocketResult{WebSocketResultType::ContinuationReceived}, + ContinuationData(body, body + size), IsFinalFrame(isFinal) + { + } + std::vector ContinuationData; + bool IsFinalFrame; + }; + + class WebSocketPeerCloseFrame : public WebSocketResult, + public std::enable_shared_from_this { + std::vector frameData_; + + public: + WebSocketPeerCloseFrame() = default; + WebSocketPeerCloseFrame( + uint16_t remoteStatusCode, + unsigned char const* closeData, + size_t closeSize) + : WebSocketResult{WebSocketResultType::PeerClosed}, RemoteStatusCode(remoteStatusCode), + frameData_(closeData, closeData + closeSize), BodyStream(frameData_) + { + } + uint16_t RemoteStatusCode; + Azure::Core::IO::MemoryBodyStream BodyStream; + }; + + struct WebSocketOptions : Azure::Core::_internal::ClientOptions + { + /** + * @brief Enable masking for this WebSocket. + * + * @detail Masking is needed to block [certain infrastructure + * attacks](https://www.rfc-editor.org/rfc/rfc6455.html#section-10.3) and is strongly + * recommended. + */ + bool EnableMasking{true}; + /** + * @brief The set of protocols which are supported by this client + */ + std::vector Protocols = {}; + std::string ServiceName; + std::string ServiceVersion; + + explicit WebSocketOptions(bool enableMasking, std::vector protocols) + : Azure::Core::_internal::ClientOptions{}, EnableMasking(enableMasking), + Protocols(protocols) + { + } + WebSocketOptions() = default; + }; + + class WebSocket { + public: + explicit WebSocket( + Azure::Core::Url const& remoteUrl, + WebSocketOptions const& options = WebSocketOptions{}); + + ~WebSocket(); + + void Open(Azure::Core::Context const& context = Azure::Core::Context{}); + void Close(Azure::Core::Context const& context = Azure::Core::Context{}); + void Close( + uint16_t closeStatus, + std::string const& closeReason = {}, + Azure::Core::Context const& context = Azure::Core::Context{}); + void SendFrame( + std::string const& textFrame, + bool isFinalFrame, + Azure::Core::Context const& context = Azure::Core::Context{}); + void SendFrame( + std::vector const& binaryFrame, + bool isFinalFrame, + Azure::Core::Context const& context = Azure::Core::Context{}); + + std::shared_ptr ReceiveFrame( + Azure::Core::Context const& context = Azure::Core::Context{}); + + void AddHeader(std::string const& headerName, std::string const& headerValue); + + bool IsOpen(); + + std::string const& GetChosenProtocol() const; + + private: + std::unique_ptr<_detail::WebSocketImplementation> m_socketImplementation; + }; +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp new file mode 100644 index 0000000000..08e9239819 --- /dev/null +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief Utilities to be used by HTTP WebSocket transport implementations. + */ + +#pragma once + +#include "azure/core/context.hpp" +#include "azure/core/http/http.hpp" + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + /** + * @brief Base class for all WebSocket transport implementations. + */ + class WebSocketTransport : public HttpTransport { + public: + /** + * @brief Web Socket Frame type, one of Text or Binary. + */ + enum class WebSocketFrameType + { + /** + * @brief Indicates that the frame is a partial UTF-8 encoded text frame - it is NOT the + * complete frame to be sent to the remote node. + */ + FrameTypeTextFragment, + /** + * @brief Indicates that the frame is either the complete UTF-8 encoded text frame to be sent + * to the remote node or the final frame of a multipart message. + */ + FrameTypeText, + /** + * @brief Indicates that the frame is either the complete binary frame to be sent + * to the remote node or the final frame of a multipart message. + */ + FrameTypeBinary, + /** + * @brief Indicates that the frame is a partial binary frame - it is NOT the + * complete frame to be sent to the remote node. + */ + FrameTypeBinaryFragment, + }; + /** + * @brief Destructs `%HttpTransport`. + * + */ + virtual ~WebSocketTransport() {} + + /** + * @brief Determines if the transport natively supports WebSockets or not. + * + * @returns true iff the transport has native websocket support, false otherwise. + */ + virtual bool NativeWebsocketSupport() = 0; + /** + * @brief Complete the WebSocket upgrade. + * + * @detail Called by the WebSocket client after the HTTP server responds with a + * SwitchingProtocols response. This method performs whatever operations are needed to + * transfer the protocol from HTTP to WebSockets. + */ + virtual void CompleteUpgrade() = 0; + + /**************/ + /* Native WebSocket support functions*/ + /**************/ + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + * @param status Status value to be sent to the remote node. Application defined. + * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. + * @param context Context for the operation. + */ + virtual void CloseSocket( + uint16_t status, + std::string const& disconnectReason, + Azure::Core::Context const& context) + = 0; + + /** + * @brief Closes the WebSocket. + * + * Does not notify the remote endpoint that the socket is being closed. + * + */ + virtual void Close() = 0; + + /** + * @brief Send a frame of data to the remote node. + * + * @brief frameType Frame type sent to the server, Text or Binary. + * @brief frameData Frame data to be sent to the server. + */ + virtual void SendFrame( + WebSocketFrameType frameType, + std::vector frameData, + Azure::Core::Context const& context) + = 0; + + /**************/ + /* Non Native WebSocket support functions */ + /**************/ + + /** + * @brief This function is used when working with streams to pull more data from the wire. + * Function will try to keep pulling data from socket until the buffer is all written or until + * there is no more data to get from the socket. + * + */ + virtual size_t ReadFromSocket(uint8_t* buffer, size_t bufferSize, Context const& context) = 0; + + /** + * @brief This method will use the raw socket to write all the bytes from buffer. + * + */ + virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) = 0; + + protected: + /** + * @brief Constructs a default instance of `%HttpTransport`. + * + */ + WebSocketTransport() = default; + + /** + * @brief Constructs `%HttpTransport` by copying another instance of `%HttpTransport`. + * + * @param other An instance to copy. + */ + WebSocketTransport(const WebSocketTransport& other) = default; + + /** + * @brief Constructs `%HttpTransport` by moving another instance of `%HttpTransport`. + * + * @param other An instance to move in. + */ + WebSocketTransport(WebSocketTransport&& other) = default; + + /** + * @brief Assigns `%HttpTransport` to another instance of `%HttpTransport`. + * + * @param other An instance to assign. + * + * @return A reference to this instance. + */ + WebSocketTransport& operator=(const WebSocketTransport& other) = default; + }; + +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index 446243fe69..ad3c028584 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -17,6 +17,57 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal { + /** + * @brief Defines #Sha256Hash. + * + */ + class Sha1Hash final : public Azure::Core::Cryptography::Hash { + public: + /** + * @brief Construct a default instance of #Sha256Hash. + * + */ + Sha1Hash(); + + /** + * @brief Cleanup any state when destroying the instance of #Sha256Hash. + * + */ + ~Sha1Hash() {} + + private: + /** + * @brief Underline implementation based on the OS. + * + */ + std::unique_ptr m_portableImplementation; + + /** + * @brief Computes the hash value of the specified binary input data, including any previously + * appended. + * @param data The pointer to binary data to compute the hash value for. + * @param length The size of the data provided. + * @return The computed SHA256 hash value corresponding to the input provided including any + * previously appended. + */ + std::vector OnFinal(const uint8_t* data, size_t length) override + { + return m_portableImplementation->Final(data, length); + } + + /** + * @brief Used to append partial binary input data to compute the SHA256 hash in a streaming + * fashion. + * @remark Once all the data has been added, call #Final() to get the computed hash value. + * @param data The pointer to the current block of binary data that is used for hash + * calculation. + * @param length The size of the data provided. + */ + void OnAppend(const uint8_t* data, size_t length) override + { + return m_portableImplementation->Append(data, length); + } + }; /** * @brief Defines #Sha256Hash. * diff --git a/sdk/core/azure-core/src/base64.cpp b/sdk/core/azure-core/src/base64.cpp index 867e796bc5..0bfffa4be7 100644 --- a/sdk/core/azure-core/src/base64.cpp +++ b/sdk/core/azure-core/src/base64.cpp @@ -313,10 +313,10 @@ static void Base64WriteIntAsFourBytes(char* destination, int32_t value) destination[0] = static_cast(value & 0xFF); } -std::string Base64Encode(const std::vector& data) +std::string Base64Encode(uint8_t const * const data, size_t length) { size_t sourceIndex = 0; - auto inputSize = data.size(); + auto inputSize = length; auto maxEncodedSize = ((inputSize + 2) / 3) * 4; // Use a string with size to the max possible result std::string encodedResult(maxEncodedSize, '0'); @@ -490,7 +490,12 @@ namespace Azure { namespace Core { std::string Convert::Base64Encode(const std::vector& data) { - return ::Base64Encode(data); + return Base64Encode(data.data(), data.size()); + } + + std::string Convert::Base64Encode(uint8_t const * const data, size_t length) + { + return ::Base64Encode(data, length); } std::vector Convert::Base64Decode(const std::string& text) diff --git a/sdk/core/azure-core/src/cryptography/sha_hash.cpp b/sdk/core/azure-core/src/cryptography/sha_hash.cpp index 3ef52b4fb9..81925ee619 100644 --- a/sdk/core/azure-core/src/cryptography/sha_hash.cpp +++ b/sdk/core/azure-core/src/cryptography/sha_hash.cpp @@ -26,6 +26,7 @@ namespace { enum class SHASize { + SHA1, SHA256, SHA384, SHA512 @@ -65,6 +66,13 @@ class SHAWithOpenSSL final : public Azure::Core::Cryptography::Hash { } switch (size) { + case SHASize::SHA1: { + if (1 != EVP_DigestInit_ex(m_context, EVP_sha1(), NULL)) + { + throw std::runtime_error("Crypto error while init Sha256Hash."); + } + break; + } case SHASize::SHA256: { if (1 != EVP_DigestInit_ex(m_context, EVP_sha256(), NULL)) { @@ -97,6 +105,11 @@ class SHAWithOpenSSL final : public Azure::Core::Cryptography::Hash { } // namespace +Azure::Core::Cryptography::_internal::Sha1Hash::Sha1Hash() + : m_portableImplementation(std::make_unique(SHASize::SHA1)) +{ +} + Azure::Core::Cryptography::_internal::Sha256Hash::Sha256Hash() : m_portableImplementation(std::make_unique(SHASize::SHA256)) { @@ -222,6 +235,11 @@ class SHAWithBCrypt final : public Azure::Core::Cryptography::Hash { } // namespace +Azure::Core::Cryptography::_internal::Sha1Hash::Sha1Hash() + : m_portableImplementation(std::make_unique(BCRYPT_SHA1_ALGORITHM)) +{ +} + Azure::Core::Cryptography::_internal::Sha256Hash::Sha256Hash() : m_portableImplementation(std::make_unique(BCRYPT_SHA256_ALGORITHM)) { diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 493f77e677..1af544a85d 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -321,6 +321,14 @@ std::unique_ptr CurlTransport::Send(Request& request, Context const throw Azure::Core::Http::TransportException( "Error while sending request. " + std::string(curl_easy_strerror(performing))); } + else + { + std::unique_ptr upgradedConnection(session->GetUpgradedConnection()); + if (upgradedConnection) + { + OnUpgradedConnection(upgradedConnection); + } + } Log::Write( Logger::Level::Verbose, @@ -421,6 +429,18 @@ CURLcode CurlSession::Perform(Context const& context) return result; } +std::unique_ptr&& CurlSession::GetUpgradedConnection() +{ + if (m_connectionUpgraded) + { + return std::move(m_connection); + } + else + { + return std::move(std::unique_ptr()); + } +} + // Creates an HTTP Response with specific bodyType static std::unique_ptr CreateHTTPResponse( uint8_t const* const begin, @@ -719,11 +739,19 @@ void CurlSession::ReadStatusLineAndHeadersFromRawResponse( auto connectionHeader = headers.find("connection"); if (connectionHeader != headers.end()) { - if (connectionHeader->second == "close") + if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + connectionHeader->second, "close")) { // Use connection shut-down so it won't be moved it back to the connection pool. m_connection->Shutdown(); } + // If the server indicated that the connection header is "upgrade", it means that this + // is a WebSocket connection so the caller will may be upgrading the connection. + if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + connectionHeader->second, "upgrade")) + { + m_connectionUpgraded = true; + } } auto isContentLengthHeaderInResponse = headers.find("content-length"); @@ -1294,7 +1322,8 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo return connection; } } - lock.unlock(); + lock.unlock(); // Why is this line here? std::unique_lock releases the lock when it leaves + // scope. } // Creating a new connection is thread safe. No need to lock mutex here. diff --git a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp index 3e39a2c739..55dafad895 100644 --- a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp @@ -274,6 +274,12 @@ namespace Azure { namespace Core { namespace Http { size_t m_sessionTotalRead = 0; + /** + * @brief If True, the connection is going to be "upgraded" into a websocket connection, so block + * moving the connection to the pool. + */ + bool m_connectionUpgraded = false; + /** * @brief Internal buffer from a session used to read bytes from a socket. This buffer is only * used while constructing an HTTP RawResponse without adding a body to it. Customers would @@ -388,7 +394,7 @@ namespace Azure { namespace Core { namespace Http { // By not moving the connection back to the pool, it gets destroyed calling the connection // destructor to clean libcurl handle and close the connection. // IsEOF will also handle a connection that fail to complete an upload request. - if (IsEOF() && m_keepAlive) + if (IsEOF() && m_keepAlive && !m_connectionUpgraded) { _detail::CurlConnectionPool::g_curlConnectionPool.MoveConnectionBackToPool( std::move(m_connection), m_lastStatusCode); @@ -418,6 +424,13 @@ namespace Azure { namespace Core { namespace Http { * @return The size of the payload. */ int64_t Length() const override { return m_contentLength; } + + /** + * @brief Return the network connection if the server indicated that the connection is upgraded. + * + * @return The network connection, or null if the connection was not upgraded. + */ + std::unique_ptr &&GetUpgradedConnection(); }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp new file mode 100644 index 0000000000..f1f1fb931a --- /dev/null +++ b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/core/http/http.hpp" +#include "azure/core/http/policies/policy.hpp" +#include "azure/core/http/transport.hpp" +#include "azure/core/http/websockets/curl_websockets_transport.hpp" +#include "azure/core/internal/diagnostics/log.hpp" +#include "azure/core/platform.hpp" + +// Private include +#include "curl_connection_private.hpp" + +#if defined(AZ_PLATFORM_POSIX) +#include // for poll() +#include // for socket shutdown +#elif defined(AZ_PLATFORM_WINDOWS) +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif +#if !defined(NOMINMAX) +#define NOMINMAX +#endif +#include +#include // for WSAPoll(); +#endif + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + void CurlWebSocketTransport::CompleteUpgrade() {} + + void CurlWebSocketTransport::Close() { m_upgradedConnection->Shutdown(); } + + // Send an HTTP request to the remote server. + std::unique_ptr CurlWebSocketTransport::Send( + Request& request, + Context const& context) + { + // CURL doesn't understand the ws and wss protocols, so change the URL to be http based. + std::string requestScheme(request.GetUrl().GetScheme()); + if (requestScheme == "wss" || requestScheme == "ws") + { + if (requestScheme == "wss") + { + request.GetUrl().SetScheme("https"); + } + else + { + request.GetUrl().SetScheme("http"); + } + } + return CurlTransport::Send(request, context); + } + + size_t CurlWebSocketTransport::ReadFromSocket( + uint8_t* buffer, + size_t bufferSize, + Context const& context) + { + return m_upgradedConnection->ReadFromSocket(buffer, bufferSize, context); + } + + /** + * @brief This method will use libcurl socket to write all the bytes from buffer. + * + */ + int CurlWebSocketTransport::SendBuffer( + uint8_t const* buffer, + size_t bufferSize, + Context const& context) + { + return m_upgradedConnection->SendBuffer(buffer, bufferSize, context); + } + + void CurlWebSocketTransport::OnUpgradedConnection( + std::unique_ptr& upgradedConnection) + { + // Note that m_upgradedConnection is a std::shared_ptr. We define it as a std::shared_ptr + // because a std::shared_ptr can be declared on an incomplete type, while a std::unique_ptr + // cannot. + m_upgradedConnection = std::move(upgradedConnection); + } + +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp b/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp new file mode 100644 index 0000000000..abd5aacd02 --- /dev/null +++ b/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "azure/core/http/websockets/websockets.hpp" +#include + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + // WebSocket opcodes. + enum class SocketOpcode : uint8_t + { + Continuation = 0x00, + TextFrame = 0x01, + BinaryFrame = 0x02, + Close = 0x08, + Ping = 0x09, + Pong = 0x0a + }; + + class WebSocketFrameEncoder { + + public: + /** + * @brief Encode a websocket frame according to RFC 6455 section 5.2. + * + * This wire format for the data transfer part is described by the ABNF + * [RFC5234] given in detail in this section. (Note that, unlike in + * other sections of this document, the ABNF in this section is + * operating on groups of bits. The length of each group of bits is + * indicated in a comment. When encoded on the wire, the most + * significant bit is the leftmost in the ABNF). A high-level overview + * of the framing is given in the following figure. In a case of + * conflict between the figure below and the ABNF specified later in + * this section, the figure is authoritative. + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-------+-+-------------+-------------------------------+ + * |F|R|R|R| opcode|M| Payload len | Extended payload length | + * |I|S|S|S| (4) |A| (7) | (16/64) | + * |N|V|V|V| |S| | (if payload len==126/127) | + * | |1|2|3| |K| | | + * +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + + * | Extended payload length continued, if payload len == 127 | + * + - - - - - - - - - - - - - - - +-------------------------------+ + * | |Masking-key, if MASK set to 1 | + * +-------------------------------+-------------------------------+ + * | Masking-key (continued) | Payload Data | + * +-------------------------------- - - - - - - - - - - - - - - - + + * : Payload Data continued ... : + * + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + * | Payload Data continued ... | + * +---------------------------------------------------------------+ + * + * FIN: 1 bit + * + * Indicates that this is the final fragment in a message. The first + * fragment MAY also be the final fragment. + * + * RSV1, RSV2, RSV3: 1 bit each + * + * MUST be 0 unless an extension is negotiated that defines meanings + * for non-zero values. If a nonzero value is received and none of + * the negotiated extensions defines the meaning of such a nonzero + * value, the receiving endpoint MUST _Fail the WebSocket + * Connection_. + * + * Opcode: 4 bits + * + * Defines the interpretation of the "Payload data". If an unknown + * opcode is received, the receiving endpoint MUST _Fail the + * WebSocket Connection_. The following values are defined. + * + * * %x0 denotes a continuation frame + * + * * %x1 denotes a text frame + * + * * %x2 denotes a binary frame + * + * * %x3-7 are reserved for further non-control frames + * + * * %x8 denotes a connection close + * + * * %x9 denotes a ping + * + * * %xA denotes a pong + * + * * %xB-F are reserved for further control frames + * + * Mask: 1 bit + * + * Defines whether the "Payload data" is masked. If set to 1, a + * masking key is present in masking-key, and this is used to unmask + * the "Payload data" as per Section 5.3. All frames sent from + * client to server have this bit set to 1. + * + * Payload length: 7 bits, 7+16 bits, or 7+64 bits + * + * The length of the "Payload data", in bytes: if 0-125, that is the + * payload length. If 126, the following 2 bytes interpreted as a + * 16-bit unsigned integer are the payload length. If 127, the + * following 8 bytes interpreted as a 64-bit unsigned integer (the + * most significant bit MUST be 0) are the payload length. Multibyte + * length quantities are expressed in network byte order. Note that + * in all cases, the minimal number of bytes MUST be used to encode + * the length, for example, the length of a 124-byte-long string + * can't be encoded as the sequence 126, 0, 124. The payload length + * is the length of the "Extension data" + the length of the + * "Application data". The length of the "Extension data" may be + * zero, in which case the payload length is the length of the + * "Application data". + * Masking-key: 0 or 4 bytes + * + * All frames sent from the client to the server are masked by a + * 32-bit value that is contained within the frame. This field is + * present if the mask bit is set to 1 and is absent if the mask bit + * is set to 0. See Section 5.3 for further information on client- + * to-server masking. + * + * Payload data: (x+y) bytes + * + * The "Payload data" is defined as "Extension data" concatenated + * with "Application data". + * + * Extension data: x bytes + * + * The "Extension data" is 0 bytes unless an extension has been + * negotiated. Any extension MUST specify the length of the + * "Extension data", or how that length may be calculated, and how + * the extension use MUST be negotiated during the opening handshake. + * If present, the "Extension data" is included in the total payload + * length. + * + * Application data: y bytes + * + * Arbitrary "Application data", taking up the remainder of the frame + * after any "Extension data". The length of the "Application data" + * is equal to the payload length minus the length of the "Extension + * data". + */ + + static std::vector EncodeFrame( + SocketOpcode opcode, + bool maskOutput, + bool isFinal, + std::vector const& payload) + { + std::vector encodedFrame; + // Add opcode+fin. + encodedFrame.push_back(static_cast(opcode) | (isFinal ? 0x80 : 0)); + uint8_t maskAndLength = 0; + if (maskOutput) + { + maskAndLength |= 0x80; + } + // Payloads smaller than 125 bytes are encoded directly in the maskAndLength field. + uint64_t payloadSize = static_cast(payload.size()); + if (payloadSize <= 125) + { + maskAndLength |= static_cast(payload.size()); + } + else if (payloadSize <= 65535) + { + // Payloads greater than 125 whose size can fit in a 16 bit integer bytes + // are encoded as a 16 bit unsigned integer in network byte order. + maskAndLength |= 126; + } + else + { + // Payloads greater than 65536 have their length are encoded as a 64 bit unsigned integer in + // network byte order. + maskAndLength |= 127; + } + encodedFrame.push_back(maskAndLength); + // Encode a 16 bit length. + if (payloadSize > 125 && payloadSize <= 65535) + { + encodedFrame.push_back(static_cast(payload.size()) >> 8); + encodedFrame.push_back(static_cast(payload.size()) & 0xff); + } + // Encode a 64 bit length. + else if (payloadSize >= 65536) + { + encodedFrame.push_back((payloadSize >> 56) & 0xff); + encodedFrame.push_back((payloadSize >> 48) & 0xff); + encodedFrame.push_back((payloadSize >> 40) & 0xff); + encodedFrame.push_back((payloadSize >> 32) & 0xff); + encodedFrame.push_back((payloadSize >> 24) & 0xff); + encodedFrame.push_back((payloadSize >> 16) & 0xff); + encodedFrame.push_back((payloadSize >> 8) & 0xff); + encodedFrame.push_back(payloadSize & 0xff); + } + // Calculate the masking key. This MUST be 4 bytes of high entropy random numbers used to mask + // the input data. + if (maskOutput) + { + // Start by generating the mask - 4 bytes of random data. + std::random_device randomEngine; + + std::array rv; + std::generate(begin(rv), end(rv), std::ref(randomEngine)); + // Append the mask to the payload. + encodedFrame.insert(encodedFrame.end(), rv.begin(), rv.end()); + size_t index = 0; + for (auto ch : payload) + { + encodedFrame.push_back(ch ^ rv[index % 4]); + index += 1; + } + } + else + { + // Since the payload is unmasked, simply append the payload to the encoded frame. + encodedFrame.insert(encodedFrame.end(), payload.begin(), payload.end()); + } + + return encodedFrame; + } + + /** + * @brief Decode a frame received from the websocket server. + * + * @param paylod Pointer to the payload returned by the service. Note that this may be shorter + * than the full data in the response message. + * @param opcode Opcode returned by the server. + * @param isFinal True if this is the final message. + * @param maskKey On Return, contains the contents of the mask key if the data is masked. + * @returns A pointer to the start of the decoded data. + */ + static uint8_t const* DecodeFrame( + std::vector const& payload, + SocketOpcode& opcode, + uint64_t& payloadLength, + bool& isFinal, + bool& isMasked, + std::array& maskKey) + { + if (payload.empty() || payload.size() <= 2) + { + throw std::runtime_error("Frame buffer is too small."); + } + uint8_t const* payloadByte = payload.data(); + opcode = static_cast(*payloadByte & 0x7f); + isFinal = (*payloadByte & 0x80) != 0; + payloadByte += 1; + isMasked = false; + if (*payloadByte & 0x80) + { + isMasked = true; + } + payloadLength = *payloadByte & 0x7f; + if (payloadLength <= 125) + { + payloadByte += 1; + } + else if (payloadLength == 126) + { + if (payload.size() < 4) + { + throw std::runtime_error("Payload is too small"); + } + payloadLength = 0; + payloadLength |= (static_cast(*payloadByte++) << 8) & 0xff; + payloadLength |= (static_cast(*payloadByte++) & 0xff); + } + else if (payloadLength == 127) + { + if (payload.size() < 10) + { + throw std::runtime_error("Payload is too small"); + } + payloadLength = 0; + payloadLength |= (static_cast(*payloadByte++) << 56) & 0xff00000000000000; + payloadLength |= (static_cast(*payloadByte++) << 48) & 0x00ff000000000000; + payloadLength |= (static_cast(*payloadByte++) << 40) & 0x0000ff0000000000; + payloadLength |= (static_cast(*payloadByte++) << 32) & 0x000000ff00000000; + payloadLength |= (static_cast(*payloadByte++) << 24) & 0x00000000ff000000; + payloadLength |= (static_cast(*payloadByte++) << 16) & 0x0000000000ff0000; + payloadLength |= (static_cast(*payloadByte++) << 8) & 0x000000000000ff00; + payloadLength |= (static_cast(*payloadByte)) & 0x000000000000ff00; + } + else + { + throw std::logic_error("Unexpected payload length."); + } + + if (isMasked) + { + maskKey = {*payloadByte++, *payloadByte++, *payloadByte++, *payloadByte++}; + } + return payloadByte; + } + }; + +}}}}} // namespace Azure::Core::Http::WebSockets::_detail diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp new file mode 100644 index 0000000000..60e738d32f --- /dev/null +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/core/http/websockets/websockets.hpp" +#include "azure/core/context.hpp" +#include "websocketsimpl.hpp" + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + WebSocket::WebSocket(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options) + : m_socketImplementation( + std::make_unique<_detail::WebSocketImplementation>(remoteUrl, options)) + + { + } + WebSocket::~WebSocket() {} + + void WebSocket::Open(Azure::Core::Context const& context) + { + m_socketImplementation->Open(context); + } + void WebSocket::Close(Azure::Core::Context const& context) + { + m_socketImplementation->Close(context); + } + void WebSocket::Close( + uint16_t closeStatus, + std::string const& closeReason, + Azure::Core::Context const& context) + { + m_socketImplementation->Close(closeStatus, closeReason, context); + } + + void WebSocket::SendFrame( + std::string const& textFrame, + bool isFinalFrame, + Azure::Core::Context const& context) + { + m_socketImplementation->SendFrame(textFrame, isFinalFrame, context); + } + + void WebSocket::SendFrame( + std::vector const& binaryFrame, + bool isFinalFrame, + Azure::Core::Context const& context) + { + m_socketImplementation->SendFrame(binaryFrame, isFinalFrame, context); + } + + std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) + { + return m_socketImplementation->ReceiveFrame(context); + } + + void WebSocket::AddHeader(std::string const& headerName, std::string const& headerValue) + { + m_socketImplementation->AddHeader(headerName, headerValue); + } + std::string const& WebSocket::GetChosenProtocol() const + { + return m_socketImplementation->GetChosenProtocol(); + } + + bool WebSocket::IsOpen() { return m_socketImplementation->IsOpen(); + } + + std::shared_ptr WebSocketResult::AsTextFrame() + { + if (ResultType != WebSocketResultType::TextFrameReceived) + { + throw std::logic_error("Cannot cast to TextFrameReceived."); + } + return static_cast(this)->shared_from_this(); + } + + std::shared_ptr WebSocketResult::AsBinaryFrame() + { + if (ResultType != WebSocketResultType::BinaryFrameReceived) + { + throw std::logic_error("Cannot cast to BinaryFrameReceived."); + } + return static_cast(this)->shared_from_this(); + } + + std::shared_ptr WebSocketResult::AsPeerCloseFrame() + { + if (ResultType != WebSocketResultType::PeerClosed) + { + throw std::logic_error("Cannot cast to PeerClose."); + } + return static_cast(this)->shared_from_this(); + } + + std::shared_ptr WebSocketResult::AsContinuationFrame() + { + if (ResultType != WebSocketResultType::ContinuationReceived) + { + throw std::logic_error("Cannot cast to ContinuationReceived."); + } + return static_cast(this)->shared_from_this(); + } + +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp new file mode 100644 index 0000000000..6fcfcca41c --- /dev/null +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -0,0 +1,417 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT +#include "websocketsimpl.hpp" +#include "azure/core/base64.hpp" +#include "azure/core/http/policies/policy.hpp" +#include "azure/core/internal/cryptography/sha_hash.hpp" +#if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) +#include "azure/core/http/websockets/win_http_websockets_transport.hpp" +#elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) +#include "azure/core/http/websockets/curl_websockets_transport.hpp" +#endif +#include "websocket_frame.hpp" + +#include +#include + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + + WebSocketImplementation::WebSocketImplementation( + Azure::Core::Url const& remoteUrl, + WebSocketOptions const& options) + : m_remoteUrl(remoteUrl), m_options(options) + { + } + + void WebSocketImplementation::Open(Azure::Core::Context const& context) + { + if (m_state != SocketState::Invalid && m_state != SocketState::Closed) + { + throw std::runtime_error("Socket is not closed."); + } + m_state = SocketState::Opening; + +#if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) + auto winHttpTransport + = std::make_shared(); + m_options.Transport.Transport = winHttpTransport; +#elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) + CurlTransportOptions transportOptions; + transportOptions.HttpKeepAlive = false; + m_transport + = std::make_shared(transportOptions); + m_options.Transport.Transport = m_transport; +#endif + + std::vector> perCallPolicies{}; + std::vector> perRetryPolicies{}; + // If the caller has told us a service name, add the telemetry policy to the pipeline to add a + // user agent header to the request. + if (!m_options.ServiceName.empty()) + { + perCallPolicies.push_back( + std::make_unique( + m_options.ServiceName, m_options.ServiceVersion, m_options.Telemetry)); + } + Azure::Core::Http::_internal::HttpPipeline openPipeline( + m_options, std::move(perRetryPolicies), std::move(perCallPolicies)); + + Azure::Core::Http::Request openSocketRequest( + Azure::Core::Http::HttpMethod::Get, m_remoteUrl, false); + + // Set the standardized WebSocket upgrade headers. + openSocketRequest.SetHeader("Upgrade", "websocket"); + openSocketRequest.SetHeader("Connection", "Upgrade"); + openSocketRequest.SetHeader("Sec-WebSocket-Version", "13"); + // Generate the random request key + auto randomKey = GenerateRandomKey(); + auto encodedKey = Azure::Core::Convert::Base64Encode(randomKey); + openSocketRequest.SetHeader("Sec-WebSocket-Key", encodedKey); + std::string protocols; + for (auto const& protocol : m_options.Protocols) + { + protocols += protocol; + protocols += ", "; + } + protocols = protocols.substr(0, protocols.size() - 2); + openSocketRequest.SetHeader("Sec-WebSocket-Protocol", protocols); + for (auto const& additionalHeader : m_headers) + { + openSocketRequest.SetHeader(additionalHeader.first, additionalHeader.second); + } + std::string remoteOrigin; + remoteOrigin = m_remoteUrl.GetScheme(); + remoteOrigin += "://"; + remoteOrigin += m_remoteUrl.GetHost(); + openSocketRequest.SetHeader("Origin", remoteOrigin); + + // Send the connect request to the WebSocket server. + auto response = openPipeline.Send(openSocketRequest, context); + + // Ensure that the server thinks we're switching protocols. If it doesn't, + // fail immediately. + if (response->GetStatusCode() != Azure::Core::Http::HttpStatusCode::SwitchingProtocols) + { + throw Azure::Core::Http::TransportException("Unexpected handshake response"); + } + + // Prove that the server received this socket request. + auto& responseHeaders = response->GetHeaders(); + auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); + if (socketAccept != responseHeaders.end()) + { + VerifySocketAccept(encodedKey, socketAccept->second); + } + + // Remember the protocol that the client chose. + auto chosenProtocol = responseHeaders.find("Sec-WebSocket-Protocol"); + if (chosenProtocol != responseHeaders.end()) + { + m_chosenProtocol = chosenProtocol->second; + } + + // Inform the transport that the upgrade is complete and that the WebSockets layer is taking + // over the HTTP connection. + m_transport->CompleteUpgrade(); + m_state = SocketState::Open; + } + + std::string const& WebSocketImplementation::GetChosenProtocol() const + { + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + return m_chosenProtocol; + } + + void WebSocketImplementation::Close(Azure::Core::Context const& context) + { + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + m_state = SocketState::Closing; + if (m_transport->NativeWebsocketSupport()) + { + m_transport->CloseSocket(0, "", context); + } + else + { + // Send a going away message to the server. + uint16_t closeReason = static_cast(WebSocketErrorCode::EndpointDisappearing); + std::vector closePayload; + closePayload.push_back(closeReason >> 8); + closePayload.push_back(closeReason & 0xff); + std::vector closeFrame = WebSocketFrameEncoder::EncodeFrame( + SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); + + auto closeResponse = ReceiveFrame(context); + if (closeResponse->ResultType != WebSocketResultType::PeerClosed) + { + throw std::runtime_error("Unexpected result type received during close()."); + } + } + m_transport->Close(); + m_state = SocketState::Closed; + } + + void WebSocketImplementation::Close( + uint16_t closeStatus, + std::string const& closeReason, + Azure::Core::Context const& context) + { + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + + m_state = SocketState::Closing; + if (m_transport->NativeWebsocketSupport()) + { + m_transport->CloseSocket(closeStatus, closeReason, context); + } + else + { + std::vector closePayload; + closePayload.push_back(closeStatus >> 8); + closePayload.push_back(closeStatus & 0xff); + closePayload.insert(closePayload.end(), closeReason.begin(), closeReason.end()); + + std::vector closeFrame = WebSocketFrameEncoder::EncodeFrame( + SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); + + auto closeResponse = ReceiveFrame(context); + if (closeResponse->ResultType != WebSocketResultType::PeerClosed) + { + throw std::runtime_error("Unexpected result type received during close()."); + } + } + m_transport->Close(); + + m_state = SocketState::Closed; + } + + void WebSocketImplementation::AddHeader(std::string const& header, std::string const& headerValue) + { + m_headers.emplace(std::make_pair(header, headerValue)); + } + void WebSocketImplementation::SendFrame( + std::string const& textFrame, + bool isFinalFrame, + Azure::Core::Context const& context) + { + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + if (m_transport->NativeWebsocketSupport()) + { + throw std::runtime_error("Not implemented"); + } + else + { + std::vector utf8text(textFrame.begin(), textFrame.end()); + std::vector sendFrame = WebSocketFrameEncoder::EncodeFrame( + SocketOpcode::TextFrame, m_options.EnableMasking, isFinalFrame, utf8text); + + m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); + } + } + + void WebSocketImplementation::SendFrame( + std::vector const& binaryFrame, + bool isFinalFrame, + Azure::Core::Context const& context) + { + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + if (m_transport->NativeWebsocketSupport()) + { + throw std::runtime_error("Not implemented"); + } + else + { + std::vector sendFrame = WebSocketFrameEncoder::EncodeFrame( + SocketOpcode::BinaryFrame, m_options.EnableMasking, isFinalFrame, binaryFrame); + + m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); + } + } + + std::shared_ptr WebSocketImplementation::ReceiveFrame( + Azure::Core::Context const& context) + { + if (m_state != SocketState::Open && m_state != SocketState::Closing) + { + throw std::runtime_error("Socket is not open."); + } + if (m_transport->NativeWebsocketSupport()) + { + throw std::runtime_error("Not implemented"); + } + else + { + uint8_t payloadByte; + // Read the first byte from the socket (the opcode and final bit). + auto bytesRead = m_transport->ReadFromSocket(&payloadByte, sizeof(payloadByte), context); + if (bytesRead != sizeof(payloadByte)) + { + throw std::runtime_error("Could not read opcode from socket."); + } + SocketOpcode opcode = static_cast(payloadByte & 0x7f); + bool isFinal = (payloadByte & 0x80) != 0; + + // Read the next byte from the socket (the size + bytesRead = m_transport->ReadFromSocket(&payloadByte, sizeof(payloadByte), context); + if (bytesRead != sizeof(payloadByte)) + { + throw std::runtime_error("Could not read size and mask from socket."); + } + + bool isMasked = false; + if (payloadByte & 0x80) + { + isMasked = true; + } + uint64_t payloadLength = payloadByte & 0x7f; + if (payloadLength == 126) + { + uint8_t shortSize[sizeof(uint16_t)]; + bytesRead = m_transport->ReadFromSocket(shortSize, sizeof(shortSize), context); + if (bytesRead != sizeof(shortSize)) + { + throw std::runtime_error("Could not read short size from socket."); + } + payloadLength = 0; + payloadLength |= (static_cast(shortSize[0]) << 8) & 0xff; + payloadLength |= (static_cast(shortSize[1]) & 0xff); + } + else if (payloadLength == 127) + { + uint8_t int64Size[sizeof(uint64_t)]; + bytesRead = m_transport->ReadFromSocket(int64Size, sizeof(int64Size), context); + if (bytesRead != sizeof(int64Size)) + { + throw std::runtime_error("Could not read short size from socket."); + } + payloadLength = 0; + payloadLength |= (static_cast(int64Size[0]) << 56) & 0xff00000000000000; + payloadLength |= (static_cast(int64Size[1]) << 48) & 0x00ff000000000000; + payloadLength |= (static_cast(int64Size[2]) << 40) & 0x0000ff0000000000; + payloadLength |= (static_cast(int64Size[3]) << 32) & 0x000000ff00000000; + payloadLength |= (static_cast(int64Size[4]) << 24) & 0x00000000ff000000; + payloadLength |= (static_cast(int64Size[5]) << 16) & 0x0000000000ff0000; + payloadLength |= (static_cast(int64Size[6]) << 8) & 0x000000000000ff00; + payloadLength |= (static_cast(int64Size[7])) & 0x00000000000000ff; + } + else if (payloadLength >= 126) + { + throw std::logic_error("Unexpected payload length."); + } + std::array maskKey{}; + if (isMasked) + { + bytesRead = m_transport->ReadFromSocket(maskKey.data(), maskKey.size(), context); + if (bytesRead != sizeof(maskKey)) + { + throw std::runtime_error("Could not read short size from socket."); + } + } + + // Now read the entire buffer from the socket. + std::vector readBuffer(payloadLength); + bytesRead = m_transport->ReadFromSocket(readBuffer.data(), readBuffer.size(), context); + + // If the buffer was masked, unmask the buffer contents. + if (isMasked) + { + int index = 0; + std::transform( + readBuffer.begin(), + readBuffer.end(), + readBuffer.begin(), + [&maskKey, &index](uint8_t val) { + val ^= maskKey[index % 4]; + index += 1; + return val; + }); + } + // At this point, readBuffer contains the actual payload from the service. + { + switch (opcode) + { + case SocketOpcode::BinaryFrame: + return std::make_shared( + isFinal, readBuffer.data(), readBuffer.size()); + case SocketOpcode::TextFrame: { + return std::make_shared( + isFinal, readBuffer.data(), readBuffer.size()); + } + case SocketOpcode::Close: { + + if (readBuffer.size() < 2) + { + throw std::runtime_error("Close response buffer is too short."); + } + uint16_t errorCode = 0; + errorCode |= (readBuffer[0] << 8) & 0xff00; + errorCode |= (readBuffer[1] & 0x00ff); + return std::make_shared( + errorCode, + readBuffer.data() + sizeof(uint16_t), + readBuffer.size() - sizeof(uint16_t)); + } + case SocketOpcode::Ping: + case SocketOpcode::Pong: + __debugbreak(); + break; + case SocketOpcode::Continuation: + return std::make_shared( + isFinal, readBuffer.data(), readBuffer.size()); + default: + throw std::runtime_error("Unknown opcode received."); + } + } + } + + return std::shared_ptr(); + context; + } + + std::array WebSocketImplementation::GenerateRandomKey() + { + std::random_device randomEngine; + + std::array rv; + std::generate(begin(rv), end(rv), std::ref(randomEngine)); + return rv; + } + + // Verify the Sec-WebSocket-Accept header as defined in RFC 6455 Section 1.3, which defines the + // opening handshake used for establishing the WebSocket connection. + std::string acceptHeaderGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + void WebSocketImplementation::VerifySocketAccept( + std::string const& encodedKey, + std::string const& acceptHeader) + { + std::string concatenatedKey(encodedKey); + concatenatedKey += acceptHeaderGuid; + Azure::Core::Cryptography::_internal::Sha1Hash sha1hash; + + sha1hash.Append( + reinterpret_cast(concatenatedKey.data()), concatenatedKey.size()); + auto keyHash = sha1hash.Final(); + std::string encodedHash = Azure::Core::Convert::Base64Encode(keyHash); + if (encodedHash != acceptHeader) + { + throw std::runtime_error( + "Hash returned by WebSocket server does not match expected hash. Aborting"); + } + } + +}}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp new file mode 100644 index 0000000000..e898f10eb1 --- /dev/null +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT +#include "azure/core/http/websockets/websockets.hpp" +#include "azure/core/http/websockets/websockets_transport.hpp" +#include "azure/core/internal/http/pipeline.hpp" + +// Implementation of WebSocket protocol. +namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + + class WebSocketImplementation { + enum class SocketState + { + Invalid, + Closed, + Opening, + Open, + Closing, + }; + + public: + WebSocketImplementation(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options); + + void Open(Azure::Core::Context const& context); + void Close(Azure::Core::Context const& context); + void Close( + uint16_t closeStatus, + std::string const& closeReason, + Azure::Core::Context const& context); + void SendFrame( + std::string const& textFrame, + bool isFinalFrame, + Azure::Core::Context const& context); + void SendFrame( + std::vector const& binaryFrame, + bool isFinalFrame, + Azure::Core::Context const& context); + + std::shared_ptr ReceiveFrame(Azure::Core::Context const& context); + + void AddHeader(std::string const& headerName, std::string const& headerValue); + + std::string const& GetChosenProtocol() const; + bool IsOpen() { return m_state == SocketState::Open; } + + private: + SocketState m_state{SocketState::Invalid}; + std::array GenerateRandomKey(); + void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); + Azure::Core::Url m_remoteUrl; + WebSocketOptions m_options; + std::map m_headers; + std::string m_chosenProtocol; + std::shared_ptr m_transport; + }; +}}}}} // namespace Azure::Core::Http::WebSockets::_detail diff --git a/sdk/core/azure-core/test/ut/CMakeLists.txt b/sdk/core/azure-core/test/ut/CMakeLists.txt index bd75517ce3..2bf3b5ffff 100644 --- a/sdk/core/azure-core/test/ut/CMakeLists.txt +++ b/sdk/core/azure-core/test/ut/CMakeLists.txt @@ -80,7 +80,7 @@ add_executable ( transport_adapter_implementation_test.cpp url_test.cpp uuid_test.cpp -) + "websocket_test.cpp") if (MSVC) # Disable warnings: diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp new file mode 100644 index 0000000000..d444e9fb1a --- /dev/null +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/core/http/websockets/websockets.hpp" +#include "azure/core/internal/json/json.hpp" +#include +#include +#include + +using namespace Azure::Core; +using namespace Azure::Core::Http::WebSockets; +using namespace std::chrono_literals; + +TEST(WebSocketTests, CreateSimpleSocket) +{ + { + WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000")); + defaultSocket.AddHeader("newHeader", "headerValue"); + } +} + +TEST(WebSocketTests, OpenSimpleSocket) +{ + { + WebSocketOptions options; + WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000/openclosetest"), options); + defaultSocket.AddHeader("newHeader", "headerValue"); + + defaultSocket.Open(); + + // Close the socket without notifying the peer. + defaultSocket.Close(); + } +} + +TEST(WebSocketTests, OpenAndCloseSocket) +{ + { + WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000/openclosetest")); + defaultSocket.AddHeader("newHeader", "headerValue"); + + defaultSocket.Open(); + + // Close the socket without notifying the peer. + defaultSocket.Close(4500); + } + + { + WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000/openclosetest")); + defaultSocket.AddHeader("newHeader", "headerValue"); + + defaultSocket.Open(); + + // Close the socket without notifying the peer. + defaultSocket.Close(4500, "This is a good reason."); + + // + // Now re-open the socket - this should work to reset everything. + defaultSocket.Open(); + defaultSocket.Close(); + } +} + +TEST(WebSocketTests, SimpleEcho) +{ + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + testSocket.SendFrame("Test message", true); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); + auto textResult = response->AsTextFrame(); + EXPECT_EQ("Test message", textResult->Text); + + // Close the socket gracefully. + testSocket.Close(); + } + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + std::vector binaryData{1, 2, 3, 4, 5, 6}; + + testSocket.SendFrame(binaryData, true); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto textResult = response->AsBinaryFrame(); + EXPECT_EQ(binaryData, textResult->Data); + + // Close the socket gracefully. + testSocket.Close(); + } +} + +TEST(WebSocketTests, CloseDuringEcho) +{ + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + testSocket.SendFrame("Test message", true); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); + auto textResult = response->AsTextFrame(); + EXPECT_EQ("Test message", textResult->Text); + + // Close the socket gracefully. + testSocket.Close(); + } +} + +// Does not work because curl rejects the wss: scheme. +class LibWebSocketIncrementProtocol { + WebSocketOptions m_options{true, {"dumb-increment-protocol"}}; + WebSocket m_socket; + +public: + LibWebSocketIncrementProtocol() : m_socket{Azure::Core::Url("wss://libwebsockets.org"), m_options} + { + } + + void Open() { m_socket.Open(); } + int GetNextNumber() + { + // Time out in 5 seconds if no activity. + Azure::Core::Context contextWithTimeout + = Azure::Core::Context().WithDeadline(std::chrono::system_clock::now() + 10s); + auto work = m_socket.ReceiveFrame(contextWithTimeout); + if (work->ResultType == WebSocketResultType::TextFrameReceived) + { + auto frame = work->AsTextFrame(); + return std::atoi(frame->Text.c_str()); + } + if (work->ResultType == WebSocketResultType::BinaryFrameReceived) + { + auto frame = work->AsBinaryFrame(); + throw std::runtime_error("Not implemented"); + } + else if (work->ResultType == WebSocketResultType::PeerClosed) + { + GTEST_LOG_(INFO) << "Remote server closed connection." << std::endl; + throw std::runtime_error("Remote server closed connection."); + } + else + { + throw std::runtime_error("Unknown result type"); + } + } + + void Reset() { m_socket.SendFrame("reset\n", true); } + void RequestClose() { m_socket.SendFrame("closeme\n", true); } + void Close() { m_socket.Close(); } + void Close(uint16_t closeCode, std::string const& reasonText = {}) + { + m_socket.Close(closeCode, reasonText); + } + void ConsumeUntilClosed() + { + while (m_socket.IsOpen()) + { + auto work = m_socket.ReceiveFrame(); + if (work->ResultType == WebSocketResultType::PeerClosed) + { + auto peerClose = work->AsPeerCloseFrame(); + GTEST_LOG_(INFO) << "Peer closed. Remote Code: " << peerClose->RemoteStatusCode; + if (peerClose->BodyStream.Length() != 0) + { + auto closeBody = peerClose->BodyStream.ReadToEnd(); + std::string closeText(closeBody.begin(), closeBody.end()); + GTEST_LOG_(INFO) << " Peer Closed Data: " << closeText; + } + GTEST_LOG_(INFO) << std::endl; + return; + } + else if (work->ResultType == WebSocketResultType::TextFrameReceived) + { + auto frame = work->AsTextFrame(); + GTEST_LOG_(INFO) << "Ignoring " << frame->Text << std::endl; + } + } + } +}; + +class LibWebSocketStatus { + +public: + std::string GetLWSStatus() + { + WebSocketOptions options; + + options.ServiceName = "websockettest"; + // Send 3 protocols to LWS. + options.Protocols.push_back("brownCow"); + options.Protocols.push_back("lws-status"); + options.Protocols.push_back("flibbityflobbidy"); + WebSocket serverSocket(Azure::Core::Url("wss://libwebsockets.org"), options); + serverSocket.Open(); + + // The server should have chosen the lws-status protocol since it doesn't understand the other protocols. + EXPECT_EQ("lws-status", serverSocket.GetChosenProtocol()); + auto lwsStatus = serverSocket.ReceiveFrame(); + if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) + { + __debugbreak(); + } + EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); + auto textFrame = lwsStatus->AsTextFrame(); + std::string returnValue = textFrame->Text; + bool isFinalFrame = textFrame->IsFinalFrame; + while (!isFinalFrame) + { + lwsStatus = serverSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::ContinuationReceived, lwsStatus->ResultType); + auto continuation = lwsStatus->AsContinuationFrame(); + returnValue.insert( + returnValue.end(), + continuation->ContinuationData.begin(), + continuation->ContinuationData.end()); + isFinalFrame = continuation->IsFinalFrame; + } + serverSocket.Close(); + return returnValue; + } +}; + +TEST(WebSocketTests, LibWebSocketOrg) +{ + { + LibWebSocketStatus lwsStatus; + auto serverStatus = lwsStatus.GetLWSStatus(); + GTEST_LOG_(INFO) << serverStatus << std::endl; + + Azure::Core::Json::_internal::json status( + Azure::Core::Json::_internal::json::parse(serverStatus)); + EXPECT_TRUE(status["conns"].is_array()); + auto connections = status["conns"].get_ref&>(); + bool foundOurConnection = false; + + // Scan through the list of connections to find a connection from the websockettest. + for (auto connection : connections) + { + EXPECT_TRUE(connection["ua"].is_string()); + auto userAgent = connection["ua"].get(); + if (userAgent.find("websockettest") != std::string::npos) + { + foundOurConnection = true; + break; + } + } + EXPECT_TRUE(foundOurConnection); + } + { + LibWebSocketIncrementProtocol incrementProtocol; + incrementProtocol.Open(); + + // Note that we cannot practically validate the numbers received from the service because + // they may be in flight at the time the "Reset" call is made. + for (auto i = 0; i < 100; i += 1) + { + if (i % 5 == 0) + { + GTEST_LOG_(INFO) << "Reset" << std::endl; + incrementProtocol.Reset(); + } + int number = incrementProtocol.GetNextNumber(); + GTEST_LOG_(INFO) << "Got next number " << number << std::endl; + } + incrementProtocol.RequestClose(); + incrementProtocol.ConsumeUntilClosed(); + } +} From ae9de9e74717d299195cd8ab698e95b4b89e7510 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 20 Jun 2022 16:23:24 -0700 Subject: [PATCH 002/149] Checkpoint --- .../src/http/websockets/websocketsimpl.cpp | 4 + sdk/core/azure-core/test/ut/SocketServer.py | 79 +++++++++++++++++++ .../azure-core/test/ut/websocket_test.cpp | 8 +- 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 sdk/core/azure-core/test/ut/SocketServer.py diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 6fcfcca41c..f2210b6c79 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -259,6 +259,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names uint8_t payloadByte; // Read the first byte from the socket (the opcode and final bit). auto bytesRead = m_transport->ReadFromSocket(&payloadByte, sizeof(payloadByte), context); + if (bytesRead == 0) + { + return nullptr; + } if (bytesRead != sizeof(payloadByte)) { throw std::runtime_error("Could not read opcode from socket."); diff --git a/sdk/core/azure-core/test/ut/SocketServer.py b/sdk/core/azure-core/test/ut/SocketServer.py new file mode 100644 index 0000000000..7f77f3865d --- /dev/null +++ b/sdk/core/azure-core/test/ut/SocketServer.py @@ -0,0 +1,79 @@ +from array import array +import asyncio +from time import sleep + +import websockets + +# create handler for each connection +customPaths = {} + +async def handleControlPath(websocket): + while (1): + data : str = await websocket.recv() + parsedCommand = data.split(' ') + if (parsedCommand[0] == "close"): + print("Closing control channel") + await websocket.send("ok") + break + elif parsedCommand[0] == "newPath": + print("Add path") + newPath = parsedCommand[1] + print(" Add path ", newPath) + customPaths[newPath] = {"path": newPath, "delay": int(parsedCommand[2]) } + await websocket.send("ok") + else: + print("Unknown command, echoing it.") + await websocket.send(data) + websocket + +async def handleCustomPath(websocket, path:dict): + print("Handle custom path", path) + data : str = await websocket.recv() + print("Received ", data) + if ("delay" in path.keys()): + sleep(path["delay"]) + print("Responding") + await websocket.send(data) + await websocket.close() + + + +async def handler(websocket, path : str): + print("Socket handler: ", path) + if (path == '/openclosetest'): + print("Open/Close Test") + try: + data = await websocket.recv() + except websockets.ConnectionClosedOK: + print("Connection closed ok.") + except websockets.ConnectionClosed as ex: + print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") + return + elif (path == '/echotest'): + data = await websocket.recv() + await websocket.send(data) + elif (path == '/closeduringecho'): + data = await websocket.recv() + await websocket.close(1001, 'closed') + elif (path=='control'): + await handleControlPath(websocket) + elif (path in customPaths.keys()): + print("Found path ", path, "in control paths.") + await handleCustomPath(websocket, customPaths[path]) + else: + data = await websocket.recv() + print("Received: ", data) + + reply = f"Data received as: {data}!" + + await websocket.send(reply) + + +async def main(): + print("Starting server") + async with websockets.serve(handler, "localhost", 8000): + await asyncio.Future() # run forever. + +if __name__=="__main__": + asyncio.run(main()) + print("Ending server") diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index d444e9fb1a..e2b10e8929 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -100,16 +100,16 @@ TEST(WebSocketTests, SimpleEcho) TEST(WebSocketTests, CloseDuringEcho) { { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/closeduringecho")); testSocket.Open(); testSocket.SendFrame("Test message", true); auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); - auto textResult = response->AsTextFrame(); - EXPECT_EQ("Test message", textResult->Text); + EXPECT_EQ(WebSocketResultType::PeerClosed, response->ResultType); + auto peerClosed = response->AsPeerCloseFrame(); + EXPECT_EQ(1001, peerClosed->RemoteStatusCode); // Close the socket gracefully. testSocket.Close(); From 3d505a0e95ffd55d207fd7e5a661bf401326b5dc Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 28 Jun 2022 11:14:59 -0700 Subject: [PATCH 003/149] Support multiple frame sizes; preliminary doxygen --- sdk/core/azure-core/inc/azure/core/base64.hpp | 5 +- .../inc/azure/core/http/curl_transport.hpp | 1 - .../azure/core/http/websockets/websockets.hpp | 77 ++++- sdk/core/azure-core/src/base64.cpp | 4 +- sdk/core/azure-core/src/http/curl/curl.cpp | 2 +- .../src/http/curl/curl_session_private.hpp | 20 +- .../src/http/websockets/websocket_frame.hpp | 296 ------------------ .../src/http/websockets/websockets.cpp | 3 +- .../src/http/websockets/websocketsimpl.cpp | 286 +++++++++++------ .../src/http/websockets/websocketsimpl.hpp | 250 ++++++++++++++- .../{SocketServer.py => websocket_server.py} | 29 +- .../azure-core/test/ut/websocket_test.cpp | 55 +++- 12 files changed, 597 insertions(+), 431 deletions(-) delete mode 100644 sdk/core/azure-core/src/http/websockets/websocket_frame.hpp rename sdk/core/azure-core/test/ut/{SocketServer.py => websocket_server.py} (72%) diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index a16678421c..41c8767209 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -10,10 +10,10 @@ #pragma once #include +#include #include #include #include -#include namespace Azure { namespace Core { @@ -48,7 +48,8 @@ namespace Azure { namespace Core { */ static std::string Base64Encode(uint8_t const* const data, size_t length); - template static std::string Base64Encode(std::array const& data) + template + static std::string Base64Encode(std::array const& data) { return Base64Encode(data.data(), data.size()); } diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index a6f1c795bb..fed11827c7 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -124,7 +124,6 @@ namespace Azure { namespace Core { namespace Http { std::chrono::milliseconds ConnectionTimeout = _detail::DefaultConnectionTimeout; }; - /** * @brief Concrete implementation of an HTTP Transport that uses libcurl. */ diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index f0a57c5246..a1bcdb1415 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -88,8 +88,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { bool IsFinalFrame; }; - class WebSocketContinuationFrame : public WebSocketResult, - public std::enable_shared_from_this { + class WebSocketContinuationFrame + : public WebSocketResult, + public std::enable_shared_from_this { public: WebSocketContinuationFrame() = default; WebSocketContinuationFrame(bool isFinal, unsigned char const* body, size_t size) @@ -133,9 +134,24 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief The set of protocols which are supported by this client */ std::vector Protocols = {}; + + /** + * @brief The protocol name of the service client. Used for the User-Agent header + * in the initial WebSocket handshake. + */ std::string ServiceName; + /** + * @brief The version of the service client. Used for the User-Agent header in the + * initial WebSocket handshake + */ std::string ServiceVersion; + /** + * @brief Construct an instance of a WebSocketOptions type. + * + * @param enableMasking If true, enable masking for the websocket + * @param protocols Supported protocols for this websocket client. + */ explicit WebSocketOptions(bool enableMasking, std::vector protocols) : Azure::Core::_internal::ClientOptions{}, EnableMasking(enableMasking), Protocols(protocols) @@ -146,34 +162,91 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocket { public: + /** @brief Constructs a new instance of a WebSocket with the specified WebSocket options. + * + * @param remoteUrl The URL of the remote WebSocket server. + * @param options The options to use for the WebSocket. + */ explicit WebSocket( Azure::Core::Url const& remoteUrl, WebSocketOptions const& options = WebSocketOptions{}); + /** @brief Destroys an instance of a WebSocket. + */ ~WebSocket(); + /** @brief Opens a WebSocket connection to a remote server. + * + * @param context Context for the operation, used for cancellation and timeout. + */ void Open(Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Closes a WebSocket connection to the remote server gracefully. + * + * @param context Context for the operation. + */ void Close(Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Closes a WebSocket connection to the remote server with additional context. + * + * @param closeStatus 16 bit WebSocket error code. + * @param closeReason String describing the reason for closing the socket. + * @param context Context for the operation. + */ void Close( uint16_t closeStatus, std::string const& closeReason = {}, Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Sends a String frame to the remote server. + * + * @param textFrame UTF-8 encoded text to send. + * @param isFinalFrame if True, this is the final frame in a multi-frame message. + * @param context Context for the operation. + */ void SendFrame( std::string const& textFrame, bool isFinalFrame, Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Sends a Binary frame to the remote server. + * + * @param binaryFrame Binary data to send. + * @param isFinalFrame if True, this is the final frame in a multi-frame message. + * @param context Context for the operation. + */ void SendFrame( std::vector const& binaryFrame, bool isFinalFrame, Azure::Core::Context const& context = Azure::Core::Context{}); + /** @brief Receive a frame from the remote server. + * + * @param context Context for the operation. + * + * @returns The received WebSocket frame. + * + */ std::shared_ptr ReceiveFrame( Azure::Core::Context const& context = Azure::Core::Context{}); + /** @brief AddHeader - Adds a header to the initial handshake. + * + * @note This API is ignored after the WebSocket is opened. + * + * @param headerName Name of header to add to the initial handshake request. + * @param headerValue Value of header to add. + */ void AddHeader(std::string const& headerName, std::string const& headerValue); + /** @brief Determine if the WebSocket is open. + * + * @returns true if the WebSocket is open, false otherwise. + */ bool IsOpen(); + /** @brief Returns the protocol chosen by the remote server during the initial handshake + */ std::string const& GetChosenProtocol() const; private: diff --git a/sdk/core/azure-core/src/base64.cpp b/sdk/core/azure-core/src/base64.cpp index 0bfffa4be7..fc29735f22 100644 --- a/sdk/core/azure-core/src/base64.cpp +++ b/sdk/core/azure-core/src/base64.cpp @@ -313,7 +313,7 @@ static void Base64WriteIntAsFourBytes(char* destination, int32_t value) destination[0] = static_cast(value & 0xFF); } -std::string Base64Encode(uint8_t const * const data, size_t length) +std::string Base64Encode(uint8_t const* const data, size_t length) { size_t sourceIndex = 0; auto inputSize = length; @@ -493,7 +493,7 @@ namespace Azure { namespace Core { return Base64Encode(data.data(), data.size()); } - std::string Convert::Base64Encode(uint8_t const * const data, size_t length) + std::string Convert::Base64Encode(uint8_t const* const data, size_t length) { return ::Base64Encode(data, length); } diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 1af544a85d..489c1bb2f0 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -326,7 +326,7 @@ std::unique_ptr CurlTransport::Send(Request& request, Context const std::unique_ptr upgradedConnection(session->GetUpgradedConnection()); if (upgradedConnection) { - OnUpgradedConnection(upgradedConnection); + OnUpgradedConnection(upgradedConnection); } } diff --git a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp index 55dafad895..e5007663ae 100644 --- a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp @@ -274,10 +274,10 @@ namespace Azure { namespace Core { namespace Http { size_t m_sessionTotalRead = 0; - /** - * @brief If True, the connection is going to be "upgraded" into a websocket connection, so block - * moving the connection to the pool. - */ + /** + * @brief If True, the connection is going to be "upgraded" into a websocket connection, so + * block moving the connection to the pool. + */ bool m_connectionUpgraded = false; /** @@ -425,12 +425,12 @@ namespace Azure { namespace Core { namespace Http { */ int64_t Length() const override { return m_contentLength; } - /** - * @brief Return the network connection if the server indicated that the connection is upgraded. - * - * @return The network connection, or null if the connection was not upgraded. - */ - std::unique_ptr &&GetUpgradedConnection(); + /** + * @brief Return the network connection if the server indicated that the connection is upgraded. + * + * @return The network connection, or null if the connection was not upgraded. + */ + std::unique_ptr&& GetUpgradedConnection(); }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp b/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp deleted file mode 100644 index abd5aacd02..0000000000 --- a/sdk/core/azure-core/src/http/websockets/websocket_frame.hpp +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: MIT - -#pragma once - -#include "azure/core/http/websockets/websockets.hpp" -#include - -namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { - // WebSocket opcodes. - enum class SocketOpcode : uint8_t - { - Continuation = 0x00, - TextFrame = 0x01, - BinaryFrame = 0x02, - Close = 0x08, - Ping = 0x09, - Pong = 0x0a - }; - - class WebSocketFrameEncoder { - - public: - /** - * @brief Encode a websocket frame according to RFC 6455 section 5.2. - * - * This wire format for the data transfer part is described by the ABNF - * [RFC5234] given in detail in this section. (Note that, unlike in - * other sections of this document, the ABNF in this section is - * operating on groups of bits. The length of each group of bits is - * indicated in a comment. When encoded on the wire, the most - * significant bit is the leftmost in the ABNF). A high-level overview - * of the framing is given in the following figure. In a case of - * conflict between the figure below and the ABNF specified later in - * this section, the figure is authoritative. - * - * 0 1 2 3 - * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - * +-+-+-+-+-------+-+-------------+-------------------------------+ - * |F|R|R|R| opcode|M| Payload len | Extended payload length | - * |I|S|S|S| (4) |A| (7) | (16/64) | - * |N|V|V|V| |S| | (if payload len==126/127) | - * | |1|2|3| |K| | | - * +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + - * | Extended payload length continued, if payload len == 127 | - * + - - - - - - - - - - - - - - - +-------------------------------+ - * | |Masking-key, if MASK set to 1 | - * +-------------------------------+-------------------------------+ - * | Masking-key (continued) | Payload Data | - * +-------------------------------- - - - - - - - - - - - - - - - + - * : Payload Data continued ... : - * + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - * | Payload Data continued ... | - * +---------------------------------------------------------------+ - * - * FIN: 1 bit - * - * Indicates that this is the final fragment in a message. The first - * fragment MAY also be the final fragment. - * - * RSV1, RSV2, RSV3: 1 bit each - * - * MUST be 0 unless an extension is negotiated that defines meanings - * for non-zero values. If a nonzero value is received and none of - * the negotiated extensions defines the meaning of such a nonzero - * value, the receiving endpoint MUST _Fail the WebSocket - * Connection_. - * - * Opcode: 4 bits - * - * Defines the interpretation of the "Payload data". If an unknown - * opcode is received, the receiving endpoint MUST _Fail the - * WebSocket Connection_. The following values are defined. - * - * * %x0 denotes a continuation frame - * - * * %x1 denotes a text frame - * - * * %x2 denotes a binary frame - * - * * %x3-7 are reserved for further non-control frames - * - * * %x8 denotes a connection close - * - * * %x9 denotes a ping - * - * * %xA denotes a pong - * - * * %xB-F are reserved for further control frames - * - * Mask: 1 bit - * - * Defines whether the "Payload data" is masked. If set to 1, a - * masking key is present in masking-key, and this is used to unmask - * the "Payload data" as per Section 5.3. All frames sent from - * client to server have this bit set to 1. - * - * Payload length: 7 bits, 7+16 bits, or 7+64 bits - * - * The length of the "Payload data", in bytes: if 0-125, that is the - * payload length. If 126, the following 2 bytes interpreted as a - * 16-bit unsigned integer are the payload length. If 127, the - * following 8 bytes interpreted as a 64-bit unsigned integer (the - * most significant bit MUST be 0) are the payload length. Multibyte - * length quantities are expressed in network byte order. Note that - * in all cases, the minimal number of bytes MUST be used to encode - * the length, for example, the length of a 124-byte-long string - * can't be encoded as the sequence 126, 0, 124. The payload length - * is the length of the "Extension data" + the length of the - * "Application data". The length of the "Extension data" may be - * zero, in which case the payload length is the length of the - * "Application data". - * Masking-key: 0 or 4 bytes - * - * All frames sent from the client to the server are masked by a - * 32-bit value that is contained within the frame. This field is - * present if the mask bit is set to 1 and is absent if the mask bit - * is set to 0. See Section 5.3 for further information on client- - * to-server masking. - * - * Payload data: (x+y) bytes - * - * The "Payload data" is defined as "Extension data" concatenated - * with "Application data". - * - * Extension data: x bytes - * - * The "Extension data" is 0 bytes unless an extension has been - * negotiated. Any extension MUST specify the length of the - * "Extension data", or how that length may be calculated, and how - * the extension use MUST be negotiated during the opening handshake. - * If present, the "Extension data" is included in the total payload - * length. - * - * Application data: y bytes - * - * Arbitrary "Application data", taking up the remainder of the frame - * after any "Extension data". The length of the "Application data" - * is equal to the payload length minus the length of the "Extension - * data". - */ - - static std::vector EncodeFrame( - SocketOpcode opcode, - bool maskOutput, - bool isFinal, - std::vector const& payload) - { - std::vector encodedFrame; - // Add opcode+fin. - encodedFrame.push_back(static_cast(opcode) | (isFinal ? 0x80 : 0)); - uint8_t maskAndLength = 0; - if (maskOutput) - { - maskAndLength |= 0x80; - } - // Payloads smaller than 125 bytes are encoded directly in the maskAndLength field. - uint64_t payloadSize = static_cast(payload.size()); - if (payloadSize <= 125) - { - maskAndLength |= static_cast(payload.size()); - } - else if (payloadSize <= 65535) - { - // Payloads greater than 125 whose size can fit in a 16 bit integer bytes - // are encoded as a 16 bit unsigned integer in network byte order. - maskAndLength |= 126; - } - else - { - // Payloads greater than 65536 have their length are encoded as a 64 bit unsigned integer in - // network byte order. - maskAndLength |= 127; - } - encodedFrame.push_back(maskAndLength); - // Encode a 16 bit length. - if (payloadSize > 125 && payloadSize <= 65535) - { - encodedFrame.push_back(static_cast(payload.size()) >> 8); - encodedFrame.push_back(static_cast(payload.size()) & 0xff); - } - // Encode a 64 bit length. - else if (payloadSize >= 65536) - { - encodedFrame.push_back((payloadSize >> 56) & 0xff); - encodedFrame.push_back((payloadSize >> 48) & 0xff); - encodedFrame.push_back((payloadSize >> 40) & 0xff); - encodedFrame.push_back((payloadSize >> 32) & 0xff); - encodedFrame.push_back((payloadSize >> 24) & 0xff); - encodedFrame.push_back((payloadSize >> 16) & 0xff); - encodedFrame.push_back((payloadSize >> 8) & 0xff); - encodedFrame.push_back(payloadSize & 0xff); - } - // Calculate the masking key. This MUST be 4 bytes of high entropy random numbers used to mask - // the input data. - if (maskOutput) - { - // Start by generating the mask - 4 bytes of random data. - std::random_device randomEngine; - - std::array rv; - std::generate(begin(rv), end(rv), std::ref(randomEngine)); - // Append the mask to the payload. - encodedFrame.insert(encodedFrame.end(), rv.begin(), rv.end()); - size_t index = 0; - for (auto ch : payload) - { - encodedFrame.push_back(ch ^ rv[index % 4]); - index += 1; - } - } - else - { - // Since the payload is unmasked, simply append the payload to the encoded frame. - encodedFrame.insert(encodedFrame.end(), payload.begin(), payload.end()); - } - - return encodedFrame; - } - - /** - * @brief Decode a frame received from the websocket server. - * - * @param paylod Pointer to the payload returned by the service. Note that this may be shorter - * than the full data in the response message. - * @param opcode Opcode returned by the server. - * @param isFinal True if this is the final message. - * @param maskKey On Return, contains the contents of the mask key if the data is masked. - * @returns A pointer to the start of the decoded data. - */ - static uint8_t const* DecodeFrame( - std::vector const& payload, - SocketOpcode& opcode, - uint64_t& payloadLength, - bool& isFinal, - bool& isMasked, - std::array& maskKey) - { - if (payload.empty() || payload.size() <= 2) - { - throw std::runtime_error("Frame buffer is too small."); - } - uint8_t const* payloadByte = payload.data(); - opcode = static_cast(*payloadByte & 0x7f); - isFinal = (*payloadByte & 0x80) != 0; - payloadByte += 1; - isMasked = false; - if (*payloadByte & 0x80) - { - isMasked = true; - } - payloadLength = *payloadByte & 0x7f; - if (payloadLength <= 125) - { - payloadByte += 1; - } - else if (payloadLength == 126) - { - if (payload.size() < 4) - { - throw std::runtime_error("Payload is too small"); - } - payloadLength = 0; - payloadLength |= (static_cast(*payloadByte++) << 8) & 0xff; - payloadLength |= (static_cast(*payloadByte++) & 0xff); - } - else if (payloadLength == 127) - { - if (payload.size() < 10) - { - throw std::runtime_error("Payload is too small"); - } - payloadLength = 0; - payloadLength |= (static_cast(*payloadByte++) << 56) & 0xff00000000000000; - payloadLength |= (static_cast(*payloadByte++) << 48) & 0x00ff000000000000; - payloadLength |= (static_cast(*payloadByte++) << 40) & 0x0000ff0000000000; - payloadLength |= (static_cast(*payloadByte++) << 32) & 0x000000ff00000000; - payloadLength |= (static_cast(*payloadByte++) << 24) & 0x00000000ff000000; - payloadLength |= (static_cast(*payloadByte++) << 16) & 0x0000000000ff0000; - payloadLength |= (static_cast(*payloadByte++) << 8) & 0x000000000000ff00; - payloadLength |= (static_cast(*payloadByte)) & 0x000000000000ff00; - } - else - { - throw std::logic_error("Unexpected payload length."); - } - - if (isMasked) - { - maskKey = {*payloadByte++, *payloadByte++, *payloadByte++, *payloadByte++}; - } - return payloadByte; - } - }; - -}}}}} // namespace Azure::Core::Http::WebSockets::_detail diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index 60e738d32f..9f5213dc48 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -60,8 +60,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return m_socketImplementation->GetChosenProtocol(); } - bool WebSocket::IsOpen() { return m_socketImplementation->IsOpen(); - } + bool WebSocket::IsOpen() { return m_socketImplementation->IsOpen(); } std::shared_ptr WebSocketResult::AsTextFrame() { diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index f2210b6c79..c5217fd788 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -9,9 +9,8 @@ #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) #include "azure/core/http/websockets/curl_websockets_transport.hpp" #endif -#include "websocket_frame.hpp" - #include +#include #include namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { @@ -41,6 +40,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_transport = std::make_shared(transportOptions); m_options.Transport.Transport = m_transport; + m_bufferedStreamReader.SetTransport(m_transport); #endif std::vector> perCallPolicies{}; @@ -116,8 +116,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Open; } - std::string const& WebSocketImplementation::GetChosenProtocol() const + std::string const& WebSocketImplementation::GetChosenProtocol() { + std::shared_lock lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -127,6 +128,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names void WebSocketImplementation::Close(Azure::Core::Context const& context) { + std::unique_lock lock(m_stateMutex); + + // If we're closing an already closed socket, we're done. + if (m_state == SocketState::Closed) + { + return; + } if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -143,11 +151,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector closePayload; closePayload.push_back(closeReason >> 8); closePayload.push_back(closeReason & 0xff); - std::vector closeFrame = WebSocketFrameEncoder::EncodeFrame( - SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + std::vector closeFrame + = EncodeFrame(SocketOpcode::Close, m_options.EnableMasking, true, closePayload); m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); - auto closeResponse = ReceiveFrame(context); + auto closeResponse = ReceiveFrame(context, true); if (closeResponse->ResultType != WebSocketResultType::PeerClosed) { throw std::runtime_error("Unexpected result type received during close()."); @@ -162,6 +170,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::string const& closeReason, Azure::Core::Context const& context) { + std::unique_lock lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -179,11 +188,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names closePayload.push_back(closeStatus & 0xff); closePayload.insert(closePayload.end(), closeReason.begin(), closeReason.end()); - std::vector closeFrame = WebSocketFrameEncoder::EncodeFrame( - SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + std::vector closeFrame + = EncodeFrame(SocketOpcode::Close, m_options.EnableMasking, true, closePayload); m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); - auto closeResponse = ReceiveFrame(context); + auto closeResponse = ReceiveFrame(context, true); if (closeResponse->ResultType != WebSocketResultType::PeerClosed) { throw std::runtime_error("Unexpected result type received during close()."); @@ -203,6 +212,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context) { + std::shared_lock lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -214,8 +224,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names else { std::vector utf8text(textFrame.begin(), textFrame.end()); - std::vector sendFrame = WebSocketFrameEncoder::EncodeFrame( - SocketOpcode::TextFrame, m_options.EnableMasking, isFinalFrame, utf8text); + std::vector sendFrame + = EncodeFrame(SocketOpcode::TextFrame, m_options.EnableMasking, isFinalFrame, utf8text); m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); } @@ -226,6 +236,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context) { + std::shared_lock lock(m_stateMutex); + if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -236,16 +248,25 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } else { - std::vector sendFrame = WebSocketFrameEncoder::EncodeFrame( + std::vector sendFrame = EncodeFrame( SocketOpcode::BinaryFrame, m_options.EnableMasking, isFinalFrame, binaryFrame); + std::unique_lock transportLock(m_transportMutex); m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); } } std::shared_ptr WebSocketImplementation::ReceiveFrame( - Azure::Core::Context const& context) + Azure::Core::Context const& context, + bool stateIsLocked) { + std::shared_lock lock(m_stateMutex, std::defer_lock); + + if (!stateIsLocked) + { + lock.lock(); + } + if (m_state != SocketState::Open && m_state != SocketState::Closing) { throw std::runtime_error("Socket is not open."); @@ -256,90 +277,21 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } else { - uint8_t payloadByte; - // Read the first byte from the socket (the opcode and final bit). - auto bytesRead = m_transport->ReadFromSocket(&payloadByte, sizeof(payloadByte), context); - if (bytesRead == 0) - { - return nullptr; - } - if (bytesRead != sizeof(payloadByte)) - { - throw std::runtime_error("Could not read opcode from socket."); - } - SocketOpcode opcode = static_cast(payloadByte & 0x7f); - bool isFinal = (payloadByte & 0x80) != 0; - - // Read the next byte from the socket (the size - bytesRead = m_transport->ReadFromSocket(&payloadByte, sizeof(payloadByte), context); - if (bytesRead != sizeof(payloadByte)) - { - throw std::runtime_error("Could not read size and mask from socket."); - } + std::vector frameData; + SocketOpcode opcode; + bool isFinal = false; bool isMasked = false; - if (payloadByte & 0x80) - { - isMasked = true; - } - uint64_t payloadLength = payloadByte & 0x7f; - if (payloadLength == 126) - { - uint8_t shortSize[sizeof(uint16_t)]; - bytesRead = m_transport->ReadFromSocket(shortSize, sizeof(shortSize), context); - if (bytesRead != sizeof(shortSize)) - { - throw std::runtime_error("Could not read short size from socket."); - } - payloadLength = 0; - payloadLength |= (static_cast(shortSize[0]) << 8) & 0xff; - payloadLength |= (static_cast(shortSize[1]) & 0xff); - } - else if (payloadLength == 127) - { - uint8_t int64Size[sizeof(uint64_t)]; - bytesRead = m_transport->ReadFromSocket(int64Size, sizeof(int64Size), context); - if (bytesRead != sizeof(int64Size)) - { - throw std::runtime_error("Could not read short size from socket."); - } - payloadLength = 0; - payloadLength |= (static_cast(int64Size[0]) << 56) & 0xff00000000000000; - payloadLength |= (static_cast(int64Size[1]) << 48) & 0x00ff000000000000; - payloadLength |= (static_cast(int64Size[2]) << 40) & 0x0000ff0000000000; - payloadLength |= (static_cast(int64Size[3]) << 32) & 0x000000ff00000000; - payloadLength |= (static_cast(int64Size[4]) << 24) & 0x00000000ff000000; - payloadLength |= (static_cast(int64Size[5]) << 16) & 0x0000000000ff0000; - payloadLength |= (static_cast(int64Size[6]) << 8) & 0x000000000000ff00; - payloadLength |= (static_cast(int64Size[7])) & 0x00000000000000ff; - } - else if (payloadLength >= 126) - { - throw std::logic_error("Unexpected payload length."); - } + uint64_t payloadLength; std::array maskKey{}; - if (isMasked) - { - bytesRead = m_transport->ReadFromSocket(maskKey.data(), maskKey.size(), context); - if (bytesRead != sizeof(maskKey)) - { - throw std::runtime_error("Could not read short size from socket."); - } - } - - // Now read the entire buffer from the socket. - std::vector readBuffer(payloadLength); - bytesRead = m_transport->ReadFromSocket(readBuffer.data(), readBuffer.size(), context); + frameData = DecodeFrame( + m_bufferedStreamReader, opcode, payloadLength, isFinal, isMasked, maskKey, context); - // If the buffer was masked, unmask the buffer contents. if (isMasked) { int index = 0; std::transform( - readBuffer.begin(), - readBuffer.end(), - readBuffer.begin(), - [&maskKey, &index](uint8_t val) { + frameData.begin(), frameData.end(), frameData.begin(), [&maskKey, &index](uint8_t val) { val ^= maskKey[index % 4]; index += 1; return val; @@ -351,24 +303,33 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { case SocketOpcode::BinaryFrame: return std::make_shared( - isFinal, readBuffer.data(), readBuffer.size()); + isFinal, frameData.data(), frameData.size()); case SocketOpcode::TextFrame: { return std::make_shared( - isFinal, readBuffer.data(), readBuffer.size()); + isFinal, frameData.data(), frameData.size()); } case SocketOpcode::Close: { - if (readBuffer.size() < 2) + if (frameData.size() < 2) { throw std::runtime_error("Close response buffer is too short."); } uint16_t errorCode = 0; - errorCode |= (readBuffer[0] << 8) & 0xff00; - errorCode |= (readBuffer[1] & 0x00ff); + errorCode |= (frameData[0] << 8) & 0xff00; + errorCode |= (frameData[1] & 0x00ff); + + // Update our state to be closed once we've received a closed frame. We only need to do + // this if our state is not currently locked. + if (!stateIsLocked) + { + lock.unlock(); + std::unique_lock closeLock(m_stateMutex); + m_state = SocketState::Closed; + } return std::make_shared( errorCode, - readBuffer.data() + sizeof(uint16_t), - readBuffer.size() - sizeof(uint16_t)); + frameData.data() + sizeof(uint16_t), + frameData.size() - sizeof(uint16_t)); } case SocketOpcode::Ping: case SocketOpcode::Pong: @@ -376,7 +337,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names break; case SocketOpcode::Continuation: return std::make_shared( - isFinal, readBuffer.data(), readBuffer.size()); + isFinal, frameData.data(), frameData.size()); default: throw std::runtime_error("Unknown opcode received."); } @@ -387,13 +348,134 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names context; } - std::array WebSocketImplementation::GenerateRandomKey() + std::vector WebSocketImplementation::EncodeFrame( + SocketOpcode opcode, + bool maskOutput, + bool isFinal, + std::vector const& payload) + { + std::vector encodedFrame; + // Add opcode+fin. + encodedFrame.push_back(static_cast(opcode) | (isFinal ? 0x80 : 0)); + uint8_t maskAndLength = 0; + if (maskOutput) + { + maskAndLength |= 0x80; + } + // Payloads smaller than 125 bytes are encoded directly in the maskAndLength field. + uint64_t payloadSize = static_cast(payload.size()); + if (payloadSize <= 125) + { + maskAndLength |= static_cast(payload.size()); + } + else if (payloadSize <= 65535) + { + // Payloads greater than 125 whose size can fit in a 16 bit integer bytes + // are encoded as a 16 bit unsigned integer in network byte order. + maskAndLength |= 126; + } + else + { + // Payloads greater than 65536 have their length are encoded as a 64 bit unsigned integer + // in network byte order. + maskAndLength |= 127; + } + encodedFrame.push_back(maskAndLength); + // Encode a 16 bit length. + if (payloadSize > 125 && payloadSize <= 65535) + { + encodedFrame.push_back(static_cast(payload.size()) >> 8); + encodedFrame.push_back(static_cast(payload.size()) & 0xff); + } + // Encode a 64 bit length. + else if (payloadSize >= 65536) + { + + encodedFrame.push_back((payloadSize >> 56) & 0xff); + encodedFrame.push_back((payloadSize >> 48) & 0xff); + encodedFrame.push_back((payloadSize >> 40) & 0xff); + encodedFrame.push_back((payloadSize >> 32) & 0xff); + encodedFrame.push_back((payloadSize >> 24) & 0xff); + encodedFrame.push_back((payloadSize >> 16) & 0xff); + encodedFrame.push_back((payloadSize >> 8) & 0xff); + encodedFrame.push_back(payloadSize & 0xff); + } + // Calculate the masking key. This MUST be 4 bytes of high entropy random numbers used to + // mask the input data. + if (maskOutput) + { + // Start by generating the mask - 4 bytes of random data. + std::array mask = GenerateRandomBytes<4>(); + + // Append the mask to the payload. + encodedFrame.insert(encodedFrame.end(), mask.begin(), mask.end()); + + // And mask the payload before transmitting it. + size_t index = 0; + for (auto ch : payload) + { + encodedFrame.push_back(ch ^ mask[index % 4]); + index += 1; + } + } + else + { + // Since the payload is unmasked, simply append the payload to the encoded frame. + encodedFrame.insert(encodedFrame.end(), payload.begin(), payload.end()); + } + + return encodedFrame; + } + + std::vector WebSocketImplementation::DecodeFrame( + WebSocketImplementation::BufferedStreamReader& streamReader, + SocketOpcode& opcode, + uint64_t& payloadLength, + bool& isFinal, + bool& isMasked, + std::array& maskKey, + Azure::Core::Context const& context) { - std::random_device randomEngine; + std::unique_lock lock(m_transportMutex); + if (streamReader.IsEof()) + { + throw std::runtime_error("Frame buffer is too small."); + } + uint8_t payloadByte = streamReader.ReadByte(context); + opcode = static_cast(payloadByte & 0x7f); + isFinal = (payloadByte & 0x80) != 0; + payloadByte = streamReader.ReadByte(context); + isMasked = false; + if (payloadByte & 0x80) + { + isMasked = true; + } + payloadLength = payloadByte & 0x7f; + if (payloadLength <= 125) + { + payloadByte += 1; + } + else if (payloadLength == 126) + { + payloadLength = streamReader.ReadShort(context); + } + else if (payloadLength == 127) + { + payloadLength = streamReader.ReadInt64(context); + } + else + { + throw std::logic_error("Unexpected payload length."); + } - std::array rv; - std::generate(begin(rv), end(rv), std::ref(randomEngine)); - return rv; + if (isMasked) + { + maskKey[0] = streamReader.ReadByte(context); + maskKey[1] = streamReader.ReadByte(context); + maskKey[2] = streamReader.ReadByte(context); + maskKey[3] = streamReader.ReadByte(context); + }; + return streamReader.ReadBytes(payloadLength, context); } // Verify the Sec-WebSocket-Accept header as defined in RFC 6455 Section 1.3, which defines the diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index e898f10eb1..d25dad2831 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -3,10 +3,23 @@ #include "azure/core/http/websockets/websockets.hpp" #include "azure/core/http/websockets/websockets_transport.hpp" #include "azure/core/internal/http/pipeline.hpp" +#include +#include +#include // Implementation of WebSocket protocol. namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + // Generator for random bytes. Used in WebSocketImplementation and tests. + template std::array GenerateRandomBytes() + { + std::random_device randomEngine; + + std::array rv; + std::generate(begin(rv), end(rv), std::ref(randomEngine)); + return rv; + } + class WebSocketImplementation { enum class SocketState { @@ -35,21 +48,252 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context); - std::shared_ptr ReceiveFrame(Azure::Core::Context const& context); + std::shared_ptr ReceiveFrame( + Azure::Core::Context const& context, + bool stateIsLocked = false); void AddHeader(std::string const& headerName, std::string const& headerValue); - std::string const& GetChosenProtocol() const; + std::string const& GetChosenProtocol(); bool IsOpen() { return m_state == SocketState::Open; } private: + // WebSocket opcodes. + enum class SocketOpcode : uint8_t + { + Continuation = 0x00, + TextFrame = 0x01, + BinaryFrame = 0x02, + Close = 0x08, + Ping = 0x09, + Pong = 0x0a + }; + + // Implement a buffered stream reader + class BufferedStreamReader { + std::shared_ptr m_transport; + constexpr static size_t m_bufferSize = 1024; + uint8_t m_buffer[m_bufferSize]{}; + size_t m_bufferPos = 0; + size_t m_bufferLen = 0; + bool m_eof = false; + + public: + explicit BufferedStreamReader() = default; + ~BufferedStreamReader() = default; + + void SetTransport( + std::shared_ptr& transport) + { + m_transport = transport; + } + + uint8_t ReadByte(Azure::Core::Context const& context) + { + if (m_bufferPos >= m_bufferLen) + { + m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + m_bufferPos = 0; + if (m_bufferLen == 0) + { + m_eof = true; + return 0; + } + } + return m_buffer[m_bufferPos++]; + } + uint16_t ReadShort(Azure::Core::Context const& context) + { + uint16_t result = ReadByte(context); + result <<= 8; + result |= ReadByte(context); + return result; + } + uint64_t ReadInt64(Azure::Core::Context const& context) + { + uint64_t result = 0; + + result |= (static_cast(ReadByte(context)) << 56 & 0xff00000000000000); + result |= (static_cast(ReadByte(context)) << 48 & 0x00ff000000000000); + result |= (static_cast(ReadByte(context)) << 40 & 0x0000ff0000000000); + result |= (static_cast(ReadByte(context)) << 32 & 0x000000ff00000000); + result |= (static_cast(ReadByte(context)) << 24 & 0x00000000ff000000); + result |= (static_cast(ReadByte(context)) << 16 & 0x0000000000ff0000); + result |= (static_cast(ReadByte(context)) << 8 & 0x000000000000ff00); + result |= static_cast(ReadByte(context)); + return result; + } + std::vector ReadBytes(size_t readLength, Azure::Core::Context const& context) + { + std::vector result; + size_t index = 0; + while (index < readLength) + { + uint8_t byte = ReadByte(context); + result.push_back(byte); + index += 1; + } + return result; + } + + bool IsEof() const { return m_eof; } + }; + + /** + * @brief Encode a websocket frame according to RFC 6455 section 5.2. + * + * This wire format for the data transfer part is described by the ABNF + * [RFC5234] given in detail in this section. (Note that, unlike in + * other sections of this document, the ABNF in this section is + * operating on groups of bits. The length of each group of bits is + * indicated in a comment. When encoded on the wire, the most + * significant bit is the leftmost in the ABNF). A high-level overview + * of the framing is given in the following figure. In a case of + * conflict between the figure below and the ABNF specified later in + * this section, the figure is authoritative. + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-------+-+-------------+-------------------------------+ + * |F|R|R|R| opcode|M| Payload len | Extended payload length | + * |I|S|S|S| (4) |A| (7) | (16/64) | + * |N|V|V|V| |S| | (if payload len==126/127) | + * | |1|2|3| |K| | | + * +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + + * | Extended payload length continued, if payload len == 127 | + * + - - - - - - - - - - - - - - - +-------------------------------+ + * | |Masking-key, if MASK set to 1 | + * +-------------------------------+-------------------------------+ + * | Masking-key (continued) | Payload Data | + * +-------------------------------- - - - - - - - - - - - - - - - + + * : Payload Data continued ... : + * + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + * | Payload Data continued ... | + * +---------------------------------------------------------------+ + * + * FIN: 1 bit + * + * Indicates that this is the final fragment in a message. The first + * fragment MAY also be the final fragment. + * + * RSV1, RSV2, RSV3: 1 bit each + * + * MUST be 0 unless an extension is negotiated that defines meanings + * for non-zero values. If a nonzero value is received and none of + * the negotiated extensions defines the meaning of such a nonzero + * value, the receiving endpoint MUST _Fail the WebSocket + * Connection_. + * + * Opcode: 4 bits + * + * Defines the interpretation of the "Payload data". If an unknown + * opcode is received, the receiving endpoint MUST _Fail the + * WebSocket Connection_. The following values are defined. + * + * * %x0 denotes a continuation frame + * + * * %x1 denotes a text frame + * + * * %x2 denotes a binary frame + * + * * %x3-7 are reserved for further non-control frames + * + * * %x8 denotes a connection close + * + * * %x9 denotes a ping + * + * * %xA denotes a pong + * + * * %xB-F are reserved for further control frames + * + * Mask: 1 bit + * + * Defines whether the "Payload data" is masked. If set to 1, a + * masking key is present in masking-key, and this is used to unmask + * the "Payload data" as per Section 5.3. All frames sent from + * client to server have this bit set to 1. + * + * Payload length: 7 bits, 7+16 bits, or 7+64 bits + * + * The length of the "Payload data", in bytes: if 0-125, that is the + * payload length. If 126, the following 2 bytes interpreted as a + * 16-bit unsigned integer are the payload length. If 127, the + * following 8 bytes interpreted as a 64-bit unsigned integer (the + * most significant bit MUST be 0) are the payload length. Multibyte + * length quantities are expressed in network byte order. Note that + * in all cases, the minimal number of bytes MUST be used to encode + * the length, for example, the length of a 124-byte-long string + * can't be encoded as the sequence 126, 0, 124. The payload length + * is the length of the "Extension data" + the length of the + * "Application data". The length of the "Extension data" may be + * zero, in which case the payload length is the length of the + * "Application data". + * Masking-key: 0 or 4 bytes + * + * All frames sent from the client to the server are masked by a + * 32-bit value that is contained within the frame. This field is + * present if the mask bit is set to 1 and is absent if the mask bit + * is set to 0. See Section 5.3 for further information on client- + * to-server masking. + * + * Payload data: (x+y) bytes + * + * The "Payload data" is defined as "Extension data" concatenated + * with "Application data". + * + * Extension data: x bytes + * + * The "Extension data" is 0 bytes unless an extension has been + * negotiated. Any extension MUST specify the length of the + * "Extension data", or how that length may be calculated, and how + * the extension use MUST be negotiated during the opening handshake. + * If present, the "Extension data" is included in the total payload + * length. + * + * Application data: y bytes + * + * Arbitrary "Application data", taking up the remainder of the frame + * after any "Extension data". The length of the "Application data" + * is equal to the payload length minus the length of the "Extension + * data". + */ + std::vector EncodeFrame( + SocketOpcode opcode, + bool maskOutput, + bool isFinal, + std::vector const& payload); + + /** + * @brief Decode a frame received from the websocket server. + * + * @param paylod Pointer to the payload returned by the service. Note that this may be shorter + * than the full data in the response message. + * @param opcode Opcode returned by the server. + * @param isFinal True if this is the final message. + * @param maskKey On Return, contains the contents of the mask key if the data is masked. + * @returns A pointer to the start of the decoded data. + */ + std::vector DecodeFrame( + BufferedStreamReader& streamReader, + SocketOpcode& opcode, + uint64_t& payloadLength, + bool& isFinal, + bool& isMasked, + std::array& maskKey, + Azure::Core::Context const& context); + SocketState m_state{SocketState::Invalid}; - std::array GenerateRandomKey(); + + std::array GenerateRandomKey() { return GenerateRandomBytes<16>(); }; void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); Azure::Core::Url m_remoteUrl; WebSocketOptions m_options; std::map m_headers; std::string m_chosenProtocol; std::shared_ptr m_transport; + BufferedStreamReader m_bufferedStreamReader; + + std::mutex m_transportMutex; + std::shared_mutex m_stateMutex; }; }}}}} // namespace Azure::Core::Http::WebSockets::_detail diff --git a/sdk/core/azure-core/test/ut/SocketServer.py b/sdk/core/azure-core/test/ut/websocket_server.py similarity index 72% rename from sdk/core/azure-core/test/ut/SocketServer.py rename to sdk/core/azure-core/test/ut/websocket_server.py index 7f77f3865d..a97ea79c59 100644 --- a/sdk/core/azure-core/test/ut/SocketServer.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -1,6 +1,8 @@ from array import array import asyncio +from operator import length_hint from time import sleep +from urllib.parse import urlparse import websockets @@ -36,11 +38,25 @@ async def handleCustomPath(websocket, path:dict): await websocket.send(data) await websocket.close() +async def handleEcho(websocket, url): + while websocket.open: + try: + data = await websocket.recv() + print(f'Echo ', length_hint(data),' bytes') + await websocket.send(data) + except websockets.ConnectionClosedOK: + print("Connection closed ok.") + return + except websockets.ConnectionClosed as ex: + print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") + return + async def handler(websocket, path : str): print("Socket handler: ", path) - if (path == '/openclosetest'): + parsedUrl = urlparse(path) + if (parsedUrl.path == '/openclosetest'): print("Open/Close Test") try: data = await websocket.recv() @@ -49,15 +65,14 @@ async def handler(websocket, path : str): except websockets.ConnectionClosed as ex: print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") return - elif (path == '/echotest'): - data = await websocket.recv() - await websocket.send(data) - elif (path == '/closeduringecho'): + elif (parsedUrl.path == '/echotest'): + await handleEcho(websocket, parsedUrl) + elif (parsedUrl.path == '/closeduringecho'): data = await websocket.recv() await websocket.close(1001, 'closed') - elif (path=='control'): + elif (parsedUrl.path =='control'): await handleControlPath(websocket) - elif (path in customPaths.keys()): + elif (parsedUrl.path in customPaths.keys()): print("Found path ", path, "in control paths.") await handleCustomPath(websocket, customPaths[path]) else: diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index e2b10e8929..37fc959423 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT +#include "../../src/http/websockets/websocketsimpl.hpp" #include "azure/core/http/websockets/websockets.hpp" #include "azure/core/internal/json/json.hpp" #include @@ -79,7 +80,7 @@ TEST(WebSocketTests, SimpleEcho) testSocket.Close(); } { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?delay=20")); testSocket.Open(); @@ -97,6 +98,53 @@ TEST(WebSocketTests, SimpleEcho) } } +template void EchoRandomData(WebSocket& socket) +{ + std::array data = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(); + std::vector sendData{data.begin(), data.end()}; + + socket.SendFrame(sendData, true); + + auto response = socket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto binaryResult = response->AsBinaryFrame(); + // Make sure we get back the data we sent in the echo request. + EXPECT_EQ(sendData.size(), binaryResult->Data.size()); + EXPECT_EQ(sendData, binaryResult->Data); +} + +TEST(WebSocketTests, VariableSizeEcho) +{ + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + { + EchoRandomData<100>(testSocket); + EchoRandomData<124>(testSocket); + EchoRandomData<125>(testSocket); + // The websocket protocol treats lengths of 125, 126 and > 127 specially. + EchoRandomData<126>(testSocket); + EchoRandomData<127>(testSocket); + EchoRandomData<128>(testSocket); + EchoRandomData<1020>(testSocket); + EchoRandomData<1021>(testSocket); + EchoRandomData<1022>(testSocket); + EchoRandomData<1023>(testSocket); + EchoRandomData<1024>(testSocket); + EchoRandomData<2048>(testSocket); + EchoRandomData<4096>(testSocket); + EchoRandomData<8192>(testSocket); + // The websocket protocol treats lengths of >65536 specially. + EchoRandomData<65535>(testSocket); + EchoRandomData<65536>(testSocket); + EchoRandomData<131072>(testSocket); + } + // Close the socket gracefully. + testSocket.Close(); + } +} + TEST(WebSocketTests, CloseDuringEcho) { { @@ -194,7 +242,7 @@ class LibWebSocketStatus { std::string GetLWSStatus() { WebSocketOptions options; - + options.ServiceName = "websockettest"; // Send 3 protocols to LWS. options.Protocols.push_back("brownCow"); @@ -203,7 +251,8 @@ class LibWebSocketStatus { WebSocket serverSocket(Azure::Core::Url("wss://libwebsockets.org"), options); serverSocket.Open(); - // The server should have chosen the lws-status protocol since it doesn't understand the other protocols. + // The server should have chosen the lws-status protocol since it doesn't understand the other + // protocols. EXPECT_EQ("lws-status", serverSocket.GetChosenProtocol()); auto lwsStatus = serverSocket.ReceiveFrame(); if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) From 7e0e9fc78cfe68220e469e312f80fc20d548a4d2 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 28 Jun 2022 13:45:13 -0700 Subject: [PATCH 004/149] Multi-threaded stress test --- .../src/http/websockets/websocketsimpl.cpp | 13 +- .../src/http/websockets/websocketsimpl.hpp | 14 +- .../azure-core/test/ut/websocket_server.py | 2 +- .../azure-core/test/ut/websocket_test.cpp | 133 +++++++++++++++++- 4 files changed, 149 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index c5217fd788..5a6dc094e8 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -405,7 +405,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names if (maskOutput) { // Start by generating the mask - 4 bytes of random data. - std::array mask = GenerateRandomBytes<4>(); + std::vector mask = GenerateRandomBytes(4); // Append the mask to the payload. encodedFrame.insert(encodedFrame.end(), mask.begin(), mask.end()); @@ -500,4 +500,15 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } + // Generator for random bytes. Used in WebSocketImplementation and tests. + std::vector GenerateRandomBytes(size_t vectorSize) + { + std::random_device randomEngine; + + std::vector rv(vectorSize); + std::generate(begin(rv), end(rv), std::ref(randomEngine)); + return rv; + } + + }}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index d25dad2831..8e9b85ff22 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -11,14 +11,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { // Generator for random bytes. Used in WebSocketImplementation and tests. - template std::array GenerateRandomBytes() - { - std::random_device randomEngine; - - std::array rv; - std::generate(begin(rv), end(rv), std::ref(randomEngine)); - return rv; - } + std::vector GenerateRandomBytes(size_t vectorSize); class WebSocketImplementation { enum class SocketState @@ -284,7 +277,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketState m_state{SocketState::Invalid}; - std::array GenerateRandomKey() { return GenerateRandomBytes<16>(); }; + std::vector GenerateRandomKey() + { + return GenerateRandomBytes(16); + }; void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); Azure::Core::Url m_remoteUrl; WebSocketOptions m_options; diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index a97ea79c59..b28d42fd9a 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -42,7 +42,7 @@ async def handleEcho(websocket, url): while websocket.open: try: data = await websocket.recv() - print(f'Echo ', length_hint(data),' bytes') +# print(f'Echo ', len(data),' bytes') await websocket.send(data) except websockets.ConnectionClosedOK: print("Connection closed ok.") diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 37fc959423..11623327b5 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -100,8 +100,7 @@ TEST(WebSocketTests, SimpleEcho) template void EchoRandomData(WebSocket& socket) { - std::array data = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(); - std::vector sendData{data.begin(), data.end()}; + std::vector sendData = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(N); socket.SendFrame(sendData, true); @@ -164,6 +163,136 @@ TEST(WebSocketTests, CloseDuringEcho) } } +std::string ToHexString(std::vector const& data) +{ + std::stringstream ss; + for (auto const& byte : data) + { + ss << std::hex << std::setfill('0') << std::setw(2) << (int)byte; + } + return ss.str(); +} + +TEST(WebSocketTests, MultiThreadedTestOnSingleSocket) +{ + constexpr size_t threadCount = 50; + constexpr size_t testDataLength = 30000; + constexpr auto testDuration = 10s; + + + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + // seed test data for the operations. + std::vector> testData(testDataLength); + std::vector> receivedData(testDataLength); + std::atomic_size_t iterationCount(0); + + // Spin up threadCount threads and hammer the echo server for 10 seconds. + std::vector threads; + for (size_t i = 0; i < threadCount; i += 1) + { + threads.push_back(std::thread([&]() { + std::chrono::time_point startTime + = std::chrono::steady_clock::now(); + + do + { + size_t i = iterationCount++; + std::vector sendData + = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(100); + { + if (i < testData.size()) + { + EXPECT_EQ(0, testData[i].size()); + testData[i] = sendData; + } + } + + testSocket.SendFrame(sendData, true); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto binaryResult = response->AsBinaryFrame(); + // Make sure we get back the data we sent in the echo request. + EXPECT_EQ(sendData.size(), binaryResult->Data.size()); + { + // There is no ordering expectation on the results, so we just remember the data + // as it comes in. We'll make sure we received everything later on. + if (i < receivedData.size()) + { + EXPECT_EQ(0, receivedData[i].size()); + receivedData[i] = binaryResult->Data; + } + } + } while (std::chrono::steady_clock::now() - startTime < testDuration); + })); + } + // std::this_thread::sleep_for(10s); + + for (auto& thread : threads) + { + thread.join(); + } + + // We no longer need to worry about synchronization since all the worker threads are done. + GTEST_LOG_(INFO) << "Total server requests: " << iterationCount.load() << std::endl; + GTEST_LOG_(INFO) << "Logged " << std::dec << testData.size() << " iterations (0x" << std::hex + << testData.size() << ")" << std::endl; + + // Close the socket gracefully. + testSocket.Close(); + + // If we've processed every iteration, let's make sure that we received everything we sent. + // If we dropped some results, then we can't check to ensure that we have received everything + // because we can't account for everything sent. + if (iterationCount <= testDataLength) + { + + // Compare testData and receivedData to ensure every element in testData is in receivedData + // and every element in receivedData is in testData. + // + // This is a bit of a hack, but it is the only way to do this without a lot of extra code. + // The problem is that the order of the elements in testData and receivedData is not + // guaranteed, so we need to sort them before we can compare them. + // We sort by the size of the vector, so the smaller vectors will be first in the sort. + std::vector testDataStrings; + std::vector receivedDataStrings; + for (auto const& data : testData) + { + testDataStrings.push_back(ToHexString(data)); + } + for (auto const& data : receivedData) + { + receivedDataStrings.push_back(ToHexString(data)); + } + std::sort(testDataStrings.begin(), testDataStrings.end()); + std::sort(receivedDataStrings.begin(), receivedDataStrings.end()); + for (size_t i = 0; i < testDataStrings.size(); ++i) + { + if (testDataStrings[i] != receivedDataStrings[i]) + { + GTEST_LOG_(ERROR) << "Mismatch at index " << i << std::endl; + GTEST_LOG_(ERROR) << "testData: " << testDataStrings[i] << std::endl; + GTEST_LOG_(ERROR) << "receivedData: " << receivedDataStrings[i] << std::endl; + } + } + + for (auto const& data : testDataStrings) + { + EXPECT_NE( + receivedDataStrings.end(), + std::find(receivedDataStrings.begin(), receivedDataStrings.end(), data)); + } + for (auto const& data : receivedDataStrings) + { + EXPECT_NE( + testDataStrings.end(), std::find(testDataStrings.begin(), testDataStrings.end(), data)); + } + } +} + // Does not work because curl rejects the wss: scheme. class LibWebSocketIncrementProtocol { WebSocketOptions m_options{true, {"dumb-increment-protocol"}}; From 97e63603cdc01cfd3047fcfb188f039c28c189b8 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 29 Jun 2022 16:16:26 -0700 Subject: [PATCH 005/149] Starting implementation of WinHTTP websocket transport --- sdk/core/azure-core/CMakeLists.txt | 11 +- .../win_http_websockets_transport.hpp | 112 +++++++++++ .../azure/core/http/win_http_transport.hpp | 103 +++++----- .../src/http/websockets/websocketsimpl.cpp | 61 ++++-- .../src/http/winhttp/win_http_transport.cpp | 186 ++++++++++-------- .../src/http/winhttp/win_http_websockets.cpp | 106 ++++++++++ 6 files changed, 427 insertions(+), 152 deletions(-) create mode 100644 sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp create mode 100644 sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index 8d99ce7867..51f49c5874 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -50,8 +50,12 @@ if(BUILD_TRANSPORT_CURL) ) endif() if(BUILD_TRANSPORT_WINHTTP) - SET(WIN_TRANSPORT_ADAPTER_SRC src/http/winhttp/win_http_transport.cpp) - SET(WIN_TRANSPORT_ADAPTER_INC inc/azure/core/http/win_http_transport.hpp) + SET(WIN_TRANSPORT_ADAPTER_SRC + src/http/winhttp/win_http_transport.cpp + src/http/winhttp/win_http_websockets.cpp) + SET(WIN_TRANSPORT_ADAPTER_INC + inc/azure/core/http/win_http_transport.hpp + inc/azure/core/http/websockets/win_http_websockets_transport.hpp) endif() set( @@ -87,7 +91,6 @@ set( inc/azure/core/internal/http/http_sanitizer.hpp inc/azure/core/internal/http/pipeline.hpp inc/azure/core/internal/http/user_agent.hpp - inc/azure/core/internal/input_sanitizer.hpp inc/azure/core/internal/io/null_body_stream.hpp inc/azure/core/internal/json/json.hpp inc/azure/core/internal/json/json_optional.hpp @@ -138,7 +141,7 @@ set( src/http/user_agent.cpp src/http/websockets/websockets.cpp src/http/websockets/websocketsimpl.cpp - src/io/body_stream.cpp + src/io/body_stream.cpp src/io/random_access_file_body_stream.cpp src/logger.cpp src/operation_status.cpp diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp new file mode 100644 index 0000000000..335059278f --- /dev/null +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @file + * @brief #Azure::Core::Http::WebSockets::WebSocketTransport implementation via WInHTTP. + */ + +#pragma once + +#include "azure/core/context.hpp" +#include "azure/core/http/http.hpp" +#include "azure/core/http/transport.hpp" +#include "azure/core/http/websockets/websockets_transport.hpp" +#include "azure/core/http/win_http_transport.hpp" +#include + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + /** + * @brief Concrete implementation of a WebSocket Transport that uses libcurl. + */ + class WinHttpWebSocketTransport : public WebSocketTransport, public WinHttpTransport { + + Azure::Core::Http::_detail::unique_HINTERNET m_socketHandle; + + // Called by the + void OnResponseReceived(Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) override; + + public: + /** + * @brief Construct a new CurlTransport object. + * + * @param options Optional parameter to override the default options. + */ + WinHttpWebSocketTransport(WinHttpTransportOptions const& options = WinHttpTransportOptions()) + : WinHttpTransport(options) + { + } + /** + * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse + * + * @param request an HTTP Request to be send. + * @param context A context to control the request lifetime. + * + * @return unique ptr to an HTTP RawResponse. + */ + virtual std::unique_ptr Send(Request& request, Context const& context) override; + + /** + * @brief Indicates if the transports supports native websockets or not. + * + * @detail For the WinHTTP websocket transport, the transport supports native websockets. + */ + virtual bool NativeWebsocketSupport() override { return true; } + + virtual void CompleteUpgrade() override; + + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + */ + virtual void Close() override; + + // Native WebSocket support methods. + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + * @param status Status value to be sent to the remote node. Application defined. + * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. + * @param context Context for the operation. + * + */ + virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override; + + /** + * @brief Send a frame of data to the remote node. + * + * @detail Not implemented for CURL websockets because CURL does not support native + * websockets. + * + * @brief frameType Frame type sent to the server, Text or Binary. + * @brief frameData Frame data to be sent to the server. + */ + virtual void SendFrame(WebSocketFrameType, std::vector, Azure::Core::Context const&) + override; + + // Non-Native WebSocket support. + /** + * @brief This function is used when working with streams to pull more data from the wire. + * Function will try to keep pulling data from socket until the buffer is all written or + * until there is no more data to get from the socket. + * + */ + virtual size_t ReadFromSocket(uint8_t*, size_t, Context const&) override + { + throw std::runtime_error("Not implemented."); + } + + /** + * @brief This method will use libcurl socket to write all the bytes from buffer. + * + */ + virtual int SendBuffer(uint8_t const*, size_t, Context const&) override + { + throw std::runtime_error("Not implemented."); + } + }; + +}}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp index 6090d6b49a..d7d970de51 100644 --- a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp @@ -24,6 +24,7 @@ #include #endif +#include #include #include #include @@ -35,41 +36,22 @@ namespace Azure { namespace Core { namespace Http { constexpr static size_t DefaultUploadChunkSize = 1024 * 64; constexpr static size_t MaximumUploadChunkSize = 1024 * 1024; - struct HandleManager final - { - Context const& m_context; - Request& m_request; - HINTERNET m_connectionHandle; - HINTERNET m_requestHandle; - - HandleManager(Request& request, Context const& context) - : m_request(request), m_context(context) - { - m_connectionHandle = NULL; - m_requestHandle = NULL; - } - - ~HandleManager() + // unique_ptr class wrapping an HINTERNET handle + class HINTERNET_deleter { + public: + void operator()(HINTERNET handle) noexcept { - // Close the handles and set them to null to avoid multiple calls to WinHTTP to close the - // handles. - if (m_requestHandle) + if (handle != nullptr) { - WinHttpCloseHandle(m_requestHandle); - m_requestHandle = NULL; - } - - if (m_connectionHandle) - { - WinHttpCloseHandle(m_connectionHandle); - m_connectionHandle = NULL; + WinHttpCloseHandle(handle); } } }; + using unique_HINTERNET = std::unique_ptr; class WinHttpStream final : public Azure::Core::IO::BodyStream { private: - std::unique_ptr m_handleManager; + _detail::unique_HINTERNET m_requestHandle; bool m_isEOF; /** @@ -99,8 +81,8 @@ namespace Azure { namespace Core { namespace Http { size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override; public: - WinHttpStream(std::unique_ptr handleManager, int64_t contentLength) - : m_handleManager(std::move(handleManager)), m_contentLength(contentLength), + WinHttpStream(_detail::unique_HINTERNET& requestHandle, int64_t contentLength) + : m_requestHandle(std::move(requestHandle)), m_contentLength(contentLength), m_isEOF(false), m_streamTotalRead(0) { } @@ -124,34 +106,64 @@ namespace Azure { namespace Core { namespace Http { * @brief When `true`, allows an invalid certificate authority. */ bool IgnoreUnknownCertificateAuthority = false; + + /** + * @brief When `true`, enables WebSocket upgrade. + */ + bool EnableWebSocketUpgrade = false; }; /** * @brief Concrete implementation of an HTTP transport that uses WinHTTP when sending and * receiving requests and responses over the wire. */ - class WinHttpTransport final : public HttpTransport { + class WinHttpTransport : public HttpTransport { private: WinHttpTransportOptions m_options; // This should remain immutable and not be modified after calling the ctor, to avoid threading // issues. - HINTERNET m_sessionHandle = NULL; - - HINTERNET CreateSessionHandle(); - void CreateConnectionHandle(std::unique_ptr<_detail::HandleManager>& handleManager); - void CreateRequestHandle(std::unique_ptr<_detail::HandleManager>& handleManager); - void Upload(std::unique_ptr<_detail::HandleManager>& handleManager); - void SendRequest(std::unique_ptr<_detail::HandleManager>& handleManager); - void ReceiveResponse(std::unique_ptr<_detail::HandleManager>& handleManager); + _detail::unique_HINTERNET m_sessionHandle; + + _detail::unique_HINTERNET CreateSessionHandle(); + _detail::unique_HINTERNET CreateConnectionHandle( + Azure::Core::Url const& url, + Azure::Core::Context const& context); + _detail::unique_HINTERNET CreateRequestHandle( + _detail::unique_HINTERNET& connectionHandle, + Azure::Core::Url const& url, + Azure::Core::Http::HttpMethod const& method); + void Upload( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Http::Request& request, + Azure::Core::Context const& context); + void SendRequest( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Http::Request& request, + Azure::Core::Context const& context); + void ReceiveResponse( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Context const& context); int64_t GetContentLength( - std::unique_ptr<_detail::HandleManager>& handleManager, + _detail::unique_HINTERNET& requestHandle, HttpMethod requestMethod, HttpStatusCode responseStatusCode); std::unique_ptr SendRequestAndGetResponse( - std::unique_ptr<_detail::HandleManager> handleManager, + _detail::unique_HINTERNET& requestHandle, HttpMethod requestMethod); + // Callback to allow a derived transport to extract the request handle. Used for WebSocket + // transports. + protected: + virtual void OnResponseReceived(_detail::unique_HINTERNET&){}; + /** + * @brief Throw an exception based on the Win32 Error code + * + * @param exceptionMessage Message describing error. + * @param error Win32 Error code. + */ + void GetErrorAndThrow(const std::string& exceptionMessage, DWORD error = GetLastError()); + public: /** * @brief Constructs `%WinHttpTransport`. @@ -170,16 +182,7 @@ namespace Azure { namespace Core { namespace Http { */ virtual std::unique_ptr Send(Request& request, Context const& context) override; - ~WinHttpTransport() - { - // Close the handles and set them to null to avoid multiple calls to WinHTTP to close the - // handles. - if (m_sessionHandle) - { - WinHttpCloseHandle(m_sessionHandle); - m_sessionHandle = NULL; - } - } + virtual ~WinHttpTransport() = default; }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 5a6dc094e8..84a0a09e6b 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -9,6 +9,7 @@ #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) #include "azure/core/http/websockets/curl_websockets_transport.hpp" #endif +#include #include #include #include @@ -31,9 +32,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Opening; #if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) - auto winHttpTransport - = std::make_shared(); - m_options.Transport.Transport = winHttpTransport; + WinHttpTransportOptions transportOptions; + // WinHTTP overrides the Connection: Upgrade header, so disable keep alives. + transportOptions.EnableWebSocketUpgrade = true; + + m_transport = std::make_shared( + transportOptions); + m_options.Transport.Transport = m_transport; #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) CurlTransportOptions transportOptions; transportOptions.HttpKeepAlive = false; @@ -59,22 +64,31 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Azure::Core::Http::Request openSocketRequest( Azure::Core::Http::HttpMethod::Get, m_remoteUrl, false); - // Set the standardized WebSocket upgrade headers. - openSocketRequest.SetHeader("Upgrade", "websocket"); - openSocketRequest.SetHeader("Connection", "Upgrade"); - openSocketRequest.SetHeader("Sec-WebSocket-Version", "13"); - // Generate the random request key + // Generate the random request key. Only used when the transport doesn't support websockets + // natively. auto randomKey = GenerateRandomKey(); auto encodedKey = Azure::Core::Convert::Base64Encode(randomKey); - openSocketRequest.SetHeader("Sec-WebSocket-Key", encodedKey); - std::string protocols; - for (auto const& protocol : m_options.Protocols) + if (!m_transport->NativeWebsocketSupport()) + { + // If the transport doesn't support WebSockets natively, set the standardized WebSocket + // upgrade headers. + openSocketRequest.SetHeader("Upgrade", "websocket"); + openSocketRequest.SetHeader("Connection", "upgrade"); + openSocketRequest.SetHeader("Sec-WebSocket-Version", "13"); + openSocketRequest.SetHeader("Sec-WebSocket-Key", encodedKey); + } + if (!m_options.Protocols.empty()) { - protocols += protocol; - protocols += ", "; + + std::string protocols; + for (auto const& protocol : m_options.Protocols) + { + protocols += protocol; + protocols += ", "; + } + protocols = protocols.substr(0, protocols.size() - 2); + openSocketRequest.SetHeader("Sec-WebSocket-Protocol", protocols); } - protocols = protocols.substr(0, protocols.size() - 2); - openSocketRequest.SetHeader("Sec-WebSocket-Protocol", protocols); for (auto const& additionalHeader : m_headers) { openSocketRequest.SetHeader(additionalHeader.first, additionalHeader.second); @@ -97,10 +111,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Prove that the server received this socket request. auto& responseHeaders = response->GetHeaders(); - auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); - if (socketAccept != responseHeaders.end()) + if (!m_transport->NativeWebsocketSupport()) { - VerifySocketAccept(encodedKey, socketAccept->second); + + auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); + if (socketAccept == responseHeaders.end()) + { + throw Azure::Core::Http::TransportException("Missing Sec-WebSocket-Accept header"); + } + // Verify that the WebSocket server received *this* open request. + else + { + VerifySocketAccept(encodedKey, socketAccept->second); + } } // Remember the protocol that the client chose. @@ -506,9 +529,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::random_device randomEngine; std::vector rv(vectorSize); +#pragma warning(suppress : 4244) std::generate(begin(rv), end(rv), std::ref(randomEngine)); return rv; } - }}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 7726305bf9..8bcad260b8 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -198,9 +198,8 @@ std::string GetHeadersAsString(Azure::Core::Http::Request const& request) } // namespace -void GetErrorAndThrow(const std::string& exceptionMessage) +void WinHttpTransport::GetErrorAndThrow(const std::string& exceptionMessage, DWORD error) { - DWORD error = GetLastError(); std::string errorMessage = exceptionMessage + " Error Code: " + std::to_string(error); char* errorMsg = nullptr; @@ -226,17 +225,19 @@ void GetErrorAndThrow(const std::string& exceptionMessage) throw Azure::Core::Http::TransportException(errorMessage); } -HINTERNET WinHttpTransport::CreateSessionHandle() +_detail::unique_HINTERNET WinHttpTransport::CreateSessionHandle() { // Use WinHttpOpen to obtain a session handle. // The dwFlags is set to 0 - all WinHTTP functions are performed synchronously. - HINTERNET sessionHandle = WinHttpOpen( - NULL, // Do not use a fallback user-agent string, and only rely on the header within the - // request itself. - WINHTTP_ACCESS_TYPE_NO_PROXY, - WINHTTP_NO_PROXY_NAME, - WINHTTP_NO_PROXY_BYPASS, - 0); + _detail::unique_HINTERNET sessionHandle( + WinHttpOpen( + NULL, // Do not use a fallback user-agent string, and only rely on the header within the + // request itself. + WINHTTP_ACCESS_TYPE_NO_PROXY, + WINHTTP_NO_PROXY_NAME, + WINHTTP_NO_PROXY_BYPASS, + 0), + _detail::HINTERNET_deleter()); if (!sessionHandle) { @@ -253,19 +254,22 @@ HINTERNET WinHttpTransport::CreateSessionHandle() #ifdef WINHTTP_OPTION_TCP_FAST_OPEN BOOL tcp_fast_open = TRUE; WinHttpSetOption( - sessionHandle, WINHTTP_OPTION_TCP_FAST_OPEN, &tcp_fast_open, sizeof(tcp_fast_open)); + sessionHandle.get(), WINHTTP_OPTION_TCP_FAST_OPEN, &tcp_fast_open, sizeof(tcp_fast_open)); #endif #ifdef WINHTTP_OPTION_TLS_FALSE_START BOOL tls_false_start = TRUE; WinHttpSetOption( - sessionHandle, WINHTTP_OPTION_TLS_FALSE_START, &tls_false_start, sizeof(tls_false_start)); + sessionHandle.get(), + WINHTTP_OPTION_TLS_FALSE_START, + &tls_false_start, + sizeof(tls_false_start)); #endif // Enforce TLS version 1.2 auto tlsOption = WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2; if (!WinHttpSetOption( - sessionHandle, WINHTTP_OPTION_SECURE_PROTOCOLS, &tlsOption, sizeof(tlsOption))) + sessionHandle.get(), WINHTTP_OPTION_SECURE_PROTOCOLS, &tlsOption, sizeof(tlsOption))) { GetErrorAndThrow("Error while enforcing TLS 1.2 for connection request."); } @@ -278,23 +282,26 @@ WinHttpTransport::WinHttpTransport(WinHttpTransportOptions const& options) { } -void WinHttpTransport::CreateConnectionHandle( - std::unique_ptr<_detail::HandleManager>& handleManager) +_detail::unique_HINTERNET WinHttpTransport::CreateConnectionHandle( + Azure::Core::Url const& url, + Azure::Core::Context const& context) { // If port is 0, i.e. INTERNET_DEFAULT_PORT, it uses port 80 for HTTP and port 443 for HTTPS. - uint16_t port = handleManager->m_request.GetUrl().GetPort(); + uint16_t port = url.GetPort(); - handleManager->m_context.ThrowIfCancelled(); + context.ThrowIfCancelled(); // Specify an HTTP server. // This function always operates synchronously. - handleManager->m_connectionHandle = WinHttpConnect( - m_sessionHandle, - StringToWideString(handleManager->m_request.GetUrl().GetHost()).c_str(), - port == 0 ? INTERNET_DEFAULT_PORT : port, - 0); - - if (!handleManager->m_connectionHandle) + _detail::unique_HINTERNET rv( + WinHttpConnect( + m_sessionHandle.get(), + StringToWideString(url.GetHost()).c_str(), + port == 0 ? INTERNET_DEFAULT_PORT : port, + 0), + _detail::HINTERNET_deleter()); + + if (!rv) { // Errors include: // ERROR_WINHTTP_INCORRECT_HANDLE_TYPE @@ -306,29 +313,33 @@ void WinHttpTransport::CreateConnectionHandle( // ERROR_NOT_ENOUGH_MEMORY GetErrorAndThrow("Error while getting a connection handle."); } + return rv; } -void WinHttpTransport::CreateRequestHandle(std::unique_ptr<_detail::HandleManager>& handleManager) +_detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( + _detail::unique_HINTERNET& connectionHandle, + Azure::Core::Url const& url, + Azure::Core::Http::HttpMethod const& method) { - const std::string& path = handleManager->m_request.GetUrl().GetRelativeUrl(); - HttpMethod requestMethod = handleManager->m_request.GetMethod(); + const std::string& path = url.GetRelativeUrl(); + HttpMethod requestMethod = method; bool const requestSecureHttp( !Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( - handleManager->m_request.GetUrl().GetScheme(), HttpScheme)); + url.GetScheme(), HttpScheme)); // Create an HTTP request handle. - handleManager->m_requestHandle = WinHttpOpenRequest( - handleManager->m_connectionHandle, - HttpMethodToWideString(requestMethod).c_str(), - path.empty() ? NULL - : StringToWideString(path) - .c_str(), // Name of the target resource of the specified HTTP verb - NULL, // Use HTTP/1.1 - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, // No media types are accepted by the client - requestSecureHttp ? WINHTTP_FLAG_SECURE : 0); // Uses secure transaction semantics (SSL/TLS) - - if (!handleManager->m_requestHandle) + _detail::unique_HINTERNET hi( + WinHttpOpenRequest( + connectionHandle.get(), + HttpMethodToWideString(requestMethod).c_str(), + path.empty() ? NULL : StringToWideString(path).c_str(), // Name of the target resource of + // the specified HTTP verb + NULL, // Use HTTP/1.1 + WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, // No media types are accepted by the client + requestSecureHttp ? WINHTTP_FLAG_SECURE : 0), + _detail::HINTERNET_deleter()); // Uses secure transaction semantics (SSL/TLS) + if (!hi) { // Errors include: // ERROR_WINHTTP_INCORRECT_HANDLE_TYPE @@ -348,10 +359,7 @@ void WinHttpTransport::CreateRequestHandle(std::unique_ptr<_detail::HandleManage // Note: If/When TLS client certificate support is added to the pipeline, this line may need to // be revisited. if (!WinHttpSetOption( - handleManager->m_requestHandle, - WINHTTP_OPTION_CLIENT_CERT_CONTEXT, - WINHTTP_NO_CLIENT_CERT_CONTEXT, - 0)) + hi.get(), WINHTTP_OPTION_CLIENT_CERT_CONTEXT, WINHTTP_NO_CLIENT_CERT_CONTEXT, 0)) { GetErrorAndThrow("Error while setting client cert context to ignore."); } @@ -360,18 +368,30 @@ void WinHttpTransport::CreateRequestHandle(std::unique_ptr<_detail::HandleManage if (m_options.IgnoreUnknownCertificateAuthority) { auto option = SECURITY_FLAG_IGNORE_UNKNOWN_CA; - if (!WinHttpSetOption( - handleManager->m_requestHandle, WINHTTP_OPTION_SECURITY_FLAGS, &option, sizeof(option))) + if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_SECURITY_FLAGS, &option, sizeof(option))) { GetErrorAndThrow("Error while setting ignore unknown server certificate."); } } + + if (m_options.EnableWebSocketUpgrade) + { + int option; + if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, &option, 0)) + { + GetErrorAndThrow("Error while Enabling WebSocket upgrade."); + } + } + return hi; } // For PUT/POST requests, send additional data using WinHttpWriteData. -void WinHttpTransport::Upload(std::unique_ptr<_detail::HandleManager>& handleManager) +void WinHttpTransport::Upload( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Http::Request& request, + Azure::Core::Context const& context) { - auto streamBody = handleManager->m_request.GetBodyStream(); + auto streamBody = request.GetBodyStream(); int64_t streamLength = streamBody->Length(); // Consider using `MaximumUploadChunkSize` here, after some perf measurements @@ -384,8 +404,7 @@ void WinHttpTransport::Upload(std::unique_ptr<_detail::HandleManager>& handleMan while (true) { - size_t rawRequestLen - = streamBody->Read(unique_buffer.get(), uploadChunkSize, handleManager->m_context); + size_t rawRequestLen = streamBody->Read(unique_buffer.get(), uploadChunkSize, context); if (rawRequestLen == 0) { break; @@ -393,11 +412,11 @@ void WinHttpTransport::Upload(std::unique_ptr<_detail::HandleManager>& handleMan DWORD dwBytesWritten = 0; - handleManager->m_context.ThrowIfCancelled(); + context.ThrowIfCancelled(); // Write data to the server. if (!WinHttpWriteData( - handleManager->m_requestHandle, + requestHandle.get(), unique_buffer.get(), static_cast(rawRequestLen), &dwBytesWritten)) @@ -407,29 +426,32 @@ void WinHttpTransport::Upload(std::unique_ptr<_detail::HandleManager>& handleMan } } -void WinHttpTransport::SendRequest(std::unique_ptr<_detail::HandleManager>& handleManager) +void WinHttpTransport::SendRequest( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Http::Request& request, + Azure::Core::Context const& context) { std::wstring encodedHeaders; int encodedHeadersLength = 0; - auto requestHeaders = handleManager->m_request.GetHeaders(); + auto requestHeaders = request.GetHeaders(); if (requestHeaders.size() != 0) { // The encodedHeaders will be null-terminated and the length is calculated. encodedHeadersLength = -1; - std::string requestHeaderString = GetHeadersAsString(handleManager->m_request); + std::string requestHeaderString = GetHeadersAsString(request); requestHeaderString.append("\0"); encodedHeaders = StringToWideString(requestHeaderString); } - int64_t streamLength = handleManager->m_request.GetBodyStream()->Length(); + int64_t streamLength = request.GetBodyStream()->Length(); - handleManager->m_context.ThrowIfCancelled(); + context.ThrowIfCancelled(); // Send a request. if (!WinHttpSendRequest( - handleManager->m_requestHandle, + requestHandle.get(), requestHeaders.size() == 0 ? WINHTTP_NO_ADDITIONAL_HEADERS : encodedHeaders.c_str(), encodedHeadersLength, WINHTTP_NO_REQUEST_DATA, @@ -468,18 +490,20 @@ void WinHttpTransport::SendRequest(std::unique_ptr<_detail::HandleManager>& hand if (streamLength > 0) { - Upload(handleManager); + Upload(requestHandle, request, context); } } -void WinHttpTransport::ReceiveResponse(std::unique_ptr<_detail::HandleManager>& handleManager) +void WinHttpTransport::ReceiveResponse( + _detail::unique_HINTERNET& requestHandle, + Azure::Core::Context const& context) { - handleManager->m_context.ThrowIfCancelled(); + context.ThrowIfCancelled(); // Wait to receive the response to the HTTP request initiated by WinHttpSendRequest. // When WinHttpReceiveResponse completes successfully, the status code and response headers have // been received. - if (!WinHttpReceiveResponse(handleManager->m_requestHandle, NULL)) + if (!WinHttpReceiveResponse(requestHandle.get(), NULL)) { // Errors include: // ERROR_WINHTTP_CANNOT_CONNECT @@ -494,7 +518,7 @@ void WinHttpTransport::ReceiveResponse(std::unique_ptr<_detail::HandleManager>& } int64_t WinHttpTransport::GetContentLength( - std::unique_ptr<_detail::HandleManager>& handleManager, + _detail::unique_HINTERNET& requestHandle, HttpMethod requestMethod, HttpStatusCode responseStatusCode) { @@ -511,7 +535,7 @@ int64_t WinHttpTransport::GetContentLength( if (requestMethod != HttpMethod::Head && responseStatusCode != HttpStatusCode::NoContent) { if (!WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER, WINHTTP_HEADER_NAME_BY_INDEX, &dwContentLength, @@ -530,14 +554,14 @@ int64_t WinHttpTransport::GetContentLength( } std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( - std::unique_ptr<_detail::HandleManager> handleManager, + _detail::unique_HINTERNET& requestHandle, HttpMethod requestMethod) { // First, use WinHttpQueryHeaders to obtain the size of the buffer. // The call is expected to fail since no destination buffer is provided. DWORD sizeOfHeaders = 0; if (WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_RAW_HEADERS, WINHTTP_HEADER_NAME_BY_INDEX, NULL, @@ -563,7 +587,7 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( // Now, use WinHttpQueryHeaders to retrieve all the headers. // Each header is terminated by "\0". An additional "\0" terminates the list of headers. if (!WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_RAW_HEADERS, WINHTTP_HEADER_NAME_BY_INDEX, outputBuffer.data(), @@ -583,7 +607,7 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( // Get the HTTP version. if (!WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_VERSION, WINHTTP_HEADER_NAME_BY_INDEX, outputBuffer.data(), @@ -606,7 +630,7 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( // Get the status code as a number. if (!WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, WINHTTP_HEADER_NAME_BY_INDEX, &statusCode, @@ -623,7 +647,7 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( DWORD sizeOfReasonPhrase = sizeOfHeaders; if (WinHttpQueryHeaders( - handleManager->m_requestHandle, + requestHandle.get(), WINHTTP_QUERY_STATUS_TEXT, WINHTTP_HEADER_NAME_BY_INDEX, outputBuffer.data(), @@ -643,25 +667,29 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( SetHeaders(responseHeaders, rawResponse); int64_t contentLength - = GetContentLength(handleManager, requestMethod, rawResponse->GetStatusCode()); + = GetContentLength(requestHandle, requestMethod, rawResponse->GetStatusCode()); rawResponse->SetBodyStream( - std::make_unique<_detail::WinHttpStream>(std::move(handleManager), contentLength)); + std::make_unique<_detail::WinHttpStream>(requestHandle, contentLength)); return rawResponse; } std::unique_ptr WinHttpTransport::Send(Request& request, Context const& context) { - auto handleManager = std::make_unique<_detail::HandleManager>(request, context); + _detail::unique_HINTERNET connectionHandle = CreateConnectionHandle(request.GetUrl(), context); + _detail::unique_HINTERNET requestHandle + = CreateRequestHandle(connectionHandle, request.GetUrl(), request.GetMethod()); - CreateConnectionHandle(handleManager); - CreateRequestHandle(handleManager); + SendRequest(requestHandle, request, context); - SendRequest(handleManager); + ReceiveResponse(requestHandle, context); - ReceiveResponse(handleManager); + if (m_options.EnableWebSocketUpgrade) + { + OnResponseReceived(requestHandle); + } - return SendRequestAndGetResponse(std::move(handleManager), request.GetMethod()); + return SendRequestAndGetResponse(requestHandle, request.GetMethod()); } // Read the response from the sent request. @@ -679,7 +707,7 @@ size_t _detail::WinHttpStream::OnRead(uint8_t* buffer, size_t count, Context con DWORD numberOfBytesRead = 0; if (!WinHttpReadData( - this->m_handleManager->m_requestHandle, + this->m_requestHandle.get(), (LPVOID)(buffer), static_cast(count), &numberOfBytesRead)) diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp new file mode 100644 index 0000000000..934130a3a0 --- /dev/null +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/core/http/http.hpp" +#include "azure/core/http/policies/policy.hpp" +#include "azure/core/http/transport.hpp" +#include "azure/core/http/websockets/win_http_websockets_transport.hpp" +#include "azure/core/internal/diagnostics/log.hpp" +#include "azure/core/platform.hpp" + +#if defined(AZ_PLATFORM_POSIX) +#include // for poll() +#include // for socket shutdown +#elif defined(AZ_PLATFORM_WINDOWS) +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif +#if !defined(NOMINMAX) +#define NOMINMAX +#endif +#include +#include // for WSAPoll(); +#endif + +namespace Azure { namespace Core { namespace Http { namespace WebSockets { + + void WinHttpWebSocketTransport::OnResponseReceived( + Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) + { + // Convert the request handle into a WebSocket handle for us to use later. + m_socketHandle = Azure::Core::Http::_detail::unique_HINTERNET( + WinHttpWebSocketCompleteUpgrade(requestHandle.get(), 0), + Azure::Core::Http::_detail::HINTERNET_deleter()); + if (!m_socketHandle) + { + GetErrorAndThrow("Error Upgrading HttpRequest handle to WebSocket handle."); + } + } + + std::unique_ptr WinHttpWebSocketTransport::Send( + Azure::Core::Http::Request& request, + Azure::Core::Context const& context) + { + return WinHttpTransport::Send(request, context); + } + + void WinHttpWebSocketTransport::CompleteUpgrade() {} + + // Close the WebSocket operation cleanly. Does not specify + void WinHttpWebSocketTransport::Close() + { + auto err = WinHttpWebSocketClose( + m_socketHandle.get(), WINHTTP_WEB_SOCKET_ENDPOINT_TERMINATED_CLOSE_STATUS, 0, 0); + if (err != 0) + { + GetErrorAndThrow("WinHttpWebSocketClose() failed", err); + } + } + + // Native WebSocket support methods. + /** + * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + * @param status Status value to be sent to the remote node. Application defined. + * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. + * @param context Context for the operation. + * + */ + void WinHttpWebSocketTransport::CloseSocket( + uint16_t status, + std::string const& disconnectReason, + Azure::Core::Context const& context) + { + context.ThrowIfCancelled(); + + auto err = WinHttpWebSocketClose( + m_socketHandle.get(), + status, + reinterpret_cast(const_cast(disconnectReason.c_str())), + static_cast(disconnectReason.size())); + if (err != 0) + { + GetErrorAndThrow("WinHttpWebSocketClose() failed", err); + } + } + + /** + * @brief Send a frame of data to the remote node. + * + * @detail Not implemented for CURL websockets because CURL does not support native + * websockets. + * + * @brief frameType Frame type sent to the server, Text or Binary. + * @brief frameData Frame data to be sent to the server. + */ + void WinHttpWebSocketTransport::SendFrame( + WebSocketFrameType, + std::vector, + Azure::Core::Context const&) + { + throw std::runtime_error("Not implemented"); + } + +}}}} // namespace Azure::Core::Http::WebSockets From 1372b0af4e49c92a9b869b5bfa10c3a74aa225bf Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 1 Jul 2022 16:00:55 -0700 Subject: [PATCH 006/149] Completed implementation of HTTP websockets --- .../websockets/curl_websockets_transport.hpp | 32 +++- .../azure/core/http/websockets/websockets.hpp | 39 +---- .../http/websockets/websockets_transport.hpp | 27 +++- .../win_http_websockets_transport.hpp | 35 ++++- .../src/http/websockets/websockets.cpp | 9 -- .../src/http/websockets/websocketsimpl.cpp | 93 ++++++++--- .../src/http/websockets/websocketsimpl.hpp | 17 +- .../src/http/winhttp/win_http_transport.cpp | 3 +- .../src/http/winhttp/win_http_websockets.cpp | 147 ++++++++++++++++-- .../azure-core/test/ut/websocket_test.cpp | 113 +++++++------- 10 files changed, 371 insertions(+), 144 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index c93f511640..94dd4d55cd 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -80,6 +80,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { throw std::runtime_error("Not implemented."); } + /** + * @brief Retrieve the status of the close socket operation. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + */ + std::pair GetCloseSocketInformation(const Azure::Core::Context&) override + { + throw std::runtime_error("Not implemented"); + } + /** * @brief Send a frame of data to the remote node. * @@ -88,12 +99,29 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief frameType Frame type sent to the server, Text or Binary. * @brief frameData Frame data to be sent to the server. */ - virtual void SendFrame(WebSocketFrameType, std::vector, Azure::Core::Context const&) - override + virtual void SendFrame( + WebSocketFrameType, + std::vector const&, + Azure::Core::Context const&) override { throw std::runtime_error("Not implemented."); } + /** + * @brief Receive a frame of data from the remote node. + * + * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * + * @brief buffer data received from the server. + * @brief bufferSize size of buffer. + * @brief bytesRead number of bytes read from server. + */ + virtual std::vector ReceiveFrame(WebSocketFrameType&, Azure::Core::Context const&) + override + { + throw std::runtime_error("Not implemented"); + } + // Non-Native WebSocket support. /** * @brief This function is used when working with streams to pull more data from the wire. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index a1bcdb1415..a80a0c86c3 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -26,7 +26,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { Unknown, TextFrameReceived, BinaryFrameReceived, - ContinuationReceived, PeerClosed, }; @@ -49,16 +48,15 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketTextFrame; class WebSocketBinaryFrame; - class WebSocketContinuationFrame; class WebSocketPeerCloseFrame; struct WebSocketResult { WebSocketResultType ResultType; + bool IsFinalFrame{false}; std::shared_ptr AsTextFrame(); std::shared_ptr AsBinaryFrame(); std::shared_ptr AsPeerCloseFrame(); - std::shared_ptr AsContinuationFrame(); }; class WebSocketTextFrame : public WebSocketResult, @@ -67,12 +65,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: WebSocketTextFrame() = default; WebSocketTextFrame(bool isFinalFrame, unsigned char const* body, size_t size) - : WebSocketResult{WebSocketResultType::TextFrameReceived}, Text(body, body + size), - IsFinalFrame(isFinalFrame) + : WebSocketResult{WebSocketResultType::TextFrameReceived, isFinalFrame}, + Text(body, body + size) { } std::string Text; - bool IsFinalFrame; }; class WebSocketBinaryFrame : public WebSocketResult, public std::enable_shared_from_this { @@ -80,44 +77,24 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: WebSocketBinaryFrame() = default; WebSocketBinaryFrame(bool isFinal, unsigned char const* body, size_t size) - : WebSocketResult{WebSocketResultType::BinaryFrameReceived}, Data(body, body + size), - IsFinalFrame(isFinal) + : WebSocketResult{WebSocketResultType::BinaryFrameReceived, isFinal}, + Data(body, body + size) { } std::vector Data; - bool IsFinalFrame; - }; - - class WebSocketContinuationFrame - : public WebSocketResult, - public std::enable_shared_from_this { - public: - WebSocketContinuationFrame() = default; - WebSocketContinuationFrame(bool isFinal, unsigned char const* body, size_t size) - : WebSocketResult{WebSocketResultType::ContinuationReceived}, - ContinuationData(body, body + size), IsFinalFrame(isFinal) - { - } - std::vector ContinuationData; - bool IsFinalFrame; }; class WebSocketPeerCloseFrame : public WebSocketResult, public std::enable_shared_from_this { - std::vector frameData_; - public: WebSocketPeerCloseFrame() = default; - WebSocketPeerCloseFrame( - uint16_t remoteStatusCode, - unsigned char const* closeData, - size_t closeSize) + WebSocketPeerCloseFrame(uint16_t remoteStatusCode, std::string const& remoteCloseReason) : WebSocketResult{WebSocketResultType::PeerClosed}, RemoteStatusCode(remoteStatusCode), - frameData_(closeData, closeData + closeSize), BodyStream(frameData_) + RemoteCloseReason(remoteCloseReason) { } uint16_t RemoteStatusCode; - Azure::Core::IO::MemoryBodyStream BodyStream; + std::string RemoteCloseReason; }; struct WebSocketOptions : Azure::Core::_internal::ClientOptions diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 08e9239819..db5e753365 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -43,6 +43,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * complete frame to be sent to the remote node. */ FrameTypeBinaryFragment, + + FrameTypeClosed, }; /** * @brief Destructs `%HttpTransport`. @@ -89,6 +91,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual void Close() = 0; + /** + * @brief Retrieve the information associated with a WebSocket close response. + * + * @param context Context for the operation. + * + * @returns a tuple containing the status code and string. + */ + virtual std::pair GetCloseSocketInformation( + Azure::Core::Context const& context) + = 0; + /** * @brief Send a frame of data to the remote node. * @@ -97,7 +110,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual void SendFrame( WebSocketFrameType frameType, - std::vector frameData, + std::vector const& frameData, + Azure::Core::Context const& context) + = 0; + + /** + * @brief Receive a frame from the remote WebSocket server. + * + * @param frameTypeReceived frame type received from the remote server. + * + * @returns Frame data received from the remote server. + */ + virtual std::vector ReceiveFrame( + WebSocketFrameType& frameTypeReceived, Azure::Core::Context const& context) = 0; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 335059278f..7b3fe8eefe 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -14,6 +14,7 @@ #include "azure/core/http/websockets/websockets_transport.hpp" #include "azure/core/http/win_http_transport.hpp" #include +#include namespace Azure { namespace Core { namespace Http { namespace WebSockets { @@ -22,11 +23,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ class WinHttpWebSocketTransport : public WebSocketTransport, public WinHttpTransport { - Azure::Core::Http::_detail::unique_HINTERNET m_socketHandle; - - // Called by the - void OnResponseReceived(Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) override; - + Azure::Core::Http::_detail::unique_HINTERNET m_socketHandle; + std::mutex m_sendMutex; + std::mutex m_receiveMutex; + + // Called by the + void OnResponseReceived(Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) override; + public: /** * @brief Construct a new CurlTransport object. @@ -75,6 +78,18 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override; + /** + * @brief Retrieve the information associated with a WebSocket close response. + * + * Should only be called when a Receive operation returns WebSocketFrameType::CloseFrameType + * + * @param context Context for the operation. + * + * @returns a tuple containing the status code and string. + */ + virtual std::pair GetCloseSocketInformation( + Azure::Core::Context const& context) override; + /** * @brief Send a frame of data to the remote node. * @@ -84,8 +99,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief frameType Frame type sent to the server, Text or Binary. * @brief frameData Frame data to be sent to the server. */ - virtual void SendFrame(WebSocketFrameType, std::vector, Azure::Core::Context const&) - override; + virtual void SendFrame( + WebSocketFrameType, + std::vector const&, + Azure::Core::Context const&) override; + + virtual std::vector ReceiveFrame( + WebSocketFrameType& frameType, + Azure::Core::Context const&) override; // Non-Native WebSocket support. /** diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index 9f5213dc48..8182903238 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -89,13 +89,4 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return static_cast(this)->shared_from_this(); } - std::shared_ptr WebSocketResult::AsContinuationFrame() - { - if (ResultType != WebSocketResultType::ContinuationReceived) - { - throw std::logic_error("Cannot cast to ContinuationReceived."); - } - return static_cast(this)->shared_from_this(); - } - }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 84a0a09e6b..e248ff2758 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -149,6 +149,16 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return m_chosenProtocol; } + void WebSocketImplementation::AddHeader(std::string const& header, std::string const& headerValue) + { + std::shared_lock lock(m_stateMutex); + if (m_state != SocketState::Closed && m_state != SocketState::Invalid) + { + throw std::runtime_error("AddHeader can only be called on closed sockets."); + } + m_headers.emplace(std::make_pair(header, headerValue)); + } + void WebSocketImplementation::Close(Azure::Core::Context const& context) { std::unique_lock lock(m_stateMutex); @@ -165,7 +175,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Closing; if (m_transport->NativeWebsocketSupport()) { - m_transport->CloseSocket(0, "", context); + m_transport->CloseSocket( + static_cast(WebSocketErrorCode::EndpointDisappearing), "", context); } else { @@ -184,6 +195,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Unexpected result type received during close()."); } } + // Close the socket - after this point, the m_transport is invalid. m_transport->Close(); m_state = SocketState::Closed; } @@ -221,15 +233,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Unexpected result type received during close()."); } } + // Close the socket - after this point, the m_transport is invalid. m_transport->Close(); m_state = SocketState::Closed; } - void WebSocketImplementation::AddHeader(std::string const& header, std::string const& headerValue) - { - m_headers.emplace(std::make_pair(header, headerValue)); - } void WebSocketImplementation::SendFrame( std::string const& textFrame, bool isFinalFrame, @@ -240,13 +249,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::runtime_error("Socket is not open."); } + std::vector utf8text(textFrame.begin(), textFrame.end()); if (m_transport->NativeWebsocketSupport()) { - throw std::runtime_error("Not implemented"); + m_transport->SendFrame( + (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeText + : WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment), + utf8text, + context); } else { - std::vector utf8text(textFrame.begin(), textFrame.end()); std::vector sendFrame = EncodeFrame(SocketOpcode::TextFrame, m_options.EnableMasking, isFinalFrame, utf8text); @@ -267,7 +280,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } if (m_transport->NativeWebsocketSupport()) { - throw std::runtime_error("Not implemented"); + m_transport->SendFrame( + (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeBinary + : WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment), + binaryFrame, + context); } else { @@ -296,18 +313,35 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } if (m_transport->NativeWebsocketSupport()) { - throw std::runtime_error("Not implemented"); + WebSocketTransport::WebSocketFrameType frameType; + std::vector payload = m_transport->ReceiveFrame(frameType, context); + switch (frameType) + { + case WebSocketTransport::WebSocketFrameType::FrameTypeBinary: + return std::make_shared(true, payload.data(), payload.size()); + case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: + return std::make_shared(false, payload.data(), payload.size()); + case WebSocketTransport::WebSocketFrameType::FrameTypeText: + return std::make_shared(true, payload.data(), payload.size()); + case WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment: + return std::make_shared(false, payload.data(), payload.size()); + case WebSocketTransport::WebSocketFrameType::FrameTypeClosed: { + auto closeResult = m_transport->GetCloseSocketInformation(context); + return std::make_shared(closeResult.first, closeResult.second); + } + default: + throw std::runtime_error("Unexpected frame type received."); + } } else { - std::vector frameData; SocketOpcode opcode; bool isFinal = false; bool isMasked = false; uint64_t payloadLength; std::array maskKey{}; - frameData = DecodeFrame( + std::vector frameData = DecodeFrame( m_bufferedStreamReader, opcode, payloadLength, isFinal, isMasked, maskKey, context); if (isMasked) @@ -325,9 +359,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names switch (opcode) { case SocketOpcode::BinaryFrame: + m_currentMessageType = SocketMessageType::Binary; return std::make_shared( isFinal, frameData.data(), frameData.size()); case SocketOpcode::TextFrame: { + m_currentMessageType = SocketMessageType::Text; return std::make_shared( isFinal, frameData.data(), frameData.size()); } @@ -341,8 +377,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names errorCode |= (frameData[0] << 8) & 0xff00; errorCode |= (frameData[1] & 0x00ff); - // Update our state to be closed once we've received a closed frame. We only need to do - // this if our state is not currently locked. + // Update our state to be closed once we've received a closed frame. We only need to + // do this if our state is not currently locked. if (!stateIsLocked) { lock.unlock(); @@ -350,17 +386,36 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Closed; } return std::make_shared( - errorCode, - frameData.data() + sizeof(uint16_t), - frameData.size() - sizeof(uint16_t)); + errorCode, std::string(frameData.begin() + 2, frameData.end())); } case SocketOpcode::Ping: case SocketOpcode::Pong: __debugbreak(); break; case SocketOpcode::Continuation: - return std::make_shared( - isFinal, frameData.data(), frameData.size()); + if (m_currentMessageType == SocketMessageType::Text) + { + if (isFinal) + { + m_currentMessageType = SocketMessageType::Unknown; + } + return std::make_shared( + isFinal, frameData.data(), frameData.size()); + } + else if (m_currentMessageType == SocketMessageType::Binary) + { + if (isFinal) + { + m_currentMessageType = SocketMessageType::Unknown; + } + return std::make_shared( + isFinal, frameData.data(), frameData.size()); + } + else + { + throw std::runtime_error("Unknown message type and received continuation opcode"); + } + break; default: throw std::runtime_error("Unknown opcode received."); } @@ -368,7 +423,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } return std::shared_ptr(); - context; } std::vector WebSocketImplementation::EncodeFrame( @@ -533,5 +587,4 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::generate(begin(rv), end(rv), std::ref(randomEngine)); return rv; } - }}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 8e9b85ff22..7cabdb8ea5 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -62,6 +62,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Pong = 0x0a }; + /** + * Indicates the type of the message currently being processed. Used when processing + * Continuation Opcode frames. + */ + enum class SocketMessageType : int + { + Unknown, + Text, + Binary, + }; + // Implement a buffered stream reader class BufferedStreamReader { std::shared_ptr m_transport; @@ -277,10 +288,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketState m_state{SocketState::Invalid}; - std::vector GenerateRandomKey() - { - return GenerateRandomBytes(16); - }; + std::vector GenerateRandomKey() { return GenerateRandomBytes(16); }; void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); Azure::Core::Url m_remoteUrl; WebSocketOptions m_options; @@ -288,6 +296,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::string m_chosenProtocol; std::shared_ptr m_transport; BufferedStreamReader m_bufferedStreamReader; + SocketMessageType m_currentMessageType{SocketMessageType::Unknown}; std::mutex m_transportMutex; std::shared_mutex m_stateMutex; diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 8bcad260b8..4239545523 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -376,8 +376,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( if (m_options.EnableWebSocketUpgrade) { - int option; - if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, &option, 0)) + if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) { GetErrorAndThrow("Error while Enabling WebSocket upgrade."); } diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 934130a3a0..8c6c2c9c77 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -21,6 +21,7 @@ #include #include // for WSAPoll(); #endif +#include namespace Azure { namespace Core { namespace Http { namespace WebSockets { @@ -46,16 +47,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { void WinHttpWebSocketTransport::CompleteUpgrade() {} - // Close the WebSocket operation cleanly. Does not specify - void WinHttpWebSocketTransport::Close() - { - auto err = WinHttpWebSocketClose( - m_socketHandle.get(), WINHTTP_WEB_SOCKET_ENDPOINT_TERMINATED_CLOSE_STATUS, 0, 0); - if (err != 0) - { - GetErrorAndThrow("WinHttpWebSocketClose() failed", err); - } - } + /** + * @brief Close the WebSocket cleanly. + */ + void WinHttpWebSocketTransport::Close() { m_socketHandle.reset(); } // Native WebSocket support methods. /** @@ -78,12 +73,57 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { auto err = WinHttpWebSocketClose( m_socketHandle.get(), status, - reinterpret_cast(const_cast(disconnectReason.c_str())), + disconnectReason.empty() + ? nullptr + : reinterpret_cast(const_cast(disconnectReason.c_str())), static_cast(disconnectReason.size())); if (err != 0) { GetErrorAndThrow("WinHttpWebSocketClose() failed", err); } + + context.ThrowIfCancelled(); + + // Make sure that the server responds gracefully to the close request. + auto closeInformation = GetCloseSocketInformation(context); + + // The server should return the same status we sent. + if (closeInformation.first != status) + { + throw std::runtime_error( + "Close status mismatch, got " + std::to_string(closeInformation.first) + " expected " + + std::to_string(status)); + } + } + /** + * @brief Retrieve the information associated with a WebSocket close response. + * + * Should only be called when a Receive operation returns WebSocketFrameType::CloseFrameType + * + * @param context Context for the operation. + * + * @returns a tuple containing the status code and string. + */ + std::pair WinHttpWebSocketTransport::GetCloseSocketInformation( + Azure::Core::Context const& context) + { + context.ThrowIfCancelled(); + uint16_t closeStatus = 0; + char closeReason[WINHTTP_WEB_SOCKET_MAX_CLOSE_REASON_LENGTH]{}; + DWORD closeReasonLength; + std::lock_guard lock(m_receiveMutex); + + auto err = WinHttpWebSocketQueryCloseStatus( + m_socketHandle.get(), + &closeStatus, + closeReason, + WINHTTP_WEB_SOCKET_MAX_CLOSE_REASON_LENGTH, + &closeReasonLength); + if (err != 0) + { + GetErrorAndThrow("WinHttpGetCloseStatus() failed", err); + } + return std::make_pair(closeStatus, std::string(closeReason)); } /** @@ -96,11 +136,88 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief frameData Frame data to be sent to the server. */ void WinHttpWebSocketTransport::SendFrame( - WebSocketFrameType, - std::vector, - Azure::Core::Context const&) + WebSocketFrameType frameType, + std::vector const& frameData, + Azure::Core::Context const& context) { - throw std::runtime_error("Not implemented"); + context.ThrowIfCancelled(); + WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; + switch (frameType) + { + case WebSocketFrameType::FrameTypeText: + bufferType = WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE; + break; + case WebSocketFrameType::FrameTypeBinary: + bufferType = WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE; + break; + case WebSocketFrameType::FrameTypeBinaryFragment: + bufferType = WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE; + break; + case WebSocketFrameType::FrameTypeTextFragment: + bufferType = WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE; + break; + default: + throw std::runtime_error("Unknown frame type."); + break; + } + // Lock the socket to prevent concurrent writes. WinHTTP gets annoyed if + // there are multiple WinHttpWebSocketSend requests outstanding. + std::lock_guard lock(m_sendMutex); + auto err = WinHttpWebSocketSend( + m_socketHandle.get(), + bufferType, + reinterpret_cast(const_cast(frameData.data())), + static_cast(frameData.size())); + if (err != 0) + { + GetErrorAndThrow("WinHttpWebSocketSend() failed", err); + } + } + + std::vector WinHttpWebSocketTransport::ReceiveFrame( + WebSocketFrameType& frameTypeReceived, + Azure::Core::Context const& context) + { + WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; + DWORD bufferBytesRead; + std::vector buffer(128); + context.ThrowIfCancelled(); + std::lock_guard lock(m_receiveMutex); + + auto err = WinHttpWebSocketReceive( + m_socketHandle.get(), + reinterpret_cast(buffer.data()), + static_cast(buffer.size()), + &bufferBytesRead, + &bufferType); + if (err != 0 && err != ERROR_INSUFFICIENT_BUFFER) + { + GetErrorAndThrow("WinHttpWebSocketReceive() failed", err); + } + buffer.resize(bufferBytesRead); + + switch (bufferType) + { + case WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE: + frameTypeReceived = WebSocketFrameType::FrameTypeText; + break; + case WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE: + frameTypeReceived = WebSocketFrameType::FrameTypeBinary; + break; + case WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE: + frameTypeReceived = WebSocketFrameType::FrameTypeBinaryFragment; + break; + case WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE: + frameTypeReceived = WebSocketFrameType::FrameTypeTextFragment; + break; + case WINHTTP_WEB_SOCKET_CLOSE_BUFFER_TYPE: + frameTypeReceived = WebSocketFrameType::FrameTypeClosed; + break; + default: + throw std::runtime_error("Unknown frame type."); + break; + } + return buffer; } }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 11623327b5..ad4dbb17f8 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -36,6 +36,7 @@ TEST(WebSocketTests, OpenSimpleSocket) TEST(WebSocketTests, OpenAndCloseSocket) { + if (false) { WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000/openclosetest")); defaultSocket.AddHeader("newHeader", "headerValue"); @@ -104,12 +105,20 @@ template void EchoRandomData(WebSocket& socket) socket.SendFrame(sendData, true); - auto response = socket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); - auto binaryResult = response->AsBinaryFrame(); + std::vector receiveData; + + std::shared_ptr response; + do + { + response = socket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto binaryResult = response->AsBinaryFrame(); + receiveData.insert(receiveData.end(), binaryResult->Data.begin(), binaryResult->Data.end()); + } while (!response->IsFinalFrame); + // Make sure we get back the data we sent in the echo request. - EXPECT_EQ(sendData.size(), binaryResult->Data.size()); - EXPECT_EQ(sendData, binaryResult->Data); + EXPECT_EQ(sendData.size(), receiveData.size()); + EXPECT_EQ(sendData, receiveData); } TEST(WebSocketTests, VariableSizeEcho) @@ -179,7 +188,6 @@ TEST(WebSocketTests, MultiThreadedTestOnSingleSocket) constexpr size_t testDataLength = 30000; constexpr auto testDuration = 10s; - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); testSocket.Open(); @@ -196,37 +204,44 @@ TEST(WebSocketTests, MultiThreadedTestOnSingleSocket) threads.push_back(std::thread([&]() { std::chrono::time_point startTime = std::chrono::steady_clock::now(); - - do + try { - size_t i = iterationCount++; - std::vector sendData - = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(100); + do { - if (i < testData.size()) + size_t i = iterationCount++; + std::vector sendData + = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(100); { - EXPECT_EQ(0, testData[i].size()); - testData[i] = sendData; + if (i < testData.size()) + { + EXPECT_EQ(0, testData[i].size()); + testData[i] = sendData; + } } - } - testSocket.SendFrame(sendData, true); + testSocket.SendFrame(sendData, true); - auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); - auto binaryResult = response->AsBinaryFrame(); - // Make sure we get back the data we sent in the echo request. - EXPECT_EQ(sendData.size(), binaryResult->Data.size()); - { - // There is no ordering expectation on the results, so we just remember the data - // as it comes in. We'll make sure we received everything later on. - if (i < receivedData.size()) + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto binaryResult = response->AsBinaryFrame(); + // Make sure we get back the data we sent in the echo request. + EXPECT_EQ(sendData.size(), binaryResult->Data.size()); { - EXPECT_EQ(0, receivedData[i].size()); - receivedData[i] = binaryResult->Data; + // There is no ordering expectation on the results, so we just remember the data + // as it comes in. We'll make sure we received everything later on. + if (i < receivedData.size()) + { + EXPECT_EQ(0, receivedData[i].size()); + receivedData[i] = binaryResult->Data; + } } - } - } while (std::chrono::steady_clock::now() - startTime < testDuration); + } while (std::chrono::steady_clock::now() - startTime < testDuration); + } + catch (std::exception const& ex) + { + GTEST_LOG_(ERROR) << "Exception: " << ex.what() << std::endl; + EXPECT_TRUE(false); + } })); } // std::this_thread::sleep_for(10s); @@ -249,7 +264,6 @@ TEST(WebSocketTests, MultiThreadedTestOnSingleSocket) // because we can't account for everything sent. if (iterationCount <= testDataLength) { - // Compare testData and receivedData to ensure every element in testData is in receivedData // and every element in receivedData is in testData. // @@ -346,12 +360,11 @@ class LibWebSocketIncrementProtocol { if (work->ResultType == WebSocketResultType::PeerClosed) { auto peerClose = work->AsPeerCloseFrame(); - GTEST_LOG_(INFO) << "Peer closed. Remote Code: " << peerClose->RemoteStatusCode; - if (peerClose->BodyStream.Length() != 0) + GTEST_LOG_(INFO) << "Peer closed. Remote Code: " << std::dec << peerClose->RemoteStatusCode + << " (0x" << std::hex << peerClose->RemoteStatusCode << ")" << std::endl; + if (!peerClose->RemoteCloseReason.empty()) { - auto closeBody = peerClose->BodyStream.ReadToEnd(); - std::string closeText(closeBody.begin(), closeBody.end()); - GTEST_LOG_(INFO) << " Peer Closed Data: " << closeText; + GTEST_LOG_(INFO) << " Peer Closed Data: " << peerClose->RemoteCloseReason; } GTEST_LOG_(INFO) << std::endl; return; @@ -383,26 +396,20 @@ class LibWebSocketStatus { // The server should have chosen the lws-status protocol since it doesn't understand the other // protocols. EXPECT_EQ("lws-status", serverSocket.GetChosenProtocol()); - auto lwsStatus = serverSocket.ReceiveFrame(); - if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) - { - __debugbreak(); - } - EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); - auto textFrame = lwsStatus->AsTextFrame(); - std::string returnValue = textFrame->Text; - bool isFinalFrame = textFrame->IsFinalFrame; - while (!isFinalFrame) + std::string returnValue; + std::shared_ptr lwsStatus; + do { + lwsStatus = serverSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::ContinuationReceived, lwsStatus->ResultType); - auto continuation = lwsStatus->AsContinuationFrame(); - returnValue.insert( - returnValue.end(), - continuation->ContinuationData.begin(), - continuation->ContinuationData.end()); - isFinalFrame = continuation->IsFinalFrame; - } + if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) + { + __debugbreak(); + } + EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); + auto textFrame = lwsStatus->AsTextFrame(); + returnValue.insert(returnValue.end(), textFrame->Text.begin(), textFrame->Text.end()); + } while (!lwsStatus->IsFinalFrame); serverSocket.Close(); return returnValue; } From 8cd1e7cfc925c25530b642d864ba77fe39ecc460 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:09:59 -0700 Subject: [PATCH 007/149] CI fixes --- .../websockets/curl_websockets_transport.hpp | 20 +++++++++---------- .../azure/core/http/websockets/websockets.hpp | 2 +- .../http/websockets/websockets_transport.hpp | 7 ++++--- .../win_http_websockets_transport.hpp | 6 +++--- sdk/core/azure-core/src/http/curl/curl.cpp | 4 ++-- .../src/http/curl/curl_session_private.hpp | 2 +- .../src/http/websockets/websocketsimpl.cpp | 3 ++- .../src/http/winhttp/win_http_websockets.cpp | 4 ++-- .../azure-core/test/ut/websocket_test.cpp | 2 +- 9 files changed, 26 insertions(+), 24 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 94dd4d55cd..bb6ef0efb0 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -50,7 +50,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Indicates if the transports supports native websockets or not. * - * @detail For the CURL websocket transport, the transport does NOT support native websockets - + * @details For the CURL websocket transport, the transport does NOT support native websockets - * it is the responsibility of the client of the WebSocketTransport to format WebSocket protocol * elements. */ @@ -70,10 +70,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @detail Not implemented for CURL websockets because CURL does not support native websockets. * - * @param status Status value to be sent to the remote node. Application defined. - * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. - * @param context Context for the operation. - * */ virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override { @@ -96,8 +92,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @detail Not implemented for CURL websockets because CURL does not support native websockets. * - * @brief frameType Frame type sent to the server, Text or Binary. - * @brief frameData Frame data to be sent to the server. */ virtual void SendFrame( WebSocketFrameType, @@ -112,9 +106,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @detail Not implemented for CURL websockets because CURL does not support native websockets. * - * @brief buffer data received from the server. - * @brief bufferSize size of buffer. - * @brief bytesRead number of bytes read from server. */ virtual std::vector ReceiveFrame(WebSocketFrameType&, Azure::Core::Context const&) override @@ -127,6 +118,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief This function is used when working with streams to pull more data from the wire. * Function will try to keep pulling data from socket until the buffer is all written or until * there is no more data to get from the socket. + * + * @param buffer Buffer to fill with data. + * @param bufferSize Size of buffer. + * @param context Context to control the request lifetime. + * + * @returns Buffer data received. * */ virtual size_t ReadFromSocket(uint8_t* buffer, size_t bufferSize, Context const& context) @@ -135,6 +132,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief This method will use libcurl socket to write all the bytes from buffer. * + * @param buffer Buffer to send. + * @param bufferSize Number of bytes to write. + * @param context Context for the operation. */ virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) override; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index a80a0c86c3..61307f9494 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -102,7 +102,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Enable masking for this WebSocket. * - * @detail Masking is needed to block [certain infrastructure + * @details Masking is needed to block [certain infrastructure * attacks](https://www.rfc-editor.org/rfc/rfc6455.html#section-10.3) and is strongly * recommended. */ diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index db5e753365..8682667d62 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -61,7 +61,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Complete the WebSocket upgrade. * - * @detail Called by the WebSocket client after the HTTP server responds with a + * @details Called by the WebSocket client after the HTTP server responds with a * SwitchingProtocols response. This method performs whatever operations are needed to * transfer the protocol from HTTP to WebSockets. */ @@ -105,8 +105,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Send a frame of data to the remote node. * - * @brief frameType Frame type sent to the server, Text or Binary. - * @brief frameData Frame data to be sent to the server. + * @param frameType Frame type sent to the server, Text or Binary. + * @param frameData Frame data to be sent to the server. + * @param context Context for the operation. */ virtual void SendFrame( WebSocketFrameType frameType, diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 7b3fe8eefe..b24379ab92 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -53,7 +53,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Indicates if the transports supports native websockets or not. * - * @detail For the WinHTTP websocket transport, the transport supports native websockets. + * @details For the WinHTTP websocket transport, the transport supports native websockets. */ virtual bool NativeWebsocketSupport() override { return true; } @@ -69,7 +69,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * * @param status Status value to be sent to the remote node. Application defined. * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. @@ -93,7 +93,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Send a frame of data to the remote node. * - * @detail Not implemented for CURL websockets because CURL does not support native + * @details Not implemented for CURL websockets because CURL does not support native * websockets. * * @brief frameType Frame type sent to the server, Text or Binary. diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 489c1bb2f0..a7a6bb8956 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -429,7 +429,7 @@ CURLcode CurlSession::Perform(Context const& context) return result; } -std::unique_ptr&& CurlSession::GetUpgradedConnection() +std::unique_ptr CurlSession::GetUpgradedConnection() { if (m_connectionUpgraded) { @@ -437,7 +437,7 @@ std::unique_ptr&& CurlSession::GetUpgradedConnection() } else { - return std::move(std::unique_ptr()); + return nullptr; } } diff --git a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp index e5007663ae..a94552dabe 100644 --- a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp @@ -430,7 +430,7 @@ namespace Azure { namespace Core { namespace Http { * * @return The network connection, or null if the connection was not upgraded. */ - std::unique_ptr&& GetUpgradedConnection(); + std::unique_ptr GetUpgradedConnection(); }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index e248ff2758..867e2cb156 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { @@ -552,7 +553,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names maskKey[2] = streamReader.ReadByte(context); maskKey[3] = streamReader.ReadByte(context); }; - return streamReader.ReadBytes(payloadLength, context); + return streamReader.ReadBytes(static_cast(payloadLength), context); } // Verify the Sec-WebSocket-Accept header as defined in RFC 6455 Section 1.3, which defines the diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 8c6c2c9c77..65fd1c9dd5 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -56,7 +56,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * * @param status Status value to be sent to the remote node. Application defined. * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. @@ -129,7 +129,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Send a frame of data to the remote node. * - * @detail Not implemented for CURL websockets because CURL does not support native + * @details Not implemented for CURL websockets because CURL does not support native * websockets. * * @brief frameType Frame type sent to the server, Text or Binary. diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index ad4dbb17f8..13abae67a0 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -404,7 +404,7 @@ class LibWebSocketStatus { lwsStatus = serverSocket.ReceiveFrame(); if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) { - __debugbreak(); + throw std::runtime_error("Expected text frame"); } EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); auto textFrame = lwsStatus->AsTextFrame(); From 4ff6584b10fa4a5ac1165c6582536869dd60cc2c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:12:43 -0700 Subject: [PATCH 008/149] cspell --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 2 +- sdk/core/azure-core/test/ut/websocket_test.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 867e2cb156..b3b12892fe 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -34,7 +34,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) WinHttpTransportOptions transportOptions; - // WinHTTP overrides the Connection: Upgrade header, so disable keep alives. + // WinHTTP overrides the Connection: Upgrade header, so disable keep-alive. transportOptions.EnableWebSocketUpgrade = true; m_transport = std::make_shared( diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 13abae67a0..272acd1e54 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -7,6 +7,7 @@ #include #include #include +// cspell::words closeme flibbityflobbidy using namespace Azure::Core; using namespace Azure::Core::Http::WebSockets; From 0f509ccd0837eb2b5b9e6459a86aac9088f20022 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:18:23 -0700 Subject: [PATCH 009/149] cspell --- .vscode/cspell.json | 1 + sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 5b01ec946c..ab9350cf27 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -99,6 +99,7 @@ "pdbs", "Piotrowski", "PUCHAR", + "PVOID", "pwsh", "Ragrs", "Ragzrs", diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 7cabdb8ea5..e446b8daa5 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -270,7 +270,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names /** * @brief Decode a frame received from the websocket server. * - * @param paylod Pointer to the payload returned by the service. Note that this may be shorter + * @param payload Pointer to the payload returned by the service. Note that this may be shorter * than the full data in the response message. * @param opcode Opcode returned by the server. * @param isFinal True if this is the final message. From 20fdade32119d3991b2343f9b99f8b9c538b568c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:19:40 -0700 Subject: [PATCH 010/149] detail->details --- .../core/http/websockets/curl_websockets_transport.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index bb6ef0efb0..6ad4be13fc 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -68,7 +68,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override @@ -79,7 +79,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Retrieve the status of the close socket operation. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ std::pair GetCloseSocketInformation(const Azure::Core::Context&) override @@ -90,7 +90,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Send a frame of data to the remote node. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ virtual void SendFrame( @@ -104,7 +104,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Receive a frame of data from the remote node. * - * @detail Not implemented for CURL websockets because CURL does not support native websockets. + * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ virtual std::vector ReceiveFrame(WebSocketFrameType&, Azure::Core::Context const&) From f709bc4306e9bc8d988998251b7671b386b47303 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:30:42 -0700 Subject: [PATCH 011/149] C++15 fixes --- .../src/http/websockets/websocketsimpl.cpp | 16 ++++++++-------- .../src/http/websockets/websocketsimpl.hpp | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index b3b12892fe..e03cde8d20 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -142,7 +142,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::string const& WebSocketImplementation::GetChosenProtocol() { - std::shared_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -152,7 +152,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names void WebSocketImplementation::AddHeader(std::string const& header, std::string const& headerValue) { - std::shared_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); if (m_state != SocketState::Closed && m_state != SocketState::Invalid) { throw std::runtime_error("AddHeader can only be called on closed sockets."); @@ -162,7 +162,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names void WebSocketImplementation::Close(Azure::Core::Context const& context) { - std::unique_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); // If we're closing an already closed socket, we're done. if (m_state == SocketState::Closed) @@ -206,7 +206,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::string const& closeReason, Azure::Core::Context const& context) { - std::unique_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -245,7 +245,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context) { - std::shared_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -273,7 +273,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context) { - std::shared_lock lock(m_stateMutex); + std::lock_guard lock(m_stateMutex); if (m_state != SocketState::Open) { @@ -301,7 +301,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Azure::Core::Context const& context, bool stateIsLocked) { - std::shared_lock lock(m_stateMutex, std::defer_lock); + std::unique_lock lock(m_stateMutex, std::defer_lock); if (!stateIsLocked) { @@ -383,7 +383,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names if (!stateIsLocked) { lock.unlock(); - std::unique_lock closeLock(m_stateMutex); + std::unique_lock closeLock(m_stateMutex); m_state = SocketState::Closed; } return std::make_shared( diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index e446b8daa5..5abf73b365 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -270,8 +270,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names /** * @brief Decode a frame received from the websocket server. * - * @param payload Pointer to the payload returned by the service. Note that this may be shorter - * than the full data in the response message. + * @param streamReader Buffered stream reader to read the frame from. * @param opcode Opcode returned by the server. * @param isFinal True if this is the final message. * @param maskKey On Return, contains the contents of the mask key if the data is masked. @@ -299,6 +298,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketMessageType m_currentMessageType{SocketMessageType::Unknown}; std::mutex m_transportMutex; - std::shared_mutex m_stateMutex; + std::mutex m_stateMutex; }; }}}}} // namespace Azure::Core::Http::WebSockets::_detail From 0b4b752c52fc0fd7d7a658886742ffd96bc76fa2 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:40:54 -0700 Subject: [PATCH 012/149] CI fixes --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index e03cde8d20..d40fd717b7 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -391,7 +391,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } case SocketOpcode::Ping: case SocketOpcode::Pong: - __debugbreak(); + throw std::runtime_error("Unexpected Ping/Pong opcode received."); break; case SocketOpcode::Continuation: if (m_currentMessageType == SocketMessageType::Text) @@ -422,8 +422,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } } - - return std::shared_ptr(); } std::vector WebSocketImplementation::EncodeFrame( @@ -584,7 +582,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::random_device randomEngine; std::vector rv(vectorSize); +#if defined(_MSC_VER) #pragma warning(suppress : 4244) +#endif std::generate(begin(rv), end(rv), std::ref(randomEngine)); return rv; } From b475efa4917c3278d4c62c74bef0bf66e6e18455 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 13:51:26 -0700 Subject: [PATCH 013/149] CI changes --- .../src/http/websockets/websocketsimpl.cpp | 115 +++++++++--------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index d40fd717b7..9078f35006 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -356,70 +356,67 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names }); } // At this point, readBuffer contains the actual payload from the service. + switch (opcode) { - switch (opcode) - { - case SocketOpcode::BinaryFrame: - m_currentMessageType = SocketMessageType::Binary; - return std::make_shared( - isFinal, frameData.data(), frameData.size()); - case SocketOpcode::TextFrame: { - m_currentMessageType = SocketMessageType::Text; - return std::make_shared( - isFinal, frameData.data(), frameData.size()); - } - case SocketOpcode::Close: { - - if (frameData.size() < 2) - { - throw std::runtime_error("Close response buffer is too short."); - } - uint16_t errorCode = 0; - errorCode |= (frameData[0] << 8) & 0xff00; - errorCode |= (frameData[1] & 0x00ff); + case SocketOpcode::BinaryFrame: + m_currentMessageType = SocketMessageType::Binary; + return std::make_shared( + isFinal, frameData.data(), frameData.size()); + case SocketOpcode::TextFrame: { + m_currentMessageType = SocketMessageType::Text; + return std::make_shared(isFinal, frameData.data(), frameData.size()); + } + case SocketOpcode::Close: { - // Update our state to be closed once we've received a closed frame. We only need to - // do this if our state is not currently locked. - if (!stateIsLocked) - { - lock.unlock(); - std::unique_lock closeLock(m_stateMutex); - m_state = SocketState::Closed; - } - return std::make_shared( - errorCode, std::string(frameData.begin() + 2, frameData.end())); + if (frameData.size() < 2) + { + throw std::runtime_error("Close response buffer is too short."); } - case SocketOpcode::Ping: - case SocketOpcode::Pong: - throw std::runtime_error("Unexpected Ping/Pong opcode received."); - break; - case SocketOpcode::Continuation: - if (m_currentMessageType == SocketMessageType::Text) - { - if (isFinal) - { - m_currentMessageType = SocketMessageType::Unknown; - } - return std::make_shared( - isFinal, frameData.data(), frameData.size()); - } - else if (m_currentMessageType == SocketMessageType::Binary) + uint16_t errorCode = 0; + errorCode |= (frameData[0] << 8) & 0xff00; + errorCode |= (frameData[1] & 0x00ff); + + // Update our state to be closed once we've received a closed frame. We only need to + // do this if our state is not currently locked. + if (!stateIsLocked) + { + lock.unlock(); + std::unique_lock closeLock(m_stateMutex); + m_state = SocketState::Closed; + } + return std::make_shared( + errorCode, std::string(frameData.begin() + 2, frameData.end())); + } + case SocketOpcode::Ping: + case SocketOpcode::Pong: + throw std::runtime_error("Unexpected Ping/Pong opcode received."); + break; + case SocketOpcode::Continuation: + if (m_currentMessageType == SocketMessageType::Text) + { + if (isFinal) { - if (isFinal) - { - m_currentMessageType = SocketMessageType::Unknown; - } - return std::make_shared( - isFinal, frameData.data(), frameData.size()); + m_currentMessageType = SocketMessageType::Unknown; } - else + return std::make_shared( + isFinal, frameData.data(), frameData.size()); + } + else if (m_currentMessageType == SocketMessageType::Binary) + { + if (isFinal) { - throw std::runtime_error("Unknown message type and received continuation opcode"); + m_currentMessageType = SocketMessageType::Unknown; } - break; - default: - throw std::runtime_error("Unknown opcode received."); - } + return std::make_shared( + isFinal, frameData.data(), frameData.size()); + } + else + { + throw std::runtime_error("Unknown message type and received continuation opcode"); + } + break; + default: + throw std::runtime_error("Unknown opcode received."); } } } @@ -583,9 +580,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector rv(vectorSize); #if defined(_MSC_VER) +#pragma warning(push) #pragma warning(suppress : 4244) #endif std::generate(begin(rv), end(rv), std::ref(randomEngine)); +#if defined(_MSC_VER) +#pragma warning(pop) +#endif return rv; } }}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file From d29fc89f62a5829cd4f8d5904c3fe87eecfe8237 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 14:31:07 -0700 Subject: [PATCH 014/149] CI fixes --- .../azure-core/src/http/websockets/websocketsimpl.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 9078f35006..fb64dc06f9 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -580,12 +580,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector rv(vectorSize); #if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(suppress : 4244) +//#pragma warning(push) +//#pragma warning(suppress : 4244) #endif - std::generate(begin(rv), end(rv), std::ref(randomEngine)); + std::generate(begin(rv), end(rv), [&randomEngine]() mutable { + return static_cast(randomEngine() % UINT8_MAX); + }); #if defined(_MSC_VER) -#pragma warning(pop) +//#pragma warning(pop) #endif return rv; } From 791550bba90377edfa379255cd169e2d16d3096b Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 14:31:53 -0700 Subject: [PATCH 015/149] clang-format --- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 272acd1e54..a2153992bf 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -405,7 +405,7 @@ class LibWebSocketStatus { lwsStatus = serverSocket.ReceiveFrame(); if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) { - throw std::runtime_error("Expected text frame"); + throw std::runtime_error("Expected text frame"); } EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); auto textFrame = lwsStatus->AsTextFrame(); From 89cf5f6da6e8bc8e32b7d48eff862550671a6218 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 14:51:14 -0700 Subject: [PATCH 016/149] ci fixes --- .../src/http/websockets/websocketsimpl.cpp | 13 +++---------- sdk/core/azure-core/test/ut/websocket_test.cpp | 3 ++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index fb64dc06f9..550f781262 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -579,16 +579,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::random_device randomEngine; std::vector rv(vectorSize); -#if defined(_MSC_VER) -//#pragma warning(push) -//#pragma warning(suppress : 4244) -#endif - std::generate(begin(rv), end(rv), [&randomEngine]() mutable { - return static_cast(randomEngine() % UINT8_MAX); - }); -#if defined(_MSC_VER) -//#pragma warning(pop) -#endif + std::generate(begin(rv), end(rv), [&randomEngine]() mutable { + return static_cast(randomEngine() % UINT8_MAX); + }); return rv; } }}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index a2153992bf..e059c1ed20 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include // cspell::words closeme flibbityflobbidy using namespace Azure::Core; @@ -178,7 +179,7 @@ std::string ToHexString(std::vector const& data) std::stringstream ss; for (auto const& byte : data) { - ss << std::hex << std::setfill('0') << std::setw(2) << (int)byte; + ss << std::hex << std::setfill('0') << std::setw(2) << static_cast(byte); } return ss.str(); } From 7e4302241a9c84ecb6eae7236efcc4f28bb8d24c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 15:02:44 -0700 Subject: [PATCH 017/149] dead code handling? --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 550f781262..28d2b660f7 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -419,6 +419,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Unknown opcode received."); } } +#if !defined(_MSC_VER) + // gcc 5 doesn't seem to detect that this is dead code, so we'll just leave it here. + return nullptr; +#endif } std::vector WebSocketImplementation::EncodeFrame( From 9c69110b2e9cbd72cdec1e0225b83c3176a0ff24 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 15:28:21 -0700 Subject: [PATCH 018/149] clang_format --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 28d2b660f7..31334def04 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -420,7 +420,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } #if !defined(_MSC_VER) - // gcc 5 doesn't seem to detect that this is dead code, so we'll just leave it here. + // gcc 5 doesn't seem to detect that this is dead code, so we'll just leave it here. return nullptr; #endif } From 95d28b142d6f93aeed4218b811cbf3a923fe6398 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 15:48:29 -0700 Subject: [PATCH 019/149] clang-format --- .../azure/core/http/websockets/curl_websockets_transport.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 6ad4be13fc..b1dd062bf6 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -118,11 +118,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief This function is used when working with streams to pull more data from the wire. * Function will try to keep pulling data from socket until the buffer is all written or until * there is no more data to get from the socket. - * + * * @param buffer Buffer to fill with data. * @param bufferSize Size of buffer. * @param context Context to control the request lifetime. - * + * * @returns Buffer data received. * */ From 59da14b0161c25a5520b7761d9b16e9f29c5ed4a Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 5 Jul 2022 16:43:25 -0700 Subject: [PATCH 020/149] Trial: Added pip install websockets --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 9f27a6f855..24955a1154 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -137,6 +137,10 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" + - pwsh: | + pip install websockets + workingDirectory: build + displayName: Install Python WebSockets. - pwsh: | ctest ` -C Debug ` From d7da11c7ee28ce3f84aa16d23bc7d158e0b3be7e Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 13:16:09 -0700 Subject: [PATCH 021/149] First try at websocket server --- eng/pipelines/templates/jobs/ci.tests.yml | 21 +++++++++++++++++--- sdk/core/azure-core/test/ut/requirements.txt | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 sdk/core/azure-core/test/ut/requirements.txt diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 24955a1154..d3fc95ece7 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -138,9 +138,19 @@ jobs: Env: "$(CmakeEnvArg)" - pwsh: | - pip install websockets - workingDirectory: build - displayName: Install Python WebSockets. + python --version + pip install -r requirements.txt + workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut/ + displayName: Install Python requirements. + condition: eq(${{parameters.ServiceDirectory}}, "core") + - pwsh: | + Start-Job -ScriptBlock { + python websocket_server.py + } + workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut + condition: eq(${{parameters.ServiceDirectory}}, "core") + displayName: Launch python websocket server. + - pwsh: | ctest ` -C Debug ` @@ -151,6 +161,11 @@ jobs: -T Test workingDirectory: build displayName: Test + - pwsh: | + Stop-Job + workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut + condition: eq(${{parameters.ServiceDirectory}}, "core") + displayName: Stop python websocket server. - task: PublishTestResults@2 inputs: diff --git a/sdk/core/azure-core/test/ut/requirements.txt b/sdk/core/azure-core/test/ut/requirements.txt new file mode 100644 index 0000000000..14774b465e --- /dev/null +++ b/sdk/core/azure-core/test/ut/requirements.txt @@ -0,0 +1 @@ +websockets From a997d8dd8170c304a9a83275fe386d45b4ab0416 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 13:44:04 -0700 Subject: [PATCH 022/149] corrected condition for websocket --- eng/pipelines/templates/jobs/ci.tests.yml | 6 +++--- .../inc/azure/attestation/attestation_client_options.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index d3fc95ece7..ede8c0b753 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -142,13 +142,13 @@ jobs: pip install -r requirements.txt workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut/ displayName: Install Python requirements. - condition: eq(${{parameters.ServiceDirectory}}, "core") + condition: eq(${{parameters.ServiceDirectory}}, core) - pwsh: | Start-Job -ScriptBlock { python websocket_server.py } workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut - condition: eq(${{parameters.ServiceDirectory}}, "core") + condition: eq(${{parameters.ServiceDirectory}}, core) displayName: Launch python websocket server. - pwsh: | @@ -164,7 +164,7 @@ jobs: - pwsh: | Stop-Job workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut - condition: eq(${{parameters.ServiceDirectory}}, "core") + condition: eq(${{parameters.ServiceDirectory}}, core) displayName: Stop python websocket server. - task: PublishTestResults@2 diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp index bcd6e12fb7..186ce13be9 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp @@ -116,7 +116,7 @@ namespace Azure { namespace Security { namespace Attestation { { /** @brief Version to use when communicating with the attestation service. */ - ServiceVersion Version; + std::string ApiVersion{"2020-10-01"}; /** @brief Options sent when validating tokens received by the attestation service. */ @@ -130,9 +130,9 @@ namespace Azure { namespace Security { namespace Attestation { * the service. */ AttestationClientOptions( - ServiceVersion version = ServiceVersion::V2020_10_01, + std::string version = "2020-10-01", AttestationTokenValidationOptions const& tokenValidationOptions = {}) - : Azure::Core::_internal::ClientOptions(), Version(version), + : Azure::Core::_internal::ClientOptions(), ApiVersion(version), TokenValidationOptions(tokenValidationOptions) { } From 5d178857ee8711892b7b111acfecb95e82a2ed83 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 14:55:37 -0700 Subject: [PATCH 023/149] Checkpoint with traces --- .../azure/attestation/attestation_client_options.hpp | 2 +- .../azure-core/src/http/websockets/websocketsimpl.hpp | 4 ++++ sdk/core/azure-core/test/ut/websocket_test.cpp | 11 +++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp index 6d0865a44c..de33572400 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp @@ -110,7 +110,7 @@ namespace Azure { namespace Security { namespace Attestation { AttestationClientOptions( std::string version = "2020-10-01", AttestationTokenValidationOptions const& tokenValidationOptions = {}) - : Azure::Core::_internal::ClientOptions(), ApiVersion(version), + : Azure::Core::_internal::ClientOptions(), Version(version), TokenValidationOptions(tokenValidationOptions) { } diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 5abf73b365..82463f8111 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -6,6 +6,7 @@ #include #include #include +#include // Implementation of WebSocket protocol. namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { @@ -94,9 +95,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names uint8_t ReadByte(Azure::Core::Context const& context) { + std::cout << "BufferedStreamReader::ReadByte "; if (m_bufferPos >= m_bufferLen) { m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + std::cout << " Read " << m_bufferLen << " bytes from socket"; m_bufferPos = 0; if (m_bufferLen == 0) { @@ -104,6 +107,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return 0; } } + std::cout << "Read byte " << std::to_string(m_buffer[m_bufferPos]) << std::endl; return m_buffer[m_bufferPos++]; } uint16_t ReadShort(Azure::Core::Context const& context) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index e059c1ed20..c0949a9fa9 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -404,13 +404,12 @@ class LibWebSocketStatus { { lwsStatus = serverSocket.ReceiveFrame(); - if (lwsStatus->ResultType != WebSocketResultType::TextFrameReceived) + EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); + if (lwsStatus->ResultType == WebSocketResultType::TextFrameReceived) { - throw std::runtime_error("Expected text frame"); + auto textFrame = lwsStatus->AsTextFrame(); + returnValue.insert(returnValue.end(), textFrame->Text.begin(), textFrame->Text.end()); } - EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); - auto textFrame = lwsStatus->AsTextFrame(); - returnValue.insert(returnValue.end(), textFrame->Text.begin(), textFrame->Text.end()); } while (!lwsStatus->IsFinalFrame); serverSocket.Close(); return returnValue; @@ -422,7 +421,7 @@ TEST(WebSocketTests, LibWebSocketOrg) { LibWebSocketStatus lwsStatus; auto serverStatus = lwsStatus.GetLWSStatus(); - GTEST_LOG_(INFO) << serverStatus << std::endl; + GTEST_LOG_(INFO) << "Server status: " << serverStatus << std::endl; Azure::Core::Json::_internal::json status( Azure::Core::Json::_internal::json::parse(serverStatus)); From ad821112d97b5c56e5ef04dd238f60c2252c664c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 15:02:28 -0700 Subject: [PATCH 024/149] Try again with conditional --- eng/pipelines/templates/jobs/ci.tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ede8c0b753..1afae95bb3 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -142,13 +142,13 @@ jobs: pip install -r requirements.txt workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut/ displayName: Install Python requirements. - condition: eq(${{parameters.ServiceDirectory}}, core) + condition: eq('${{parameters.ServiceDirectory}}', 'core') - pwsh: | Start-Job -ScriptBlock { python websocket_server.py } workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut - condition: eq(${{parameters.ServiceDirectory}}, core) + condition: eq('${{parameters.ServiceDirectory}}', 'core') displayName: Launch python websocket server. - pwsh: | @@ -164,7 +164,7 @@ jobs: - pwsh: | Stop-Job workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut - condition: eq(${{parameters.ServiceDirectory}}, core) + condition: eq('${{parameters.ServiceDirectory}}', 'core') displayName: Stop python websocket server. - task: PublishTestResults@2 From 9b684a39595144606fd2c95d102e08ee8db4df38 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 15:16:01 -0700 Subject: [PATCH 025/149] updated working directory --- eng/pipelines/templates/jobs/ci.tests.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 1afae95bb3..da00294519 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -130,6 +130,13 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} TestPipeline: ${{ parameters.TestPipeline }} + - pwsh: | + python --version + pip install -r requirements.txt + workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut/ + displayName: Install Python requirements. + condition: eq('${{parameters.ServiceDirectory}}', 'core') + - template: /eng/pipelines/templates/steps/cmake-build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -137,17 +144,11 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut/ - displayName: Install Python requirements. - condition: eq('${{parameters.ServiceDirectory}}', 'core') - pwsh: | Start-Job -ScriptBlock { python websocket_server.py } - workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut + workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut condition: eq('${{parameters.ServiceDirectory}}', 'core') displayName: Launch python websocket server. @@ -163,7 +164,7 @@ jobs: displayName: Test - pwsh: | Stop-Job - workingDirectory: $(Build.SourcesDirectory)/azure-core/test/ut + workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut condition: eq('${{parameters.ServiceDirectory}}', 'core') displayName: Stop python websocket server. From b85c7631c2535e20aa90587ca5aae9c679391de7 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 15:23:42 -0700 Subject: [PATCH 026/149] Try updating Python version --- eng/pipelines/templates/jobs/ci.tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index da00294519..458db6c5fe 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -79,6 +79,11 @@ jobs: value: ${{ testEnvVar.Value }} steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: '3.6' + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: AgentImage: $(OsVmImage) From 9379b0b8fd3fda88857e2c6e2aa02dcc7d3197b1 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 15:29:53 -0700 Subject: [PATCH 027/149] Try updating Python version v2 --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 458db6c5fe..65cae011be 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -80,9 +80,9 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python 3.10' inputs: - versionSpec: '3.6' + versionSpec: '3.10' - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: From de6cfee28f7f0862148728d4fa6a6f6fb712f54d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 16:04:48 -0700 Subject: [PATCH 028/149] Parameterize websocket server --- .../templates/jobs/archetype-sdk-client.yml | 8 +++++ eng/pipelines/templates/jobs/ci.tests.yml | 29 ++++++++++++------- .../templates/stages/archetype-sdk-client.yml | 8 +++++ .../templates/stages/archetype-sdk-tests.yml | 8 +++++ sdk/core/ci.yml | 2 ++ 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 14883215a9..3501323123 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -32,6 +32,12 @@ parameters: - name: TestEnv type: object default: [] + - name: EnablePythonWebSocketServer + type: boolean + default: false + - name: PythonWebSocketServerName + type: string + default: '' jobs: - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml @@ -53,6 +59,8 @@ jobs: LineCoverageTarget: ${{ parameters.LineCoverageTarget }} BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }} TestEnv: ${{ parameters.TestEnv }} + EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} + PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} # Disable build for cpp - client - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 65cae011be..647ddfaadc 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -46,6 +46,12 @@ parameters: - name: UsePlatformContainer type: boolean default: false + - name: EnablePythonWebSocketServer + type: boolean + default: false + - name: PythonWebSocketServerName + type: string + default: '' jobs: - job: @@ -83,6 +89,7 @@ jobs: displayName: 'Use Python 3.10' inputs: versionSpec: '3.10' + condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: @@ -135,13 +142,6 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} TestPipeline: ${{ parameters.TestPipeline }} - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut/ - displayName: Install Python requirements. - condition: eq('${{parameters.ServiceDirectory}}', 'core') - - template: /eng/pipelines/templates/steps/cmake-build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -150,11 +150,18 @@ jobs: Env: "$(CmakeEnvArg)" - pwsh: | - Start-Job -ScriptBlock { + python --version + pip install -r requirements.txt + workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut/ + displayName: Install Python requirements. + condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) + + - pwsh: | + Start-Job -Name PythonServer -ScriptBlock { python websocket_server.py } workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut - condition: eq('${{parameters.ServiceDirectory}}', 'core') + condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) displayName: Launch python websocket server. - pwsh: | @@ -168,9 +175,9 @@ jobs: workingDirectory: build displayName: Test - pwsh: | - Stop-Job + Stop-Job -Name PythonServer workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut - condition: eq('${{parameters.ServiceDirectory}}', 'core') + condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) displayName: Stop python websocket server. - task: PublishTestResults@2 diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 5b4ae1ba76..3494d1fc58 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -73,6 +73,12 @@ parameters: - name: UnsupportedClouds type: string default: '' +- name: EnablePythonWebSocketServer + type: boolean + default: false +- name: PythonWebSocketServerName + type: string + default: '' stages: @@ -116,6 +122,8 @@ stages: Clouds: ${{ parameters.Clouds }} SupportedClouds: ${{ parameters.SupportedClouds }} UnsupportedClouds: ${{ parameters.UnsupportedClouds }} + EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} + PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), not(endsWith(variables['Build.DefinitionName'], ' - tests'))) }}: - template: archetype-cpp-release.yml diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 737de7b297..9ddf86ebee 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -29,6 +29,12 @@ parameters: - name: UnsupportedClouds type: string default: '' +- name: EnablePythonWebSocketServer + type: boolean + default: false +- name: PythonWebSocketServerName + type: string + default: '' stages: - ${{ each cloud in parameters.CloudConfig }}: @@ -57,3 +63,5 @@ stages: Coverage: ${{ parameters.Coverage}} CoverageReportPath: ${{ parameters.CoverageReportPath}} TimeoutInMinutes: ${{ parameters.TimeoutInMinutes}} + EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer}} + PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName}} diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 8547868d47..66513fcc2b 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -42,6 +42,8 @@ stages: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 93 BranchCoverageTarget: 55 + EnablePythonWebSocketServer: true + PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket-server.py Artifacts: - Name: azure-core Path: azure-core From a37acf26296eb940c9ce445f7fb7cc116892e3d5 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 16:20:29 -0700 Subject: [PATCH 029/149] Thread parameters to build job --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 3494d1fc58..9cb6e41d97 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -108,6 +108,8 @@ stages: ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true TestEnv: ${{ parameters.TestEnv }} + EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} + PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(parameters.LiveTestCtestRegex, '')) }}: - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml From 392be84486c20979d407f3f98baf2925f3431efe Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 6 Jul 2022 16:53:37 -0700 Subject: [PATCH 030/149] Check to ensure the websocket server is running at test start --- eng/pipelines/templates/jobs/ci.tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 647ddfaadc..456219fdcb 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -158,13 +158,14 @@ jobs: - pwsh: | Start-Job -Name PythonServer -ScriptBlock { - python websocket_server.py + python ${{parameters.PythonWebSocketServerName}} } - workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut + workingDirectory: build condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) displayName: Launch python websocket server. - pwsh: | + Get-Job ctest ` -C Debug ` -V ` @@ -175,7 +176,8 @@ jobs: workingDirectory: build displayName: Test - pwsh: | - Stop-Job -Name PythonServer + Get-Job + Stop-Job -Name PythonServer workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) displayName: Stop python websocket server. From cccb5667b6fe76b322edcd073cd209ed75f6e6b2 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 11:12:22 -0700 Subject: [PATCH 031/149] Copy websocket server to build output directory --- eng/pipelines/templates/jobs/ci.tests.yml | 5 ++++- sdk/core/azure-core/test/ut/CMakeLists.txt | 16 +++++++++++++++- sdk/core/azure-core/test/ut/websocket_server.py | 11 ++++------- sdk/core/ci.yml | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 456219fdcb..ba3b7bd209 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -152,7 +152,7 @@ jobs: - pwsh: | python --version pip install -r requirements.txt - workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut/ + workingDirectory: build displayName: Install Python requirements. condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) @@ -160,12 +160,15 @@ jobs: Start-Job -Name PythonServer -ScriptBlock { python ${{parameters.PythonWebSocketServerName}} } + Get-Job workingDirectory: build condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) displayName: Launch python websocket server. - pwsh: | + echo "Check Jobs" Get-Job + echo "Run Tests" ctest ` -C Debug ` -V ` diff --git a/sdk/core/azure-core/test/ut/CMakeLists.txt b/sdk/core/azure-core/test/ut/CMakeLists.txt index 2bf3b5ffff..2451d9153d 100644 --- a/sdk/core/azure-core/test/ut/CMakeLists.txt +++ b/sdk/core/azure-core/test/ut/CMakeLists.txt @@ -80,7 +80,8 @@ add_executable ( transport_adapter_implementation_test.cpp url_test.cpp uuid_test.cpp - "websocket_test.cpp") + websocket_test.cpp + ) if (MSVC) # Disable warnings: @@ -98,6 +99,19 @@ if (MSVC) target_compile_options(azure-core-test PUBLIC /wd26495 /wd26812 /wd6326 /wd28204 /wd28020 /wd6330 /wd4389) endif() +#add_custom_target(copy_non_source_outputs +# COMMAND ${CMAKE_COMMAND} -E copy websocket_server.py requirements.txt ${CMAKE_BINARY_DIR} +# SOURCES ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt) +#add_dependencies(azure-core-test copy_non_source_outputs) + +add_custom_command(TARGET azure-core-test POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt + COMMENT 'Copying non-source output files') + +#file(COPY_FILE requirements.txt ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt ONLY_IF_DIFFERENT) +#file(COPY_FILE websocket_server.py ${CMAKE_CURRENT_BINARY_DIR}/websocket_server.py ONLY_IF_DIFFERENT) + # Adding private headers from CORE to the tests so we can test the private APIs with no relative paths include. target_include_directories (azure-core-test PRIVATE $) diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index b28d42fd9a..256269d725 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -28,6 +28,7 @@ async def handleControlPath(websocket): await websocket.send(data) websocket + async def handleCustomPath(websocket, path:dict): print("Handle custom path", path) data : str = await websocket.recv() @@ -38,7 +39,7 @@ async def handleCustomPath(websocket, path:dict): await websocket.send(data) await websocket.close() -async def handleEcho(websocket, url): + async def handleEcho(websocket, url): while websocket.open: try: data = await websocket.recv() @@ -50,9 +51,7 @@ async def handleEcho(websocket, url): except websockets.ConnectionClosed as ex: print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") return - - - + async def handler(websocket, path : str): print("Socket handler: ", path) parsedUrl = urlparse(path) @@ -80,10 +79,8 @@ async def handler(websocket, path : str): print("Received: ", data) reply = f"Data received as: {data}!" - await websocket.send(reply) - - + async def main(): print("Starting server") async with websockets.serve(handler, "localhost", 8000): diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 66513fcc2b..5dd87c0a8a 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -43,7 +43,7 @@ stages: LineCoverageTarget: 93 BranchCoverageTarget: 55 EnablePythonWebSocketServer: true - PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket-server.py + PythonWebSocketServerName: websocket-server.py Artifacts: - Name: azure-core Path: azure-core From 7bca96eb84c6885208ff64c88da26d631f96d5c4 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 11:37:48 -0700 Subject: [PATCH 032/149] Try to find requirements.txt --- eng/pipelines/templates/jobs/ci.tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ba3b7bd209..5074a07b9b 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -151,6 +151,7 @@ jobs: - pwsh: | python --version + dir -r requirements.txt pip install -r requirements.txt workingDirectory: build displayName: Install Python requirements. From 87e60952946748f344d333a00fc9e8d87ab9f4ac Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 11:45:58 -0700 Subject: [PATCH 033/149] Diagnostics on prerequisites --- eng/pipelines/templates/jobs/ci.tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 5074a07b9b..abb3f1a430 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -151,7 +151,9 @@ jobs: - pwsh: | python --version + echo 'Search for requirements.' dir -r requirements.txt + echo 'Install requirements.' pip install -r requirements.txt workingDirectory: build displayName: Install Python requirements. From 4bae1013de6f278eb3999ca9194a9baa3be69f6f Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 12:14:06 -0700 Subject: [PATCH 034/149] Diagnostics on prerequisites --- eng/pipelines/templates/jobs/ci.tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index abb3f1a430..3edcaae005 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -152,6 +152,7 @@ jobs: - pwsh: | python --version echo 'Search for requirements.' + get-location dir -r requirements.txt echo 'Install requirements.' pip install -r requirements.txt From fa67d1e5f856ca9017b4cae0edbf9d319ebfee1c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 12:54:29 -0700 Subject: [PATCH 035/149] Copy websocket server to build output directory --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 3edcaae005..38e5bd2fcd 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -156,7 +156,7 @@ jobs: dir -r requirements.txt echo 'Install requirements.' pip install -r requirements.txt - workingDirectory: build + workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) From 8fa2eae1eb3eb1868958a03eeeebf9872f6bf4f6 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:05:34 -0700 Subject: [PATCH 036/149] Yet another try at the CI pipeline --- eng/pipelines/templates/jobs/ci.tests.yml | 29 +++++++++---------- .../azure-core/test/ut/websocket_server.py | 2 +- .../azure-core/test/ut/websocket_test.cpp | 25 +++++++++++----- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 38e5bd2fcd..707114f438 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -160,14 +160,20 @@ jobs: displayName: Install Python requirements. condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - - pwsh: | - Start-Job -Name PythonServer -ScriptBlock { - python ${{parameters.PythonWebSocketServerName}} - } - Get-Job - workingDirectory: build - condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - displayName: Launch python websocket server. + - ${{ if (eq(parameters.EnablePythonWebSocketServer, true))}}: + - pwsh: | + Start-Process 'python.exe' + -ArgumentList '${{parameters.PythonWebSocketServerName}}' + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + workingDirectory: ${{parameters.rootFolder}} + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + displayName: Launch python websocket server (Windows). + + - bash: | + nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & + workingDirectory: ${{parameters.rootFolder}} + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + displayName: Launch python websocket server (Linux). - pwsh: | echo "Check Jobs" @@ -182,13 +188,6 @@ jobs: -T Test workingDirectory: build displayName: Test - - pwsh: | - Get-Job - Stop-Job -Name PythonServer - workingDirectory: $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut - condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - displayName: Stop python websocket server. - - task: PublishTestResults@2 inputs: testResultsFormat: cTest diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 256269d725..da548c034c 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -39,7 +39,7 @@ async def handleCustomPath(websocket, path:dict): await websocket.send(data) await websocket.close() - async def handleEcho(websocket, url): +async def handleEcho(websocket, url): while websocket.open: try: data = await websocket.recv() diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index c0949a9fa9..3649f002b0 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -14,7 +14,16 @@ using namespace Azure::Core; using namespace Azure::Core::Http::WebSockets; using namespace std::chrono_literals; -TEST(WebSocketTests, CreateSimpleSocket) +class WebSocketTests : public testing::Test { +private: +protected: + // Create + virtual void SetUp() override + { + } +}; + +TEST_F(WebSocketTests, CreateSimpleSocket) { { WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000")); @@ -22,7 +31,7 @@ TEST(WebSocketTests, CreateSimpleSocket) } } -TEST(WebSocketTests, OpenSimpleSocket) +TEST_F(WebSocketTests, OpenSimpleSocket) { { WebSocketOptions options; @@ -36,7 +45,7 @@ TEST(WebSocketTests, OpenSimpleSocket) } } -TEST(WebSocketTests, OpenAndCloseSocket) +TEST_F(WebSocketTests, OpenAndCloseSocket) { if (false) { @@ -65,7 +74,7 @@ TEST(WebSocketTests, OpenAndCloseSocket) } } -TEST(WebSocketTests, SimpleEcho) +TEST_F(WebSocketTests, SimpleEcho) { { WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); @@ -123,7 +132,7 @@ template void EchoRandomData(WebSocket& socket) EXPECT_EQ(sendData, receiveData); } -TEST(WebSocketTests, VariableSizeEcho) +TEST_F(WebSocketTests, VariableSizeEcho) { { WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); @@ -155,7 +164,7 @@ TEST(WebSocketTests, VariableSizeEcho) } } -TEST(WebSocketTests, CloseDuringEcho) +TEST_F(WebSocketTests, CloseDuringEcho) { { WebSocket testSocket(Azure::Core::Url("http://localhost:8000/closeduringecho")); @@ -184,7 +193,7 @@ std::string ToHexString(std::vector const& data) return ss.str(); } -TEST(WebSocketTests, MultiThreadedTestOnSingleSocket) +TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; constexpr size_t testDataLength = 30000; @@ -416,7 +425,7 @@ class LibWebSocketStatus { } }; -TEST(WebSocketTests, LibWebSocketOrg) +TEST_F(WebSocketTests, LibWebSocketOrg) { { LibWebSocketStatus lwsStatus; From 6741a78ab4543c4d7291402c651126812cf9dfb0 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:09:10 -0700 Subject: [PATCH 037/149] Yet another try at the CI pipeline 2 --- eng/pipelines/templates/jobs/ci.tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 707114f438..55f78163f8 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -160,7 +160,7 @@ jobs: displayName: Install Python requirements. condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - - ${{ if (eq(parameters.EnablePythonWebSocketServer, true))}}: + - ${{ if eq(parameters.EnablePythonWebSocketServer, true) }}: - pwsh: | Start-Process 'python.exe' -ArgumentList '${{parameters.PythonWebSocketServerName}}' @@ -176,9 +176,6 @@ jobs: displayName: Launch python websocket server (Linux). - pwsh: | - echo "Check Jobs" - Get-Job - echo "Run Tests" ctest ` -C Debug ` -V ` From 530a92fcd2199981c4e6edfe511e3ce1d5b77203 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:10:03 -0700 Subject: [PATCH 038/149] Update sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp Co-authored-by: Rick Winter --- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index ad3c028584..e39b63193c 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -18,7 +18,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal { /** - * @brief Defines #Sha256Hash. + * @brief Defines #Sha1Hash. * */ class Sha1Hash final : public Azure::Core::Cryptography::Hash { From 05eba24cca71d4ceca66068ad7147b212c4f1684 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:10:12 -0700 Subject: [PATCH 039/149] Update sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp Co-authored-by: Rick Winter --- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index e39b63193c..4cb0ab4eb4 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -24,7 +24,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal class Sha1Hash final : public Azure::Core::Cryptography::Hash { public: /** - * @brief Construct a default instance of #Sha256Hash. + * @brief Construct a default instance of #Sha1Hash. * */ Sha1Hash(); From 802d4c126e5333352078998d7f500c0b746c6378 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:10:21 -0700 Subject: [PATCH 040/149] Update sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp Co-authored-by: Rick Winter --- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index 4cb0ab4eb4..df82cfbd90 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -30,7 +30,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal Sha1Hash(); /** - * @brief Cleanup any state when destroying the instance of #Sha256Hash. + * @brief Cleanup any state when destroying the instance of #Sha1Hash. * */ ~Sha1Hash() {} From af0f8d98897b060f6a963c8683de7949a97fc3b9 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:10:29 -0700 Subject: [PATCH 041/149] Update sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp Co-authored-by: Rick Winter --- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index df82cfbd90..4c1172a65a 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -47,7 +47,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal * appended. * @param data The pointer to binary data to compute the hash value for. * @param length The size of the data provided. - * @return The computed SHA256 hash value corresponding to the input provided including any + * @return The computed SHA1 hash value corresponding to the input provided including any * previously appended. */ std::vector OnFinal(const uint8_t* data, size_t length) override From a3f4278be8c1977835bbea2e8f4b8543518be45c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:10:36 -0700 Subject: [PATCH 042/149] Update sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp Co-authored-by: Rick Winter --- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index 4c1172a65a..c1b8475478 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -56,7 +56,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal } /** - * @brief Used to append partial binary input data to compute the SHA256 hash in a streaming + * @brief Used to append partial binary input data to compute the SHA1 hash in a streaming * fashion. * @remark Once all the data has been added, call #Final() to get the computed hash value. * @param data The pointer to the current block of binary data that is used for hash From a1173f9ed6a825b25b2e96a881b78fc3a4632958 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:12:38 -0700 Subject: [PATCH 043/149] Update sdk/core/azure-core/src/cryptography/sha_hash.cpp Co-authored-by: Rick Winter --- sdk/core/azure-core/src/cryptography/sha_hash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/cryptography/sha_hash.cpp b/sdk/core/azure-core/src/cryptography/sha_hash.cpp index 81925ee619..95b06b7d7a 100644 --- a/sdk/core/azure-core/src/cryptography/sha_hash.cpp +++ b/sdk/core/azure-core/src/cryptography/sha_hash.cpp @@ -69,7 +69,7 @@ class SHAWithOpenSSL final : public Azure::Core::Cryptography::Hash { case SHASize::SHA1: { if (1 != EVP_DigestInit_ex(m_context, EVP_sha1(), NULL)) { - throw std::runtime_error("Crypto error while init Sha256Hash."); + throw std::runtime_error("Crypto error while initializing Sha1Hash."); } break; } From 07d554cd025e74b85dc67e57f95ffb440383b56e Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:15:58 -0700 Subject: [PATCH 044/149] another CI try --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ++-- sdk/core/ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 55f78163f8..4f0aed541f 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -165,13 +165,13 @@ jobs: Start-Process 'python.exe' -ArgumentList '${{parameters.PythonWebSocketServerName}}' -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: ${{parameters.rootFolder}} + workingDirectory: build condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) displayName: Launch python websocket server (Windows). - bash: | nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & - workingDirectory: ${{parameters.rootFolder}} + workingDirectory: build condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) displayName: Launch python websocket server (Linux). diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 5dd87c0a8a..66513fcc2b 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -43,7 +43,7 @@ stages: LineCoverageTarget: 93 BranchCoverageTarget: 55 EnablePythonWebSocketServer: true - PythonWebSocketServerName: websocket-server.py + PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket-server.py Artifacts: - Name: azure-core Path: azure-core From e4fef5153b4b3b42aa687f00b39d15d106b42b8d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:30:51 -0700 Subject: [PATCH 045/149] Try to find requirements and python server --- .vscode/cspell.json | 1 + eng/pipelines/templates/jobs/ci.tests.yml | 35 ++++++++++++----------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index b2e6beece5..f4dc3a9d1a 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -84,6 +84,7 @@ "ncus", "Niels", "nlohmann", + "nohup", "nostd", "noclean", "NOCLOSE", diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 4f0aed541f..48d73c7865 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -85,12 +85,6 @@ jobs: value: ${{ testEnvVar.Value }} steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' - condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: AgentImage: $(OsVmImage) @@ -149,18 +143,25 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - pwsh: | - python --version - echo 'Search for requirements.' - get-location - dir -r requirements.txt - echo 'Install requirements.' - pip install -r requirements.txt - workingDirectory: build/sdk/core/azure-core/test/ut - displayName: Install Python requirements. - condition: eq(true, ${{parameters.EnablePythonWebSocketServer}}) - - ${{ if eq(parameters.EnablePythonWebSocketServer, true) }}: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + + - pwsh: | + python --version + dir -r + pip install -r requirements.txt + workingDirectory: build/sdk/core/azure-core/test/ut + displayName: Install Python requirements. + + - pwsh: | + dir -r + workingDirectory: build + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + displayName: Dir -r Launch python websocket server (Windows). + - pwsh: | Start-Process 'python.exe' -ArgumentList '${{parameters.PythonWebSocketServerName}}' From 4aa0751a9e36cc59ecbd3de1ab9fcecd5e958f61 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 14:46:23 -0700 Subject: [PATCH 046/149] Correct multi-line pwsh syntax --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 48d73c7865..d0488364a8 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -163,8 +163,8 @@ jobs: displayName: Dir -r Launch python websocket server (Windows). - pwsh: | - Start-Process 'python.exe' - -ArgumentList '${{parameters.PythonWebSocketServerName}}' + Start-Process 'python.exe' ` + -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) From 1b8cebe788a43a9aee57bfb896b632018630c8bb Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 15:06:48 -0700 Subject: [PATCH 047/149] Fixed name of websocket server --- eng/pipelines/templates/jobs/ci.tests.yml | 6 ------ sdk/core/ci.yml | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index d0488364a8..9a2b8888c3 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -156,12 +156,6 @@ jobs: workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. - - pwsh: | - dir -r - workingDirectory: build - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Dir -r Launch python websocket server (Windows). - - pwsh: | Start-Process 'python.exe' ` -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 66513fcc2b..eb50714b74 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -43,7 +43,7 @@ stages: LineCoverageTarget: 93 BranchCoverageTarget: 55 EnablePythonWebSocketServer: true - PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket-server.py + PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket_server.py Artifacts: - Name: azure-core Path: azure-core From db9c7ac7b1a610dcd8eba35921c5274cdd847f20 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 15:20:35 -0700 Subject: [PATCH 048/149] Skip PythonTests when not building tests --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 9a2b8888c3..ee27c4c6b3 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,7 +143,7 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - ${{ if eq(parameters.EnablePythonWebSocketServer, true) }}: + - ${{ if and(contains($(CmakeArgs), 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: From 343311669e2c70b46d41a4f607472ae70c985865 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 15:22:46 -0700 Subject: [PATCH 049/149] Skip PythonTests when not building tests2 --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ee27c4c6b3..ee874bf731 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,7 +143,7 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - ${{ if and(contains($(CmakeArgs), 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: + - ${{ if and(contains(variables['CmakeArgs'], 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: From c2d7ade09f61f2d8c6bab8cf973a47586a3b61dd Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 15:43:50 -0700 Subject: [PATCH 050/149] Skip PythonTests when not building tests3 --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ee874bf731..3939dc2149 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,7 +143,7 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - ${{ if and(contains(variables['CmakeArgs'], 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: + - ${{ if and(contains(variables.CmakeArgs, 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: From ceed005624ff147d8fd3ee401dc589801358a5eb Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 15:53:29 -0700 Subject: [PATCH 051/149] Skip PythonTests when not building tests4 --- eng/pipelines/templates/jobs/ci.tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 3939dc2149..4c92392b8f 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,7 +143,7 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - ${{ if and(contains(variables.CmakeArgs, 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, true)) }}: + - ${{ if eq(parameters.EnablePythonWebSocketServer, true) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: @@ -155,6 +155,7 @@ jobs: pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. + condition: and(succeeded(), contains(parameters['CmakeArgs'], "BUILD_TESTING=ON")) - pwsh: | Start-Process 'python.exe' ` From 71263b79952b5dc41a15b0ab3eb6ff7c80fc63fe Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:00:41 -0700 Subject: [PATCH 052/149] Skip PythonTests when not building tests5 --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 4c92392b8f..699f7064a1 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -155,7 +155,7 @@ jobs: pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. - condition: and(succeeded(), contains(parameters['CmakeArgs'], "BUILD_TESTING=ON")) + condition: and(succeeded(), contains(${{parameters.CmakeArgs}}, "BUILD_TESTING=ON")) - pwsh: | Start-Process 'python.exe' ` From e2b595658995dae9936681765e2ec300fb90aecb Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:02:14 -0700 Subject: [PATCH 053/149] Skip PythonTests when not building tests6 --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 699f7064a1..a565a016ae 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -155,7 +155,7 @@ jobs: pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. - condition: and(succeeded(), contains(${{parameters.CmakeArgs}}, "BUILD_TESTING=ON")) + condition: and(succeeded(), contains($(CmakeArgs), "BUILD_TESTING=ON")) - pwsh: | Start-Process 'python.exe' ` From 679f02cbf22749e51468e3f6502699212d01a0dc Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:08:22 -0700 Subject: [PATCH 054/149] Skip PythonTests when not building tests7 --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index a565a016ae..78db7aaea5 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -155,7 +155,7 @@ jobs: pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. - condition: and(succeeded(), contains($(CmakeArgs), "BUILD_TESTING=ON")) + condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - pwsh: | Start-Process 'python.exe' ` From c364959213527f287b13743e1fc0a1d577940e7c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:17:11 -0700 Subject: [PATCH 055/149] Another try at conditional --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 78db7aaea5..80d760f261 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,7 +143,7 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - - ${{ if eq(parameters.EnablePythonWebSocketServer, true) }}: + - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: From b435dc488c17c6d5f6712f421925b5d3279eb71d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:31:30 -0700 Subject: [PATCH 056/149] Turn off python webserver if build_testing is not on --- eng/pipelines/templates/jobs/ci.tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 80d760f261..337c7d16a1 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -151,7 +151,6 @@ jobs: - pwsh: | python --version - dir -r pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. @@ -162,13 +161,13 @@ jobs: -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) displayName: Launch python websocket server (Windows). - bash: | nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & workingDirectory: build - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) displayName: Launch python websocket server (Linux). - pwsh: | From 05e0da0ef7f73296e2c1529286a5425b4758c576 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 16:58:48 -0700 Subject: [PATCH 057/149] clang-format --- sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp | 2 +- sdk/core/azure-core/test/ut/websocket_test.cpp | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 82463f8111..437169c6d1 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -4,9 +4,9 @@ #include "azure/core/http/websockets/websockets_transport.hpp" #include "azure/core/internal/http/pipeline.hpp" #include +#include #include #include -#include // Implementation of WebSocket protocol. namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 3649f002b0..06753dae19 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -18,9 +18,7 @@ class WebSocketTests : public testing::Test { private: protected: // Create - virtual void SetUp() override - { - } + virtual void SetUp() override {} }; TEST_F(WebSocketTests, CreateSimpleSocket) From b19b42681c6f0e2e8eecf2e14ad8fa1dfd5b16a3 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 7 Jul 2022 17:48:44 -0700 Subject: [PATCH 058/149] Added coverage tests for curl websocket transport --- eng/pipelines/templates/jobs/ci.tests.yml | 10 ++++-- eng/pipelines/templates/jobs/live.tests.yml | 32 +++++++++++++++++++ .../azure-core/test/ut/websocket_test.cpp | 29 ++++++++++++++--- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 337c7d16a1..384370b45d 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -143,6 +143,9 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" + # If the pipeline is configured to enable a Python WebSocket server, then + # configure the server and launch it. Note that we only launch the server if + # we're not running tests locally. - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' @@ -165,7 +168,7 @@ jobs: displayName: Launch python websocket server (Windows). - bash: | - nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & + nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/WebSocketServer.log & workingDirectory: build condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) displayName: Launch python websocket server (Linux). @@ -190,7 +193,10 @@ jobs: publishRunAttachments: true displayName: Publish test results condition: succeededOrFailed() - + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/WebSocketServer.Log' + ArtifactName: '$(Build.SourcesDirectory)/WebSocketServer.log' - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index e5272766e3..e27091fd0e 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -126,6 +126,38 @@ jobs: Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} + # If the pipeline is configured to enable a Python WebSocket server, then + # configure the server and launch it. Note that we only launch the server if + # we're not running tests locally. + - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + + - pwsh: | + python --version + pip install -r requirements.txt + workingDirectory: build/sdk/core/azure-core/test/ut + displayName: Install Python requirements. + condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + + - pwsh: | + Start-Process 'python.exe' ` + -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + workingDirectory: build + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + displayName: Launch python websocket server (Windows). + + - bash: | + nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & + workingDirectory: build + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + displayName: Launch python websocket server (Linux). + + + # For non multi-config generator use the same build configuration to run tests # We don't need to set it to invoke ctest # Visual Studio generator used in CI is a multi-config generator. diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 06753dae19..9f08a19a2f 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -8,6 +8,9 @@ #include #include #include +#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) +#include "azure/core/http/websockets/curl_websockets_transport.hpp" +#endif // cspell::words closeme flibbityflobbidy using namespace Azure::Core; @@ -430,14 +433,14 @@ TEST_F(WebSocketTests, LibWebSocketOrg) auto serverStatus = lwsStatus.GetLWSStatus(); GTEST_LOG_(INFO) << "Server status: " << serverStatus << std::endl; - Azure::Core::Json::_internal::json status( - Azure::Core::Json::_internal::json::parse(serverStatus)); + Azure::Core::Json::_internal::json status; + EXPECT_NO_THROW(status = Azure::Core::Json::_internal::json::parse(serverStatus)); EXPECT_TRUE(status["conns"].is_array()); - auto connections = status["conns"].get_ref&>(); + auto& connections = status["conns"].get_ref&>(); bool foundOurConnection = false; // Scan through the list of connections to find a connection from the websockettest. - for (auto connection : connections) + for (auto& connection : connections) { EXPECT_TRUE(connection["ua"].is_string()); auto userAgent = connection["ua"].get(); @@ -469,3 +472,21 @@ TEST_F(WebSocketTests, LibWebSocketOrg) incrementProtocol.ConsumeUntilClosed(); } } +#if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) +TEST_F(WebSocketTests, CurlTransportCoverage) +{ + Azure::Core::Http::CurlTransportOptions transportOptions; + transportOptions.HttpKeepAlive = false; + auto transport + = std::make_shared(transportOptions); + + EXPECT_THROW(transport->CloseSocket(1001, {}, {}), std::runtime_error); + EXPECT_THROW(transport->GetCloseSocketInformation({}), std::runtime_error); + EXPECT_THROW( + transport->SendFrame(WebSocketTransport::WebSocketFrameType::FrameTypeBinary, {}, {}), + std::runtime_error); + WebSocketTransport::WebSocketFrameType ft; + EXPECT_THROW(transport->ReceiveFrame(ft, {}), std::runtime_error); +} + +#endif \ No newline at end of file From 4a555913af9d450540dad56a9c916fda773d4f76 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 09:52:18 -0700 Subject: [PATCH 059/149] Artifact name fix --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 384370b45d..b9a861d56a 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -195,8 +195,8 @@ jobs: condition: succeededOrFailed() - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/WebSocketServer.Log' - ArtifactName: '$(Build.SourcesDirectory)/WebSocketServer.log' + ArtifactPath: '$(Build.SourcesDirectory)/' + ArtifactName: 'WebSocketServer.log' - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" From d84dd287c511947d2a5f62ff05a23b9966723165 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 10:18:38 -0700 Subject: [PATCH 060/149] Log websocket outputs --- eng/pipelines/templates/jobs/ci.tests.yml | 9 ++++--- sdk/core/azure-core/src/http/curl/curl.cpp | 27 ++++++++++++------- .../src/http/curl/curl_connection_private.hpp | 26 ++++++++++++++---- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index b9a861d56a..946d30330f 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -193,10 +193,11 @@ jobs: publishRunAttachments: true displayName: Publish test results condition: succeededOrFailed() - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/' - ArtifactName: 'WebSocketServer.log' + - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' + ArtifactName: 'WebSocketLogs' - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index a7a6bb8956..0fecfcbb00 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -39,9 +39,13 @@ template // C26812: The enum type 'CURLoption' is un-scoped. Prefer 'enum class' over 'enum' (Enum.3) #pragma warning(disable : 26812) #endif -inline bool SetLibcurlOption(CURL* handle, CURLoption option, T value, CURLcode* outError) +inline bool SetLibcurlOption( + Azure::Core::Http::_detail::unique_CURL const& handle, + CURLoption option, + T value, + CURLcode* outError) { - *outError = curl_easy_setopt(handle, option, value); + *outError = curl_easy_setopt(handle.get(), option, value); return *outError == CURLE_OK; } #if defined(_MSC_VER) @@ -504,7 +508,10 @@ CURLcode CurlConnection::SendBuffer( { size_t sentBytesPerRequest = 0; sendResult = curl_easy_send( - m_handle, buffer + sentBytesTotal, bufferSize - sentBytesTotal, &sentBytesPerRequest); + m_handle.get(), + buffer + sentBytesTotal, + bufferSize - sentBytesTotal, + &sentBytesPerRequest); switch (sendResult) { @@ -960,7 +967,7 @@ size_t CurlConnection::ReadFromSocket(uint8_t* buffer, size_t bufferSize, Contex size_t readBytes = 0; for (CURLcode readResult = CURLE_AGAIN; readResult == CURLE_AGAIN;) { - readResult = curl_easy_recv(m_handle, buffer, bufferSize, &readBytes); + readResult = curl_easy_recv(m_handle.get(), buffer, bufferSize, &readBytes); switch (readResult) { @@ -1329,7 +1336,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo // Creating a new connection is thread safe. No need to lock mutex here. // No available connection for the pool for the required host. Create one Log::Write(Logger::Level::Verbose, LogMsgPrefix + "Spawn new connection."); - CURL* newHandle = curl_easy_init(); + unique_CURL newHandle(curl_easy_init(), CURL_deleter()); if (!newHandle) { throw Azure::Core::Http::TransportException( @@ -1339,7 +1346,8 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo CURLcode result; // Libcurl setup before open connection (url, connect_only, timeout) - if (!SetLibcurlOption(newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) + if (!SetLibcurlOption( + newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " @@ -1372,7 +1380,8 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (options.ConnectionTimeout != Azure::Core::Http::_detail::DefaultConnectionTimeout) { - if (!SetLibcurlOption(newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) + if (!SetLibcurlOption( + newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1460,7 +1469,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo + ". Failed enforcing TLS v1.2 or greater. " + std::string(curl_easy_strerror(result))); } - auto performResult = curl_easy_perform(newHandle); + auto performResult = curl_easy_perform(newHandle.get()); if (performResult != CURLE_OK) { throw Http::TransportException( @@ -1468,7 +1477,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo + std::string(curl_easy_strerror(performResult))); } - return std::make_unique(newHandle, connectionKey); + return std::make_unique(std::move(newHandle), connectionKey); } // Move the connection back to the connection pool. Push it to the front so it becomes the diff --git a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp index 4965848104..36e37d7081 100644 --- a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp @@ -47,6 +47,22 @@ namespace Azure { namespace Core { namespace Http { // Define the maximun allowed connections per host-index in the pool. If this number is reached // for the host-index, next connections trying to be added to the pool will be ignored. constexpr static int32_t MaxConnectionsPerIndex = 1024; + + // unique_ptr class wrapping an HINTERNET handle + class CURL_deleter { + public: + void operator()(CURL* handle) noexcept + { + if (handle != nullptr) + { + curl_easy_cleanup(handle); + } + } + }; + using unique_CURL = std::unique_ptr; + + + } // namespace _detail /** @@ -122,7 +138,7 @@ namespace Azure { namespace Core { namespace Http { */ class CurlConnection final : public CurlNetworkConnection { private: - CURL* m_handle; + _detail::unique_CURL m_handle; curl_socket_t m_curlSocket; std::chrono::steady_clock::time_point m_lastUseTime; std::string m_connectionKey; @@ -135,8 +151,8 @@ namespace Azure { namespace Core { namespace Http { * * @param connectionPropertiesKey CURL connection properties key */ - CurlConnection(CURL* handle, std::string connectionPropertiesKey) - : m_handle(handle), m_connectionKey(std::move(connectionPropertiesKey)) + CurlConnection(_detail::unique_CURL&& handle, std::string connectionPropertiesKey) + : m_handle(std::move(handle)), m_connectionKey(std::move(connectionPropertiesKey)) { // Get the socket that libcurl is using from handle. Will use this to wait while // reading/writing @@ -146,7 +162,7 @@ namespace Azure { namespace Core { namespace Http { // C26812: The enum type 'CURLcode' is un-scoped. Prefer 'enum class' over 'enum' (Enum.3) #pragma warning(disable : 26812) #endif - auto result = curl_easy_getinfo(m_handle, CURLINFO_ACTIVESOCKET, &m_curlSocket); + auto result = curl_easy_getinfo(m_handle.get(), CURLINFO_ACTIVESOCKET, &m_curlSocket); #if defined(_MSC_VER) #pragma warning(pop) #endif @@ -162,7 +178,7 @@ namespace Azure { namespace Core { namespace Http { * @brief Destructor. * @details Cleans up CURL (invokes `curl_easy_cleanup()`). */ - ~CurlConnection() override { curl_easy_cleanup(this->m_handle); } + ~CurlConnection() override { curl_easy_cleanup(this->m_handle.get()); } std::string const& GetConnectionKey() const override { return this->m_connectionKey; } From 0776a20a2deba1cf5a0531920c18ad1893ad308e Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 10:23:01 -0700 Subject: [PATCH 061/149] Use RAII pointer for CURL transport --- .../azure-core/src/http/curl/curl_connection_private.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp index 36e37d7081..0f7763c084 100644 --- a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp @@ -47,8 +47,8 @@ namespace Azure { namespace Core { namespace Http { // Define the maximun allowed connections per host-index in the pool. If this number is reached // for the host-index, next connections trying to be added to the pool will be ignored. constexpr static int32_t MaxConnectionsPerIndex = 1024; - - // unique_ptr class wrapping an HINTERNET handle + + // unique_ptr class wrapping a CURL handle class CURL_deleter { public: void operator()(CURL* handle) noexcept @@ -61,8 +61,6 @@ namespace Azure { namespace Core { namespace Http { }; using unique_CURL = std::unique_ptr; - - } // namespace _detail /** @@ -178,7 +176,7 @@ namespace Azure { namespace Core { namespace Http { * @brief Destructor. * @details Cleans up CURL (invokes `curl_easy_cleanup()`). */ - ~CurlConnection() override { curl_easy_cleanup(this->m_handle.get()); } + ~CurlConnection() override {} std::string const& GetConnectionKey() const override { return this->m_connectionKey; } From 6e3c4b2c1a3b362f747f422974876d93110a8062 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 10:44:12 -0700 Subject: [PATCH 062/149] Noise reduction in test code --- eng/pipelines/templates/jobs/ci.tests.yml | 2 ++ sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp | 4 ---- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 946d30330f..d0d65fdc40 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -193,11 +193,13 @@ jobs: publishRunAttachments: true displayName: Publish test results condition: succeededOrFailed() + - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' ArtifactName: 'WebSocketLogs' + - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 437169c6d1..5abf73b365 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -4,7 +4,6 @@ #include "azure/core/http/websockets/websockets_transport.hpp" #include "azure/core/internal/http/pipeline.hpp" #include -#include #include #include @@ -95,11 +94,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names uint8_t ReadByte(Azure::Core::Context const& context) { - std::cout << "BufferedStreamReader::ReadByte "; if (m_bufferPos >= m_bufferLen) { m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); - std::cout << " Read " << m_bufferLen << " bytes from socket"; m_bufferPos = 0; if (m_bufferLen == 0) { @@ -107,7 +104,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return 0; } } - std::cout << "Read byte " << std::to_string(m_buffer[m_bufferPos]) << std::endl; return m_buffer[m_bufferPos++]; } uint16_t ReadShort(Azure::Core::Context const& context) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 9f08a19a2f..831e227a1e 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -267,7 +267,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) GTEST_LOG_(INFO) << "Total server requests: " << iterationCount.load() << std::endl; GTEST_LOG_(INFO) << "Logged " << std::dec << testData.size() << " iterations (0x" << std::hex << testData.size() << ")" << std::endl; - + EXPECT_GE(testData.size(), iterationCount.load()); // Close the socket gracefully. testSocket.Close(); From 1899c3199a7953b2039a50592f00c31d4696bcff Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 11:02:23 -0700 Subject: [PATCH 063/149] More iterations in multithreaded test; dont publish if no tests --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index d0d65fdc40..57b2a43d06 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -194,7 +194,7 @@ jobs: displayName: Publish test results condition: succeededOrFailed() - - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: + - ${{ if and(contains(variables.CmakeArgs, 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, True)) }}: - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 831e227a1e..a4d305457e 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -197,7 +197,7 @@ std::string ToHexString(std::vector const& data) TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; - constexpr size_t testDataLength = 30000; + constexpr size_t testDataLength = 95000; constexpr auto testDuration = 10s; WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); From f4b7685aca275d2206ccfffcd6c7563acb58ec8a Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 11:17:18 -0700 Subject: [PATCH 064/149] Disclaimers on SHA1 and MD5 hash algorithms --- eng/pipelines/templates/jobs/ci.tests.yml | 3 ++- sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp | 7 +++++++ .../inc/azure/core/internal/cryptography/sha_hash.hpp | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 57b2a43d06..f1d705275a 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -194,11 +194,12 @@ jobs: displayName: Publish test results condition: succeededOrFailed() - - ${{ if and(contains(variables.CmakeArgs, 'BUILD_TESTING=ON'), eq(parameters.EnablePythonWebSocketServer, True)) }}: + - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' ArtifactName: 'WebSocketLogs' + condition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | diff --git a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp index 9ef494cab1..089b59dec1 100644 --- a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp @@ -120,6 +120,13 @@ namespace Azure { namespace Core { namespace Cryptography { /** * @brief Represents the class for the MD5 hash function which maps binary data of an arbitrary * length to small binary data of a fixed length. + * + * @remarks NOTE: MD5 is a deprecated hashing algorithm and SHOULD NOT be used, + * unless it is used to implement a specific protocol (See RFC 6151 for more information + * about the weaknesses of the MD5 hash function). + * + * Client implementers should strongly prefer the SHA256, SHA384, and SHA512 hash + * functions. */ class Md5Hash final : public Hash { diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index c1b8475478..f9c04b7164 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -19,6 +19,12 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal /** * @brief Defines #Sha1Hash. + * + * @remarks NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, + * unless it is used to implement a specific protocol (for instance, RFC 6455 and + * RFC 7517 both require the use of SHA1 hashes). + * + * SHA256, SHA384, and SHA512 are all preferred to SHA1. * */ class Sha1Hash final : public Azure::Core::Cryptography::Hash { From d8de4c29c1453826bbbaa3d6882f178ecf794346 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 11:27:02 -0700 Subject: [PATCH 065/149] clang-format; improved conditional for websocketserver log --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp | 2 +- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 6 +++--- sdk/core/azure-core/src/http/curl/curl.cpp | 6 ++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index f1d705275a..383e18f8f6 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -199,7 +199,7 @@ jobs: parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' ArtifactName: 'WebSocketLogs' - condition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') + CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | diff --git a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp index 089b59dec1..b81f287b05 100644 --- a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp @@ -124,7 +124,7 @@ namespace Azure { namespace Core { namespace Cryptography { * @remarks NOTE: MD5 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (See RFC 6151 for more information * about the weaknesses of the MD5 hash function). - * + * * Client implementers should strongly prefer the SHA256, SHA384, and SHA512 hash * functions. */ diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index f9c04b7164..ba34679dbe 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -19,11 +19,11 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal /** * @brief Defines #Sha1Hash. - * - * @remarks NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, + * + * @remarks NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (for instance, RFC 6455 and * RFC 7517 both require the use of SHA1 hashes). - * + * * SHA256, SHA384, and SHA512 are all preferred to SHA1. * */ diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 0fecfcbb00..5f93052b9b 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -1346,8 +1346,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo CURLcode result; // Libcurl setup before open connection (url, connect_only, timeout) - if (!SetLibcurlOption( - newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " @@ -1380,8 +1379,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (options.ConnectionTimeout != Azure::Core::Http::_detail::DefaultConnectionTimeout) { - if (!SetLibcurlOption( - newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName From a92f5d599461639a21e54690294303acdafb2d0d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 11:34:06 -0700 Subject: [PATCH 066/149] Chagned remarks to warning on SHA/MD5 deprecation --- sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp | 2 +- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp index b81f287b05..5d29bc8a30 100644 --- a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp @@ -121,7 +121,7 @@ namespace Azure { namespace Core { namespace Cryptography { * @brief Represents the class for the MD5 hash function which maps binary data of an arbitrary * length to small binary data of a fixed length. * - * @remarks NOTE: MD5 is a deprecated hashing algorithm and SHOULD NOT be used, + * @warning NOTE: MD5 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (See RFC 6151 for more information * about the weaknesses of the MD5 hash function). * diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index ba34679dbe..a12374582c 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -20,7 +20,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal /** * @brief Defines #Sha1Hash. * - * @remarks NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, + * @warning NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (for instance, RFC 6455 and * RFC 7517 both require the use of SHA1 hashes). * From 07339d528302305a9b0137e7a6c15f889c3cd66c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 12:30:45 -0700 Subject: [PATCH 067/149] Updated hash comments --- sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp | 8 +++----- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp index 5d29bc8a30..a2225574cf 100644 --- a/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/cryptography/hash.hpp @@ -121,12 +121,10 @@ namespace Azure { namespace Core { namespace Cryptography { * @brief Represents the class for the MD5 hash function which maps binary data of an arbitrary * length to small binary data of a fixed length. * - * @warning NOTE: MD5 is a deprecated hashing algorithm and SHOULD NOT be used, + * @warning MD5 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (See RFC 6151 for more information - * about the weaknesses of the MD5 hash function). - * - * Client implementers should strongly prefer the SHA256, SHA384, and SHA512 hash - * functions. + * about the weaknesses of the MD5 hash function). Client implementers should strongly prefer the + * SHA256, SHA384, and SHA512 hash functions. */ class Md5Hash final : public Hash { diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index a12374582c..a389f7e837 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -20,11 +20,10 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal /** * @brief Defines #Sha1Hash. * - * @warning NOTE: SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, + * @warning SHA1 is a deprecated hashing algorithm and SHOULD NOT be used, * unless it is used to implement a specific protocol (for instance, RFC 6455 and - * RFC 7517 both require the use of SHA1 hashes). - * - * SHA256, SHA384, and SHA512 are all preferred to SHA1. + * RFC 7517 both require the use of SHA1 hashes). SHA256, SHA384, and SHA512 are all preferred to + * SHA1. * */ class Sha1Hash final : public Azure::Core::Cryptography::Hash { From 8d225cc56b8a84a99c800f2a4e1f7c0a60578617 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 12:30:54 -0700 Subject: [PATCH 068/149] Stop websocket server after tests complete --- sdk/core/azure-core/test/ut/websocket_server.py | 12 ++++++++---- sdk/core/azure-core/test/ut/websocket_test.cpp | 8 ++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index da548c034c..93cdb353fc 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -8,6 +8,7 @@ # create handler for each connection customPaths = {} +stop = False async def handleControlPath(websocket): while (1): @@ -16,6 +17,8 @@ async def handleControlPath(websocket): if (parsedCommand[0] == "close"): print("Closing control channel") await websocket.send("ok") + print("Terminating WebSocket server.") + stop.set_result(0) break elif parsedCommand[0] == "newPath": print("Add path") @@ -26,9 +29,7 @@ async def handleControlPath(websocket): else: print("Unknown command, echoing it.") await websocket.send(data) - websocket - async def handleCustomPath(websocket, path:dict): print("Handle custom path", path) data : str = await websocket.recv() @@ -69,7 +70,7 @@ async def handler(websocket, path : str): elif (parsedUrl.path == '/closeduringecho'): data = await websocket.recv() await websocket.close(1001, 'closed') - elif (parsedUrl.path =='control'): + elif (parsedUrl.path =='/control'): await handleControlPath(websocket) elif (parsedUrl.path in customPaths.keys()): print("Found path ", path, "in control paths.") @@ -82,9 +83,12 @@ async def handler(websocket, path : str): await websocket.send(reply) async def main(): + global stop print("Starting server") + loop = asyncio.get_running_loop() + stop = loop.create_future() async with websockets.serve(handler, "localhost", 8000): - await asyncio.Future() # run forever. + await stop # run forever. if __name__=="__main__": asyncio.run(main()) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index a4d305457e..9b7fabaaf5 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -22,6 +22,14 @@ class WebSocketTests : public testing::Test { protected: // Create virtual void SetUp() override {} + +static void TearDownTestSuite() { + WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); + controlSocket.Open(); + controlSocket.SendFrame("close", true); + auto controlResponse = controlSocket.ReceiveFrame(); + EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); +} }; TEST_F(WebSocketTests, CreateSimpleSocket) From de390ee9295d2066242cf3a6570056865c3f428c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 13:37:18 -0700 Subject: [PATCH 069/149] Use unique name for websocket logs --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- .../azure-core/test/ut/websocket_test.cpp | 21 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 383e18f8f6..e32da09d1c 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -198,7 +198,7 @@ jobs: - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' - ArtifactName: 'WebSocketLogs' + ArtifactName: 'WebSocketLogs-$(Agent.JobName)_attempt_$(System.JobAttempt)' CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - ${{ if eq(parameters.CoverageEnabled, true) }}: diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 9b7fabaaf5..d667a7ee07 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -21,15 +21,18 @@ class WebSocketTests : public testing::Test { private: protected: // Create - virtual void SetUp() override {} - -static void TearDownTestSuite() { - WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); - controlSocket.Open(); - controlSocket.SendFrame("close", true); - auto controlResponse = controlSocket.ReceiveFrame(); - EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); -} + static void SetUpTestSuite() + { + } + static void TearDownTestSuite() + { +// GTEST_LOG_(INFO) << "Shut down test server" << std::endl; +// WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); +// controlSocket.Open(); +// controlSocket.SendFrame("close", true); +// auto controlResponse = controlSocket.ReceiveFrame(); +// EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); + } }; TEST_F(WebSocketTests, CreateSimpleSocket) From 3808eb4ba444b8e9420d29eee4fa9a9e31390bd8 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 13:55:01 -0700 Subject: [PATCH 070/149] clang-format --- sdk/core/azure-core/test/ut/websocket_test.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index d667a7ee07..d5bbf169ee 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -21,17 +21,15 @@ class WebSocketTests : public testing::Test { private: protected: // Create - static void SetUpTestSuite() - { - } + static void SetUpTestSuite() {} static void TearDownTestSuite() { -// GTEST_LOG_(INFO) << "Shut down test server" << std::endl; -// WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); -// controlSocket.Open(); -// controlSocket.SendFrame("close", true); -// auto controlResponse = controlSocket.ReceiveFrame(); -// EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); + // GTEST_LOG_(INFO) << "Shut down test server" << std::endl; + // WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); + // controlSocket.Open(); + // controlSocket.SendFrame("close", true); + // auto controlResponse = controlSocket.ReceiveFrame(); + // EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); } }; From 01db8dd245208d5a92959a2f43076143f778a03d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 14:07:18 -0700 Subject: [PATCH 071/149] First try at core specific pipelines. --- sdk/core/ci.yml | 8 +++-- .../templates/stages/core-test-post.yml | 15 ++++++++ .../templates/stages/core-test-pre.yml | 34 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 sdk/core/pipelines/templates/stages/core-test-post.yml create mode 100644 sdk/core/pipelines/templates/stages/core-test-pre.yml diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index eb50714b74..84355b9c86 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -31,6 +31,9 @@ pr: - sdk/core stages: + - template: pipelines/templates/stages/core-test-pre.yml + parameters: + ServiceDirectory: core - template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core @@ -42,8 +45,6 @@ stages: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 93 BranchCoverageTarget: 55 - EnablePythonWebSocketServer: true - PythonWebSocketServerName: sdk/core/azure-core/test/ut/websocket_server.py Artifacts: - Name: azure-core Path: azure-core @@ -91,3 +92,6 @@ stages: Value: '' - Name: Test Value: '-DBUILD_TESTING=ON' + - template: pipelines/templates/stages/core-test-post.yml + parameters: + ServiceDirectory: core diff --git a/sdk/core/pipelines/templates/stages/core-test-post.yml b/sdk/core/pipelines/templates/stages/core-test-post.yml new file mode 100644 index 0000000000..941c74410f --- /dev/null +++ b/sdk/core/pipelines/templates/stages/core-test-post.yml @@ -0,0 +1,15 @@ +parameters: +- name: ServiceDirectory + type: string + default: not-specified + + +stages: + - stage: + jobs: + job: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + diff --git a/sdk/core/pipelines/templates/stages/core-test-pre.yml b/sdk/core/pipelines/templates/stages/core-test-pre.yml new file mode 100644 index 0000000000..f5cb3ee548 --- /dev/null +++ b/sdk/core/pipelines/templates/stages/core-test-pre.yml @@ -0,0 +1,34 @@ +parameters: +- name: ServiceDirectory + type: string + default: not-specified + + +stages: + - stage: + jobs: + job: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + + - pwsh: | + python --version + pip install -r requirements.txt + workingDirectory: build/sdk/core/azure-core/test/ut + displayName: Install Python requirements. + + - pwsh: | + Start-Process 'python.exe' ` + -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + workingDirectory: build + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Windows). + + - bash: | + nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & + workingDirectory: build + condition: ne(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Linux). From 4453714cfde4a44a0a9dae69567a9bffe1bb8253 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 8 Jul 2022 14:48:31 -0700 Subject: [PATCH 072/149] Pre and Post test steps --- .../templates/jobs/archetype-sdk-client.yml | 8 +++ eng/pipelines/templates/jobs/ci.tests.yml | 40 ++++---------- eng/pipelines/templates/jobs/live.tests.yml | 40 ++++---------- .../templates/stages/archetype-sdk-client.yml | 10 ++++ .../templates/stages/archetype-sdk-tests.yml | 8 +++ sdk/core/ci.yml | 10 ++-- .../templates/stages/core-test-post.yml | 19 ++----- .../templates/stages/core-test-pre.yml | 53 ++++++++----------- 8 files changed, 77 insertions(+), 111 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 3501323123..7fb2ff5250 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -38,6 +38,12 @@ parameters: - name: PythonWebSocketServerName type: string default: '' + - name: PreTestSteps + type: object + default: [] + - name: PostTestSteps + type: object + default: [] jobs: - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml @@ -61,6 +67,8 @@ jobs: TestEnv: ${{ parameters.TestEnv }} EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} + PreTestSteps: ${{ parameters.PreTestSteps }} + PostTestSteps: ${{ parameters.PostTestSteps }} # Disable build for cpp - client - ${{ if ne(parameters.ServiceDirectory, 'not-specified' )}}: diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index e32da09d1c..eaed86fbce 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -52,6 +52,13 @@ parameters: - name: PythonWebSocketServerName type: string default: '' + - name: PreTestSteps + type: object + default: [] + - name: PostTestSteps + type: object + default: [] + jobs: - job: @@ -143,36 +150,8 @@ jobs: BuildArgs: "$(BuildArgs)" Env: "$(CmakeEnvArg)" - # If the pipeline is configured to enable a Python WebSocket server, then - # configure the server and launch it. Note that we only launch the server if - # we're not running tests locally. - - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' - - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: build/sdk/core/azure-core/test/ut - displayName: Install Python requirements. - condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + - ${{ parameters.PreTestSteps }} - - pwsh: | - Start-Process 'python.exe' ` - -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: build - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Windows). - - - bash: | - nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/WebSocketServer.log & - workingDirectory: build - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Linux). - - pwsh: | ctest ` -C Debug ` @@ -183,6 +162,9 @@ jobs: -T Test workingDirectory: build displayName: Test + + - ${{ parameters.PostTestSteps }} + - task: PublishTestResults@2 inputs: testResultsFormat: cTest diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index e27091fd0e..19e73bb0b5 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -36,6 +36,12 @@ parameters: - name: UsePlatformContainer type: boolean default: false +- name: PreTestSteps + type: object + default: [] +- name: PostTestSteps + type: object + default: [] jobs: - job: ValidateLive @@ -126,37 +132,7 @@ jobs: Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} - # If the pipeline is configured to enable a Python WebSocket server, then - # configure the server and launch it. Note that we only launch the server if - # we're not running tests locally. - - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' - - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: build/sdk/core/azure-core/test/ut - displayName: Install Python requirements. - condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - - - pwsh: | - Start-Process 'python.exe' ` - -ArgumentList '${{parameters.PythonWebSocketServerName}}' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: build - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Windows). - - - bash: | - nohup python ${{parameters.PythonWebSocketServerName}} > $(Build.SourcesDirectory)/test-proxy.log & - workingDirectory: build - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Linux). - - + - ${{ parameters.PreTestSteps }} # For non multi-config generator use the same build configuration to run tests # We don't need to set it to invoke ctest @@ -173,6 +149,8 @@ jobs: succeeded(), ne(variables['RunSamples'], '1')) + - ${{ parameters.PostTestSteps }} + - task: PublishTestResults@2 inputs: testResultsFormat: cTest diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 9cb6e41d97..6dfc2d00df 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -79,6 +79,12 @@ parameters: - name: PythonWebSocketServerName type: string default: '' +- name: PreTestSteps + type: object + default: [] +- name: PostTestSteps + type: object + default: [] stages: @@ -110,6 +116,8 @@ stages: TestEnv: ${{ parameters.TestEnv }} EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} + PreTestSteps: ${{ parameters.PreTestSteps }} + PostTestSteps: ${{ parameters.PostTestSteps }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(parameters.LiveTestCtestRegex, '')) }}: - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -126,6 +134,8 @@ stages: UnsupportedClouds: ${{ parameters.UnsupportedClouds }} EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} + PreTestSteps: ${{ parameters.PreTestSteps }} + PostTestSteps: ${{ parameters.PostTestSteps }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), not(endsWith(variables['Build.DefinitionName'], ' - tests'))) }}: - template: archetype-cpp-release.yml diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 9ddf86ebee..fa6c27a587 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -35,6 +35,12 @@ parameters: - name: PythonWebSocketServerName type: string default: '' +- name: PreTestSteps + type: object + default: [] +- name: PostTestSteps + type: object + default: [] stages: - ${{ each cloud in parameters.CloudConfig }}: @@ -65,3 +71,5 @@ stages: TimeoutInMinutes: ${{ parameters.TimeoutInMinutes}} EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer}} PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName}} + PreTestSteps: ${{ parameters.PreTestSteps }} + PostTestSteps: ${{ parameters.PostTestSteps }} diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 84355b9c86..00e167699f 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -31,9 +31,6 @@ pr: - sdk/core stages: - - template: pipelines/templates/stages/core-test-pre.yml - parameters: - ServiceDirectory: core - template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core @@ -45,6 +42,10 @@ stages: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 93 BranchCoverageTarget: 55 + PreTestSteps: + - template: /sdk/core/pipelines/templates/stages/core-test-pre.yml + PostTestSteps: + - template: /sdk/core/pipelines/templates/stages/core-test-post.yml Artifacts: - Name: azure-core Path: azure-core @@ -92,6 +93,3 @@ stages: Value: '' - Name: Test Value: '-DBUILD_TESTING=ON' - - template: pipelines/templates/stages/core-test-post.yml - parameters: - ServiceDirectory: core diff --git a/sdk/core/pipelines/templates/stages/core-test-post.yml b/sdk/core/pipelines/templates/stages/core-test-post.yml index 941c74410f..c2e3158462 100644 --- a/sdk/core/pipelines/templates/stages/core-test-post.yml +++ b/sdk/core/pipelines/templates/stages/core-test-post.yml @@ -1,15 +1,6 @@ -parameters: -- name: ServiceDirectory - type: string - default: not-specified - - -stages: - - stage: - jobs: - job: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' +steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' diff --git a/sdk/core/pipelines/templates/stages/core-test-pre.yml b/sdk/core/pipelines/templates/stages/core-test-pre.yml index f5cb3ee548..001fdbe8ad 100644 --- a/sdk/core/pipelines/templates/stages/core-test-pre.yml +++ b/sdk/core/pipelines/templates/stages/core-test-pre.yml @@ -1,34 +1,25 @@ -parameters: -- name: ServiceDirectory - type: string - default: not-specified +steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + - pwsh: | + python --version + pip install -r requirements.txt + workingDirectory: build/sdk/core/azure-core/test/ut + displayName: Install Python requirements. -stages: - - stage: - jobs: - job: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' + - pwsh: | + Start-Process 'python.exe' ` + -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + workingDirectory: build + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Windows). - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: build/sdk/core/azure-core/test/ut - displayName: Install Python requirements. - - - pwsh: | - Start-Process 'python.exe' ` - -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: build - condition: eq(variables['Agent.OS'], 'Windows_NT') - displayName: Launch python websocket server (Windows). - - - bash: | - nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & - workingDirectory: build - condition: ne(variables['Agent.OS'], 'Windows_NT') - displayName: Launch python websocket server (Linux). + - bash: | + nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & + workingDirectory: build + condition: ne(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Linux). From 9311f261fb2ebce3c098a0d1e5f9ad353ac12242 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 8 Jul 2022 14:50:03 -0700 Subject: [PATCH 073/149] Do not insert template --- sdk/core/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 00e167699f..e755dbe661 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -43,9 +43,9 @@ stages: LineCoverageTarget: 93 BranchCoverageTarget: 55 PreTestSteps: - - template: /sdk/core/pipelines/templates/stages/core-test-pre.yml + - pwsh: Write-Host "Hello Pre-test" PostTestSteps: - - template: /sdk/core/pipelines/templates/stages/core-test-post.yml + - pwsh: Write-Host "Hello Post-test" Artifacts: - Name: azure-core Path: azure-core From f022cdd285e2cbe489757954761586a901d30123 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 8 Jul 2022 14:56:43 -0700 Subject: [PATCH 074/149] Clean up most references to earlier parameters --- .../templates/jobs/archetype-sdk-client.yml | 8 -------- eng/pipelines/templates/jobs/ci.tests.yml | 18 ++++++------------ .../templates/stages/archetype-sdk-client.yml | 10 ---------- .../templates/stages/archetype-sdk-tests.yml | 8 -------- sdk/core/ci.yml | 8 ++++---- 5 files changed, 10 insertions(+), 42 deletions(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 7fb2ff5250..6a1652faec 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -32,12 +32,6 @@ parameters: - name: TestEnv type: object default: [] - - name: EnablePythonWebSocketServer - type: boolean - default: false - - name: PythonWebSocketServerName - type: string - default: '' - name: PreTestSteps type: object default: [] @@ -65,8 +59,6 @@ jobs: LineCoverageTarget: ${{ parameters.LineCoverageTarget }} BranchCoverageTarget: ${{ parameters.BranchCoverageTarget }} TestEnv: ${{ parameters.TestEnv }} - EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} - PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} PreTestSteps: ${{ parameters.PreTestSteps }} PostTestSteps: ${{ parameters.PostTestSteps }} diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index eaed86fbce..ff670e9123 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -46,12 +46,6 @@ parameters: - name: UsePlatformContainer type: boolean default: false - - name: EnablePythonWebSocketServer - type: boolean - default: false - - name: PythonWebSocketServerName - type: string - default: '' - name: PreTestSteps type: object default: [] @@ -176,12 +170,12 @@ jobs: displayName: Publish test results condition: succeededOrFailed() - - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' - ArtifactName: 'WebSocketLogs-$(Agent.JobName)_attempt_$(System.JobAttempt)' - CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') + # - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: + # - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + # parameters: + # ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' + # ArtifactName: 'WebSocketLogs-$(Agent.JobName)_attempt_$(System.JobAttempt)' + # CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 6dfc2d00df..1ae0a90238 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -73,12 +73,6 @@ parameters: - name: UnsupportedClouds type: string default: '' -- name: EnablePythonWebSocketServer - type: boolean - default: false -- name: PythonWebSocketServerName - type: string - default: '' - name: PreTestSteps type: object default: [] @@ -114,8 +108,6 @@ stages: ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true TestEnv: ${{ parameters.TestEnv }} - EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} - PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} PreTestSteps: ${{ parameters.PreTestSteps }} PostTestSteps: ${{ parameters.PostTestSteps }} @@ -132,8 +124,6 @@ stages: Clouds: ${{ parameters.Clouds }} SupportedClouds: ${{ parameters.SupportedClouds }} UnsupportedClouds: ${{ parameters.UnsupportedClouds }} - EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer }} - PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName }} PreTestSteps: ${{ parameters.PreTestSteps }} PostTestSteps: ${{ parameters.PostTestSteps }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index fa6c27a587..a4aba5565b 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -29,12 +29,6 @@ parameters: - name: UnsupportedClouds type: string default: '' -- name: EnablePythonWebSocketServer - type: boolean - default: false -- name: PythonWebSocketServerName - type: string - default: '' - name: PreTestSteps type: object default: [] @@ -69,7 +63,5 @@ stages: Coverage: ${{ parameters.Coverage}} CoverageReportPath: ${{ parameters.CoverageReportPath}} TimeoutInMinutes: ${{ parameters.TimeoutInMinutes}} - EnablePythonWebSocketServer: ${{ parameters.EnablePythonWebSocketServer}} - PythonWebSocketServerName: ${{ parameters.PythonWebSocketServerName}} PreTestSteps: ${{ parameters.PreTestSteps }} PostTestSteps: ${{ parameters.PostTestSteps }} diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index e755dbe661..b6ee70992d 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -42,10 +42,10 @@ stages: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 93 BranchCoverageTarget: 55 - PreTestSteps: - - pwsh: Write-Host "Hello Pre-test" - PostTestSteps: - - pwsh: Write-Host "Hello Post-test" + # PreTestSteps: + # - pwsh: Write-Host "Hello Pre-test" + # PostTestSteps: + # - pwsh: Write-Host "Hello Post-test" Artifacts: - Name: azure-core Path: azure-core From a617e3e3f33983cf45fd1e445a4f7ee68599a829 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 8 Jul 2022 15:16:22 -0700 Subject: [PATCH 075/149] Add pre and post steps back in --- sdk/core/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index b6ee70992d..e755dbe661 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -42,10 +42,10 @@ stages: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 93 BranchCoverageTarget: 55 - # PreTestSteps: - # - pwsh: Write-Host "Hello Pre-test" - # PostTestSteps: - # - pwsh: Write-Host "Hello Post-test" + PreTestSteps: + - pwsh: Write-Host "Hello Pre-test" + PostTestSteps: + - pwsh: Write-Host "Hello Post-test" Artifacts: - Name: azure-core Path: azure-core From cd8acdc2f7f3adbbbdfd192583d31e2be2f317dc Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 15:19:42 -0700 Subject: [PATCH 076/149] Build fixes from merge --- sdk/core/azure-core/src/http/curl/curl.cpp | 24 +++++++++---------- .../curl/curl_connection_pool_private.hpp | 14 ----------- .../src/http/curl/curl_connection_private.hpp | 5 +++- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index e44f160bf7..0fecfcbb00 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -1347,21 +1347,21 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo // Libcurl setup before open connection (url, connect_only, timeout) if (!SetLibcurlOption( - newHandle.get(), CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) + newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " + std::string(curl_easy_strerror(result))); } - if (port != 0 && !SetLibcurlOption(newHandle.get(), CURLOPT_PORT, port, &result)) + if (port != 0 && !SetLibcurlOption(newHandle, CURLOPT_PORT, port, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " + std::string(curl_easy_strerror(result))); } - if (!SetLibcurlOption(newHandle.get(), CURLOPT_CONNECT_ONLY, 1L, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_CONNECT_ONLY, 1L, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " @@ -1371,7 +1371,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo // Set timeout to 24h. Libcurl will fail uploading on windows if timeout is: // timeout >= 25 days. Fails as soon as trying to upload any data // 25 days < timeout > 1 days. Fail on huge uploads ( > 1GB) - if (!SetLibcurlOption(newHandle.get(), CURLOPT_TIMEOUT, 60L * 60L * 24L, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_TIMEOUT, 60L * 60L * 24L, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " @@ -1381,7 +1381,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (options.ConnectionTimeout != Azure::Core::Http::_detail::DefaultConnectionTimeout) { if (!SetLibcurlOption( - newHandle.get(), CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) + newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1396,7 +1396,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo */ if (options.Proxy) { - if (!SetLibcurlOption(newHandle.get(), CURLOPT_PROXY, options.Proxy->c_str(), &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_PROXY, options.Proxy->c_str(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1407,7 +1407,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (!options.CAInfo.empty()) { - if (!SetLibcurlOption(newHandle.get(), CURLOPT_CAINFO, options.CAInfo.c_str(), &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_CAINFO, options.CAInfo.c_str(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1422,7 +1422,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo sslOption |= CURLSSLOPT_NO_REVOKE; } - if (!SetLibcurlOption(newHandle.get(), CURLOPT_SSL_OPTIONS, sslOption, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_SSL_OPTIONS, sslOption, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1432,7 +1432,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (!options.SslVerifyPeer) { - if (!SetLibcurlOption(newHandle.get(), CURLOPT_SSL_VERIFYPEER, 0L, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_SSL_VERIFYPEER, 0L, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1442,7 +1442,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (options.NoSignal) { - if (!SetLibcurlOption(newHandle.get(), CURLOPT_NOSIGNAL, 1L, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_NOSIGNAL, 1L, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1454,7 +1454,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo // curl-transport adapter supports only HTTP/1.1 // https://github.com/Azure/azure-sdk-for-cpp/issues/2848 // The libcurl uses HTTP/2 by default, if it can be negotiated with a server on handshake. - if (!SetLibcurlOption(newHandle.get(), CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName @@ -1462,7 +1462,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo } // Make libcurl to support only TLS v1.2 or later - if (!SetLibcurlOption(newHandle.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName diff --git a/sdk/core/azure-core/src/http/curl/curl_connection_pool_private.hpp b/sdk/core/azure-core/src/http/curl/curl_connection_pool_private.hpp index dff3fad2f1..fdf1334360 100644 --- a/sdk/core/azure-core/src/http/curl/curl_connection_pool_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_connection_pool_private.hpp @@ -126,18 +126,4 @@ namespace Azure { namespace Core { namespace Http { namespace _detail { std::thread m_cleanThread; }; - /** - * @brief std::default_delete for the CURL * type , used for std::unique_ptr - * - */ - class CURL_deleter { - public: - void operator()(CURL* handle) noexcept - { - if (handle != nullptr) - { - curl_easy_cleanup(handle); - } - } - }; }}}} // namespace Azure::Core::Http::_detail diff --git a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp index 0f7763c084..705ec2d862 100644 --- a/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_connection_private.hpp @@ -48,7 +48,10 @@ namespace Azure { namespace Core { namespace Http { // for the host-index, next connections trying to be added to the pool will be ignored. constexpr static int32_t MaxConnectionsPerIndex = 1024; - // unique_ptr class wrapping a CURL handle + /** + * @brief std::default_delete for the CURL * type , used for std::unique_ptr + * + */ class CURL_deleter { public: void operator()(CURL* handle) noexcept From 5e1c78de8f422f781bbc32bc69a6afcd9745b7ce Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 8 Jul 2022 15:23:20 -0700 Subject: [PATCH 077/149] Inline pre and post-test steps --- sdk/core/ci.yml | 33 +++++++++++++++++-- .../templates/stages/core-test-post.yml | 6 ---- .../templates/stages/core-test-pre.yml | 25 -------------- 3 files changed, 31 insertions(+), 33 deletions(-) delete mode 100644 sdk/core/pipelines/templates/stages/core-test-post.yml delete mode 100644 sdk/core/pipelines/templates/stages/core-test-pre.yml diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index e755dbe661..0b52c0aabb 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -43,9 +43,38 @@ stages: LineCoverageTarget: 93 BranchCoverageTarget: 55 PreTestSteps: - - pwsh: Write-Host "Hello Pre-test" + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: '3.10' + + - pwsh: | + python --version + pip install -r requirements.txt + workingDirectory: build/sdk/core/azure-core/test/ut + displayName: Install Python requirements. + + - pwsh: | + Start-Process 'python.exe' ` + -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + workingDirectory: build + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Windows). + + - bash: | + nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & + workingDirectory: build + condition: ne(variables['Agent.OS'], 'Windows_NT') + displayName: Launch python websocket server (Linux). + PostTestSteps: - - pwsh: Write-Host "Hello Post-test" + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' + ArtifactName: 'WebSocketLogs-$(Agent.JobName)_attempt_$(System.JobAttempt)' + CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') + Artifacts: - Name: azure-core Path: azure-core diff --git a/sdk/core/pipelines/templates/stages/core-test-post.yml b/sdk/core/pipelines/templates/stages/core-test-post.yml deleted file mode 100644 index c2e3158462..0000000000 --- a/sdk/core/pipelines/templates/stages/core-test-post.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' - diff --git a/sdk/core/pipelines/templates/stages/core-test-pre.yml b/sdk/core/pipelines/templates/stages/core-test-pre.yml deleted file mode 100644 index 001fdbe8ad..0000000000 --- a/sdk/core/pipelines/templates/stages/core-test-pre.yml +++ /dev/null @@ -1,25 +0,0 @@ -steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: '3.10' - - - pwsh: | - python --version - pip install -r requirements.txt - workingDirectory: build/sdk/core/azure-core/test/ut - displayName: Install Python requirements. - - - pwsh: | - Start-Process 'python.exe' ` - -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: build - condition: eq(variables['Agent.OS'], 'Windows_NT') - displayName: Launch python websocket server (Windows). - - - bash: | - nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & - workingDirectory: build - condition: ne(variables['Agent.OS'], 'Windows_NT') - displayName: Launch python websocket server (Linux). From 6e39b3bc11123d307b634058f0eb709d5d3db395 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 15:54:38 -0700 Subject: [PATCH 078/149] Shut down websocket server; clang-format --- sdk/core/azure-core/src/http/curl/curl.cpp | 6 ++---- sdk/core/azure-core/test/ut/websocket_server.py | 3 +++ sdk/core/ci.yml | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 0fecfcbb00..5f93052b9b 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -1346,8 +1346,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo CURLcode result; // Libcurl setup before open connection (url, connect_only, timeout) - if (!SetLibcurlOption( - newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_URL, request.GetUrl().GetAbsoluteUrl().data(), &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName + ". " @@ -1380,8 +1379,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo if (options.ConnectionTimeout != Azure::Core::Http::_detail::DefaultConnectionTimeout) { - if (!SetLibcurlOption( - newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) + if (!SetLibcurlOption(newHandle, CURLOPT_CONNECTTIMEOUT_MS, options.ConnectionTimeout, &result)) { throw Azure::Core::Http::TransportException( _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 93cdb353fc..3cb7bcc542 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -75,6 +75,9 @@ async def handler(websocket, path : str): elif (parsedUrl.path in customPaths.keys()): print("Found path ", path, "in control paths.") await handleCustomPath(websocket, customPaths[path]) + elif (parsedUrl.path == '/terminateserver'): + print("Terminating WebSocket server.") + stop.set_result(0) else: data = await websocket.recv() print("Received: ", data) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 0b52c0aabb..31195dd1c3 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -46,7 +46,7 @@ stages: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: - versionSpec: '3.10' + versionSpec: '3.10.5' - pwsh: | python --version @@ -69,6 +69,20 @@ stages: displayName: Launch python websocket server (Linux). PostTestSteps: + # Shut down the test server. + - pwsh: | + curl.exe ` + --include ` + --no-buffer ` + --header "Connection: Upgrade" ` + --header "Upgrade: websocket" ` + --header "Host: localhost:8000" ` + --header "Origin: http://localhost:8000" ` + --header "Sec-WebSocket-Key: eaQZ9ed+LnT0zs5EvI04aQ==" ` + --header "Sec-WebSocket-Version: 13" ` + http://localhost:8000/terminateserver + + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' From 3cc6182a178f55ad727df79e95ef3efaa3a35f02 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 16:12:31 -0700 Subject: [PATCH 079/149] Added conditions on test steps --- sdk/core/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 31195dd1c3..fc143b3f98 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -47,29 +47,30 @@ stages: displayName: 'Use Python 3.10' inputs: versionSpec: '3.10.5' + condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - pwsh: | python --version pip install -r requirements.txt workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. - + condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - pwsh: | Start-Process 'python.exe' ` -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) displayName: Launch python websocket server (Windows). - bash: | nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & workingDirectory: build - condition: ne(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) displayName: Launch python websocket server (Linux). PostTestSteps: - # Shut down the test server. + # Shut down the test server. - pwsh: | curl.exe ` --include ` @@ -81,8 +82,7 @@ stages: --header "Sec-WebSocket-Key: eaQZ9ed+LnT0zs5EvI04aQ==" ` --header "Sec-WebSocket-Version: 13" ` http://localhost:8000/terminateserver - - + displayName: Shutdown WebSocket server. - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' From d7eed260f29c3e306c1e6e00f5b56061de80bbde Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 16:23:22 -0700 Subject: [PATCH 080/149] Only terminate websocket server if it was started --- sdk/core/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index fc143b3f98..873ea68271 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -83,6 +83,7 @@ stages: --header "Sec-WebSocket-Version: 13" ` http://localhost:8000/terminateserver displayName: Shutdown WebSocket server. + condition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' From f653b1801650e24a9512b896ac989a1b5da573ee Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 16:43:52 -0700 Subject: [PATCH 081/149] 110000 multithreaded requests --- sdk/core/azure-core/test/ut/websocket_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index d5bbf169ee..66790b32e1 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -206,7 +206,7 @@ std::string ToHexString(std::vector const& data) TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; - constexpr size_t testDataLength = 95000; + constexpr size_t testDataLength = 110000; constexpr auto testDuration = 10s; WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); @@ -274,7 +274,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) // We no longer need to worry about synchronization since all the worker threads are done. GTEST_LOG_(INFO) << "Total server requests: " << iterationCount.load() << std::endl; - GTEST_LOG_(INFO) << "Logged " << std::dec << testData.size() << " iterations (0x" << std::hex + GTEST_LOG_(INFO) << "Estimated " << std::dec << testData.size() << " iterations (0x" << std::hex << testData.size() << ")" << std::endl; EXPECT_GE(testData.size(), iterationCount.load()); // Close the socket gracefully. From ea35542d37f54cf14daf578ccf6a0b53cba78811 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 16:54:05 -0700 Subject: [PATCH 082/149] curl.exe -> curl --- sdk/core/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 873ea68271..9f08c5b6f9 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -70,9 +70,10 @@ stages: displayName: Launch python websocket server (Linux). PostTestSteps: - # Shut down the test server. + # Shut down the test server. This uses curl to send a request to the "terminateserver" websocket endpoint. + # When the test server receives a request on terminateserver, it shuts down gracefully. - pwsh: | - curl.exe ` + curl ` --include ` --no-buffer ` --header "Connection: Upgrade" ` From d7ef5a0eb4f23b9f316a02b54308eda17ca91093 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 8 Jul 2022 17:16:46 -0700 Subject: [PATCH 083/149] More test iterations --- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 66790b32e1..544f07e50a 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -206,7 +206,7 @@ std::string ToHexString(std::vector const& data) TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; - constexpr size_t testDataLength = 110000; + constexpr size_t testDataLength = 150000; constexpr auto testDuration = 10s; WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); From dfc3d03db976488115c49cd4a4b5cd1488faddd7 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 10:08:29 -0700 Subject: [PATCH 084/149] Code coverage improvements --- .../src/http/curl/curl_websockets.cpp | 1 + sdk/core/azure-core/test/ut/sha_test.cpp | 123 ++++++++++++++---- .../azure-core/test/ut/websocket_test.cpp | 39 ++++-- 3 files changed, 127 insertions(+), 36 deletions(-) diff --git a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp index f1f1fb931a..3cbd7ed980 100644 --- a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp +++ b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp @@ -75,6 +75,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { void CurlWebSocketTransport::OnUpgradedConnection( std::unique_ptr& upgradedConnection) { + CurlTransport::OnUpgradedConnection(upgradedConnection); // Note that m_upgradedConnection is a std::shared_ptr. We define it as a std::shared_ptr // because a std::shared_ptr can be declared on an incomplete type, while a std::unique_ptr // cannot. diff --git a/sdk/core/azure-core/test/ut/sha_test.cpp b/sdk/core/azure-core/test/ut/sha_test.cpp index 32bb33a26e..4c66ceada6 100644 --- a/sdk/core/azure-core/test/ut/sha_test.cpp +++ b/sdk/core/azure-core/test/ut/sha_test.cpp @@ -7,38 +7,113 @@ using namespace Azure::Core::Cryptography::_internal; +TEST(SHA, SHA1Test) +{ + { + Sha1Hash sha; + Sha1Hash sha2; + uint8_t data[] = "A"; + auto shaResult = sha.Final(data, sizeof(data)); + auto shaResult2 = sha2.Final(data, sizeof(data)); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } + { + Sha1Hash sha; + Sha1Hash sha2; + std::string data1 = "ABCDE"; + std::string data2 = "FGHIJ"; + sha.Append(reinterpret_cast(data1.data()), data1.size()); + auto shaResult = sha.Final(reinterpret_cast(data2.data()), data2.size()); + auto shaResult2 = sha2.Final( + reinterpret_cast((data1 + data2).data()), data1.size() + data2.size()); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } +} + TEST(SHA, SHA256Test) { - Sha256Hash sha; - Sha256Hash sha2; - uint8_t data[] = "A"; - auto shaResult = sha.Final(data, sizeof(data)); - auto shaResult2 = sha2.Final(data, sizeof(data)); - EXPECT_EQ(shaResult, shaResult2); - for (size_t i = 0; i != shaResult.size(); i++) - printf("%02x", shaResult[i]); + { + + Sha256Hash sha; + Sha256Hash sha2; + uint8_t data[] = "A"; + auto shaResult = sha.Final(data, sizeof(data)); + auto shaResult2 = sha2.Final(data, sizeof(data)); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } + { + Sha256Hash sha; + Sha256Hash sha2; + std::string data1 = "ABCDE"; + std::string data2 = "FGHIJ"; + sha.Append(reinterpret_cast(data1.data()), data1.size()); + auto shaResult = sha.Final(reinterpret_cast(data2.data()), data2.size()); + auto shaResult2 = sha2.Final( + reinterpret_cast((data1 + data2).data()), data1.size() + data2.size()); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } } TEST(SHA, SHA384Test) { - Sha384Hash sha; - Sha384Hash sha2; - uint8_t data[] = "A"; - auto shaResult = sha.Final(data, sizeof(data)); - auto shaResult2 = sha2.Final(data, sizeof(data)); - EXPECT_EQ(shaResult, shaResult2); - for (size_t i = 0; i != shaResult.size(); i++) - printf("%02x", shaResult[i]); + { + + Sha384Hash sha; + Sha384Hash sha2; + uint8_t data[] = "A"; + auto shaResult = sha.Final(data, sizeof(data)); + auto shaResult2 = sha2.Final(data, sizeof(data)); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } + { + Sha384Hash sha; + Sha384Hash sha2; + std::string data1 = "ABCDE"; + std::string data2 = "FGHIJ"; + sha.Append(reinterpret_cast(data1.data()), data1.size()); + auto shaResult = sha.Final(reinterpret_cast(data2.data()), data2.size()); + auto shaResult2 = sha2.Final( + reinterpret_cast((data1 + data2).data()), data1.size() + data2.size()); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } } TEST(SHA, SHA512Test) { - Sha512Hash sha; - Sha512Hash sha2; - uint8_t data[] = "A"; - auto shaResult = sha.Final(data, sizeof(data)); - auto shaResult2 = sha2.Final(data, sizeof(data)); - EXPECT_EQ(shaResult, shaResult2); - for (size_t i = 0; i != shaResult.size(); i++) - printf("%02x", shaResult[i]); + { + + Sha512Hash sha; + Sha512Hash sha2; + uint8_t data[] = "A"; + auto shaResult = sha.Final(data, sizeof(data)); + auto shaResult2 = sha2.Final(data, sizeof(data)); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } + { + Sha384Hash sha; + Sha384Hash sha2; + std::string data1 = "ABCDE"; + std::string data2 = "FGHIJ"; + sha.Append(reinterpret_cast(data1.data()), data1.size()); + auto shaResult = sha.Final(reinterpret_cast(data2.data()), data2.size()); + auto shaResult2 = sha2.Final( + reinterpret_cast((data1 + data2).data()), data1.size() + data2.size()); + EXPECT_EQ(shaResult, shaResult2); + for (size_t i = 0; i != shaResult.size(); i++) + printf("%02x", shaResult[i]); + } } diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 544f07e50a..7babdf1db8 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -38,6 +38,7 @@ TEST_F(WebSocketTests, CreateSimpleSocket) { WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000")); defaultSocket.AddHeader("newHeader", "headerValue"); + EXPECT_THROW(defaultSocket.GetChosenProtocol(), std::runtime_error); } } @@ -50,9 +51,19 @@ TEST_F(WebSocketTests, OpenSimpleSocket) defaultSocket.Open(); + EXPECT_THROW(defaultSocket.AddHeader("newHeader", "headerValue"), std::runtime_error); + // Close the socket without notifying the peer. defaultSocket.Close(); } + + { + WebSocketOptions options; + WebSocket defaultSocket(Azure::Core::Url("http://microsoft.com/index.htm"), options); + defaultSocket.AddHeader("newHeader", "headerValue"); + + EXPECT_THROW(defaultSocket.Open(), std::runtime_error); + } } TEST_F(WebSocketTests, OpenAndCloseSocket) @@ -80,6 +91,7 @@ TEST_F(WebSocketTests, OpenAndCloseSocket) // // Now re-open the socket - this should work to reset everything. defaultSocket.Open(); + EXPECT_THROW(defaultSocket.Open(), std::runtime_error); defaultSocket.Close(); } } @@ -484,18 +496,21 @@ TEST_F(WebSocketTests, LibWebSocketOrg) #if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) TEST_F(WebSocketTests, CurlTransportCoverage) { - Azure::Core::Http::CurlTransportOptions transportOptions; - transportOptions.HttpKeepAlive = false; - auto transport - = std::make_shared(transportOptions); - - EXPECT_THROW(transport->CloseSocket(1001, {}, {}), std::runtime_error); - EXPECT_THROW(transport->GetCloseSocketInformation({}), std::runtime_error); - EXPECT_THROW( - transport->SendFrame(WebSocketTransport::WebSocketFrameType::FrameTypeBinary, {}, {}), - std::runtime_error); - WebSocketTransport::WebSocketFrameType ft; - EXPECT_THROW(transport->ReceiveFrame(ft, {}), std::runtime_error); + { + + Azure::Core::Http::CurlTransportOptions transportOptions; + transportOptions.HttpKeepAlive = false; + auto transport + = std::make_shared(transportOptions); + + EXPECT_THROW(transport->CloseSocket(1001, {}, {}), std::runtime_error); + EXPECT_THROW(transport->GetCloseSocketInformation({}), std::runtime_error); + EXPECT_THROW( + transport->SendFrame(WebSocketTransport::WebSocketFrameType::FrameTypeBinary, {}, {}), + std::runtime_error); + WebSocketTransport::WebSocketFrameType ft; + EXPECT_THROW(transport->ReceiveFrame(ft, {}), std::runtime_error); + } } #endif \ No newline at end of file From d374e13d4d03f567ffeaade1caf67eff6e007e11 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 10:12:16 -0700 Subject: [PATCH 085/149] cspell --- sdk/core/azure-core/test/ut/sha_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/azure-core/test/ut/sha_test.cpp b/sdk/core/azure-core/test/ut/sha_test.cpp index 4c66ceada6..1ac049379c 100644 --- a/sdk/core/azure-core/test/ut/sha_test.cpp +++ b/sdk/core/azure-core/test/ut/sha_test.cpp @@ -7,6 +7,7 @@ using namespace Azure::Core::Cryptography::_internal; +// cspell: words ABCDE FGHIJ TEST(SHA, SHA1Test) { { From d34ca222ff322e4531a902f08cb95cb7cf1c2d5a Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 12:07:50 -0700 Subject: [PATCH 086/149] Faster multithreaded testl; increased more coverage --- sdk/core/azure-core/src/cryptography/md5.cpp | 1 - sdk/core/azure-core/test/ut/sha_test.cpp | 4 +- .../azure-core/test/ut/websocket_test.cpp | 63 +++++++------------ 3 files changed, 24 insertions(+), 44 deletions(-) diff --git a/sdk/core/azure-core/src/cryptography/md5.cpp b/sdk/core/azure-core/src/cryptography/md5.cpp index a4bbaa217c..79a4a1210f 100644 --- a/sdk/core/azure-core/src/cryptography/md5.cpp +++ b/sdk/core/azure-core/src/cryptography/md5.cpp @@ -194,7 +194,6 @@ class Md5OpenSSL final : public Azure::Core::Cryptography::Hash { } if (1 != EVP_DigestInit_ex(m_context, EVP_md5(), NULL)) { - EVP_MD_CTX_free(m_context); throw std::runtime_error("Crypto error while init Md5Hash."); } } diff --git a/sdk/core/azure-core/test/ut/sha_test.cpp b/sdk/core/azure-core/test/ut/sha_test.cpp index 1ac049379c..826f805503 100644 --- a/sdk/core/azure-core/test/ut/sha_test.cpp +++ b/sdk/core/azure-core/test/ut/sha_test.cpp @@ -105,8 +105,8 @@ TEST(SHA, SHA512Test) printf("%02x", shaResult[i]); } { - Sha384Hash sha; - Sha384Hash sha2; + Sha512Hash sha; + Sha512Hash sha2; std::string data1 = "ABCDE"; std::string data2 = "FGHIJ"; sha.Append(reinterpret_cast(data1.data()), data1.size()); diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 7babdf1db8..4e2842ce6e 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -279,6 +279,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) } // std::this_thread::sleep_for(10s); + // Wait for all the threads to exit. for (auto& thread : threads) { thread.join(); @@ -288,55 +289,35 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) GTEST_LOG_(INFO) << "Total server requests: " << iterationCount.load() << std::endl; GTEST_LOG_(INFO) << "Estimated " << std::dec << testData.size() << " iterations (0x" << std::hex << testData.size() << ")" << std::endl; - EXPECT_GE(testData.size(), iterationCount.load()); + EXPECT_GE(testDataLength, iterationCount.load()); // Close the socket gracefully. testSocket.Close(); + // Resize the test data to the number of actual iterations. + testData.resize(iterationCount.load()); + receivedData.resize(iterationCount.load()); + // If we've processed every iteration, let's make sure that we received everything we sent. // If we dropped some results, then we can't check to ensure that we have received everything // because we can't account for everything sent. - if (iterationCount <= testDataLength) + std::set testDataStrings; + std::set receivedDataStrings; + for (auto const& data : testData) { - // Compare testData and receivedData to ensure every element in testData is in receivedData - // and every element in receivedData is in testData. - // - // This is a bit of a hack, but it is the only way to do this without a lot of extra code. - // The problem is that the order of the elements in testData and receivedData is not - // guaranteed, so we need to sort them before we can compare them. - // We sort by the size of the vector, so the smaller vectors will be first in the sort. - std::vector testDataStrings; - std::vector receivedDataStrings; - for (auto const& data : testData) - { - testDataStrings.push_back(ToHexString(data)); - } - for (auto const& data : receivedData) - { - receivedDataStrings.push_back(ToHexString(data)); - } - std::sort(testDataStrings.begin(), testDataStrings.end()); - std::sort(receivedDataStrings.begin(), receivedDataStrings.end()); - for (size_t i = 0; i < testDataStrings.size(); ++i) - { - if (testDataStrings[i] != receivedDataStrings[i]) - { - GTEST_LOG_(ERROR) << "Mismatch at index " << i << std::endl; - GTEST_LOG_(ERROR) << "testData: " << testDataStrings[i] << std::endl; - GTEST_LOG_(ERROR) << "receivedData: " << receivedDataStrings[i] << std::endl; - } - } + testDataStrings.emplace(ToHexString(data)); + } + for (auto const& data : receivedData) + { + receivedDataStrings.emplace(ToHexString(data)); + } - for (auto const& data : testDataStrings) - { - EXPECT_NE( - receivedDataStrings.end(), - std::find(receivedDataStrings.begin(), receivedDataStrings.end(), data)); - } - for (auto const& data : receivedDataStrings) - { - EXPECT_NE( - testDataStrings.end(), std::find(testDataStrings.begin(), testDataStrings.end(), data)); - } + for (auto const& data : testDataStrings) + { + EXPECT_NE(receivedDataStrings.end(), receivedDataStrings.find(data)); + } + for (auto const& data : receivedDataStrings) + { + EXPECT_NE(testDataStrings.end(), testDataStrings.find(data)); } } From fa3e8dee002549b8febadbe57eb7d51c9956dbc2 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 13:33:09 -0700 Subject: [PATCH 087/149] Reduce code complexity upon closer reading of RFC 6455 --- .../azure/core/http/websockets/websockets.hpp | 13 +---- sdk/core/azure-core/src/http/curl/curl.cpp | 5 ++ .../src/http/websockets/websocketsimpl.cpp | 56 ++++--------------- .../src/http/websockets/websocketsimpl.hpp | 16 ++++-- .../azure-core/test/ut/websocket_test.cpp | 7 ++- 5 files changed, 37 insertions(+), 60 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 61307f9494..8a0b58a23e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -99,14 +99,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { struct WebSocketOptions : Azure::Core::_internal::ClientOptions { - /** - * @brief Enable masking for this WebSocket. - * - * @details Masking is needed to block [certain infrastructure - * attacks](https://www.rfc-editor.org/rfc/rfc6455.html#section-10.3) and is strongly - * recommended. - */ - bool EnableMasking{true}; /** * @brief The set of protocols which are supported by this client */ @@ -129,9 +121,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param enableMasking If true, enable masking for the websocket * @param protocols Supported protocols for this websocket client. */ - explicit WebSocketOptions(bool enableMasking, std::vector protocols) - : Azure::Core::_internal::ClientOptions{}, EnableMasking(enableMasking), - Protocols(protocols) + explicit WebSocketOptions(std::vector protocols) + : Azure::Core::_internal::ClientOptions{}, Protocols(protocols) { } WebSocketOptions() = default; diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 5f93052b9b..eacbc7b6bb 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -915,6 +915,11 @@ size_t CurlSession::OnRead(uint8_t* buffer, size_t count, Context const& context return 0; } + // If we no longer have a connection, read 0 bytes. + if (!m_connection) + { + return 0; + } // Read from socket when no more data on internal buffer // For chunk request, read a chunk based on chunk size totalRead = m_connection->ReadFromSocket(buffer, static_cast(readRequestLength), context); diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 31334def04..a6ce4a40a8 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -125,6 +125,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { VerifySocketAccept(encodedKey, socketAccept->second); } + auto bodyStream = response->ExtractBodyStream(); + m_bufferedStreamReader.SetInitialStream(bodyStream); } // Remember the protocol that the client chose. @@ -186,8 +188,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector closePayload; closePayload.push_back(closeReason >> 8); closePayload.push_back(closeReason & 0xff); - std::vector closeFrame - = EncodeFrame(SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + std::vector closeFrame = EncodeFrame(SocketOpcode::Close, true, closePayload); m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); auto closeResponse = ReceiveFrame(context, true); @@ -224,8 +225,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names closePayload.push_back(closeStatus & 0xff); closePayload.insert(closePayload.end(), closeReason.begin(), closeReason.end()); - std::vector closeFrame - = EncodeFrame(SocketOpcode::Close, m_options.EnableMasking, true, closePayload); + std::vector closeFrame = EncodeFrame(SocketOpcode::Close, true, closePayload); m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); auto closeResponse = ReceiveFrame(context, true); @@ -261,8 +261,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } else { - std::vector sendFrame - = EncodeFrame(SocketOpcode::TextFrame, m_options.EnableMasking, isFinalFrame, utf8text); + std::vector sendFrame = EncodeFrame(SocketOpcode::TextFrame, isFinalFrame, utf8text); m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); } @@ -289,8 +288,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } else { - std::vector sendFrame = EncodeFrame( - SocketOpcode::BinaryFrame, m_options.EnableMasking, isFinalFrame, binaryFrame); + std::vector sendFrame + = EncodeFrame(SocketOpcode::BinaryFrame, isFinalFrame, binaryFrame); std::unique_lock transportLock(m_transportMutex); m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); @@ -339,22 +338,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketOpcode opcode; bool isFinal = false; - bool isMasked = false; uint64_t payloadLength; - std::array maskKey{}; - std::vector frameData = DecodeFrame( - m_bufferedStreamReader, opcode, payloadLength, isFinal, isMasked, maskKey, context); + std::vector frameData + = DecodeFrame(m_bufferedStreamReader, opcode, payloadLength, isFinal, context); - if (isMasked) - { - int index = 0; - std::transform( - frameData.begin(), frameData.end(), frameData.begin(), [&maskKey, &index](uint8_t val) { - val ^= maskKey[index % 4]; - index += 1; - return val; - }); - } // At this point, readBuffer contains the actual payload from the service. switch (opcode) { @@ -427,7 +414,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector WebSocketImplementation::EncodeFrame( SocketOpcode opcode, - bool maskOutput, bool isFinal, std::vector const& payload) { @@ -435,10 +421,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Add opcode+fin. encodedFrame.push_back(static_cast(opcode) | (isFinal ? 0x80 : 0)); uint8_t maskAndLength = 0; - if (maskOutput) - { - maskAndLength |= 0x80; - } + maskAndLength |= 0x80; + // Payloads smaller than 125 bytes are encoded directly in the maskAndLength field. uint64_t payloadSize = static_cast(payload.size()); if (payloadSize <= 125) @@ -479,7 +463,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } // Calculate the masking key. This MUST be 4 bytes of high entropy random numbers used to // mask the input data. - if (maskOutput) { // Start by generating the mask - 4 bytes of random data. std::vector mask = GenerateRandomBytes(4); @@ -495,11 +478,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names index += 1; } } - else - { - // Since the payload is unmasked, simply append the payload to the encoded frame. - encodedFrame.insert(encodedFrame.end(), payload.begin(), payload.end()); - } return encodedFrame; } @@ -509,8 +487,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketOpcode& opcode, uint64_t& payloadLength, bool& isFinal, - bool& isMasked, - std::array& maskKey, Azure::Core::Context const& context) { std::unique_lock lock(m_transportMutex); @@ -522,10 +498,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names opcode = static_cast(payloadByte & 0x7f); isFinal = (payloadByte & 0x80) != 0; payloadByte = streamReader.ReadByte(context); - isMasked = false; if (payloadByte & 0x80) { - isMasked = true; + throw std::runtime_error("Server sent a frame with a reserved bit set."); } payloadLength = payloadByte & 0x7f; if (payloadLength <= 125) @@ -545,13 +520,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::logic_error("Unexpected payload length."); } - if (isMasked) - { - maskKey[0] = streamReader.ReadByte(context); - maskKey[1] = streamReader.ReadByte(context); - maskKey[2] = streamReader.ReadByte(context); - maskKey[3] = streamReader.ReadByte(context); - }; return streamReader.ReadBytes(static_cast(payloadLength), context); } diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 5abf73b365..b61915ced5 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -76,6 +76,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Implement a buffered stream reader class BufferedStreamReader { std::shared_ptr m_transport; + std::unique_ptr m_initialBodyStream; constexpr static size_t m_bufferSize = 1024; uint8_t m_buffer[m_bufferSize]{}; size_t m_bufferPos = 0; @@ -86,6 +87,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names explicit BufferedStreamReader() = default; ~BufferedStreamReader() = default; + void SetInitialStream(std::unique_ptr& stream) + { + m_initialBodyStream = std::move(stream); + } void SetTransport( std::shared_ptr& transport) { @@ -96,7 +101,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { if (m_bufferPos >= m_bufferLen) { - m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + // Start by reading data from our initial body stream. + m_bufferLen = m_initialBodyStream->ReadToCount(m_buffer, m_bufferSize, context); + if (m_bufferLen == 0) + { + // If we run out of the initial stream, we need to read from the transport. + m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + } m_bufferPos = 0; if (m_bufferLen == 0) { @@ -263,7 +274,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names */ std::vector EncodeFrame( SocketOpcode opcode, - bool maskOutput, bool isFinal, std::vector const& payload); @@ -281,8 +291,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names SocketOpcode& opcode, uint64_t& payloadLength, bool& isFinal, - bool& isMasked, - std::array& maskKey, Azure::Core::Context const& context); SocketState m_state{SocketState::Invalid}; diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 4e2842ce6e..9c22f57fd5 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -191,6 +191,11 @@ TEST_F(WebSocketTests, CloseDuringEcho) { WebSocket testSocket(Azure::Core::Url("http://localhost:8000/closeduringecho")); + EXPECT_THROW(testSocket.SendFrame("Foo", true), std::runtime_error); + std::vector data{1, 2, 3, 4}; + EXPECT_THROW(testSocket.SendFrame(data, true), std::runtime_error); + EXPECT_THROW(testSocket.ReceiveFrame(), std::runtime_error); + testSocket.Open(); testSocket.SendFrame("Test message", true); @@ -323,7 +328,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) // Does not work because curl rejects the wss: scheme. class LibWebSocketIncrementProtocol { - WebSocketOptions m_options{true, {"dumb-increment-protocol"}}; + WebSocketOptions m_options{{"dumb-increment-protocol"}}; WebSocket m_socket; public: From 23efd9122f925ede0090e284c24abe36f4e38a79 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 13:43:49 -0700 Subject: [PATCH 088/149] fixed docstrings --- .../azure-core/inc/azure/core/http/websockets/websockets.hpp | 1 - sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp | 1 - 2 files changed, 2 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 8a0b58a23e..6ad1f41dad 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -118,7 +118,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Construct an instance of a WebSocketOptions type. * - * @param enableMasking If true, enable masking for the websocket * @param protocols Supported protocols for this websocket client. */ explicit WebSocketOptions(std::vector protocols) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index b61915ced5..532efa29b2 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -283,7 +283,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names * @param streamReader Buffered stream reader to read the frame from. * @param opcode Opcode returned by the server. * @param isFinal True if this is the final message. - * @param maskKey On Return, contains the contents of the mask key if the data is masked. * @returns A pointer to the start of the decoded data. */ std::vector DecodeFrame( From 46fb880e15c127f8717779553db9c873fb2891b3 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 14:49:35 -0700 Subject: [PATCH 089/149] Drain in-flight messages when processing close --- .../src/http/websockets/websocketsimpl.cpp | 30 +++++++++++++++++-- .../src/http/websockets/websocketsimpl.hpp | 7 +++++ .../azure-core/test/ut/websocket_test.cpp | 5 +++- sdk/core/ci.yml | 4 +-- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index a6ce4a40a8..b0cc38f703 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -9,6 +9,7 @@ #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) #include "azure/core/http/websockets/curl_websockets_transport.hpp" #endif +#include "azure/core/internal/diagnostics/log.hpp" #include #include #include @@ -16,6 +17,9 @@ #include namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + using namespace Azure::Core::Diagnostics::_internal; + using namespace Azure::Core::Diagnostics; + using namespace std::chrono_literals; WebSocketImplementation::WebSocketImplementation( Azure::Core::Url const& remoteUrl, @@ -191,10 +195,27 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector closeFrame = EncodeFrame(SocketOpcode::Close, true, closePayload); m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); - auto closeResponse = ReceiveFrame(context, true); - if (closeResponse->ResultType != WebSocketResultType::PeerClosed) + // To ensure that we process the responses in a "timely" fashion, limit the close reception to + // 20 seconds if we don't already have a timeout. + Azure::Core::Context closeContext = context; + auto cancelTimepoint = closeContext.GetDeadline(); + if (cancelTimepoint == Azure::DateTime::max()) { - throw std::runtime_error("Unexpected result type received during close()."); + closeContext = closeContext.WithDeadline(std::chrono::system_clock::now() + 20s); + } + // Drain the incoming series of frames from the server. + // Note that there might be in-flight frames that were sent from the other end of the + // WebSocket that we don't care about any more (since we're closing the WebSocket). So drain + // those frames. + auto closeResponse = ReceiveFrame(closeContext, true); + while (closeResponse->ResultType != WebSocketResultType::PeerClosed) + { + auto textResult = closeResponse->AsTextFrame(); + Log::Write( + Logger::Level::Warning, + "Received unexpected frame during close. Frame type: " + + std::to_string(static_cast(closeResponse->ResultType))); + closeResponse = ReceiveFrame(closeContext, true); } } // Close the socket - after this point, the m_transport is invalid. @@ -342,6 +363,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector frameData = DecodeFrame(m_bufferedStreamReader, opcode, payloadLength, isFinal, context); + Log::Write( + Azure::Core::Diagnostics::Logger::Level::Informational, + "Received frame with opcode: " + std::to_string(static_cast(opcode))); // At this point, readBuffer contains the actual payload from the service. switch (opcode) { diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 532efa29b2..0979b04dd8 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT #include "azure/core/http/websockets/websockets.hpp" #include "azure/core/http/websockets/websockets_transport.hpp" +#include "azure/core/internal/diagnostics/log.hpp" #include "azure/core/internal/http/pipeline.hpp" #include #include @@ -108,6 +109,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // If we run out of the initial stream, we need to read from the transport. m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); } + else + { + Azure::Core::Diagnostics::_internal::Log::Write( + Azure::Core::Diagnostics::Logger::Level::Informational, + "Read data from initial stream"); + } m_bufferPos = 0; if (m_bufferLen == 0) { diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 9c22f57fd5..af0315876d 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -433,7 +433,7 @@ class LibWebSocketStatus { } }; -TEST_F(WebSocketTests, LibWebSocketOrg) +TEST_F(WebSocketTests, LibWebSocketOrgLwsStatus) { { LibWebSocketStatus lwsStatus; @@ -459,6 +459,9 @@ TEST_F(WebSocketTests, LibWebSocketOrg) } EXPECT_TRUE(foundOurConnection); } +} +TEST_F(WebSocketTests, LibWebSocketOrgIncrement) +{ { LibWebSocketIncrementProtocol incrementProtocol; incrementProtocol.Open(); diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 9f08c5b6f9..8aa8662cd7 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -44,9 +44,9 @@ stages: BranchCoverageTarget: 55 PreTestSteps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' + displayName: 'Use Python 3' inputs: - versionSpec: '3.10.5' + versionSpec: '3' condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - pwsh: | From b3b75b8affb20ec2013c7c4f7e2fc587f8d664c4 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 16:08:28 -0700 Subject: [PATCH 090/149] Added test for binary fragmentation --- .../azure-core/test/ut/websocket_server.py | 6 ++++- .../azure-core/test/ut/websocket_test.cpp | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 3cb7bcc542..7b95f7b6ff 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -45,7 +45,11 @@ async def handleEcho(websocket, url): try: data = await websocket.recv() # print(f'Echo ', len(data),' bytes') - await websocket.send(data) + + if (url.query == 'fragment=true'): + await websocket.send(data.split()) + else: + await websocket.send(data) except websockets.ConnectionClosedOK: print("Connection closed ok.") return diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index af0315876d..b2bec1ff4e 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -130,6 +130,32 @@ TEST_F(WebSocketTests, SimpleEcho) // Close the socket gracefully. testSocket.Close(); } + + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?fragment=true")); + + testSocket.Open(); + + std::vector binaryData{1, 2, 3, 4, 5, 6}; + + testSocket.SendFrame(binaryData, true); + + std::vector responseData; + std::shared_ptr response; + do + { + response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + auto binaryResult = response->AsBinaryFrame(); + responseData.insert(responseData.end(), binaryResult->Data.begin(), binaryResult->Data.end()); + } while (!response->IsFinalFrame); + + auto textResult = response->AsBinaryFrame(); + EXPECT_EQ(binaryData, responseData); + + // Close the socket gracefully. + testSocket.Close(); + } } template void EchoRandomData(WebSocket& socket) From efe18bf63466650a08d53d2edae9ee67effc4bf9 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 17:01:54 -0700 Subject: [PATCH 091/149] More code coverage tweaks --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 3 --- sdk/core/azure-core/test/ut/websocket_test.cpp | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index b0cc38f703..e1b4e5d247 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -363,9 +363,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector frameData = DecodeFrame(m_bufferedStreamReader, opcode, payloadLength, isFinal, context); - Log::Write( - Azure::Core::Diagnostics::Logger::Level::Informational, - "Received frame with opcode: " + std::to_string(static_cast(opcode))); // At this point, readBuffer contains the actual payload from the service. switch (opcode) { diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index b2bec1ff4e..b1811cb9c9 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -107,6 +107,7 @@ TEST_F(WebSocketTests, SimpleEcho) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); + EXPECT_THROW(response->AsBinaryFrame(), std::runtime_error); auto textResult = response->AsTextFrame(); EXPECT_EQ("Test message", textResult->Text); @@ -124,6 +125,8 @@ TEST_F(WebSocketTests, SimpleEcho) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + EXPECT_THROW(response->AsPeerCloseFrame(), std::runtime_error); + EXPECT_THROW(response->AsTextFrame(), std::runtime_error); auto textResult = response->AsBinaryFrame(); EXPECT_EQ(binaryData, textResult->Data); @@ -215,7 +218,7 @@ TEST_F(WebSocketTests, VariableSizeEcho) TEST_F(WebSocketTests, CloseDuringEcho) { { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/closeduringecho")); + WebSocket testSocket(Azure::Core::Url("ws://localhost:8000/closeduringecho")); EXPECT_THROW(testSocket.SendFrame("Foo", true), std::runtime_error); std::vector data{1, 2, 3, 4}; @@ -288,6 +291,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); auto binaryResult = response->AsBinaryFrame(); + // Make sure we get back the data we sent in the echo request. EXPECT_EQ(sendData.size(), binaryResult->Data.size()); { From a5821505ea4a776f58f2e43ef40d171a9c5120ba Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 17:48:38 -0700 Subject: [PATCH 092/149] Fixed exception thrown on wrong AsXxx --- sdk/core/azure-core/test/ut/websocket_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index b1811cb9c9..4d743b193c 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -107,7 +107,7 @@ TEST_F(WebSocketTests, SimpleEcho) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); - EXPECT_THROW(response->AsBinaryFrame(), std::runtime_error); + EXPECT_THROW(response->AsBinaryFrame(), std::logic_error); auto textResult = response->AsTextFrame(); EXPECT_EQ("Test message", textResult->Text); @@ -125,8 +125,8 @@ TEST_F(WebSocketTests, SimpleEcho) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); - EXPECT_THROW(response->AsPeerCloseFrame(), std::runtime_error); - EXPECT_THROW(response->AsTextFrame(), std::runtime_error); + EXPECT_THROW(response->AsPeerCloseFrame(), std::logic_error); + EXPECT_THROW(response->AsTextFrame(), std::logic_error); auto textResult = response->AsBinaryFrame(); EXPECT_EQ(binaryData, textResult->Data); From ff6a811d89f70402bfef7e68cf3c822f45c92a71 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 11 Jul 2022 20:35:41 -0700 Subject: [PATCH 093/149] clang_format --- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 4d743b193c..981f5b3a3b 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -291,7 +291,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) auto response = testSocket.ReceiveFrame(); EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); auto binaryResult = response->AsBinaryFrame(); - + // Make sure we get back the data we sent in the echo request. EXPECT_EQ(sendData.size(), binaryResult->Data.size()); { From 6cfc9d7595dd180ae552539c933bb336fb5e949c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 12 Jul 2022 06:09:10 -0700 Subject: [PATCH 094/149] More code coverage --- .../test/ut/azure_core_otel_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core-tracing-opentelemetry/test/ut/azure_core_otel_test.cpp b/sdk/core/azure-core-tracing-opentelemetry/test/ut/azure_core_otel_test.cpp index bba341df18..ded42c49d3 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/test/ut/azure_core_otel_test.cpp +++ b/sdk/core/azure-core-tracing-opentelemetry/test/ut/azure_core_otel_test.cpp @@ -531,6 +531,9 @@ TEST_F(OpenTelemetryTests, SetStatus) span->SetStatus(Azure::Core::Tracing::_internal::SpanStatus::Error, {}); span->SetStatus(Azure::Core::Tracing::_internal::SpanStatus::Ok, {}); + EXPECT_THROW( + span->SetStatus(static_cast(357), {}), + std::runtime_error); span->End({}); @@ -553,7 +556,7 @@ TEST_F(OpenTelemetryTests, SetStatus) span->SetStatus(Azure::Core::Tracing::_internal::SpanStatus::Error, "Something went wrong."); - span->End({}); + span->End(Azure::DateTime(std::chrono::system_clock::now())); // Return the collected spans. auto spans = m_spanData->GetSpans(); From 50f02ce2676df5bdbdbbffe5586b41361052bd04 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 09:41:51 -0700 Subject: [PATCH 095/149] Teach WinHTTP transport that WebSockets is also a non-https scheme --- sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 4239545523..10fda41f0c 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -20,6 +20,7 @@ using namespace Azure::Core::Http; namespace { const std::string HttpScheme = "http"; +const std::string WebSocketScheme = "ws"; inline std::wstring HttpMethodToWideString(HttpMethod method) { @@ -325,7 +326,9 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( HttpMethod requestMethod = method; bool const requestSecureHttp( !Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( - url.GetScheme(), HttpScheme)); + url.GetScheme(), HttpScheme) + && !Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + url.GetScheme(), WebSocketScheme)); // Create an HTTP request handle. _detail::unique_HINTERNET hi( From 9044b5da15032f1f3334237de380628aa9a5b7d5 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 09:49:37 -0700 Subject: [PATCH 096/149] Disable native transport support for curl transports to improve code coverage numbers --- .../src/http/websockets/websocketsimpl.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index e1b4e5d247..35642dfd6d 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -4,10 +4,15 @@ #include "azure/core/base64.hpp" #include "azure/core/http/policies/policy.hpp" #include "azure/core/internal/cryptography/sha_hash.hpp" +// SUPPORT_NATIVE_TRANSPORT indicates if WinHTTP should be compiled with native transport support +// or not. +// Note that this is primarily required to improve the code coverage numbers in the CI pipeline. #if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) #include "azure/core/http/websockets/win_http_websockets_transport.hpp" +#define SUPPORT_NATIVE_TRANSPORT 1 #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) #include "azure/core/http/websockets/curl_websockets_transport.hpp" +#define SUPPORT_NATIVE_TRANSPORT 0 #endif #include "azure/core/internal/diagnostics/log.hpp" #include @@ -118,7 +123,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names auto& responseHeaders = response->GetHeaders(); if (!m_transport->NativeWebsocketSupport()) { - auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); if (socketAccept == responseHeaders.end()) { @@ -180,12 +184,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Socket is not open."); } m_state = SocketState::Closing; +#if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { m_transport->CloseSocket( static_cast(WebSocketErrorCode::EndpointDisappearing), "", context); } else +#endif { // Send a going away message to the server. uint16_t closeReason = static_cast(WebSocketErrorCode::EndpointDisappearing); @@ -235,11 +241,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } m_state = SocketState::Closing; +#if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { m_transport->CloseSocket(closeStatus, closeReason, context); } else +#endif { std::vector closePayload; closePayload.push_back(closeStatus >> 8); @@ -272,6 +280,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Socket is not open."); } std::vector utf8text(textFrame.begin(), textFrame.end()); +#if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { m_transport->SendFrame( @@ -281,6 +290,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names context); } else +#endif { std::vector sendFrame = EncodeFrame(SocketOpcode::TextFrame, isFinalFrame, utf8text); @@ -299,6 +309,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::runtime_error("Socket is not open."); } +#if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { m_transport->SendFrame( @@ -308,6 +319,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names context); } else +#endif { std::vector sendFrame = EncodeFrame(SocketOpcode::BinaryFrame, isFinalFrame, binaryFrame); @@ -332,6 +344,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::runtime_error("Socket is not open."); } +#if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { WebSocketTransport::WebSocketFrameType frameType; @@ -355,6 +368,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } else +#endif { SocketOpcode opcode; From ac056f6c9470338e86be6213d305df24a4b77a8f Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 10:24:04 -0700 Subject: [PATCH 097/149] Removed WinHTTP transport option for websockets, moved it to code --- .../inc/azure/core/http/transport.hpp | 6 ++++++ .../websockets/curl_websockets_transport.hpp | 1 + .../http/websockets/websockets_transport.hpp | 2 ++ .../win_http_websockets_transport.hpp | 2 ++ .../inc/azure/core/http/win_http_transport.hpp | 5 ----- sdk/core/azure-core/src/http/curl/curl.cpp | 15 +++++++++------ .../src/http/websockets/websocketsimpl.cpp | 3 --- .../src/http/winhttp/win_http_transport.cpp | 4 ++-- sdk/core/azure-core/test/ut/websocket_test.cpp | 17 ++++++++++++----- 9 files changed, 34 insertions(+), 21 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/transport.hpp b/sdk/core/azure-core/inc/azure/core/http/transport.hpp index 64d6f6da15..4d7c39bf6c 100644 --- a/sdk/core/azure-core/inc/azure/core/http/transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/transport.hpp @@ -66,6 +66,12 @@ namespace Azure { namespace Core { namespace Http { * @return A reference to this instance. */ HttpTransport& operator=(const HttpTransport& other) = default; + + /** + * @brief Returns true if the HttpTransport supports WebSockets (the ability to + * communicate bidirectionally on the TCP connection used by the HTTP transport). + */ + virtual bool SupportsWebSockets() const { return false; } }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index b1dd062bf6..28457caf36 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -138,6 +138,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) override; + bool SupportsWebSockets() const override { return true; } }; }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 8682667d62..e583240525 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -145,6 +145,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) = 0; + bool SupportsWebSockets() const override { return true; } + protected: /** * @brief Constructs a default instance of `%HttpTransport`. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index b24379ab92..02a0379190 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -128,6 +128,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { { throw std::runtime_error("Not implemented."); } + + bool SupportsWebSockets() const override { return true; } }; }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp index d7d970de51..8de11c98e5 100644 --- a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp @@ -106,11 +106,6 @@ namespace Azure { namespace Core { namespace Http { * @brief When `true`, allows an invalid certificate authority. */ bool IgnoreUnknownCertificateAuthority = false; - - /** - * @brief When `true`, enables WebSocket upgrade. - */ - bool EnableWebSocketUpgrade = false; }; /** diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index eacbc7b6bb..b584087b45 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -80,7 +80,9 @@ int pollSocketUntilEventOrTimeout( throw TransportException("Error while sending request. Platform does not support Poll()"); #endif - struct pollfd poller; + struct pollfd poller + { + }; poller.fd = socketFileDescriptor; // set direction @@ -327,10 +329,13 @@ std::unique_ptr CurlTransport::Send(Request& request, Context const } else { - std::unique_ptr upgradedConnection(session->GetUpgradedConnection()); - if (upgradedConnection) + if (SupportsWebSockets()) { - OnUpgradedConnection(upgradedConnection); + std::unique_ptr upgradedConnection(session->GetUpgradedConnection()); + if (upgradedConnection) + { + OnUpgradedConnection(upgradedConnection); + } } } @@ -1334,8 +1339,6 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo return connection; } } - lock.unlock(); // Why is this line here? std::unique_lock releases the lock when it leaves - // scope. } // Creating a new connection is thread safe. No need to lock mutex here. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 35642dfd6d..12f6be710e 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -43,9 +43,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) WinHttpTransportOptions transportOptions; - // WinHTTP overrides the Connection: Upgrade header, so disable keep-alive. - transportOptions.EnableWebSocketUpgrade = true; - m_transport = std::make_shared( transportOptions); m_options.Transport.Transport = m_transport; diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 10fda41f0c..8df8102156 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -377,7 +377,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( } } - if (m_options.EnableWebSocketUpgrade) + if (SupportsWebSockets()) { if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) { @@ -686,7 +686,7 @@ std::unique_ptr WinHttpTransport::Send(Request& request, Context co ReceiveResponse(requestHandle, context); - if (m_options.EnableWebSocketUpgrade) + if (SupportsWebSockets()) { OnResponseReceived(requestHandle); } diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 981f5b3a3b..566b2d99f1 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -220,11 +220,6 @@ TEST_F(WebSocketTests, CloseDuringEcho) { WebSocket testSocket(Azure::Core::Url("ws://localhost:8000/closeduringecho")); - EXPECT_THROW(testSocket.SendFrame("Foo", true), std::runtime_error); - std::vector data{1, 2, 3, 4}; - EXPECT_THROW(testSocket.SendFrame(data, true), std::runtime_error); - EXPECT_THROW(testSocket.ReceiveFrame(), std::runtime_error); - testSocket.Open(); testSocket.SendFrame("Test message", true); @@ -239,6 +234,18 @@ TEST_F(WebSocketTests, CloseDuringEcho) } } +TEST_F(WebSocketTests, ExpectThrow) +{ + { + WebSocket testSocket(Azure::Core::Url("ws://localhost:8000/closeduringecho")); + + EXPECT_THROW(testSocket.SendFrame("Foo", true), std::runtime_error); + std::vector data{1, 2, 3, 4}; + EXPECT_THROW(testSocket.SendFrame(data, true), std::runtime_error); + EXPECT_THROW(testSocket.ReceiveFrame(), std::runtime_error); + } +} + std::string ToHexString(std::vector const& data) { std::stringstream ss; From 51a63de0d0b5f99a17ae3b82b478b4a0d7da2380 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 10:36:55 -0700 Subject: [PATCH 098/149] Time out long running failure test --- sdk/core/azure-core/test/ut/websocket_test.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 566b2d99f1..3c679a88cf 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -62,7 +62,11 @@ TEST_F(WebSocketTests, OpenSimpleSocket) WebSocket defaultSocket(Azure::Core::Url("http://microsoft.com/index.htm"), options); defaultSocket.AddHeader("newHeader", "headerValue"); - EXPECT_THROW(defaultSocket.Open(), std::runtime_error); + // When running this test locally, the call times out, so drop in a 5 second timeout on + // the request. + Azure::Core::Context requestContext = Azure::Core::Context::ApplicationContext.WithDeadline( + std::chrono::system_clock::now() + 15s); + EXPECT_THROW(defaultSocket.Open(requestContext), std::runtime_error); } } From 0e77d3a2f20bb9381fba12e4b6ba11de8040d92f Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 13:56:09 -0700 Subject: [PATCH 099/149] Removed commented out code --- eng/pipelines/templates/jobs/ci.tests.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ff670e9123..87b5eb0f82 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -170,13 +170,6 @@ jobs: displayName: Publish test results condition: succeededOrFailed() - # - ${{ if eq(parameters.EnablePythonWebSocketServer, True) }}: - # - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - # parameters: - # ArtifactPath: '$(Build.SourcesDirectory)/WebSocketServer.log' - # ArtifactName: 'WebSocketLogs-$(Agent.JobName)_attempt_$(System.JobAttempt)' - # CustomCondition: contains(variables.CmakeArgs, 'BUILD_TESTING=ON') - - ${{ if eq(parameters.CoverageEnabled, true) }}: - pwsh: | $toolsDirectory = "$(Agent.TempDirectory)/coveragetools" From d891283bbc9ff15b0d379e18128ff23d45523335 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 13 Jul 2022 16:53:48 -0700 Subject: [PATCH 100/149] Added ping support to websockets --- .../azure/core/http/websockets/websockets.hpp | 49 +++++++++--- .../src/http/websockets/websockets.cpp | 29 +++++-- .../src/http/websockets/websocketsimpl.cpp | 44 ++++++++-- .../src/http/websockets/websocketsimpl.hpp | 5 +- .../azure-core/test/ut/websocket_server.py | 2 +- .../azure-core/test/ut/websocket_test.cpp | 80 ++++++++++++++----- 6 files changed, 162 insertions(+), 47 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 6ad1f41dad..b81521db71 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -21,12 +21,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketImplementation; } - enum class WebSocketResultType : int + enum class WebSocketFrameType : int { Unknown, TextFrameReceived, BinaryFrameReceived, - PeerClosed, + PeerClosedReceived, + PongReceived, }; enum class WebSocketErrorCode : uint16_t @@ -49,48 +50,61 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketTextFrame; class WebSocketBinaryFrame; class WebSocketPeerCloseFrame; + class WebSocketPongFrame; - struct WebSocketResult + struct WebSocketFrame { - WebSocketResultType ResultType; + WebSocketFrameType FrameType; bool IsFinalFrame{false}; std::shared_ptr AsTextFrame(); std::shared_ptr AsBinaryFrame(); std::shared_ptr AsPeerCloseFrame(); + std::shared_ptr AsPongFrame(); }; - class WebSocketTextFrame : public WebSocketResult, + class WebSocketTextFrame : public WebSocketFrame, public std::enable_shared_from_this { private: public: WebSocketTextFrame() = default; WebSocketTextFrame(bool isFinalFrame, unsigned char const* body, size_t size) - : WebSocketResult{WebSocketResultType::TextFrameReceived, isFinalFrame}, + : WebSocketFrame{WebSocketFrameType::TextFrameReceived, isFinalFrame}, Text(body, body + size) { } std::string Text; }; - class WebSocketBinaryFrame : public WebSocketResult, + class WebSocketBinaryFrame : public WebSocketFrame, public std::enable_shared_from_this { private: public: WebSocketBinaryFrame() = default; WebSocketBinaryFrame(bool isFinal, unsigned char const* body, size_t size) - : WebSocketResult{WebSocketResultType::BinaryFrameReceived, isFinal}, - Data(body, body + size) + : WebSocketFrame{WebSocketFrameType::BinaryFrameReceived, isFinal}, Data(body, body + size) { } std::vector Data; }; - class WebSocketPeerCloseFrame : public WebSocketResult, + class WebSocketPongFrame : public WebSocketFrame, + public std::enable_shared_from_this { + private: + public: + WebSocketPongFrame() = default; + WebSocketPongFrame(unsigned char const* body, size_t size) + : WebSocketFrame{WebSocketFrameType::PongReceived, true}, Data(body, body + size) + { + } + std::vector Data; + }; + + class WebSocketPeerCloseFrame : public WebSocketFrame, public std::enable_shared_from_this { public: WebSocketPeerCloseFrame() = default; WebSocketPeerCloseFrame(uint16_t remoteStatusCode, std::string const& remoteCloseReason) - : WebSocketResult{WebSocketResultType::PeerClosed}, RemoteStatusCode(remoteStatusCode), - RemoteCloseReason(remoteCloseReason) + : WebSocketFrame{WebSocketFrameType::PeerClosedReceived}, + RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) { } uint16_t RemoteStatusCode; @@ -194,7 +208,16 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @returns The received WebSocket frame. * */ - std::shared_ptr ReceiveFrame( + std::shared_ptr ReceiveFrame( + Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Send a 'Ping' frame to the remote server. + * + * @param pingData data to be sent in the ping operation. + * @param context Context for the operation. + */ + void SendPing( + std::vector const& pingData, Azure::Core::Context const& context = Azure::Core::Context{}); /** @brief AddHeader - Adds a header to the initial handshake. diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index 8182903238..d06badc8c6 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -46,11 +46,18 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { m_socketImplementation->SendFrame(binaryFrame, isFinalFrame, context); } - std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) + std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) { return m_socketImplementation->ReceiveFrame(context); } + void WebSocket::SendPing( + std::vector const& pingData, + Azure::Core::Context const& context) + { + m_socketImplementation->SendPing(pingData, context); + } + void WebSocket::AddHeader(std::string const& headerName, std::string const& headerValue) { m_socketImplementation->AddHeader(headerName, headerValue); @@ -62,31 +69,39 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { bool WebSocket::IsOpen() { return m_socketImplementation->IsOpen(); } - std::shared_ptr WebSocketResult::AsTextFrame() + std::shared_ptr WebSocketFrame::AsTextFrame() { - if (ResultType != WebSocketResultType::TextFrameReceived) + if (FrameType != WebSocketFrameType::TextFrameReceived) { throw std::logic_error("Cannot cast to TextFrameReceived."); } return static_cast(this)->shared_from_this(); } - std::shared_ptr WebSocketResult::AsBinaryFrame() + std::shared_ptr WebSocketFrame::AsBinaryFrame() { - if (ResultType != WebSocketResultType::BinaryFrameReceived) + if (FrameType != WebSocketFrameType::BinaryFrameReceived) { throw std::logic_error("Cannot cast to BinaryFrameReceived."); } return static_cast(this)->shared_from_this(); } - std::shared_ptr WebSocketResult::AsPeerCloseFrame() + std::shared_ptr WebSocketFrame::AsPeerCloseFrame() { - if (ResultType != WebSocketResultType::PeerClosed) + if (FrameType != WebSocketFrameType::PeerClosedReceived) { throw std::logic_error("Cannot cast to PeerClose."); } return static_cast(this)->shared_from_this(); } + std::shared_ptr WebSocketFrame::AsPongFrame() + { + if (FrameType != WebSocketFrameType::PongReceived) + { + throw std::logic_error("Cannot cast to PongReceived."); + } + return static_cast(this)->shared_from_this(); + } }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 12f6be710e..c2eebeeca6 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -211,13 +211,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // WebSocket that we don't care about any more (since we're closing the WebSocket). So drain // those frames. auto closeResponse = ReceiveFrame(closeContext, true); - while (closeResponse->ResultType != WebSocketResultType::PeerClosed) + while (closeResponse->FrameType != WebSocketFrameType::PeerClosedReceived) { auto textResult = closeResponse->AsTextFrame(); Log::Write( Logger::Level::Warning, "Received unexpected frame during close. Frame type: " - + std::to_string(static_cast(closeResponse->ResultType))); + + std::to_string(static_cast(closeResponse->FrameType))); closeResponse = ReceiveFrame(closeContext, true); } } @@ -255,7 +255,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); auto closeResponse = ReceiveFrame(context, true); - if (closeResponse->ResultType != WebSocketResultType::PeerClosed) + if (closeResponse->FrameType != WebSocketFrameType::PeerClosedReceived) { throw std::runtime_error("Unexpected result type received during close()."); } @@ -326,7 +326,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } - std::shared_ptr WebSocketImplementation::ReceiveFrame( + std::shared_ptr WebSocketImplementation::ReceiveFrame( Azure::Core::Context const& context, bool stateIsLocked) { @@ -335,6 +335,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names if (!stateIsLocked) { lock.lock(); + stateIsLocked = true; } if (m_state != SocketState::Open && m_state != SocketState::Closing) @@ -407,9 +408,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names errorCode, std::string(frameData.begin() + 2, frameData.end())); } case SocketOpcode::Ping: + // Respond to the ping and then recurse to process the next frame. + SendPong(frameData, context); + return ReceiveFrame(context, stateIsLocked); case SocketOpcode::Pong: - throw std::runtime_error("Unexpected Ping/Pong opcode received."); - break; + return std::make_shared(frameData.data(), frameData.size()); + case SocketOpcode::Continuation: if (m_currentMessageType == SocketMessageType::Text) { @@ -444,6 +448,34 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #endif } + void WebSocketImplementation::SendPing( + std::vector const& pingData, + Azure::Core::Context const& context) + { + { + + std::unique_lock lock(m_stateMutex); + + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + } + std::vector pingFrame = EncodeFrame(SocketOpcode::Ping, true, pingData); + + std::unique_lock transportLock(m_transportMutex); + m_transport->SendBuffer(pingFrame.data(), pingFrame.size(), context); + } + void WebSocketImplementation::SendPong( + std::vector const& pongData, + Azure::Core::Context const& context) + { + std::vector pongFrame = EncodeFrame(SocketOpcode::Pong, true, pongData); + + std::unique_lock transportLock(m_transportMutex); + m_transport->SendBuffer(pongFrame.data(), pongFrame.size(), context); + } + std::vector WebSocketImplementation::EncodeFrame( SocketOpcode opcode, bool isFinal, diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 0979b04dd8..1ea87c73e3 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -42,9 +42,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context); - std::shared_ptr ReceiveFrame( + std::shared_ptr ReceiveFrame( Azure::Core::Context const& context, bool stateIsLocked = false); + void SendPing(std::vector const& pingData, Azure::Core::Context const& context); void AddHeader(std::string const& headerName, std::string const& headerValue); @@ -299,6 +300,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool& isFinal, Azure::Core::Context const& context); + void SendPong(std::vector const& pongData, Azure::Core::Context const& context); + SocketState m_state{SocketState::Invalid}; std::vector GenerateRandomKey() { return GenerateRandomBytes(16); }; diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 7b95f7b6ff..29c1dd4613 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -94,7 +94,7 @@ async def main(): print("Starting server") loop = asyncio.get_running_loop() stop = loop.create_future() - async with websockets.serve(handler, "localhost", 8000): + async with websockets.serve(handler, "localhost", 8000, ping_interval=3): await stop # run forever. if __name__=="__main__": diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 3c679a88cf..addc570651 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -29,7 +29,7 @@ class WebSocketTests : public testing::Test { // controlSocket.Open(); // controlSocket.SendFrame("close", true); // auto controlResponse = controlSocket.ReceiveFrame(); - // EXPECT_EQ(controlResponse->ResultType, WebSocketResultType::TextFrameReceived); + // EXPECT_EQ(controlResponse->FrameType, WebSocketFrameType::TextFrameReceived); } }; @@ -110,7 +110,7 @@ TEST_F(WebSocketTests, SimpleEcho) testSocket.SendFrame("Test message", true); auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::TextFrameReceived, response->ResultType); + EXPECT_EQ(WebSocketFrameType::TextFrameReceived, response->FrameType); EXPECT_THROW(response->AsBinaryFrame(), std::logic_error); auto textResult = response->AsTextFrame(); EXPECT_EQ("Test message", textResult->Text); @@ -128,7 +128,7 @@ TEST_F(WebSocketTests, SimpleEcho) testSocket.SendFrame(binaryData, true); auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); EXPECT_THROW(response->AsPeerCloseFrame(), std::logic_error); EXPECT_THROW(response->AsTextFrame(), std::logic_error); auto textResult = response->AsBinaryFrame(); @@ -148,11 +148,11 @@ TEST_F(WebSocketTests, SimpleEcho) testSocket.SendFrame(binaryData, true); std::vector responseData; - std::shared_ptr response; + std::shared_ptr response; do { response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); auto binaryResult = response->AsBinaryFrame(); responseData.insert(responseData.end(), binaryResult->Data.begin(), binaryResult->Data.end()); } while (!response->IsFinalFrame); @@ -165,6 +165,48 @@ TEST_F(WebSocketTests, SimpleEcho) } } +TEST_F(WebSocketTests, PingTest) +{ + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + testSocket.SendFrame("Test message", true); + // Sleep for 10s - this should trigger a ping. + // The websocket server is configured to ping every 5 seconds, so the 10 second sleep should + // force the next frame received to be a ping frame. + std::this_thread::sleep_for(10s); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketFrameType::TextFrameReceived, response->FrameType); + auto textResult = response->AsTextFrame(); + EXPECT_EQ("Test message", textResult->Text); + + // Close the socket gracefully. + testSocket.Close(); + } + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + // Sleep for 10s - this should trigger a ping. + // The websocket server is configured to ping every 5 seconds, so the 10 second sleep should + // force the next frame received to be a ping frame. + testSocket.SendPing({1, 2, 3, 4}); + + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketFrameType::PongReceived, response->FrameType); + auto pongResult = response->AsPongFrame(); + std::vector pongData{1, 2, 3, 4}; + EXPECT_EQ(pongResult->Data, pongData); + + // Close the socket gracefully. + testSocket.Close(); + } +} + template void EchoRandomData(WebSocket& socket) { std::vector sendData = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(N); @@ -173,11 +215,11 @@ template void EchoRandomData(WebSocket& socket) std::vector receiveData; - std::shared_ptr response; + std::shared_ptr response; do { response = socket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); auto binaryResult = response->AsBinaryFrame(); receiveData.insert(receiveData.end(), binaryResult->Data.begin(), binaryResult->Data.end()); } while (!response->IsFinalFrame); @@ -229,9 +271,9 @@ TEST_F(WebSocketTests, CloseDuringEcho) testSocket.SendFrame("Test message", true); auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::PeerClosed, response->ResultType); - auto peerClosed = response->AsPeerCloseFrame(); - EXPECT_EQ(1001, peerClosed->RemoteStatusCode); + EXPECT_EQ(WebSocketFrameType::PeerClosedReceived, response->FrameType); + auto PeerClosedReceived = response->AsPeerCloseFrame(); + EXPECT_EQ(1001, PeerClosedReceived->RemoteStatusCode); // Close the socket gracefully. testSocket.Close(); @@ -300,7 +342,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) testSocket.SendFrame(sendData, true); auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::BinaryFrameReceived, response->ResultType); + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); auto binaryResult = response->AsBinaryFrame(); // Make sure we get back the data we sent in the echo request. @@ -384,17 +426,17 @@ class LibWebSocketIncrementProtocol { Azure::Core::Context contextWithTimeout = Azure::Core::Context().WithDeadline(std::chrono::system_clock::now() + 10s); auto work = m_socket.ReceiveFrame(contextWithTimeout); - if (work->ResultType == WebSocketResultType::TextFrameReceived) + if (work->FrameType == WebSocketFrameType::TextFrameReceived) { auto frame = work->AsTextFrame(); return std::atoi(frame->Text.c_str()); } - if (work->ResultType == WebSocketResultType::BinaryFrameReceived) + if (work->FrameType == WebSocketFrameType::BinaryFrameReceived) { auto frame = work->AsBinaryFrame(); throw std::runtime_error("Not implemented"); } - else if (work->ResultType == WebSocketResultType::PeerClosed) + else if (work->FrameType == WebSocketFrameType::PeerClosedReceived) { GTEST_LOG_(INFO) << "Remote server closed connection." << std::endl; throw std::runtime_error("Remote server closed connection."); @@ -417,7 +459,7 @@ class LibWebSocketIncrementProtocol { while (m_socket.IsOpen()) { auto work = m_socket.ReceiveFrame(); - if (work->ResultType == WebSocketResultType::PeerClosed) + if (work->FrameType == WebSocketFrameType::PeerClosedReceived) { auto peerClose = work->AsPeerCloseFrame(); GTEST_LOG_(INFO) << "Peer closed. Remote Code: " << std::dec << peerClose->RemoteStatusCode @@ -429,7 +471,7 @@ class LibWebSocketIncrementProtocol { GTEST_LOG_(INFO) << std::endl; return; } - else if (work->ResultType == WebSocketResultType::TextFrameReceived) + else if (work->FrameType == WebSocketFrameType::TextFrameReceived) { auto frame = work->AsTextFrame(); GTEST_LOG_(INFO) << "Ignoring " << frame->Text << std::endl; @@ -457,13 +499,13 @@ class LibWebSocketStatus { // protocols. EXPECT_EQ("lws-status", serverSocket.GetChosenProtocol()); std::string returnValue; - std::shared_ptr lwsStatus; + std::shared_ptr lwsStatus; do { lwsStatus = serverSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketResultType::TextFrameReceived, lwsStatus->ResultType); - if (lwsStatus->ResultType == WebSocketResultType::TextFrameReceived) + EXPECT_EQ(WebSocketFrameType::TextFrameReceived, lwsStatus->FrameType); + if (lwsStatus->FrameType == WebSocketFrameType::TextFrameReceived) { auto textFrame = lwsStatus->AsTextFrame(); returnValue.insert(returnValue.end(), textFrame->Text.begin(), textFrame->Text.end()); From a11167be4b857bcf9baec9f3f16b18b22cf10a75 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 10:25:30 -0700 Subject: [PATCH 101/149] Set state to closed when receiving closed frame --- .../azure-core/src/http/websockets/websocketsimpl.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index c2eebeeca6..4445f75599 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -396,14 +396,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names errorCode |= (frameData[0] << 8) & 0xff00; errorCode |= (frameData[1] & 0x00ff); - // Update our state to be closed once we've received a closed frame. We only need to - // do this if our state is not currently locked. - if (!stateIsLocked) - { - lock.unlock(); - std::unique_lock closeLock(m_stateMutex); - m_state = SocketState::Closed; - } + // Update our state to be closed once we've received a closed frame. + m_state = SocketState::Closed; return std::make_shared( errorCode, std::string(frameData.begin() + 2, frameData.end())); } From a0c4d08b6e448a40456e6a495a7111fd66bf616b Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 10:59:53 -0700 Subject: [PATCH 102/149] Ping support on winhttp --- .../azure/core/http/websockets/websockets.hpp | 5 ++++- .../src/http/websockets/websockets.cpp | 4 ++-- .../src/http/websockets/websocketsimpl.cpp | 9 ++++++-- .../src/http/websockets/websocketsimpl.hpp | 9 +++++++- .../azure-core/test/ut/websocket_test.cpp | 21 +++++++++---------- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index b81521db71..bcbfaf428f 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -215,8 +215,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @param pingData data to be sent in the ping operation. * @param context Context for the operation. + * + * @returns True if the "Ping" was sent to the server, False if the underlying transport + * does not support ping operations. */ - void SendPing( + bool SendPing( std::vector const& pingData, Azure::Core::Context const& context = Azure::Core::Context{}); diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index d06badc8c6..34a9304fba 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -51,11 +51,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return m_socketImplementation->ReceiveFrame(context); } - void WebSocket::SendPing( + bool WebSocket::SendPing( std::vector const& pingData, Azure::Core::Context const& context) { - m_socketImplementation->SendPing(pingData, context); + return m_socketImplementation->SendPing(pingData, context); } void WebSocket::AddHeader(std::string const& headerName, std::string const& headerValue) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 4445f75599..ff0dc26433 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -442,12 +442,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #endif } - void WebSocketImplementation::SendPing( + bool WebSocketImplementation::SendPing( std::vector const& pingData, Azure::Core::Context const& context) { { - std::unique_lock lock(m_stateMutex); if (m_state != SocketState::Open) @@ -455,10 +454,16 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Socket is not open."); } } + if (m_transport->NativeWebsocketSupport()) + { + return false; + } + std::vector pingFrame = EncodeFrame(SocketOpcode::Ping, true, pingData); std::unique_lock transportLock(m_transportMutex); m_transport->SendBuffer(pingFrame.data(), pingFrame.size(), context); + return true; } void WebSocketImplementation::SendPong( std::vector const& pongData, diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 1ea87c73e3..736e4a4433 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -45,7 +45,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::shared_ptr ReceiveFrame( Azure::Core::Context const& context, bool stateIsLocked = false); - void SendPing(std::vector const& pingData, Azure::Core::Context const& context); + + /** + * @brief Send a "ping" frame to the other side of the WebSocket. + * + * @returns True if the ping was sent, false if the underlying transport didn't support "Ping" + * operations. + */ + bool SendPing(std::vector const& pingData, Azure::Core::Context const& context); void AddHeader(std::string const& headerName, std::string const& headerValue); diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index addc570651..197f2f3ee2 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -191,17 +191,16 @@ TEST_F(WebSocketTests, PingTest) testSocket.Open(); - // Sleep for 10s - this should trigger a ping. - // The websocket server is configured to ping every 5 seconds, so the 10 second sleep should - // force the next frame received to be a ping frame. - testSocket.SendPing({1, 2, 3, 4}); - - auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketFrameType::PongReceived, response->FrameType); - auto pongResult = response->AsPongFrame(); - std::vector pongData{1, 2, 3, 4}; - EXPECT_EQ(pongResult->Data, pongData); - + // Send a "Ping" to the remote server. If the "Ping" operation is supported, + // wait until the corresponding "Pong" response is received. + if (testSocket.SendPing({1, 2, 3, 4})) + { + auto response = testSocket.ReceiveFrame(); + EXPECT_EQ(WebSocketFrameType::PongReceived, response->FrameType); + auto pongResult = response->AsPongFrame(); + std::vector pongData{1, 2, 3, 4}; + EXPECT_EQ(pongResult->Data, pongData); + } // Close the socket gracefully. testSocket.Close(); } From b9785826374824d7662f1f52afbc05311aa27dac Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 12:19:44 -0700 Subject: [PATCH 103/149] Comment updates --- sdk/core/azure-core/inc/azure/core/base64.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index 41c8767209..4f742fbcf8 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -48,6 +48,13 @@ namespace Azure { namespace Core { */ static std::string Base64Encode(uint8_t const* const data, size_t length); + /** + * @brief Encodes an array of binary data into a Base64 encoded string. + * + * @param data The binary data to be encoded. + * @param length The length of the binaryData parameter. + * @return The UTF-8 encoded text in Base64. + */ template static std::string Base64Encode(std::array const& data) { From 75b9487ab21ad87bdbf63881b6d147bfe4fb959b Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 13:49:38 -0700 Subject: [PATCH 104/149] Code review feedback --- sdk/core/azure-core/inc/azure/core/base64.hpp | 35 +++++-------------- .../inc/azure/core/http/curl_transport.hpp | 6 ++++ .../websockets/curl_websockets_transport.hpp | 30 +++++++++++----- .../http/websockets/websockets_transport.hpp | 6 ++-- .../win_http_websockets_transport.hpp | 3 +- sdk/core/azure-core/src/base64.cpp | 7 +--- .../src/http/websockets/websocketsimpl.cpp | 16 +++++---- .../src/http/winhttp/win_http_websockets.cpp | 10 +++--- .../azure-core/test/ut/websocket_test.cpp | 3 +- 9 files changed, 56 insertions(+), 60 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index 4f742fbcf8..e44b0da0b5 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -20,6 +20,9 @@ namespace Azure { namespace Core { /** * @brief Used to convert one form of data into another, for example encoding binary data into * Base64 text. + * + * @note Base64 encoded data is a subset of the ASCII encoding (characters 0-127). As such, + * it can be considered a subset of UTF-8. */ class Convert final { private: @@ -32,39 +35,17 @@ namespace Azure { namespace Core { public: /** - * @brief Encodes the vector of binary data into UTF-8 encoded text represented as Base64. + * @brief Base64 encodes a vector of binary data. * - * @param data The input vector that contains binary data that needs to be encoded. - * @return The UTF-8 encoded text in Base64. + * @param data The input vector that contains binary data to be encoded. + * @return The Base64 encoded contents of the vector. */ static std::string Base64Encode(const std::vector& data); /** - * @brief Encodes the vector of binary data into UTF-8 encoded text represented as Base64. + * @brief Decodes the Base64 encoded text into binary data. * - * @param data The binary data to be encoded. - * @param length The length of the binaryData parameter. - * @return The UTF-8 encoded text in Base64. - */ - static std::string Base64Encode(uint8_t const* const data, size_t length); - - /** - * @brief Encodes an array of binary data into a Base64 encoded string. - * - * @param data The binary data to be encoded. - * @param length The length of the binaryData parameter. - * @return The UTF-8 encoded text in Base64. - */ - template - static std::string Base64Encode(std::array const& data) - { - return Base64Encode(data.data(), data.size()); - } - - /** - * @brief Decodes the UTF-8 encoded text represented as Base64 into binary data. - * - * @param text The input UTF-8 encoded text in Base64 that needs to be decoded. + * @param text Base64 encoded text to be decoded. * @return The decoded binary data. */ static std::vector Base64Decode(const std::string& text); diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index fed11827c7..aad53a5e15 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -132,6 +132,12 @@ namespace Azure { namespace Core { namespace Http { CurlTransportOptions m_options; protected: + /** + * @brief Called when an HTTP response indicates that the connection should be upgraded to + * a websocket. + * + * @param The connection that is being upgraded. + */ virtual void OnUpgradedConnection(std::unique_ptr&){}; public: diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 28457caf36..c9e039279b 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -21,15 +21,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief Concrete implementation of a WebSocket Transport that uses libcurl. */ class CurlWebSocketTransport : public WebSocketTransport, public CurlTransport { - // std::unique_ptr cannot be constructed on an incomplete type (CurlNetworkConnection), but - // std::shared_ptr can be. - std::shared_ptr m_upgradedConnection; - void OnUpgradedConnection( - std::unique_ptr& upgradedConnection) override; - public: /** - * @brief Construct a new CurlTransport object. + * @brief Construct a new CurlWebSocketTransport object. * * @param options Optional parameter to override the default options. */ @@ -56,6 +50,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual bool NativeWebsocketSupport() override { return false; } + /** + * @brief Complete the WebSocket upgrade. + * + * @details Called by the WebSocket client after the HTTP server responds with a + * SwitchingProtocols response. This method performs whatever operations are needed to + * transfer the protocol from HTTP to WebSockets. + */ virtual void CompleteUpgrade() override; /** @@ -107,8 +108,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - virtual std::vector ReceiveFrame(WebSocketFrameType&, Azure::Core::Context const&) - override + virtual std::pair> ReceiveFrame( + Azure::Core::Context const&) override { throw std::runtime_error("Not implemented"); } @@ -138,7 +139,18 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) override; + + /** + * @brief returns true if this transport supports WebSockets, false otherwise. + */ bool SupportsWebSockets() const override { return true; } + + private: + // std::unique_ptr cannot be constructed on an incomplete type (CurlNetworkConnection), but + // std::shared_ptr can be. + std::shared_ptr m_upgradedConnection; + void OnUpgradedConnection( + std::unique_ptr& upgradedConnection) override; }; }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index e583240525..1d83d98d3c 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -120,10 +120,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @param frameTypeReceived frame type received from the remote server. * - * @returns Frame data received from the remote server. + * @returns a tuple containing the Frame data received from the remote server and the type of + * data returned from the remote endpoint */ - virtual std::vector ReceiveFrame( - WebSocketFrameType& frameTypeReceived, + virtual std::pair> ReceiveFrame( Azure::Core::Context const& context) = 0; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 02a0379190..291e1da7fb 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -104,8 +104,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { std::vector const&, Azure::Core::Context const&) override; - virtual std::vector ReceiveFrame( - WebSocketFrameType& frameType, + virtual std::pair> ReceiveFrame( Azure::Core::Context const&) override; // Non-Native WebSocket support. diff --git a/sdk/core/azure-core/src/base64.cpp b/sdk/core/azure-core/src/base64.cpp index fc29735f22..a2e214c5bc 100644 --- a/sdk/core/azure-core/src/base64.cpp +++ b/sdk/core/azure-core/src/base64.cpp @@ -490,12 +490,7 @@ namespace Azure { namespace Core { std::string Convert::Base64Encode(const std::vector& data) { - return Base64Encode(data.data(), data.size()); - } - - std::string Convert::Base64Encode(uint8_t const* const data, size_t length) - { - return ::Base64Encode(data, length); + return ::Base64Encode(data.data(), data.size()); } std::vector Convert::Base64Decode(const std::string& text) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index ff0dc26433..453d74a9a2 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -345,18 +345,20 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { - WebSocketTransport::WebSocketFrameType frameType; - std::vector payload = m_transport->ReceiveFrame(frameType, context); - switch (frameType) + auto payload = m_transport->ReceiveFrame(context); + switch (payload.first) { case WebSocketTransport::WebSocketFrameType::FrameTypeBinary: - return std::make_shared(true, payload.data(), payload.size()); + return std::make_shared( + true, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: - return std::make_shared(false, payload.data(), payload.size()); + return std::make_shared(false, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeText: - return std::make_shared(true, payload.data(), payload.size()); + return std::make_shared( + true, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment: - return std::make_shared(false, payload.data(), payload.size()); + return std::make_shared( + false, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeClosed: { auto closeResult = m_transport->GetCloseSocketInformation(context); return std::make_shared(closeResult.first, closeResult.second); diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 65fd1c9dd5..43fb4aa900 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -174,11 +174,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } } - std::vector WinHttpWebSocketTransport::ReceiveFrame( - WebSocketFrameType& frameTypeReceived, - Azure::Core::Context const& context) + std::pair< + Azure::Core::Http::WebSockets::WebSocketTransport::WebSocketFrameType, + std::vector> + WinHttpWebSocketTransport::ReceiveFrame(Azure::Core::Context const& context) { WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; + WebSocketFrameType frameTypeReceived; DWORD bufferBytesRead; std::vector buffer(128); context.ThrowIfCancelled(); @@ -217,7 +219,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { throw std::runtime_error("Unknown frame type."); break; } - return buffer; + return std::make_pair(frameTypeReceived, buffer); } }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 197f2f3ee2..c4efddc900 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -579,8 +579,7 @@ TEST_F(WebSocketTests, CurlTransportCoverage) EXPECT_THROW( transport->SendFrame(WebSocketTransport::WebSocketFrameType::FrameTypeBinary, {}, {}), std::runtime_error); - WebSocketTransport::WebSocketFrameType ft; - EXPECT_THROW(transport->ReceiveFrame(ft, {}), std::runtime_error); + EXPECT_THROW(transport->ReceiveFrame({}), std::runtime_error); } } From 76813050d569a235e212051204321b5e420451dd Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 13:58:55 -0700 Subject: [PATCH 105/149] clang fixes --- sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp | 2 -- .../inc/azure/core/http/websockets/websockets_transport.hpp | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index aad53a5e15..0eb6a27bc1 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -135,8 +135,6 @@ namespace Azure { namespace Core { namespace Http { /** * @brief Called when an HTTP response indicates that the connection should be upgraded to * a websocket. - * - * @param The connection that is being upgraded. */ virtual void OnUpgradedConnection(std::unique_ptr&){}; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 1d83d98d3c..41f68cf4ff 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -118,7 +118,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Receive a frame from the remote WebSocket server. * - * @param frameTypeReceived frame type received from the remote server. + * @param context Context for the operation. * * @returns a tuple containing the Frame data received from the remote server and the type of * data returned from the remote endpoint From 1915b76f2e488626f95ac94b900d1ffa1c15dc96 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 14:22:21 -0700 Subject: [PATCH 106/149] clang_format --- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 453d74a9a2..6c143a7f77 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -352,7 +352,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return std::make_shared( true, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: - return std::make_shared(false, payload.second.data(), payload.second.size()); + return std::make_shared( + false, payload.second.data(), payload.second.size()); case WebSocketTransport::WebSocketFrameType::FrameTypeText: return std::make_shared( true, payload.second.data(), payload.second.size()); From ab6b497dd5fdb1ca2357f8b6b5f7136f7d3a6d67 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 17:09:17 -0700 Subject: [PATCH 107/149] Get rid of diamond anti-pattern in WebSocketTransport --- .../http/websockets/curl_websockets_transport.hpp | 2 +- .../core/http/websockets/websockets_transport.hpp | 4 +--- .../src/http/websockets/websocketsimpl.cpp | 14 +++++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index c9e039279b..ba08f6d328 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -20,7 +20,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Concrete implementation of a WebSocket Transport that uses libcurl. */ - class CurlWebSocketTransport : public WebSocketTransport, public CurlTransport { + class CurlWebSocketTransport : public CurlTransport, public WebSocketTransport { public: /** * @brief Construct a new CurlWebSocketTransport object. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 41f68cf4ff..f067f112c4 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -16,7 +16,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Base class for all WebSocket transport implementations. */ - class WebSocketTransport : public HttpTransport { + class WebSocketTransport { public: /** * @brief Web Socket Frame type, one of Text or Binary. @@ -145,8 +145,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual int SendBuffer(uint8_t const* buffer, size_t bufferSize, Context const& context) = 0; - bool SupportsWebSockets() const override { return true; } - protected: /** * @brief Constructs a default instance of `%HttpTransport`. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 6c143a7f77..60b0a713f0 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -43,15 +43,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if defined(BUILD_TRANSPORT_WINHTTP_ADAPTER) WinHttpTransportOptions transportOptions; - m_transport = std::make_shared( - transportOptions); - m_options.Transport.Transport = m_transport; + auto winHttpTransport + = std::make_shared( + transportOptions); + m_transport = std::static_pointer_cast(winHttpTransport); + m_options.Transport.Transport = std::static_pointer_cast(winHttpTransport); #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) CurlTransportOptions transportOptions; transportOptions.HttpKeepAlive = false; - m_transport + auto curlWebSockets = std::make_shared(transportOptions); - m_options.Transport.Transport = m_transport; + + m_transport = std::static_pointer_cast(curlWebSockets); + m_options.Transport.Transport = std::static_pointer_cast(curlWebSockets); m_bufferedStreamReader.SetTransport(m_transport); #endif From 9f835545214cd363c7511b87c3c793275ff5c929 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 14 Jul 2022 17:14:27 -0700 Subject: [PATCH 108/149] Removed CompleteUpgrade since it did nothing --- .../core/http/websockets/curl_websockets_transport.hpp | 9 --------- .../core/http/websockets/websockets_transport.hpp | 10 +--------- .../http/websockets/win_http_websockets_transport.hpp | 2 -- sdk/core/azure-core/src/http/curl/curl_websockets.cpp | 2 -- .../azure-core/src/http/websockets/websocketsimpl.cpp | 3 --- .../src/http/winhttp/win_http_websockets.cpp | 2 -- 6 files changed, 1 insertion(+), 27 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index ba08f6d328..ab64d233fb 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -50,15 +50,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual bool NativeWebsocketSupport() override { return false; } - /** - * @brief Complete the WebSocket upgrade. - * - * @details Called by the WebSocket client after the HTTP server responds with a - * SwitchingProtocols response. This method performs whatever operations are needed to - * transfer the protocol from HTTP to WebSockets. - */ - virtual void CompleteUpgrade() override; - /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index f067f112c4..1223aedf5e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -47,7 +47,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { FrameTypeClosed, }; /** - * @brief Destructs `%HttpTransport`. + * @brief Destructs `%WebSocketTransport`. * */ virtual ~WebSocketTransport() {} @@ -58,14 +58,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @returns true iff the transport has native websocket support, false otherwise. */ virtual bool NativeWebsocketSupport() = 0; - /** - * @brief Complete the WebSocket upgrade. - * - * @details Called by the WebSocket client after the HTTP server responds with a - * SwitchingProtocols response. This method performs whatever operations are needed to - * transfer the protocol from HTTP to WebSockets. - */ - virtual void CompleteUpgrade() = 0; /**************/ /* Native WebSocket support functions*/ diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 291e1da7fb..ef8305c347 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -57,8 +57,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual bool NativeWebsocketSupport() override { return true; } - virtual void CompleteUpgrade() override; - /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * diff --git a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp index 3cbd7ed980..2e448a58e2 100644 --- a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp +++ b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp @@ -27,8 +27,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { - void CurlWebSocketTransport::CompleteUpgrade() {} - void CurlWebSocketTransport::Close() { m_upgradedConnection->Shutdown(); } // Send an HTTP request to the remote server. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 60b0a713f0..f03ce8fed8 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -145,9 +145,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_chosenProtocol = chosenProtocol->second; } - // Inform the transport that the upgrade is complete and that the WebSockets layer is taking - // over the HTTP connection. - m_transport->CompleteUpgrade(); m_state = SocketState::Open; } diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 43fb4aa900..e2f234f095 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -45,8 +45,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return WinHttpTransport::Send(request, context); } - void WinHttpWebSocketTransport::CompleteUpgrade() {} - /** * @brief Close the WebSocket cleanly. */ From 9ebc88f93859fd2d6be596b8958d91f1c80836b8 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 20 Jul 2022 15:47:06 -0700 Subject: [PATCH 109/149] Implemented ping support; some internal cleanup and simplification; Use RAII for all curl handles --- .../websockets/curl_websockets_transport.hpp | 2 +- .../azure/core/http/websockets/websockets.hpp | 145 ++++- .../http/websockets/websockets_transport.hpp | 85 ++- .../src/environment_log_level_listener.cpp | 4 +- sdk/core/azure-core/src/http/curl/curl.cpp | 19 +- .../src/http/websockets/websockets.cpp | 28 +- .../src/http/websockets/websocketsimpl.cpp | 566 ++++++++++++------ .../src/http/websockets/websocketsimpl.hpp | 251 ++++---- .../src/http/winhttp/win_http_transport.cpp | 6 +- .../src/http/winhttp/win_http_websockets.cpp | 3 +- .../azure-core/test/ut/websocket_server.py | 51 +- .../azure-core/test/ut/websocket_test.cpp | 234 ++++++-- 12 files changed, 993 insertions(+), 401 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index ab64d233fb..5bd01f8533 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -48,7 +48,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * it is the responsibility of the client of the WebSocketTransport to format WebSocket protocol * elements. */ - virtual bool NativeWebsocketSupport() override { return false; } + virtual bool HasNativeWebsocketSupport() override { return false; } /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index bcbfaf428f..334e89d4c3 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -27,7 +27,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { TextFrameReceived, BinaryFrameReceived, PeerClosedReceived, - PongReceived, }; enum class WebSocketErrorCode : uint16_t @@ -50,64 +49,145 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketTextFrame; class WebSocketBinaryFrame; class WebSocketPeerCloseFrame; - class WebSocketPongFrame; + /** @brief Statistics about data sent and received by the WebSocket. + * + * @remarks This class is primarily intended for test collateral and debugging to allow + * a caller to determine information about the status of a WebSocket. + * + * Note: Some of these statistics are not available if the underlying transport supports native + * websockets. + */ + struct WebSocketStatistics + { + /** @brief The number of WebSocket frames sent on this WebSocket. */ + uint32_t FramesSent; + /** @brief The number of bytes of data sent to the peer on this WebSocket. */ + uint32_t BytesSent; + /** @brief The number of WebSocket frames received from the peer. */ + uint32_t FramesReceived; + /** @brief The number of bytes received from the peer. */ + uint32_t BytesReceived; + /** @brief The number of "Ping" frames received from the peer. */ + uint32_t PingFramesReceived; + /** @brief The number of "Ping" frames sent to the peer. */ + uint32_t PingFramesSent; + /** @brief The number of "Pong" frames received from the peer. */ + uint32_t PongFramesReceived; + /** @brief The number of "Pong" frames sent to the peer. */ + uint32_t PongFramesSent; + /** @brief The number of "Text" frames received from the peer. */ + uint32_t TextFramesReceived; + /** @brief The number of "Text" frames sent to the peer. */ + uint32_t TextFramesSent; + /** @brief The number of "Binary" frames received from the peer. */ + uint32_t BinaryFramesReceived; + /** @brief The number of "Binary" frames sent to the peer. */ + uint32_t BinaryFramesSent; + /** @brief The number of "Continuation" frames sent to the peer. */ + uint32_t ContinuationFramesSent; + /** @brief The number of "Continuation" frames received from the peer. */ + uint32_t ContinuationFramesReceived; + /** @brief The number of "Close" frames received from the peer. */ + uint32_t CloseFramesReceived; + /** @brief The number of frames received which were not processed. */ + uint32_t FramesDropped; + /** @brief The number of frames received which were not returned because they were received + * after the Close() method was called. */ + uint32_t FramesDroppedByClose; + /** @brief The number of frames dropped because they were over the maximum payload size. */ + uint32_t FramesDroppedByPayloadSizeLimit; + /** @brief The number of frames dropped because they were out of compliance with the protocol. + */ + uint32_t FramesDroppedByProtocolError; + /** @brief The number of reads performed on the transport.*/ + uint32_t TransportReads; + /** @brief The number of bytes read from the transport. */ + uint32_t TransportReadBytes; + }; + + /** @brief A frame of data received from a WebSocket. + */ struct WebSocketFrame { + /** @brief The type of frame received: Text, Binary or Close. */ WebSocketFrameType FrameType; + /** @brief True if the frame received is a "final" frame */ bool IsFinalFrame{false}; + /** @brief Returns the contents of the frame as a Text frame. + * @returns A WebSocketTextFrame containing the contents of the frame. + */ std::shared_ptr AsTextFrame(); + /** @brief Returns the contents of the frame as a Binary frame. + * @returns A WebSocketBinaryFrame containing the contents of the frame. + */ std::shared_ptr AsBinaryFrame(); + /** @brief Returns the contents of the frame as a Peer Close frame. + * @returns A WebSocketPeerCloseFrame containing the contents of the frame. + */ std::shared_ptr AsPeerCloseFrame(); - std::shared_ptr AsPongFrame(); }; + /** @brief Contains the contents of a WebSocket Text frame.*/ class WebSocketTextFrame : public WebSocketFrame, public std::enable_shared_from_this { private: public: + /** @brief Constructs a new WebSocketTextFrame */ WebSocketTextFrame() = default; + /** @brief Constructs a new WebSocketTextFrame + * @param isFinalFrame True if this is the final frame in a multi-frame message. + * @param body UTF-8 encoded text of the frame data. + * @param size Length in bytes of the frame body. + */ WebSocketTextFrame(bool isFinalFrame, unsigned char const* body, size_t size) : WebSocketFrame{WebSocketFrameType::TextFrameReceived, isFinalFrame}, Text(body, body + size) { } + /** @brief Text of the frame received from the remote peer. */ std::string Text; }; + + /** @brief Contains the contents of a WebSocket Binary frame.*/ class WebSocketBinaryFrame : public WebSocketFrame, public std::enable_shared_from_this { private: public: + /** @brief Constructs a new WebSocketBinaryFrame */ WebSocketBinaryFrame() = default; + /** @brief Constructs a new WebSocketBinaryFrame + * @param isFinal True if this is the final frame in a multi-frame message. + * @param body binary of the frame data. + * @param size Length in bytes of the frame body. + */ WebSocketBinaryFrame(bool isFinal, unsigned char const* body, size_t size) : WebSocketFrame{WebSocketFrameType::BinaryFrameReceived, isFinal}, Data(body, body + size) { } + /** @brief Binary frame data received from the remote peer. */ std::vector Data; }; - class WebSocketPongFrame : public WebSocketFrame, - public std::enable_shared_from_this { - private: - public: - WebSocketPongFrame() = default; - WebSocketPongFrame(unsigned char const* body, size_t size) - : WebSocketFrame{WebSocketFrameType::PongReceived, true}, Data(body, body + size) - { - } - std::vector Data; - }; - + /** @brief Contains the contents of a WebSocket Close frame.*/ class WebSocketPeerCloseFrame : public WebSocketFrame, public std::enable_shared_from_this { public: + /** @brief Constructs a new WebSocketPeerCloseFrame */ WebSocketPeerCloseFrame() = default; + /** @brief Constructs a new WebSocketBinaryFrame + * @param remoteStatusCode Status code sent by the remote peer. + * @param remoteCloseReason Optional reason sent by the remote peer. + */ WebSocketPeerCloseFrame(uint16_t remoteStatusCode, std::string const& remoteCloseReason) : WebSocketFrame{WebSocketFrameType::PeerClosedReceived}, RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) { } + /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode + * enumeration */ uint16_t RemoteStatusCode; + /** @brief Optional text sent from the remote peer. */ std::string RemoteCloseReason; }; @@ -129,6 +209,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ std::string ServiceVersion; + /** + * @brief The period of time between ping operations, default is 60 seconds. + */ + std::chrono::duration PingInterval{std::chrono::seconds{60}}; + /** * @brief Construct an instance of a WebSocketOptions type. * @@ -211,18 +296,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { std::shared_ptr ReceiveFrame( Azure::Core::Context const& context = Azure::Core::Context{}); - /** @brief Send a 'Ping' frame to the remote server. - * - * @param pingData data to be sent in the ping operation. - * @param context Context for the operation. - * - * @returns True if the "Ping" was sent to the server, False if the underlying transport - * does not support ping operations. - */ - bool SendPing( - std::vector const& pingData, - Azure::Core::Context const& context = Azure::Core::Context{}); - /** @brief AddHeader - Adds a header to the initial handshake. * * @note This API is ignored after the WebSocket is opened. @@ -236,12 +309,28 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @returns true if the WebSocket is open, false otherwise. */ - bool IsOpen(); + bool IsOpen() const; + + /** @brief Returns "true" if the configured websocket transport + * supports websockets in the transport, or if the websocket implementation + * is providing websocket protocol support. + * + * @returns true if the websocket transport supports websockets natively. + */ + bool HasNativeWebSocketSupport() const; /** @brief Returns the protocol chosen by the remote server during the initial handshake + * + * @returns The protocol negotiated between client and server. */ std::string const& GetChosenProtocol() const; + /** @brief Returns statistics about the WebSocket. + * + * @returns The statistics about the WebSocket. + */ + WebSocketStatistics GetStatistics() const; + private: std::unique_ptr<_detail::WebSocketImplementation> m_socketImplementation; }; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 1223aedf5e..0a98f1045d 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -55,9 +55,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Determines if the transport natively supports WebSockets or not. * - * @returns true iff the transport has native websocket support, false otherwise. + * @returns true if the transport has native websocket support, false otherwise. */ - virtual bool NativeWebsocketSupport() = 0; + virtual bool HasNativeWebsocketSupport() = 0; /**************/ /* Native WebSocket support functions*/ @@ -123,6 +123,87 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /* Non Native WebSocket support functions */ /**************/ + // Implement a buffered stream reader + class BufferedStreamReader { + std::shared_ptr m_transport; + std::unique_ptr m_initialBodyStream; + constexpr static size_t m_bufferSize = 1024; + uint8_t m_buffer[m_bufferSize]{}; + size_t m_bufferPos = 0; + size_t m_bufferLen = 0; + bool m_eof = false; + + public: + explicit BufferedStreamReader() = default; + ~BufferedStreamReader() = default; + + void SetInitialStream(std::unique_ptr& stream) + { + m_initialBodyStream = std::move(stream); + } + void SetTransport( + std::shared_ptr& transport) + { + m_transport = transport; + } + + uint8_t ReadByte(Azure::Core::Context const& context) + { + if (m_bufferPos >= m_bufferLen) + { + // Start by reading data from our initial body stream. + m_bufferLen = m_initialBodyStream->ReadToCount(m_buffer, m_bufferSize, context); + if (m_bufferLen == 0) + { + // If we run out of the initial stream, we need to read from the transport. + m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + } + m_bufferPos = 0; + if (m_bufferLen == 0) + { + m_eof = true; + return 0; + } + } + return m_buffer[m_bufferPos++]; + } + uint16_t ReadShort(Azure::Core::Context const& context) + { + uint16_t result = ReadByte(context); + result <<= 8; + result |= ReadByte(context); + return result; + } + uint64_t ReadInt64(Azure::Core::Context const& context) + { + uint64_t result = 0; + + result |= (static_cast(ReadByte(context)) << 56 & 0xff00000000000000); + result |= (static_cast(ReadByte(context)) << 48 & 0x00ff000000000000); + result |= (static_cast(ReadByte(context)) << 40 & 0x0000ff0000000000); + result |= (static_cast(ReadByte(context)) << 32 & 0x000000ff00000000); + result |= (static_cast(ReadByte(context)) << 24 & 0x00000000ff000000); + result |= (static_cast(ReadByte(context)) << 16 & 0x0000000000ff0000); + result |= (static_cast(ReadByte(context)) << 8 & 0x000000000000ff00); + result |= static_cast(ReadByte(context)); + return result; + } + std::vector ReadBytes(size_t readLength, Azure::Core::Context const& context) + { + std::vector result; + size_t index = 0; + while (index < readLength) + { + uint8_t byte = ReadByte(context); + result.push_back(byte); + index += 1; + } + return result; + } + + bool IsEof() const { return m_eof; } + }; + /** * @brief This function is used when working with streams to pull more data from the wire. * Function will try to keep pulling data from socket until the buffer is all written or until diff --git a/sdk/core/azure-core/src/environment_log_level_listener.cpp b/sdk/core/azure-core/src/environment_log_level_listener.cpp index ffcf1e6076..98ef17dcf3 100644 --- a/sdk/core/azure-core/src/environment_log_level_listener.cpp +++ b/sdk/core/azure-core/src/environment_log_level_listener.cpp @@ -9,6 +9,7 @@ #include #include +#include using Azure::Core::_internal::Environment; using namespace Azure::Core::Diagnostics; @@ -114,7 +115,8 @@ EnvironmentLogLevelListener::GetLogListener() << Azure::DateTime(std::chrono::system_clock::now()) .ToString( DateTime::DateFormat::Rfc3339, DateTime::TimeFractionFormat::AllDigits) - << "] " << LogLevelToConsoleString(level) << " : " << message << std::endl; + << " T: " << std::this_thread::get_id() << "] " << LogLevelToConsoleString(level) + << " : " << message << std::endl; }; return consoleLogger; diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index b584087b45..001c91eb07 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -141,14 +141,14 @@ void WinSocketSetBuffSize(curl_socket_t socket) // Specifies the total per-socket buffer space reserved for sends. // https://docs.microsoft.com/windows/win32/api/winsock/nf-winsock-setsockopt auto result = setsockopt(socket, SOL_SOCKET, SO_SNDBUF, (const char*)&ideal, sizeof(ideal)); - - if (Log::ShouldWrite(Logger::Level::Verbose)) - { - Log::Write( - Logger::Level::Verbose, - LogMsgPrefix + "Windows - calling setsockopt after uploading chunk. ideal = " - + std::to_string(ideal) + " result = " + std::to_string(result)); - } + result; + // if (Log::ShouldWrite(Logger::Level::Verbose)) + // { + // Log::Write( + // Logger::Level::Verbose, + // LogMsgPrefix + "Windows - calling setsockopt after uploading chunk. ideal = " + // + std::to_string(ideal) + " result = " + std::to_string(result)); + // } } } #endif @@ -978,7 +978,6 @@ size_t CurlConnection::ReadFromSocket(uint8_t* buffer, size_t bufferSize, Contex for (CURLcode readResult = CURLE_AGAIN; readResult == CURLE_AGAIN;) { readResult = curl_easy_recv(m_handle.get(), buffer, bufferSize, &readBytes); - switch (readResult) { case CURLE_AGAIN: { @@ -1344,7 +1343,7 @@ std::unique_ptr CurlConnectionPool::ExtractOrCreateCurlCo // Creating a new connection is thread safe. No need to lock mutex here. // No available connection for the pool for the required host. Create one Log::Write(Logger::Level::Verbose, LogMsgPrefix + "Spawn new connection."); - unique_CURL newHandle(curl_easy_init(), CURL_deleter()); + unique_CURL newHandle(curl_easy_init(), CURL_deleter{}); if (!newHandle) { throw Azure::Core::Http::TransportException( diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index 34a9304fba..ad785993dc 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -20,7 +20,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } void WebSocket::Close(Azure::Core::Context const& context) { - m_socketImplementation->Close(context); + m_socketImplementation->Close( + static_cast(WebSocketErrorCode::EndpointDisappearing), {}, context); } void WebSocket::Close( uint16_t closeStatus, @@ -46,16 +47,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { m_socketImplementation->SendFrame(binaryFrame, isFinalFrame, context); } - std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) + WebSocketStatistics WebSocket::GetStatistics() const { - return m_socketImplementation->ReceiveFrame(context); + return m_socketImplementation->GetStatistics(); } - bool WebSocket::SendPing( - std::vector const& pingData, - Azure::Core::Context const& context) + bool WebSocket::HasNativeWebSocketSupport() const + { + return m_socketImplementation->HasNativeWebSocketSupport(); + } + + std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) { - return m_socketImplementation->SendPing(pingData, context); + return m_socketImplementation->ReceiveFrame(context); } void WebSocket::AddHeader(std::string const& headerName, std::string const& headerValue) @@ -67,7 +71,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return m_socketImplementation->GetChosenProtocol(); } - bool WebSocket::IsOpen() { return m_socketImplementation->IsOpen(); } + bool WebSocket::IsOpen() const { return m_socketImplementation->IsOpen(); } std::shared_ptr WebSocketFrame::AsTextFrame() { @@ -95,13 +99,5 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } return static_cast(this)->shared_from_this(); } - std::shared_ptr WebSocketFrame::AsPongFrame() - { - if (FrameType != WebSocketFrameType::PongReceived) - { - throw std::logic_error("Cannot cast to PongReceived."); - } - return static_cast(this)->shared_from_this(); - } }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index f03ce8fed8..8ea657de0a 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -17,19 +17,33 @@ #include "azure/core/internal/diagnostics/log.hpp" #include #include +#include #include #include #include +#include namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { using namespace Azure::Core::Diagnostics::_internal; using namespace Azure::Core::Diagnostics; using namespace std::chrono_literals; + namespace { + std::string HexEncode(std::vector const& data, size_t length) + { + std::stringstream ss; + for (size_t i = 0; i < std::min(data.size(), length); i++) + { + ss << std::hex << std::setfill('0') << std::setw(2) << static_cast(data[i]); + } + return ss.str(); + } + } // namespace + WebSocketImplementation::WebSocketImplementation( Azure::Core::Url const& remoteUrl, WebSocketOptions const& options) - : m_remoteUrl(remoteUrl), m_options(options) + : m_remoteUrl(remoteUrl), m_options(options), m_pingThread(this, m_options.PingInterval) { } @@ -56,13 +70,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_transport = std::static_pointer_cast(curlWebSockets); m_options.Transport.Transport = std::static_pointer_cast(curlWebSockets); - m_bufferedStreamReader.SetTransport(m_transport); #endif std::vector> perCallPolicies{}; std::vector> perRetryPolicies{}; - // If the caller has told us a service name, add the telemetry policy to the pipeline to add a - // user agent header to the request. + // If the caller has told us a service name, add the telemetry policy to the pipeline to add + // a user agent header to the request. if (!m_options.ServiceName.empty()) { perCallPolicies.push_back( @@ -79,7 +92,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // natively. auto randomKey = GenerateRandomKey(); auto encodedKey = Azure::Core::Convert::Base64Encode(randomKey); - if (!m_transport->NativeWebsocketSupport()) + if (!m_transport->HasNativeWebsocketSupport()) { // If the transport doesn't support WebSockets natively, set the standardized WebSocket // upgrade headers. @@ -122,7 +135,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Prove that the server received this socket request. auto& responseHeaders = response->GetHeaders(); - if (!m_transport->NativeWebsocketSupport()) + if (!m_transport->HasNativeWebsocketSupport()) { auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); if (socketAccept == responseHeaders.end()) @@ -134,8 +147,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { VerifySocketAccept(encodedKey, socketAccept->second); } - auto bodyStream = response->ExtractBodyStream(); - m_bufferedStreamReader.SetInitialStream(bodyStream); + m_initialBodyStream = response->ExtractBodyStream(); + m_pingThread.Start(m_transport); } // Remember the protocol that the client chose. @@ -147,10 +160,21 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Open; } + bool WebSocketImplementation::HasNativeWebSocketSupport() + { + std::lock_guard lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); + if (m_state != SocketState::Open) + { + throw std::runtime_error("Socket is not open."); + } + return m_transport->HasNativeWebsocketSupport(); + } std::string const& WebSocketImplementation::GetChosenProtocol() { std::lock_guard lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); @@ -161,6 +185,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names void WebSocketImplementation::AddHeader(std::string const& header, std::string const& headerValue) { std::lock_guard lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Closed && m_state != SocketState::Invalid) { throw std::runtime_error("AddHeader can only be called on closed sockets."); @@ -168,9 +193,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_headers.emplace(std::make_pair(header, headerValue)); } - void WebSocketImplementation::Close(Azure::Core::Context const& context) + void WebSocketImplementation::Close( + uint16_t closeStatus, + std::string const& closeReason, + Azure::Core::Context const& context) { - std::lock_guard lock(m_stateMutex); + std::unique_lock lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); // If we're closing an already closed socket, we're done. if (m_state == SocketState::Closed) @@ -185,22 +214,23 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { - m_transport->CloseSocket( - static_cast(WebSocketErrorCode::EndpointDisappearing), "", context); + m_transport->CloseSocket(closeStatus, closeReason.c_str(), context); } else #endif { // Send a going away message to the server. - uint16_t closeReason = static_cast(WebSocketErrorCode::EndpointDisappearing); std::vector closePayload; - closePayload.push_back(closeReason >> 8); - closePayload.push_back(closeReason & 0xff); + closePayload.push_back(closeStatus >> 8); + closePayload.push_back(closeStatus & 0xff); + closePayload.insert(closePayload.end(), closeReason.begin(), closeReason.end()); std::vector closeFrame = EncodeFrame(SocketOpcode::Close, true, closePayload); - m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); + SendTransportBuffer(closeFrame, context); - // To ensure that we process the responses in a "timely" fashion, limit the close reception to - // 20 seconds if we don't already have a timeout. + // Unlock the state mutex before waiting for the close response to be received. + lock.unlock(); + // To ensure that we process the responses in a "timely" fashion, limit the close + // reception to 20 seconds if we don't already have a timeout. Azure::Core::Context closeContext = context; auto cancelTimepoint = closeContext.GetDeadline(); if (cancelTimepoint == Azure::DateTime::max()) @@ -209,61 +239,25 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } // Drain the incoming series of frames from the server. // Note that there might be in-flight frames that were sent from the other end of the - // WebSocket that we don't care about any more (since we're closing the WebSocket). So drain - // those frames. - auto closeResponse = ReceiveFrame(closeContext, true); + // WebSocket that we don't care about any more (since we're closing the WebSocket). So + // drain those frames. + auto closeResponse = ReceiveFrame(context); while (closeResponse->FrameType != WebSocketFrameType::PeerClosedReceived) { - auto textResult = closeResponse->AsTextFrame(); + m_receiveStatistics.FramesDroppedByClose++; Log::Write( Logger::Level::Warning, - "Received unexpected frame during close. Frame type: " + "Received unexpected frame during close. Opcode: " + std::to_string(static_cast(closeResponse->FrameType))); - closeResponse = ReceiveFrame(closeContext, true); + closeResponse = ReceiveFrame(closeContext); } - } - // Close the socket - after this point, the m_transport is invalid. - m_transport->Close(); - m_state = SocketState::Closed; - } - void WebSocketImplementation::Close( - uint16_t closeStatus, - std::string const& closeReason, - Azure::Core::Context const& context) - { - std::lock_guard lock(m_stateMutex); - if (m_state != SocketState::Open) - { - throw std::runtime_error("Socket is not open."); - } - - m_state = SocketState::Closing; -#if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) - { - m_transport->CloseSocket(closeStatus, closeReason, context); - } - else -#endif - { - std::vector closePayload; - closePayload.push_back(closeStatus >> 8); - closePayload.push_back(closeStatus & 0xff); - closePayload.insert(closePayload.end(), closeReason.begin(), closeReason.end()); - - std::vector closeFrame = EncodeFrame(SocketOpcode::Close, true, closePayload); - m_transport->SendBuffer(closeFrame.data(), closeFrame.size(), context); - - auto closeResponse = ReceiveFrame(context, true); - if (closeResponse->FrameType != WebSocketFrameType::PeerClosedReceived) - { - throw std::runtime_error("Unexpected result type received during close()."); - } + // Re-acquire the state lock once we've received the close lock. + lock.lock(); } // Close the socket - after this point, the m_transport is invalid. + m_pingThread.Shutdown(); m_transport->Close(); - m_state = SocketState::Closed; } @@ -273,11 +267,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Azure::Core::Context const& context) { std::lock_guard lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); } std::vector utf8text(textFrame.begin(), textFrame.end()); + m_receiveStatistics.TextFramesSent++; #if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { @@ -291,8 +287,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #endif { std::vector sendFrame = EncodeFrame(SocketOpcode::TextFrame, isFinalFrame, utf8text); - - m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); + SendTransportBuffer(sendFrame, context); } } @@ -302,11 +297,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Azure::Core::Context const& context) { std::lock_guard lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { throw std::runtime_error("Socket is not open."); } + m_receiveStatistics.BinaryFramesSent++; #if SUPPORT_NATIVE_TRANSPORT if (m_transport->NativeWebsocketSupport()) { @@ -319,164 +316,225 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names else #endif { + // Log::Write(Logger::Level::Verbose, "Send Binary Frame " + HexEncode(binaryFrame, 16)); std::vector sendFrame = EncodeFrame(SocketOpcode::BinaryFrame, isFinalFrame, binaryFrame); - std::unique_lock transportLock(m_transportMutex); - m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); + SendTransportBuffer(sendFrame, context); } } std::shared_ptr WebSocketImplementation::ReceiveFrame( - Azure::Core::Context const& context, - bool stateIsLocked) + Azure::Core::Context const& context) { - std::unique_lock lock(m_stateMutex, std::defer_lock); - - if (!stateIsLocked) - { - lock.lock(); - stateIsLocked = true; - } + std::unique_lock lock(m_stateMutex); + m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open && m_state != SocketState::Closing) { throw std::runtime_error("Socket is not open."); } -#if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) - { - auto payload = m_transport->ReceiveFrame(context); - switch (payload.first) - { - case WebSocketTransport::WebSocketFrameType::FrameTypeBinary: - return std::make_shared( - true, payload.second.data(), payload.second.size()); - case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: - return std::make_shared( - false, payload.second.data(), payload.second.size()); - case WebSocketTransport::WebSocketFrameType::FrameTypeText: - return std::make_shared( - true, payload.second.data(), payload.second.size()); - case WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment: - return std::make_shared( - false, payload.second.data(), payload.second.size()); - case WebSocketTransport::WebSocketFrameType::FrameTypeClosed: { - auto closeResult = m_transport->GetCloseSocketInformation(context); - return std::make_shared(closeResult.first, closeResult.second); - } - default: - throw std::runtime_error("Unexpected frame type received."); - } - } - else -#endif - { - SocketOpcode opcode; - bool isFinal = false; - uint64_t payloadLength; - std::vector frameData - = DecodeFrame(m_bufferedStreamReader, opcode, payloadLength, isFinal, context); + // Unlock the state lock to allow other threads to run. If we don't, we might end up in in a + // situation where the server won't respond to the this client because all the client threads + // are blocked on the state lock. + lock.unlock(); - // At this point, readBuffer contains the actual payload from the service. - switch (opcode) + std::shared_ptr frame; + // Loop until we receive an returnable incoming frame. + // If the incoming frame is returnable, we return the value from the frame. + while (true) + { + frame = ReceiveTransportFrame(context); + switch (frame->Opcode) { + // When we receive a "ping" frame, we want to send a Pong frame back to the server. + case SocketOpcode::Ping: + Log::Write( + Logger::Level::Verbose, "Received Ping frame: " + HexEncode(frame->Payload, 16)); + SendPong(frame->Payload, context); + break; + // We want to ignore all incoming "Pong" frames. + case SocketOpcode::Pong: + Log::Write( + Logger::Level::Verbose, "Received Pong frame: " + HexEncode(frame->Payload, 16)); + break; + case SocketOpcode::BinaryFrame: m_currentMessageType = SocketMessageType::Binary; return std::make_shared( - isFinal, frameData.data(), frameData.size()); - case SocketOpcode::TextFrame: { + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); + + case SocketOpcode::TextFrame: m_currentMessageType = SocketMessageType::Text; - return std::make_shared(isFinal, frameData.data(), frameData.size()); - } - case SocketOpcode::Close: { + return std::make_shared( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); - if (frameData.size() < 2) + case SocketOpcode::Close: { + if (frame->Payload.size() < 2) { throw std::runtime_error("Close response buffer is too short."); } uint16_t errorCode = 0; - errorCode |= (frameData[0] << 8) & 0xff00; - errorCode |= (frameData[1] & 0x00ff); + errorCode |= (frame->Payload[0] << 8) & 0xff00; + errorCode |= (frame->Payload[1] & 0x00ff); - // Update our state to be closed once we've received a closed frame. + // We received a close frame, mark the socket as closed. Make sure we + // reacquire the state lock before setting the state to closed. + lock.lock(); m_state = SocketState::Closed; + return std::make_shared( - errorCode, std::string(frameData.begin() + 2, frameData.end())); + errorCode, std::string(frame->Payload.begin() + 2, frame->Payload.end())); } - case SocketOpcode::Ping: - // Respond to the ping and then recurse to process the next frame. - SendPong(frameData, context); - return ReceiveFrame(context, stateIsLocked); - case SocketOpcode::Pong: - return std::make_shared(frameData.data(), frameData.size()); + // Continuation frames need to be treated somewhat specially. + // We depend on the fact that the protocol requires that a Continuation frame + // only be sent if it is part of a multi-frame message whose previous frame was a Text or + // Binary frame. case SocketOpcode::Continuation: if (m_currentMessageType == SocketMessageType::Text) { - if (isFinal) + if (frame->IsFinalFrame) { m_currentMessageType = SocketMessageType::Unknown; } return std::make_shared( - isFinal, frameData.data(), frameData.size()); + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); } else if (m_currentMessageType == SocketMessageType::Binary) { - if (isFinal) + if (frame->IsFinalFrame) { m_currentMessageType = SocketMessageType::Unknown; } return std::make_shared( - isFinal, frameData.data(), frameData.size()); + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); } else { + m_receiveStatistics.FramesDroppedByProtocolError++; throw std::runtime_error("Unknown message type and received continuation opcode"); } - break; default: - throw std::runtime_error("Unknown opcode received."); + throw std::runtime_error("Unknown frame type received."); } + context.ThrowIfCancelled(); } -#if !defined(_MSC_VER) - // gcc 5 doesn't seem to detect that this is dead code, so we'll just leave it here. - return nullptr; -#endif } - bool WebSocketImplementation::SendPing( - std::vector const& pingData, - Azure::Core::Context const& context) + std::shared_ptr + WebSocketImplementation::ReceiveTransportFrame(Azure::Core::Context const& context) { +#if SUPPORT_NATIVE_TRANSPORT + if (m_transport->NativeWebsocketSupport()) { - std::unique_lock lock(m_stateMutex); - - if (m_state != SocketState::Open) + auto payload = m_transport->ReceiveFrame(context); + m_receiveStatistics.FramesReceived++; + switch (payload.first) { - throw std::runtime_error("Socket is not open."); + case WebSocketTransport::WebSocketFrameType::FrameTypeBinary: + m_receiveStatistics.BinaryFramesReceived++; + return std::make_shared( + SocketOpcode::BinaryFrame, true, payload.second); + case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: + m_receiveStatistics.BinaryFramesReceived++; + return std::make_shared( + SocketOpcode::BinaryFrame, false, payload.second); + case WebSocketTransport::WebSocketFrameType::FrameTypeText: + m_receiveStatistics.TextFramesReceived++; + return std::make_shared( + SocketOpcode::TextFrame, true, payload.second); + case WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment: + m_receiveStatistics.TextFramesReceived++; + return std::make_shared( + SocketOpcode::TextFrame, false, payload.second); + case WebSocketTransport::WebSocketFrameType::FrameTypeClosed: { + m_receiveStatistics.CloseFramesReceived++; + auto closeResult = m_transport->GetCloseSocketInformation(context); + std::vector closePayload; + closePayload.push_back(closeResult.first >> 8); + closePayload.push_back(closeResult.first & 0xff); + closePayload.insert( + closePayload.end(), closeResult.second.begin(), closeResult.second.end()); + return std::make_shared(SocketOpcode::Close, true, closePayload); + } + default: + throw std::runtime_error("Unexpected frame type received."); } } - if (m_transport->NativeWebsocketSupport()) + else +#endif { - return false; - } + SocketOpcode opcode; - std::vector pingFrame = EncodeFrame(SocketOpcode::Ping, true, pingData); + bool isFinal = false; + std::vector frameData = DecodeFrame(opcode, isFinal, context); + // At this point, frameData contains the actual payload from the service. + auto frame = std::make_shared(opcode, isFinal, frameData); - std::unique_lock transportLock(m_transportMutex); - m_transport->SendBuffer(pingFrame.data(), pingFrame.size(), context); - return true; + // Handle statistics for the incoming frame. + m_receiveStatistics.FramesReceived++; + switch (frame->Opcode) + { + case SocketOpcode::Ping: { + m_receiveStatistics.PingFramesReceived++; + break; + } + case SocketOpcode::Pong: { + m_receiveStatistics.PongFramesReceived++; + break; + } + case SocketOpcode::TextFrame: { + m_receiveStatistics.TextFramesReceived++; + break; + } + case SocketOpcode::BinaryFrame: { + m_receiveStatistics.BinaryFramesReceived++; + break; + } + case SocketOpcode::Close: { + m_receiveStatistics.CloseFramesReceived++; + break; + } + case SocketOpcode::Continuation: { + m_receiveStatistics.ContinuationFramesReceived++; + break; + } + default: { + m_receiveStatistics.UnknownFramesReceived++; + break; + } + } + return frame; + } } - void WebSocketImplementation::SendPong( - std::vector const& pongData, - Azure::Core::Context const& context) - { - std::vector pongFrame = EncodeFrame(SocketOpcode::Pong, true, pongData); - std::unique_lock transportLock(m_transportMutex); - m_transport->SendBuffer(pongFrame.data(), pongFrame.size(), context); + WebSocketStatistics WebSocketImplementation::GetStatistics() const + { + WebSocketStatistics returnValue{}; + returnValue.FramesSent = m_receiveStatistics.FramesSent.load(); + returnValue.FramesReceived = m_receiveStatistics.FramesReceived.load(); + returnValue.BinaryFramesReceived = m_receiveStatistics.BinaryFramesReceived.load(); + returnValue.TextFramesReceived = m_receiveStatistics.TextFramesReceived.load(); + returnValue.BinaryFramesSent = m_receiveStatistics.BinaryFramesSent.load(); + returnValue.TextFramesSent = m_receiveStatistics.TextFramesSent.load(); + returnValue.PingFramesReceived = m_receiveStatistics.PingFramesReceived.load(); + returnValue.PongFramesReceived = m_receiveStatistics.PongFramesReceived.load(); + returnValue.PingFramesSent = m_receiveStatistics.PingFramesSent.load(); + returnValue.PongFramesSent = m_receiveStatistics.PongFramesSent.load(); + + returnValue.BytesSent = m_receiveStatistics.BytesSent.load(); + returnValue.BytesReceived = m_receiveStatistics.BytesReceived.load(); + returnValue.FramesDropped = m_receiveStatistics.FramesDropped.load(); + returnValue.FramesDroppedByClose = m_receiveStatistics.FramesDroppedByClose.load(); + returnValue.FramesDroppedByPayloadSizeLimit + = m_receiveStatistics.FramesDroppedByPayloadSizeLimit.load(); + returnValue.FramesDroppedByProtocolError + = m_receiveStatistics.FramesDroppedByProtocolError.load(); + returnValue.TransportReadBytes = m_receiveStatistics.TransportReadBytes.load(); + returnValue.TransportReads = m_receiveStatistics.TransportReads.load(); + return returnValue; } std::vector WebSocketImplementation::EncodeFrame( @@ -548,50 +606,133 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return encodedFrame; } - std::vector WebSocketImplementation::DecodeFrame( - WebSocketImplementation::BufferedStreamReader& streamReader, SocketOpcode& opcode, - uint64_t& payloadLength, bool& isFinal, Azure::Core::Context const& context) { + // Ensure single threaded access to receive this frame. std::unique_lock lock(m_transportMutex); - if (streamReader.IsEof()) + if (IsTransportEof()) { throw std::runtime_error("Frame buffer is too small."); } - uint8_t payloadByte = streamReader.ReadByte(context); + uint8_t payloadByte = ReadTransportByte(context); opcode = static_cast(payloadByte & 0x7f); isFinal = (payloadByte & 0x80) != 0; - payloadByte = streamReader.ReadByte(context); + payloadByte = ReadTransportByte(context); if (payloadByte & 0x80) { throw std::runtime_error("Server sent a frame with a reserved bit set."); } - payloadLength = payloadByte & 0x7f; + int64_t payloadLength = payloadByte & 0x7f; if (payloadLength <= 125) { payloadByte += 1; } else if (payloadLength == 126) { - payloadLength = streamReader.ReadShort(context); + payloadLength = ReadTransportShort(context); } else if (payloadLength == 127) { - payloadLength = streamReader.ReadInt64(context); + payloadLength = ReadTransportInt64(context); } else { throw std::logic_error("Unexpected payload length."); } - return streamReader.ReadBytes(static_cast(payloadLength), context); + return ReadTransportBytes(static_cast(payloadLength), context); + } + + uint8_t WebSocketImplementation::ReadTransportByte(Azure::Core::Context const& context) + { + if (m_bufferPos >= m_bufferLen) + { + // Start by reading data from our initial body stream. + m_bufferLen = m_initialBodyStream->ReadToCount(m_buffer, m_bufferSize, context); + if (m_bufferLen == 0) + { + // If we run out of the initial stream, we need to read from the transport. + m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); + m_receiveStatistics.TransportReads++; + m_receiveStatistics.TransportReadBytes += static_cast(m_bufferLen); + // Log::Write( + // Logger::Level::Verbose, + // "Read #" + std::to_string(m_receiveStatistics.TransportReads.load()) + // + "from transport: " + std::to_string(m_bufferLen)); + } + else + { + Azure::Core::Diagnostics::_internal::Log::Write( + Azure::Core::Diagnostics::Logger::Level::Informational, + "Read data from initial stream"); + } + m_bufferPos = 0; + if (m_bufferLen == 0) + { + m_eof = true; + return 0; + } + } + + m_receiveStatistics.BytesReceived++; + return m_buffer[m_bufferPos++]; + } + uint16_t WebSocketImplementation::ReadTransportShort(Azure::Core::Context const& context) + { + uint16_t result = ReadTransportByte(context); + result <<= 8; + result |= ReadTransportByte(context); + return result; + } + uint64_t WebSocketImplementation::ReadTransportInt64(Azure::Core::Context const& context) + { + uint64_t result = 0; + + result |= (static_cast(ReadTransportByte(context)) << 56 & 0xff00000000000000); + result |= (static_cast(ReadTransportByte(context)) << 48 & 0x00ff000000000000); + result |= (static_cast(ReadTransportByte(context)) << 40 & 0x0000ff0000000000); + result |= (static_cast(ReadTransportByte(context)) << 32 & 0x000000ff00000000); + result |= (static_cast(ReadTransportByte(context)) << 24 & 0x00000000ff000000); + result |= (static_cast(ReadTransportByte(context)) << 16 & 0x0000000000ff0000); + result |= (static_cast(ReadTransportByte(context)) << 8 & 0x000000000000ff00); + result |= static_cast(ReadTransportByte(context)); + return result; + } + std::vector WebSocketImplementation::ReadTransportBytes( + size_t readLength, + Azure::Core::Context const& context) + { + std::vector result; + size_t index = 0; + while (index < readLength) + { + uint8_t byte = ReadTransportByte(context); + result.push_back(byte); + index += 1; + } + return result; + } + + void WebSocketImplementation::SendTransportBuffer( + std::vector const& sendFrame, + Azure::Core::Context const& context) + { + std::unique_lock transportLock(m_transportMutex); + m_receiveStatistics.BytesSent += static_cast(sendFrame.size()); + m_receiveStatistics.FramesSent += 1; + // Log::Write( + // Logger::Level::Verbose, + // "Send #" + std::to_string(m_receiveStatistics.FramesSent.load()) + "to + // transport:" + // + std::to_string(sendFrame.size()) + "Data: " + HexEncode(sendFrame, 0x10)); + m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); } - // Verify the Sec-WebSocket-Accept header as defined in RFC 6455 Section 1.3, which defines the - // opening handshake used for establishing the WebSocket connection. + // Verify the Sec-WebSocket-Accept header as defined in RFC 6455 Section 1.3, which defines + // the opening handshake used for establishing the WebSocket connection. std::string acceptHeaderGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; void WebSocketImplementation::VerifySocketAccept( std::string const& encodedKey, @@ -612,6 +753,89 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } } + WebSocketImplementation::PingThread::PingThread( + WebSocketImplementation* socketImplementation, + std::chrono::duration pingInterval) + : m_webSocketImplementation(socketImplementation), m_pingInterval(pingInterval) + { + } + void WebSocketImplementation::PingThread::Start(std::shared_ptr transport) + { + m_stop = false; + // Spin up a thread to receive data from the transport. + if (!transport->HasNativeWebsocketSupport()) + { + std::unique_lock lock(m_pingThreadStarted); + m_pingThread = std::thread{&PingThread::PingThreadLoop, this}; + m_pingThreadReady.wait(lock); + } + } + + WebSocketImplementation::PingThread::~PingThread() + { + // Ensure that the receive thread is stopped. + Shutdown(); + } + void WebSocketImplementation::PingThread::Shutdown() + { + if (m_pingThread.joinable()) + { + std::unique_lock lock(m_stopMutex); + m_stop = true; + lock.unlock(); + m_pingThreadStopped.notify_all(); + + m_pingThread.join(); + } + } + + void WebSocketImplementation::PingThread::PingThreadLoop() + { + Log::Write(Logger::Level::Verbose, "Start Ping Thread Loop."); + { + std::unique_lock lock(m_pingThreadStarted); + m_pingThreadReady.notify_all(); + } + while (true) + { + std::unique_lock lock(m_stopMutex); + if (this->m_pingThreadStopped.wait_for(lock, m_pingInterval) == std::cv_status::timeout) + { + Log::Write(Logger::Level::Verbose, "Send Ping to peer."); + + // The receiveContext timed out, this means we timed out our "ping" timeout. + // Send a "Ping" request to the remote node. + auto pingData = GenerateRandomBytes(4); + SendPing(pingData, Azure::Core::Context{}); + } + if (m_stop) + { + Log::Write(Logger::Level::Verbose, "Exiting ping thread"); + return; + } + } + } + + bool WebSocketImplementation::PingThread::SendPing( + std::vector const& pingData, + Azure::Core::Context const& context) + { + std::vector pingFrame = EncodeFrame(SocketOpcode::Ping, true, pingData); + m_webSocketImplementation->m_receiveStatistics.PingFramesSent++; + m_webSocketImplementation->SendTransportBuffer(pingFrame, context); + return true; + } + + void WebSocketImplementation::SendPong( + std::vector const& pongData, + Azure::Core::Context const& context) + { + std::vector pongFrame = EncodeFrame(SocketOpcode::Pong, true, pongData); + + m_receiveStatistics.PongFramesSent++; + SendTransportBuffer(pongFrame, context); + } + // Generator for random bytes. Used in WebSocketImplementation and tests. std::vector GenerateRandomBytes(size_t vectorSize) { diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 736e4a4433..1b4109fd99 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -5,8 +5,10 @@ #include "azure/core/internal/diagnostics/log.hpp" #include "azure/core/internal/http/pipeline.hpp" #include +#include #include #include +#include // Implementation of WebSocket protocol. namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { @@ -28,7 +30,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names WebSocketImplementation(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options); void Open(Azure::Core::Context const& context); - void Close(Azure::Core::Context const& context); void Close( uint16_t closeStatus, std::string const& closeReason, @@ -42,22 +43,15 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context); - std::shared_ptr ReceiveFrame( - Azure::Core::Context const& context, - bool stateIsLocked = false); - - /** - * @brief Send a "ping" frame to the other side of the WebSocket. - * - * @returns True if the ping was sent, false if the underlying transport didn't support "Ping" - * operations. - */ - bool SendPing(std::vector const& pingData, Azure::Core::Context const& context); + std::shared_ptr ReceiveFrame(Azure::Core::Context const& context); void AddHeader(std::string const& headerName, std::string const& headerValue); std::string const& GetChosenProtocol(); bool IsOpen() { return m_state == SocketState::Open; } + bool HasNativeWebSocketSupport(); + + WebSocketStatistics GetStatistics() const; private: // WebSocket opcodes. @@ -82,91 +76,124 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Binary, }; - // Implement a buffered stream reader - class BufferedStreamReader { - std::shared_ptr m_transport; - std::unique_ptr m_initialBodyStream; - constexpr static size_t m_bufferSize = 1024; - uint8_t m_buffer[m_bufferSize]{}; - size_t m_bufferPos = 0; - size_t m_bufferLen = 0; - bool m_eof = false; - + class WebSocketInternalFrame { public: - explicit BufferedStreamReader() = default; - ~BufferedStreamReader() = default; - - void SetInitialStream(std::unique_ptr& stream) - { - m_initialBodyStream = std::move(stream); - } - void SetTransport( - std::shared_ptr& transport) + SocketOpcode Opcode{}; + bool IsFinalFrame{false}; + std::vector Payload; + std::exception_ptr Exception; + WebSocketInternalFrame( + SocketOpcode opcode, + bool isFinalFrame, + std::vector const& payload) + : Opcode(opcode), IsFinalFrame(isFinalFrame), Payload(payload) { - m_transport = transport; } + WebSocketInternalFrame(std::exception_ptr exception) : Exception(exception) {} + }; - uint8_t ReadByte(Azure::Core::Context const& context) - { - if (m_bufferPos >= m_bufferLen) - { - // Start by reading data from our initial body stream. - m_bufferLen = m_initialBodyStream->ReadToCount(m_buffer, m_bufferSize, context); - if (m_bufferLen == 0) - { - // If we run out of the initial stream, we need to read from the transport. - m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); - } - else - { - Azure::Core::Diagnostics::_internal::Log::Write( - Azure::Core::Diagnostics::Logger::Level::Informational, - "Read data from initial stream"); - } - m_bufferPos = 0; - if (m_bufferLen == 0) - { - m_eof = true; - return 0; - } - } - return m_buffer[m_bufferPos++]; - } - uint16_t ReadShort(Azure::Core::Context const& context) - { - uint16_t result = ReadByte(context); - result <<= 8; - result |= ReadByte(context); - return result; - } - uint64_t ReadInt64(Azure::Core::Context const& context) - { - uint64_t result = 0; + struct ReceiveStatistics + { + std::atomic_uint32_t FramesSent; + std::atomic_uint32_t FramesReceived; + std::atomic_uint32_t BytesSent; + std::atomic_uint32_t BytesReceived; + std::atomic_uint32_t PingFramesSent; + std::atomic_uint32_t PingFramesReceived; + std::atomic_uint32_t PongFramesSent; + std::atomic_uint32_t PongFramesReceived; + std::atomic_uint32_t TextFramesReceived; + std::atomic_uint32_t BinaryFramesReceived; + std::atomic_uint32_t ContinuationFramesReceived; + std::atomic_uint32_t CloseFramesReceived; + std::atomic_uint32_t UnknownFramesReceived; + std::atomic_uint32_t FramesDropped; + std::atomic_uint32_t FramesDroppedByPayloadSizeLimit; + std::atomic_uint32_t FramesDroppedByProtocolError; + std::atomic_uint32_t TransportReads; + std::atomic_uint32_t TransportReadBytes; + std::atomic_uint32_t BinaryFramesSent; + std::atomic_uint32_t TextFramesSent; + std::atomic_uint32_t FramesDroppedByClose; - result |= (static_cast(ReadByte(context)) << 56 & 0xff00000000000000); - result |= (static_cast(ReadByte(context)) << 48 & 0x00ff000000000000); - result |= (static_cast(ReadByte(context)) << 40 & 0x0000ff0000000000); - result |= (static_cast(ReadByte(context)) << 32 & 0x000000ff00000000); - result |= (static_cast(ReadByte(context)) << 24 & 0x00000000ff000000); - result |= (static_cast(ReadByte(context)) << 16 & 0x0000000000ff0000); - result |= (static_cast(ReadByte(context)) << 8 & 0x000000000000ff00); - result |= static_cast(ReadByte(context)); - return result; - } - std::vector ReadBytes(size_t readLength, Azure::Core::Context const& context) + void Reset() { - std::vector result; - size_t index = 0; - while (index < readLength) - { - uint8_t byte = ReadByte(context); - result.push_back(byte); - index += 1; - } - return result; + FramesSent = 0; + BytesSent = 0; + FramesReceived = 0; + BytesReceived = 0; + PingFramesReceived = 0; + PingFramesSent = 0; + PongFramesReceived = 0; + PongFramesSent = 0; + TextFramesReceived = 0; + TextFramesSent = 0; + BinaryFramesReceived = 0; + BinaryFramesSent = 0; + ContinuationFramesReceived = 0; + CloseFramesReceived = 0; + UnknownFramesReceived = 0; + FramesDropped = 0; + FramesDroppedByClose = 0; + FramesDroppedByPayloadSizeLimit = 0; + FramesDroppedByProtocolError = 0; + TransportReads = 0; + TransportReadBytes = 0; } + }; + /** + * @brief The PingThread handles sending Ping operations from the WebSocket server. + * + */ + class PingThread { + public: + /** + * @brief Construct a new ReceiveQueue object. + * + * @param webSocketImplementation Parent object, used to send Ping threads. + * @param pingInterval Interval to wait between sending pings. + */ + PingThread( + WebSocketImplementation* webSocketImplementation, + std::chrono::duration pingInterval); + /** + * @brief Destroys a ReceiveQueue object. Blocks until the queue thread is completed. + */ + ~PingThread(); + + /** + * @brief Start the receive queue. This will start a thread that will process incoming frames. + * + * @param transport The websocket transport to use for receiving frames. + */ + void Start(std::shared_ptr transport); + /** + * @brief Stop the receive queue. This will stop the thread that processes incoming frames. + */ + void Shutdown(); - bool IsEof() const { return m_eof; } + private: + /** + * @brief The receive queue thread. + */ + void PingThreadLoop(); + /** + * @brief Send a "ping" frame to the other side of the WebSocket. + * + * @returns True if the ping was sent, false if the underlying transport didn't support "Ping" + * operations. + */ + bool SendPing(std::vector const& pingData, Azure::Core::Context const& context); + + WebSocketImplementation* m_webSocketImplementation; + std::chrono::duration m_pingInterval; + std::thread m_pingThread; + std::mutex m_pingThreadStarted; + std::condition_variable m_pingThreadReady; + + std::mutex m_stopMutex; + std::condition_variable m_pingThreadStopped; + bool m_stop = false; }; /** @@ -287,41 +314,63 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names * is equal to the payload length minus the length of the "Extension * data". */ - std::vector EncodeFrame( + static std::vector EncodeFrame( SocketOpcode opcode, bool isFinal, std::vector const& payload); + SocketState m_state{SocketState::Invalid}; + + std::vector GenerateRandomKey() { return GenerateRandomBytes(16); }; + void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); + + /********* + * Buffered Read Support. Read data from the underlying transport into a buffer. + */ + uint8_t ReadTransportByte(Azure::Core::Context const& context); + uint16_t ReadTransportShort(Azure::Core::Context const& context); + uint64_t ReadTransportInt64(Azure::Core::Context const& context); + std::vector ReadTransportBytes(size_t readLength, Azure::Core::Context const& context); + bool IsTransportEof() const { return m_eof; } + void SendPong(std::vector const& pongData, Azure::Core::Context const& context); + void SendTransportBuffer( + std::vector const& payload, + Azure::Core::Context const& context); + std::shared_ptr ReceiveTransportFrame( + Azure::Core::Context const& context); + /** * @brief Decode a frame received from the websocket server. * - * @param streamReader Buffered stream reader to read the frame from. * @param opcode Opcode returned by the server. * @param isFinal True if this is the final message. + * @param context Context for reads if necessary. * @returns A pointer to the start of the decoded data. */ std::vector DecodeFrame( - BufferedStreamReader& streamReader, SocketOpcode& opcode, - uint64_t& payloadLength, bool& isFinal, Azure::Core::Context const& context); - void SendPong(std::vector const& pongData, Azure::Core::Context const& context); - - SocketState m_state{SocketState::Invalid}; - - std::vector GenerateRandomKey() { return GenerateRandomBytes(16); }; - void VerifySocketAccept(std::string const& encodedKey, std::string const& acceptHeader); Azure::Core::Url m_remoteUrl; WebSocketOptions m_options; std::map m_headers; std::string m_chosenProtocol; std::shared_ptr m_transport; - BufferedStreamReader m_bufferedStreamReader; + PingThread m_pingThread; SocketMessageType m_currentMessageType{SocketMessageType::Unknown}; + std::mutex m_stateMutex; + std::thread::id m_stateOwner; + + ReceiveStatistics m_receiveStatistics{}; std::mutex m_transportMutex; - std::mutex m_stateMutex; + + std::unique_ptr m_initialBodyStream; + constexpr static size_t m_bufferSize = 1024; + uint8_t m_buffer[m_bufferSize]{}; + size_t m_bufferPos = 0; + size_t m_bufferLen = 0; + bool m_eof = false; }; }}}}} // namespace Azure::Core::Http::WebSockets::_detail diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 8df8102156..ecbcc0978b 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -238,7 +238,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateSessionHandle() WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0), - _detail::HINTERNET_deleter()); + _detail::HINTERNET_deleter{}); if (!sessionHandle) { @@ -300,7 +300,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateConnectionHandle( StringToWideString(url.GetHost()).c_str(), port == 0 ? INTERNET_DEFAULT_PORT : port, 0), - _detail::HINTERNET_deleter()); + _detail::HINTERNET_deleter{}); if (!rv) { @@ -341,7 +341,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, // No media types are accepted by the client requestSecureHttp ? WINHTTP_FLAG_SECURE : 0), - _detail::HINTERNET_deleter()); // Uses secure transaction semantics (SSL/TLS) + _detail::HINTERNET_deleter{}); // Uses secure transaction semantics (SSL/TLS) if (!hi) { // Errors include: diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index e2f234f095..6750b4d07c 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -31,7 +31,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { // Convert the request handle into a WebSocket handle for us to use later. m_socketHandle = Azure::Core::Http::_detail::unique_HINTERNET( WinHttpWebSocketCompleteUpgrade(requestHandle.get(), 0), - Azure::Core::Http::_detail::HINTERNET_deleter()); + Azure::Core::Http::_detail::HINTERNET_deleter{}); if (!m_socketHandle) { GetErrorAndThrow("Error Upgrading HttpRequest handle to WebSocket handle."); @@ -109,7 +109,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { uint16_t closeStatus = 0; char closeReason[WINHTTP_WEB_SOCKET_MAX_CLOSE_REASON_LENGTH]{}; DWORD closeReasonLength; - std::lock_guard lock(m_receiveMutex); auto err = WinHttpWebSocketQueryCloseStatus( m_socketHandle.get(), diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 29c1dd4613..352e4d516c 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -1,6 +1,7 @@ from array import array import asyncio from operator import length_hint +import threading from time import sleep from urllib.parse import urlparse @@ -40,36 +41,76 @@ async def handleCustomPath(websocket, path:dict): await websocket.send(data) await websocket.close() +def HexEncode(data: bytes)->str: + rv="" + for val in data: +# rv+= hex(val).removeprefix("0x") + rv+= '{:02X}'.format(val) + return rv + + +echo_count_lock = threading.Lock() +echo_count_recv = 0 +echo_count_send = 0 +client_count = 0 async def handleEcho(websocket, url): + global client_count + global echo_count_recv + global echo_count_send + global echo_count_lock while websocket.open: try: data = await websocket.recv() -# print(f'Echo ', len(data),' bytes') + with echo_count_lock: + echo_count_recv+=1 +# if type(data) is bytes: +# print(f'Echo {HexEncode(data)}') if (url.query == 'fragment=true'): await websocket.send(data.split()) else: await websocket.send(data) + with echo_count_lock: + echo_count_send+=1 except websockets.ConnectionClosedOK: print("Connection closed ok.") + with echo_count_lock: + client_count -= 1 + print(f"Echo count: {echo_count_recv}, {echo_count_send} client_count {client_count}") + if client_count == 0: + echo_count_send = 0 + echo_count_recv = 0 return except websockets.ConnectionClosed as ex: - print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") + if (ex.rcvd): + print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") + else: + print(f"Connection closed. No close information.") + with echo_count_lock: + client_count -= 1 + print(f"Echo count: recv: {echo_count_recv}, send: {echo_count_send} client_count {client_count}") + if client_count == 0: + echo_count_send = 0 + echo_count_recv = 0 return async def handler(websocket, path : str): + global client_count print("Socket handler: ", path) parsedUrl = urlparse(path) if (parsedUrl.path == '/openclosetest'): print("Open/Close Test") try: data = await websocket.recv() + print(f"OpenCloseTest: Received {data}") except websockets.ConnectionClosedOK: - print("Connection closed ok.") + print("OpenCloseTest: Connection closed ok.") except websockets.ConnectionClosed as ex: - print(f"Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") + print(f"OpenCloseTest: Connection closed exception: {ex.rcvd.code} {ex.rcvd.reason}") return elif (parsedUrl.path == '/echotest'): + with echo_count_lock: + client_count+= 1 await handleEcho(websocket, parsedUrl) elif (parsedUrl.path == '/closeduringecho'): data = await websocket.recv() @@ -94,7 +135,7 @@ async def main(): print("Starting server") loop = asyncio.get_running_loop() stop = loop.create_future() - async with websockets.serve(handler, "localhost", 8000, ping_interval=3): + async with websockets.serve(handler, "localhost", 8000, ping_interval=7): await stop # run forever. if __name__=="__main__": diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index c4efddc900..fb4463355a 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #if defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) #include "azure/core/http/websockets/curl_websockets_transport.hpp" @@ -85,7 +86,6 @@ TEST_F(WebSocketTests, OpenAndCloseSocket) { WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000/openclosetest")); - defaultSocket.AddHeader("newHeader", "headerValue"); defaultSocket.Open(); @@ -165,47 +165,6 @@ TEST_F(WebSocketTests, SimpleEcho) } } -TEST_F(WebSocketTests, PingTest) -{ - { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); - - testSocket.Open(); - - testSocket.SendFrame("Test message", true); - // Sleep for 10s - this should trigger a ping. - // The websocket server is configured to ping every 5 seconds, so the 10 second sleep should - // force the next frame received to be a ping frame. - std::this_thread::sleep_for(10s); - - auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketFrameType::TextFrameReceived, response->FrameType); - auto textResult = response->AsTextFrame(); - EXPECT_EQ("Test message", textResult->Text); - - // Close the socket gracefully. - testSocket.Close(); - } - { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); - - testSocket.Open(); - - // Send a "Ping" to the remote server. If the "Ping" operation is supported, - // wait until the corresponding "Pong" response is received. - if (testSocket.SendPing({1, 2, 3, 4})) - { - auto response = testSocket.ReceiveFrame(); - EXPECT_EQ(WebSocketFrameType::PongReceived, response->FrameType); - auto pongResult = response->AsPongFrame(); - std::vector pongData{1, 2, 3, 4}; - EXPECT_EQ(pongResult->Data, pongData); - } - // Close the socket gracefully. - testSocket.Close(); - } -} - template void EchoRandomData(WebSocket& socket) { std::vector sendData = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(N); @@ -301,10 +260,92 @@ std::string ToHexString(std::vector const& data) return ss.str(); } +// Generator for random bytes. Used in WebSocketImplementation and tests. +std::vector GenerateRandomBytes(size_t index, size_t vectorSize) +{ + std::random_device randomEngine; + + std::vector rv(vectorSize + 4); + rv[0] = index & 0xff; + rv[1] = (index >> 8) & 0xff; + rv[2] = (index >> 16) & 0xff; + rv[3] = (index >> 24) & 0xff; + std::generate(std::begin(rv) + 4, std::end(rv), [&randomEngine]() mutable { + return static_cast(randomEngine() % UINT8_MAX); + }); + return rv; +} + +TEST_F(WebSocketTests, PingReceiveTest) +{ + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + if (!testSocket.HasNativeWebSocketSupport()) + { + + GTEST_LOG_(INFO) << "Sleeping for 15 seconds to collect pings."; + Azure::Core::Context receiveContext = Azure::Core::Context::ApplicationContext.WithDeadline( + Azure::DateTime{std::chrono::system_clock::now() + 15s}); + EXPECT_THROW(testSocket.ReceiveFrame(receiveContext), Azure::Core::OperationCancelledException); + auto statistics = testSocket.GetStatistics(); + GTEST_LOG_(INFO) << "Total bytes sent: " << std::dec << statistics.BytesSent; + GTEST_LOG_(INFO) << "Total bytes received: " << std::dec << statistics.BytesReceived; + GTEST_LOG_(INFO) << "Ping Frames received: " << std::dec << statistics.PingFramesReceived; + GTEST_LOG_(INFO) << "Ping Frames sent: " << std::dec << statistics.PingFramesSent; + GTEST_LOG_(INFO) << "Pong Frames received: " << std::dec << statistics.PongFramesReceived; + GTEST_LOG_(INFO) << "Pong Frames sent: " << std::dec << statistics.PongFramesSent; + GTEST_LOG_(INFO) << "Binary frames sent: " << std::dec << statistics.BinaryFramesSent; + GTEST_LOG_(INFO) << "Binary frames received: " << std::dec << statistics.BinaryFramesReceived; + GTEST_LOG_(INFO) << "Total frames lost: " << std::dec << statistics.FramesDropped; + GTEST_LOG_(INFO) << "Transport Reads " << std::dec << statistics.TransportReads; + GTEST_LOG_(INFO) << "Transport Bytes Read " << std::dec << statistics.TransportReadBytes; + EXPECT_NE(0, statistics.PingFramesReceived); + EXPECT_NE(0, statistics.PongFramesSent); + } +} + +TEST_F(WebSocketTests, PingSendTest) +{ + // Configure the socket to ping every second. + WebSocketOptions socketOptions; + socketOptions.PingInterval = std::chrono::seconds(1); + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest"), socketOptions); + + testSocket.Open(); + if (!testSocket.HasNativeWebSocketSupport()) + { + + GTEST_LOG_(INFO) << "Sleeping for 10 seconds to collect pings."; + // Note that we cannot collect incoming pings or outgoing pongs unless we are receiving + // data from the server. + Azure::Core::Context receiveContext = Azure::Core::Context::ApplicationContext.WithDeadline( + Azure::DateTime{std::chrono::system_clock::now() + 10s}); + EXPECT_THROW(testSocket.ReceiveFrame(receiveContext), Azure::Core::OperationCancelledException); + auto statistics = testSocket.GetStatistics(); + GTEST_LOG_(INFO) << "Total bytes sent: " << std::dec << statistics.BytesSent; + GTEST_LOG_(INFO) << "Total bytes received: " << std::dec << statistics.BytesReceived; + GTEST_LOG_(INFO) << "Ping Frames received: " << std::dec << statistics.PingFramesReceived; + GTEST_LOG_(INFO) << "Ping Frames sent: " << std::dec << statistics.PingFramesSent; + GTEST_LOG_(INFO) << "Pong Frames received: " << std::dec << statistics.PongFramesReceived; + GTEST_LOG_(INFO) << "Pong Frames sent: " << std::dec << statistics.PongFramesSent; + GTEST_LOG_(INFO) << "Binary frames sent: " << std::dec << statistics.BinaryFramesSent; + GTEST_LOG_(INFO) << "Binary frames received: " << std::dec << statistics.BinaryFramesReceived; + GTEST_LOG_(INFO) << "Total frames lost: " << std::dec << statistics.FramesDropped; + GTEST_LOG_(INFO) << "Transport Reads " << std::dec << statistics.TransportReads; + GTEST_LOG_(INFO) << "Transport Bytes Read " << std::dec << statistics.TransportReadBytes; + EXPECT_NE(0, statistics.PingFramesSent); + EXPECT_NE(0, statistics.PongFramesReceived); + EXPECT_NE(0, statistics.PingFramesReceived); + EXPECT_NE(0, statistics.PongFramesSent); + } +} + TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; constexpr size_t testDataLength = 150000; + constexpr size_t testDataSize = 100; constexpr auto testDuration = 10s; WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); @@ -318,49 +359,85 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) // Spin up threadCount threads and hammer the echo server for 10 seconds. std::vector threads; - for (size_t i = 0; i < threadCount; i += 1) + std::atomic_int32_t cancellationExceptions{0}; + std::atomic_int32_t exceptions{0}; + for (size_t threadIndex = 0; threadIndex < threadCount; threadIndex += 1) { threads.push_back(std::thread([&]() { - std::chrono::time_point startTime - = std::chrono::steady_clock::now(); + std::chrono::time_point startTime + = std::chrono::system_clock::now(); + // Set the context to expire *after* the test is supposed to finish. + Azure::Core::Context context = Azure::Core::Context::ApplicationContext.WithDeadline( + Azure::DateTime{startTime} + testDuration + 10s); + size_t iteration = 0; try { do { - size_t i = iterationCount++; - std::vector sendData - = Azure::Core::Http::WebSockets::_detail::GenerateRandomBytes(100); + iteration = iterationCount++; + std::vector sendData = GenerateRandomBytes(iteration, testDataSize); { - if (i < testData.size()) + if (iteration < testData.size()) + { + if (testData[iteration].size() != 0) + { + GTEST_LOG_(ERROR) << "Overwriting send frame at offset " << iteration << std::endl; + } + EXPECT_EQ(0, testData[iteration].size()); + testData[iteration] = sendData; + } + else { - EXPECT_EQ(0, testData[i].size()); - testData[i] = sendData; + GTEST_LOG_(ERROR) << "Dropping test result at offset " << iteration << std::endl; } } - testSocket.SendFrame(sendData, true); - - auto response = testSocket.ReceiveFrame(); + testSocket.SendFrame(sendData, true /*, context*/); + auto response = testSocket.ReceiveFrame(context); EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); auto binaryResult = response->AsBinaryFrame(); // Make sure we get back the data we sent in the echo request. + if (binaryResult->Data.size() == 0) + { + GTEST_LOG_(ERROR) << "Received empty frame at offset " << iteration << std::endl; + } EXPECT_EQ(sendData.size(), binaryResult->Data.size()); { // There is no ordering expectation on the results, so we just remember the data // as it comes in. We'll make sure we received everything later on. - if (i < receivedData.size()) + if (iteration < receivedData.size()) { - EXPECT_EQ(0, receivedData[i].size()); - receivedData[i] = binaryResult->Data; + if (receivedData[iteration].size() != 0) + { + GTEST_LOG_(ERROR) << "Overwriting receive frame at offset " << iteration + << std::endl; + } + + EXPECT_EQ(0, receivedData[iteration].size()); + receivedData[iteration] = binaryResult->Data; + } + else + { + GTEST_LOG_(ERROR) << "Dropping test result at offset " << iteration << std::endl; } } - } while (std::chrono::steady_clock::now() - startTime < testDuration); + if (receivedData[iteration].size() == 0) + { + GTEST_LOG_(ERROR) << "Received empty frame at offset " << iteration << std::endl; + } + } while (std::chrono::system_clock::now() - startTime < testDuration); + } + catch (Azure::Core::OperationCancelledException& ex) + { + GTEST_LOG_(ERROR) << "Cancelled Exception: " << ex.what() << " at index " << iteration + << " Current Thread: " << std::this_thread::get_id() << std::endl; + cancellationExceptions++; } catch (std::exception const& ex) { GTEST_LOG_(ERROR) << "Exception: " << ex.what() << std::endl; - EXPECT_TRUE(false); + exceptions++; } })); } @@ -377,9 +454,26 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) GTEST_LOG_(INFO) << "Estimated " << std::dec << testData.size() << " iterations (0x" << std::hex << testData.size() << ")" << std::endl; EXPECT_GE(testDataLength, iterationCount.load()); + + auto statistics = testSocket.GetStatistics(); + GTEST_LOG_(INFO) << "Total bytes sent: " << std::dec << statistics.BytesSent; + GTEST_LOG_(INFO) << "Total bytes received: " << std::dec << statistics.BytesReceived; + GTEST_LOG_(INFO) << "Ping Frames received: " << std::dec << statistics.PingFramesReceived; + GTEST_LOG_(INFO) << "Ping Frames sent: " << std::dec << statistics.PingFramesSent; + GTEST_LOG_(INFO) << "Pong Frames received: " << std::dec << statistics.PongFramesReceived; + GTEST_LOG_(INFO) << "Pong Frames sent: " << std::dec << statistics.PongFramesSent; + GTEST_LOG_(INFO) << "Binary frames sent: " << std::dec << statistics.BinaryFramesSent; + GTEST_LOG_(INFO) << "Binary frames received: " << std::dec << statistics.BinaryFramesReceived; + GTEST_LOG_(INFO) << "Total frames lost: " << std::dec << statistics.FramesDropped; + GTEST_LOG_(INFO) << "Transport Reads " << std::dec << statistics.TransportReads; + GTEST_LOG_(INFO) << "Transport Bytes Read " << std::dec << statistics.TransportReadBytes; + // Close the socket gracefully. testSocket.Close(); + EXPECT_EQ(iterationCount.load(), statistics.BinaryFramesSent); + EXPECT_EQ(iterationCount.load(), statistics.BinaryFramesReceived); + // Resize the test data to the number of actual iterations. testData.resize(iterationCount.load()); receivedData.resize(iterationCount.load()); @@ -387,8 +481,8 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) // If we've processed every iteration, let's make sure that we received everything we sent. // If we dropped some results, then we can't check to ensure that we have received everything // because we can't account for everything sent. - std::set testDataStrings; - std::set receivedDataStrings; + std::multiset testDataStrings; + std::multiset receivedDataStrings; for (auto const& data : testData) { testDataStrings.emplace(ToHexString(data)); @@ -398,14 +492,32 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) receivedDataStrings.emplace(ToHexString(data)); } + // EXPECT_EQ(testDataStrings, receivedDataStrings); for (auto const& data : testDataStrings) { + if (receivedDataStrings.count(data) != testDataStrings.count(data)) + { + GTEST_LOG_(INFO) << "Missing data. TestDataCount: " << testDataStrings.count(data) + << " ReceivedDataCount: " << receivedDataStrings.count(data) + << " Missing Data: " << data << std::endl; + } EXPECT_NE(receivedDataStrings.end(), receivedDataStrings.find(data)); } for (auto const& data : receivedDataStrings) { + if (testDataStrings.count(data) != receivedDataStrings.count(data)) + { + GTEST_LOG_(INFO) << "Extra data. TestDataCount: " << testDataStrings.count(data) + << " ReceivedDataCount: " << receivedDataStrings.count(data) + << " Missing Data: " << data << std::endl; + } + EXPECT_NE(testDataStrings.end(), testDataStrings.find(data)); } + + // We shouldn't have seen any exceptions during the run. + EXPECT_EQ(0, exceptions.load()); + EXPECT_EQ(0, cancellationExceptions.load()); } // Does not work because curl rejects the wss: scheme. From 0b964da162a7ef3ea117eaee5f0439523425443f Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 20 Jul 2022 16:04:13 -0700 Subject: [PATCH 110/149] Compilation errors with winhttp --- .../http/websockets/win_http_websockets_transport.hpp | 2 +- .../azure-core/src/http/websockets/websocketsimpl.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index ef8305c347..01d1a80d07 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -55,7 +55,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @details For the WinHTTP websocket transport, the transport supports native websockets. */ - virtual bool NativeWebsocketSupport() override { return true; } + virtual bool HasNativeWebsocketSupport() override { return true; } /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 8ea657de0a..50f30fab3e 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -212,7 +212,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } m_state = SocketState::Closing; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) + if (m_transport->HasNativeWebsocketSupport()) { m_transport->CloseSocket(closeStatus, closeReason.c_str(), context); } @@ -275,7 +275,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector utf8text(textFrame.begin(), textFrame.end()); m_receiveStatistics.TextFramesSent++; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) + if (m_transport->HasNativeWebsocketSupport()) { m_transport->SendFrame( (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeText @@ -305,7 +305,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } m_receiveStatistics.BinaryFramesSent++; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) + if (m_transport->HasNativeWebsocketSupport()) { m_transport->SendFrame( (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeBinary @@ -427,7 +427,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names WebSocketImplementation::ReceiveTransportFrame(Azure::Core::Context const& context) { #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->NativeWebsocketSupport()) + if (m_transport->HasNativeWebsocketSupport()) { auto payload = m_transport->ReceiveFrame(context); m_receiveStatistics.FramesReceived++; From 48cca807726049b182d6afc16ce5939651f85229 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 20 Jul 2022 16:17:15 -0700 Subject: [PATCH 111/149] gpp 5 doesn't have std::atomic_uint32_t --- .../src/http/websockets/websocketsimpl.hpp | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 1b4109fd99..e6efd84fb7 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -94,27 +94,27 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names struct ReceiveStatistics { - std::atomic_uint32_t FramesSent; - std::atomic_uint32_t FramesReceived; - std::atomic_uint32_t BytesSent; - std::atomic_uint32_t BytesReceived; - std::atomic_uint32_t PingFramesSent; - std::atomic_uint32_t PingFramesReceived; - std::atomic_uint32_t PongFramesSent; - std::atomic_uint32_t PongFramesReceived; - std::atomic_uint32_t TextFramesReceived; - std::atomic_uint32_t BinaryFramesReceived; - std::atomic_uint32_t ContinuationFramesReceived; - std::atomic_uint32_t CloseFramesReceived; - std::atomic_uint32_t UnknownFramesReceived; - std::atomic_uint32_t FramesDropped; - std::atomic_uint32_t FramesDroppedByPayloadSizeLimit; - std::atomic_uint32_t FramesDroppedByProtocolError; - std::atomic_uint32_t TransportReads; - std::atomic_uint32_t TransportReadBytes; - std::atomic_uint32_t BinaryFramesSent; - std::atomic_uint32_t TextFramesSent; - std::atomic_uint32_t FramesDroppedByClose; + std::atomic FramesSent; + std::atomic FramesReceived; + std::atomic BytesSent; + std::atomic BytesReceived; + std::atomic PingFramesSent; + std::atomic PingFramesReceived; + std::atomic PongFramesSent; + std::atomic PongFramesReceived; + std::atomic TextFramesReceived; + std::atomic BinaryFramesReceived; + std::atomic ContinuationFramesReceived; + std::atomic CloseFramesReceived; + std::atomic UnknownFramesReceived; + std::atomic FramesDropped; + std::atomic FramesDroppedByPayloadSizeLimit; + std::atomic FramesDroppedByProtocolError; + std::atomic TransportReads; + std::atomic TransportReadBytes; + std::atomic BinaryFramesSent; + std::atomic TextFramesSent; + std::atomic FramesDroppedByClose; void Reset() { From 227228996d028ef4443d7fcf47c84b95150b182d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 20 Jul 2022 16:39:26 -0700 Subject: [PATCH 112/149] Increased iteration count for multithreaded test --- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index fb4463355a..cd4b8dc92a 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -344,7 +344,7 @@ TEST_F(WebSocketTests, PingSendTest) TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; - constexpr size_t testDataLength = 150000; + constexpr size_t testDataLength = 175000; constexpr size_t testDataSize = 100; constexpr auto testDuration = 10s; From 8cf06ddd419ef9a86958e2005835de9f6c122818 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 20 Jul 2022 17:05:14 -0700 Subject: [PATCH 113/149] Noise reduction; Increased test size --- sdk/core/azure-core/test/ut/websocket_test.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index cd4b8dc92a..73db224693 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -344,7 +344,7 @@ TEST_F(WebSocketTests, PingSendTest) TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) { constexpr size_t threadCount = 50; - constexpr size_t testDataLength = 175000; + constexpr size_t testDataLength = 200000; constexpr size_t testDataSize = 100; constexpr auto testDuration = 10s; @@ -386,10 +386,6 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) EXPECT_EQ(0, testData[iteration].size()); testData[iteration] = sendData; } - else - { - GTEST_LOG_(ERROR) << "Dropping test result at offset " << iteration << std::endl; - } } testSocket.SendFrame(sendData, true /*, context*/); @@ -417,14 +413,6 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) EXPECT_EQ(0, receivedData[iteration].size()); receivedData[iteration] = binaryResult->Data; } - else - { - GTEST_LOG_(ERROR) << "Dropping test result at offset " << iteration << std::endl; - } - } - if (receivedData[iteration].size() == 0) - { - GTEST_LOG_(ERROR) << "Received empty frame at offset " << iteration << std::endl; } } while (std::chrono::system_clock::now() - startTime < testDuration); } From 43081644374e3680544938ee8982933dd5be52d3 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 15:34:35 -0700 Subject: [PATCH 114/149] code review feedback --- .../inc/azure/core/http/curl_transport.hpp | 6 +- .../websockets/curl_websockets_transport.hpp | 16 +-- .../azure/core/http/websockets/websockets.hpp | 44 ++++--- .../http/websockets/websockets_transport.hpp | 114 ++++-------------- .../win_http_websockets_transport.hpp | 24 ++-- sdk/core/azure-core/src/http/curl/curl.cpp | 23 +--- .../src/http/curl/curl_session_private.hpp | 2 +- .../src/http/curl/curl_websockets.cpp | 5 +- .../src/http/websockets/websocketsimpl.cpp | 66 +++++----- .../src/http/winhttp/win_http_websockets.cpp | 50 ++++---- .../test/ut/Start-WebSocketServer.ps1 | 13 ++ .../azure-core/test/ut/websocket_test.cpp | 10 +- sdk/core/ci.yml | 20 ++- 13 files changed, 169 insertions(+), 224 deletions(-) create mode 100644 sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index 0eb6a27bc1..f2fc456421 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -133,10 +133,10 @@ namespace Azure { namespace Core { namespace Http { protected: /** - * @brief Called when an HTTP response indicates that the connection should be upgraded to - * a websocket. + * @brief Called when an HTTP response indicates the connection should be upgraded to + * a websocket. Takes ownership of the CurlNetworkConnection object. */ - virtual void OnUpgradedConnection(std::unique_ptr&){}; + virtual void OnUpgradedConnection(std::unique_ptr&&){}; public: /** diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 5bd01f8533..3824c1b72e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -3,7 +3,7 @@ /** * @file - * @brief #Azure::Core::Http::HttpTransport implementation via CURL. + * @brief #Azure::Core::Http::WebSockets::WebSocketTransport implementation via CURL. */ #pragma once @@ -54,7 +54,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * */ - virtual void Close() override; + virtual void NativeClose() override; // Native WebSocket support methods. /** @@ -63,7 +63,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override + virtual void NativeCloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override { throw std::runtime_error("Not implemented."); } @@ -74,7 +74,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - std::pair GetCloseSocketInformation(const Azure::Core::Context&) override + NativeWebSocketCloseInformation NativeGetCloseSocketInformation(const Azure::Core::Context&) override { throw std::runtime_error("Not implemented"); } @@ -85,8 +85,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - virtual void SendFrame( - WebSocketFrameType, + virtual void NativeSendFrame( + NativeWebSocketFrameType, std::vector const&, Azure::Core::Context const&) override { @@ -99,7 +99,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - virtual std::pair> ReceiveFrame( + virtual NativeWebSocketReceiveInformation NativeReceiveFrame( Azure::Core::Context const&) override { throw std::runtime_error("Not implemented"); @@ -141,7 +141,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { // std::shared_ptr can be. std::shared_ptr m_upgradedConnection; void OnUpgradedConnection( - std::unique_ptr& upgradedConnection) override; + std::unique_ptr&& upgradedConnection) override; }; }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 334e89d4c3..dfb4222bfc 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -3,7 +3,8 @@ /** * @file - * @brief Utilities to be used by HTTP transport implementations. + * @brief Azure Core APIs implementing the WebSocket protocol [RFC 6455] + * (https://www.rfc-editor.org/rfc/rfc6455.html). */ #pragma once @@ -50,6 +51,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketBinaryFrame; class WebSocketPeerCloseFrame; +namespace _detail { + class WebSocketImplementation; + } /** @brief Statistics about data sent and received by the WebSocket. * * @remarks This class is primarily intended for test collateral and debugging to allow @@ -131,50 +135,66 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** @brief Contains the contents of a WebSocket Text frame.*/ class WebSocketTextFrame : public WebSocketFrame, public std::enable_shared_from_this { + friend _detail::WebSocketImplementation; private: public: /** @brief Constructs a new WebSocketTextFrame */ WebSocketTextFrame() = default; + /** @brief Text of the frame received from the remote peer. */ + std::string Text; + + private: /** @brief Constructs a new WebSocketTextFrame * @param isFinalFrame True if this is the final frame in a multi-frame message. * @param body UTF-8 encoded text of the frame data. * @param size Length in bytes of the frame body. */ - WebSocketTextFrame(bool isFinalFrame, unsigned char const* body, size_t size) + WebSocketTextFrame(bool isFinalFrame, uint8_t const* body, size_t size) : WebSocketFrame{WebSocketFrameType::TextFrameReceived, isFinalFrame}, Text(body, body + size) { } - /** @brief Text of the frame received from the remote peer. */ - std::string Text; }; /** @brief Contains the contents of a WebSocket Binary frame.*/ class WebSocketBinaryFrame : public WebSocketFrame, public std::enable_shared_from_this { + friend _detail::WebSocketImplementation; + private: public: /** @brief Constructs a new WebSocketBinaryFrame */ WebSocketBinaryFrame() = default; + /** @brief Binary frame data received from the remote peer. */ + std::vector Data; + /** @brief Constructs a new WebSocketBinaryFrame * @param isFinal True if this is the final frame in a multi-frame message. * @param body binary of the frame data. * @param size Length in bytes of the frame body. */ - WebSocketBinaryFrame(bool isFinal, unsigned char const* body, size_t size) + private: + WebSocketBinaryFrame(bool isFinal, uint8_t const* body, size_t size) : WebSocketFrame{WebSocketFrameType::BinaryFrameReceived, isFinal}, Data(body, body + size) { } - /** @brief Binary frame data received from the remote peer. */ - std::vector Data; }; /** @brief Contains the contents of a WebSocket Close frame.*/ class WebSocketPeerCloseFrame : public WebSocketFrame, public std::enable_shared_from_this { + friend _detail::WebSocketImplementation; + public: /** @brief Constructs a new WebSocketPeerCloseFrame */ WebSocketPeerCloseFrame() = default; + /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode + * enumeration */ + uint16_t RemoteStatusCode; + /** @brief Optional text sent from the remote peer. */ + std::string RemoteCloseReason; + + private: /** @brief Constructs a new WebSocketBinaryFrame * @param remoteStatusCode Status code sent by the remote peer. * @param remoteCloseReason Optional reason sent by the remote peer. @@ -184,11 +204,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) { } - /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode - * enumeration */ - uint16_t RemoteStatusCode; - /** @brief Optional text sent from the remote peer. */ - std::string RemoteCloseReason; + }; struct WebSocketOptions : Azure::Core::_internal::ClientOptions @@ -272,7 +288,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ void SendFrame( std::string const& textFrame, - bool isFinalFrame, + bool isFinalFrame = false, Azure::Core::Context const& context = Azure::Core::Context{}); /** @brief Sends a Binary frame to the remote server. @@ -283,7 +299,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ void SendFrame( std::vector const& binaryFrame, - bool isFinalFrame, + bool isFinalFrame = false, Azure::Core::Context const& context = Azure::Core::Context{}); /** @brief Receive a frame from the remote server. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 0a98f1045d..d00986f5d4 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -21,7 +21,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Web Socket Frame type, one of Text or Binary. */ - enum class WebSocketFrameType + enum class NativeWebSocketFrameType { /** * @brief Indicates that the frame is a partial UTF-8 encoded text frame - it is NOT the @@ -46,6 +46,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { FrameTypeClosed, }; + + struct NativeWebSocketCloseInformation + { + uint16_t CloseReason; + std::string CloseReasonDescription; + }; + struct NativeWebSocketReceiveInformation + { + NativeWebSocketFrameType FrameType; + std::vector FrameData; + }; /** * @brief Destructs `%WebSocketTransport`. * @@ -69,7 +80,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param disconnectReason UTF-8 encoded reason for the disconnection. Optional. * @param context Context for the operation. */ - virtual void CloseSocket( + virtual void NativeCloseSocket( uint16_t status, std::string const& disconnectReason, Azure::Core::Context const& context) @@ -81,7 +92,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * Does not notify the remote endpoint that the socket is being closed. * */ - virtual void Close() = 0; + virtual void NativeClose() = 0; /** * @brief Retrieve the information associated with a WebSocket close response. @@ -90,7 +101,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @returns a tuple containing the status code and string. */ - virtual std::pair GetCloseSocketInformation( + virtual NativeWebSocketCloseInformation NativeGetCloseSocketInformation( Azure::Core::Context const& context) = 0; @@ -101,8 +112,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param frameData Frame data to be sent to the server. * @param context Context for the operation. */ - virtual void SendFrame( - WebSocketFrameType frameType, + virtual void NativeSendFrame( + NativeWebSocketFrameType frameType, std::vector const& frameData, Azure::Core::Context const& context) = 0; @@ -115,7 +126,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @returns a tuple containing the Frame data received from the remote server and the type of * data returned from the remote endpoint */ - virtual std::pair> ReceiveFrame( + virtual NativeWebSocketReceiveInformation NativeReceiveFrame( Azure::Core::Context const& context) = 0; @@ -123,87 +134,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /* Non Native WebSocket support functions */ /**************/ - // Implement a buffered stream reader - class BufferedStreamReader { - std::shared_ptr m_transport; - std::unique_ptr m_initialBodyStream; - constexpr static size_t m_bufferSize = 1024; - uint8_t m_buffer[m_bufferSize]{}; - size_t m_bufferPos = 0; - size_t m_bufferLen = 0; - bool m_eof = false; - - public: - explicit BufferedStreamReader() = default; - ~BufferedStreamReader() = default; - - void SetInitialStream(std::unique_ptr& stream) - { - m_initialBodyStream = std::move(stream); - } - void SetTransport( - std::shared_ptr& transport) - { - m_transport = transport; - } - - uint8_t ReadByte(Azure::Core::Context const& context) - { - if (m_bufferPos >= m_bufferLen) - { - // Start by reading data from our initial body stream. - m_bufferLen = m_initialBodyStream->ReadToCount(m_buffer, m_bufferSize, context); - if (m_bufferLen == 0) - { - // If we run out of the initial stream, we need to read from the transport. - m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); - } - m_bufferPos = 0; - if (m_bufferLen == 0) - { - m_eof = true; - return 0; - } - } - return m_buffer[m_bufferPos++]; - } - uint16_t ReadShort(Azure::Core::Context const& context) - { - uint16_t result = ReadByte(context); - result <<= 8; - result |= ReadByte(context); - return result; - } - uint64_t ReadInt64(Azure::Core::Context const& context) - { - uint64_t result = 0; - - result |= (static_cast(ReadByte(context)) << 56 & 0xff00000000000000); - result |= (static_cast(ReadByte(context)) << 48 & 0x00ff000000000000); - result |= (static_cast(ReadByte(context)) << 40 & 0x0000ff0000000000); - result |= (static_cast(ReadByte(context)) << 32 & 0x000000ff00000000); - result |= (static_cast(ReadByte(context)) << 24 & 0x00000000ff000000); - result |= (static_cast(ReadByte(context)) << 16 & 0x0000000000ff0000); - result |= (static_cast(ReadByte(context)) << 8 & 0x000000000000ff00); - result |= static_cast(ReadByte(context)); - return result; - } - std::vector ReadBytes(size_t readLength, Azure::Core::Context const& context) - { - std::vector result; - size_t index = 0; - while (index < readLength) - { - uint8_t byte = ReadByte(context); - result.push_back(byte); - index += 1; - } - return result; - } - - bool IsEof() const { return m_eof; } - }; - /** * @brief This function is used when working with streams to pull more data from the wire. * Function will try to keep pulling data from socket until the buffer is all written or until @@ -220,7 +150,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { protected: /** - * @brief Constructs a default instance of `%HttpTransport`. + * @brief Constructs a default instance of `%WebSocketTransport`. * */ WebSocketTransport() = default; @@ -233,15 +163,15 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { WebSocketTransport(const WebSocketTransport& other) = default; /** - * @brief Constructs `%HttpTransport` by moving another instance of `%HttpTransport`. + * @brief Constructs a WebSocketTransport from another WebSocketTransport. * * @param other An instance to move in. */ WebSocketTransport(WebSocketTransport&& other) = default; /** - * @brief Assigns `%HttpTransport` to another instance of `%HttpTransport`. - * + * @brief Assigns one WebSocketTransport to another. + * * @param other An instance to assign. * * @return A reference to this instance. diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 01d1a80d07..6b4dc7aaa4 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -19,7 +19,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** - * @brief Concrete implementation of a WebSocket Transport that uses libcurl. + * @brief Concrete implementation of a WebSocket Transport that uses WinHTTP. */ class WinHttpWebSocketTransport : public WebSocketTransport, public WinHttpTransport { @@ -32,7 +32,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** - * @brief Construct a new CurlTransport object. + * @brief Construct a new WinHTTP WebSocket Transport. * * @param options Optional parameter to override the default options. */ @@ -61,7 +61,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. * */ - virtual void Close() override; + virtual void NativeClose() override; // Native WebSocket support methods. /** @@ -74,7 +74,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param context Context for the operation. * */ - virtual void CloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override; + virtual void NativeCloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) + override; /** * @brief Retrieve the information associated with a WebSocket close response. @@ -85,7 +86,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @returns a tuple containing the status code and string. */ - virtual std::pair GetCloseSocketInformation( + virtual NativeWebSocketCloseInformation NativeGetCloseSocketInformation( Azure::Core::Context const& context) override; /** @@ -97,12 +98,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief frameType Frame type sent to the server, Text or Binary. * @brief frameData Frame data to be sent to the server. */ - virtual void SendFrame( - WebSocketFrameType, + virtual void NativeSendFrame( + NativeWebSocketFrameType, std::vector const&, Azure::Core::Context const&) override; - virtual std::pair> ReceiveFrame( + virtual NativeWebSocketReceiveInformation NativeReceiveFrame( Azure::Core::Context const&) override; // Non-Native WebSocket support. @@ -111,6 +112,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * Function will try to keep pulling data from socket until the buffer is all written or * until there is no more data to get from the socket. * + * @details Not implemented for WinHTTP websockets because WinHTTP implements websockets + * natively. */ virtual size_t ReadFromSocket(uint8_t*, size_t, Context const&) override { @@ -118,7 +121,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } /** - * @brief This method will use libcurl socket to write all the bytes from buffer. + * @brief This method will use sockets to write all the bytes from buffer. + * + * @details Not implemented for WinHTTP websockets because WinHTTP implements websockets + * natively. * */ virtual int SendBuffer(uint8_t const*, size_t, Context const&) override diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 001c91eb07..82a4b7bbb1 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -140,15 +140,7 @@ void WinSocketSetBuffSize(curl_socket_t socket) // if WSAloctl succeeded (returned 0), set the socket buffer size. // Specifies the total per-socket buffer space reserved for sends. // https://docs.microsoft.com/windows/win32/api/winsock/nf-winsock-setsockopt - auto result = setsockopt(socket, SOL_SOCKET, SO_SNDBUF, (const char*)&ideal, sizeof(ideal)); - result; - // if (Log::ShouldWrite(Logger::Level::Verbose)) - // { - // Log::Write( - // Logger::Level::Verbose, - // LogMsgPrefix + "Windows - calling setsockopt after uploading chunk. ideal = " - // + std::to_string(ideal) + " result = " + std::to_string(result)); - // } + setsockopt(socket, SOL_SOCKET, SO_SNDBUF, (const char*)&ideal, sizeof(ideal)); } } #endif @@ -327,15 +319,12 @@ std::unique_ptr CurlTransport::Send(Request& request, Context const throw Azure::Core::Http::TransportException( "Error while sending request. " + std::string(curl_easy_strerror(performing))); } - else + if (SupportsWebSockets()) { - if (SupportsWebSockets()) + std::unique_ptr upgradedConnection(session->ExtractConnection()); + if (upgradedConnection) { - std::unique_ptr upgradedConnection(session->GetUpgradedConnection()); - if (upgradedConnection) - { - OnUpgradedConnection(upgradedConnection); - } + OnUpgradedConnection(std::move(upgradedConnection)); } } @@ -438,7 +427,7 @@ CURLcode CurlSession::Perform(Context const& context) return result; } -std::unique_ptr CurlSession::GetUpgradedConnection() +std::unique_ptr CurlSession::ExtractConnection() { if (m_connectionUpgraded) { diff --git a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp index a94552dabe..be77f85fbe 100644 --- a/sdk/core/azure-core/src/http/curl/curl_session_private.hpp +++ b/sdk/core/azure-core/src/http/curl/curl_session_private.hpp @@ -430,7 +430,7 @@ namespace Azure { namespace Core { namespace Http { * * @return The network connection, or null if the connection was not upgraded. */ - std::unique_ptr GetUpgradedConnection(); + std::unique_ptr ExtractConnection(); }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp index 2e448a58e2..edd10355d9 100644 --- a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp +++ b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp @@ -27,7 +27,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { - void CurlWebSocketTransport::Close() { m_upgradedConnection->Shutdown(); } + void CurlWebSocketTransport::NativeClose() { m_upgradedConnection->Shutdown(); } // Send an HTTP request to the remote server. std::unique_ptr CurlWebSocketTransport::Send( @@ -71,9 +71,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } void CurlWebSocketTransport::OnUpgradedConnection( - std::unique_ptr& upgradedConnection) + std::unique_ptr&& upgradedConnection) { - CurlTransport::OnUpgradedConnection(upgradedConnection); // Note that m_upgradedConnection is a std::shared_ptr. We define it as a std::shared_ptr // because a std::shared_ptr can be declared on an incomplete type, while a std::unique_ptr // cannot. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 50f30fab3e..d96926c08d 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -214,7 +214,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->HasNativeWebsocketSupport()) { - m_transport->CloseSocket(closeStatus, closeReason.c_str(), context); + m_transport->NativeCloseSocket(closeStatus, closeReason.c_str(), context); } else #endif @@ -257,7 +257,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } // Close the socket - after this point, the m_transport is invalid. m_pingThread.Shutdown(); - m_transport->Close(); + m_transport->NativeClose(); m_state = SocketState::Closed; } @@ -277,9 +277,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->HasNativeWebsocketSupport()) { - m_transport->SendFrame( - (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeText - : WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment), + m_transport->NativeSendFrame( + (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::FrameTypeText + : WebSocketTransport::NativeWebSocketFrameType::FrameTypeTextFragment), utf8text, context); } @@ -307,9 +307,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->HasNativeWebsocketSupport()) { - m_transport->SendFrame( - (isFinalFrame ? WebSocketTransport::WebSocketFrameType::FrameTypeBinary - : WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment), + m_transport->NativeSendFrame( + (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary + : WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinaryFragment), binaryFrame, context); } @@ -362,13 +362,13 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names case SocketOpcode::BinaryFrame: m_currentMessageType = SocketMessageType::Binary; - return std::make_shared( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); + return std::shared_ptr(new WebSocketBinaryFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); case SocketOpcode::TextFrame: m_currentMessageType = SocketMessageType::Text; - return std::make_shared( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); + return std::shared_ptr(new WebSocketTextFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); case SocketOpcode::Close: { if (frame->Payload.size() < 2) @@ -384,8 +384,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names lock.lock(); m_state = SocketState::Closed; - return std::make_shared( - errorCode, std::string(frame->Payload.begin() + 2, frame->Payload.end())); + return std::shared_ptr(new WebSocketPeerCloseFrame( + errorCode, std::string(frame->Payload.begin() + 2, frame->Payload.end()))); } // Continuation frames need to be treated somewhat specially. @@ -399,8 +399,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { m_currentMessageType = SocketMessageType::Unknown; } - return std::make_shared( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); + return std::shared_ptr(new WebSocketTextFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); } else if (m_currentMessageType == SocketMessageType::Binary) { @@ -408,8 +408,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { m_currentMessageType = SocketMessageType::Unknown; } - return std::make_shared( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size()); + return std::shared_ptr(new WebSocketBinaryFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); } else { @@ -429,34 +429,34 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names #if SUPPORT_NATIVE_TRANSPORT if (m_transport->HasNativeWebsocketSupport()) { - auto payload = m_transport->ReceiveFrame(context); + auto payload = m_transport->NativeReceiveFrame(context); m_receiveStatistics.FramesReceived++; - switch (payload.first) + switch (payload.FrameType) { - case WebSocketTransport::WebSocketFrameType::FrameTypeBinary: + case WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary: m_receiveStatistics.BinaryFramesReceived++; return std::make_shared( - SocketOpcode::BinaryFrame, true, payload.second); - case WebSocketTransport::WebSocketFrameType::FrameTypeBinaryFragment: + SocketOpcode::BinaryFrame, true, payload.FrameData); + case WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinaryFragment: m_receiveStatistics.BinaryFramesReceived++; return std::make_shared( - SocketOpcode::BinaryFrame, false, payload.second); - case WebSocketTransport::WebSocketFrameType::FrameTypeText: + SocketOpcode::BinaryFrame, false, payload.FrameData); + case WebSocketTransport::NativeWebSocketFrameType::FrameTypeText: m_receiveStatistics.TextFramesReceived++; return std::make_shared( - SocketOpcode::TextFrame, true, payload.second); - case WebSocketTransport::WebSocketFrameType::FrameTypeTextFragment: + SocketOpcode::TextFrame, true, payload.FrameData); + case WebSocketTransport::NativeWebSocketFrameType::FrameTypeTextFragment: m_receiveStatistics.TextFramesReceived++; return std::make_shared( - SocketOpcode::TextFrame, false, payload.second); - case WebSocketTransport::WebSocketFrameType::FrameTypeClosed: { + SocketOpcode::TextFrame, false, payload.FrameData); + case WebSocketTransport::NativeWebSocketFrameType::FrameTypeClosed: { m_receiveStatistics.CloseFramesReceived++; - auto closeResult = m_transport->GetCloseSocketInformation(context); + auto closeResult = m_transport->NativeGetCloseSocketInformation(context); std::vector closePayload; - closePayload.push_back(closeResult.first >> 8); - closePayload.push_back(closeResult.first & 0xff); + closePayload.push_back(closeResult.CloseReason >> 8); + closePayload.push_back(closeResult.CloseReason& 0xff); closePayload.insert( - closePayload.end(), closeResult.second.begin(), closeResult.second.end()); + closePayload.end(), closeResult.CloseReasonDescription.begin(), closeResult.CloseReasonDescription.end()); return std::make_shared(SocketOpcode::Close, true, closePayload); } default: diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 6750b4d07c..36083673fd 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -48,7 +48,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Close the WebSocket cleanly. */ - void WinHttpWebSocketTransport::Close() { m_socketHandle.reset(); } + void WinHttpWebSocketTransport::NativeClose() { m_socketHandle.reset(); } // Native WebSocket support methods. /** @@ -61,7 +61,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param context Context for the operation. * */ - void WinHttpWebSocketTransport::CloseSocket( + void WinHttpWebSocketTransport::NativeCloseSocket( uint16_t status, std::string const& disconnectReason, Azure::Core::Context const& context) @@ -83,14 +83,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { context.ThrowIfCancelled(); // Make sure that the server responds gracefully to the close request. - auto closeInformation = GetCloseSocketInformation(context); + auto closeInformation = NativeGetCloseSocketInformation(context); // The server should return the same status we sent. - if (closeInformation.first != status) + if (closeInformation.CloseReason != status) { throw std::runtime_error( - "Close status mismatch, got " + std::to_string(closeInformation.first) + " expected " - + std::to_string(status)); + "Close status mismatch, got " + std::to_string(closeInformation.CloseReason) + + " expected " + std::to_string(status)); } } /** @@ -102,8 +102,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @returns a tuple containing the status code and string. */ - std::pair WinHttpWebSocketTransport::GetCloseSocketInformation( - Azure::Core::Context const& context) + WinHttpWebSocketTransport::NativeWebSocketCloseInformation + WinHttpWebSocketTransport::NativeGetCloseSocketInformation(Azure::Core::Context const& context) { context.ThrowIfCancelled(); uint16_t closeStatus = 0; @@ -120,7 +120,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { { GetErrorAndThrow("WinHttpGetCloseStatus() failed", err); } - return std::make_pair(closeStatus, std::string(closeReason)); + return NativeWebSocketCloseInformation{closeStatus, std::string(closeReason)}; } /** @@ -132,8 +132,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief frameType Frame type sent to the server, Text or Binary. * @brief frameData Frame data to be sent to the server. */ - void WinHttpWebSocketTransport::SendFrame( - WebSocketFrameType frameType, + void WinHttpWebSocketTransport::NativeSendFrame( + NativeWebSocketFrameType frameType, std::vector const& frameData, Azure::Core::Context const& context) { @@ -141,16 +141,16 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; switch (frameType) { - case WebSocketFrameType::FrameTypeText: + case NativeWebSocketFrameType::FrameTypeText: bufferType = WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE; break; - case WebSocketFrameType::FrameTypeBinary: + case NativeWebSocketFrameType::FrameTypeBinary: bufferType = WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE; break; - case WebSocketFrameType::FrameTypeBinaryFragment: + case NativeWebSocketFrameType::FrameTypeBinaryFragment: bufferType = WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE; break; - case WebSocketFrameType::FrameTypeTextFragment: + case NativeWebSocketFrameType::FrameTypeTextFragment: bufferType = WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE; break; default: @@ -171,13 +171,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { } } - std::pair< - Azure::Core::Http::WebSockets::WebSocketTransport::WebSocketFrameType, - std::vector> - WinHttpWebSocketTransport::ReceiveFrame(Azure::Core::Context const& context) + WinHttpWebSocketTransport::NativeWebSocketReceiveInformation + WinHttpWebSocketTransport::NativeReceiveFrame(Azure::Core::Context const& context) { WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; - WebSocketFrameType frameTypeReceived; + NativeWebSocketFrameType frameTypeReceived; DWORD bufferBytesRead; std::vector buffer(128); context.ThrowIfCancelled(); @@ -198,25 +196,25 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { switch (bufferType) { case WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE: - frameTypeReceived = WebSocketFrameType::FrameTypeText; + frameTypeReceived = NativeWebSocketFrameType::FrameTypeText; break; case WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE: - frameTypeReceived = WebSocketFrameType::FrameTypeBinary; + frameTypeReceived = NativeWebSocketFrameType::FrameTypeBinary; break; case WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE: - frameTypeReceived = WebSocketFrameType::FrameTypeBinaryFragment; + frameTypeReceived = NativeWebSocketFrameType::FrameTypeBinaryFragment; break; case WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE: - frameTypeReceived = WebSocketFrameType::FrameTypeTextFragment; + frameTypeReceived = NativeWebSocketFrameType::FrameTypeTextFragment; break; case WINHTTP_WEB_SOCKET_CLOSE_BUFFER_TYPE: - frameTypeReceived = WebSocketFrameType::FrameTypeClosed; + frameTypeReceived = NativeWebSocketFrameType::FrameTypeClosed; break; default: throw std::runtime_error("Unknown frame type."); break; } - return std::make_pair(frameTypeReceived, buffer); + return NativeWebSocketReceiveInformation{frameTypeReceived, buffer}; } }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 new file mode 100644 index 0000000000..d32e2c108c --- /dev/null +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT +param( + [string] $LogFileLocation = "$($env:BUILD_SOURCESDIRECTORY)/WebSocketServer.log" +) + +if ($IsWindows) { + Start-Process 'python.exe' ` + -ArgumentList 'websocket_server.py' ` + -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log +} else { + nohup python websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & +} diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 73db224693..e89458cfe8 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -60,7 +60,7 @@ TEST_F(WebSocketTests, OpenSimpleSocket) { WebSocketOptions options; - WebSocket defaultSocket(Azure::Core::Url("http://microsoft.com/index.htm"), options); + WebSocket defaultSocket(Azure::Core::Url("http://www.microsoft.com/"), options); defaultSocket.AddHeader("newHeader", "headerValue"); // When running this test locally, the call times out, so drop in a 5 second timeout on @@ -674,12 +674,12 @@ TEST_F(WebSocketTests, CurlTransportCoverage) auto transport = std::make_shared(transportOptions); - EXPECT_THROW(transport->CloseSocket(1001, {}, {}), std::runtime_error); - EXPECT_THROW(transport->GetCloseSocketInformation({}), std::runtime_error); + EXPECT_THROW(transport->NativeCloseSocket(1001, {}, {}), std::runtime_error); + EXPECT_THROW(transport->NativeGetCloseSocketInformation({}), std::runtime_error); EXPECT_THROW( - transport->SendFrame(WebSocketTransport::WebSocketFrameType::FrameTypeBinary, {}, {}), + transport->NativeSendFrame(WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary, {}, {}), std::runtime_error); - EXPECT_THROW(transport->ReceiveFrame({}), std::runtime_error); + EXPECT_THROW(transport->NativeReceiveFrame({}), std::runtime_error); } } diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 8aa8662cd7..8f2a2e78ad 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -55,19 +55,13 @@ stages: workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - - pwsh: | - Start-Process 'python.exe' ` - -ArgumentList 'sdk/core/azure-core/test/ut/websocket_server.py' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log - workingDirectory: build - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Windows). - - - bash: | - nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & - workingDirectory: build - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server (Linux). + - pwsh: + inputs: + pwsh: true + filePath: Start-WebSocketServer.ps1 + workingDirectory: build/sdk/core/azure-core/test/ut + condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + displayName: Launch python websocket server PostTestSteps: # Shut down the test server. This uses curl to send a request to the "terminateserver" websocket endpoint. From 9df1b134d1b17cebe0d91a1b7b199d75e9ae1303 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 15:56:07 -0700 Subject: [PATCH 115/149] Pull request feedback --- .../inc/azure/core/http/curl_transport.hpp | 2 +- .../inc/azure/core/http/transport.hpp | 2 +- .../websockets/curl_websockets_transport.hpp | 14 +++-- .../win_http_websockets_transport.hpp | 2 +- sdk/core/azure-core/src/http/curl/curl.cpp | 2 +- .../src/http/websockets/websocketsimpl.cpp | 2 +- .../src/http/winhttp/win_http_transport.cpp | 4 +- .../src/http/winhttp/win_http_websockets.cpp | 4 +- sdk/core/azure-core/test/ut/CMakeLists.txt | 16 +++--- .../test/ut/Start-WebSocketServer.ps1 | 4 +- .../azure-core/test/ut/websocket_server.py | 3 -- .../azure-core/test/ut/websocket_test.cpp | 52 ++++++++----------- 12 files changed, 51 insertions(+), 56 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index f2fc456421..2816f57581 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -47,7 +47,7 @@ namespace Azure { namespace Core { namespace Http { /** * @brief Set the libcurl connection options like a proxy and CA path. */ - struct CurlTransportOptions final + struct CurlTransportOptions { /** * @brief The string for the proxy is passed directly to the libcurl handle without any parsing. diff --git a/sdk/core/azure-core/inc/azure/core/http/transport.hpp b/sdk/core/azure-core/inc/azure/core/http/transport.hpp index 4d7c39bf6c..448f060319 100644 --- a/sdk/core/azure-core/inc/azure/core/http/transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/transport.hpp @@ -71,7 +71,7 @@ namespace Azure { namespace Core { namespace Http { * @brief Returns true if the HttpTransport supports WebSockets (the ability to * communicate bidirectionally on the TCP connection used by the HTTP transport). */ - virtual bool SupportsWebSockets() const { return false; } + virtual bool HasWebSocketSupport() const { return false; } }; }}} // namespace Azure::Core::Http diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 3824c1b72e..1b74484cf2 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -17,6 +17,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { + struct CurlWebSocketTransportOptions : public Azure::Core::Http::CurlTransportOptions + { + }; /** * @brief Concrete implementation of a WebSocket Transport that uses libcurl. */ @@ -27,10 +30,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @param options Optional parameter to override the default options. */ - CurlWebSocketTransport(CurlTransportOptions const& options = CurlTransportOptions()) + CurlWebSocketTransport(CurlWebSocketTransportOptions const& options = CurlWebSocketTransportOptions()) : CurlTransport(options) { } + /** * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse * @@ -63,7 +67,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - virtual void NativeCloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override + virtual void NativeCloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) + override { throw std::runtime_error("Not implemented."); } @@ -74,7 +79,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @details Not implemented for CURL websockets because CURL does not support native websockets. * */ - NativeWebSocketCloseInformation NativeGetCloseSocketInformation(const Azure::Core::Context&) override + NativeWebSocketCloseInformation NativeGetCloseSocketInformation( + const Azure::Core::Context&) override { throw std::runtime_error("Not implemented"); } @@ -134,7 +140,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief returns true if this transport supports WebSockets, false otherwise. */ - bool SupportsWebSockets() const override { return true; } + bool HasWebSocketSupport() const override { return true; } private: // std::unique_ptr cannot be constructed on an incomplete type (CurlNetworkConnection), but diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 6b4dc7aaa4..364cf70e6a 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -132,7 +132,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { throw std::runtime_error("Not implemented."); } - bool SupportsWebSockets() const override { return true; } + bool HasWebSocketSupport() const override { return true; } }; }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 82a4b7bbb1..f8e23cf006 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -319,7 +319,7 @@ std::unique_ptr CurlTransport::Send(Request& request, Context const throw Azure::Core::Http::TransportException( "Error while sending request. " + std::string(curl_easy_strerror(performing))); } - if (SupportsWebSockets()) + if (HasWebSocketSupport()) { std::unique_ptr upgradedConnection(session->ExtractConnection()); if (upgradedConnection) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index d96926c08d..aa2b9779ca 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -63,7 +63,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_transport = std::static_pointer_cast(winHttpTransport); m_options.Transport.Transport = std::static_pointer_cast(winHttpTransport); #elif defined(BUILD_CURL_HTTP_TRANSPORT_ADAPTER) - CurlTransportOptions transportOptions; + CurlWebSocketTransportOptions transportOptions; transportOptions.HttpKeepAlive = false; auto curlWebSockets = std::make_shared(transportOptions); diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index ecbcc0978b..5520b3e1de 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -377,7 +377,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( } } - if (SupportsWebSockets()) + if (HasWebSocketSupport()) { if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) { @@ -686,7 +686,7 @@ std::unique_ptr WinHttpTransport::Send(Request& request, Context co ReceiveResponse(requestHandle, context); - if (SupportsWebSockets()) + if (HasWebSocketSupport()) { OnResponseReceived(requestHandle); } diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 36083673fd..d8521bba07 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -154,7 +154,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { bufferType = WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE; break; default: - throw std::runtime_error("Unknown frame type."); + throw std::runtime_error("Unknown frame type: " + std::to_string(static_cast(frameType))); break; } // Lock the socket to prevent concurrent writes. WinHTTP gets annoyed if @@ -211,7 +211,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { frameTypeReceived = NativeWebSocketFrameType::FrameTypeClosed; break; default: - throw std::runtime_error("Unknown frame type."); + throw std::runtime_error("Unknown frame type: " + std::to_string(bufferType)); break; } return NativeWebSocketReceiveInformation{frameTypeReceived, buffer}; diff --git a/sdk/core/azure-core/test/ut/CMakeLists.txt b/sdk/core/azure-core/test/ut/CMakeLists.txt index 2451d9153d..ac0023446b 100644 --- a/sdk/core/azure-core/test/ut/CMakeLists.txt +++ b/sdk/core/azure-core/test/ut/CMakeLists.txt @@ -99,19 +99,17 @@ if (MSVC) target_compile_options(azure-core-test PUBLIC /wd26495 /wd26812 /wd6326 /wd28204 /wd28020 /wd6330 /wd4389) endif() -#add_custom_target(copy_non_source_outputs -# COMMAND ${CMAKE_COMMAND} -E copy websocket_server.py requirements.txt ${CMAKE_BINARY_DIR} -# SOURCES ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt) -#add_dependencies(azure-core-test copy_non_source_outputs) +# Additional test files to be copied to the output directory. +set(TEST_ADDITIONAL_FILES ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py + ${CMAKE_CURRENT_LIST_DIR}/requirements.txt + ${CMAKE_CURRENT_LIST_DIR}/Start-WebSocketServer.ps1 +) add_custom_command(TARGET azure-core-test POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${CMAKE_CURRENT_LIST_DIR}/websocket_server.py ${CMAKE_CURRENT_LIST_DIR}/requirements.txt + COMMAND ${CMAKE_COMMAND} -E copy ${TEST_ADDITIONAL_FILES} ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${TEST_ADDITIONAL_FILES} COMMENT 'Copying non-source output files') -#file(COPY_FILE requirements.txt ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt ONLY_IF_DIFFERENT) -#file(COPY_FILE websocket_server.py ${CMAKE_CURRENT_BINARY_DIR}/websocket_server.py ONLY_IF_DIFFERENT) - # Adding private headers from CORE to the tests so we can test the private APIs with no relative paths include. target_include_directories (azure-core-test PRIVATE $) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index d32e2c108c..69b6622309 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -7,7 +7,7 @@ param( if ($IsWindows) { Start-Process 'python.exe' ` -ArgumentList 'websocket_server.py' ` - -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/WebSocketServer.log + -NoNewWindow -PassThru -RedirectStandardOutput $LogFileLocation } else { - nohup python websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & + nohup python websocket_server.py > $LogFileLocation } diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 352e4d516c..9aa8e165c8 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -44,7 +44,6 @@ async def handleCustomPath(websocket, path:dict): def HexEncode(data: bytes)->str: rv="" for val in data: -# rv+= hex(val).removeprefix("0x") rv+= '{:02X}'.format(val) return rv @@ -63,8 +62,6 @@ async def handleEcho(websocket, url): data = await websocket.recv() with echo_count_lock: echo_count_recv+=1 -# if type(data) is bytes: -# print(f'Echo {HexEncode(data)}') if (url.query == 'fragment=true'): await websocket.send(data.split()) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index e89458cfe8..47ef2e6065 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -18,20 +18,14 @@ using namespace Azure::Core; using namespace Azure::Core::Http::WebSockets; using namespace std::chrono_literals; +constexpr uint16_t UndefinedButLegalCloseReason = 4500; + class WebSocketTests : public testing::Test { private: protected: // Create static void SetUpTestSuite() {} - static void TearDownTestSuite() - { - // GTEST_LOG_(INFO) << "Shut down test server" << std::endl; - // WebSocket controlSocket(Azure::Core::Url("http://localhost:8000/control")); - // controlSocket.Open(); - // controlSocket.SendFrame("close", true); - // auto controlResponse = controlSocket.ReceiveFrame(); - // EXPECT_EQ(controlResponse->FrameType, WebSocketFrameType::TextFrameReceived); - } + static void TearDownTestSuite() {} }; TEST_F(WebSocketTests, CreateSimpleSocket) @@ -66,7 +60,7 @@ TEST_F(WebSocketTests, OpenSimpleSocket) // When running this test locally, the call times out, so drop in a 5 second timeout on // the request. Azure::Core::Context requestContext = Azure::Core::Context::ApplicationContext.WithDeadline( - std::chrono::system_clock::now() + 15s); + std::chrono::system_clock::now() + 5s); EXPECT_THROW(defaultSocket.Open(requestContext), std::runtime_error); } } @@ -81,7 +75,7 @@ TEST_F(WebSocketTests, OpenAndCloseSocket) defaultSocket.Open(); // Close the socket without notifying the peer. - defaultSocket.Close(4500); + defaultSocket.Close(UndefinedButLegalCloseReason); } { @@ -90,7 +84,7 @@ TEST_F(WebSocketTests, OpenAndCloseSocket) defaultSocket.Open(); // Close the socket without notifying the peer. - defaultSocket.Close(4500, "This is a good reason."); + defaultSocket.Close(UndefinedButLegalCloseReason, "This is a good reason."); // // Now re-open the socket - this should work to reset everything. @@ -201,18 +195,19 @@ TEST_F(WebSocketTests, VariableSizeEcho) EchoRandomData<126>(testSocket); EchoRandomData<127>(testSocket); EchoRandomData<128>(testSocket); - EchoRandomData<1020>(testSocket); - EchoRandomData<1021>(testSocket); - EchoRandomData<1022>(testSocket); - EchoRandomData<1023>(testSocket); - EchoRandomData<1024>(testSocket); - EchoRandomData<2048>(testSocket); - EchoRandomData<4096>(testSocket); - EchoRandomData<8192>(testSocket); + EchoRandomData<1020>(testSocket); // 1K-4 + EchoRandomData<1021>(testSocket); // 1K-3 + EchoRandomData<1022>(testSocket); // 1K-2 + EchoRandomData<1023>(testSocket); // 1K-1 + EchoRandomData<1024>(testSocket); // 1K + EchoRandomData<2048>(testSocket); // 2K + EchoRandomData<4096>(testSocket); // 4K + EchoRandomData<8192>(testSocket); // 8K // The websocket protocol treats lengths of >65536 specially. - EchoRandomData<65535>(testSocket); - EchoRandomData<65536>(testSocket); - EchoRandomData<131072>(testSocket); + EchoRandomData<65535>(testSocket); // 64K-1 + EchoRandomData<65536>(testSocket); // 64K + EchoRandomData<65537>(testSocket); // 64K+1 + EchoRandomData<131072>(testSocket); // 128K } // Close the socket gracefully. testSocket.Close(); @@ -429,7 +424,6 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) } })); } - // std::this_thread::sleep_for(10s); // Wait for all the threads to exit. for (auto& thread : threads) @@ -480,7 +474,7 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) receivedDataStrings.emplace(ToHexString(data)); } - // EXPECT_EQ(testDataStrings, receivedDataStrings); + EXPECT_EQ(testDataStrings, receivedDataStrings); for (auto const& data : testDataStrings) { if (receivedDataStrings.count(data) != testDataStrings.count(data)) @@ -669,7 +663,7 @@ TEST_F(WebSocketTests, CurlTransportCoverage) { { - Azure::Core::Http::CurlTransportOptions transportOptions; + Azure::Core::Http::WebSockets::CurlWebSocketTransportOptions transportOptions; transportOptions.HttpKeepAlive = false; auto transport = std::make_shared(transportOptions); @@ -677,10 +671,10 @@ TEST_F(WebSocketTests, CurlTransportCoverage) EXPECT_THROW(transport->NativeCloseSocket(1001, {}, {}), std::runtime_error); EXPECT_THROW(transport->NativeGetCloseSocketInformation({}), std::runtime_error); EXPECT_THROW( - transport->NativeSendFrame(WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary, {}, {}), + transport->NativeSendFrame( + WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary, {}, {}), std::runtime_error); EXPECT_THROW(transport->NativeReceiveFrame({}), std::runtime_error); } } - -#endif \ No newline at end of file +#endif From dddac79726e08300578dbf950d174d68f4351cab Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 15:58:43 -0700 Subject: [PATCH 116/149] CI changes --- sdk/core/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 8f2a2e78ad..268deafdc8 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -55,13 +55,13 @@ stages: workingDirectory: build/sdk/core/azure-core/test/ut displayName: Install Python requirements. condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - - pwsh: + - task: PowerShell@2 + displayName: 'Launch python websocket server' inputs: pwsh: true filePath: Start-WebSocketServer.ps1 workingDirectory: build/sdk/core/azure-core/test/ut condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - displayName: Launch python websocket server PostTestSteps: # Shut down the test server. This uses curl to send a request to the "terminateserver" websocket endpoint. From 1711fcabe6e2c39f9e6a798c146f9e8923cb6b01 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 16:08:27 -0700 Subject: [PATCH 117/149] Better path to PS1 file --- sdk/core/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 268deafdc8..818598e3a5 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -59,7 +59,7 @@ stages: displayName: 'Launch python websocket server' inputs: pwsh: true - filePath: Start-WebSocketServer.ps1 + filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 workingDirectory: build/sdk/core/azure-core/test/ut condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) From 05c43c318e6ff71949bf474300e99dd2acee2379 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 16:24:50 -0700 Subject: [PATCH 118/149] Powershell comments are # not // --- sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index 69b6622309..7c3981c8b4 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -1,5 +1,5 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: MIT +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT param( [string] $LogFileLocation = "$($env:BUILD_SOURCESDIRECTORY)/WebSocketServer.log" ) From 48486e85b8e7287f5c8e59a20cd5d7c60362e30c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 16:48:19 -0700 Subject: [PATCH 119/149] clang-format and set socketserver log file --- .../core/http/websockets/curl_websockets_transport.hpp | 5 +++-- .../inc/azure/core/http/websockets/websockets.hpp | 6 +++--- .../inc/azure/core/http/websockets/websockets_transport.hpp | 2 +- sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp | 6 ++++-- .../azure-core/src/http/winhttp/win_http_websockets.cpp | 3 ++- sdk/core/ci.yml | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 1b74484cf2..f3a2e6c9f9 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -30,11 +30,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @param options Optional parameter to override the default options. */ - CurlWebSocketTransport(CurlWebSocketTransportOptions const& options = CurlWebSocketTransportOptions()) + CurlWebSocketTransport( + CurlWebSocketTransportOptions const& options = CurlWebSocketTransportOptions()) : CurlTransport(options) { } - + /** * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse * diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index dfb4222bfc..ab0070e2bb 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -51,8 +51,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { class WebSocketBinaryFrame; class WebSocketPeerCloseFrame; -namespace _detail { - class WebSocketImplementation; + namespace _detail { + class WebSocketImplementation; } /** @brief Statistics about data sent and received by the WebSocket. * @@ -136,6 +136,7 @@ namespace _detail { class WebSocketTextFrame : public WebSocketFrame, public std::enable_shared_from_this { friend _detail::WebSocketImplementation; + private: public: /** @brief Constructs a new WebSocketTextFrame */ @@ -204,7 +205,6 @@ namespace _detail { RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) { } - }; struct WebSocketOptions : Azure::Core::_internal::ClientOptions diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index d00986f5d4..e56b431e28 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -171,7 +171,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Assigns one WebSocketTransport to another. - * + * * @param other An instance to assign. * * @return A reference to this instance. diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index aa2b9779ca..9da99e72c4 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -454,9 +454,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names auto closeResult = m_transport->NativeGetCloseSocketInformation(context); std::vector closePayload; closePayload.push_back(closeResult.CloseReason >> 8); - closePayload.push_back(closeResult.CloseReason& 0xff); + closePayload.push_back(closeResult.CloseReason & 0xff); closePayload.insert( - closePayload.end(), closeResult.CloseReasonDescription.begin(), closeResult.CloseReasonDescription.end()); + closePayload.end(), + closeResult.CloseReasonDescription.begin(), + closeResult.CloseReasonDescription.end()); return std::make_shared(SocketOpcode::Close, true, closePayload); } default: diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index d8521bba07..2125912cc4 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -154,7 +154,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { bufferType = WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE; break; default: - throw std::runtime_error("Unknown frame type: " + std::to_string(static_cast(frameType))); + throw std::runtime_error( + "Unknown frame type: " + std::to_string(static_cast(frameType))); break; } // Lock the socket to prevent concurrent writes. WinHTTP gets annoyed if diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 818598e3a5..38705d1edc 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -59,7 +59,7 @@ stages: displayName: 'Launch python websocket server' inputs: pwsh: true - filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 + filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build/sdk/core/azure-core/test/ut condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) From 011b10d1e8de28110c69f9f5947187db460f3841 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 17:04:22 -0700 Subject: [PATCH 120/149] Moved WebSocket API to _internal namespace; implementation is still in _detail; --- .../azure/core/http/websockets/websockets.hpp | 645 +++++++++--------- .../src/http/websockets/websockets.cpp | 7 +- .../src/http/websockets/websocketsimpl.cpp | 1 + .../src/http/websockets/websocketsimpl.hpp | 8 +- .../azure-core/test/ut/websocket_test.cpp | 3 +- sdk/core/ci.yml | 3 +- 6 files changed, 337 insertions(+), 330 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index ab0070e2bb..41119d07ec 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -17,337 +17,340 @@ #include namespace Azure { namespace Core { namespace Http { namespace WebSockets { - namespace _detail { class WebSocketImplementation; } + namespace _internal { - enum class WebSocketFrameType : int - { - Unknown, - TextFrameReceived, - BinaryFrameReceived, - PeerClosedReceived, - }; - - enum class WebSocketErrorCode : uint16_t - { - OK = 1000, - EndpointDisappearing = 1001, - ProtocolError = 1002, - UnknownDataType = 1003, - Reserved1 = 1004, - NoStatusCodePresent = 1005, - ConnectionClosedWithoutCloseFrame = 1006, - InvalidMessageData = 1007, - PolicyViolation = 1008, - MessageTooLarge = 1009, - ExtensionNotFound = 1010, - UnexpectedError = 1011, - TlsHandshakeFailure = 1015, - }; - - class WebSocketTextFrame; - class WebSocketBinaryFrame; - class WebSocketPeerCloseFrame; - - namespace _detail { - class WebSocketImplementation; - } - /** @brief Statistics about data sent and received by the WebSocket. - * - * @remarks This class is primarily intended for test collateral and debugging to allow - * a caller to determine information about the status of a WebSocket. - * - * Note: Some of these statistics are not available if the underlying transport supports native - * websockets. - */ - struct WebSocketStatistics - { - /** @brief The number of WebSocket frames sent on this WebSocket. */ - uint32_t FramesSent; - /** @brief The number of bytes of data sent to the peer on this WebSocket. */ - uint32_t BytesSent; - /** @brief The number of WebSocket frames received from the peer. */ - uint32_t FramesReceived; - /** @brief The number of bytes received from the peer. */ - uint32_t BytesReceived; - /** @brief The number of "Ping" frames received from the peer. */ - uint32_t PingFramesReceived; - /** @brief The number of "Ping" frames sent to the peer. */ - uint32_t PingFramesSent; - /** @brief The number of "Pong" frames received from the peer. */ - uint32_t PongFramesReceived; - /** @brief The number of "Pong" frames sent to the peer. */ - uint32_t PongFramesSent; - /** @brief The number of "Text" frames received from the peer. */ - uint32_t TextFramesReceived; - /** @brief The number of "Text" frames sent to the peer. */ - uint32_t TextFramesSent; - /** @brief The number of "Binary" frames received from the peer. */ - uint32_t BinaryFramesReceived; - /** @brief The number of "Binary" frames sent to the peer. */ - uint32_t BinaryFramesSent; - /** @brief The number of "Continuation" frames sent to the peer. */ - uint32_t ContinuationFramesSent; - /** @brief The number of "Continuation" frames received from the peer. */ - uint32_t ContinuationFramesReceived; - /** @brief The number of "Close" frames received from the peer. */ - uint32_t CloseFramesReceived; - /** @brief The number of frames received which were not processed. */ - uint32_t FramesDropped; - /** @brief The number of frames received which were not returned because they were received - * after the Close() method was called. */ - uint32_t FramesDroppedByClose; - /** @brief The number of frames dropped because they were over the maximum payload size. */ - uint32_t FramesDroppedByPayloadSizeLimit; - /** @brief The number of frames dropped because they were out of compliance with the protocol. - */ - uint32_t FramesDroppedByProtocolError; - /** @brief The number of reads performed on the transport.*/ - uint32_t TransportReads; - /** @brief The number of bytes read from the transport. */ - uint32_t TransportReadBytes; - }; - - /** @brief A frame of data received from a WebSocket. - */ - struct WebSocketFrame - { - /** @brief The type of frame received: Text, Binary or Close. */ - WebSocketFrameType FrameType; - /** @brief True if the frame received is a "final" frame */ - bool IsFinalFrame{false}; - /** @brief Returns the contents of the frame as a Text frame. - * @returns A WebSocketTextFrame containing the contents of the frame. - */ - std::shared_ptr AsTextFrame(); - /** @brief Returns the contents of the frame as a Binary frame. - * @returns A WebSocketBinaryFrame containing the contents of the frame. - */ - std::shared_ptr AsBinaryFrame(); - /** @brief Returns the contents of the frame as a Peer Close frame. - * @returns A WebSocketPeerCloseFrame containing the contents of the frame. - */ - std::shared_ptr AsPeerCloseFrame(); - }; - - /** @brief Contains the contents of a WebSocket Text frame.*/ - class WebSocketTextFrame : public WebSocketFrame, - public std::enable_shared_from_this { - friend _detail::WebSocketImplementation; - - private: - public: - /** @brief Constructs a new WebSocketTextFrame */ - WebSocketTextFrame() = default; - /** @brief Text of the frame received from the remote peer. */ - std::string Text; - - private: - /** @brief Constructs a new WebSocketTextFrame - * @param isFinalFrame True if this is the final frame in a multi-frame message. - * @param body UTF-8 encoded text of the frame data. - * @param size Length in bytes of the frame body. - */ - WebSocketTextFrame(bool isFinalFrame, uint8_t const* body, size_t size) - : WebSocketFrame{WebSocketFrameType::TextFrameReceived, isFinalFrame}, - Text(body, body + size) - { - } - }; - - /** @brief Contains the contents of a WebSocket Binary frame.*/ - class WebSocketBinaryFrame : public WebSocketFrame, - public std::enable_shared_from_this { - friend _detail::WebSocketImplementation; - - private: - public: - /** @brief Constructs a new WebSocketBinaryFrame */ - WebSocketBinaryFrame() = default; - /** @brief Binary frame data received from the remote peer. */ - std::vector Data; - - /** @brief Constructs a new WebSocketBinaryFrame - * @param isFinal True if this is the final frame in a multi-frame message. - * @param body binary of the frame data. - * @param size Length in bytes of the frame body. - */ - private: - WebSocketBinaryFrame(bool isFinal, uint8_t const* body, size_t size) - : WebSocketFrame{WebSocketFrameType::BinaryFrameReceived, isFinal}, Data(body, body + size) + enum class WebSocketFrameType : int { - } - }; - - /** @brief Contains the contents of a WebSocket Close frame.*/ - class WebSocketPeerCloseFrame : public WebSocketFrame, - public std::enable_shared_from_this { - friend _detail::WebSocketImplementation; - - public: - /** @brief Constructs a new WebSocketPeerCloseFrame */ - WebSocketPeerCloseFrame() = default; - /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode - * enumeration */ - uint16_t RemoteStatusCode; - /** @brief Optional text sent from the remote peer. */ - std::string RemoteCloseReason; - - private: - /** @brief Constructs a new WebSocketBinaryFrame - * @param remoteStatusCode Status code sent by the remote peer. - * @param remoteCloseReason Optional reason sent by the remote peer. - */ - WebSocketPeerCloseFrame(uint16_t remoteStatusCode, std::string const& remoteCloseReason) - : WebSocketFrame{WebSocketFrameType::PeerClosedReceived}, - RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) - { - } - }; + Unknown, + TextFrameReceived, + BinaryFrameReceived, + PeerClosedReceived, + }; - struct WebSocketOptions : Azure::Core::_internal::ClientOptions - { - /** - * @brief The set of protocols which are supported by this client - */ - std::vector Protocols = {}; - - /** - * @brief The protocol name of the service client. Used for the User-Agent header - * in the initial WebSocket handshake. - */ - std::string ServiceName; - /** - * @brief The version of the service client. Used for the User-Agent header in the - * initial WebSocket handshake - */ - std::string ServiceVersion; - - /** - * @brief The period of time between ping operations, default is 60 seconds. - */ - std::chrono::duration PingInterval{std::chrono::seconds{60}}; - - /** - * @brief Construct an instance of a WebSocketOptions type. - * - * @param protocols Supported protocols for this websocket client. - */ - explicit WebSocketOptions(std::vector protocols) - : Azure::Core::_internal::ClientOptions{}, Protocols(protocols) + enum class WebSocketErrorCode : uint16_t { + OK = 1000, + EndpointDisappearing = 1001, + ProtocolError = 1002, + UnknownDataType = 1003, + Reserved1 = 1004, + NoStatusCodePresent = 1005, + ConnectionClosedWithoutCloseFrame = 1006, + InvalidMessageData = 1007, + PolicyViolation = 1008, + MessageTooLarge = 1009, + ExtensionNotFound = 1010, + UnexpectedError = 1011, + TlsHandshakeFailure = 1015, + }; + + class WebSocketTextFrame; + class WebSocketBinaryFrame; + class WebSocketPeerCloseFrame; + + namespace _detail { + class WebSocketImplementation; } - WebSocketOptions() = default; - }; - - class WebSocket { - public: - /** @brief Constructs a new instance of a WebSocket with the specified WebSocket options. - * - * @param remoteUrl The URL of the remote WebSocket server. - * @param options The options to use for the WebSocket. - */ - explicit WebSocket( - Azure::Core::Url const& remoteUrl, - WebSocketOptions const& options = WebSocketOptions{}); - - /** @brief Destroys an instance of a WebSocket. - */ - ~WebSocket(); - - /** @brief Opens a WebSocket connection to a remote server. + /** @brief Statistics about data sent and received by the WebSocket. * - * @param context Context for the operation, used for cancellation and timeout. - */ - void Open(Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief Closes a WebSocket connection to the remote server gracefully. + * @remarks This class is primarily intended for test collateral and debugging to allow + * a caller to determine information about the status of a WebSocket. * - * @param context Context for the operation. + * Note: Some of these statistics are not available if the underlying transport supports native + * websockets. */ - void Close(Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief Closes a WebSocket connection to the remote server with additional context. - * - * @param closeStatus 16 bit WebSocket error code. - * @param closeReason String describing the reason for closing the socket. - * @param context Context for the operation. - */ - void Close( - uint16_t closeStatus, - std::string const& closeReason = {}, - Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief Sends a String frame to the remote server. - * - * @param textFrame UTF-8 encoded text to send. - * @param isFinalFrame if True, this is the final frame in a multi-frame message. - * @param context Context for the operation. - */ - void SendFrame( - std::string const& textFrame, - bool isFinalFrame = false, - Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief Sends a Binary frame to the remote server. - * - * @param binaryFrame Binary data to send. - * @param isFinalFrame if True, this is the final frame in a multi-frame message. - * @param context Context for the operation. - */ - void SendFrame( - std::vector const& binaryFrame, - bool isFinalFrame = false, - Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief Receive a frame from the remote server. - * - * @param context Context for the operation. - * - * @returns The received WebSocket frame. - * - */ - std::shared_ptr ReceiveFrame( - Azure::Core::Context const& context = Azure::Core::Context{}); - - /** @brief AddHeader - Adds a header to the initial handshake. - * - * @note This API is ignored after the WebSocket is opened. - * - * @param headerName Name of header to add to the initial handshake request. - * @param headerValue Value of header to add. - */ - void AddHeader(std::string const& headerName, std::string const& headerValue); - - /** @brief Determine if the WebSocket is open. - * - * @returns true if the WebSocket is open, false otherwise. - */ - bool IsOpen() const; - - /** @brief Returns "true" if the configured websocket transport - * supports websockets in the transport, or if the websocket implementation - * is providing websocket protocol support. - * - * @returns true if the websocket transport supports websockets natively. - */ - bool HasNativeWebSocketSupport() const; - - /** @brief Returns the protocol chosen by the remote server during the initial handshake - * - * @returns The protocol negotiated between client and server. - */ - std::string const& GetChosenProtocol() const; - - /** @brief Returns statistics about the WebSocket. - * - * @returns The statistics about the WebSocket. + struct WebSocketStatistics + { + /** @brief The number of WebSocket frames sent on this WebSocket. */ + uint32_t FramesSent; + /** @brief The number of bytes of data sent to the peer on this WebSocket. */ + uint32_t BytesSent; + /** @brief The number of WebSocket frames received from the peer. */ + uint32_t FramesReceived; + /** @brief The number of bytes received from the peer. */ + uint32_t BytesReceived; + /** @brief The number of "Ping" frames received from the peer. */ + uint32_t PingFramesReceived; + /** @brief The number of "Ping" frames sent to the peer. */ + uint32_t PingFramesSent; + /** @brief The number of "Pong" frames received from the peer. */ + uint32_t PongFramesReceived; + /** @brief The number of "Pong" frames sent to the peer. */ + uint32_t PongFramesSent; + /** @brief The number of "Text" frames received from the peer. */ + uint32_t TextFramesReceived; + /** @brief The number of "Text" frames sent to the peer. */ + uint32_t TextFramesSent; + /** @brief The number of "Binary" frames received from the peer. */ + uint32_t BinaryFramesReceived; + /** @brief The number of "Binary" frames sent to the peer. */ + uint32_t BinaryFramesSent; + /** @brief The number of "Continuation" frames sent to the peer. */ + uint32_t ContinuationFramesSent; + /** @brief The number of "Continuation" frames received from the peer. */ + uint32_t ContinuationFramesReceived; + /** @brief The number of "Close" frames received from the peer. */ + uint32_t CloseFramesReceived; + /** @brief The number of frames received which were not processed. */ + uint32_t FramesDropped; + /** @brief The number of frames received which were not returned because they were received + * after the Close() method was called. */ + uint32_t FramesDroppedByClose; + /** @brief The number of frames dropped because they were over the maximum payload size. */ + uint32_t FramesDroppedByPayloadSizeLimit; + /** @brief The number of frames dropped because they were out of compliance with the protocol. + */ + uint32_t FramesDroppedByProtocolError; + /** @brief The number of reads performed on the transport.*/ + uint32_t TransportReads; + /** @brief The number of bytes read from the transport. */ + uint32_t TransportReadBytes; + }; + + /** @brief A frame of data received from a WebSocket. */ - WebSocketStatistics GetStatistics() const; - - private: - std::unique_ptr<_detail::WebSocketImplementation> m_socketImplementation; - }; + struct WebSocketFrame + { + /** @brief The type of frame received: Text, Binary or Close. */ + WebSocketFrameType FrameType; + /** @brief True if the frame received is a "final" frame */ + bool IsFinalFrame{false}; + /** @brief Returns the contents of the frame as a Text frame. + * @returns A WebSocketTextFrame containing the contents of the frame. + */ + std::shared_ptr AsTextFrame(); + /** @brief Returns the contents of the frame as a Binary frame. + * @returns A WebSocketBinaryFrame containing the contents of the frame. + */ + std::shared_ptr AsBinaryFrame(); + /** @brief Returns the contents of the frame as a Peer Close frame. + * @returns A WebSocketPeerCloseFrame containing the contents of the frame. + */ + std::shared_ptr AsPeerCloseFrame(); + }; + + /** @brief Contains the contents of a WebSocket Text frame.*/ + class WebSocketTextFrame : public WebSocketFrame, + public std::enable_shared_from_this { + friend Azure::Core::Http::WebSockets::_detail::WebSocketImplementation; + + private: + public: + /** @brief Constructs a new WebSocketTextFrame */ + WebSocketTextFrame() = default; + /** @brief Text of the frame received from the remote peer. */ + std::string Text; + + private: + /** @brief Constructs a new WebSocketTextFrame + * @param isFinalFrame True if this is the final frame in a multi-frame message. + * @param body UTF-8 encoded text of the frame data. + * @param size Length in bytes of the frame body. + */ + WebSocketTextFrame(bool isFinalFrame, uint8_t const* body, size_t size) + : WebSocketFrame{WebSocketFrameType::TextFrameReceived, isFinalFrame}, + Text(body, body + size) + { + } + }; + + /** @brief Contains the contents of a WebSocket Binary frame.*/ + class WebSocketBinaryFrame : public WebSocketFrame, + public std::enable_shared_from_this { + friend Azure::Core::Http::WebSockets::_detail::WebSocketImplementation; + + private: + public: + /** @brief Constructs a new WebSocketBinaryFrame */ + WebSocketBinaryFrame() = default; + /** @brief Binary frame data received from the remote peer. */ + std::vector Data; + + /** @brief Constructs a new WebSocketBinaryFrame + * @param isFinal True if this is the final frame in a multi-frame message. + * @param body binary of the frame data. + * @param size Length in bytes of the frame body. + */ + private: + WebSocketBinaryFrame(bool isFinal, uint8_t const* body, size_t size) + : WebSocketFrame{WebSocketFrameType::BinaryFrameReceived, isFinal}, + Data(body, body + size) + { + } + }; + + /** @brief Contains the contents of a WebSocket Close frame.*/ + class WebSocketPeerCloseFrame : public WebSocketFrame, + public std::enable_shared_from_this { + friend Azure::Core::Http::WebSockets::_detail::WebSocketImplementation; + + public: + /** @brief Constructs a new WebSocketPeerCloseFrame */ + WebSocketPeerCloseFrame() = default; + /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode + * enumeration */ + uint16_t RemoteStatusCode; + /** @brief Optional text sent from the remote peer. */ + std::string RemoteCloseReason; + + private: + /** @brief Constructs a new WebSocketBinaryFrame + * @param remoteStatusCode Status code sent by the remote peer. + * @param remoteCloseReason Optional reason sent by the remote peer. + */ + WebSocketPeerCloseFrame(uint16_t remoteStatusCode, std::string const& remoteCloseReason) + : WebSocketFrame{WebSocketFrameType::PeerClosedReceived}, + RemoteStatusCode(remoteStatusCode), RemoteCloseReason(remoteCloseReason) + { + } + }; + + struct WebSocketOptions : Azure::Core::_internal::ClientOptions + { + /** + * @brief The set of protocols which are supported by this client + */ + std::vector Protocols = {}; + + /** + * @brief The protocol name of the service client. Used for the User-Agent header + * in the initial WebSocket handshake. + */ + std::string ServiceName; + /** + * @brief The version of the service client. Used for the User-Agent header in the + * initial WebSocket handshake + */ + std::string ServiceVersion; + + /** + * @brief The period of time between ping operations, default is 60 seconds. + */ + std::chrono::duration PingInterval{std::chrono::seconds{60}}; + + /** + * @brief Construct an instance of a WebSocketOptions type. + * + * @param protocols Supported protocols for this websocket client. + */ + explicit WebSocketOptions(std::vector protocols) + : Azure::Core::_internal::ClientOptions{}, Protocols(protocols) + { + } + WebSocketOptions() = default; + }; + + class WebSocket { + public: + /** @brief Constructs a new instance of a WebSocket with the specified WebSocket options. + * + * @param remoteUrl The URL of the remote WebSocket server. + * @param options The options to use for the WebSocket. + */ + explicit WebSocket( + Azure::Core::Url const& remoteUrl, + WebSocketOptions const& options = WebSocketOptions{}); + + /** @brief Destroys an instance of a WebSocket. + */ + ~WebSocket(); + + /** @brief Opens a WebSocket connection to a remote server. + * + * @param context Context for the operation, used for cancellation and timeout. + */ + void Open(Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Closes a WebSocket connection to the remote server gracefully. + * + * @param context Context for the operation. + */ + void Close(Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Closes a WebSocket connection to the remote server with additional context. + * + * @param closeStatus 16 bit WebSocket error code. + * @param closeReason String describing the reason for closing the socket. + * @param context Context for the operation. + */ + void Close( + uint16_t closeStatus, + std::string const& closeReason = {}, + Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Sends a String frame to the remote server. + * + * @param textFrame UTF-8 encoded text to send. + * @param isFinalFrame if True, this is the final frame in a multi-frame message. + * @param context Context for the operation. + */ + void SendFrame( + std::string const& textFrame, + bool isFinalFrame = false, + Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Sends a Binary frame to the remote server. + * + * @param binaryFrame Binary data to send. + * @param isFinalFrame if True, this is the final frame in a multi-frame message. + * @param context Context for the operation. + */ + void SendFrame( + std::vector const& binaryFrame, + bool isFinalFrame = false, + Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief Receive a frame from the remote server. + * + * @param context Context for the operation. + * + * @returns The received WebSocket frame. + * + */ + std::shared_ptr ReceiveFrame( + Azure::Core::Context const& context = Azure::Core::Context{}); + + /** @brief AddHeader - Adds a header to the initial handshake. + * + * @note This API is ignored after the WebSocket is opened. + * + * @param headerName Name of header to add to the initial handshake request. + * @param headerValue Value of header to add. + */ + void AddHeader(std::string const& headerName, std::string const& headerValue); + + /** @brief Determine if the WebSocket is open. + * + * @returns true if the WebSocket is open, false otherwise. + */ + bool IsOpen() const; + + /** @brief Returns "true" if the configured websocket transport + * supports websockets in the transport, or if the websocket implementation + * is providing websocket protocol support. + * + * @returns true if the websocket transport supports websockets natively. + */ + bool HasNativeWebSocketSupport() const; + + /** @brief Returns the protocol chosen by the remote server during the initial handshake + * + * @returns The protocol negotiated between client and server. + */ + std::string const& GetChosenProtocol() const; + + /** @brief Returns statistics about the WebSocket. + * + * @returns The statistics about the WebSocket. + */ + WebSocketStatistics GetStatistics() const; + + private: + std::unique_ptr + m_socketImplementation; + }; + } // namespace _internal }}}} // namespace Azure::Core::Http::WebSockets diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index ad785993dc..b18e3ca5c8 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -5,10 +5,11 @@ #include "azure/core/context.hpp" #include "websocketsimpl.hpp" -namespace Azure { namespace Core { namespace Http { namespace WebSockets { +namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _internal { + WebSocket::WebSocket(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options) : m_socketImplementation( - std::make_unique<_detail::WebSocketImplementation>(remoteUrl, options)) + std::make_unique(remoteUrl, options)) { } @@ -100,4 +101,4 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { return static_cast(this)->shared_from_this(); } -}}}} // namespace Azure::Core::Http::WebSockets +}}}}} // namespace Azure::Core::Http::WebSockets::_internal diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 9da99e72c4..9b30b52543 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -24,6 +24,7 @@ #include namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _detail { + using namespace Azure::Core::Http::WebSockets::_internal; using namespace Azure::Core::Diagnostics::_internal; using namespace Azure::Core::Diagnostics; using namespace std::chrono_literals; diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index e6efd84fb7..9cc042efb4 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -27,7 +27,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names }; public: - WebSocketImplementation(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options); + WebSocketImplementation(Azure::Core::Url const& remoteUrl, _internal::WebSocketOptions const& options); void Open(Azure::Core::Context const& context); void Close( @@ -43,7 +43,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool isFinalFrame, Azure::Core::Context const& context); - std::shared_ptr ReceiveFrame(Azure::Core::Context const& context); + std::shared_ptr<_internal::WebSocketFrame> ReceiveFrame(Azure::Core::Context const& context); void AddHeader(std::string const& headerName, std::string const& headerValue); @@ -51,7 +51,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names bool IsOpen() { return m_state == SocketState::Open; } bool HasNativeWebSocketSupport(); - WebSocketStatistics GetStatistics() const; + _internal::WebSocketStatistics GetStatistics() const; private: // WebSocket opcodes. @@ -353,7 +353,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Azure::Core::Context const& context); Azure::Core::Url m_remoteUrl; - WebSocketOptions m_options; + _internal::WebSocketOptions m_options; std::map m_headers; std::string m_chosenProtocol; std::shared_ptr m_transport; diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 47ef2e6065..270671bd53 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -16,6 +16,7 @@ using namespace Azure::Core; using namespace Azure::Core::Http::WebSockets; +using namespace Azure::Core::Http::WebSockets::_internal; using namespace std::chrono_literals; constexpr uint16_t UndefinedButLegalCloseReason = 4500; @@ -142,7 +143,7 @@ TEST_F(WebSocketTests, SimpleEcho) testSocket.SendFrame(binaryData, true); std::vector responseData; - std::shared_ptr response; + std::shared_ptr response; do { response = testSocket.ReceiveFrame(); diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 38705d1edc..7ac0be30b5 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -59,7 +59,8 @@ stages: displayName: 'Launch python websocket server' inputs: pwsh: true - filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 $(Build.SourcesDirectory)/WebSocketServer.log + filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 + arguments: $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build/sdk/core/azure-core/test/ut condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) From 3925f04ced7c247f813fbbfcec544d1fcf75f5f7 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 17:19:45 -0700 Subject: [PATCH 121/149] clang-format --- sdk/core/azure-core/src/http/websockets/websockets.cpp | 4 +++- sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index b18e3ca5c8..a39c4f873e 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -9,7 +9,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names WebSocket::WebSocket(Azure::Core::Url const& remoteUrl, WebSocketOptions const& options) : m_socketImplementation( - std::make_unique(remoteUrl, options)) + std::make_unique( + remoteUrl, + options)) { } diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index 9cc042efb4..a488df06a5 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -27,7 +27,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names }; public: - WebSocketImplementation(Azure::Core::Url const& remoteUrl, _internal::WebSocketOptions const& options); + WebSocketImplementation( + Azure::Core::Url const& remoteUrl, + _internal::WebSocketOptions const& options); void Open(Azure::Core::Context const& context); void Close( From ec10454ed1e57371f0269e9bd97f6184acb57330 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 17:28:32 -0700 Subject: [PATCH 122/149] Force python3 usage for linux. --- sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index 7c3981c8b4..a761c2a038 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -9,5 +9,5 @@ if ($IsWindows) { -ArgumentList 'websocket_server.py' ` -NoNewWindow -PassThru -RedirectStandardOutput $LogFileLocation } else { - nohup python websocket_server.py > $LogFileLocation + nohup python3 websocket_server.py > $LogFileLocation } From e491cadc27234c665606cf2d3ca6816c4fbc3122 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 17:38:00 -0700 Subject: [PATCH 123/149] Different attempt at launching nohup --- sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index a761c2a038..5db3e506ff 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -9,5 +9,5 @@ if ($IsWindows) { -ArgumentList 'websocket_server.py' ` -NoNewWindow -PassThru -RedirectStandardOutput $LogFileLocation } else { - nohup python3 websocket_server.py > $LogFileLocation + Start-Process nohup -ArgumentList 'python3 websocket_server.py' -RedirectStandardOutput $LogFileLocation } From d40286cac61d3a976845e4f355acc95139b02b11 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Thu, 21 Jul 2022 18:08:23 -0700 Subject: [PATCH 124/149] Another attempt at launching nohup --- sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index 5db3e506ff..a60964fc01 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -9,5 +9,5 @@ if ($IsWindows) { -ArgumentList 'websocket_server.py' ` -NoNewWindow -PassThru -RedirectStandardOutput $LogFileLocation } else { - Start-Process nohup -ArgumentList 'python3 websocket_server.py' -RedirectStandardOutput $LogFileLocation + Start-Process nohup 'python3 websocket_server.py > $LogFileLocation' } From 2e8346b0b5d5c94d9d1e104f4f0206eabff8a545 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 22 Jul 2022 08:55:33 -0700 Subject: [PATCH 125/149] Still another attempt at launching nohup --- sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 index a60964fc01..7e08280988 100644 --- a/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 +++ b/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 @@ -9,5 +9,5 @@ if ($IsWindows) { -ArgumentList 'websocket_server.py' ` -NoNewWindow -PassThru -RedirectStandardOutput $LogFileLocation } else { - Start-Process nohup 'python3 websocket_server.py > $LogFileLocation' + Start-Process nohup 'python3 websocket_server.py' -RedirectStandardOutput $LogFileLocation } From 013d000b355375972762c3b6dfebb97f621159f9 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 22 Jul 2022 09:12:40 -0700 Subject: [PATCH 126/149] Put back original linux python launch action to get CI pipeline working again --- sdk/core/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 7ac0be30b5..34be7a63ef 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -62,8 +62,14 @@ stages: filePath: build/sdk/core/azure-core/test/ut/Start-WebSocketServer.ps1 arguments: $(Build.SourcesDirectory)/WebSocketServer.log workingDirectory: build/sdk/core/azure-core/test/ut - condition: and(succeeded(), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) - + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + # It would be nice to collapse this branch with the previous one, but nohup doesn't seem to + # behave when called from powershell. + - bash: | + nohup python sdk/core/azure-core/test/ut/websocket_server.py > $(Build.SourcesDirectory)/WebSocketServer.log & + workingDirectory: build + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'), contains(variables.CmakeArgs, 'BUILD_TESTING=ON')) + displayName: Launch python websocket server (Linux). PostTestSteps: # Shut down the test server. This uses curl to send a request to the "terminateserver" websocket endpoint. # When the test server receives a request on terminateserver, it shuts down gracefully. From a5ef68e018c4f3330379fe44073118695d4425a1 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 22 Jul 2022 11:31:20 -0700 Subject: [PATCH 127/149] Added test case to close socket while a receive is outstanding --- .../azure/core/http/websockets/websockets.hpp | 24 +- .../src/http/websockets/websocketsimpl.cpp | 255 ++++++++++-------- .../src/http/websockets/websocketsimpl.hpp | 8 +- .../azure-core/test/ut/websocket_server.py | 20 +- .../azure-core/test/ut/websocket_test.cpp | 82 ++++-- 5 files changed, 243 insertions(+), 146 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 41119d07ec..6c6c6fea51 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -112,10 +112,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** @brief A frame of data received from a WebSocket. */ - struct WebSocketFrame - { + class WebSocketFrame { + public: /** @brief The type of frame received: Text, Binary or Close. */ - WebSocketFrameType FrameType; + WebSocketFrameType FrameType{}; /** @brief True if the frame received is a "final" frame */ bool IsFinalFrame{false}; /** @brief Returns the contents of the frame as a Text frame. @@ -130,6 +130,18 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @returns A WebSocketPeerCloseFrame containing the contents of the frame. */ std::shared_ptr AsPeerCloseFrame(); + + /** @brief Construct a new instance of a WebSocketFrame.*/ + WebSocketFrame() = default; + + /** @brief Construct a new instance of a WebSocketFrame with a specific frame type. + * @param frameType The type of frame received. + */ + WebSocketFrame(WebSocketFrameType frameType) : FrameType{frameType} {} + WebSocketFrame(WebSocketFrameType frameType, bool isFinalFrame) + : FrameType{frameType}, IsFinalFrame{isFinalFrame} + { + } }; /** @brief Contains the contents of a WebSocket Text frame.*/ @@ -140,7 +152,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { private: public: /** @brief Constructs a new WebSocketTextFrame */ - WebSocketTextFrame() = default; + WebSocketTextFrame() : WebSocketFrame(WebSocketFrameType::TextFrameReceived){}; /** @brief Text of the frame received from the remote peer. */ std::string Text; @@ -165,7 +177,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { private: public: /** @brief Constructs a new WebSocketBinaryFrame */ - WebSocketBinaryFrame() = default; + WebSocketBinaryFrame() : WebSocketFrame(WebSocketFrameType::BinaryFrameReceived){}; /** @brief Binary frame data received from the remote peer. */ std::vector Data; @@ -189,7 +201,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** @brief Constructs a new WebSocketPeerCloseFrame */ - WebSocketPeerCloseFrame() = default; + WebSocketPeerCloseFrame() : WebSocketFrame(WebSocketFrameType::PeerClosedReceived){}; /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode * enumeration */ uint16_t RemoteStatusCode; diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp index 9b30b52543..72c5e673a7 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp @@ -242,15 +242,15 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Note that there might be in-flight frames that were sent from the other end of the // WebSocket that we don't care about any more (since we're closing the WebSocket). So // drain those frames. - auto closeResponse = ReceiveFrame(context); - while (closeResponse->FrameType != WebSocketFrameType::PeerClosedReceived) + auto closeResponse = ReceiveTransportFrame(context); + while (closeResponse && closeResponse->Opcode != SocketOpcode::Close) { m_receiveStatistics.FramesDroppedByClose++; Log::Write( Logger::Level::Warning, "Received unexpected frame during close. Opcode: " - + std::to_string(static_cast(closeResponse->FrameType))); - closeResponse = ReceiveFrame(closeContext); + + std::to_string(static_cast(closeResponse->Opcode))); + closeResponse = ReceiveTransportFrame(closeContext); } // Re-acquire the state lock once we've received the close lock. @@ -347,79 +347,92 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names while (true) { frame = ReceiveTransportFrame(context); - switch (frame->Opcode) + if (frame) { - // When we receive a "ping" frame, we want to send a Pong frame back to the server. - case SocketOpcode::Ping: - Log::Write( - Logger::Level::Verbose, "Received Ping frame: " + HexEncode(frame->Payload, 16)); - SendPong(frame->Payload, context); - break; - // We want to ignore all incoming "Pong" frames. - case SocketOpcode::Pong: - Log::Write( - Logger::Level::Verbose, "Received Pong frame: " + HexEncode(frame->Payload, 16)); - break; - - case SocketOpcode::BinaryFrame: - m_currentMessageType = SocketMessageType::Binary; - return std::shared_ptr(new WebSocketBinaryFrame( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); - - case SocketOpcode::TextFrame: - m_currentMessageType = SocketMessageType::Text; - return std::shared_ptr(new WebSocketTextFrame( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); - - case SocketOpcode::Close: { - if (frame->Payload.size() < 2) - { - throw std::runtime_error("Close response buffer is too short."); - } - uint16_t errorCode = 0; - errorCode |= (frame->Payload[0] << 8) & 0xff00; - errorCode |= (frame->Payload[1] & 0x00ff); - - // We received a close frame, mark the socket as closed. Make sure we - // reacquire the state lock before setting the state to closed. - lock.lock(); - m_state = SocketState::Closed; + switch (frame->Opcode) + { + // When we receive a "ping" frame, we want to send a Pong frame back to the server. + case SocketOpcode::Ping: + Log::Write( + Logger::Level::Verbose, "Received Ping frame: " + HexEncode(frame->Payload, 16)); + SendPong(frame->Payload, context); + break; + // We want to ignore all incoming "Pong" frames. + case SocketOpcode::Pong: + Log::Write( + Logger::Level::Verbose, "Received Pong frame: " + HexEncode(frame->Payload, 16)); + break; + + case SocketOpcode::BinaryFrame: + m_currentMessageType = SocketMessageType::Binary; + return std::shared_ptr(new WebSocketBinaryFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); - return std::shared_ptr(new WebSocketPeerCloseFrame( - errorCode, std::string(frame->Payload.begin() + 2, frame->Payload.end()))); - } + case SocketOpcode::TextFrame: + m_currentMessageType = SocketMessageType::Text; + return std::shared_ptr(new WebSocketTextFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); - // Continuation frames need to be treated somewhat specially. - // We depend on the fact that the protocol requires that a Continuation frame - // only be sent if it is part of a multi-frame message whose previous frame was a Text or - // Binary frame. - case SocketOpcode::Continuation: - if (m_currentMessageType == SocketMessageType::Text) - { - if (frame->IsFinalFrame) + case SocketOpcode::Close: { + if (frame->Payload.size() < 2) { - m_currentMessageType = SocketMessageType::Unknown; + throw std::runtime_error("Close response buffer is too short."); } - return std::shared_ptr(new WebSocketTextFrame( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); + uint16_t errorCode = 0; + errorCode |= (frame->Payload[0] << 8) & 0xff00; + errorCode |= (frame->Payload[1] & 0x00ff); + + // We received a close frame, mark the socket as closed. Make sure we + // reacquire the state lock before setting the state to closed. + lock.lock(); + m_state = SocketState::Closed; + + return std::shared_ptr(new WebSocketPeerCloseFrame( + errorCode, std::string(frame->Payload.begin() + 2, frame->Payload.end()))); } - else if (m_currentMessageType == SocketMessageType::Binary) - { - if (frame->IsFinalFrame) + + // Continuation frames need to be treated somewhat specially. + // We depend on the fact that the protocol requires that a Continuation frame + // only be sent if it is part of a multi-frame message whose previous frame was a Text + // or Binary frame. + case SocketOpcode::Continuation: + if (m_currentMessageType == SocketMessageType::Text) { - m_currentMessageType = SocketMessageType::Unknown; + if (frame->IsFinalFrame) + { + m_currentMessageType = SocketMessageType::Unknown; + } + return std::shared_ptr(new WebSocketTextFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); } - return std::shared_ptr(new WebSocketBinaryFrame( - frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); - } - else - { - m_receiveStatistics.FramesDroppedByProtocolError++; - throw std::runtime_error("Unknown message type and received continuation opcode"); - } - default: - throw std::runtime_error("Unknown frame type received."); + else if (m_currentMessageType == SocketMessageType::Binary) + { + if (frame->IsFinalFrame) + { + m_currentMessageType = SocketMessageType::Unknown; + } + return std::shared_ptr(new WebSocketBinaryFrame( + frame->IsFinalFrame, frame->Payload.data(), frame->Payload.size())); + } + else + { + m_receiveStatistics.FramesDroppedByProtocolError++; + throw std::runtime_error("Unknown message type and received continuation opcode"); + } + default: + throw std::runtime_error("Unknown frame type received."); + } + } + else + { + if (m_state != SocketState::Closed && m_state != SocketState::Closing) + { + throw std::runtime_error("Transport is at EOF, no frame to receive."); + } + // The socket was closed, most likely locally, so fake a close frame response. + return std::shared_ptr(new WebSocketPeerCloseFrame()); } + context.ThrowIfCancelled(); } } @@ -469,46 +482,48 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names else #endif { - SocketOpcode opcode; - - bool isFinal = false; - std::vector frameData = DecodeFrame(opcode, isFinal, context); - // At this point, frameData contains the actual payload from the service. - auto frame = std::make_shared(opcode, isFinal, frameData); - - // Handle statistics for the incoming frame. - m_receiveStatistics.FramesReceived++; - switch (frame->Opcode) + std::shared_ptr frame = DecodeFrame(context); + if (frame) { - case SocketOpcode::Ping: { - m_receiveStatistics.PingFramesReceived++; - break; - } - case SocketOpcode::Pong: { - m_receiveStatistics.PongFramesReceived++; - break; - } - case SocketOpcode::TextFrame: { - m_receiveStatistics.TextFramesReceived++; - break; - } - case SocketOpcode::BinaryFrame: { - m_receiveStatistics.BinaryFramesReceived++; - break; - } - case SocketOpcode::Close: { - m_receiveStatistics.CloseFramesReceived++; - break; - } - case SocketOpcode::Continuation: { - m_receiveStatistics.ContinuationFramesReceived++; - break; - } - default: { - m_receiveStatistics.UnknownFramesReceived++; - break; + + // Handle statistics for the incoming frame. + m_receiveStatistics.FramesReceived++; + switch (frame->Opcode) + { + case SocketOpcode::Ping: { + m_receiveStatistics.PingFramesReceived++; + break; + } + case SocketOpcode::Pong: { + m_receiveStatistics.PongFramesReceived++; + break; + } + case SocketOpcode::TextFrame: { + m_receiveStatistics.TextFramesReceived++; + break; + } + case SocketOpcode::BinaryFrame: { + m_receiveStatistics.BinaryFramesReceived++; + break; + } + case SocketOpcode::Close: { + m_receiveStatistics.CloseFramesReceived++; + break; + } + case SocketOpcode::Continuation: { + m_receiveStatistics.ContinuationFramesReceived++; + break; + } + default: { + m_receiveStatistics.UnknownFramesReceived++; + break; + } } } + else + { + m_receiveStatistics.FramesDropped++; + } return frame; } } @@ -609,10 +624,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return encodedFrame; } - std::vector WebSocketImplementation::DecodeFrame( - SocketOpcode& opcode, - bool& isFinal, - Azure::Core::Context const& context) + std::shared_ptr + WebSocketImplementation::DecodeFrame(Azure::Core::Context const& context) { // Ensure single threaded access to receive this frame. std::unique_lock lock(m_transportMutex); @@ -621,9 +634,18 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names throw std::runtime_error("Frame buffer is too small."); } uint8_t payloadByte = ReadTransportByte(context); - opcode = static_cast(payloadByte & 0x7f); - isFinal = (payloadByte & 0x80) != 0; + // If the transport is at EOF, then there is no payload data, so just return null. + if (IsTransportEof()) + { + return nullptr; + } + SocketOpcode opcode = static_cast(payloadByte & 0x7f); + bool isFinal = (payloadByte & 0x80) != 0; payloadByte = ReadTransportByte(context); + if (IsTransportEof()) + { + return nullptr; + } if (payloadByte & 0x80) { throw std::runtime_error("Server sent a frame with a reserved bit set."); @@ -645,8 +667,17 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::logic_error("Unexpected payload length."); } + if (IsTransportEof()) + { + return nullptr; + } - return ReadTransportBytes(static_cast(payloadLength), context); + std::vector payload(ReadTransportBytes(static_cast(payloadLength), context)); + if (IsTransportEof()) + { + return nullptr; + } + return std::make_shared(opcode, isFinal, payload); } uint8_t WebSocketImplementation::ReadTransportByte(Azure::Core::Context const& context) diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp index a488df06a5..6a5fcf7cfe 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp @@ -344,15 +344,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names /** * @brief Decode a frame received from the websocket server. * - * @param opcode Opcode returned by the server. - * @param isFinal True if this is the final message. - * @param context Context for reads if necessary. * @returns A pointer to the start of the decoded data. */ - std::vector DecodeFrame( - SocketOpcode& opcode, - bool& isFinal, - Azure::Core::Context const& context); + std::shared_ptr DecodeFrame(Azure::Core::Context const& context); Azure::Core::Url m_remoteUrl; _internal::WebSocketOptions m_options; diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 9aa8e165c8..3cf745865c 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -1,9 +1,10 @@ from array import array import asyncio +#from curses import has_key from operator import length_hint import threading from time import sleep -from urllib.parse import urlparse +from urllib.parse import ParseResult, urlparse import websockets @@ -47,23 +48,36 @@ def HexEncode(data: bytes)->str: rv+= '{:02X}'.format(val) return rv +def ParseQuery(url : ParseResult) -> dict: + rv={} + if len(url.query)!=0: + args = url.query.split('&') + for arg in args: + vals=arg.split('=') + rv[vals[0]]=vals[1] + return rv echo_count_lock = threading.Lock() echo_count_recv = 0 echo_count_send = 0 client_count = 0 -async def handleEcho(websocket, url): +async def handleEcho(websocket, url:ParseResult): global client_count global echo_count_recv global echo_count_send global echo_count_lock + queryValues = ParseQuery(url) while websocket.open: try: data = await websocket.recv() with echo_count_lock: echo_count_recv+=1 + if 'delay' in queryValues: + print(f"sleeping for {queryValues['delay']} seconds") + await asyncio.sleep(float(queryValues['delay'])) + print("woken up.") - if (url.query == 'fragment=true'): + if 'fragment' in queryValues and queryValues['fragment']=='true': await websocket.send(data.split()) else: await websocket.send(data) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 270671bd53..ec62fcc572 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -114,7 +114,7 @@ TEST_F(WebSocketTests, SimpleEcho) testSocket.Close(); } { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?delay=20")); + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?delay=5")); testSocket.Open(); @@ -134,7 +134,7 @@ TEST_F(WebSocketTests, SimpleEcho) } { - WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?fragment=true")); + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest?fragment=true&delay=5")); testSocket.Open(); @@ -215,6 +215,22 @@ TEST_F(WebSocketTests, VariableSizeEcho) } } +// Generator for random bytes. Used in WebSocketImplementation and tests. +std::vector GenerateRandomBytes(size_t index, size_t vectorSize) +{ + std::random_device randomEngine; + + std::vector rv(vectorSize + 4); + rv[0] = index & 0xff; + rv[1] = (index >> 8) & 0xff; + rv[2] = (index >> 16) & 0xff; + rv[3] = (index >> 24) & 0xff; + std::generate(std::begin(rv) + 4, std::end(rv), [&randomEngine]() mutable { + return static_cast(randomEngine() % UINT8_MAX); + }); + return rv; +} + TEST_F(WebSocketTests, CloseDuringEcho) { { @@ -232,6 +248,52 @@ TEST_F(WebSocketTests, CloseDuringEcho) // Close the socket gracefully. testSocket.Close(); } + + // Close the websocket while a thread is waiting for a response. + { + WebSocket testSocket(Azure::Core::Url("ws://localhost:8000/echotest?delay=10")); + + testSocket.Open(); + + std::thread testThread([&]() { + try + { + std::vector sendData = GenerateRandomBytes(0, 100); + testSocket.SendFrame(sendData); + GTEST_LOG_(INFO) << "Receive frame." ; + auto response = testSocket.ReceiveFrame(); + GTEST_LOG_(INFO) << "Received frame."; + if (response->FrameType == WebSocketFrameType::PeerClosedReceived) + { + GTEST_LOG_(INFO) << "Peer closed the socket; Terminating thread."; + return; + } + else if (response->FrameType != WebSocketFrameType::BinaryFrameReceived) + { + GTEST_LOG_(INFO) << "Unexpected frame type received."; + } + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); + auto binaryResult = response->AsBinaryFrame(); + } + catch (Azure::Core::OperationCancelledException& ex) + { + GTEST_LOG_(ERROR) << "Cancelled Exception: " << ex.what() + << " Current Thread: " << std::this_thread::get_id() << std::endl; + } + catch (std::exception const& ex) + { + GTEST_LOG_(ERROR) << "Exception: " << ex.what() << std::endl; + } + }); + + std::this_thread::sleep_for(100ms); + + // Close the socket gracefully. + GTEST_LOG_(INFO) << "Closing Socket."; + EXPECT_NO_THROW(testSocket.Close(UndefinedButLegalCloseReason, "Close Reason.")); + GTEST_LOG_(INFO) << "Closed Socket."; + testThread.join(); + } } TEST_F(WebSocketTests, ExpectThrow) @@ -256,22 +318,6 @@ std::string ToHexString(std::vector const& data) return ss.str(); } -// Generator for random bytes. Used in WebSocketImplementation and tests. -std::vector GenerateRandomBytes(size_t index, size_t vectorSize) -{ - std::random_device randomEngine; - - std::vector rv(vectorSize + 4); - rv[0] = index & 0xff; - rv[1] = (index >> 8) & 0xff; - rv[2] = (index >> 16) & 0xff; - rv[3] = (index >> 24) & 0xff; - std::generate(std::begin(rv) + 4, std::end(rv), [&randomEngine]() mutable { - return static_cast(randomEngine() % UINT8_MAX); - }); - return rv; -} - TEST_F(WebSocketTests, PingReceiveTest) { WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); From 83a48857bb738306bcc5fa2873cfbdb28a80d751 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 22 Jul 2022 11:57:31 -0700 Subject: [PATCH 128/149] clang-format --- sdk/core/azure-core/test/ut/websocket_test.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index ec62fcc572..5fe3506c6e 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -260,7 +260,7 @@ TEST_F(WebSocketTests, CloseDuringEcho) { std::vector sendData = GenerateRandomBytes(0, 100); testSocket.SendFrame(sendData); - GTEST_LOG_(INFO) << "Receive frame." ; + GTEST_LOG_(INFO) << "Receive frame."; auto response = testSocket.ReceiveFrame(); GTEST_LOG_(INFO) << "Received frame."; if (response->FrameType == WebSocketFrameType::PeerClosedReceived) @@ -285,14 +285,14 @@ TEST_F(WebSocketTests, CloseDuringEcho) GTEST_LOG_(ERROR) << "Exception: " << ex.what() << std::endl; } }); - - std::this_thread::sleep_for(100ms); + + std::this_thread::sleep_for(100ms); // Close the socket gracefully. - GTEST_LOG_(INFO) << "Closing Socket."; - EXPECT_NO_THROW(testSocket.Close(UndefinedButLegalCloseReason, "Close Reason.")); - GTEST_LOG_(INFO) << "Closed Socket."; - testThread.join(); + GTEST_LOG_(INFO) << "Closing Socket."; + EXPECT_NO_THROW(testSocket.Close(UndefinedButLegalCloseReason, "Close Reason.")); + GTEST_LOG_(INFO) << "Closed Socket."; + testThread.join(); } } From 23675772b99db0bab4fcaaf2bb2dd51e1bc547d1 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Fri, 22 Jul 2022 15:56:23 -0700 Subject: [PATCH 129/149] Added multithreaded test with multiple websockets --- .../azure-core/test/ut/websocket_test.cpp | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 5fe3506c6e..d75259b1f8 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -549,6 +549,156 @@ TEST_F(WebSocketTests, MultiThreadedTestOnSingleSocket) EXPECT_EQ(0, cancellationExceptions.load()); } +TEST_F(WebSocketTests, MultiThreadedTestOnMultipleSockets) +{ + constexpr size_t threadCount = 50; + constexpr size_t testDataLength = 200000; + constexpr size_t testDataSize = 100; + constexpr auto testDuration = 10s; + + // seed test data for the operations. + std::vector> testData(testDataLength); + std::vector> receivedData(testDataLength); + std::atomic_size_t iterationCount(0); + + // Spin up threadCount threads and hammer the echo server for 10 seconds. + std::vector threads; + std::atomic_int32_t cancellationExceptions{0}; + std::atomic_int32_t exceptions{0}; + for (size_t threadIndex = 0; threadIndex < threadCount; threadIndex += 1) + { + threads.push_back(std::thread([&]() { + std::chrono::time_point startTime + = std::chrono::system_clock::now(); + // Set the context to expire *after* the test is supposed to finish. + Azure::Core::Context context = Azure::Core::Context::ApplicationContext.WithDeadline( + Azure::DateTime{startTime} + testDuration + 10s); + size_t iteration = 0; + try + { + WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); + + testSocket.Open(); + + do + { + iteration = iterationCount++; + std::vector sendData = GenerateRandomBytes(iteration, testDataSize); + { + if (iteration < testData.size()) + { + if (testData[iteration].size() != 0) + { + GTEST_LOG_(ERROR) << "Overwriting send frame at offset " << iteration << std::endl; + } + EXPECT_EQ(0, testData[iteration].size()); + testData[iteration] = sendData; + } + } + + testSocket.SendFrame(sendData, true /*, context*/); + auto response = testSocket.ReceiveFrame(context); + EXPECT_EQ(WebSocketFrameType::BinaryFrameReceived, response->FrameType); + auto binaryResult = response->AsBinaryFrame(); + + // Make sure we get back the data we sent in the echo request. + if (binaryResult->Data.size() == 0) + { + GTEST_LOG_(ERROR) << "Received empty frame at offset " << iteration << std::endl; + } + EXPECT_EQ(sendData.size(), binaryResult->Data.size()); + { + // There is no ordering expectation on the results, so we just remember the data + // as it comes in. We'll make sure we received everything later on. + if (iteration < receivedData.size()) + { + if (receivedData[iteration].size() != 0) + { + GTEST_LOG_(ERROR) << "Overwriting receive frame at offset " << iteration + << std::endl; + } + + EXPECT_EQ(0, receivedData[iteration].size()); + receivedData[iteration] = binaryResult->Data; + } + } + } while (std::chrono::system_clock::now() - startTime < testDuration); + // Close the socket gracefully. + testSocket.Close(); + } + catch (Azure::Core::OperationCancelledException& ex) + { + GTEST_LOG_(ERROR) << "Cancelled Exception: " << ex.what() << " at index " << iteration + << " Current Thread: " << std::this_thread::get_id() << std::endl; + cancellationExceptions++; + } + catch (std::exception const& ex) + { + GTEST_LOG_(ERROR) << "Exception: " << ex.what() << std::endl; + exceptions++; + } + })); + } + + // Wait for all the threads to exit. + for (auto& thread : threads) + { + thread.join(); + } + + // We no longer need to worry about synchronization since all the worker threads are done. + GTEST_LOG_(INFO) << "Total server requests: " << iterationCount.load() << std::endl; + GTEST_LOG_(INFO) << "Estimated " << std::dec << testData.size() << " iterations (0x" << std::hex + << testData.size() << ")" << std::endl; + EXPECT_GE(testDataLength, iterationCount.load()); + + // Resize the test data to the number of actual iterations. + testData.resize(iterationCount.load()); + receivedData.resize(iterationCount.load()); + + // If we've processed every iteration, let's make sure that we received everything we sent. + // If we dropped some results, then we can't check to ensure that we have received everything + // because we can't account for everything sent. + std::multiset testDataStrings; + std::multiset receivedDataStrings; + for (auto const& data : testData) + { + testDataStrings.emplace(ToHexString(data)); + } + for (auto const& data : receivedData) + { + receivedDataStrings.emplace(ToHexString(data)); + } + + EXPECT_EQ(testDataStrings, receivedDataStrings); + for (auto const& data : testDataStrings) + { + if (receivedDataStrings.count(data) != testDataStrings.count(data)) + { + GTEST_LOG_(INFO) << "Missing data. TestDataCount: " << testDataStrings.count(data) + << " ReceivedDataCount: " << receivedDataStrings.count(data) + << " Missing Data: " << data << std::endl; + } + EXPECT_NE(receivedDataStrings.end(), receivedDataStrings.find(data)); + } + for (auto const& data : receivedDataStrings) + { + if (testDataStrings.count(data) != receivedDataStrings.count(data)) + { + GTEST_LOG_(INFO) << "Extra data. TestDataCount: " << testDataStrings.count(data) + << " ReceivedDataCount: " << receivedDataStrings.count(data) + << " Missing Data: " << data << std::endl; + } + + EXPECT_NE(testDataStrings.end(), testDataStrings.find(data)); + } + + // We shouldn't have seen any exceptions during the run. + EXPECT_EQ(0, exceptions.load()); + EXPECT_EQ(0, cancellationExceptions.load()); +} + + // Does not work because curl rejects the wss: scheme. class LibWebSocketIncrementProtocol { WebSocketOptions m_options{{"dumb-increment-protocol"}}; From d9cd38c6a6f364d33e40b4ad7b534ad216a28e4d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 25 Jul 2022 12:01:54 -0700 Subject: [PATCH 130/149] clang-format; changed name of the connection upgrade message for WinHTTP websockets --- .../azure/core/http/websockets/websockets.hpp | 2 +- .../win_http_websockets_transport.hpp | 2 +- .../azure/core/http/win_http_transport.hpp | 12 ++++---- .../src/http/winhttp/win_http_transport.cpp | 30 ++++++++++--------- .../src/http/winhttp/win_http_websockets.cpp | 4 +-- .../azure-core/test/ut/websocket_test.cpp | 1 - 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 6c6c6fea51..9dc8c4a8d7 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -204,7 +204,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { WebSocketPeerCloseFrame() : WebSocketFrame(WebSocketFrameType::PeerClosedReceived){}; /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode * enumeration */ - uint16_t RemoteStatusCode; + uint16_t RemoteStatusCode{}; /** @brief Optional text sent from the remote peer. */ std::string RemoteCloseReason; diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 364cf70e6a..e254041165 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -28,7 +28,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { std::mutex m_receiveMutex; // Called by the - void OnResponseReceived(Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) override; + void OnUpgradedConnection(Azure::Core::Http::_detail::unique_HINTERNET const& requestHandle) override; public: /** diff --git a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp index 8de11c98e5..207a39efeb 100644 --- a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp @@ -125,22 +125,22 @@ namespace Azure { namespace Core { namespace Http { Azure::Core::Url const& url, Azure::Core::Context const& context); _detail::unique_HINTERNET CreateRequestHandle( - _detail::unique_HINTERNET& connectionHandle, + _detail::unique_HINTERNET const& connectionHandle, Azure::Core::Url const& url, Azure::Core::Http::HttpMethod const& method); void Upload( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Http::Request& request, Azure::Core::Context const& context); void SendRequest( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Http::Request& request, Azure::Core::Context const& context); void ReceiveResponse( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Context const& context); int64_t GetContentLength( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, HttpMethod requestMethod, HttpStatusCode responseStatusCode); std::unique_ptr SendRequestAndGetResponse( @@ -150,7 +150,7 @@ namespace Azure { namespace Core { namespace Http { // Callback to allow a derived transport to extract the request handle. Used for WebSocket // transports. protected: - virtual void OnResponseReceived(_detail::unique_HINTERNET&){}; + virtual void OnUpgradedConnection(_detail::unique_HINTERNET const&){}; /** * @brief Throw an exception based on the Win32 Error code * diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 5520b3e1de..5ab5f1b6c9 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -318,7 +318,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateConnectionHandle( } _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( - _detail::unique_HINTERNET& connectionHandle, + _detail::unique_HINTERNET const& connectionHandle, Azure::Core::Url const& url, Azure::Core::Http::HttpMethod const& method) { @@ -389,7 +389,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( // For PUT/POST requests, send additional data using WinHttpWriteData. void WinHttpTransport::Upload( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Http::Request& request, Azure::Core::Context const& context) { @@ -429,7 +429,7 @@ void WinHttpTransport::Upload( } void WinHttpTransport::SendRequest( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Http::Request& request, Azure::Core::Context const& context) { @@ -497,7 +497,7 @@ void WinHttpTransport::SendRequest( } void WinHttpTransport::ReceiveResponse( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, Azure::Core::Context const& context) { context.ThrowIfCancelled(); @@ -520,7 +520,7 @@ void WinHttpTransport::ReceiveResponse( } int64_t WinHttpTransport::GetContentLength( - _detail::unique_HINTERNET& requestHandle, + _detail::unique_HINTERNET const& requestHandle, HttpMethod requestMethod, HttpStatusCode responseStatusCode) { @@ -668,11 +668,18 @@ std::unique_ptr WinHttpTransport::SendRequestAndGetResponse( SetHeaders(responseHeaders, rawResponse); - int64_t contentLength - = GetContentLength(requestHandle, requestMethod, rawResponse->GetStatusCode()); + if (HasWebSocketSupport() && (httpStatusCode == HttpStatusCode::SwitchingProtocols)) + { + OnUpgradedConnection(requestHandle); + } + else + { + int64_t contentLength + = GetContentLength(requestHandle, requestMethod, rawResponse->GetStatusCode()); - rawResponse->SetBodyStream( - std::make_unique<_detail::WinHttpStream>(requestHandle, contentLength)); + rawResponse->SetBodyStream( + std::make_unique<_detail::WinHttpStream>(requestHandle, contentLength)); + } return rawResponse; } @@ -686,11 +693,6 @@ std::unique_ptr WinHttpTransport::Send(Request& request, Context co ReceiveResponse(requestHandle, context); - if (HasWebSocketSupport()) - { - OnResponseReceived(requestHandle); - } - return SendRequestAndGetResponse(requestHandle, request.GetMethod()); } diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 2125912cc4..e5a274faa1 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -25,8 +25,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { - void WinHttpWebSocketTransport::OnResponseReceived( - Azure::Core::Http::_detail::unique_HINTERNET& requestHandle) + void WinHttpWebSocketTransport::OnUpgradedConnection( + Azure::Core::Http::_detail::unique_HINTERNET const& requestHandle) { // Convert the request handle into a WebSocket handle for us to use later. m_socketHandle = Azure::Core::Http::_detail::unique_HINTERNET( diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index d75259b1f8..aab0264221 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -698,7 +698,6 @@ TEST_F(WebSocketTests, MultiThreadedTestOnMultipleSockets) EXPECT_EQ(0, cancellationExceptions.load()); } - // Does not work because curl rejects the wss: scheme. class LibWebSocketIncrementProtocol { WebSocketOptions m_options{{"dumb-increment-protocol"}}; From 3e353679b20cf40f2ac2af4b030a490d35848ed3 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 25 Jul 2022 12:26:30 -0700 Subject: [PATCH 131/149] clang-format --- .../core/http/websockets/win_http_websockets_transport.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index e254041165..63f6567021 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -28,7 +28,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { std::mutex m_receiveMutex; // Called by the - void OnUpgradedConnection(Azure::Core::Http::_detail::unique_HINTERNET const& requestHandle) override; + void OnUpgradedConnection( + Azure::Core::Http::_detail::unique_HINTERNET const& requestHandle) override; public: /** From 549d2bd1413b074d7fc4881f5007ab032e2b7907 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 25 Jul 2022 16:52:12 -0700 Subject: [PATCH 132/149] Update sdk/core/azure-core/src/http/curl/curl.cpp Co-authored-by: Ahson Khan --- sdk/core/azure-core/src/http/curl/curl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index f8e23cf006..bfb8693641 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -747,7 +747,7 @@ void CurlSession::ReadStatusLineAndHeadersFromRawResponse( m_connection->Shutdown(); } // If the server indicated that the connection header is "upgrade", it means that this - // is a WebSocket connection so the caller will may be upgrading the connection. + // is a WebSocket connection so the caller may be upgrading the connection. if (Azure::Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( connectionHeader->second, "upgrade")) { From 15e0afe52b1c392bd4e08904a3d3b3e1b9f443c6 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Mon, 25 Jul 2022 17:10:46 -0700 Subject: [PATCH 133/149] Pull request feedback --- sdk/core/azure-core/CMakeLists.txt | 2 +- sdk/core/azure-core/inc/azure/core/base64.hpp | 8 ++++---- .../inc/azure/core/internal/cryptography/sha_hash.hpp | 2 +- sdk/core/azure-core/src/http/curl/curl.cpp | 1 - sdk/core/azure-core/src/http/websockets/websockets.cpp | 2 +- .../{websocketsimpl.cpp => websockets_impl.cpp} | 2 +- .../{websocketsimpl.hpp => websockets_impl.hpp} | 0 sdk/core/azure-core/test/ut/websocket_server.py | 3 ++- sdk/core/azure-core/test/ut/websocket_test.cpp | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) rename sdk/core/azure-core/src/http/websockets/{websocketsimpl.cpp => websockets_impl.cpp} (99%) rename sdk/core/azure-core/src/http/websockets/{websocketsimpl.hpp => websockets_impl.hpp} (100%) diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index a50f87b9c5..6d94397403 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -141,7 +141,7 @@ set( src/http/url.cpp src/http/user_agent.cpp src/http/websockets/websockets.cpp - src/http/websockets/websocketsimpl.cpp + src/http/websockets/websockets_impl.cpp src/io/body_stream.cpp src/io/random_access_file_body_stream.cpp src/logger.cpp diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index e44b0da0b5..b84451dd9d 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -19,7 +19,7 @@ namespace Azure { namespace Core { /** * @brief Used to convert one form of data into another, for example encoding binary data into - * Base64 text. + * Base64 encoded octets. * * @note Base64 encoded data is a subset of the ASCII encoding (characters 0-127). As such, * it can be considered a subset of UTF-8. @@ -35,7 +35,7 @@ namespace Azure { namespace Core { public: /** - * @brief Base64 encodes a vector of binary data. + * @brief Encodes a vector of binary data using Base64. * * @param data The input vector that contains binary data to be encoded. * @return The Base64 encoded contents of the vector. @@ -43,9 +43,9 @@ namespace Azure { namespace Core { static std::string Base64Encode(const std::vector& data); /** - * @brief Decodes the Base64 encoded text into binary data. + * @brief Decodes a Base64 encoded data into a vector of binary data. * - * @param text Base64 encoded text to be decoded. + * @param text Base64 encoded data to be decoded. * @return The decoded binary data. */ static std::vector Base64Decode(const std::string& text); diff --git a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp index a389f7e837..87ecf1ddc1 100644 --- a/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp +++ b/sdk/core/azure-core/inc/azure/core/internal/cryptography/sha_hash.hpp @@ -42,7 +42,7 @@ namespace Azure { namespace Core { namespace Cryptography { namespace _internal private: /** - * @brief Underline implementation based on the OS. + * @brief Underlying implementation based on the OS. * */ std::unique_ptr m_portableImplementation; diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index bfb8693641..45ca2d52c6 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -1535,5 +1535,4 @@ void CurlConnectionPool::MoveConnectionBackToPool( { Log::Write(Logger::Level::Verbose, "Clean thread running. Won't start a new one."); } - lock.unlock(); } diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index a39c4f873e..0d65a5196a 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -3,7 +3,7 @@ #include "azure/core/http/websockets/websockets.hpp" #include "azure/core/context.hpp" -#include "websocketsimpl.hpp" +#include "websockets_impl.hpp" namespace Azure { namespace Core { namespace Http { namespace WebSockets { namespace _internal { diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp similarity index 99% rename from sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp rename to sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 72c5e673a7..080a6de256 100644 --- a/sdk/core/azure-core/src/http/websockets/websocketsimpl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "websocketsimpl.hpp" +#include "websockets_impl.hpp" #include "azure/core/base64.hpp" #include "azure/core/http/policies/policy.hpp" #include "azure/core/internal/cryptography/sha_hash.hpp" diff --git a/sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp similarity index 100% rename from sdk/core/azure-core/src/http/websockets/websocketsimpl.hpp rename to sdk/core/azure-core/src/http/websockets/websockets_impl.hpp diff --git a/sdk/core/azure-core/test/ut/websocket_server.py b/sdk/core/azure-core/test/ut/websocket_server.py index 3cf745865c..ca7bcc077b 100644 --- a/sdk/core/azure-core/test/ut/websocket_server.py +++ b/sdk/core/azure-core/test/ut/websocket_server.py @@ -1,6 +1,7 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT from array import array import asyncio -#from curses import has_key from operator import length_hint import threading from time import sleep diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index aab0264221..6308a4fab8 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "../../src/http/websockets/websocketsimpl.hpp" +#include "../../src/http/websockets/websockets_impl.hpp" #include "azure/core/http/websockets/websockets.hpp" #include "azure/core/internal/json/json.hpp" #include From d5e02ef6ba48612fd5d3efc9a37d791a331800d7 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 10:09:28 -0700 Subject: [PATCH 134/149] Fixed CI build problem --- sdk/core/azure-core-tracing-opentelemetry/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core-tracing-opentelemetry/CMakeLists.txt b/sdk/core/azure-core-tracing-opentelemetry/CMakeLists.txt index bf3077e676..6d0dc15151 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/CMakeLists.txt +++ b/sdk/core/azure-core-tracing-opentelemetry/CMakeLists.txt @@ -36,7 +36,7 @@ if (BUILD_AZURE_CORE_TRACING_OPENTELEMETRY) find_package(azure-core-cpp REQUIRED) endif() endif() - find_package(opentelemetry-cpp "1.3.0" CONFIG REQUIRED) + find_package(opentelemetry-cpp CONFIG REQUIRED) set( AZURE_CORE_OPENTELEMETRY_HEADER From afbab8a8b7bd80f23a4deca428613130549e4bac Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 14:18:41 -0700 Subject: [PATCH 135/149] Improved text for HasNativeWebsocketSupport. --- .../azure/core/http/websockets/curl_websockets_transport.hpp | 2 +- .../core/http/websockets/win_http_websockets_transport.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index f3a2e6c9f9..efae1bdca2 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -47,7 +47,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { virtual std::unique_ptr Send(Request& request, Context const& context) override; /** - * @brief Indicates if the transports supports native websockets or not. + * @brief Indicates if the transport natively supports websockets or not. * * @details For the CURL websocket transport, the transport does NOT support native websockets - * it is the responsibility of the client of the WebSocketTransport to format WebSocket protocol diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 63f6567021..cdecb36f4f 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -52,9 +52,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { virtual std::unique_ptr Send(Request& request, Context const& context) override; /** - * @brief Indicates if the transports supports native websockets or not. + * @brief Indicates if the transports natively websockets or not. * - * @details For the WinHTTP websocket transport, the transport supports native websockets. + * @details For the WinHTTP websocket transport, the WinHTTP API supports websockets. */ virtual bool HasNativeWebsocketSupport() override { return true; } From 91ea0a1f5069a6b6c42c321515bb9bf2f108ead8 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 14:39:09 -0700 Subject: [PATCH 136/149] PR changes --- .../core/http/websockets/curl_websockets_transport.hpp | 3 ++- .../inc/azure/core/http/websockets/websockets.hpp | 5 +++-- .../core/http/websockets/websockets_transport.hpp | 2 +- sdk/core/azure-core/src/http/websockets/websockets.cpp | 4 ++-- .../azure-core/src/http/websockets/websockets_impl.cpp | 10 +++++----- .../azure-core/src/http/websockets/websockets_impl.hpp | 2 +- sdk/core/azure-core/test/ut/websocket_test.cpp | 4 ++-- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index efae1bdca2..7d6ef9c4f7 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -53,7 +53,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * it is the responsibility of the client of the WebSocketTransport to format WebSocket protocol * elements. */ - virtual bool HasNativeWebsocketSupport() override { return false; } + virtual bool HasBuiltInWebSocketSupport() override { return false; } /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. @@ -67,6 +67,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @details Not implemented for CURL websockets because CURL does not support native websockets. * + * The first param is the close reason, the second is descriptive text. */ virtual void NativeCloseSocket(uint16_t, std::string const&, Azure::Core::Context const&) override diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index 9dc8c4a8d7..e4a65d12e2 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -344,9 +344,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * supports websockets in the transport, or if the websocket implementation * is providing websocket protocol support. * - * @returns true if the websocket transport supports websockets natively. + * @returns true if the HTTP transport used for WebSocket support directly supports the + * WebSocket API. */ - bool HasNativeWebSocketSupport() const; + bool HasBuiltInWebSocketSupport() const; /** @brief Returns the protocol chosen by the remote server during the initial handshake * diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index e56b431e28..e163c3cecd 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -68,7 +68,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @returns true if the transport has native websocket support, false otherwise. */ - virtual bool HasNativeWebsocketSupport() = 0; + virtual bool HasBuiltInWebSocketSupport() = 0; /**************/ /* Native WebSocket support functions*/ diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index 0d65a5196a..fdf9f118b4 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -55,9 +55,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return m_socketImplementation->GetStatistics(); } - bool WebSocket::HasNativeWebSocketSupport() const + bool WebSocket::HasBuiltInWebSocketSupport() const { - return m_socketImplementation->HasNativeWebSocketSupport(); + return m_socketImplementation->HasBuiltInWebSocketSupport(); } std::shared_ptr WebSocket::ReceiveFrame(Azure::Core::Context const& context) diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 080a6de256..8e57e6c9a0 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -93,7 +93,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // natively. auto randomKey = GenerateRandomKey(); auto encodedKey = Azure::Core::Convert::Base64Encode(randomKey); - if (!m_transport->HasNativeWebsocketSupport()) + if (!m_transport->HasBuiltInWebSocketSupport()) { // If the transport doesn't support WebSockets natively, set the standardized WebSocket // upgrade headers. @@ -136,7 +136,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Prove that the server received this socket request. auto& responseHeaders = response->GetHeaders(); - if (!m_transport->HasNativeWebsocketSupport()) + if (!m_transport->HasBuiltInWebSocketSupport()) { auto socketAccept(responseHeaders.find("Sec-WebSocket-Accept")); if (socketAccept == responseHeaders.end()) @@ -161,7 +161,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_state = SocketState::Open; } - bool WebSocketImplementation::HasNativeWebSocketSupport() + bool WebSocketImplementation::HasBuiltInWebSocketSupport() { std::lock_guard lock(m_stateMutex); m_stateOwner = std::this_thread::get_id(); @@ -169,7 +169,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::runtime_error("Socket is not open."); } - return m_transport->HasNativeWebsocketSupport(); + return m_transport->HasBuiltInWebSocketSupport(); } std::string const& WebSocketImplementation::GetChosenProtocol() @@ -797,7 +797,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { m_stop = false; // Spin up a thread to receive data from the transport. - if (!transport->HasNativeWebsocketSupport()) + if (!transport->HasBuiltInWebSocketSupport()) { std::unique_lock lock(m_pingThreadStarted); m_pingThread = std::thread{&PingThread::PingThreadLoop, this}; diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp index 6a5fcf7cfe..65b1391299 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp @@ -51,7 +51,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::string const& GetChosenProtocol(); bool IsOpen() { return m_state == SocketState::Open; } - bool HasNativeWebSocketSupport(); + bool HasBuiltInWebSocketSupport(); _internal::WebSocketStatistics GetStatistics() const; diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 6308a4fab8..78f4efaeab 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -323,7 +323,7 @@ TEST_F(WebSocketTests, PingReceiveTest) WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest")); testSocket.Open(); - if (!testSocket.HasNativeWebSocketSupport()) + if (!testSocket.HasBuiltInWebSocketSupport()) { GTEST_LOG_(INFO) << "Sleeping for 15 seconds to collect pings."; @@ -355,7 +355,7 @@ TEST_F(WebSocketTests, PingSendTest) WebSocket testSocket(Azure::Core::Url("http://localhost:8000/echotest"), socketOptions); testSocket.Open(); - if (!testSocket.HasNativeWebSocketSupport()) + if (!testSocket.HasBuiltInWebSocketSupport()) { GTEST_LOG_(INFO) << "Sleeping for 10 seconds to collect pings."; From d8e175b1b7fe76b1e40deeec0499c75d093b1ce3 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 14:50:01 -0700 Subject: [PATCH 137/149] more feedback --- .../http/websockets/websockets_transport.hpp | 10 +++---- .../win_http_websockets_transport.hpp | 2 +- .../src/http/websockets/websockets_impl.cpp | 26 +++++++++---------- .../src/http/winhttp/win_http_websockets.cpp | 18 ++++++------- .../azure-core/test/ut/websocket_test.cpp | 2 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index e163c3cecd..8343ba727e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -27,24 +27,24 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @brief Indicates that the frame is a partial UTF-8 encoded text frame - it is NOT the * complete frame to be sent to the remote node. */ - FrameTypeTextFragment, + TextFragment, /** * @brief Indicates that the frame is either the complete UTF-8 encoded text frame to be sent * to the remote node or the final frame of a multipart message. */ - FrameTypeText, + Text, /** * @brief Indicates that the frame is either the complete binary frame to be sent * to the remote node or the final frame of a multipart message. */ - FrameTypeBinary, + Binary, /** * @brief Indicates that the frame is a partial binary frame - it is NOT the * complete frame to be sent to the remote node. */ - FrameTypeBinaryFragment, + BinaryFragment, - FrameTypeClosed, + Closed, }; struct NativeWebSocketCloseInformation diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index cdecb36f4f..b5aa480af7 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -56,7 +56,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * * @details For the WinHTTP websocket transport, the WinHTTP API supports websockets. */ - virtual bool HasNativeWebsocketSupport() override { return true; } + virtual bool HasBuiltInWebSocketSupport() override { return true; } /** * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 8e57e6c9a0..201e301f4e 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -213,7 +213,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } m_state = SocketState::Closing; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->HasNativeWebsocketSupport()) + if (m_transport->HasBuiltInWebSocketSupport()) { m_transport->NativeCloseSocket(closeStatus, closeReason.c_str(), context); } @@ -276,11 +276,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::vector utf8text(textFrame.begin(), textFrame.end()); m_receiveStatistics.TextFramesSent++; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->HasNativeWebsocketSupport()) + if (m_transport->HasBuiltInWebSocketSupport()) { m_transport->NativeSendFrame( - (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::FrameTypeText - : WebSocketTransport::NativeWebSocketFrameType::FrameTypeTextFragment), + (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::Text + : WebSocketTransport::NativeWebSocketFrameType::TextFragment), utf8text, context); } @@ -306,11 +306,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } m_receiveStatistics.BinaryFramesSent++; #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->HasNativeWebsocketSupport()) + if (m_transport->HasBuiltInWebSocketSupport()) { m_transport->NativeSendFrame( - (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary - : WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinaryFragment), + (isFinalFrame ? WebSocketTransport::NativeWebSocketFrameType::Binary + : WebSocketTransport::NativeWebSocketFrameType::BinaryFragment), binaryFrame, context); } @@ -441,29 +441,29 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names WebSocketImplementation::ReceiveTransportFrame(Azure::Core::Context const& context) { #if SUPPORT_NATIVE_TRANSPORT - if (m_transport->HasNativeWebsocketSupport()) + if (m_transport->HasBuiltInWebSocketSupport()) { auto payload = m_transport->NativeReceiveFrame(context); m_receiveStatistics.FramesReceived++; switch (payload.FrameType) { - case WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary: + case WebSocketTransport::NativeWebSocketFrameType::Binary: m_receiveStatistics.BinaryFramesReceived++; return std::make_shared( SocketOpcode::BinaryFrame, true, payload.FrameData); - case WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinaryFragment: + case WebSocketTransport::NativeWebSocketFrameType::BinaryFragment: m_receiveStatistics.BinaryFramesReceived++; return std::make_shared( SocketOpcode::BinaryFrame, false, payload.FrameData); - case WebSocketTransport::NativeWebSocketFrameType::FrameTypeText: + case WebSocketTransport::NativeWebSocketFrameType::Text: m_receiveStatistics.TextFramesReceived++; return std::make_shared( SocketOpcode::TextFrame, true, payload.FrameData); - case WebSocketTransport::NativeWebSocketFrameType::FrameTypeTextFragment: + case WebSocketTransport::NativeWebSocketFrameType::TextFragment: m_receiveStatistics.TextFramesReceived++; return std::make_shared( SocketOpcode::TextFrame, false, payload.FrameData); - case WebSocketTransport::NativeWebSocketFrameType::FrameTypeClosed: { + case WebSocketTransport::NativeWebSocketFrameType::Closed: { m_receiveStatistics.CloseFramesReceived++; auto closeResult = m_transport->NativeGetCloseSocketInformation(context); std::vector closePayload; diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index e5a274faa1..93cd9e568c 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -141,16 +141,16 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { WINHTTP_WEB_SOCKET_BUFFER_TYPE bufferType; switch (frameType) { - case NativeWebSocketFrameType::FrameTypeText: + case NativeWebSocketFrameType::Text: bufferType = WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE; break; - case NativeWebSocketFrameType::FrameTypeBinary: + case NativeWebSocketFrameType::Binary: bufferType = WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE; break; - case NativeWebSocketFrameType::FrameTypeBinaryFragment: + case NativeWebSocketFrameType::BinaryFragment: bufferType = WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE; break; - case NativeWebSocketFrameType::FrameTypeTextFragment: + case NativeWebSocketFrameType::TextFragment: bufferType = WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE; break; default: @@ -197,19 +197,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { switch (bufferType) { case WINHTTP_WEB_SOCKET_UTF8_MESSAGE_BUFFER_TYPE: - frameTypeReceived = NativeWebSocketFrameType::FrameTypeText; + frameTypeReceived = NativeWebSocketFrameType::Text; break; case WINHTTP_WEB_SOCKET_BINARY_MESSAGE_BUFFER_TYPE: - frameTypeReceived = NativeWebSocketFrameType::FrameTypeBinary; + frameTypeReceived = NativeWebSocketFrameType::Binary; break; case WINHTTP_WEB_SOCKET_BINARY_FRAGMENT_BUFFER_TYPE: - frameTypeReceived = NativeWebSocketFrameType::FrameTypeBinaryFragment; + frameTypeReceived = NativeWebSocketFrameType::BinaryFragment; break; case WINHTTP_WEB_SOCKET_UTF8_FRAGMENT_BUFFER_TYPE: - frameTypeReceived = NativeWebSocketFrameType::FrameTypeTextFragment; + frameTypeReceived = NativeWebSocketFrameType::TextFragment; break; case WINHTTP_WEB_SOCKET_CLOSE_BUFFER_TYPE: - frameTypeReceived = NativeWebSocketFrameType::FrameTypeClosed; + frameTypeReceived = NativeWebSocketFrameType::Closed; break; default: throw std::runtime_error("Unknown frame type: " + std::to_string(bufferType)); diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 78f4efaeab..92b706a1ac 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -868,7 +868,7 @@ TEST_F(WebSocketTests, CurlTransportCoverage) EXPECT_THROW(transport->NativeGetCloseSocketInformation({}), std::runtime_error); EXPECT_THROW( transport->NativeSendFrame( - WebSocketTransport::NativeWebSocketFrameType::FrameTypeBinary, {}, {}), + WebSocketTransport::NativeWebSocketFrameType::Binary, {}, {}), std::runtime_error); EXPECT_THROW(transport->NativeReceiveFrame({}), std::runtime_error); } From ad4db258bf04201561ae78582a1692939a708d45 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 14:53:34 -0700 Subject: [PATCH 138/149] clang-format --- sdk/core/azure-core/test/ut/websocket_test.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 92b706a1ac..809340df2f 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -867,8 +867,7 @@ TEST_F(WebSocketTests, CurlTransportCoverage) EXPECT_THROW(transport->NativeCloseSocket(1001, {}, {}), std::runtime_error); EXPECT_THROW(transport->NativeGetCloseSocketInformation({}), std::runtime_error); EXPECT_THROW( - transport->NativeSendFrame( - WebSocketTransport::NativeWebSocketFrameType::Binary, {}, {}), + transport->NativeSendFrame(WebSocketTransport::NativeWebSocketFrameType::Binary, {}, {}), std::runtime_error); EXPECT_THROW(transport->NativeReceiveFrame({}), std::runtime_error); } From 980b54718493557c0a28e5c4803f9970feaeeacd Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 14:57:46 -0700 Subject: [PATCH 139/149] code review feedback --- .../http/websockets/websockets_transport.hpp | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index 8343ba727e..e7371283d5 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -44,17 +44,39 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ BinaryFragment, + /** + * @brief Indicates that the frame is a "close" frame - the remote node + * sent a close frame. + */ Closed, }; + /** @brief Close information returned from a WebSocket transport that has builtin support + * for WebSockets. + */ struct NativeWebSocketCloseInformation { + /** + * @brief Close response code. + */ uint16_t CloseReason; + /** + * @brief Close reason. + */ std::string CloseReasonDescription; }; + /** @brief Frame information returned from a WebSocket transport that has builtin support + * for WebSockets. + */ struct NativeWebSocketReceiveInformation { + /** + * @brief Type of frame received. + */ NativeWebSocketFrameType FrameType; + /** + * @brief Data received. + */ std::vector FrameData; }; /** @@ -64,7 +86,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { virtual ~WebSocketTransport() {} /** - * @brief Determines if the transport natively supports WebSockets or not. + * @brief Indicates whether the transport natively supports WebSockets. * * @returns true if the transport has native websocket support, false otherwise. */ From 41c49c016293cccf87b0d51c31b3a3f28c194801 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 15:01:07 -0700 Subject: [PATCH 140/149] Renamed GetChosenProtocl to GetNegotiatedProtocol --- .../azure-core/inc/azure/core/http/websockets/websockets.hpp | 4 ++-- sdk/core/azure-core/src/http/websockets/websockets.cpp | 4 ++-- sdk/core/azure-core/src/http/websockets/websockets_impl.cpp | 2 +- sdk/core/azure-core/src/http/websockets/websockets_impl.hpp | 2 +- sdk/core/azure-core/test/ut/websocket_test.cpp | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index e4a65d12e2..b4007384f2 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -349,11 +349,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ bool HasBuiltInWebSocketSupport() const; - /** @brief Returns the protocol chosen by the remote server during the initial handshake + /** @brief Returns the protocol chosen by the remote server during the initial handshake. * * @returns The protocol negotiated between client and server. */ - std::string const& GetChosenProtocol() const; + std::string const& GetNegotiatedProtocol() const; /** @brief Returns statistics about the WebSocket. * diff --git a/sdk/core/azure-core/src/http/websockets/websockets.cpp b/sdk/core/azure-core/src/http/websockets/websockets.cpp index fdf9f118b4..65102f31ab 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets.cpp @@ -69,9 +69,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { m_socketImplementation->AddHeader(headerName, headerValue); } - std::string const& WebSocket::GetChosenProtocol() const + std::string const& WebSocket::GetNegotiatedProtocol() const { - return m_socketImplementation->GetChosenProtocol(); + return m_socketImplementation->GetNegotiatedProtocol(); } bool WebSocket::IsOpen() const { return m_socketImplementation->IsOpen(); } diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 201e301f4e..bf5c68595e 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -172,7 +172,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names return m_transport->HasBuiltInWebSocketSupport(); } - std::string const& WebSocketImplementation::GetChosenProtocol() + std::string const& WebSocketImplementation::GetNegotiatedProtocol() { std::lock_guard lock(m_stateMutex); m_stateOwner = std::this_thread::get_id(); diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp index 65b1391299..73a10ec143 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.hpp @@ -49,7 +49,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names void AddHeader(std::string const& headerName, std::string const& headerValue); - std::string const& GetChosenProtocol(); + std::string const& GetNegotiatedProtocol(); bool IsOpen() { return m_state == SocketState::Open; } bool HasBuiltInWebSocketSupport(); diff --git a/sdk/core/azure-core/test/ut/websocket_test.cpp b/sdk/core/azure-core/test/ut/websocket_test.cpp index 809340df2f..3160934962 100644 --- a/sdk/core/azure-core/test/ut/websocket_test.cpp +++ b/sdk/core/azure-core/test/ut/websocket_test.cpp @@ -34,7 +34,7 @@ TEST_F(WebSocketTests, CreateSimpleSocket) { WebSocket defaultSocket(Azure::Core::Url("http://localhost:8000")); defaultSocket.AddHeader("newHeader", "headerValue"); - EXPECT_THROW(defaultSocket.GetChosenProtocol(), std::runtime_error); + EXPECT_THROW(defaultSocket.GetNegotiatedProtocol(), std::runtime_error); } } @@ -786,7 +786,7 @@ class LibWebSocketStatus { // The server should have chosen the lws-status protocol since it doesn't understand the other // protocols. - EXPECT_EQ("lws-status", serverSocket.GetChosenProtocol()); + EXPECT_EQ("lws-status", serverSocket.GetNegotiatedProtocol()); std::string returnValue; std::shared_ptr lwsStatus; do From 7f3d8fe50d938067f549a136bdc6577851a17ccf Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 15:49:48 -0700 Subject: [PATCH 141/149] Renamed NativeClose to Close since it's valid for all WebSocket transports --- .../inc/azure/core/http/curl_transport.hpp | 6 ++++++ .../websockets/curl_websockets_transport.hpp | 4 ++-- .../http/websockets/websockets_transport.hpp | 16 ++++++++-------- .../websockets/win_http_websockets_transport.hpp | 4 ++-- .../inc/azure/core/http/win_http_transport.hpp | 4 ++++ .../azure-core/src/http/curl/curl_websockets.cpp | 2 +- .../src/http/websockets/websockets_impl.cpp | 2 +- .../src/http/winhttp/win_http_websockets.cpp | 2 +- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp index 2816f57581..398708399e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp @@ -148,6 +148,12 @@ namespace Azure { namespace Core { namespace Http { { } + // See also: + // [Core Guidelines C.35: "A base class destructor should be either public + // and virtual or protected and + // non-virtual"](http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#c35-a-base-class-destructor-should-be-either-public-and-virtual-or-protected-and-non-virtual) + virtual ~CurlTransport() = default; + /** * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse * diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp index 7d6ef9c4f7..d3b1ecb1ad 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/curl_websockets_transport.hpp @@ -56,10 +56,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { virtual bool HasBuiltInWebSocketSupport() override { return false; } /** - * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * @brief Closes the WebSocket handle. * */ - virtual void NativeClose() override; + virtual void Close() override; // Native WebSocket support methods. /** diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp index e7371283d5..1afda3c0d2 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets_transport.hpp @@ -92,6 +92,14 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { */ virtual bool HasBuiltInWebSocketSupport() = 0; + /** + * @brief Closes the WebSocket. + * + * Does not notify the remote endpoint that the socket is being closed. + * + */ + virtual void Close() = 0; + /**************/ /* Native WebSocket support functions*/ /**************/ @@ -108,14 +116,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { Azure::Core::Context const& context) = 0; - /** - * @brief Closes the WebSocket. - * - * Does not notify the remote endpoint that the socket is being closed. - * - */ - virtual void NativeClose() = 0; - /** * @brief Retrieve the information associated with a WebSocket close response. * diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index b5aa480af7..7324662f54 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -59,10 +59,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { virtual bool HasBuiltInWebSocketSupport() override { return true; } /** - * @brief Gracefully closes the WebSocket, notifying the remote node of the close reason. + * @brief Close the underlying WebSocket handle. * */ - virtual void NativeClose() override; + virtual void Close() override; // Native WebSocket support methods. /** diff --git a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp index 207a39efeb..459c400db3 100644 --- a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp @@ -177,6 +177,10 @@ namespace Azure { namespace Core { namespace Http { */ virtual std::unique_ptr Send(Request& request, Context const& context) override; + // See also: + // [Core Guidelines C.35: "A base class destructor should be either public + // and virtual or protected and + // non-virtual"](http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#c35-a-base-class-destructor-should-be-either-public-and-virtual-or-protected-and-non-virtual) virtual ~WinHttpTransport() = default; }; diff --git a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp index edd10355d9..49e182caee 100644 --- a/sdk/core/azure-core/src/http/curl/curl_websockets.cpp +++ b/sdk/core/azure-core/src/http/curl/curl_websockets.cpp @@ -27,7 +27,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { - void CurlWebSocketTransport::NativeClose() { m_upgradedConnection->Shutdown(); } + void CurlWebSocketTransport::Close() { m_upgradedConnection->Shutdown(); } // Send an HTTP request to the remote server. std::unique_ptr CurlWebSocketTransport::Send( diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index bf5c68595e..e5cc49b3fa 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -258,7 +258,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } // Close the socket - after this point, the m_transport is invalid. m_pingThread.Shutdown(); - m_transport->NativeClose(); + m_transport->Close(); m_state = SocketState::Closed; } diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp index 93cd9e568c..7d869ca709 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_websockets.cpp @@ -48,7 +48,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { /** * @brief Close the WebSocket cleanly. */ - void WinHttpWebSocketTransport::NativeClose() { m_socketHandle.reset(); } + void WinHttpWebSocketTransport::Close() { m_socketHandle.reset(); } // Native WebSocket support methods. /** From 0a80a7ec6a2a331c6db52d14fe57ace0b0769daf Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 15:58:55 -0700 Subject: [PATCH 142/149] Fixed horrible variable name --- .../src/http/winhttp/win_http_transport.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 5ab5f1b6c9..a1469deb22 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -331,7 +331,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( url.GetScheme(), WebSocketScheme)); // Create an HTTP request handle. - _detail::unique_HINTERNET hi( + _detail::unique_HINTERNET request( WinHttpOpenRequest( connectionHandle.get(), HttpMethodToWideString(requestMethod).c_str(), @@ -342,7 +342,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( WINHTTP_DEFAULT_ACCEPT_TYPES, // No media types are accepted by the client requestSecureHttp ? WINHTTP_FLAG_SECURE : 0), _detail::HINTERNET_deleter{}); // Uses secure transaction semantics (SSL/TLS) - if (!hi) + if (!request) { // Errors include: // ERROR_WINHTTP_INCORRECT_HANDLE_TYPE @@ -362,7 +362,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( // Note: If/When TLS client certificate support is added to the pipeline, this line may need to // be revisited. if (!WinHttpSetOption( - hi.get(), WINHTTP_OPTION_CLIENT_CERT_CONTEXT, WINHTTP_NO_CLIENT_CERT_CONTEXT, 0)) + request.get(), WINHTTP_OPTION_CLIENT_CERT_CONTEXT, WINHTTP_NO_CLIENT_CERT_CONTEXT, 0)) { GetErrorAndThrow("Error while setting client cert context to ignore."); } @@ -371,7 +371,7 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( if (m_options.IgnoreUnknownCertificateAuthority) { auto option = SECURITY_FLAG_IGNORE_UNKNOWN_CA; - if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_SECURITY_FLAGS, &option, sizeof(option))) + if (!WinHttpSetOption(request.get(), WINHTTP_OPTION_SECURITY_FLAGS, &option, sizeof(option))) { GetErrorAndThrow("Error while setting ignore unknown server certificate."); } @@ -379,12 +379,12 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( if (HasWebSocketSupport()) { - if (!WinHttpSetOption(hi.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) + if (!WinHttpSetOption(request.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) { GetErrorAndThrow("Error while Enabling WebSocket upgrade."); } } - return hi; + return request; } // For PUT/POST requests, send additional data using WinHttpWriteData. From 755fc8026ff5dad2fc2c224cf2a89802c53dbfd8 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 16:18:11 -0700 Subject: [PATCH 143/149] Simplified expression --- .../azure-core/src/http/winhttp/win_http_transport.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index a1469deb22..9131eb98df 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -377,12 +377,12 @@ _detail::unique_HINTERNET WinHttpTransport::CreateRequestHandle( } } - if (HasWebSocketSupport()) + // If we are supporting WebSockets, then let WinHTTP know that it should + // prepare to upgrade the HttpRequest to a WebSocket. + if (HasWebSocketSupport() + && !WinHttpSetOption(request.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) { - if (!WinHttpSetOption(request.get(), WINHTTP_OPTION_UPGRADE_TO_WEB_SOCKET, nullptr, 0)) - { - GetErrorAndThrow("Error while Enabling WebSocket upgrade."); - } + GetErrorAndThrow("Error while Enabling WebSocket upgrade."); } return request; } From 1072f76beec128ee604dc6abc5939c6a1e8696ae Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 16:20:26 -0700 Subject: [PATCH 144/149] Update sdk/core/azure-core/src/http/websockets/websockets_impl.cpp Co-authored-by: Rick Winter --- sdk/core/azure-core/src/http/websockets/websockets_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index e5cc49b3fa..30d765f0ce 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -881,4 +881,4 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names }); return rv; } -}}}}} // namespace Azure::Core::Http::WebSockets::_detail \ No newline at end of file +}}}}} // namespace Azure::Core::Http::WebSockets::_detail From fa8f236aca2b17ccc15ba9b83d908cef42527db1 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 17:05:35 -0700 Subject: [PATCH 145/149] Code review feedback --- sdk/core/azure-core/inc/azure/core/base64.hpp | 1 - .../azure/core/http/websockets/websockets.hpp | 35 +++++++++++++++++++ .../src/http/websockets/websockets_impl.cpp | 33 +++++++++-------- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/base64.hpp b/sdk/core/azure-core/inc/azure/core/base64.hpp index b84451dd9d..cb5daadcf5 100644 --- a/sdk/core/azure-core/inc/azure/core/base64.hpp +++ b/sdk/core/azure-core/inc/azure/core/base64.hpp @@ -10,7 +10,6 @@ #pragma once #include -#include #include #include #include diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp index b4007384f2..a0a55d3bd4 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/websockets.hpp @@ -66,46 +66,66 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { { /** @brief The number of WebSocket frames sent on this WebSocket. */ uint32_t FramesSent; + /** @brief The number of bytes of data sent to the peer on this WebSocket. */ uint32_t BytesSent; + /** @brief The number of WebSocket frames received from the peer. */ uint32_t FramesReceived; + /** @brief The number of bytes received from the peer. */ uint32_t BytesReceived; + /** @brief The number of "Ping" frames received from the peer. */ uint32_t PingFramesReceived; + /** @brief The number of "Ping" frames sent to the peer. */ uint32_t PingFramesSent; + /** @brief The number of "Pong" frames received from the peer. */ uint32_t PongFramesReceived; + /** @brief The number of "Pong" frames sent to the peer. */ uint32_t PongFramesSent; + /** @brief The number of "Text" frames received from the peer. */ uint32_t TextFramesReceived; + /** @brief The number of "Text" frames sent to the peer. */ uint32_t TextFramesSent; + /** @brief The number of "Binary" frames received from the peer. */ uint32_t BinaryFramesReceived; + /** @brief The number of "Binary" frames sent to the peer. */ uint32_t BinaryFramesSent; + /** @brief The number of "Continuation" frames sent to the peer. */ uint32_t ContinuationFramesSent; + /** @brief The number of "Continuation" frames received from the peer. */ uint32_t ContinuationFramesReceived; + /** @brief The number of "Close" frames received from the peer. */ uint32_t CloseFramesReceived; + /** @brief The number of frames received which were not processed. */ uint32_t FramesDropped; + /** @brief The number of frames received which were not returned because they were received * after the Close() method was called. */ + uint32_t FramesDroppedByClose; /** @brief The number of frames dropped because they were over the maximum payload size. */ + uint32_t FramesDroppedByPayloadSizeLimit; /** @brief The number of frames dropped because they were out of compliance with the protocol. */ uint32_t FramesDroppedByProtocolError; + /** @brief The number of reads performed on the transport.*/ uint32_t TransportReads; + /** @brief The number of bytes read from the transport. */ uint32_t TransportReadBytes; }; @@ -116,15 +136,19 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** @brief The type of frame received: Text, Binary or Close. */ WebSocketFrameType FrameType{}; + /** @brief True if the frame received is a "final" frame */ bool IsFinalFrame{false}; + /** @brief Returns the contents of the frame as a Text frame. * @returns A WebSocketTextFrame containing the contents of the frame. */ std::shared_ptr AsTextFrame(); + /** @brief Returns the contents of the frame as a Binary frame. * @returns A WebSocketBinaryFrame containing the contents of the frame. */ + std::shared_ptr AsBinaryFrame(); /** @brief Returns the contents of the frame as a Peer Close frame. * @returns A WebSocketPeerCloseFrame containing the contents of the frame. @@ -138,6 +162,12 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * @param frameType The type of frame received. */ WebSocketFrame(WebSocketFrameType frameType) : FrameType{frameType} {} + + /** @brief Construct a new instance of a WebSocketFrame with a specific frame type and final + * flag. + * @param frameType The type of frame received. + * @param isFinalFrame true if the frame is the final frame. + */ WebSocketFrame(WebSocketFrameType frameType, bool isFinalFrame) : FrameType{frameType}, IsFinalFrame{isFinalFrame} { @@ -153,6 +183,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** @brief Constructs a new WebSocketTextFrame */ WebSocketTextFrame() : WebSocketFrame(WebSocketFrameType::TextFrameReceived){}; + /** @brief Text of the frame received from the remote peer. */ std::string Text; @@ -178,6 +209,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** @brief Constructs a new WebSocketBinaryFrame */ WebSocketBinaryFrame() : WebSocketFrame(WebSocketFrameType::BinaryFrameReceived){}; + /** @brief Binary frame data received from the remote peer. */ std::vector Data; @@ -202,9 +234,11 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { public: /** @brief Constructs a new WebSocketPeerCloseFrame */ WebSocketPeerCloseFrame() : WebSocketFrame(WebSocketFrameType::PeerClosedReceived){}; + /** @brief Status code sent from the remote peer. Typically a member of the WebSocketErrorCode * enumeration */ uint16_t RemoteStatusCode{}; + /** @brief Optional text sent from the remote peer. */ std::string RemoteCloseReason; @@ -232,6 +266,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { * in the initial WebSocket handshake. */ std::string ServiceName; + /** * @brief The version of the service client. Used for the User-Agent header in the * initial WebSocket handshake diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index e5cc49b3fa..83a99eb8da 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -52,7 +52,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { if (m_state != SocketState::Invalid && m_state != SocketState::Closed) { - throw std::runtime_error("Socket is not closed."); + throw std::runtime_error( + "Socket in unexpected state: " + std::to_string(static_cast(m_state))); } m_state = SocketState::Opening; @@ -167,7 +168,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } return m_transport->HasBuiltInWebSocketSupport(); } @@ -178,7 +180,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } return m_chosenProtocol; } @@ -209,7 +212,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names } if (m_state != SocketState::Open) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } m_state = SocketState::Closing; #if SUPPORT_NATIVE_TRANSPORT @@ -230,14 +234,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Unlock the state mutex before waiting for the close response to be received. lock.unlock(); - // To ensure that we process the responses in a "timely" fashion, limit the close - // reception to 20 seconds if we don't already have a timeout. - Azure::Core::Context closeContext = context; - auto cancelTimepoint = closeContext.GetDeadline(); - if (cancelTimepoint == Azure::DateTime::max()) - { - closeContext = closeContext.WithDeadline(std::chrono::system_clock::now() + 20s); - } // Drain the incoming series of frames from the server. // Note that there might be in-flight frames that were sent from the other end of the // WebSocket that we don't care about any more (since we're closing the WebSocket). So @@ -250,10 +246,10 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Logger::Level::Warning, "Received unexpected frame during close. Opcode: " + std::to_string(static_cast(closeResponse->Opcode))); - closeResponse = ReceiveTransportFrame(closeContext); + closeResponse = ReceiveTransportFrame(context); } - // Re-acquire the state lock once we've received the close lock. + // Re-acquire the state lock once we've received the close response. lock.lock(); } // Close the socket - after this point, the m_transport is invalid. @@ -271,7 +267,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_stateOwner = std::this_thread::get_id(); if (m_state != SocketState::Open) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } std::vector utf8text(textFrame.begin(), textFrame.end()); m_receiveStatistics.TextFramesSent++; @@ -302,7 +299,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names if (m_state != SocketState::Open) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } m_receiveStatistics.BinaryFramesSent++; #if SUPPORT_NATIVE_TRANSPORT @@ -333,7 +331,8 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names if (m_state != SocketState::Open && m_state != SocketState::Closing) { - throw std::runtime_error("Socket is not open."); + throw std::runtime_error( + "Socket is not open." + std::to_string(static_cast(m_state))); } // Unlock the state lock to allow other threads to run. If we don't, we might end up in in a From 868e68f79a556c2b01b5f84493f324dd658522ff Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 17:28:56 -0700 Subject: [PATCH 146/149] Update sdk/core/azure-core/src/http/websockets/websockets_impl.cpp Co-authored-by: Rick Winter --- sdk/core/azure-core/src/http/websockets/websockets_impl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 30d765f0ce..e2f54cfbbd 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -357,6 +357,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Logger::Level::Verbose, "Received Ping frame: " + HexEncode(frame->Payload, 16)); SendPong(frame->Payload, context); break; + // We want to ignore all incoming "Pong" frames. case SocketOpcode::Pong: Log::Write( From 74f40c06f731f9cb226ea1557ae31aa60366745c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 17:29:19 -0700 Subject: [PATCH 147/149] Pull request feedback --- .../core/http/websockets/win_http_websockets_transport.hpp | 1 + sdk/core/azure-core/src/http/curl/curl.cpp | 4 +--- sdk/core/azure-core/src/http/websockets/websockets_impl.cpp | 6 +++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 7324662f54..61bb469bc7 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -41,6 +41,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { : WinHttpTransport(options) { } + /** * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse * diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index 45ca2d52c6..ea65dbd39a 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -80,9 +80,7 @@ int pollSocketUntilEventOrTimeout( throw TransportException("Error while sending request. Platform does not support Poll()"); #endif - struct pollfd poller - { - }; + struct pollfd poller; poller.fd = socketFileDescriptor; // set direction diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 83a99eb8da..9a62a22b8a 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -251,6 +251,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // Re-acquire the state lock once we've received the close response. lock.lock(); + m_stateOwner = std::this_thread::get_id(); } // Close the socket - after this point, the m_transport is invalid. m_pingThread.Shutdown(); @@ -315,7 +316,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names else #endif { - // Log::Write(Logger::Level::Verbose, "Send Binary Frame " + HexEncode(binaryFrame, 16)); std::vector sendFrame = EncodeFrame(SocketOpcode::BinaryFrame, isFinalFrame, binaryFrame); @@ -377,6 +377,9 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names { throw std::runtime_error("Close response buffer is too short."); } + // Encode the payload for close according to RFC 6455 + // section 5.5.1. The first two bytes of the payload contain the status code. + // The remainder of the payload is a UTF-8 encoded string. uint16_t errorCode = 0; errorCode |= (frame->Payload[0] << 8) & 0xff00; errorCode |= (frame->Payload[1] & 0x00ff); @@ -384,6 +387,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names // We received a close frame, mark the socket as closed. Make sure we // reacquire the state lock before setting the state to closed. lock.lock(); + m_stateOwner = std::this_thread::get_id(); m_state = SocketState::Closed; return std::shared_ptr(new WebSocketPeerCloseFrame( From 81555061e859efb03c89b2bcc402d748c4821d6c Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 17:32:34 -0700 Subject: [PATCH 148/149] Code Review feedback --- .../src/http/websockets/websockets_impl.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp index 6341e92686..90a9a69e33 100644 --- a/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp +++ b/sdk/core/azure-core/src/http/websockets/websockets_impl.cpp @@ -356,7 +356,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names Logger::Level::Verbose, "Received Ping frame: " + HexEncode(frame->Payload, 16)); SendPong(frame->Payload, context); break; - + // We want to ignore all incoming "Pong" frames. case SocketOpcode::Pong: Log::Write( @@ -696,10 +696,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names m_bufferLen = m_transport->ReadFromSocket(m_buffer, m_bufferSize, context); m_receiveStatistics.TransportReads++; m_receiveStatistics.TransportReadBytes += static_cast(m_bufferLen); - // Log::Write( - // Logger::Level::Verbose, - // "Read #" + std::to_string(m_receiveStatistics.TransportReads.load()) - // + "from transport: " + std::to_string(m_bufferLen)); } else { @@ -761,11 +757,6 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { names std::unique_lock transportLock(m_transportMutex); m_receiveStatistics.BytesSent += static_cast(sendFrame.size()); m_receiveStatistics.FramesSent += 1; - // Log::Write( - // Logger::Level::Verbose, - // "Send #" + std::to_string(m_receiveStatistics.FramesSent.load()) + "to - // transport:" - // + std::to_string(sendFrame.size()) + "Data: " + HexEncode(sendFrame, 0x10)); m_transport->SendBuffer(sendFrame.data(), sendFrame.size(), context); } From 965538ee1dbed2e4321346c0e7d732a4b3e926f0 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 26 Jul 2022 21:20:37 -0700 Subject: [PATCH 149/149] clang-format --- .../core/http/websockets/win_http_websockets_transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp index 61bb469bc7..8fdf5b533c 100644 --- a/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/websockets/win_http_websockets_transport.hpp @@ -41,7 +41,7 @@ namespace Azure { namespace Core { namespace Http { namespace WebSockets { : WinHttpTransport(options) { } - + /** * @brief Implements interface to send an HTTP Request and produce an HTTP RawResponse *