diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index cef6a7452d..bed39c5995 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_57a579efb6" + "Tag": "cpp/storage_532bd2d370" } diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index 960d4564f1..b08054d1fc 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -434,6 +434,10 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Nullable m_customerProvidedKey; /** @brief Encryption scope. */ Azure::Nullable m_encryptionScope; + /** @brief Upload TransferValidationOptions */ + Azure::Nullable m_uploadValidationOptions; + /** @brief Download TransferValidationOptions */ + Azure::Nullable m_downloadValidationOptions; private: explicit BlobClient( diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 31ef7c26d5..a72eaf4211 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -168,6 +168,17 @@ namespace Azure { namespace Storage { namespace Blobs { Models::EncryptionAlgorithmType Algorithm; }; + /** + * Configures whether to do content validation for blob uploads and downloads. + */ + struct TransferValidationOptions + { + /** + * @brief The algorithm used for storage checksum. + */ + StorageChecksumAlgorithm Algorithm = StorageChecksumAlgorithm::None; + }; + /** * @brief Client options used to initialize all kinds of blob clients. */ @@ -210,6 +221,16 @@ namespace Azure { namespace Storage { namespace Blobs { * not set. */ Azure::Nullable Audience; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable UploadValidationOptions; + + /** + * @brief Optional. Configures whether to do content validation for blob downloads. + */ + Azure::Nullable DownloadValidationOptions; }; /** @@ -673,6 +694,11 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ BlobAccessConditions AccessConditions; + + /** + * @brief Optional. Configures whether to do content validation for blob downloads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -707,6 +733,11 @@ namespace Azure { namespace Storage { namespace Blobs { */ int32_t Concurrency = 5; } TransferOptions; + + /** + * @brief Optional. Configures whether to do content validation for blob downloads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -889,6 +920,11 @@ namespace Azure { namespace Storage { namespace Blobs { * Indicates whether the blob has a legal hold. */ Azure::Nullable HasLegalHold; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -948,6 +984,11 @@ namespace Azure { namespace Storage { namespace Blobs { * Indicates whether the blob has a legal hold. */ Azure::Nullable HasLegalHold; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -1035,6 +1076,11 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ LeaseAccessConditions AccessConditions; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -1335,6 +1381,11 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ AppendBlobAccessConditions AccessConditions; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable ValidationOptions; }; /** @@ -1444,6 +1495,11 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ PageBlobAccessConditions AccessConditions; + + /** + * @brief Optional. Configures whether to do content validation for blob uploads. + */ + Azure::Nullable ValidationOptions; }; /** diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index 389f3d8fd6..7180fe967b 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -32,7 +32,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2024-08-04"; + constexpr static const char* ApiVersion = "2025-01-05"; } // namespace _detail namespace Models { /** @@ -1612,6 +1612,16 @@ namespace Azure { namespace Storage { namespace Blobs { * The blob's type. */ Models::BlobType BlobType; + /** + * Indicates the response body contains a structured message and specifies the message schema + * version and properties. + */ + Nullable StructuredBodyType; + /** + * The length of the blob/file content inside the message body when the response body is + * returned as a structured message. Will always be smaller than Content-Length. + */ + Nullable StructuredContentLength; }; /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetProperties. @@ -2576,6 +2586,11 @@ namespace Azure { namespace Storage { namespace Blobs { * encryption scope. */ Nullable EncryptionScope; + /** + * Indicates the structured message body was accepted and mirrors back the message schema + * version and properties. + */ + Nullable StructuredBodyType; }; /** * @brief Response type for #Azure::Storage::Blobs::PageBlobClient::ClearPages. @@ -2872,6 +2887,11 @@ namespace Azure { namespace Storage { namespace Blobs { * encryption scope. */ Nullable EncryptionScope; + /** + * Indicates the structured message body was accepted and mirrors back the message schema + * version and properties. + */ + Nullable StructuredBodyType; }; /** * @brief Response type for #Azure::Storage::Blobs::AppendBlobClient::AppendBlockFromUri. @@ -2985,6 +3005,11 @@ namespace Azure { namespace Storage { namespace Blobs { * encryption scope. */ Nullable EncryptionScope; + /** + * Indicates the structured message body was accepted and mirrors back the message schema + * version and properties. + */ + Nullable StructuredBodyType; }; /** * @brief Response type for #Azure::Storage::Blobs::BlockBlobClient::UploadFromUri. @@ -3057,6 +3082,11 @@ namespace Azure { namespace Storage { namespace Blobs { * encryption scope. */ Nullable EncryptionScope; + /** + * Indicates the structured message body was accepted and mirrors back the message schema + * version and properties. + */ + Nullable StructuredBodyType; }; /** * @brief Response type for #Azure::Storage::Blobs::BlockBlobClient::StageBlockFromUri. @@ -3499,6 +3529,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable LeaseId; Nullable RangeGetContentMD5; Nullable RangeGetContentCRC64; + Nullable StructuredBodyType; Nullable EncryptionKey; Nullable> EncryptionKeySha256; Nullable EncryptionAlgorithm; @@ -3917,6 +3948,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable StructuredBodyType; + Nullable StructuredContentLength; }; static Response UploadPages( Core::Http::_internal::HttpPipeline& pipeline, @@ -4112,6 +4145,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable StructuredBodyType; + Nullable StructuredContentLength; }; static Response AppendBlock( Core::Http::_internal::HttpPipeline& pipeline, @@ -4192,6 +4227,8 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable ImmutabilityPolicyMode; Nullable LegalHold; Nullable> TransactionalContentCrc64; + Nullable StructuredBodyType; + Nullable StructuredContentLength; }; static Response Upload( Core::Http::_internal::HttpPipeline& pipeline, @@ -4247,6 +4284,8 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable> EncryptionKeySha256; Nullable EncryptionAlgorithm; Nullable EncryptionScope; + Nullable StructuredBodyType; + Nullable StructuredContentLength; }; static Response StageBlock( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-blobs/src/append_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/append_blob_client.cpp index ba1cf35566..96dafe524f 100644 --- a/sdk/storage/azure-storage-blobs/src/append_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/append_blob_client.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -141,19 +142,6 @@ namespace Azure { namespace Storage { namespace Blobs { const Azure::Core::Context& context) const { _detail::AppendBlobClient::AppendAppendBlobBlockOptions protocolLayerOptions; - if (options.TransactionalContentHash.HasValue()) - { - if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) - { - protocolLayerOptions.TransactionalContentMD5 - = options.TransactionalContentHash.Value().Value; - } - else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) - { - protocolLayerOptions.TransactionalContentCrc64 - = options.TransactionalContentHash.Value().Value; - } - } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.MaxSize = options.AccessConditions.IfMaxSizeLessThanOrEqual; protocolLayerOptions.AppendPosition = options.AccessConditions.IfAppendPositionEqual; @@ -169,6 +157,37 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); } protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (options.TransactionalContentHash.HasValue()) + { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) + { + protocolLayerOptions.TransactionalContentMD5 + = options.TransactionalContentHash.Value().Value; + } + else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) + { + protocolLayerOptions.TransactionalContentCrc64 + = options.TransactionalContentHash.Value().Value; + } + } + else + { + Azure::Nullable validationOptions + = options.ValidationOptions.HasValue() ? options.ValidationOptions + : m_uploadValidationOptions; + if (validationOptions.HasValue() + && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None) + { + protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage; + protocolLayerOptions.StructuredContentLength = content.Length(); + _internal::StructuredMessageEncodingStreamOptions encodingStreamOptions; + encodingStreamOptions.Flags = _internal::StructuredMessageFlags::Crc64; + auto structuredContent + = _internal::StructuredMessageEncodingStream(&content, encodingStreamOptions); + return _detail::AppendBlobClient::AppendBlock( + *m_pipeline, m_blobUrl, structuredContent, protocolLayerOptions, context); + } + } return _detail::AppendBlobClient::AppendBlock( *m_pipeline, m_blobUrl, content, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index a96f2c98a8..e4556b9ba8 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -110,7 +111,9 @@ namespace Azure { namespace Storage { namespace Blobs { BlobClient::BlobClient(const std::string& blobUrl, const BlobClientOptions& options) : m_blobUrl(blobUrl), m_customerProvidedKey(options.CustomerProvidedKey), - m_encryptionScope(options.EncryptionScope) + m_encryptionScope(options.EncryptionScope), + m_uploadValidationOptions(options.UploadValidationOptions), + m_downloadValidationOptions(options.DownloadValidationOptions) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -167,6 +170,7 @@ namespace Azure { namespace Storage { namespace Blobs { const DownloadBlobOptions& options, const Azure::Core::Context& context) const { + bool isStructuredMessage = false; _detail::BlobClient::DownloadBlobOptions protocolLayerOptions; if (options.Range.HasValue()) { @@ -189,6 +193,18 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.RangeGetContentCRC64 = true; } } + else + { + Azure::Nullable validationOptions + = options.ValidationOptions.HasValue() ? options.ValidationOptions + : m_downloadValidationOptions; + if (validationOptions.HasValue() + && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None) + { + isStructuredMessage = true; + protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage; + } + } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; @@ -243,8 +259,23 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::ReliableStreamOptions reliableStreamOptions; reliableStreamOptions.MaxRetryRequests = _internal::ReliableStreamRetryCount; - downloadResponse.Value.BodyStream = std::make_unique<_internal::ReliableStream>( + auto reliableStream = std::make_unique<_internal::ReliableStream>( std::move(downloadResponse.Value.BodyStream), reliableStreamOptions, retryFunction); + if (isStructuredMessage) + { + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + if (downloadResponse.Value.StructuredContentLength.HasValue()) + { + decodingOptions.ContentLength = downloadResponse.Value.StructuredContentLength.Value(); + } + downloadResponse.Value.BodyStream + = std::make_unique<_internal::StructuredMessageDecodingStream>( + std::move(reliableStream), decodingOptions); + } + else + { + downloadResponse.Value.BodyStream = std::move(reliableStream); + } } if (downloadResponse.RawResponse->GetStatusCode() == Azure::Core::Http::HttpStatusCode::Ok) { diff --git a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp index 0ffcefe748..e50fb857fd 100644 --- a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -98,19 +99,7 @@ namespace Azure { namespace Storage { namespace Blobs { const Azure::Core::Context& context) const { _detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions; - if (options.TransactionalContentHash.HasValue()) - { - if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) - { - protocolLayerOptions.TransactionalContentMD5 - = options.TransactionalContentHash.Value().Value; - } - else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) - { - protocolLayerOptions.TransactionalContentCrc64 - = options.TransactionalContentHash.Value().Value; - } - } + protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; @@ -141,6 +130,38 @@ namespace Azure { namespace Storage { namespace Blobs { } protocolLayerOptions.LegalHold = options.HasLegalHold; + if (options.TransactionalContentHash.HasValue()) + { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) + { + protocolLayerOptions.TransactionalContentMD5 + = options.TransactionalContentHash.Value().Value; + } + else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) + { + protocolLayerOptions.TransactionalContentCrc64 + = options.TransactionalContentHash.Value().Value; + } + } + else + { + Azure::Nullable validationOptions + = options.ValidationOptions.HasValue() ? options.ValidationOptions + : m_uploadValidationOptions; + if (validationOptions.HasValue() + && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None) + { + protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage; + protocolLayerOptions.StructuredContentLength = content.Length(); + _internal::StructuredMessageEncodingStreamOptions encodingStreamOptions; + encodingStreamOptions.Flags = _internal::StructuredMessageFlags::Crc64; + auto structuredContent + = _internal::StructuredMessageEncodingStream(&content, encodingStreamOptions); + return _detail::BlockBlobClient::Upload( + *m_pipeline, m_blobUrl, structuredContent, protocolLayerOptions, context); + } + } + return _detail::BlockBlobClient::Upload( *m_pipeline, m_blobUrl, content, protocolLayerOptions, context); } @@ -171,6 +192,7 @@ namespace Azure { namespace Storage { namespace Blobs { uploadBlockBlobOptions.AccessTier = options.AccessTier; uploadBlockBlobOptions.ImmutabilityPolicy = options.ImmutabilityPolicy; uploadBlockBlobOptions.HasLegalHold = options.HasLegalHold; + uploadBlockBlobOptions.ValidationOptions = options.ValidationOptions; return Upload(contentStream, uploadBlockBlobOptions, context); } @@ -396,6 +418,15 @@ namespace Azure { namespace Storage { namespace Blobs { { _detail::BlockBlobClient::StageBlockBlobBlockOptions protocolLayerOptions; protocolLayerOptions.BlockId = blockId; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (options.TransactionalContentHash.HasValue()) { if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) @@ -409,14 +440,24 @@ namespace Azure { namespace Storage { namespace Blobs { = options.TransactionalContentHash.Value().Value; } } - protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; - if (m_customerProvidedKey.HasValue()) + else { - protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; - protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; - protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + Azure::Nullable validationOptions + = options.ValidationOptions.HasValue() ? options.ValidationOptions + : m_uploadValidationOptions; + if (validationOptions.HasValue() + && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None) + { + protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage; + protocolLayerOptions.StructuredContentLength = content.Length(); + _internal::StructuredMessageEncodingStreamOptions encodingStreamOptions; + encodingStreamOptions.Flags = _internal::StructuredMessageFlags::Crc64; + auto structuredContent + = _internal::StructuredMessageEncodingStream(&content, encodingStreamOptions); + return _detail::BlockBlobClient::StageBlock( + *m_pipeline, m_blobUrl, structuredContent, protocolLayerOptions, context); + } } - protocolLayerOptions.EncryptionScope = m_encryptionScope; return _detail::BlockBlobClient::StageBlock( *m_pipeline, m_blobUrl, content, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp index 9204eb12d1..e784a9ad96 100644 --- a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -152,19 +153,6 @@ namespace Azure { namespace Storage { namespace Blobs { _detail::PageBlobClient::UploadPageBlobPagesOptions protocolLayerOptions; protocolLayerOptions.Range = "bytes=" + std::to_string(offset) + "-" + std::to_string(offset + content.Length() - 1); - if (options.TransactionalContentHash.HasValue()) - { - if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) - { - protocolLayerOptions.TransactionalContentMD5 - = options.TransactionalContentHash.Value().Value; - } - else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) - { - protocolLayerOptions.TransactionalContentCrc64 - = options.TransactionalContentHash.Value().Value; - } - } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; @@ -183,6 +171,38 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); } protocolLayerOptions.EncryptionScope = m_encryptionScope; + + if (options.TransactionalContentHash.HasValue()) + { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) + { + protocolLayerOptions.TransactionalContentMD5 + = options.TransactionalContentHash.Value().Value; + } + else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) + { + protocolLayerOptions.TransactionalContentCrc64 + = options.TransactionalContentHash.Value().Value; + } + } + else + { + Azure::Nullable validationOptions + = options.ValidationOptions.HasValue() ? options.ValidationOptions + : m_uploadValidationOptions; + if (validationOptions.HasValue() + && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None) + { + protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage; + protocolLayerOptions.StructuredContentLength = content.Length(); + _internal::StructuredMessageEncodingStreamOptions encodingStreamOptions; + encodingStreamOptions.Flags = _internal::StructuredMessageFlags::Crc64; + auto structuredContent + = _internal::StructuredMessageEncodingStream(&content, encodingStreamOptions); + return _detail::PageBlobClient::UploadPages( + *m_pipeline, m_blobUrl, structuredContent, protocolLayerOptions, context); + } + } return _detail::PageBlobClient::UploadPages( *m_pipeline, m_blobUrl, content, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index f42493bd9d..f6ddab9ce0 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -377,7 +377,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -397,7 +397,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -693,7 +693,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "stats"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -793,7 +793,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobContainersIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1078,7 +1078,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "userdelegationkey"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1193,7 +1193,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "account"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -1223,7 +1223,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1244,7 +1244,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "blobs"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.Where.HasValue() && !options.Where.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1400,7 +1400,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-public-access", options.Access.ToString()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.DefaultEncryptionScope.HasValue() && !options.DefaultEncryptionScope.Value().empty()) { @@ -1437,7 +1437,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1515,7 +1515,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1549,7 +1549,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Modified-Since", options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1576,7 +1576,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1745,7 +1745,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1768,7 +1768,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.DeletedContainerName.HasValue() && !options.DeletedContainerName.Value().empty()) { request.SetHeader("x-ms-deleted-container-name", options.DeletedContainerName.Value()); @@ -1798,7 +1798,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (!options.SourceContainerName.empty()) { request.SetHeader("x-ms-source-container-name", options.SourceContainerName); @@ -1832,7 +1832,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1854,7 +1854,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "blobs"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.Where.HasValue() && !options.Where.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -2024,7 +2024,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -2065,7 +2065,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2105,7 +2105,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2146,7 +2146,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -2191,7 +2191,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2238,7 +2238,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2917,7 +2917,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.ShowOnly.HasValue() && !options.ShowOnly.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -3606,7 +3606,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "account"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -3658,6 +3658,10 @@ namespace Azure { namespace Storage { namespace Blobs { "x-ms-range-get-content-crc64", options.RangeGetContentCRC64.Value() ? "true" : "false"); } + if (options.StructuredBodyType.HasValue() && !options.StructuredBodyType.Value().empty()) + { + request.SetHeader("x-ms-structured-body", options.StructuredBodyType.Value()); + } if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) { request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); @@ -3697,7 +3701,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.UserPrincipalName.HasValue()) { request.SetHeader("x-ms-upn", options.UserPrincipalName.Value() ? "true" : "false"); @@ -3870,6 +3874,15 @@ namespace Azure { namespace Storage { namespace Blobs { response.Details.HasLegalHold = pRawResponse->GetHeaders().at("x-ms-legal-hold") == std::string("true"); } + if (pRawResponse->GetHeaders().count("x-ms-structured-body") != 0) + { + response.StructuredBodyType = pRawResponse->GetHeaders().at("x-ms-structured-body"); + } + if (pRawResponse->GetHeaders().count("x-ms-structured-content-length") != 0) + { + response.StructuredContentLength + = std::stoll(pRawResponse->GetHeaders().at("x-ms-structured-content-length")); + } if (httpStatusCode == Core::Http::HttpStatusCode::Ok) { if (pRawResponse->GetHeaders().count("Content-MD5") != 0) @@ -3977,7 +3990,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.UserPrincipalName.HasValue()) { request.SetHeader("x-ms-upn", options.UserPrincipalName.Value() ? "true" : "false"); @@ -4243,7 +4256,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -4261,7 +4274,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4280,7 +4293,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "expiry"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (!options.ExpiryOptions.ToString().empty()) { request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.ToString()); @@ -4368,7 +4381,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-content-disposition", options.BlobContentDisposition); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4401,7 +4414,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.IfUnmodifiedSince.HasValue()) { request.SetHeader( @@ -4444,7 +4457,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Delete, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4464,7 +4477,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "legalhold"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); request.SetHeader("x-ms-legal-hold", options.LegalHold ? "true" : "false"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4536,7 +4549,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4611,7 +4624,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -4669,7 +4682,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4726,7 +4739,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4788,7 +4801,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4846,7 +4859,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -4926,7 +4939,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5033,7 +5046,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -5152,7 +5165,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -5262,7 +5275,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::NoContent) @@ -5300,7 +5313,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-rehydrate-priority", options.RehydratePriority.Value().ToString()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -5329,7 +5342,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "account"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -5627,7 +5640,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.EncryptionScope.HasValue() && !options.EncryptionScope.Value().empty()) { request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); @@ -5676,7 +5689,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5803,7 +5816,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Type", "application/xml; charset=UTF-8"); request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.VersionId.HasValue() && !options.VersionId.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5934,7 +5947,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -6080,7 +6093,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); + if (options.StructuredBodyType.HasValue() && !options.StructuredBodyType.Value().empty()) + { + request.SetHeader("x-ms-structured-body", options.StructuredBodyType.Value()); + } + if (options.StructuredContentLength.HasValue()) + { + request.SetHeader( + "x-ms-structured-content-length", + std::to_string(options.StructuredContentLength.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6127,6 +6150,10 @@ namespace Azure { namespace Storage { namespace Blobs { { response.EncryptionScope = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); } + if (pRawResponse->GetHeaders().count("x-ms-structured-body") != 0) + { + response.StructuredBodyType = pRawResponse->GetHeaders().at("x-ms-structured-body"); + } return Response(std::move(response), std::move(pRawResponse)); } Response PageBlobClient::ClearPages( @@ -6206,7 +6233,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6349,7 +6376,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.CopySourceAuthorization.HasValue() && !options.CopySourceAuthorization.Value().empty()) { @@ -6450,7 +6477,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.Marker.HasValue() && !options.Marker.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -6628,7 +6655,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.Marker.HasValue() && !options.Marker.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -6808,7 +6835,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } request.SetHeader("x-ms-blob-content-length", std::to_string(options.BlobContentLength)); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6874,7 +6901,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6932,7 +6959,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-copy-source", options.CopySource); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -7043,7 +7070,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -7171,7 +7198,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); + if (options.StructuredBodyType.HasValue() && !options.StructuredBodyType.Value().empty()) + { + request.SetHeader("x-ms-structured-body", options.StructuredBodyType.Value()); + } + if (options.StructuredContentLength.HasValue()) + { + request.SetHeader( + "x-ms-structured-content-length", + std::to_string(options.StructuredContentLength.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7210,6 +7247,10 @@ namespace Azure { namespace Storage { namespace Blobs { { response.EncryptionScope = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); } + if (pRawResponse->GetHeaders().count("x-ms-structured-body") != 0) + { + response.StructuredBodyType = pRawResponse->GetHeaders().at("x-ms-structured-body"); + } return Response(std::move(response), std::move(pRawResponse)); } Response AppendBlobClient::AppendBlockFromUri( @@ -7325,7 +7366,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.CopySourceAuthorization.HasValue() && !options.CopySourceAuthorization.Value().empty()) { @@ -7381,7 +7422,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "seal"); - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -7520,7 +7561,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -7549,6 +7590,16 @@ namespace Azure { namespace Storage { namespace Blobs { "x-ms-content-crc64", Core::Convert::Base64Encode(options.TransactionalContentCrc64.Value())); } + if (options.StructuredBodyType.HasValue() && !options.StructuredBodyType.Value().empty()) + { + request.SetHeader("x-ms-structured-body", options.StructuredBodyType.Value()); + } + if (options.StructuredContentLength.HasValue()) + { + request.SetHeader( + "x-ms-structured-content-length", + std::to_string(options.StructuredContentLength.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7581,6 +7632,10 @@ namespace Azure { namespace Storage { namespace Blobs { { response.EncryptionScope = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); } + if (pRawResponse->GetHeaders().count("x-ms-structured-body") != 0) + { + response.StructuredBodyType = pRawResponse->GetHeaders().at("x-ms-structured-body"); + } if (pRawResponse->GetHeaders().count("x-ms-content-crc64") != 0) { response.TransactionalContentHash = ContentHash(); @@ -7703,7 +7758,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-tags", options.SourceIfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -7835,7 +7890,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); + if (options.StructuredBodyType.HasValue() && !options.StructuredBodyType.Value().empty()) + { + request.SetHeader("x-ms-structured-body", options.StructuredBodyType.Value()); + } + if (options.StructuredContentLength.HasValue()) + { + request.SetHeader( + "x-ms-structured-content-length", + std::to_string(options.StructuredContentLength.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7868,6 +7933,10 @@ namespace Azure { namespace Storage { namespace Blobs { { response.EncryptionScope = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); } + if (pRawResponse->GetHeaders().count("x-ms-structured-body") != 0) + { + response.StructuredBodyType = pRawResponse->GetHeaders().at("x-ms-structured-body"); + } return Response(std::move(response), std::move(pRawResponse)); } Response BlockBlobClient::StageBlockFromUri( @@ -7949,7 +8018,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.CopySourceAuthorization.HasValue() && !options.CopySourceAuthorization.Value().empty()) { @@ -8116,7 +8185,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -8205,7 +8274,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2024-08-04"); + request.SetHeader("x-ms-version", "2025-01-05"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index c3bbde3790..55769069af 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-blobs namespace: Azure::Storage::Blobs output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.BlobStorage/stable/2024-08-04/blob.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/refs/heads/main/specification/storage/data-plane/Microsoft.BlobStorage/stable/2025-01-05/blob.json ``` ## ModelFour Options @@ -100,12 +100,12 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2024-08-04"] + "enum": ["2025-01-05"] }; - from: swagger-document where: $.parameters transform: > - $.ApiVersionParameter.enum[0] = "2024-08-04"; + $.ApiVersionParameter.enum[0] = "2025-01-05"; ``` ### Rename Operations @@ -382,6 +382,9 @@ directive: if (h === "x-ms-meta") { $[h]["x-ms-format"] = "caseinsensitivemap"; } + if (h === "x-ms-structured-body" || h === "x-ms-structured-content-length") { + $[h]["x-nullable"] = true; + } } - from: swagger-document where: $.parameters diff --git a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp index c72ff20fa4..a4e291573f 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp @@ -467,4 +467,48 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(blobContent.size(), properties.BlobSize); } + TEST_F(AppendBlobClientTest, StructuredMessageTest_PLAYBACKONLY_) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto bodyStream = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); + Blobs::TransferValidationOptions validationOptions; + validationOptions.Algorithm = StorageChecksumAlgorithm::Crc64; + + auto appendBlob = GetAppendBlobClientForTest(LowercaseRandomString()); + appendBlob.Create(); + + // Append + Blobs::AppendBlockOptions appendOptions; + appendOptions.ValidationOptions = validationOptions; + Blobs::Models::AppendBlockResult appendResult; + EXPECT_NO_THROW(appendResult = appendBlob.AppendBlock(bodyStream, appendOptions).Value); + EXPECT_TRUE(appendResult.StructuredBodyType.HasValue()); + EXPECT_EQ(appendResult.StructuredBodyType.Value(), _internal::CrcStructuredMessage); + validationOptions.Algorithm = StorageChecksumAlgorithm::None; + appendOptions.ValidationOptions = validationOptions; + bodyStream.Rewind(); + EXPECT_NO_THROW(appendResult = appendBlob.AppendBlock(bodyStream, appendOptions).Value); + EXPECT_FALSE(appendResult.StructuredBodyType.HasValue()); + validationOptions.Algorithm = StorageChecksumAlgorithm::Auto; + appendOptions.ValidationOptions = validationOptions; + bodyStream.Rewind(); + EXPECT_NO_THROW(appendResult = appendBlob.AppendBlock(bodyStream, appendOptions).Value); + EXPECT_TRUE(appendResult.StructuredBodyType.HasValue()); + EXPECT_EQ(appendResult.StructuredBodyType.Value(), _internal::CrcStructuredMessage); + + // Download + Blobs::DownloadBlobOptions downloadOptions; + downloadOptions.ValidationOptions = validationOptions; + Blobs::Models::DownloadBlobResult downloadResult; + EXPECT_NO_THROW(downloadResult = appendBlob.Download(downloadOptions).Value); + auto downloadedData = downloadResult.BodyStream->ReadToEnd(); + EXPECT_EQ( + content, + std::vector(downloadedData.begin(), downloadedData.begin() + contentSize)); + EXPECT_TRUE(downloadResult.StructuredContentLength.HasValue()); + EXPECT_EQ(downloadResult.StructuredContentLength.Value(), contentSize * 3); + EXPECT_TRUE(downloadResult.StructuredBodyType.HasValue()); + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index 03c76ae6d8..b5440fcf55 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -1387,6 +1387,21 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW( destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options)); } + { + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest5"); + Blobs::TransferValidationOptions validationOptions; + validationOptions.Algorithm = StorageChecksumAlgorithm::Crc64; + Blobs::UploadBlockBlobOptions options; + options.ValidationOptions + = validationOptions; // ValidationOptions should not work when ContentHash is set + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = contentMd5; + stream.Rewind(); + Blobs::Models::UploadBlockBlobResult uploadResult; + EXPECT_NO_THROW(uploadResult = destBlobClient.Upload(stream, options).Value); + EXPECT_FALSE(uploadResult.StructuredBodyType.HasValue()); + } } TEST_F(BlockBlobClientTest, UploadFromUri) @@ -2182,4 +2197,62 @@ namespace Azure { namespace Storage { namespace Test { blobProperties = versionClient.GetProperties().Value; EXPECT_TRUE(blobProperties.HasLegalHold); } + + TEST_F(BlockBlobClientTest, StructuredMessageTest_PLAYBACKONLY_) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto bodyStream = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); + Blobs::TransferValidationOptions validationOptions; + validationOptions.Algorithm = StorageChecksumAlgorithm::Crc64; + + // Upload Download + Blobs::UploadBlockBlobOptions uploadOptions; + uploadOptions.ValidationOptions = validationOptions; + Blobs::Models::UploadBlockBlobResult uploadResult; + EXPECT_NO_THROW(uploadResult = m_blockBlobClient->Upload(bodyStream, uploadOptions).Value); + EXPECT_TRUE(uploadResult.StructuredBodyType.HasValue()); + EXPECT_EQ(uploadResult.StructuredBodyType.Value(), _internal::CrcStructuredMessage); + Blobs::DownloadBlobOptions downloadOptions; + downloadOptions.ValidationOptions = validationOptions; + Blobs::Models::DownloadBlobResult downloadResult; + EXPECT_NO_THROW(downloadResult = m_blockBlobClient->Download(downloadOptions).Value); + auto downloadedData = downloadResult.BodyStream->ReadToEnd(); + EXPECT_EQ(content, downloadedData); + EXPECT_TRUE(downloadResult.StructuredContentLength.HasValue()); + EXPECT_EQ(downloadResult.StructuredContentLength.Value(), contentSize); + EXPECT_TRUE(downloadResult.StructuredBodyType.HasValue()); + + // Stage Block + auto blobClient = m_blobContainerClient->GetBlockBlobClient(LowercaseRandomString()); + const std::vector dataPart1 = RandomBuffer(contentSize); + const std::vector dataPart2 = RandomBuffer(contentSize); + + const std::string blockId1 = Base64EncodeText("0"); + const std::string blockId2 = Base64EncodeText("1"); + + Blobs::StageBlockOptions stageBlockOptions; + stageBlockOptions.ValidationOptions = validationOptions; + auto blockContent = Azure::Core::IO::MemoryBodyStream(dataPart1.data(), dataPart1.size()); + Blobs::Models::StageBlockResult stageResult; + EXPECT_NO_THROW( + stageResult = blobClient.StageBlock(blockId1, blockContent, stageBlockOptions).Value); + EXPECT_TRUE(stageResult.StructuredBodyType.HasValue()); + EXPECT_EQ(stageResult.StructuredBodyType.Value(), _internal::CrcStructuredMessage); + validationOptions.Algorithm = StorageChecksumAlgorithm::None; + stageBlockOptions.ValidationOptions = validationOptions; + blockContent = Azure::Core::IO::MemoryBodyStream(dataPart2.data(), dataPart2.size()); + EXPECT_NO_THROW( + stageResult = blobClient.StageBlock(blockId2, blockContent, stageBlockOptions).Value); + EXPECT_FALSE(stageResult.StructuredBodyType.HasValue()); + EXPECT_NO_THROW(blobClient.CommitBlockList({blockId1, blockId2})); + downloadOptions.ValidationOptions = validationOptions; + EXPECT_NO_THROW(downloadResult = blobClient.Download(downloadOptions).Value); + downloadedData = downloadResult.BodyStream->ReadToEnd(); + EXPECT_EQ( + dataPart1, + std::vector(downloadedData.begin(), downloadedData.begin() + contentSize)); + EXPECT_FALSE(downloadResult.StructuredContentLength.HasValue()); + EXPECT_FALSE(downloadResult.StructuredBodyType.HasValue()); + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 44bb71f86a..237fbe77cc 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -780,4 +780,37 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(blockBlobClient.GetProperties()); } + TEST_F(PageBlobClientTest, StructuredMessageTest_PLAYBACKONLY_) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto bodyStream = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); + Blobs::TransferValidationOptions validationOptions; + validationOptions.Algorithm = StorageChecksumAlgorithm::Crc64; + + auto pageBlob = GetPageBlobClientTestForTest(LowercaseRandomString()); + pageBlob.Create(contentSize); + + // Append + Blobs::UploadPagesOptions uploadOptions; + uploadOptions.ValidationOptions = validationOptions; + Blobs::Models::UploadPagesResult uploadResult; + EXPECT_NO_THROW(uploadResult = pageBlob.UploadPages(0, bodyStream, uploadOptions).Value); + EXPECT_TRUE(uploadResult.StructuredBodyType.HasValue()); + EXPECT_EQ(uploadResult.StructuredBodyType.Value(), _internal::CrcStructuredMessage); + + // Download + Blobs::DownloadBlobOptions downloadOptions; + downloadOptions.ValidationOptions = validationOptions; + Blobs::Models::DownloadBlobResult downloadResult; + EXPECT_NO_THROW(downloadResult = pageBlob.Download(downloadOptions).Value); + auto downloadedData = downloadResult.BodyStream->ReadToEnd(); + EXPECT_EQ( + content, + std::vector(downloadedData.begin(), downloadedData.begin() + contentSize)); + EXPECT_TRUE(downloadResult.StructuredContentLength.HasValue()); + EXPECT_EQ(downloadResult.StructuredContentLength.Value(), contentSize); + EXPECT_TRUE(downloadResult.StructuredBodyType.HasValue()); + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index 76afb17a3c..a91e1baaf7 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -57,6 +57,9 @@ set( inc/azure/storage/common/internal/storage_per_retry_policy.hpp inc/azure/storage/common/internal/storage_service_version_policy.hpp inc/azure/storage/common/internal/storage_switch_to_secondary_policy.hpp + inc/azure/storage/common/internal/structured_message_decoding_stream.hpp + inc/azure/storage/common/internal/structured_message_encoding_stream.hpp + inc/azure/storage/common/internal/structured_message_helper.hpp inc/azure/storage/common/internal/xml_wrapper.hpp inc/azure/storage/common/rtti.hpp inc/azure/storage/common/storage_common.hpp @@ -77,6 +80,9 @@ set( src/storage_exception.cpp src/storage_per_retry_policy.cpp src/storage_switch_to_secondary_policy.cpp + src/structured_message_decoding_stream.cpp + src/structured_message_encoding_stream.cpp + src/structured_message_helper.cpp src/xml_wrapper.cpp ) diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/constants.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/constants.hpp index df0486cf72..a18abccda6 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/constants.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/constants.hpp @@ -21,6 +21,7 @@ namespace Azure { namespace Storage { namespace _internal { constexpr static const char* HttpHeaderContentType = "content-type"; constexpr static const char* HttpHeaderContentLength = "content-length"; constexpr static const char* HttpHeaderContentRange = "content-range"; + constexpr static const char* CrcStructuredMessage = "XSM/1.0; properties=crc64"; constexpr int ReliableStreamRetryCount = 3; }}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_decoding_stream.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_decoding_stream.hpp new file mode 100644 index 0000000000..9633a7c79a --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_decoding_stream.hpp @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/common/crypt.hpp" +#include "azure/storage/common/dll_import_export.hpp" +#include "constants.hpp" +#include "structured_message_helper.hpp" + +#include +#include + +#include +#include +#include + +namespace Azure { namespace Storage { namespace _internal { + + // Options used by structured message encode stream + struct StructuredMessageDecodingStreamOptions final + { + /** + * Required. The length of the real data in the structured message. + */ + int64_t ContentLength = 0; + }; + + /** + * @brief TODO + * + */ + class StructuredMessageDecodingStream final : public Azure::Core::IO::BodyStream { + private: + // initial bodyStream. + std::unique_ptr m_inner; + // Configuration for the encode stream + StructuredMessageDecodingStreamOptions const m_options; + + size_t m_streamHeaderLength; + size_t m_segmentHeaderLength; + size_t m_segmentFooterLength; + size_t m_streamFooterLength; + + // Length of the stream + uint64_t m_length; + StructuredMessageFlags m_flags; + uint16_t m_segmentCount; + + int64_t m_offset; + + StructuredMessageCurrentRegion m_currentRegion; + uint16_t m_currentSegmentNumber; + uint64_t m_currentSegmentOffset; + uint64_t m_currentSegmentLength; + + std::vector m_segmentHeaderBuffer; + std::vector m_segmentFooterBuffer; + + std::unique_ptr m_segmentCrc64Hash; + std::unique_ptr m_streamCrc64Hash; + + size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override; + + public: + explicit StructuredMessageDecodingStream( + std::unique_ptr inner, + StructuredMessageDecodingStreamOptions const options) + : m_inner(std::move(inner)), m_options(options), + m_streamHeaderLength(StructuredMessageHelper::StreamHeaderLength), + m_segmentHeaderLength(StructuredMessageHelper::SegmentHeaderLength), + m_segmentFooterLength(0), m_streamFooterLength(0), m_length(0), + m_flags(StructuredMessageFlags::None), m_segmentCount(0), m_offset(0), + m_currentRegion(StructuredMessageCurrentRegion::StreamHeader), m_currentSegmentNumber(0), + m_currentSegmentOffset(0), m_currentSegmentLength(0), + m_segmentHeaderBuffer(StructuredMessageHelper::SegmentHeaderLength), + m_segmentFooterBuffer(0), m_segmentCrc64Hash(std::make_unique()), + m_streamCrc64Hash(std::make_unique()) + { + } + + int64_t Length() const override { return m_options.ContentLength; } + + void Rewind() override + { + // Rewind directly from a transportAdapter body stream (like libcurl) would throw + this->m_inner->Rewind(); + this->m_segmentFooterLength = 0; + this->m_streamFooterLength = 0; + this->m_length = 0; + this->m_segmentCount = 0; + this->m_flags = StructuredMessageFlags::None; + this->m_offset = 0; + this->m_currentRegion = StructuredMessageCurrentRegion::StreamHeader; + this->m_currentSegmentNumber = 0; + this->m_currentSegmentOffset = 0; + this->m_currentSegmentLength = 0; + this->m_segmentHeaderBuffer.clear(); + this->m_segmentFooterBuffer.clear(); + this->m_segmentCrc64Hash = std::make_unique(); + this->m_streamCrc64Hash = std::make_unique(); + } + }; + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_encoding_stream.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_encoding_stream.hpp new file mode 100644 index 0000000000..d75785fcac --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_encoding_stream.hpp @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/common/crypt.hpp" +#include "azure/storage/common/dll_import_export.hpp" +#include "constants.hpp" +#include "structured_message_helper.hpp" + +#include +#include + +#include +#include +#include + +namespace Azure { namespace Storage { namespace _internal { + + // Options used by structured message encode stream + struct StructuredMessageEncodingStreamOptions final + { + // configures the maximun segment length + int64_t MaxSegmentLength = 4 * 1024 * 1024; + + StructuredMessageFlags Flags = StructuredMessageFlags::None; + }; + + /** + * @brief TODO + * + */ + class StructuredMessageEncodingStream final : public Azure::Core::IO::BodyStream { + private: + // initial bodyStream. + Azure::Core::IO::BodyStream* m_inner; + // Configuration for the encode stream + StructuredMessageEncodingStreamOptions const m_options; + + size_t m_streamHeaderLength; + size_t m_segmentHeaderLength; + size_t m_segmentFooterLength; + size_t m_streamFooterLength; + + uint16_t m_segmentCount; + uint16_t m_segmentNumber; + + int64_t m_offset; + int64_t m_innerOffset; + + StructuredMessageCurrentRegion m_currentRegion; + uint64_t m_currentRegionOffset; + + std::vector m_streamHeaderCache; + std::vector m_segmentHeaderCache; + std::vector m_segmentFooterCache; + std::vector m_streamFooterCache; + + std::unique_ptr m_segmentCrc64Hash; + std::unique_ptr m_streamCrc64Hash; + + size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override; + + public: + explicit StructuredMessageEncodingStream( + Azure::Core::IO::BodyStream* inner, + StructuredMessageEncodingStreamOptions const options) + : m_inner(inner), m_options(options), + m_streamHeaderLength(StructuredMessageHelper::StreamHeaderLength), + m_segmentHeaderLength(StructuredMessageHelper::SegmentHeaderLength), m_segmentCount(0), + m_segmentNumber(0), m_offset(0), m_innerOffset(0), + m_currentRegion(StructuredMessageCurrentRegion::StreamHeader), m_currentRegionOffset(0), + m_streamHeaderCache(0), m_segmentHeaderCache(0), m_segmentFooterCache(0), + m_streamFooterCache(0), m_segmentCrc64Hash(std::make_unique()), + m_streamCrc64Hash(std::make_unique()) + { + m_segmentFooterLength = m_options.Flags == StructuredMessageFlags::Crc64 + ? StructuredMessageHelper::Crc64Length + : 0; + m_streamFooterLength = m_options.Flags == StructuredMessageFlags::Crc64 + ? StructuredMessageHelper::Crc64Length + : 0; + m_segmentCount = static_cast( + (m_inner->Length() + m_options.MaxSegmentLength - 1) / m_options.MaxSegmentLength); + } + + int64_t Length() const override + { + return m_streamHeaderLength + m_streamFooterLength + + (m_segmentHeaderLength + m_segmentFooterLength) * m_segmentCount + + this->m_inner->Length(); + } + + void Rewind() override + { + // Rewind directly from a transportAdapter body stream (like libcurl) would throw + this->m_inner->Rewind(); + this->m_segmentNumber = 0; + this->m_offset = 0; + this->m_innerOffset = 0; + this->m_currentRegion = StructuredMessageCurrentRegion::StreamHeader; + this->m_currentRegionOffset = 0; + this->m_streamHeaderCache.clear(); + this->m_segmentHeaderCache.clear(); + this->m_segmentFooterCache.clear(); + this->m_streamFooterCache.clear(); + this->m_segmentCrc64Hash = std::make_unique(); + this->m_streamCrc64Hash = std::make_unique(); + } + }; + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_helper.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_helper.hpp new file mode 100644 index 0000000000..9343ad597f --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/structured_message_helper.hpp @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/common/dll_import_export.hpp" + +#include +#include +#include + +namespace Azure { namespace Storage { namespace _internal { + + enum StructuredMessageFlags : uint16_t + { + None = 0x0, + Crc64 = 0x1, + }; + + enum StructuredMessageCurrentRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + }; + + class StructuredMessageHelper final { + public: + AZ_STORAGE_COMMON_DLLEXPORT static const size_t Crc64Length; + + AZ_STORAGE_COMMON_DLLEXPORT static const uint8_t StructuredMessageVersion; + + AZ_STORAGE_COMMON_DLLEXPORT static const size_t StreamHeaderLength; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t StreamHeaderVersionOffset; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t StreamHeaderMessageLengthOffset; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t StreamHeaderFlagsOffset; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t StreamHeaderSegmentCountOffset; + + AZ_STORAGE_COMMON_DLLEXPORT static const size_t SegmentHeaderLength; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t SegmentHeaderNumOffset; + AZ_STORAGE_COMMON_DLLEXPORT static const int64_t SegmentHeaderContentLengthOffset; + + static void WriteStreamHeader( + uint8_t* buffer, + const uint64_t& messageLength, + const uint16_t& flags, + const uint16_t& segmentCount); + + static void WriteSegmentHeader( + uint8_t* buffer, + const uint16_t& segmentCount, + const uint64_t& segmentLength); + + static void WriteSegmentFooter(uint8_t* buffer, const uint8_t* crc64); + + static void WriteStreamFooter(uint8_t* buffer, const uint8_t* crc64); + + static void ReadStreamHeader( + const uint8_t* buffer, + uint64_t& messageLength, + StructuredMessageFlags& flags, + uint16_t& segmentCount); + + static void ReadSegmentHeader( + const uint8_t* buffer, + uint16_t& segmentCount, + uint64_t& segmentLength); + + static void ReadSegmentFooter(const uint8_t* buffer, uint8_t* crc64); + + static void ReadStreamFooter(const uint8_t* buffer, uint8_t* crc64); + }; + +}}} // namespace Azure::Storage::_internal \ No newline at end of file diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp index 5ae312ed3b..f1ade0e0fd 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/storage_common.hpp @@ -54,4 +54,25 @@ namespace Azure { namespace Storage { using Metadata = Azure::Core::CaseInsensitiveMap; + /** + * @brief The algorithm used for storage checksum. + */ + enum class StorageChecksumAlgorithm + { + /** + * @brief No selected algorithm. Do not calculate or request checksums. + */ + None, + + /** + * @brief Recommended. Allow the library to choose an algorithm. Different library versions may + * make different choices. + */ + Auto, + + /** + * @brief Cyclic redundancy check. + */ + Crc64 + }; }} // namespace Azure::Storage diff --git a/sdk/storage/azure-storage-common/src/structured_message_decoding_stream.cpp b/sdk/storage/azure-storage-common/src/structured_message_decoding_stream.cpp new file mode 100644 index 0000000000..07a06f3569 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/structured_message_decoding_stream.cpp @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/structured_message_decoding_stream.hpp" + +#include "azure/storage/common/crypt.hpp" +#include "azure/storage/common/storage_exception.hpp" + +#include + +using Azure::Core::Context; +using Azure::Core::IO::BodyStream; + +namespace Azure { namespace Storage { namespace _internal { + + size_t StructuredMessageDecodingStream::OnRead( + uint8_t* buffer, + size_t count, + Context const& context) + { + size_t totalReadContent = 0; + while (totalReadContent < count && m_offset < m_inner->Length()) + { + switch (m_currentRegion) + { + case StructuredMessageCurrentRegion::StreamHeader: { + std::vector streamHeaderBuffer(m_streamHeaderLength); + auto bytesRead = m_inner->Read(streamHeaderBuffer.data(), m_streamHeaderLength, context); + StructuredMessageHelper::ReadStreamHeader( + streamHeaderBuffer.data(), m_length, m_flags, m_segmentCount); + m_streamFooterLength + = m_flags == StructuredMessageFlags::Crc64 ? StructuredMessageHelper::Crc64Length : 0; + m_segmentFooterLength + = m_flags == StructuredMessageFlags::Crc64 ? StructuredMessageHelper::Crc64Length : 0; + m_segmentFooterBuffer.resize(m_segmentFooterLength); + m_offset += bytesRead; + m_currentRegion = StructuredMessageCurrentRegion::SegmentHeader; + break; + } + case StructuredMessageCurrentRegion::SegmentHeader: { + auto bytesRead + = m_inner->Read(m_segmentHeaderBuffer.data(), m_segmentHeaderLength, context); + StructuredMessageHelper::ReadSegmentHeader( + m_segmentHeaderBuffer.data(), m_currentSegmentNumber, m_currentSegmentLength); + m_offset += bytesRead; + m_currentRegion = StructuredMessageCurrentRegion::SegmentContent; + break; + } + case StructuredMessageCurrentRegion::SegmentContent: { + size_t readBytes = std::min( + count - totalReadContent, + static_cast(m_currentSegmentLength - m_currentSegmentOffset)); + auto bytesRead + = m_inner->Read(buffer + totalReadContent, static_cast(readBytes), context); + if (m_flags == StructuredMessageFlags::Crc64) + { + m_segmentCrc64Hash->Append(buffer + totalReadContent, bytesRead); + } + m_offset += bytesRead; + m_currentSegmentOffset += bytesRead; + totalReadContent += bytesRead; + if (m_currentSegmentOffset == m_currentSegmentLength) + { + m_currentRegion = StructuredMessageCurrentRegion::SegmentFooter; + } + break; + } + case StructuredMessageCurrentRegion::SegmentFooter: { + if (m_flags == StructuredMessageFlags::Crc64) + { + auto bytesRead + = m_inner->Read(m_segmentFooterBuffer.data(), m_segmentFooterLength, context); + // in current version, segment footer contains crc64 hash of the segment content. + auto calculatedCrc64 = m_segmentCrc64Hash->Final(); + if (calculatedCrc64 != m_segmentFooterBuffer) + { + throw StorageException( + "Segment Compared checksums did not match. Invalid data may have been written to " + "the " + "destination. calculatedChecksum:" + + std::string(calculatedCrc64.begin(), calculatedCrc64.end()) + + "reportedChecksum: " + + std::string(m_segmentFooterBuffer.begin(), m_segmentFooterBuffer.end())); + } + m_offset += bytesRead; + m_streamCrc64Hash->Concatenate(*m_segmentCrc64Hash); + m_segmentCrc64Hash = std::make_unique(); + } + + m_currentSegmentOffset = 0; + m_currentRegion = m_currentSegmentNumber == m_segmentCount + ? StructuredMessageCurrentRegion::StreamFooter + : StructuredMessageCurrentRegion::SegmentHeader; + break; + } + case StructuredMessageCurrentRegion::StreamFooter: { + if (m_flags == StructuredMessageFlags::Crc64) + { + std::vector streamFooterBuffer(m_streamFooterLength); + auto bytesRead + = m_inner->Read(streamFooterBuffer.data(), m_streamFooterLength, context); + // in current version, segment footer contains crc64 hash of the segment content. + auto calculatedCrc64 = m_streamCrc64Hash->Final(); + if (calculatedCrc64 != streamFooterBuffer) + { + throw StorageException( + "Stream Compared checksums did not match. Invalid data may have been written to " + "the " + "destination. calculatedChecksum:" + + std::string(calculatedCrc64.begin(), calculatedCrc64.end()) + + "reportedChecksum: " + + std::string(streamFooterBuffer.begin(), streamFooterBuffer.end())); + } + m_offset += bytesRead; + m_streamCrc64Hash->Concatenate(*m_segmentCrc64Hash); + m_segmentCrc64Hash = std::make_unique(); + } + break; + } + } + } + return totalReadContent; + } +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/structured_message_encoding_stream.cpp b/sdk/storage/azure-storage-common/src/structured_message_encoding_stream.cpp new file mode 100644 index 0000000000..14dc2862ef --- /dev/null +++ b/sdk/storage/azure-storage-common/src/structured_message_encoding_stream.cpp @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/structured_message_encoding_stream.hpp" + +#include "azure/storage/common/crypt.hpp" + +#include + +using Azure::Core::Context; +using Azure::Core::IO::BodyStream; + +namespace Azure { namespace Storage { namespace _internal { + + size_t StructuredMessageEncodingStream::OnRead( + uint8_t* buffer, + size_t count, + Context const& context) + { + size_t totalRead = 0; + while (totalRead < count && m_offset < this->Length()) + { + size_t subreadOffset = totalRead; + switch (m_currentRegion) + { + case StructuredMessageCurrentRegion::StreamHeader: { + size_t readBytes = std::min( + count - totalRead, static_cast(m_streamHeaderLength - m_currentRegionOffset)); + if (m_streamHeaderCache.empty()) + { + m_streamHeaderCache.resize(m_streamHeaderLength); + StructuredMessageHelper::WriteStreamHeader( + m_streamHeaderCache.data(), this->Length(), m_options.Flags, m_segmentCount); + } + std::memcpy( + buffer + subreadOffset, + m_streamHeaderCache.data() + m_currentRegionOffset, + readBytes); + m_offset += readBytes; + m_currentRegionOffset += readBytes; + totalRead += readBytes; + if (m_currentRegionOffset == m_streamHeaderLength) + { + m_currentRegion = StructuredMessageCurrentRegion::SegmentHeader; + m_currentRegionOffset = 0; + } + break; + } + case StructuredMessageCurrentRegion::SegmentHeader: { + size_t readBytes = std::min( + count - totalRead, + static_cast(m_segmentHeaderLength - m_currentRegionOffset)); + if (m_segmentHeaderCache.empty()) + { + m_segmentHeaderCache.resize(m_segmentHeaderLength); + m_segmentNumber += 1; + StructuredMessageHelper::WriteSegmentHeader( + m_segmentHeaderCache.data(), + m_segmentNumber, + std::min(m_options.MaxSegmentLength, m_inner->Length() - m_innerOffset)); + } + std::memcpy( + buffer + subreadOffset, + m_segmentHeaderCache.data() + m_currentRegionOffset, + readBytes); + m_offset += readBytes; + m_currentRegionOffset += readBytes; + totalRead += readBytes; + if (m_currentRegionOffset == m_segmentHeaderLength) + { + m_currentRegion = StructuredMessageCurrentRegion::SegmentContent; + m_currentRegionOffset = 0; + } + break; + } + case StructuredMessageCurrentRegion::SegmentContent: { + size_t readBytes = std::min( + count - totalRead, + static_cast(m_options.MaxSegmentLength - m_currentRegionOffset)); + auto bytesRead + = m_inner->Read(buffer + subreadOffset, static_cast(readBytes), context); + m_offset += bytesRead; + m_innerOffset += bytesRead; + m_currentRegionOffset += bytesRead; + totalRead += bytesRead; + if (m_options.Flags == StructuredMessageFlags::Crc64) + { + m_segmentCrc64Hash->Append(buffer + subreadOffset, bytesRead); + } + if (m_currentRegionOffset == static_cast(m_options.MaxSegmentLength) + || m_innerOffset >= m_inner->Length()) + { + m_currentRegion = StructuredMessageCurrentRegion::SegmentFooter; + m_currentRegionOffset = 0; + } + break; + } + case StructuredMessageCurrentRegion::SegmentFooter: { + if (m_options.Flags == StructuredMessageFlags::Crc64) + { + size_t readBytes = std::min( + count - totalRead, + static_cast(m_segmentFooterLength - m_currentRegionOffset)); + if (m_segmentFooterCache.empty()) + { + m_segmentFooterCache.resize(m_segmentFooterLength); + StructuredMessageHelper::WriteSegmentFooter( + m_segmentFooterCache.data(), m_segmentCrc64Hash->Final().data()); + } + std::memcpy( + buffer + subreadOffset, + m_segmentFooterCache.data() + m_currentRegionOffset, + readBytes); + m_offset += readBytes; + m_currentRegionOffset += readBytes; + totalRead += readBytes; + m_streamCrc64Hash->Concatenate(*m_segmentCrc64Hash); + m_segmentCrc64Hash = std::make_unique(); + } + + if (m_currentRegionOffset == m_segmentFooterLength) + { + m_currentRegion = m_innerOffset == m_inner->Length() + ? StructuredMessageCurrentRegion::StreamFooter + : StructuredMessageCurrentRegion::SegmentHeader; + m_currentRegionOffset = 0; + m_segmentHeaderCache.clear(); + m_segmentFooterCache.clear(); + } + break; + } + case StructuredMessageCurrentRegion::StreamFooter: { + if (m_options.Flags == StructuredMessageFlags::Crc64) + { + size_t readBytes = std::min( + count - totalRead, + static_cast(m_streamFooterLength - m_currentRegionOffset)); + if (readBytes <= 0) + { + return totalRead; + } + if (m_streamFooterCache.empty()) + { + m_streamFooterCache.resize(m_streamFooterLength); + StructuredMessageHelper::WriteStreamFooter( + m_streamFooterCache.data(), m_streamCrc64Hash->Final().data()); + } + std::memcpy( + buffer + subreadOffset, + m_streamFooterCache.data() + m_currentRegionOffset, + readBytes); + m_offset += readBytes; + m_currentRegionOffset += readBytes; + totalRead += readBytes; + } + break; + } + } + } + return totalRead; + } +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/structured_message_helper.cpp b/sdk/storage/azure-storage-common/src/structured_message_helper.cpp new file mode 100644 index 0000000000..71e76eb494 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/structured_message_helper.cpp @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/structured_message_helper.hpp" + +namespace Azure { namespace Storage { namespace _internal { + const size_t StructuredMessageHelper::Crc64Length = 8; + const uint8_t StructuredMessageHelper::StructuredMessageVersion = 1; + const size_t StructuredMessageHelper::StreamHeaderLength = 13; + const int64_t StructuredMessageHelper::StreamHeaderVersionOffset = 0; + const int64_t StructuredMessageHelper::StreamHeaderMessageLengthOffset = 1; + const int64_t StructuredMessageHelper::StreamHeaderFlagsOffset = 9; + const int64_t StructuredMessageHelper::StreamHeaderSegmentCountOffset = 11; + const size_t StructuredMessageHelper::SegmentHeaderLength = 10; + const int64_t StructuredMessageHelper::SegmentHeaderNumOffset = 0; + const int64_t StructuredMessageHelper::SegmentHeaderContentLengthOffset = 2; + + void StructuredMessageHelper::WriteStreamHeader( + uint8_t* buffer, + const uint64_t& messageLength, + const uint16_t& flags, + const uint16_t& segmentCount) + { + buffer[StructuredMessageHelper::StreamHeaderVersionOffset] + = StructuredMessageHelper::StructuredMessageVersion; + std::copy( + reinterpret_cast(&messageLength), + reinterpret_cast(&messageLength) + sizeof(uint64_t), + buffer + StructuredMessageHelper::StreamHeaderMessageLengthOffset); + std::copy( + reinterpret_cast(&flags), + reinterpret_cast(&flags) + sizeof(uint16_t), + buffer + StructuredMessageHelper::StreamHeaderFlagsOffset); + std::copy( + reinterpret_cast(&segmentCount), + reinterpret_cast(&segmentCount) + sizeof(uint16_t), + buffer + StructuredMessageHelper::StreamHeaderSegmentCountOffset); + } + + void StructuredMessageHelper::WriteSegmentHeader( + uint8_t* buffer, + const uint16_t& segmentNum, + const uint64_t& segmentLength) + { + std::copy( + reinterpret_cast(&segmentNum), + reinterpret_cast(&segmentNum) + sizeof(uint16_t), + buffer + StructuredMessageHelper::SegmentHeaderNumOffset); + std::copy( + reinterpret_cast(&segmentLength), + reinterpret_cast(&segmentLength) + sizeof(uint64_t), + buffer + StructuredMessageHelper::SegmentHeaderContentLengthOffset); + } + + void StructuredMessageHelper::WriteSegmentFooter(uint8_t* buffer, const uint8_t* crc64) + { + if (crc64 == nullptr) + { + return; + } + std::copy(crc64, crc64 + Crc64Length, buffer); + } + + void StructuredMessageHelper::WriteStreamFooter(uint8_t* buffer, const uint8_t* crc64) + { + if (crc64 == nullptr) + { + return; + } + std::copy(crc64, crc64 + Crc64Length, buffer); + } + + void StructuredMessageHelper::ReadStreamHeader( + const uint8_t* buffer, + uint64_t& messageLength, + StructuredMessageFlags& flags, + uint16_t& segmentCount) + { + messageLength = *reinterpret_cast( + buffer + StructuredMessageHelper::StreamHeaderMessageLengthOffset); + flags = static_cast(*reinterpret_cast( + buffer + StructuredMessageHelper::StreamHeaderFlagsOffset)); + segmentCount = *reinterpret_cast( + buffer + StructuredMessageHelper::StreamHeaderSegmentCountOffset); + } + + void StructuredMessageHelper::ReadSegmentHeader( + const uint8_t* buffer, + uint16_t& segmentNumber, + uint64_t& segmentLength) + { + segmentNumber = *reinterpret_cast( + buffer + StructuredMessageHelper::SegmentHeaderNumOffset); + segmentLength = *reinterpret_cast( + buffer + StructuredMessageHelper::SegmentHeaderContentLengthOffset); + } + + void StructuredMessageHelper::ReadSegmentFooter(const uint8_t* buffer, uint8_t* crc64) + { + if (buffer == nullptr) + { + return; + } + std::copy(buffer, buffer + Crc64Length, crc64); + } + + void StructuredMessageHelper::ReadStreamFooter(const uint8_t* buffer, uint8_t* crc64) + { + if (buffer == nullptr) + { + return; + } + std::copy(buffer, buffer + Crc64Length, crc64); + } +}}} // namespace Azure::Storage::_internal \ No newline at end of file diff --git a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt index 22b8ed9cb1..67415f3b51 100644 --- a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable ( crypt_functions_test.cpp metadata_test.cpp storage_credential_test.cpp + structured_message_test.cpp test_base.cpp test_base.hpp ) diff --git a/sdk/storage/azure-storage-common/test/ut/structured_message_test.cpp b/sdk/storage/azure-storage-common/test/ut/structured_message_test.cpp new file mode 100644 index 0000000000..8fb583b3aa --- /dev/null +++ b/sdk/storage/azure-storage-common/test/ut/structured_message_test.cpp @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "test_base.hpp" + +#include +#include + +namespace Azure { namespace Storage { namespace Test { + + class StructuredMessageTest : public StorageTest { + }; + + std::vector ReadToEnd(Azure::Core::IO::BodyStream& stream, const size_t chunkSize) + { + auto buffer = std::vector(); + + for (auto chunkNumber = 0;; chunkNumber++) + { + buffer.resize((static_cast(chunkNumber) + 1) * chunkSize); + size_t readBytes = stream.ReadToCount(buffer.data() + (chunkNumber * chunkSize), chunkSize); + + if (readBytes < chunkSize) + { + buffer.resize(static_cast((chunkNumber * chunkSize) + readBytes)); + return buffer; + } + } + } + + TEST_F(StructuredMessageTest, BasicFunction) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::Crc64; + encodingOptions.MaxSegmentLength = 1024; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + auto encodedData = encodingStream.ReadToEnd(); + + innerStream = std::make_unique( + encodedData.data(), encodedData.size()); + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + _internal::StructuredMessageDecodingStream decodingStream( + std::move(innerStream), decodingOptions); + auto decodedData = decodingStream.ReadToEnd(); + + EXPECT_EQ(content, decodedData); + } + + TEST_F(StructuredMessageTest, SmallSegment) + { + const size_t contentSize = 2 * 1024 * 1024 + 5122; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::Crc64; + encodingOptions.MaxSegmentLength = 33; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + auto encodedData = ReadToEnd(encodingStream, 4096); + + innerStream = std::make_unique( + encodedData.data(), encodedData.size()); + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + _internal::StructuredMessageDecodingStream decodingStream( + std::move(innerStream), decodingOptions); + auto decodedData = ReadToEnd(decodingStream, 513); + + EXPECT_EQ(content, decodedData); + } + + TEST_F(StructuredMessageTest, ReadSmallRange) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::Crc64; + encodingOptions.MaxSegmentLength = 1024; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + std::vector encodedData = ReadToEnd(encodingStream, 7); + + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + _internal::StructuredMessageDecodingStream decodingStream( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + auto decodedData = ReadToEnd(decodingStream, 7); + + EXPECT_EQ(content, decodedData); + + // Large encode range + encodingStream.Rewind(); + encodedData = ReadToEnd(encodingStream, 4096); + _internal::StructuredMessageDecodingStream decodingStream1( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + decodedData = ReadToEnd(decodingStream1, 5); + + EXPECT_EQ(content, decodedData); + + decodingStream1.Rewind(); + decodedData = ReadToEnd(decodingStream1, 6); + + EXPECT_EQ(content, decodedData); + + // Large decode range + encodingStream.Rewind(); + encodedData = ReadToEnd(encodingStream, 8); + _internal::StructuredMessageDecodingStream decodingStream2( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + decodedData = ReadToEnd(decodingStream2, 4096); + + EXPECT_EQ(content, decodedData); + } + + TEST_F(StructuredMessageTest, ReadBigRange) + { + const size_t contentSize = 4 * 1024 * 1024 + 2 * 1024 + 512 + 3; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::Crc64; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + std::vector encodedData = ReadToEnd(encodingStream, 4 * 1024 * 1024); + + innerStream = std::make_unique( + encodedData.data(), encodedData.size()); + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + _internal::StructuredMessageDecodingStream decodingStream( + std::move(innerStream), decodingOptions); + auto decodedData = ReadToEnd(decodingStream, 4 * 1024 * 1024); + + EXPECT_EQ(content, decodedData); + } + + TEST_F(StructuredMessageTest, NotCrc64) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::None; + encodingOptions.MaxSegmentLength = 1024; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + std::vector encodedData = ReadToEnd(encodingStream, 4096); + + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + innerStream = std::make_unique( + encodedData.data(), encodedData.size()); + _internal::StructuredMessageDecodingStream decodingStream( + std::move(innerStream), decodingOptions); + auto decodedData = ReadToEnd(decodingStream, 4096); + + EXPECT_EQ(content, decodedData); + } + + TEST_F(StructuredMessageTest, NotCrc64SmallRange) + { + const size_t contentSize = 2 * 1024 + 512; + auto content = RandomBuffer(contentSize); + auto innerStream + = std::make_unique(content.data(), content.size()); + + _internal::StructuredMessageEncodingStreamOptions encodingOptions; + encodingOptions.Flags = _internal::StructuredMessageFlags::None; + encodingOptions.MaxSegmentLength = 1024; + _internal::StructuredMessageEncodingStream encodingStream(innerStream.get(), encodingOptions); + std::vector encodedData = ReadToEnd(encodingStream, 7); + + _internal::StructuredMessageDecodingStreamOptions decodingOptions; + decodingOptions.ContentLength = contentSize; + _internal::StructuredMessageDecodingStream decodingStream( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + auto decodedData = ReadToEnd(decodingStream, 7); + + EXPECT_EQ(content, decodedData); + + // Large encode range + encodingStream.Rewind(); + encodedData = ReadToEnd(encodingStream, 4096); + _internal::StructuredMessageDecodingStream decodingStream1( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + decodedData = ReadToEnd(decodingStream1, 5); + + EXPECT_EQ(content, decodedData); + + decodingStream1.Rewind(); + decodedData = ReadToEnd(decodingStream1, 6); + + EXPECT_EQ(content, decodedData); + + // Large decode range + encodingStream.Rewind(); + encodedData = ReadToEnd(encodingStream, 8); + _internal::StructuredMessageDecodingStream decodingStream2( + std::make_unique(encodedData.data(), encodedData.size()), + decodingOptions); + decodedData = ReadToEnd(decodingStream2, 4096); + + EXPECT_EQ(content, decodedData); + } + +}}} // namespace Azure::Storage::Test