Storage STG99/Content Validation - Main Function and Blob Support

sdk/storage/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "cpp",
   "TagPrefix": "cpp/storage",
-  "Tag": "cpp/storage_57a579efb6"
+  "Tag": "cpp/storage_532bd2d370"
 }

sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp

@@ -434,6 +434,10 @@ namespace Azure { namespace Storage { namespace Blobs {
     Azure::Nullable<EncryptionKey> m_customerProvidedKey;
     /** @brief Encryption scope. */
     Azure::Nullable<std::string> m_encryptionScope;
+    /** @brief Upload TransferValidationOptions */
+    Azure::Nullable<TransferValidationOptions> m_uploadValidationOptions;
+    /** @brief Download TransferValidationOptions */
+    Azure::Nullable<TransferValidationOptions> m_downloadValidationOptions;
 
   private:
     explicit BlobClient(

sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp

@@ -168,6 +168,17 @@ namespace Azure { namespace Storage { namespace Blobs {
     Models::EncryptionAlgorithmType Algorithm;
   };
 
+  /**
+   * Configures whether to do content validation for blob uploads and downloads.
+   */
+  struct TransferValidationOptions
+  {
+    /**
+     * @brief The algorithm used for storage checksum.
+     */
+    StorageChecksumAlgorithm Algorithm = StorageChecksumAlgorithm::None;
+  };
+
   /**
    * @brief Client options used to initialize all kinds of blob clients.
    */
@@ -210,6 +221,16 @@ namespace Azure { namespace Storage { namespace Blobs {
      * not set.
      */
     Azure::Nullable<BlobAudience> Audience;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> UploadValidationOptions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob downloads.
+     */
+    Azure::Nullable<TransferValidationOptions> DownloadValidationOptions;
   };
 
   /**
@@ -673,6 +694,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * @brief Optional conditions that must be met to perform this operation.
      */
     BlobAccessConditions AccessConditions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob downloads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -707,6 +733,11 @@ namespace Azure { namespace Storage { namespace Blobs {
        */
       int32_t Concurrency = 5;
     } TransferOptions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob downloads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -889,6 +920,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * Indicates whether the blob has a legal hold.
      */
     Azure::Nullable<bool> HasLegalHold;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -948,6 +984,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * Indicates whether the blob has a legal hold.
      */
     Azure::Nullable<bool> HasLegalHold;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -1035,6 +1076,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * @brief Optional conditions that must be met to perform this operation.
      */
     LeaseAccessConditions AccessConditions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -1335,6 +1381,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * @brief Optional conditions that must be met to perform this operation.
      */
     AppendBlobAccessConditions AccessConditions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -1444,6 +1495,11 @@ namespace Azure { namespace Storage { namespace Blobs {
      * @brief Optional conditions that must be met to perform this operation.
      */
     PageBlobAccessConditions AccessConditions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for blob uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**

sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp

@@ -32,7 +32,7 @@ namespace Azure { namespace Storage { namespace Blobs {
     /**
      * The version used for the operations to Azure storage services.
      */
-    constexpr static const char* ApiVersion = "2024-08-04";
+    constexpr static const char* ApiVersion = "2025-01-05";
   } // namespace _detail
   namespace Models {
     /**
@@ -1612,6 +1612,16 @@ namespace Azure { namespace Storage { namespace Blobs {
        * The blob's type.
        */
       Models::BlobType BlobType;
+      /**
+       * Indicates the response body contains a structured message and specifies the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
+      /**
+       * The length of the blob/file content inside the message body when the response body is
+       * returned as a structured message. Will always be smaller than Content-Length.
+       */
+      Nullable<std::int64_t> StructuredContentLength;
     };
     /**
      * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetProperties.
@@ -2576,6 +2586,11 @@ namespace Azure { namespace Storage { namespace Blobs {
        * encryption scope.
        */
       Nullable<std::string> EncryptionScope;
+      /**
+       * Indicates the structured message body was accepted and mirrors back the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
     };
     /**
      * @brief Response type for #Azure::Storage::Blobs::PageBlobClient::ClearPages.
@@ -2872,6 +2887,11 @@ namespace Azure { namespace Storage { namespace Blobs {
        * encryption scope.
        */
       Nullable<std::string> EncryptionScope;
+      /**
+       * Indicates the structured message body was accepted and mirrors back the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
     };
     /**
      * @brief Response type for #Azure::Storage::Blobs::AppendBlobClient::AppendBlockFromUri.
@@ -2985,6 +3005,11 @@ namespace Azure { namespace Storage { namespace Blobs {
        * encryption scope.
        */
       Nullable<std::string> EncryptionScope;
+      /**
+       * Indicates the structured message body was accepted and mirrors back the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
     };
     /**
      * @brief Response type for #Azure::Storage::Blobs::BlockBlobClient::UploadFromUri.
@@ -3057,6 +3082,11 @@ namespace Azure { namespace Storage { namespace Blobs {
        * encryption scope.
        */
       Nullable<std::string> EncryptionScope;
+      /**
+       * Indicates the structured message body was accepted and mirrors back the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
     };
     /**
      * @brief Response type for #Azure::Storage::Blobs::BlockBlobClient::StageBlockFromUri.
@@ -3499,6 +3529,7 @@ namespace Azure { namespace Storage { namespace Blobs {
         Nullable<std::string> LeaseId;
         Nullable<bool> RangeGetContentMD5;
         Nullable<bool> RangeGetContentCRC64;
+        Nullable<std::string> StructuredBodyType;
         Nullable<std::string> EncryptionKey;
         Nullable<std::vector<std::uint8_t>> EncryptionKeySha256;
         Nullable<std::string> EncryptionAlgorithm;
@@ -3917,6 +3948,8 @@ namespace Azure { namespace Storage { namespace Blobs {
         ETag IfMatch;
         ETag IfNoneMatch;
         Nullable<std::string> IfTags;
+        Nullable<std::string> StructuredBodyType;
+        Nullable<std::int64_t> StructuredContentLength;
       };
       static Response<Models::UploadPagesResult> UploadPages(
           Core::Http::_internal::HttpPipeline& pipeline,
@@ -4112,6 +4145,8 @@ namespace Azure { namespace Storage { namespace Blobs {
         ETag IfMatch;
         ETag IfNoneMatch;
         Nullable<std::string> IfTags;
+        Nullable<std::string> StructuredBodyType;
+        Nullable<std::int64_t> StructuredContentLength;
       };
       static Response<Models::AppendBlockResult> AppendBlock(
           Core::Http::_internal::HttpPipeline& pipeline,
@@ -4192,6 +4227,8 @@ namespace Azure { namespace Storage { namespace Blobs {
         Nullable<Models::BlobImmutabilityPolicyMode> ImmutabilityPolicyMode;
         Nullable<bool> LegalHold;
         Nullable<std::vector<std::uint8_t>> TransactionalContentCrc64;
+        Nullable<std::string> StructuredBodyType;
+        Nullable<std::int64_t> StructuredContentLength;
       };
       static Response<Models::UploadBlockBlobResult> Upload(
           Core::Http::_internal::HttpPipeline& pipeline,
@@ -4247,6 +4284,8 @@ namespace Azure { namespace Storage { namespace Blobs {
         Nullable<std::vector<std::uint8_t>> EncryptionKeySha256;
         Nullable<std::string> EncryptionAlgorithm;
         Nullable<std::string> EncryptionScope;
+        Nullable<std::string> StructuredBodyType;
+        Nullable<std::int64_t> StructuredContentLength;
       };
       static Response<Models::StageBlockResult> StageBlock(
           Core::Http::_internal::HttpPipeline& pipeline,

sdk/storage/azure-storage-blobs/src/append_blob_client.cpp

@@ -5,6 +5,7 @@
 
 #include <azure/storage/common/crypt.hpp>
 #include <azure/storage/common/internal/constants.hpp>
+#include <azure/storage/common/internal/structured_message_encoding_stream.hpp>
 #include <azure/storage/common/storage_common.hpp>
 #include <azure/storage/common/storage_exception.hpp>
 
@@ -141,19 +142,6 @@ namespace Azure { namespace Storage { namespace Blobs {
       const Azure::Core::Context& context) const
   {
     _detail::AppendBlobClient::AppendAppendBlobBlockOptions protocolLayerOptions;
-    if (options.TransactionalContentHash.HasValue())
-    {
-      if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5)
-      {
-        protocolLayerOptions.TransactionalContentMD5
-            = options.TransactionalContentHash.Value().Value;
-      }
-      else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64)
-      {
-        protocolLayerOptions.TransactionalContentCrc64
-            = options.TransactionalContentHash.Value().Value;
-      }
-    }
     protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
     protocolLayerOptions.MaxSize = options.AccessConditions.IfMaxSizeLessThanOrEqual;
     protocolLayerOptions.AppendPosition = options.AccessConditions.IfAppendPositionEqual;
@@ -169,6 +157,37 @@ namespace Azure { namespace Storage { namespace Blobs {
       protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
     }
     protocolLayerOptions.EncryptionScope = m_encryptionScope;
+    if (options.TransactionalContentHash.HasValue())
+    {
+      if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5)
+      {
+        protocolLayerOptions.TransactionalContentMD5
+            = options.TransactionalContentHash.Value().Value;
+      }
+      else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64)
+      {
+        protocolLayerOptions.TransactionalContentCrc64
+            = options.TransactionalContentHash.Value().Value;
+      }
+    }
+    else
+    {
+      Azure::Nullable<TransferValidationOptions> validationOptions
+          = options.ValidationOptions.HasValue() ? options.ValidationOptions
+                                                 : m_uploadValidationOptions;
+      if (validationOptions.HasValue()
+          && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None)
+      {
+        protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage;
+        protocolLayerOptions.StructuredContentLength = content.Length();
+        _internal::StructuredMessageEncodingStreamOptions encodingStreamOptions;
+        encodingStreamOptions.Flags = _internal::StructuredMessageFlags::Crc64;
+        auto structuredContent
+            = _internal::StructuredMessageEncodingStream(&content, encodingStreamOptions);
+        return _detail::AppendBlobClient::AppendBlock(
+            *m_pipeline, m_blobUrl, structuredContent, protocolLayerOptions, context);
+      }
+    }
     return _detail::AppendBlobClient::AppendBlock(
         *m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
   }

sdk/storage/azure-storage-blobs/src/blob_client.cpp

@@ -20,6 +20,7 @@
 #include <azure/storage/common/internal/storage_per_retry_policy.hpp>
 #include <azure/storage/common/internal/storage_service_version_policy.hpp>
 #include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
+#include <azure/storage/common/internal/structured_message_decoding_stream.hpp>
 #include <azure/storage/common/storage_common.hpp>
 #include <azure/storage/common/storage_exception.hpp>
 
@@ -110,7 +111,9 @@ namespace Azure { namespace Storage { namespace Blobs {
 
   BlobClient::BlobClient(const std::string& blobUrl, const BlobClientOptions& options)
       : m_blobUrl(blobUrl), m_customerProvidedKey(options.CustomerProvidedKey),
-        m_encryptionScope(options.EncryptionScope)
+        m_encryptionScope(options.EncryptionScope),
+        m_uploadValidationOptions(options.UploadValidationOptions),
+        m_downloadValidationOptions(options.DownloadValidationOptions)
   {
     std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
     std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
@@ -167,6 +170,7 @@ namespace Azure { namespace Storage { namespace Blobs {
       const DownloadBlobOptions& options,
       const Azure::Core::Context& context) const
   {
+    bool isStructuredMessage = false;
     _detail::BlobClient::DownloadBlobOptions protocolLayerOptions;
     if (options.Range.HasValue())
     {
@@ -189,6 +193,18 @@ namespace Azure { namespace Storage { namespace Blobs {
         protocolLayerOptions.RangeGetContentCRC64 = true;
       }
     }
+    else
+    {
+      Azure::Nullable<TransferValidationOptions> validationOptions
+          = options.ValidationOptions.HasValue() ? options.ValidationOptions
+                                                 : m_downloadValidationOptions;
+      if (validationOptions.HasValue()
+          && validationOptions.Value().Algorithm != StorageChecksumAlgorithm::None)
+      {
+        isStructuredMessage = true;
+        protocolLayerOptions.StructuredBodyType = _internal::CrcStructuredMessage;
+      }
+    }
     protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
     protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
     protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
@@ -243,8 +259,23 @@ namespace Azure { namespace Storage { namespace Blobs {
 
       _internal::ReliableStreamOptions reliableStreamOptions;
       reliableStreamOptions.MaxRetryRequests = _internal::ReliableStreamRetryCount;
-      downloadResponse.Value.BodyStream = std::make_unique<_internal::ReliableStream>(
+      auto reliableStream = std::make_unique<_internal::ReliableStream>(
           std::move(downloadResponse.Value.BodyStream), reliableStreamOptions, retryFunction);
+      if (isStructuredMessage)
+      {
+        _internal::StructuredMessageDecodingStreamOptions decodingOptions;
+        if (downloadResponse.Value.StructuredContentLength.HasValue())
+        {
+          decodingOptions.ContentLength = downloadResponse.Value.StructuredContentLength.Value();
+        }
+        downloadResponse.Value.BodyStream
+            = std::make_unique<_internal::StructuredMessageDecodingStream>(
+                std::move(reliableStream), decodingOptions);
+      }
+      else
+      {
+        downloadResponse.Value.BodyStream = std::move(reliableStream);
+      }
     }
     if (downloadResponse.RawResponse->GetStatusCode() == Azure::Core::Http::HttpStatusCode::Ok)
     {