Storage/Content Validation Part2

sdk/storage/azure-storage-blobs/src/blob_client.cpp

@@ -279,8 +279,20 @@ namespace Azure { namespace Storage { namespace Blobs {
     }
     if (downloadResponse.RawResponse->GetStatusCode() == Azure::Core::Http::HttpStatusCode::Ok)
     {
-      downloadResponse.Value.BlobSize = std::stoll(
-          downloadResponse.RawResponse->GetHeaders().at(_internal::HttpHeaderContentLength));
+      if (isStructuredMessage)
+      {
+        if (!downloadResponse.Value.StructuredContentLength.HasValue())
+        {
+          throw StorageException(
+              "Structured message response without x-ms-structured-content-length header.");
+        }
+        downloadResponse.Value.BlobSize = downloadResponse.Value.StructuredContentLength.Value();
+      }
+      else
+      {
+        downloadResponse.Value.BlobSize = std::stoll(
+            downloadResponse.RawResponse->GetHeaders().at(_internal::HttpHeaderContentLength));
+      }
       downloadResponse.Value.ContentRange.Offset = 0;
       downloadResponse.Value.ContentRange.Length = downloadResponse.Value.BlobSize;
     }
@@ -366,6 +378,7 @@ namespace Azure { namespace Storage { namespace Blobs {
     {
       firstChunkOptions.Range.Value().Length = firstChunkLength;
     }
+    firstChunkOptions.ValidationOptions = options.ValidationOptions;
 
     auto firstChunk = Download(firstChunkOptions, context);
     const Azure::ETag eTag = firstChunk.Value.Details.ETag;
@@ -421,6 +434,7 @@ namespace Azure { namespace Storage { namespace Blobs {
             chunkOptions.Range.Value().Offset = offset;
             chunkOptions.Range.Value().Length = length;
             chunkOptions.AccessConditions.IfMatch = eTag;
+            chunkOptions.ValidationOptions = options.ValidationOptions;
             auto chunk = Download(chunkOptions, context);
             int64_t bytesRead = chunk.Value.BodyStream->ReadToCount(
                 buffer + (offset - firstChunkOffset),
@@ -473,6 +487,7 @@ namespace Azure { namespace Storage { namespace Blobs {
     {
       firstChunkOptions.Range.Value().Length = firstChunkLength;
     }
+    firstChunkOptions.ValidationOptions = options.ValidationOptions;
 
     auto firstChunk = Download(firstChunkOptions, context);
     const Azure::ETag eTag = firstChunk.Value.Details.ETag;
@@ -538,6 +553,7 @@ namespace Azure { namespace Storage { namespace Blobs {
             chunkOptions.Range.Value().Offset = offset;
             chunkOptions.Range.Value().Length = length;
             chunkOptions.AccessConditions.IfMatch = eTag;
+            chunkOptions.ValidationOptions = options.ValidationOptions;
             auto chunk = Download(chunkOptions, context);
             bodyStreamToFile(
                 *(chunk.Value.BodyStream),

sdk/storage/azure-storage-blobs/src/block_blob_client.cpp

@@ -224,6 +224,7 @@ namespace Azure { namespace Storage { namespace Blobs {
     auto uploadBlockFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) {
       Azure::Core::IO::MemoryBodyStream contentStream(buffer + offset, static_cast<size_t>(length));
       StageBlockOptions chunkOptions;
+      chunkOptions.ValidationOptions = options.ValidationOptions;
       auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
       if (chunkId == numChunks - 1)
       {
@@ -299,6 +300,7 @@ namespace Azure { namespace Storage { namespace Blobs {
       Azure::Core::IO::_internal::RandomAccessFileBodyStream contentStream(
           fileReader.GetHandle(), offset, length);
       StageBlockOptions chunkOptions;
+      chunkOptions.ValidationOptions = options.ValidationOptions;
       auto blockInfo = StageBlock(getBlockId(chunkId), contentStream, chunkOptions, context);
       if (chunkId == numChunks - 1)
       {

sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp

@@ -2198,11 +2198,13 @@ namespace Azure { namespace Storage { namespace Test {
     EXPECT_TRUE(blobProperties.HasLegalHold);
   }
 
-  TEST_F(BlockBlobClientTest, StructuredMessageTest_PLAYBACKONLY_)
+  TEST_F(BlockBlobClientTest, DISABLED_StructuredMessageTest)
   {
     const size_t contentSize = 2 * 1024 + 512;
     auto content = RandomBuffer(contentSize);
     auto bodyStream = Azure::Core::IO::MemoryBodyStream(content.data(), content.size());
+    const std::string tempFileName = RandomString();
+    WriteFile(tempFileName, content);
     Blobs::TransferValidationOptions validationOptions;
     validationOptions.Algorithm = StorageChecksumAlgorithm::Crc64;
 
@@ -2222,9 +2224,61 @@ namespace Azure { namespace Storage { namespace Test {
     EXPECT_TRUE(downloadResult.StructuredContentLength.HasValue());
     EXPECT_EQ(downloadResult.StructuredContentLength.Value(), contentSize);
     EXPECT_TRUE(downloadResult.StructuredBodyType.HasValue());
+    EXPECT_EQ(downloadResult.BlobSize, contentSize);
+    // partial download
+    downloadOptions.Range = Core::Http::HttpRange();
+    downloadOptions.Range.Value().Length = contentSize / 2;
+    EXPECT_NO_THROW(downloadResult = m_blockBlobClient->Download(downloadOptions).Value);
+    downloadedData = downloadResult.BodyStream->ReadToEnd();
+    EXPECT_EQ(
+        downloadedData, std::vector<uint8_t>(content.begin(), content.begin() + contentSize / 2));
+    EXPECT_TRUE(downloadResult.StructuredContentLength.HasValue());
+    EXPECT_EQ(downloadResult.StructuredContentLength.Value(), contentSize / 2);
+    EXPECT_TRUE(downloadResult.StructuredBodyType.HasValue());
+    EXPECT_EQ(downloadResult.BlobSize, contentSize);
+    downloadOptions.Range.Reset();
+
+    // UploadFrom DownloadTo
+    Blobs::UploadBlockBlobFromOptions uploadFromOptions;
+    Blobs::Models::UploadBlockBlobFromResult uploadFromResult;
+    Blobs::DownloadBlobToOptions downloadToOptions;
+    Blobs::Models::DownloadBlobToResult downloadToResult;
+
+    // Stream
+    uploadFromOptions.ValidationOptions = validationOptions;
+    auto blobClient
+        = m_blobContainerClient->GetBlockBlobClient("uploadfromstream_" + LowercaseRandomString());
+    EXPECT_NO_THROW(
+        uploadFromResult
+        = blobClient.UploadFrom(content.data(), contentSize, uploadFromOptions).Value);
+    downloadToOptions.ValidationOptions = validationOptions;
+    auto downloadBuffer = std::vector<uint8_t>(contentSize, '\x00');
+    EXPECT_NO_THROW(
+        downloadToResult
+        = blobClient.DownloadTo(downloadBuffer.data(), contentSize, downloadToOptions).Value);
+    EXPECT_EQ(downloadBuffer, content);
+    // partial downloadTo
+    downloadToOptions.Range = Core::Http::HttpRange();
+    downloadToOptions.Range.Value().Length = contentSize / 2;
+    downloadBuffer.resize(static_cast<size_t>(contentSize / 2), '\x00');
+    EXPECT_NO_THROW(
+        downloadToResult
+        = blobClient.DownloadTo(downloadBuffer.data(), contentSize / 2, downloadToOptions).Value);
+    EXPECT_EQ(
+        downloadBuffer, std::vector<uint8_t>(content.begin(), content.begin() + contentSize / 2));
+    downloadToOptions.Range.Reset();
+
+    // File
+    blobClient
+        = m_blobContainerClient->GetBlockBlobClient("uploadfromfile_" + LowercaseRandomString());
+    EXPECT_NO_THROW(blobClient.UploadFrom(tempFileName, uploadFromOptions));
+    std::string downloadToFileName = RandomString();
+    EXPECT_NO_THROW(
+        downloadToResult = blobClient.DownloadTo(downloadToFileName, downloadToOptions).Value);
+    EXPECT_EQ(ReadFile(downloadToFileName), content);
 
     // Stage Block
-    auto blobClient = m_blobContainerClient->GetBlockBlobClient(LowercaseRandomString());
+    blobClient = m_blobContainerClient->GetBlockBlobClient(LowercaseRandomString());
     const std::vector<uint8_t> dataPart1 = RandomBuffer(contentSize);
     const std::vector<uint8_t> dataPart2 = RandomBuffer(contentSize);
 

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp

@@ -295,6 +295,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     {
     }
 
+    /** @brief Upload TransferValidationOptions */
+    Azure::Nullable<TransferValidationOptions> m_uploadValidationOptions;
+    /** @brief Download TransferValidationOptions */
+    Azure::Nullable<TransferValidationOptions> m_downloadValidationOptions;
+
     friend class DataLakeFileSystemClient;
     friend class DataLakeDirectoryClient;
   };

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp

@@ -153,6 +153,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     AZ_STORAGE_FILES_DATALAKE_DLLEXPORT const static DataLakeAudience DefaultAudience;
   };
 
+  /**
+   * Configures whether to do content validation for file uploads and downloads.
+   */
+  struct TransferValidationOptions
+  {
+    /**
+     * @brief The algorithm used for storage checksum.
+     */
+    StorageChecksumAlgorithm Algorithm = StorageChecksumAlgorithm::None;
+  };
+
   /**
    * @brief Client options used to initialize all DataLake clients.
    */
@@ -190,6 +201,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * if Audience is not set.
      */
     Azure::Nullable<DataLakeAudience> Audience;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for file uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> UploadValidationOptions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for file downloads.
+     */
+    Azure::Nullable<TransferValidationOptions> DownloadValidationOptions;
   };
 
   /**
@@ -481,6 +502,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * be changed using renew or change.
      */
     Azure::Nullable<std::chrono::seconds> LeaseDuration;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for file appends.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -837,6 +863,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * https://learn.microsoft.com/entra/identity/hybrid/connect/plan-connect-userprincipalname#what-is-userprincipalname
      */
     Nullable<bool> IncludeUserPrincipalName;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for file downloads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   /**
@@ -963,6 +994,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        */
       int32_t Concurrency = 5;
     } TransferOptions;
+
+    /**
+     * @brief Optional. Configures whether to do content validation for file uploads.
+     */
+    Azure::Nullable<TransferValidationOptions> ValidationOptions;
   };
 
   using AcquireLeaseOptions = Blobs::AcquireLeaseOptions;

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp

@@ -731,6 +731,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        * The permissions of the file.
        */
       Azure::Nullable<std::string> Permissions;
+
+      /**
+       * Indicates the response body contains a structured message and specifies the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
+
+      /**
+       * The length of the blob/file content inside the message body when the response body is
+       * returned as a structured message. Will always be smaller than Content-Length.
+       */
+      Nullable<std::int64_t> StructuredContentLength;
     };
 
     /**
@@ -758,6 +770,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        */
       Azure::Nullable<Storage::ContentHash> TransactionalContentHash;
 
+      /**
+       * Indicates the response body contains a structured message and specifies the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
+      /**
+       * The length of the blob/file content inside the message body when the response body is
+       * returned as a structured message. Will always be smaller than Content-Length.
+       */
+      Nullable<std::int64_t> StructuredContentLength;
+
       /**
        * The detailed information of the downloaded file.
        */

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp

@@ -27,7 +27,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     /**
      * The version used for the operations to Azure storage services.
      */
-    constexpr static const char* ApiVersion = "2024-08-04";
+    constexpr static const char* ApiVersion = "2025-01-05";
   } // namespace _detail
   namespace Models {
     namespace _detail {
@@ -354,6 +354,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        * If the lease was auto-renewed with this request.
        */
       Nullable<bool> IsLeaseRenewed;
+      /**
+       * Indicates the structured message body was accepted and mirrors back the message schema
+       * version and properties.
+       */
+      Nullable<std::string> StructuredBodyType;
     };
   } // namespace Models
   namespace _detail {
@@ -536,6 +541,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         Nullable<std::vector<std::uint8_t>> EncryptionKeySha256;
         Nullable<std::string> EncryptionAlgorithm;
         Nullable<bool> Flush;
+        Nullable<std::string> StructuredBodyType;
+        Nullable<std::int64_t> StructuredContentLength;
       };
       static Response<Models::AppendFileResult> Append(
           Core::Http::_internal::HttpPipeline& pipeline,