diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 3ca4d85de9..4e66c0eef0 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -4,6 +4,12 @@ ### Features Added +- Bumped up API version to `2020-10-02`. +- Added new API: `BlockBlobClient::Query()`. +- Added `ContinuationToken` and `PageSizeHint` in `GetPageRangesOptions`. +- Added support for listing system containers. +- Added support for listing deleted root blob with active versions. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 209d582507..e005b3df5d 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -53,6 +53,7 @@ set( inc/azure/storage/blobs/page_blob_client.hpp inc/azure/storage/blobs/rest_client.hpp inc/azure/storage/blobs.hpp + src/private/avro_parser.hpp src/private/package_version.hpp ) @@ -62,11 +63,13 @@ set( src/blob_client.cpp src/blob_container_client.cpp src/blob_lease_client.cpp + src/blob_options.cpp src/blob_responses.cpp src/blob_sas_builder.cpp src/blob_service_client.cpp src/block_blob_client.cpp src/page_blob_client.cpp + src/private/avro_parser.cpp src/rest_client.cpp ) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 944097626c..25dc5ae5aa 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -1042,6 +1042,154 @@ namespace Azure { namespace Storage { namespace Blobs { } AccessConditions; }; + /** + * @brief Blob Query text configuration for input. + */ + class BlobQueryInputTextOptions final { + public: + /** + * @brief Creates CSV text configuration. + * + * @param recordSeparator Record separator. + * @param columnSeparator Column sepeartor. + * @param quotationCharacter Field quote. + * @param escapeCharacter Escape character. + * @param hasHeaders If CSV file has headers. + * @return CSV text configuration. + */ + static BlobQueryInputTextOptions CreateCsvTextOptions( + const std::string& recordSeparator = std::string(), + const std::string& columnSeparator = std::string(), + const std::string& quotationCharacter = std::string(), + const std::string& escapeCharacter = std::string(), + bool hasHeaders = false); + /** + * @brief Creates Json text configuration. + * + * @param recordSeparator Record separator. + * @return Json text configuration. + */ + static BlobQueryInputTextOptions CreateJsonTextOptions( + const std::string& recordSeparator = std::string()); + /** + * @brief Creates Parquet text configuration. + * + * @return Parquet text configuration + */ + static BlobQueryInputTextOptions CreateParquetTextOptions(); + + private: + Models::_detail::QueryFormatType m_format; + std::string m_recordSeparator; + std::string m_columnSeparator; + std::string m_quotationCharacter; + std::string m_escapeCharacter; + bool m_hasHeaders = false; + + friend class BlockBlobClient; + }; + + /** + * @brief Blob Query text configuration for output. + */ + class BlobQueryOutputTextOptions final { + public: + /** + * @brief Creates CSV text configuration. + * + * @param recordSeparator Record separator. + * @param columnSeparator Column sepeartor. + * @param quotationCharacter Field quote. + * @param escapeCharacter Escape character. + * @param hasHeaders If CSV file has headers. + * @return CSV text configuration. + */ + static BlobQueryOutputTextOptions CreateCsvTextOptions( + const std::string& recordSeparator = std::string(), + const std::string& columnSeparator = std::string(), + const std::string& quotationCharacter = std::string(), + const std::string& escapeCharacter = std::string(), + bool hasHeaders = false); + /** + * @brief Creates Json text configuration. + * + * @param recordSeparator Record separator. + * @return Json text configuration. + */ + static BlobQueryOutputTextOptions CreateJsonTextOptions( + const std::string& recordSeparator = std::string()); + /** + * @brief Creates Arrow text configuration. + * + * @param schema A list of fields describing the schema. + * @return Arrow text configuration. + */ + static BlobQueryOutputTextOptions CreateArrowTextOptions( + std::vector schema); + + private: + Models::_detail::QueryFormatType m_format; + std::string m_recordSeparator; + std::string m_columnSeparator; + std::string m_quotationCharacter; + std::string m_escapeCharacter; + bool m_hasHeaders = false; + std::vector m_schema; + + friend class BlockBlobClient; + }; + + /** + * @brief Blob Query Error. + */ + struct BlobQueryError final + { + /** + * @brief Error name. + */ + std::string Name; + /** + * @brief Error description. + */ + std::string Description; + /** + * @brief If the error is a fatal error. + */ + bool IsFatal = false; + /** + * The position of the error.. + */ + int64_t Position; + }; + + /** + * @brief Optional parameters for #Azure::Storage::Blobs::BlockBlobClient::Query. + */ + struct QueryBlobOptions final + { + /** + * @brief Input text configuration. + */ + BlobQueryInputTextOptions InputTextConfiguration; + /** + * @brief Output text configuration. + */ + BlobQueryOutputTextOptions OutputTextConfiguration; + /** + * @brief Optional conditions that must be met to perform this operation. + */ + BlobAccessConditions AccessConditions; + /** + * @brief Callback for progress handling. + */ + std::function ProgressHandler; + /** + * @brief Callback for error handling. If you don't specify one, the default will be used, which + * will ignore all non-fatal errors and throw for fatal errors. + */ + std::function ErrorHandler; + }; + /** * @brief Optional parameters for #Azure::Storage::Blobs::AppendBlobClient::Create. */ @@ -1275,6 +1423,22 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ BlobAccessConditions AccessConditions; + + /** + * @brief This parameter identifies the portion of the ranges to be returned with the next + * operation. The operation returns a marker value within the response body if the ranges + * returned were not complete. The marker value may then be used in a subsequent call to request + * the next set of ranges.This value is opaque to the client. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief This parameter specifies the maximum number of page ranges to return. If the request + * specifies a value greater than 10000, the server will return up to 10000 items. If there are + * additional results to return, the service returns a continuation token in the NextMarker + * response element. + */ + Azure::Nullable PageSizeHint; }; /** diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp index 1b948dd77a..33b7b39d1b 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp @@ -241,6 +241,19 @@ namespace Azure { namespace Storage { namespace Blobs { const GetBlockListOptions& options = GetBlockListOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Returns the result of a query against the blob. + * + * @param querySqlExpression The query expression in SQL. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A QueryBlobResult describing the query result. + */ + Azure::Response Query( + const std::string& querySqlExpression, + const QueryBlobOptions& options = QueryBlobOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: explicit BlockBlobClient(BlobClient blobClient); friend class BlobClient; diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index e484896365..316f370646 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -31,7 +31,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2020-08-04"; + constexpr static const char* ApiVersion = "2020-10-02"; } // namespace _detail namespace Models { /** @@ -402,6 +402,10 @@ namespace Azure { namespace Storage { namespace Blobs { * container was deleted. */ Nullable RemainingRetentionDays; + /** + * Indicates if version level worm is enabled on this container. + */ + bool HasImmutableStorageWithVersioning = false; /** * A set of name-value pairs associated with this blob or blob container. */ @@ -410,10 +414,6 @@ namespace Azure { namespace Storage { namespace Blobs { std::string, Core::_internal::StringExtensions::CaseInsensitiveComparator> Metadata; - /** - * Indicates if version level worm is enabled on this container. - */ - bool HasImmutableStorageWithVersioning = false; }; /** * @brief An Azure Storage container. @@ -446,6 +446,7 @@ namespace Azure { namespace Storage { namespace Blobs { None = 0, Metadata = 1, Deleted = 2, + System = 4, }; inline ListBlobContainersIncludeFlags operator|( ListBlobContainersIncludeFlags lhs, @@ -966,6 +967,7 @@ namespace Azure { namespace Storage { namespace Blobs { AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Hot; AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Cool; AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Archive; + AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Premium; private: std::string m_value; @@ -1341,6 +1343,10 @@ namespace Azure { namespace Storage { namespace Blobs { * Properties of a blob. */ BlobItemDetails Details; + /** + * Indicates that this root blob has been deleted, but it has versions that are active. + */ + Nullable HasVersionsOnly; /** * Size in bytes. */ @@ -1365,6 +1371,7 @@ namespace Azure { namespace Storage { namespace Blobs { Tags = 64, ImmutabilityPolicy = 128, LegalHold = 256, + DeletedWithVersions = 512, }; inline ListBlobsIncludeFlags operator|(ListBlobsIncludeFlags lhs, ListBlobsIncludeFlags rhs) { @@ -1989,7 +1996,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2027,7 +2034,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2048,7 +2055,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2200,6 +2207,218 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetBlobAccessTierResult final { }; + namespace _detail { + /** + * @brief Required. The type of the provided query expression. + */ + class QueryRequestQueryType final { + public: + QueryRequestQueryType() = default; + explicit QueryRequestQueryType(std::string value) : m_value(std::move(value)) {} + bool operator==(const QueryRequestQueryType& other) const + { + return m_value == other.m_value; + } + bool operator!=(const QueryRequestQueryType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryRequestQueryType SQL; + + private: + std::string m_value; + }; + /** + * @brief The quick query format type. + */ + class QueryFormatType final { + public: + QueryFormatType() = default; + explicit QueryFormatType(std::string value) : m_value(std::move(value)) {} + bool operator==(const QueryFormatType& other) const { return m_value == other.m_value; } + bool operator!=(const QueryFormatType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Delimited; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Json; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Arrow; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Parquet; + + private: + std::string m_value; + }; + /** + * @brief Groups the settings used for interpreting the blob data if the blob is delimited + * text formatted. + */ + struct DelimitedTextConfiguration final + { + /** + * The string used to separate columns. + */ + std::string ColumnSeparator; + /** + * The string used to quote a specific field. + */ + std::string FieldQuote; + /** + * The string used to separate records. + */ + std::string RecordSeparator; + /** + * The string used as an escape character. + */ + std::string EscapeChar; + /** + * Represents whether the data has headers. + */ + bool HeadersPresent = bool(); + }; + /** + * @brief Json text configuration. + */ + struct JsonTextConfiguration final + { + /** + * The string used to separate records. + */ + std::string RecordSeparator; + }; + } // namespace _detail + /** + * @brief Type of blob query arrow field. + */ + class BlobQueryArrowFieldType final { + public: + BlobQueryArrowFieldType() = default; + explicit BlobQueryArrowFieldType(std::string value) : m_value(std::move(value)) {} + bool operator==(const BlobQueryArrowFieldType& other) const + { + return m_value == other.m_value; + } + bool operator!=(const BlobQueryArrowFieldType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Int64; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Bool; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Timestamp; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType String; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Double; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Decimal; + + private: + std::string m_value; + }; + /** + * @brief Groups settings regarding specific field of an arrow schema. + */ + struct BlobQueryArrowField final + { + /** + * Type of blob query arrow field. + */ + BlobQueryArrowFieldType Type; + Nullable Name; + Nullable Precision; + Nullable Scale; + }; + namespace _detail { + /** + * @brief Groups the settings used for formatting the response if the response should be Arrow + * formatted. + */ + struct ArrowConfiguration final + { + /** + * Array of BlobQueryArrowField. + */ + std::vector Schema; + }; + /** + * @brief Parquet configuration. + */ + struct ParquetConfiguration final + { + }; + struct QueryFormat final + { + /** + * The quick query format type. + */ + QueryFormatType Type; + /** + * Groups the settings used for interpreting the blob data if the blob is delimited text + * formatted. + */ + Nullable<_detail::DelimitedTextConfiguration> DelimitedTextConfiguration; + /** + * Json text configuration. + */ + Nullable<_detail::JsonTextConfiguration> JsonTextConfiguration; + /** + * Groups the settings used for formatting the response if the response should be Arrow + * formatted. + */ + Nullable<_detail::ArrowConfiguration> ArrowConfiguration; + /** + * Parquet configuration. + */ + Nullable ParquetTextConfiguration; + }; + struct QuerySerialization final + { + QueryFormat Format; + }; + /** + * @brief Groups the set of query request settings. + */ + struct QueryRequest final + { + /** + * Required. The type of the provided query expression. + */ + QueryRequestQueryType QueryType; + /** + * The query expression in SQL. The maximum size of the query expression is 256KiB. + */ + std::string Expression; + Nullable InputSerialization; + Nullable OutputSerialization; + }; + } // namespace _detail + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::Query. + */ + struct QueryBlobResult final + { + std::unique_ptr BodyStream; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified + * time of the blob. + */ + DateTime LastModified; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the + * request version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + Azure::ETag ETag; + /** + * When a blob is leased, specifies whether the lease is of infinite or fixed duration. + */ + Nullable LeaseDuration; + /** + * Lease state of the blob. + */ + Models::LeaseState LeaseState; + /** + * The current lease status of the blob. + */ + Models::LeaseStatus LeaseStatus; + /** + * The value of this header is set to true if the blob data and application metadata are + * completely encrypted using the specified algorithm. Otherwise, the value is set to false + * (when the blob is unencrypted, or if only parts of the blob/application metadata are + * encrypted). + */ + bool IsServerEncrypted = bool(); + }; /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::SetTags. */ @@ -2379,6 +2598,7 @@ namespace Azure { namespace Storage { namespace Blobs { * Array of ClearRange. */ std::vector ClearRanges; + std::string ContinuationToken; }; /** * @brief Response type for #Azure::Storage::Blobs::PageBlobClient::GetPageRangesDiff. @@ -2399,6 +2619,7 @@ namespace Azure { namespace Storage { namespace Blobs { * Array of ClearRange. */ std::vector ClearRanges; + std::string ContinuationToken; }; } // namespace _detail /** @@ -3470,6 +3691,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable ImmutabilityPolicyExpiry; Nullable ImmutabilityPolicyMode; Nullable LegalHold; + Nullable CopySourceAuthorization; Nullable> SourceContentcrc64; }; static Response CopyFromUri( @@ -3501,6 +3723,26 @@ namespace Azure { namespace Storage { namespace Blobs { const Core::Url& url, const SetBlobTierOptions& options, const Core::Context& context); + struct QueryBlobOptions final + { + Models::_detail::QueryRequest QueryRequest; + Nullable Snapshot; + Nullable LeaseId; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; + Nullable IfModifiedSince; + Nullable IfUnmodifiedSince; + ETag IfMatch; + ETag IfNoneMatch; + Nullable IfTags; + Nullable EncryptionScope; + }; + static Response Query( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const QueryBlobOptions& options, + const Core::Context& context); struct GetBlobTagsOptions final { Nullable Snapshot; @@ -3633,6 +3875,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response UploadPagesFromUri( Core::Http::_internal::HttpPipeline& pipeline, @@ -3649,6 +3892,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable Marker; + Nullable MaxResults; }; static Response GetPageRanges( Core::Http::_internal::HttpPipeline& pipeline, @@ -3667,6 +3912,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable Marker; + Nullable MaxResults; }; static Response GetPageRangesDiff( Core::Http::_internal::HttpPipeline& pipeline, @@ -3800,6 +4047,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response AppendBlockFromUri( Core::Http::_internal::HttpPipeline& pipeline, @@ -3885,6 +4133,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable BlobTagsString; std::string CopySource; Nullable CopySourceBlobProperties; + Nullable CopySourceAuthorization; Nullable> SourceContentcrc64; }; static Response UploadFromUri( @@ -3925,6 +4174,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response StageBlockFromUri( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt b/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt index 546a795da2..9f98c022d3 100644 --- a/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt @@ -18,3 +18,7 @@ create_per_service_target_build_for_sample(storage blob-sas) add_executable(transactional-checksum transactional_checksum.cpp) target_link_libraries(transactional-checksum PRIVATE azure-storage-blobs get-env-helper) create_per_service_target_build_for_sample(storage transactional-checksum) + +add_executable(blob-query blob_query.cpp) +target_link_libraries(blob-query PRIVATE azure-storage-blobs get-env-helper) +create_per_service_target_build_for_sample(storage blob-query) diff --git a/sdk/storage/azure-storage-blobs/samples/blob_query.cpp b/sdk/storage/azure-storage-blobs/samples/blob_query.cpp new file mode 100644 index 0000000000..f01c0de632 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/samples/blob_query.cpp @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "get_env.hpp" + +#include + +#include + +std::string GetConnectionString() +{ + const static std::string ConnectionString = ""; + + if (!ConnectionString.empty()) + { + return ConnectionString; + } + const static std::string envConnectionString = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); + if (!envConnectionString.empty()) + { + return envConnectionString; + } + throw std::runtime_error("Cannot find connection string."); +} + +int main() +{ + using namespace Azure::Storage::Blobs; + + const std::string containerName = "sample-container"; + const std::string blobName = "sample-blob"; + + auto containerClient + = BlobContainerClient::CreateFromConnectionString(GetConnectionString(), containerName); + containerClient.CreateIfNotExists(); + BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); + + const std::string blobContent = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": 50} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": 99} +{"id": 104, "name": "clementines", "price": 399} +{"id": 105, "name": "grapes", "price": 150} +{"id": 106, "name": "lemons", "price": 69} +{"id": 107, "name": "pears", "price": 100} +{"id": 108, "name": "cherries", "price": 281} +{"id": 109, "name": "coconut", "price": 178} +{"id": 110, "name": "bananas", "price": 39} +{"id": 111, "name": "peaches", "price": 117} +)json"; + + std::vector buffer(blobContent.begin(), blobContent.end()); + blobClient.UploadFrom(buffer.data(), buffer.size()); + + QueryBlobOptions queryOptions; + // input can be one of csv, json, parquet + queryOptions.InputTextConfiguration = BlobQueryInputTextOptions::CreateJsonTextOptions(); + // output can be one of csv, json, arrow, parquet + queryOptions.OutputTextConfiguration = BlobQueryOutputTextOptions::CreateCsvTextOptions(); + auto queryResponse + = blobClient.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + std::cout << std::string(data.begin(), data.end()); + /* The output is: + * 103,apples,99 + * 106,lemons,69 + * 110,bananas,39 + */ + + return 0; +} diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index da5dd8f808..d934f023f3 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -254,9 +254,16 @@ namespace Azure { namespace Storage { namespace Blobs { { i.Details.IsAccessTierInferred = false; } - if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + if (i.VersionId.HasValue()) { - i.IsCurrentVersion = false; + if (!i.HasVersionsOnly.HasValue()) + { + i.HasVersionsOnly = false; + } + if (!i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } } if (i.BlobType == Models::BlobType::AppendBlob && !i.Details.IsSealed.HasValue()) { @@ -327,9 +334,16 @@ namespace Azure { namespace Storage { namespace Blobs { { i.Details.IsAccessTierInferred = false; } - if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + if (i.VersionId.HasValue()) { - i.IsCurrentVersion = false; + if (!i.HasVersionsOnly.HasValue()) + { + i.HasVersionsOnly = false; + } + if (!i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } } if (i.BlobType == Models::BlobType::AppendBlob && !i.Details.IsSealed.HasValue()) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_options.cpp b/sdk/storage/azure-storage-blobs/src/blob_options.cpp new file mode 100644 index 0000000000..317650c0a6 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/blob_options.cpp @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/storage/blobs/blob_options.hpp" + +namespace Azure { namespace Storage { namespace Blobs { + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateCsvTextOptions( + const std::string& recordSeparator, + const std::string& columnSeparator, + const std::string& quotationCharacter, + const std::string& escapeCharacter, + bool hasHeaders) + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Delimited; + options.m_recordSeparator = recordSeparator; + options.m_columnSeparator = columnSeparator; + options.m_quotationCharacter = quotationCharacter; + options.m_escapeCharacter = escapeCharacter; + options.m_hasHeaders = hasHeaders; + return options; + } + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateJsonTextOptions( + const std::string& recordSeparator) + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Json; + options.m_recordSeparator = recordSeparator; + return options; + } + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateParquetTextOptions() + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Parquet; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateCsvTextOptions( + const std::string& recordSeparator, + const std::string& columnSeparator, + const std::string& quotationCharacter, + const std::string& escapeCharacter, + bool hasHeaders) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Delimited; + options.m_recordSeparator = recordSeparator; + options.m_columnSeparator = columnSeparator; + options.m_quotationCharacter = quotationCharacter; + options.m_escapeCharacter = escapeCharacter; + options.m_hasHeaders = hasHeaders; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateJsonTextOptions( + const std::string& recordSeparator) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Json; + options.m_recordSeparator = recordSeparator; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateArrowTextOptions( + std::vector schema) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Arrow; + options.m_schema = std::move(schema); + return options; + } + +}}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 7a52c5fe9a..a8f5269d65 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -88,11 +88,13 @@ namespace Azure { namespace Storage { namespace Blobs { void GetPageRangesPagedResponse::OnNextPage(const Azure::Core::Context& context) { + m_operationOptions.ContinuationToken = NextPageToken; *this = m_pageBlobClient->GetPageRanges(m_operationOptions, context); } void GetPageRangesDiffPagedResponse::OnNextPage(const Azure::Core::Context& context) { + m_operationOptions.ContinuationToken = NextPageToken; if (m_previousSnapshot.HasValue()) { *this = m_pageBlobClient->GetPageRangesDiff( diff --git a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp index 020269f353..35d7677df0 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp @@ -11,7 +11,7 @@ namespace Azure { namespace Storage { namespace Sas { namespace { - constexpr static const char* SasVersion = "2020-08-04"; + constexpr static const char* SasVersion = Blobs::_detail::ApiVersion; std::string BlobSasResourceToString(BlobSasResource resource) { diff --git a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp index 0bc3fa0e66..9e14727dfd 100644 --- a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp @@ -22,6 +22,9 @@ #include #include #include +#include + +#include "private/avro_parser.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -511,4 +514,147 @@ namespace Azure { namespace Storage { namespace Blobs { *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); } + Azure::Response BlockBlobClient::Query( + const std::string& querySqlExpression, + const QueryBlobOptions& options, + const Azure::Core::Context& context) const + { + _detail::BlobClient::QueryBlobOptions protocolLayerOptions; + protocolLayerOptions.QueryRequest.QueryType = Models::_detail::QueryRequestQueryType::SQL; + protocolLayerOptions.QueryRequest.Expression = querySqlExpression; + if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) + { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.InputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.InputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.InputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.InputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) + { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) + { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format.ToString().empty()) + { + } + else + { + AZURE_UNREACHABLE_CODE(); + } + if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) + { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.OutputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.OutputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.OutputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.OutputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) + { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) + { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Arrow) + { + Models::_detail::ArrowConfiguration c; + c.Schema = options.OutputTextConfiguration.m_schema; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ArrowConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format.ToString().empty()) + { + } + else + { + AZURE_UNREACHABLE_CODE(); + } + + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + auto response = _detail::BlobClient::Query( + *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + + const auto stautsCode = response.RawResponse->GetStatusCode(); + const auto reasonPhrase = response.RawResponse->GetReasonPhrase(); + const auto requestId + = response.RawResponse->GetHeaders().count(_internal::HttpHeaderRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId) + : std::string(); + + const auto clientRequestId + = response.RawResponse->GetHeaders().count(_internal::HttpHeaderClientRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderClientRequestId) + : std::string(); + + auto defaultErrorHandler + = [stautsCode, reasonPhrase, requestId, clientRequestId](BlobQueryError e) { + if (e.IsFatal) + { + StorageException exception("Fatal " + e.Name + " at " + std::to_string(e.Position)); + exception.StatusCode = stautsCode; + exception.ReasonPhrase = reasonPhrase; + exception.RequestId = requestId; + exception.ClientRequestId = clientRequestId; + exception.ErrorCode = e.Name; + exception.Message = e.Description; + + throw exception; + } + }; + + response.Value.BodyStream = std::make_unique<_detail::AvroStreamParser>( + std::move(response.Value.BodyStream), + options.ProgressHandler, + options.ErrorHandler ? options.ErrorHandler : defaultErrorHandler); + return response; + } + }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp index c2e4d35de4..4cdaf51a5b 100644 --- a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp @@ -322,6 +322,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRanges( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -333,8 +335,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.PageRanges = std::move(response.Value.PageRanges); pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; @@ -363,6 +365,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRangesDiff( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -376,8 +380,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; pagedResponse.m_previousSnapshot = previousSnapshot; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; @@ -406,6 +410,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRangesDiff( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -419,8 +425,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; pagedResponse.m_previousSnapshotUrl = previousSnapshotUrl; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; diff --git a/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp b/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp new file mode 100644 index 0000000000..5ff4846023 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp @@ -0,0 +1,680 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "avro_parser.hpp" + +#include +#include + +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + namespace { + int64_t parseInt(AvroStreamReader::ReaderPos& data) + { + uint64_t r = 0; + int nb = 0; + while (true) + { + uint8_t c = (*data.BufferPtr)[data.Offset++]; + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) + { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); + } + + AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) + { + const static std::map BuiltinNameSchemaMap = { + {"string", AvroSchema::StringSchema}, + {"bytes", AvroSchema::BytesSchema}, + {"int", AvroSchema::IntSchema}, + {"long", AvroSchema::LongSchema}, + {"float", AvroSchema::FloatSchema}, + {"double", AvroSchema::DoubleSchema}, + {"boolean", AvroSchema::BoolSchema}, + {"null", AvroSchema::NullSchema}, + {"string", AvroSchema::StringSchema}, + }; + std::map nameSchemaMap = BuiltinNameSchemaMap; + + std::function parseSchemaFromJsonObject; + parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema { + if (obj.is_string()) + { + auto typeName = obj.get(); + return nameSchemaMap.find(typeName)->second; + } + else if (obj.is_array()) + { + std::vector unionSchemas; + for (const auto& s : obj) + { + unionSchemas.push_back(parseSchemaFromJsonObject(s)); + } + return AvroSchema::UnionSchema(std::move(unionSchemas)); + } + else if (obj.is_object()) + { + if (obj.count("namespace") != 0) + { + throw std::runtime_error("Namespace isn't supported yet in Avro schema."); + } + if (obj.count("aliases") != 0) + { + throw std::runtime_error("Alias isn't supported yet in Avro schema."); + } + auto typeName = obj["type"].get(); + auto i = nameSchemaMap.find(typeName); + if (i != nameSchemaMap.end()) + { + return i->second; + } + if (typeName == "record") + { + std::vector> fieldsSchema; + for (const auto& field : obj["fields"]) + { + fieldsSchema.push_back(std::make_pair( + field["name"].get(), parseSchemaFromJsonObject(field["type"]))); + } + + const std::string recordName = obj["name"].get(); + auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema)); + nameSchemaMap.insert(std::make_pair(recordName, recordSchema)); + return recordSchema; + } + else if (typeName == "enum") + { + throw std::runtime_error("Enum type isn't supported yet in Avro schema."); + } + else if (typeName == "array") + { + return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"])); + } + else if (typeName == "map") + { + return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"])); + } + else if (typeName == "fixed") + { + const std::string fixedName = obj["name"].get(); + auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get()); + nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema)); + return fixedSchema; + } + else + { + throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema."); + } + } + AZURE_UNREACHABLE_CODE(); + }; + + auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end()); + return parseSchemaFromJsonObject(jsonRoot); + } + } // namespace + + int64_t AvroStreamReader::ParseInt(const Core::Context& context) + { + uint64_t r = 0; + int nb = 0; + while (true) + { + Preload(1, context); + uint8_t c = m_streambuffer[m_pos.Offset++]; + + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) + { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); + } + + void AvroStreamReader::Advance(size_t n, const Core::Context& context) + { + Preload(n, context); + m_pos.Offset += n; + } + + size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) + { + size_t oldAvailable = AvailableBytes(); + while (true) + { + size_t newAvailable = TryPreload(n, context); + if (newAvailable >= n) + { + return newAvailable; + } + if (oldAvailable == newAvailable) + { + throw std::runtime_error("Unexpected EOF of Avro stream."); + } + oldAvailable = newAvailable; + } + AZURE_UNREACHABLE_CODE(); + } + + size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) + { + size_t availableBytes = AvailableBytes(); + if (availableBytes >= n) + { + return availableBytes; + } + const size_t MinRead = 4096; + size_t tryReadSize = std::max(n, MinRead); + size_t currSize = m_streambuffer.size(); + m_streambuffer.resize(m_streambuffer.size() + tryReadSize); + size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context); + m_streambuffer.resize(currSize + actualReadSize); + return AvailableBytes(); + } + + void AvroStreamReader::Discard() + { + constexpr size_t MinimumReleaseMemory = 128 * 1024; + if (m_pos.Offset < MinimumReleaseMemory) + { + return; + } + const size_t availableBytes = AvailableBytes(); + std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes); + m_streambuffer.resize(availableBytes); + m_pos.Offset = 0; + } + + const AvroSchema AvroSchema::StringSchema(AvroDatumType::String); + const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes); + const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int); + const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long); + const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float); + const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double); + const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool); + const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null); + + AvroSchema AvroSchema::RecordSchema( + std::string name, + const std::vector>& fieldsSchema) + { + AvroSchema recordSchema(AvroDatumType::Record); + recordSchema.m_name = std::move(name); + recordSchema.m_status = std::make_shared(); + for (auto& i : fieldsSchema) + { + recordSchema.m_status->m_keys.push_back(i.first); + recordSchema.m_status->m_schemas.push_back(i.second); + } + return recordSchema; + } + + AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) + { + AvroSchema arraySchema(AvroDatumType::Array); + arraySchema.m_status = std::make_shared(); + arraySchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return arraySchema; + } + + AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) + { + AvroSchema mapSchema(AvroDatumType::Map); + mapSchema.m_status = std::make_shared(); + mapSchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return mapSchema; + } + + AvroSchema AvroSchema::UnionSchema(std::vector schemas) + { + AvroSchema unionSchema(AvroDatumType::Union); + unionSchema.m_status = std::make_shared(); + unionSchema.m_status->m_schemas = std::move(schemas); + return unionSchema; + } + + AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) + { + AvroSchema fixedSchema(AvroDatumType::Fixed); + fixedSchema.m_name = std::move(name); + fixedSchema.m_status = std::make_shared(); + fixedSchema.m_status->m_size = size; + return fixedSchema; + } + + void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) + { + m_data = reader.m_pos; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + int64_t stringSize = reader.ParseInt(context); + reader.Advance(static_cast(stringSize), context); + } + else if ( + m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long + || m_schema.Type() == AvroDatumType::Enum) + { + reader.ParseInt(context); + } + else if (m_schema.Type() == AvroDatumType::Float) + { + reader.Advance(4, context); + } + else if (m_schema.Type() == AvroDatumType::Double) + { + reader.Advance(8, context); + } + else if (m_schema.Type() == AvroDatumType::Bool) + { + reader.Advance(1, context); + } + else if (m_schema.Type() == AvroDatumType::Null) + { + reader.Advance(0, context); + } + else if (m_schema.Type() == AvroDatumType::Record) + { + for (const auto& s : m_schema.FieldSchemas()) + { + AvroDatum(s).Fill(reader, context); + } + } + else if (m_schema.Type() == AvroDatumType::Array) + { + while (true) + { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } + else + { + for (auto i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Map) + { + while (true) + { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } + else + { + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(AvroSchema::StringSchema).Fill(reader, context); + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = reader.ParseInt(context); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(reader, context); + } + else if (m_schema.Type() == AvroDatumType::Fixed) + { + reader.Advance(m_schema.Size(), context); + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + + void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) + { + m_data = data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + int64_t stringSize = parseInt(data); + data.Offset += static_cast(stringSize); + } + else if ( + m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long + || m_schema.Type() == AvroDatumType::Enum) + { + parseInt(data); + } + else if (m_schema.Type() == AvroDatumType::Float) + { + data.Offset += 4; + } + else if (m_schema.Type() == AvroDatumType::Double) + { + data.Offset += 8; + } + else if (m_schema.Type() == AvroDatumType::Bool) + { + data.Offset += 1; + } + else if (m_schema.Type() == AvroDatumType::Null) + { + data.Offset += 0; + } + else if (m_schema.Type() == AvroDatumType::Record) + { + for (const auto& s : m_schema.FieldSchemas()) + { + AvroDatum(s).Fill(data); + } + } + else if (m_schema.Type() == AvroDatumType::Array) + { + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } + else + { + for (auto i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Map) + { + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } + else + { + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(AvroSchema::StringSchema).Fill(data); + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = parseInt(data); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(data); + } + else if (m_schema.Type() == AvroDatumType::Fixed) + { + data.Offset += m_schema.Size(); + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + + template <> AvroDatum::StringView AvroDatum::Value() const + { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + const int64_t length = parseInt(data); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, static_cast(length)}; + data.Offset += static_cast(length); + return ret; + } + if (m_schema.Type() == AvroDatumType::Fixed) + { + const size_t fixedSize = m_schema.Size(); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, fixedSize}; + data.Offset += fixedSize; + return ret; + } + AZURE_UNREACHABLE_CODE(); + } + + template <> std::string AvroDatum::Value() const + { + auto stringView = Value(); + return std::string(stringView.Data, stringView.Data + stringView.Length); + } + + template <> std::vector AvroDatum::Value() const + { + auto stringView = Value(); + return std::vector(stringView.Data, stringView.Data + stringView.Length); + } + + template <> int64_t AvroDatum::Value() const + { + auto data = m_data; + return parseInt(data); + } + + template <> int32_t AvroDatum::Value() const { return static_cast(Value()); } + + template <> bool AvroDatum::Value() const { return Value(); } + + template <> std::nullptr_t AvroDatum::Value() const { return nullptr; } + + template <> AvroRecord AvroDatum::Value() const + { + auto data = m_data; + + AvroRecord r; + r.m_keys = &m_schema.FieldNames(); + for (const auto& schema : m_schema.FieldSchemas()) + { + auto datum = AvroDatum(schema); + datum.Fill(data); + r.m_values.push_back(std::move(datum)); + } + + return r; + } + + template <> AvroMap AvroDatum::Value() const + { + auto data = m_data; + + AvroMap m; + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + if (numElementsInBlock < 0) + { + numElementsInBlock = -numElementsInBlock; + parseInt(data); + } + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + auto keyDatum = AvroDatum(AvroSchema::StringSchema); + keyDatum.Fill(data); + auto valueDatum = AvroDatum(m_schema.ItemSchema()); + valueDatum.Fill(data); + m[keyDatum.Value()] = valueDatum; + } + } + return m; + } + + template <> AvroDatum AvroDatum::Value() const + { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = parseInt(data); + auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast(i)]); + datum.Fill(data); + return datum; + } + AZURE_UNREACHABLE_CODE(); + } + + AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream) + : m_reader(std::make_unique(stream)) + { + } + + AvroDatum AvroObjectContainerReader::NextImpl( + const AvroSchema* schema, + const Core::Context& context) + { + AZURE_ASSERT_FALSE(m_eof); + static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16); + if (!schema) + { + static AvroSchema FileHeaderSchema = []() { + std::vector> fieldsSchema; + fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4))); + fieldsSchema.push_back( + std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema))); + fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema)); + return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema)); + }(); + auto fileHeaderDatum = AvroDatum(FileHeaderSchema); + fileHeaderDatum.Fill(*m_reader, context); + auto fileHeader = fileHeaderDatum.Value(); + if (fileHeader.Field("magic").Value() != "Obj\01") + { + throw std::runtime_error("Invalid Avro object container magic."); + } + AvroMap meta = fileHeader.Field("meta").Value(); + std::string objectSchemaJson = meta["avro.schema"].Value(); + std::string codec = "null"; + if (meta.count("avro.codec") != 0) + { + codec = meta["avro.codec"].Value(); + } + if (codec != "null") + { + throw std::runtime_error("Unsupported Avro codec: " + codec); + } + m_syncMarker = fileHeader.Field("sync").Value(); + m_objectSchema = std::make_unique(ParseSchemaFromJsonString(objectSchemaJson)); + schema = m_objectSchema.get(); + } + + if (m_remainingObjectInCurrentBlock == 0) + { + m_reader->Discard(); + m_remainingObjectInCurrentBlock = m_reader->ParseInt(context); + int64_t ObjectsSize = m_reader->ParseInt(context); + m_reader->Preload(static_cast(ObjectsSize), context); + } + + auto objectDatum = AvroDatum(*m_objectSchema); + objectDatum.Fill(*m_reader, context); + if (--m_remainingObjectInCurrentBlock == 0) + { + auto markerDatum = AvroDatum(SyncMarkerSchema); + markerDatum.Fill(*m_reader, context); + auto marker = markerDatum.Value(); + if (marker != m_syncMarker) + { + throw std::runtime_error("Sync marker doesn't match."); + } + m_eof = m_reader->TryPreload(1, context) == 0; + } + return objectDatum; + } + + size_t AvroStreamParser::OnRead( + uint8_t* buffer, + size_t count, + Azure::Core::Context const& context) + { + if (m_parserBuffer.Length != 0) + { + size_t bytesToCopy = std::min(m_parserBuffer.Length, count); + std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy); + m_parserBuffer.Data += bytesToCopy; + m_parserBuffer.Length -= bytesToCopy; + return bytesToCopy; + } + while (!m_parser.End()) + { + auto datum = m_parser.Next(context); + if (datum.Schema().Type() == AvroDatumType::Union) + { + datum = datum.Value(); + } + if (datum.Schema().Type() != AvroDatumType::Record) + { + continue; + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") + { + auto record = datum.Value(); + auto dataDatum = record.Field("data"); + m_parserBuffer = dataDatum.Value(); + return OnRead(buffer, count, context); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" + && m_progressCallback) + { + auto record = datum.Value(); + auto bytesScanned = record.Field("bytesScanned").Value(); + auto totalBytes = record.Field("totalBytes").Value(); + m_progressCallback(bytesScanned, totalBytes); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" + && m_errorCallback) + { + auto record = datum.Value(); + BlobQueryError e; + e.Name = record.Field("name").Value(); + e.Description = record.Field("description").Value(); + e.IsFatal = record.Field("fatal").Value(); + e.Position = record.Field("position").Value(); + m_errorCallback(std::move(e)); + } + } + return 0; + } +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp b/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp new file mode 100644 index 0000000000..0d6554053a --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include + +#include "azure/storage/blobs/blob_options.hpp" + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + enum class AvroDatumType + { + String, + Bytes, + Int, + Long, + Float, + Double, + Bool, + Null, + Record, + Enum, + Array, + Map, + Union, + Fixed, + }; + + class AvroStreamReader final { + public: + // position of a vector that lives through vector resizing + struct ReaderPos final + { + const std::vector* BufferPtr = nullptr; + size_t Offset = 0; + }; + explicit AvroStreamReader(Core::IO::BodyStream& stream) + : m_stream(&stream), m_pos{&m_streambuffer, 0} + { + } + AvroStreamReader(const AvroStreamReader&) = delete; + AvroStreamReader& operator=(const AvroStreamReader&) = delete; + + int64_t ParseInt(const Core::Context& context); + void Advance(size_t n, const Core::Context& context); + // Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes + // available in m_streambuffer; + size_t Preload(size_t n, const Core::Context& context); + size_t TryPreload(size_t n, const Core::Context& context); + // discards data that's before m_pos + void Discard(); + + private: + size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; } + + private: + Core::IO::BodyStream* m_stream; + std::vector m_streambuffer; + ReaderPos m_pos; + + friend class AvroDatum; + }; + + class AvroSchema final { + public: + static const AvroSchema StringSchema; + static const AvroSchema BytesSchema; + static const AvroSchema IntSchema; + static const AvroSchema LongSchema; + static const AvroSchema FloatSchema; + static const AvroSchema DoubleSchema; + static const AvroSchema BoolSchema; + static const AvroSchema NullSchema; + static AvroSchema RecordSchema( + std::string name, + const std::vector>& fieldsSchema); + static AvroSchema ArraySchema(AvroSchema elementSchema); + static AvroSchema MapSchema(AvroSchema elementSchema); + static AvroSchema UnionSchema(std::vector schemas); + static AvroSchema FixedSchema(std::string name, int64_t size); + + const std::string& Name() const { return m_name; } + AvroDatumType Type() const { return m_type; } + const std::vector& FieldNames() const { return m_status->m_keys; } + AvroSchema ItemSchema() const { return m_status->m_schemas[0]; } + const std::vector& FieldSchemas() const { return m_status->m_schemas; } + size_t Size() const { return static_cast(m_status->m_size); } + + private: + explicit AvroSchema(AvroDatumType type) : m_type(type) {} + + private: + AvroDatumType m_type; + std::string m_name; + + struct SharedStatus + { + std::vector m_keys; + std::vector m_schemas; + int64_t m_size = 0; + }; + std::shared_ptr m_status; + }; + + class AvroDatum; + + class AvroRecord final { + public: + bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); } + const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); } + AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); } + const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); } + AvroDatum& FieldAt(size_t i) { return m_values.at(i); } + + private: + size_t FindField(const std::string& key) const + { + auto i = find(m_keys->begin(), m_keys->end(), key); + return i - m_keys->begin(); + } + const std::vector* m_keys = nullptr; + std::vector m_values; + + friend class AvroDatum; + }; + + using AvroMap = std::map; + + class AvroDatum final { + public: + AvroDatum() : m_schema(AvroSchema::NullSchema) {} + explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {} + + void Fill(AvroStreamReader& reader, const Core::Context& context); + void Fill(AvroStreamReader::ReaderPos& data); + + const AvroSchema& Schema() const { return m_schema; } + + template T Value() const; + struct StringView + { + const uint8_t* Data = nullptr; + size_t Length = 0; + }; + + private: + AvroSchema m_schema; + AvroStreamReader::ReaderPos m_data; + }; + + class AvroObjectContainerReader final { + public: + explicit AvroObjectContainerReader(Core::IO::BodyStream& stream); + + bool End() const { return m_eof; } + // Calling Next() will invalidates the previous AvroDatum returned by this function and all + // AvroDatums propagated from there. + AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); } + + private: + AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context); + + private: + std::unique_ptr m_reader; + std::unique_ptr m_objectSchema; + std::string m_syncMarker; + int64_t m_remainingObjectInCurrentBlock = 0; + bool m_eof = false; + }; + + class AvroStreamParser final : public Core::IO::BodyStream { + public: + explicit AvroStreamParser( + std::unique_ptr inner, + std::function progressCallback, + std::function errorCallback) + : m_inner(std::move(inner)), m_parser(*m_inner), + m_progressCallback(std::move(progressCallback)), m_errorCallback(std::move(errorCallback)) + { + } + + int64_t Length() const override { return -1; } + void Rewind() override { this->m_inner->Rewind(); } + + private: + size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override; + + private: + std::unique_ptr m_inner; + AvroObjectContainerReader m_parser; + std::function m_progressCallback; + std::function m_errorCallback; + AvroDatum::StringView m_parserBuffer; + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index a2b55c9fef..4fdcad2249 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -33,13 +33,15 @@ std::string ListBlobContainersIncludeFlagsToString( const Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags valueList[] = { Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::Metadata, Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::Deleted, + Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::System, }; const char* stringList[] = { "metadata", "deleted", + "system", }; std::string ret; - for (size_t i = 0; i < 2; ++i) + for (size_t i = 0; i < 3; ++i) { if ((val & valueList[i]) == valueList[i]) { @@ -65,6 +67,7 @@ std::string ListBlobsIncludeFlagsToString( Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::Tags, Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy, Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::LegalHold, + Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions, }; const char* stringList[] = { "copy", @@ -76,9 +79,10 @@ std::string ListBlobsIncludeFlagsToString( "tags", "immutabilitypolicy", "legalhold", + "deletedwithversions", }; std::string ret; - for (size_t i = 0; i < 9; ++i) + for (size_t i = 0; i < 10; ++i) { if ((val & valueList[i]) == valueList[i]) { @@ -147,6 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { const AccessTier AccessTier::Hot("Hot"); const AccessTier AccessTier::Cool("Cool"); const AccessTier AccessTier::Archive("Archive"); + const AccessTier AccessTier::Premium("Premium"); const ArchiveStatus ArchiveStatus::RehydratePendingToHot("rehydrate-pending-to-hot"); const ArchiveStatus ArchiveStatus::RehydratePendingToCool("rehydrate-pending-to-cool"); const RehydratePriority RehydratePriority::High("High"); @@ -165,6 +170,19 @@ namespace Azure { namespace Storage { namespace Blobs { "RelativeToCreation"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::RelativeToNow("RelativeToNow"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::Absolute("Absolute"); + namespace _detail { + const QueryRequestQueryType QueryRequestQueryType::SQL("SQL"); + const QueryFormatType QueryFormatType::Delimited("delimited"); + const QueryFormatType QueryFormatType::Json("json"); + const QueryFormatType QueryFormatType::Arrow("arrow"); + const QueryFormatType QueryFormatType::Parquet("parquet"); + } // namespace _detail + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Int64("int64"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Bool("bool"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Timestamp("timestamp[ms]"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::String("string"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Double("double"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Decimal("decimal"); const SequenceNumberAction SequenceNumberAction::Max("max"); const SequenceNumberAction SequenceNumberAction::Update("update"); const SequenceNumberAction SequenceNumberAction::Increment("increment"); @@ -356,7 +374,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -376,7 +394,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -672,7 +690,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "stats"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -772,7 +790,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobContainersIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -808,8 +826,8 @@ namespace Azure { namespace Storage { namespace Blobs { kDenyEncryptionScopeOverride, kDeletedTime, kRemainingRetentionDays, - kMetadata, kImmutableStorageWithVersioningEnabled, + kMetadata, }; const std::unordered_map XmlTagEnumMap{ {"EnumerationResults", XmlTagEnum::kEnumerationResults}, @@ -833,9 +851,9 @@ namespace Azure { namespace Storage { namespace Blobs { {"DenyEncryptionScopeOverride", XmlTagEnum::kDenyEncryptionScopeOverride}, {"DeletedTime", XmlTagEnum::kDeletedTime}, {"RemainingRetentionDays", XmlTagEnum::kRemainingRetentionDays}, - {"Metadata", XmlTagEnum::kMetadata}, {"ImmutableStorageWithVersioningEnabled", XmlTagEnum::kImmutableStorageWithVersioningEnabled}, + {"Metadata", XmlTagEnum::kMetadata}, }; std::vector xmlPath; Models::BlobContainerItem vectorElement1; @@ -988,18 +1006,18 @@ namespace Azure { namespace Storage { namespace Blobs { else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kContainers && xmlPath[2] == XmlTagEnum::kContainer - && xmlPath[3] == XmlTagEnum::kMetadata) + && xmlPath[3] == XmlTagEnum::kProperties + && xmlPath[4] == XmlTagEnum::kImmutableStorageWithVersioningEnabled) { - mapValue3 = node.Value; + vectorElement1.Details.HasImmutableStorageWithVersioning + = node.Value == std::string("true"); } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kContainers && xmlPath[2] == XmlTagEnum::kContainer - && xmlPath[3] == XmlTagEnum::kProperties - && xmlPath[4] == XmlTagEnum::kImmutableStorageWithVersioningEnabled) + && xmlPath[3] == XmlTagEnum::kMetadata) { - vectorElement1.Details.HasImmutableStorageWithVersioning - = node.Value == std::string("true"); + mapValue3 = node.Value; } } else if (node.Type == _internal::XmlNodeType::Attribute) @@ -1057,7 +1075,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "userdelegationkey"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1172,7 +1190,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "account"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -1202,7 +1220,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1223,7 +1241,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "blobs"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.Where.HasValue() && !options.Where.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1379,7 +1397,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-public-access", options.Access.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.DefaultEncryptionScope.HasValue() && !options.DefaultEncryptionScope.Value().empty()) { @@ -1416,7 +1434,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1494,7 +1512,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1528,7 +1546,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Modified-Since", options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1555,7 +1573,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1724,7 +1742,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1747,7 +1765,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.DeletedContainerName.HasValue() && !options.DeletedContainerName.Value().empty()) { request.SetHeader("x-ms-deleted-container-name", options.DeletedContainerName.Value()); @@ -1777,7 +1795,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (!options.SourceContainerName.empty()) { request.SetHeader("x-ms-source-container-name", options.SourceContainerName); @@ -1811,7 +1829,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1854,7 +1872,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -1895,7 +1913,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1935,7 +1953,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1976,7 +1994,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -2021,7 +2039,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2068,7 +2086,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2138,6 +2156,7 @@ namespace Azure { namespace Storage { namespace Blobs { kOrMetadata, kImmutabilityPolicyUntilDate, kImmutabilityPolicyMode, + kHasVersionsOnly, kContentLength, kBlobType, }; @@ -2197,6 +2216,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"OrMetadata", XmlTagEnum::kOrMetadata}, {"ImmutabilityPolicyUntilDate", XmlTagEnum::kImmutabilityPolicyUntilDate}, {"ImmutabilityPolicyMode", XmlTagEnum::kImmutabilityPolicyMode}, + {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, }; @@ -2614,6 +2634,13 @@ namespace Azure { namespace Storage { namespace Blobs { vectorElement1.Details.ImmutabilityPolicy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kHasVersionsOnly) + { + vectorElement1.HasVersionsOnly = node.Value == std::string("true"); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob @@ -2722,7 +2749,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2793,6 +2820,7 @@ namespace Azure { namespace Storage { namespace Blobs { kOrMetadata, kImmutabilityPolicyUntilDate, kImmutabilityPolicyMode, + kHasVersionsOnly, kContentLength, kBlobType, kBlobPrefix, @@ -2854,6 +2882,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"OrMetadata", XmlTagEnum::kOrMetadata}, {"ImmutabilityPolicyUntilDate", XmlTagEnum::kImmutabilityPolicyUntilDate}, {"ImmutabilityPolicyMode", XmlTagEnum::kImmutabilityPolicyMode}, + {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, {"BlobPrefix", XmlTagEnum::kBlobPrefix}, @@ -3279,6 +3308,13 @@ namespace Azure { namespace Storage { namespace Blobs { vectorElement1.Details.ImmutabilityPolicy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kHasVersionsOnly) + { + vectorElement1.HasVersionsOnly = node.Value == std::string("true"); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob @@ -3440,7 +3476,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok @@ -3710,7 +3746,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -3966,7 +4002,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -3984,7 +4020,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4003,7 +4039,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "expiry"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (!options.ExpiryOptions.ToString().empty()) { request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.ToString()); @@ -4085,7 +4121,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-content-disposition", options.BlobContentDisposition); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4112,7 +4148,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.IfUnmodifiedSince.HasValue()) { request.SetHeader( @@ -4155,7 +4191,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Delete, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4175,7 +4211,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "legalhold"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); request.SetHeader("x-ms-legal-hold", options.LegalHold ? "true" : "false"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4247,7 +4283,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4316,7 +4352,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -4368,7 +4404,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4419,7 +4455,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4475,7 +4511,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4527,7 +4563,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -4601,7 +4637,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -4702,7 +4738,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -4815,7 +4851,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -4844,6 +4880,12 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-legal-hold", options.LegalHold.Value() ? "true" : "false"); } + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } if (options.SourceContentcrc64.HasValue() && !Core::Convert::Base64Encode(options.SourceContentcrc64.Value()).empty()) { @@ -4901,7 +4943,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::NoContent) @@ -4939,7 +4981,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-rehydrate-priority", options.RehydratePriority.Value().ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4959,6 +5001,317 @@ namespace Azure { namespace Storage { namespace Blobs { return Response( std::move(response), std::move(pRawResponse)); } + Response BlobClient::Query( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const QueryBlobOptions& options, + const Core::Context& context) + { + std::string xmlBody; + { + _internal::XmlWriter writer; + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "QueryRequest"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "QueryType", + options.QueryRequest.QueryType.ToString()}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Expression", options.QueryRequest.Expression}); + if (options.QueryRequest.InputSerialization.HasValue()) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "InputSerialization"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Format"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Type", + options.QueryRequest.InputSerialization.Value().Format.Type.ToString()}); + if (options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "DelimitedTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "ColumnSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .ColumnSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "FieldQuote", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .FieldQuote}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "EscapeChar", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .EscapeChar}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "HasHeaders", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .HeadersPresent + ? "true" + : "false"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value() + .Format.JsonTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "JsonTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.JsonTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value().Format.ArrowConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ArrowConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Schema"}); + for (const auto& i1 : options.QueryRequest.InputSerialization.Value() + .Format.ArrowConfiguration.Value() + .Schema) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Field"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Type", i1.Type.ToString()}); + if (i1.Name.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Name", i1.Name.Value()}); + } + if (i1.Precision.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Precision", + std::to_string(i1.Precision.Value())}); + } + if (i1.Scale.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Scale", std::to_string(i1.Scale.Value())}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value() + .Format.ParquetTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ParquetTextConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.HasValue()) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "OutputSerialization"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Format"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Type", + options.QueryRequest.OutputSerialization.Value().Format.Type.ToString()}); + if (options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "DelimitedTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "ColumnSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .ColumnSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "FieldQuote", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .FieldQuote}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "EscapeChar", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .EscapeChar}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "HasHeaders", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .HeadersPresent + ? "true" + : "false"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value() + .Format.JsonTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "JsonTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.JsonTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value().Format.ArrowConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ArrowConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Schema"}); + for (const auto& i2 : options.QueryRequest.OutputSerialization.Value() + .Format.ArrowConfiguration.Value() + .Schema) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Field"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Type", i2.Type.ToString()}); + if (i2.Name.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Name", i2.Name.Value()}); + } + if (i2.Precision.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Precision", + std::to_string(i2.Precision.Value())}); + } + if (i2.Scale.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Scale", std::to_string(i2.Scale.Value())}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value() + .Format.ParquetTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ParquetTextConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::End}); + xmlBody = writer.GetDocument(); + } + Core::IO::MemoryBodyStream requestBody( + reinterpret_cast(xmlBody.data()), xmlBody.length()); + auto request = Core::Http::Request(Core::Http::HttpMethod::Post, url, &requestBody, false); + request.SetHeader("Content-Type", "application/xml; charset=UTF-8"); + request.SetHeader("Content-Length", std::to_string(requestBody.Length())); + request.GetUrl().AppendQueryParameter("comp", "query"); + if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "snapshot", _internal::UrlEncodeQueryParameter(options.Snapshot.Value())); + } + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } + if (options.IfModifiedSince.HasValue()) + { + request.SetHeader( + "If-Modified-Since", + options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfUnmodifiedSince.HasValue()) + { + request.SetHeader( + "If-Unmodified-Since", + options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfMatch.HasValue() && !options.IfMatch.ToString().empty()) + { + request.SetHeader("If-Match", options.IfMatch.ToString()); + } + if (options.IfNoneMatch.HasValue() && !options.IfNoneMatch.ToString().empty()) + { + request.SetHeader("If-None-Match", options.IfNoneMatch.ToString()); + } + if (options.IfTags.HasValue() && !options.IfTags.Value().empty()) + { + request.SetHeader("x-ms-if-tags", options.IfTags.Value()); + } + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.EncryptionScope.HasValue() && !options.EncryptionScope.Value().empty()) + { + request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); + } + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok + || httpStatusCode == Core::Http::HttpStatusCode::PartialContent)) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::QueryBlobResult response; + response.BodyStream = pRawResponse->ExtractBodyStream(); + response.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + response.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + response.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + return Response(std::move(response), std::move(pRawResponse)); + } Response> BlobClient::GetTags( Core::Http::_internal::HttpPipeline& pipeline, const Core::Url& url, @@ -4967,7 +5320,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5094,7 +5447,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Type", "application/xml; charset=UTF-8"); request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.VersionId.HasValue() && !options.VersionId.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5225,7 +5578,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -5365,7 +5718,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5482,7 +5835,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5616,7 +5969,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5702,7 +6061,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -5722,6 +6091,7 @@ namespace Azure { namespace Storage { namespace Blobs { kStart, kEnd, kClearRange, + kNextMarker, }; const std::unordered_map XmlTagEnumMap{ {"PageList", XmlTagEnum::kPageList}, @@ -5729,6 +6099,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"Start", XmlTagEnum::kStart}, {"End", XmlTagEnum::kEnd}, {"ClearRange", XmlTagEnum::kClearRange}, + {"NextMarker", XmlTagEnum::kNextMarker}, }; std::vector xmlPath; Core::Http::HttpRange vectorElement1; @@ -5770,6 +6141,12 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement2.Length = std::stoll(node.Value); } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kPageList + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.ContinuationToken = node.Value; + } } else if (node.Type == _internal::XmlNodeType::Attribute) { @@ -5856,7 +6233,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -5876,6 +6263,7 @@ namespace Azure { namespace Storage { namespace Blobs { kStart, kEnd, kClearRange, + kNextMarker, }; const std::unordered_map XmlTagEnumMap{ {"PageList", XmlTagEnum::kPageList}, @@ -5883,6 +6271,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"Start", XmlTagEnum::kStart}, {"End", XmlTagEnum::kEnd}, {"ClearRange", XmlTagEnum::kClearRange}, + {"NextMarker", XmlTagEnum::kNextMarker}, }; std::vector xmlPath; Core::Http::HttpRange vectorElement1; @@ -5924,6 +6313,12 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement2.Length = std::stoll(node.Value); } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kPageList + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.ContinuationToken = node.Value; + } } else if (node.Type == _internal::XmlNodeType::Attribute) { @@ -6012,7 +6407,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } request.SetHeader("x-ms-blob-content-length", std::to_string(options.BlobContentLength)); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6072,7 +6467,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6124,7 +6519,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-copy-source", options.CopySource); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -6229,7 +6624,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -6357,7 +6752,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6511,7 +6906,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6561,7 +6962,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "seal"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -6700,7 +7101,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -6883,7 +7284,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-tags", options.SourceIfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -6905,6 +7306,12 @@ namespace Azure { namespace Storage { namespace Blobs { "x-ms-copy-source-blob-properties", options.CopySourceBlobProperties.Value() ? "true" : "false"); } + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } if (options.SourceContentcrc64.HasValue() && !Core::Convert::Base64Encode(options.SourceContentcrc64.Value()).empty()) { @@ -7005,7 +7412,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7119,7 +7526,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7280,7 +7693,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -7369,7 +7782,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index dc73ddd28e..2a31bd673f 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-blobs namespace: Azure::Storage::Blobs output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Jinming-Hu/azure-storage-api-specs/main/Microsoft.BlobStorage/preview/2020-08-04/blob.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.BlobStorage/preview/2020-10-02/blob.json ``` ## ModelFour Options @@ -75,7 +75,6 @@ directive: delete $["/{filesystem}/{path}?action=setAccessControl&blob"]; delete $["/{filesystem}/{path}?action=getAccessControl&blob"]; delete $["/{filesystem}/{path}?FileRename"]; - delete $["/{containerName}/{blob}?comp=query"]; for (const operation in $) { for (const verb in $[operation]) { @@ -103,7 +102,7 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2020-08-04"], + "enum": ["2020-10-02"], "description": "The version used for the operations to Azure storage services." }; ``` @@ -123,7 +122,7 @@ directive: $["/{containerName}?restype=container&comp=undelete"].put.operationId = "BlobContainer_Undelete"; $["/{containerName}/{blob}?comp=copy"].put.operationId = "Blob_StartCopyFromUri"; $["/{containerName}/{blob}?comp=copy&sync"].put.operationId = "Blob_CopyFromUri"; - $["/{containerName}/{blob}?comp=copy©id={CopyId}"].put.operationId = "Blob_AbortCopyFromUri"; + $["/{containerName}/{blob}?comp=copy©id"].put.operationId = "Blob_AbortCopyFromUri"; $["/{containerName}/{blob}?comp=block&fromURL"].put.operationId = "BlockBlob_StageBlockFromUri"; $["/{containerName}/{blob}?comp=page&update&fromUrl"].put.operationId = "PageBlob_UploadPagesFromUri"; $["/{containerName}/{blob}?comp=appendblock&fromUrl"].put.operationId = "AppendBlob_AppendBlockFromUri"; @@ -270,7 +269,8 @@ directive: {"value": "versions", "name": "Versions"}, {"value": "tags", "name": "Tags"}, {"value": "immutabilitypolicy", "name": "ImmutabilityPolicy"}, - {"value": "legalhold", "name": "LegalHold"} + {"value": "legalhold", "name": "LegalHold"}, + {"value": "deletedwithversions", "name": "DeletedWithVersions"} ]; $.DeleteSnapshots["x-ms-enum"]["name"] = "DeleteSnapshotsOption"; $.DeleteSnapshots["x-ms-enum"]["values"] = [{"value": "include", "name": "IncludeSnapshots"},{"value":"only", "name": "OnlySnapshots"}]; @@ -353,6 +353,9 @@ directive: if (h === "x-ms-meta") { $[h]["x-ms-format"] = "caseinsensitivemap"; } + if (h === "x-ms-lease-id" && $[h].description === "Uniquely identifies a blobs' lease") { + $[h].description = "Uniquely identifies a blob's lease"; + } } - from: swagger-document where: $.parameters @@ -552,8 +555,6 @@ directive: delete $.ContainerItem.properties["Metadata"]; $.ContainerProperties.properties["Metadata"]["x-ms-xml"] = {"name": "../Metadata"}; $.ContainerProperties.properties["DeletedTime"]["x-ms-client-name"] = "DeletedOn"; - $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"] = $.ContainerProperties.properties["VersionLevelWormEnabled"]; - delete $.ContainerProperties.properties["VersionLevelWormEnabled"]; $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"]["x-ms-client-name"] = "HasImmutableStorageWithVersioning "; $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"]["x-ms-client-default"] = false; delete $.ContainerProperties.required; @@ -658,8 +659,6 @@ directive: $["x-ms-deny-encryption-scope-override"]["x-nullable"] = true; $["x-ms-deny-encryption-scope-override"]["x-ms-client-default"] = "false"; $["x-ms-meta"].description = "A set of name-value pair associated with this blob container."; - $["x-ms-immutable-storage-with-versioning-enabled"] = $["x-ms-version-level-worm-enabled"]; - delete $["x-ms-version-level-worm-enabled"]; $["x-ms-immutable-storage-with-versioning-enabled"]["x-ms-client-name"] = "HasImmutableStorageWithVersioning"; $["x-ms-immutable-storage-with-versioning-enabled"]["x-ms-client-default"] = false; $["x-ms-immutable-storage-with-versioning-enabled"]["x-nullable"] = true; @@ -762,6 +761,7 @@ directive: $.BlobItemInternal.properties["VersionId"].description = "A string value that uniquely identifies a blob version."; $.BlobItemInternal.properties["IsCurrentVersion"].description = "Indicates if this is the current version of the blob."; $.BlobItemInternal.properties["BlobType"].description = "Type of the blob."; + $.BlobItemInternal.properties["HasVersionsOnly"].description = "Indicates that this root blob has been deleted, but it has versions that are active."; $.BlobPropertiesInternal.properties["Etag"]["x-ms-client-name"] = "ETag"; $.BlobPropertiesInternal["x-ms-client-name"] = "BlobItemDetails"; @@ -1172,9 +1172,71 @@ directive: - from: swagger-document where: $.definitions transform: > + $.BlobQueryArrowFieldType = { + "type": "string", + "enum": ["Int64", "Bool", "Timestamp", "String", "Double", "Decimal"], + "x-ms-enum": { + "name": "BlobQueryArrowFieldType", + "modelAsString": false, + "values": [ + {"value": "int64", "name": "Int64"}, + {"value": "bool", "name": "Bool"}, + {"value": "timestamp[ms]", "name": "Timestamp"}, + {"value": "string", "name": "String"}, + {"value": "double", "name": "Double"}, + {"value": "decimal", "name": "Decimal"} + ] + }, + "description": "Type of blob query arrow field." + }; if ($.ParquetConfiguration) { $.ParquetConfiguration.properties = {"__placeHolder" : { "type": "integer"}}; } + $.QuerySerialization["x-namespace"] = "_detail"; + $.QueryFormat["x-namespace"] = "_detail"; + $.QueryType["x-namespace"] = "_detail"; + $.DelimitedTextConfiguration["x-namespace"] = "_detail"; + $.JsonTextConfiguration["x-namespace"] = "_detail"; + $.ArrowConfiguration["x-namespace"] = "_detail"; + $.ParquetConfiguration["x-namespace"] = "_detail"; + $.QueryRequest["x-namespace"] = "_detail"; + $.QueryRequest.properties["QueryType"]["x-namespace"] = "_detail"; + $.ArrowField["x-ms-client-name"] = "BlobQueryArrowField"; + $.ArrowField.properties["Type"] = {"$ref": "#/definitions/BlobQueryArrowFieldType"}; + $.DelimitedTextConfiguration.properties["HeadersPresent"]["x-ms-xml"] = $.DelimitedTextConfiguration.properties["HeadersPresent"]["xml"]; + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=query"].post.parameters + transform: > + $.push({"$ref": "#/parameters/EncryptionScope"}); + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=query"].post.responses + transform: > + for (const status_code of ["200", "206"]) { + delete $[status_code].headers["x-ms-meta"]; + delete $[status_code].headers["Content-Length"]; + delete $[status_code].headers["Content-Type"]; + delete $[status_code].headers["Content-Range"]; + delete $[status_code].headers["Content-MD5"]; + delete $[status_code].headers["Content-Encoding"]; + delete $[status_code].headers["Cache-Control"]; + delete $[status_code].headers["Content-Disposition"]; + delete $[status_code].headers["Content-Language"]; + delete $[status_code].headers["x-ms-blob-sequence-number"]; + delete $[status_code].headers["x-ms-blob-type"]; + delete $[status_code].headers["x-ms-copy-completion-time"]; + delete $[status_code].headers["x-ms-copy-status-description"]; + delete $[status_code].headers["x-ms-copy-id"]; + delete $[status_code].headers["x-ms-copy-progress"]; + delete $[status_code].headers["x-ms-copy-source"]; + delete $[status_code].headers["x-ms-copy-status"]; + delete $[status_code].headers["Accept-Ranges"]; + delete $[status_code].headers["x-ms-blob-committed-block-count"]; + delete $[status_code].headers["x-ms-encryption-key-sha256"]; + delete $[status_code].headers["x-ms-encryption-scope"]; + delete $[status_code].headers["x-ms-blob-content-md5"]; + delete $[status_code].headers["x-ms-content-crc64"]; + $[status_code].headers["x-ms-lease-duration"]["x-nullable"] = true; + } ``` ### PutBlockList @@ -1300,6 +1362,10 @@ directive: "x-ms-client-name": "ClearRanges", "x-ms-xml": {"name": "PageList"}, "items": {"$ref": "#/definitions/ClearRange"} + }, + "ContinuationToken": { + "type": "string", + "x-ms-xml": {"name": "PageList/NextMarker"} } } } @@ -1333,6 +1399,10 @@ directive: "x-ms-client-name": "ClearRanges", "x-ms-xml": {"name": "PageList"}, "items": {"$ref": "#/definitions/ClearRange"} + }, + "ContinuationToken": { + "type": "string", + "x-ms-xml": {"name": "PageList/NextMarker"} } } } diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index 6e5d30accc..8b5ca8b38b 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable ( append_blob_client_test.hpp blob_container_client_test.cpp blob_container_client_test.hpp + blob_query_test.cpp blob_sas_test.cpp blob_service_client_test.cpp block_blob_client_test.cpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 08aa9c65e1..deb79a8728 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -1274,4 +1274,25 @@ namespace Azure { namespace Storage { namespace Test { blobClient.Delete(); EXPECT_THROW(blobClient.GetProperties(), StorageException); } + + TEST_F(BlobContainerClientTest, ListBlobsDeletedWithActiveVersions) + { + auto client = GetBlobContainerTestClient(); + client.Create(); + + std::string blobName = "blob" + m_containerName; + auto blobClient = client.GetAppendBlobClient(blobName); + blobClient.Create(); + + auto blobItem + = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions); + ASSERT_TRUE(blobItem.HasVersionsOnly.HasValue()); + EXPECT_FALSE(blobItem.HasVersionsOnly.Value()); + + blobClient.Delete(); + + blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions); + ASSERT_TRUE(blobItem.HasVersionsOnly.HasValue()); + EXPECT_TRUE(blobItem.HasVersionsOnly.Value()); + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp new file mode 100644 index 0000000000..7a139f8093 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "block_blob_client_test.hpp" + +#include +#include +#include + +// cspell:ignore sapote + +namespace Azure { namespace Storage { namespace Test { + + const std::string JsonQueryTestData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": 50} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": 99} +{"id": 104, "name": "clementines", "price": 399} +{"id": 105, "name": "grapes", "price": 150} +{"id": 106, "name": "lemons", "price": 69} +{"id": 107, "name": "pears", "price": 100} +{"id": 108, "name": "cherries", "price": 281} +{"id": 109, "name": "coconut", "price": 178} +{"id": 110, "name": "bananas", "price": 39} +{"id": 111, "name": "peaches", "price": 117} +{"id": 112, "name": "sapote,\"mamey", "price": 50} +)json"; + + const std::string CsvQueryTestData = R"csv( +id,name,price +100,oranges,100 +101,limes,50 +102,berries,199 +103,apples,99 +104,clementines,399 +105,grapes,150 +106,lemons,69 +107,pears,100 +108,cherries,281 +109,coconut,178 +110,bananas,39 +111,peaches,117 +112,sapote\,mamey,50 +)csv"; + + const std::vector ParquetQueryTestData = Core::Convert::Base64Decode( + "UEFSMRUAFewBFewBLBUaFQAVBhUIAAACAAAAGgFkAAAAAAAAAGUAAAAAAAAAZgAAAAAAAABnAAAAAAAAAGgAAAAAAAAA" + "aQAAAAAAAABqAAAAAAAAAGsAAAAAAAAAbAAAAAAAAABtAAAAAAAAAG4AAAAAAAAAbwAAAAAAAABwAAAAAAAAAAAAAAAA" + "AAAAFQAVxAIVxAIsFRoVABUGFQgAAAIAAAAaAQcAAABvcmFuZ2VzBQAAAGxpbWVzBwAAAGJlcnJpZXMGAAAAYXBwbGVz" + "CwAAAGNsZW1lbnRpbmVzBgAAAGdyYXBlcwYAAABsZW1vbnMFAAAAcGVhcnMIAAAAY2hlcnJpZXMHAAAAY29jb251dAcA" + "AABiYW5hbmFzBwAAAHBlYWNoZXMOAAAAc2Fwb3RlLCJtYW1leSIAAAAAAAAAABUAFewBFewBLBUaFQAVBhUIAAACAAAA" + "GgFkAAAAAAAAADIAAAAAAAAAxwAAAAAAAABjAAAAAAAAAI8BAAAAAAAAlgAAAAAAAABFAAAAAAAAAGQAAAAAAAAAGQEA" + "AAAAAACyAAAAAAAAACcAAAAAAAAAdQAAAAAAAAAyAAAAAAAAAAAAAAAAAAAAFQIZTEgGc2NoZW1hFQYAFQQVgAEVAhgC" + "aWQAFQwlAhgEbmFtZSUAABUEFYABFQIYBXByaWNlABYaGRwZPCaaAhwVBBkVABkYAmlkFQAWGhaSAhaSAhkAFgg8GAhw" + "AAAAAAAAABgIZAAAAAAAAAAWAAAZHBUAFQAVAgAAACaEBRwVDBkVABkYBG5hbWUVABYaFuoCFuoCGQAWmgI8GA5zYXBv" + "dGUsIm1hbWV5IhgGYXBwbGVzFgAAGRwVABUAFQIAAAAmlgccFQQZFQAZGAVwcmljZRUAFhoWkgIWkgIZABaEBTwYCI8B" + "AAAAAAAAGAgnAAAAAAAAABYAABkcFQAVABUCAAAAFo4HFhoAGRwYBnBhbmRhcxiRBXsiY29sdW1uX2luZGV4ZXMiOiBb" + "eyJmaWVsZF9uYW1lIjogbnVsbCwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiBudWxsLCAibnVtcHlfdHlwZSI6ICJv" + "YmplY3QiLCAicGFuZGFzX3R5cGUiOiAibWl4ZWQtaW50ZWdlciJ9XSwgImNvbHVtbnMiOiBbeyJmaWVsZF9uYW1lIjog" + "ImlkIiwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiAiaWQiLCAibnVtcHlfdHlwZSI6ICJpbnQ2NCIsICJwYW5kYXNf" + "dHlwZSI6ICJpbnQ2NCJ9LCB7ImZpZWxkX25hbWUiOiAibmFtZSIsICJtZXRhZGF0YSI6IG51bGwsICJuYW1lIjogIm5h" + "bWUiLCAibnVtcHlfdHlwZSI6ICJvYmplY3QiLCAicGFuZGFzX3R5cGUiOiAidW5pY29kZSJ9LCB7ImZpZWxkX25hbWUi" + "OiAicHJpY2UiLCAibWV0YWRhdGEiOiBudWxsLCAibmFtZSI6ICJwcmljZSIsICJudW1weV90eXBlIjogImludDY0Iiwg" + "InBhbmRhc190eXBlIjogImludDY0In1dLCAiY3JlYXRvciI6IHsibGlicmFyeSI6ICJmYXN0cGFycXVldCIsICJ2ZXJz" + "aW9uIjogIjAuOC4xIn0sICJpbmRleF9jb2x1bW5zIjogW3sia2luZCI6ICJyYW5nZSIsICJuYW1lIjogbnVsbCwgInN0" + "YXJ0IjogMCwgInN0ZXAiOiAxLCAic3RvcCI6IDEzfV0sICJwYW5kYXNfdmVyc2lvbiI6ICIxLjQuMiIsICJwYXJ0aXRp" + "b25fY29sdW1ucyI6IFtdfQAYKmZhc3RwYXJxdWV0LXB5dGhvbiB2ZXJzaW9uIDAuOC4xIChidWlsZCAwKQDXAwAAUEFS" + "MQ=="); + + TEST_F(BlockBlobClientTest, QueryJsonInputCsvOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(JsonQueryTestData.data()), JsonQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateJsonTextOptions(); + + { + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions(); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103,apples,99 +106,lemons,69 +110,bananas,39 +112,"sapote,""mamey",50 +)csv"); + } + + { + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions("|", ".", "[", "\\", true); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103.apples.99|106.lemons.69|110.bananas.39|112.sapote,"mamey.50|)csv"); + } + } + + TEST_F(BlockBlobClientTest, QueryCsvInputJsonOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions("\n", ",", "\"", "\\", true); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions("|"); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); + } + + TEST_F(BlockBlobClientTest, DISABLED_QueryCsvInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions("\n", ",", "\"", "\\", true); + std::vector fields; + Blobs::Models::BlobQueryArrowField field; + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::Decimal; + field.Name = "price"; + field.Precision = 10; + field.Scale = 2; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+gAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAACAAAAAQAAAAAQAAAC" + "c////AAABBxAAAAAgAAAABAAAAAAAAAAFAAAAcHJpY2UAAAAIAAwABAAIAAgAAAAKAAAAAgAAANT///" + "8AAAEFEAAAABwAAAAEAAAAAAAAAAQAAABuYW1lAAAAAAQABAAEAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEC" + "EAAAABwAAAAEAAAAAAAAAAIAAABpZAAACAAMAAgABwAIAAAAAAAAAUAAAAAAAAAA//////" + "AAAAAUAAAAAAAAAAwAGgAGAAUACAAMAAwAAAAAAwQAHAAAAAgAAAAAAAAAAAAAAAAACgAMAAAABAAIAAoAAACA" + "AAAABAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAQAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAMA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/////" + "4AAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAACYAAAAAAAAAAAACgAYAAwABAAIAAoAAACMAAAAE" + "AAAAAQAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAA" + "AAAAgAAAAAAAAABQAAAAAAAAAOAAAAAAAAAAfAAAAAAAAAFgAAAAAAAAAAAAAAAAAAABYAAAAAAAAAEAAAAAAA" + "AAAAAAAAAMAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAABnAAAAAAA" + "AAGoAAAAAAAAAbgAAAAAAAABwAAAAAAAAAAAAAAAGAAAADAAAABMAAAAfAAAAAAAAAGFwcGxlc2xlbW9uc2Jhb" + "mFuYXNzYXBvdGUsbWFtZXkAYwAAAAAAAAAAAAAAAAAAAEUAAAAAAAAAAAAAAAAAAAAnAAAAAAAAAAAAAAAAAAA" + "AMgAAAAAAAAAAAAAAAAAAAA=="); + EXPECT_EQ(data, expectedData); + } + + TEST_F(BlockBlobClientTest, DISABLED_QueryParquetInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateParquetTextOptions(); + std::vector fields; + Blobs::Models::BlobQueryArrowField field; + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "price"; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAAB4AAAAOAAAAAQAAAC" + "k////AAABAhAAAAAYAAAABAAAAAAAAAAFAAAAcHJpY2UAAACY////AAAAAUAAAADU////" + "AAABBRAAAAAcAAAABAAAAAAAAAAEAAAAbmFtZQAAAAAEAAQABAAAABAAFAAIAAYABwAMAAAAEAAQAAAAAAABAh" + "AAAAAcAAAABAAAAAAAAAACAAAAaWQAAAgADAAIAAcACAAAAAAAAAFAAAAAAAAAAP/////" + "wAAAAFAAAAAAAAAAMABoABgAFAAgADAAMAAAAAAMEABwAAAAIAAAAAAAAAAAAAAAAAAoADAAAAAQACAAKAAAAg" + "AAAAAQAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAADA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAZwAAAAAA" + "AAAAAAAABgAAAGFwcGxlcwAAYwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAagAAAAAA" + "AAAAAAAABgAAAGxlbW9ucwAARQAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABwAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAbgAAAAAA" + "AAAAAAAABwAAAGJhbmFuYXMAJwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAKAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAADgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAcAAAAAAA" + "AAAAAAAADgAAAHNhcG90ZSwibWFtZXkiAAAyAAAAAAAAAP////8AAAAA"); + EXPECT_EQ(data, expectedData); + } + + TEST_F(BlockBlobClientTest, QueryWithError_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + const std::string malformedData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": "aa"} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": "bb"} +{"id": 104, "name": "clementines", "price": 399} +xx +)json"; + client.UploadFrom(reinterpret_cast(malformedData.data()), malformedData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateJsonTextOptions(); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + + try + { + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Core::Http::HttpStatusCode::Ok); + EXPECT_EQ(e.ReasonPhrase, "OK"); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "ParseError"); + EXPECT_FALSE(e.Message.empty()); + EXPECT_FALSE(std::string(e.what()).empty()); + } + + bool progressCallbackCalled = false; + queryOptions.ProgressHandler + = [&malformedData, &progressCallbackCalled](int64_t offset, int64_t totalBytes) { + EXPECT_EQ(totalBytes, static_cast(malformedData.size())); + EXPECT_TRUE(offset >= 0 && offset <= totalBytes); + progressCallbackCalled = true; + }; + int numNonFatalErrors = 0; + int numFatalErrors = 0; + queryOptions.ErrorHandler = [&numNonFatalErrors, &numFatalErrors](Blobs::BlobQueryError e) { + if (e.IsFatal) + { + ++numFatalErrors; + } + else + { + ++numNonFatalErrors; + } + }; + queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + queryResponse.Value.BodyStream->ReadToEnd(); + + EXPECT_EQ(numNonFatalErrors, 2); + EXPECT_EQ(numFatalErrors, 1); + EXPECT_TRUE(progressCallbackCalled); + } + + TEST_F(BlockBlobClientTest, QueryDefaultInputOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + const std::string csvData = "100,oranges,100"; + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + auto queryResponse = client.Query("SELECT * from BlobStorage;"); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + } + + TEST_F(BlockBlobClientTest, QueryLargeBlob_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + constexpr size_t DataSize = static_cast(32_MB); + + int recordCounter = 0; + std::string csvData; + std::string jsonData; + while (csvData.size() < DataSize) + { + std::string counter = std::to_string(recordCounter++); + std::string record = RandomString(static_cast(RandomInt(1, 3000))); + csvData += counter + "," + record + "\n"; + jsonData += "{\"_1\":\"" + counter + "\",\"_2\":\"" + record + "\"}\n"; + } + + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions(); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage;", queryOptions); + + size_t comparePos = 0; + std::vector readBuffer(4096); + while (true) + { + auto s = queryResponse.Value.BodyStream->Read(readBuffer.data(), readBuffer.size()); + if (s == 0) + { + break; + } + ASSERT_TRUE(comparePos + s <= jsonData.size()); + ASSERT_EQ( + std::string(readBuffer.begin(), readBuffer.begin() + s), jsonData.substr(comparePos, s)); + comparePos += s; + } + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLeaseId_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + Blobs::BlobLeaseClient leaseClient(client, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + leaseClient.Acquire(Blobs::BlobLeaseClient::InfiniteLeaseDuration); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionTags_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + std::map tags = {{"k1", "value1"}}; + client.SetTags(tags); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.TagConditions = "k1 = 'value1'"; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.TagConditions = "k1 = 'dummy'"; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLastModifiedTime_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + auto lastModifiedTime = client.GetProperties().Value.LastModified; + auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); + auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.IfModifiedSince = timeBeforeStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfModifiedSince = timeAfterStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Blobs::QueryBlobOptions(); + queryOptions.AccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + queryOptions.AccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionETag_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + auto etag = client.GetProperties().Value.ETag; + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.IfMatch = etag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfMatch = DummyETag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Blobs::QueryBlobOptions(); + queryOptions.AccessConditions.IfNoneMatch = DummyETag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfNoneMatch = etag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + } +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp index 8be9c7d257..b2d28d8075 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp @@ -193,6 +193,28 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(BlobServiceClientTest, ListSystemContainers) + { + const std::string testName = GetTestNameLowerCase(); + auto client = GetClientForTest(testName); + Azure::Storage::Blobs::ListBlobContainersOptions options; + options.Include = Blobs::Models::ListBlobContainersIncludeFlags::System; + std::vector containers; + for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + for (const auto& c : pageResult.BlobContainers) + { + if (c.Name[0] == '$') + { + containers.push_back(c.Name); + } + } + } + + EXPECT_FALSE(containers.empty()); + } + TEST_F(BlobServiceClientTest, GetProperties) { const std::string testName = GetTestName(); diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 0b49b12633..b1bae299eb 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -138,6 +138,33 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(static_cast(clearRanges[0].Length.Value()), 1_KB); } + TEST_F(PageBlobClientTest, GetPageRangesContinuation) + { + auto const testName(GetTestName()); + auto pageBlobClient = GetPageBlobClient(testName); + + std::vector blobContent = std::vector(static_cast(512), 'x'); + + pageBlobClient.Create(8_KB, m_blobUploadOptions); + auto pageContent = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); + pageBlobClient.UploadPages(0, pageContent); + pageContent.Rewind(); + pageBlobClient.UploadPages(1024, pageContent); + pageContent.Rewind(); + pageBlobClient.UploadPages(4096, pageContent); + + Blobs::GetPageRangesOptions options; + options.PageSizeHint = 1; + size_t numRanges = 0; + for (auto pageResult = pageBlobClient.GetPageRanges(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + EXPECT_EQ(pageResult.PageRanges.size(), static_cast(1)); + numRanges += pageResult.PageRanges.size(); + } + EXPECT_EQ(numRanges, static_cast(3)); + } + TEST_F(PageBlobClientTest, UploadFromUri) { auto const testName(GetTestName()); diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json new file mode 100644 index 0000000000..84bac2a971 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json @@ -0,0 +1,135 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "34530d0a-1ebb-4c9b-5ba3-8fa5df36b8ad", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "etag": "\"0x8DA36E07CE8971C\"", + "last-modified": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "34530d0a-1ebb-4c9b-5ba3-8fa5df36b8ad", + "x-ms-request-id": "d7f9d950-b01e-001e-74c9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "5e3c0bdc-9018-4c50-6e26-eec118c34b35", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "etag": "\"0x8DA36E07D1B2448\"", + "last-modified": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "5e3c0bdc-9018-4c50-6e26-eec118c34b35", + "x-ms-request-id": "d7f9da01-b01e-001e-18c9-68361a000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-05-16T02:05:02.5861463Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions/blobblobcontainerclienttestlistblobsdeletedwithactiveversions" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ff54697b-8d1a-4358-7067-e137517b491a", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "blobblobcontainerclienttestlistblobsdeletedwithactiveversionsblobblobcontainerclienttestlistblobsdeletedwithactiveversions2022-05-16T02:05:02.5861463ZtrueMon, 16 May 2022 02:05:02 GMTMon, 16 May 2022 02:05:02 GMT0x8DA36E07D1B24480application/octet-streamAppendBlobunlockedavailabletrue", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "ff54697b-8d1a-4358-7067-e137517b491a", + "x-ms-request-id": "d7f9dae8-b01e-001e-6fc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?comp=list&include=deletedwithversions&prefix=blobblobcontainerclienttestlistblobsdeletedwithactiveversions&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "57212c10-680b-408d-772d-a4209c2d9aaf", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "57212c10-680b-408d-772d-a4209c2d9aaf", + "x-ms-delete-type-permanent": "false", + "x-ms-request-id": "d7f9dbb7-b01e-001e-2dc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions/blobblobcontainerclienttestlistblobsdeletedwithactiveversions" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "072dc394-f953-4f03-45b9-562e379ac7f2", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "blobblobcontainerclienttestlistblobsdeletedwithactiveversionsblobblobcontainerclienttestlistblobsdeletedwithactiveversionstrueMon, 16 May 2022 02:05:02 GMTMon, 16 May 2022 02:05:02 GMT0x8DA36E07D1B24480application/octet-streamAppendBlobunlockedavailabletrue", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "072dc394-f953-4f03-45b9-562e379ac7f2", + "x-ms-request-id": "d7f9dcbd-b01e-001e-1fc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?comp=list&include=deletedwithversions&prefix=blobblobcontainerclienttestlistblobsdeletedwithactiveversions&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "f342e63c-21d5-4a16-5017-616ea9f7798c", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:03 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f342e63c-21d5-4a16-5017-616ea9f7798c", + "x-ms-request-id": "d7f9dd35-b01e-001e-0bc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json index 666b2b7bfb..429767bcfd 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json @@ -2,32 +2,32 @@ "networkCallRecords": [ { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "dcd6fe54-0c7e-4e74-5b20-f839a7d7d241", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "35419e25-3165-4f93-51f9-5bab0eaec1ae", + "x-ms-version": "2020-10-02" }, "Method": "GET", "Response": { - "BODY": "1.0truetruetruetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true1falsetrueindex.html404.html2019-12-12", + "BODY": "1.0truetruetruetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-02-10", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Sun, 20 Feb 2022 05:27:42 GMT", + "date": "Mon, 16 May 2022 12:49:31 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", "vary": "Origin", - "x-ms-client-request-id": "dcd6fe54-0c7e-4e74-5b20-f839a7d7d241", - "x-ms-request-id": "9fced8f8-d01e-0015-121a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "35419e25-3165-4f93-51f9-5bab0eaec1ae", + "x-ms-request-id": "da10e09f-c01e-0082-5723-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { "content-type": "application/xml; charset=UTF-8", - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "2f4337b3-0234-44ab-7609-8d9a59404ede", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6944decb-5ada-41b4-6365-2f9b90220ebe", + "x-ms-version": "2020-10-02" }, "Method": "PUT", "Response": { @@ -35,42 +35,42 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Sun, 20 Feb 2022 05:27:43 GMT", + "date": "Mon, 16 May 2022 12:49:33 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "2f4337b3-0234-44ab-7609-8d9a59404ede", - "x-ms-request-id": "9fced97e-d01e-0015-091a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "6944decb-5ada-41b4-6365-2f9b90220ebe", + "x-ms-request-id": "da10e162-c01e-0082-1423-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "8103e424-9d4d-46eb-6c22-2adc1c5a81c4", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "a72f237f-cb76-4e90-52a6-eea437169f93", + "x-ms-version": "2020-10-02" }, "Method": "GET", "Response": { - "BODY": "1.0falsefalsefalsetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-08-04", + "BODY": "1.0falsefalsefalsetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-10-02", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Sun, 20 Feb 2022 05:27:53 GMT", + "date": "Mon, 16 May 2022 12:49:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", "vary": "Origin", - "x-ms-client-request-id": "8103e424-9d4d-46eb-6c22-2adc1c5a81c4", - "x-ms-request-id": "9fceee3a-d01e-0015-751a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "a72f237f-cb76-4e90-52a6-eea437169f93", + "x-ms-request-id": "da10fdaf-c01e-0082-1623-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { "content-type": "application/xml; charset=UTF-8", - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "251ae95b-e55f-40ce-7935-edc2ecf28444", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ff874718-2fc2-479f-6308-fe94f0c824c9", + "x-ms-version": "2020-10-02" }, "Method": "PUT", "Response": { @@ -78,11 +78,11 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Sun, 20 Feb 2022 05:27:53 GMT", + "date": "Mon, 16 May 2022 12:49:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "251ae95b-e55f-40ce-7935-edc2ecf28444", - "x-ms-request-id": "9fceeec3-d01e-0015-771a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "ff874718-2fc2-479f-6308-fe94f0c824c9", + "x-ms-request-id": "da10fe3e-c01e-0082-1b23-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" } diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json new file mode 100644 index 0000000000..55c2b9d6b0 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json @@ -0,0 +1,26 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "966941db-bfd0-44ac-6a24-e4c192e1c693", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "$blobchangefeedTue, 07 Jul 2020 10:10:36 GMT\"0x8D8225DFE721DFD\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse$logsTue, 07 Jul 2020 10:10:36 GMT\"0x8D8225DFE6A2D4B\"unlockedavailable$account-encryption-keyfalsefalsefalsefalseblobcontainerclienttestblobsastestSat, 19 Feb 2022 15:08:02 GMT\"0x8D9F3B99F86B533\"unlockedavailableblob$account-encryption-keyfalsefalsefalsetrueblockblobclienttestimmutabilitySat, 19 Feb 2022 04:12:29 GMT\"0x8D9F35E0B94285B\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestimmutabilityaccessconditionSat, 19 Feb 2022 06:24:17 GMT\"0x8D9F37074DCCA0C\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestlegalholdSat, 19 Feb 2022 05:37:18 GMT\"0x8D9F369E4FEB44E\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestqueryparquetinputparquetoutputSat, 14 May 2022 14:28:57 GMT\"0x8DA35B6149118ED\"unlockedavailable$account-encryption-keyfalsefalsefalsefalseblockblobclienttestuploaddownloadTue, 10 May 2022 15:45:35 GMT\"0x8DA329C1FD1864E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsesample-containerTue, 10 May 2022 10:52:48 GMT\"0x8DA327338F99283\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:42 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "966941db-bfd0-44ac-6a24-e4c192e1c693", + "x-ms-request-id": "b3fadb96-b01e-0053-2ec9-68f9f6000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=list&include=system" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json new file mode 100644 index 0000000000..94ea774849 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json @@ -0,0 +1,220 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "3e5a06de-602d-414d-6b45-d25f91578ffa", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:59 GMT", + "etag": "\"0x8DA36E09F89EDFA\"", + "last-modified": "Mon, 16 May 2022 02:06:00 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3e5a06de-602d-414d-6b45-d25f91578ffa", + "x-ms-request-id": "cda9954f-401e-0047-71c9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "b211fc89-cee6-49d2-529d-ee204b60eb53", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:59 GMT", + "etag": "\"0x8DA36E09FBD275F\"", + "last-modified": "Mon, 16 May 2022 02:06:00 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "b211fc89-cee6-49d2-529d-ee204b60eb53", + "x-ms-request-id": "cda9961a-401e-0047-20c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-05-16T02:06:00.6894431Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "48c04419-42e4-4417-7e3e-0cc102b855a3", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:00 GMT", + "etag": "\"0x8DA36E0A00374C8\"", + "last-modified": "Mon, 16 May 2022 02:06:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "48c04419-42e4-4417-7e3e-0cc102b855a3", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda99731-401e-0047-22c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ec267c55-a846-4933-60fb-37430b21e4cf", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:00 GMT", + "etag": "\"0x8DA36E0A049E933\"", + "last-modified": "Mon, 16 May 2022 02:06:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "ec267c55-a846-4933-60fb-37430b21e4cf", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda998a4-401e-0047-7ac9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "55fb6437-b537-4d3e-6b91-a9f357d54a10", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:01 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "55fb6437-b537-4d3e-6b91-a9f357d54a10", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda999d1-401e-0047-15c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ae8b224a-7150-45bf-7e9a-93d441962963", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n05112!16!MDAwMDA0ITEwMjQh", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:02 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "ae8b224a-7150-45bf-7e9a-93d441962963", + "x-ms-request-id": "cda99b34-401e-0047-55c9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "d673efcc-de67-47f7-72fa-94634566af41", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n102415352!16!MDAwMDA0ITQwOTYh", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:02 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "d673efcc-de67-47f7-72fa-94634566af41", + "x-ms-request-id": "cda99e3d-401e-0047-1bc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&marker=2!16!MDAwMDA0ITEwMjQh&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "3cee12a6-f7ed-4e2d-5ada-da0207493646", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n40964607", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:03 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "3cee12a6-f7ed-4e2d-5ada-da0207493646", + "x-ms-request-id": "cda99eee-401e-0047-3dc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&marker=2!16!MDAwMDA0ITQwOTYh&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6340d172-a8eb-46bc-642b-a90af0c9b640", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:03 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "6340d172-a8eb-46bc-642b-a90af0c9b640", + "x-ms-request-id": "cda99fa5-401e-0047-5fc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation?restype=container" + } + ] +}