diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 836ad5972a..a31ee36f7c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,7 +6,7 @@ ################ # Git Hub integration and bot rules -/.github/ @AlexGhiondea @jsquire @rickwinter +/.github/ @jsquire @rickwinter ########### # SDK diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 9c6a632f34..b522f7d9c6 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -67,6 +67,8 @@ "Intel", "itfactor", "iusg", + "jepio", + "Jeremi", "Kirilov", "lcov", "LIBCMTD", @@ -89,6 +91,7 @@ "okhttp", "PBYTE", "pdbs", + "Piotrowski", "PUCHAR", "pwsh", "Ragrs", diff --git a/CMakeLists.txt b/CMakeLists.txt index f56c610fc2..6b64d46954 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,7 @@ include(AzureVersion) if(BUILD_SAMPLES) add_subdirectory(samples/helpers/get-env) + add_subdirectory(samples/helpers/service) endif() # sub-projects diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 4c83ad3164..b7785df9dd 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -375,7 +375,7 @@ try { $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) Log "Generated base name '$BaseName' for CI build" } else { - $BaseName = GetBaseName $UserName $ServiceDirectory + $BaseName = GetBaseName $UserName (GetServiceLeafDirectoryName $ServiceDirectory) Log "BaseName was not set. Using default base name '$BaseName'" } } @@ -520,7 +520,7 @@ try { $ResourceGroupName } elseif ($CI) { # Format the resource group name based on resource group naming recommendations and limitations. - "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') + "rg-{0}-$BaseName" -f ($serviceName -replace '[\.\\\/:]', '-').ToLowerInvariant().Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-') } else { "rg-$BaseName" } diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md index 8463501c0f..957a1f9c35 100644 --- a/eng/common/TestResources/README.md +++ b/eng/common/TestResources/README.md @@ -77,6 +77,49 @@ setx KEYVAULT_SKU ${env:KEYVAULT_SKU} setx AZURE_KEYVAULT_URL ${env:AZURE_KEYVAULT_URL} ``` +### Pre- and Post- Scripts + +Sometimes creating test resources requires either some work to be done prior to or after the main test-resources.json script is executed. +For these scenarios a `test-resources-pre.ps1` or `test-resources-post.ps1`, respectively, can be created in the same folder as the `test-resources.json` file. + +For example, it may be necessary to create artifacts prior to provisioning the actual resource, such as a certificate. +Typically the created artifact will need to be passed to `test-resources.json` to be used in the ARM template or as output (or both). + +Below is an example of how `$templateFileParameters` can be used to pass data from the `pre-` script to `test-resources.json`. + +**Snippet from `test-resources-pre.ps1`** +```powershell +$cert = New-X509Certificate2 -SubjectName 'E=opensource@microsoft.com, CN=Azure SDK, OU=Azure SDK, O=Microsoft, L=Frisco, S=TX, C=US' -ValidDays 3652 +# Create new entries in $templateFileParameters +$templateFileParameters['ConfidentialLedgerPrincipalPEM'] = Format-X509Certificate2 -Certificate $cert +$templateFileParameters['ConfidentialLedgerPrincipalPEMPK'] = Format-X509Certificate2 -Type Pkcs8 -Certificate $cert +``` + +**Snippet from the corresponding `test-resources.json`.** + +Note that the values present in `$templateFileParameters` will map to parameters of the same name. +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "_comment": "Other required parameters would go here... (this is not part of the actual test-resources.json)", + "ConfidentialLedgerPrincipalPEM": { + "type": "string", + "metadata": { + "description": "The certificate to configure as a certBasedSecurityPrincipal." + } + }, + "ConfidentialLedgerPrincipalPEMPK": { + "type": "string", + "metadata": { + "description": "The certificate to configure as a certBasedSecurityPrincipal." + } + } + }, +} +``` + ### Cleaning up Resources By default, resource groups are tagged with a `DeleteAfter` value and date according to the default or specified diff --git a/eng/common/TestResources/SubConfig-Helpers.ps1 b/eng/common/TestResources/SubConfig-Helpers.ps1 index 04cc7d3729..cc93def6aa 100644 --- a/eng/common/TestResources/SubConfig-Helpers.ps1 +++ b/eng/common/TestResources/SubConfig-Helpers.ps1 @@ -1,4 +1,5 @@ function BuildServiceDirectoryPrefix([string]$serviceName) { + $serviceName = $serviceName -replace '[\./\\]', '_' return $serviceName.ToUpperInvariant() + "_" } @@ -17,8 +18,8 @@ function GetUserName() { function GetBaseName([string]$user, [string]$serviceDirectoryName) { # Handle service directories in nested directories, e.g. `data/aztables` - $serviceDirectorySafeName = $serviceDirectoryName -replace '[/\\]', '' - return "$user$serviceDirectorySafeName" + $serviceDirectorySafeName = $serviceDirectoryName -replace '[\./\\]', '' + return "$user$serviceDirectorySafeName".ToLowerInvariant() } function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$value, [array]$allowedValues = @()) diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index 6c02706d22..cf5c0c5ac4 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -34,5 +34,5 @@ steps: -Force ` -Verbose displayName: Remove test resources - condition: eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true') + condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true diff --git a/eng/common/pipelines/templates/steps/install-pipeline-generation.yml b/eng/common/pipelines/templates/steps/install-pipeline-generation.yml index b86f541a05..266de6197e 100644 --- a/eng/common/pipelines/templates/steps/install-pipeline-generation.yml +++ b/eng/common/pipelines/templates/steps/install-pipeline-generation.yml @@ -9,7 +9,7 @@ steps: - script: > dotnet tool install Azure.Sdk.Tools.PipelineGenerator - --version 1.0.2-dev.20220321.1 + --version 1.0.2-dev.20220504.1 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --tool-path ${{parameters.ToolPath}} workingDirectory: $(Pipeline.Workspace)/pipeline-generator diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index 1ca5881196..090818cc9b 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -89,9 +89,6 @@ } }, "TargetArchitecture": { - "x86": { - "CMAKE_GENERATOR_PLATFORM": "Win32" - }, "x64": { "CMAKE_GENERATOR_PLATFORM": "x64", "VCPKG_DEFAULT_TRIPLET": "x64-uwp" diff --git a/samples/helpers/service/CMakeLists.txt b/samples/helpers/service/CMakeLists.txt new file mode 100644 index 0000000000..8f25a20ca3 --- /dev/null +++ b/samples/helpers/service/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT + +cmake_minimum_required (VERSION 3.12) +project(service LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) + +add_library( + service + INTERFACE + inc/azure/service/client.hpp +) + +target_include_directories(service INTERFACE inc) diff --git a/samples/helpers/service/LICENSE b/samples/helpers/service/LICENSE new file mode 100644 index 0000000000..51b6a76e54 --- /dev/null +++ b/samples/helpers/service/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/samples/helpers/service/README.md b/samples/helpers/service/README.md new file mode 100644 index 0000000000..b6bdc92151 --- /dev/null +++ b/samples/helpers/service/README.md @@ -0,0 +1,3 @@ +# Generic Service for Samples + +This is a helper library for samples that provides a generic service client library. diff --git a/sdk/identity/azure-identity/samples/azure/service/client.hpp b/samples/helpers/service/inc/azure/service/client.hpp similarity index 94% rename from sdk/identity/azure-identity/samples/azure/service/client.hpp rename to samples/helpers/service/inc/azure/service/client.hpp index 960d3b6b16..b489445ce3 100644 --- a/sdk/identity/azure-identity/samples/azure/service/client.hpp +++ b/samples/helpers/service/inc/azure/service/client.hpp @@ -19,11 +19,11 @@ namespace Azure { namespace Service { public: explicit Client( - const std::string& someParameter, + const std::string& serviceUrl, std::shared_ptr credential) : m_credential(std::move(credential)) { - static_cast(someParameter); // to suppress the "unused variable" warning. + static_cast(serviceUrl); // to suppress the "unused variable" warning. } void DoSomething(const Core::Context& context) const diff --git a/samples/integration/vcpkg-all-smoke/src/main.cpp b/samples/integration/vcpkg-all-smoke/src/main.cpp index b17694e3ad..09f2445512 100644 --- a/samples/integration/vcpkg-all-smoke/src/main.cpp +++ b/samples/integration/vcpkg-all-smoke/src/main.cpp @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include @@ -67,10 +67,11 @@ int main() ShareLeaseClient shareLeaseClient(shareFileClient, leaseID); ShareServiceClient shareServiceClient(smokeUrl); - //Attestation + // Attestation std::cout << "Creating Attestation Clients" << std::endl; AttestationClient attestationClient(AttestationClient::Create(smokeUrl)); - AttestationAdministrationClient attestationAdminClient(AttestationAdministrationClient::Create(smokeUrl, credential)); + AttestationAdministrationClient attestationAdminClient( + AttestationAdministrationClient::Create(smokeUrl, credential)); std::cout << "Successfully Created the Clients" << std::endl; } diff --git a/samples/integration/vcpkg-keyvault/src/main.cpp b/samples/integration/vcpkg-keyvault/src/main.cpp index b8d2c69232..541a95160c 100644 --- a/samples/integration/vcpkg-keyvault/src/main.cpp +++ b/samples/integration/vcpkg-keyvault/src/main.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/attestation/azure-security-attestation/CHANGELOG.md b/sdk/attestation/azure-security-attestation/CHANGELOG.md index 77f0ee27ef..b6dc5d1a46 100644 --- a/sdk/attestation/azure-security-attestation/CHANGELOG.md +++ b/sdk/attestation/azure-security-attestation/CHANGELOG.md @@ -1,14 +1,18 @@ # Release History -## 1.0.0-beta.2 (Unreleased) +## 1.0.0-beta.3 (Unreleased) ### Features Added -No new features, addressed API Review Feedback. - ### Breaking Changes -Breaking Changes from API Review +### Bugs Fixed + +### Other Changes + +## 1.0.0-beta.2 (2022-05-10) + +### Breaking Changes - Renamed `AttestationOpenIdMetadata` type to `OpenIdMetadata`. - Renamed `AttestationSigningCertificateResult` type to `TokenValidationCertificateResult` to more accurately reflect the @@ -36,8 +40,6 @@ Breaking Changes from API Review - `AttestationAdministrationClient::RemovePolicyManagementCertificate` becomes `AttestationAdministrationClient::RemoveIsolatedModeCertificate`. - Removed `ClientVersion` API from `AttestationClient` and `AttestationAdministrationClient` -### Bugs Fixed - ### Other Changes - Added `Endpoint` property to `AttestationClient` and `AttestationAdministrationClient` diff --git a/sdk/attestation/azure-security-attestation/CMakeLists.txt b/sdk/attestation/azure-security-attestation/CMakeLists.txt index 413ca7d523..0fd43b0753 100644 --- a/sdk/attestation/azure-security-attestation/CMakeLists.txt +++ b/sdk/attestation/azure-security-attestation/CMakeLists.txt @@ -31,38 +31,38 @@ find_package(OpenSSL REQUIRED) set( AZURE_ATTESTATION_HEADER - inc/azure/attestation/dll_import_export.hpp - inc/azure/attestation/attestation_client.hpp inc/azure/attestation/attestation_administration_client.hpp - inc/azure/attestation.hpp + inc/azure/attestation/attestation_client.hpp inc/azure/attestation/attestation_client_models.hpp inc/azure/attestation/attestation_client_options.hpp - - src/private/package_version.hpp - src/private/attestation_common_request.hpp - src/private/attestation_deserializers_private.hpp - - src/private/attestation_client_models_private.hpp - src/private/crypto/inc/crypto.hpp - - src/private/attestation_client_private.hpp) + inc/azure/attestation/dll_import_export.hpp + inc/azure/attestation/rtti.hpp + inc/azure/attestation.hpp +) set( AZURE_ATTESTATION_SOURCE - src/attestation_client.cpp - src/attestation_administration_client.cpp - src/attestation_client_options.cpp + src/private/crypto/inc/crypto.hpp + src/private/crypto/openssl/openssl_helpers.hpp + src/private/crypto/openssl/opensslcert.cpp + src/private/crypto/openssl/opensslcert.hpp + src/private/crypto/openssl/opensslcrypto.cpp + src/private/crypto/openssl/opensslkeys.cpp + src/private/crypto/openssl/opensslkeys.hpp + src/private/attestation_client_models_private.hpp + src/private/attestation_client_private.cpp + src/private/attestation_client_private.hpp src/private/attestation_common_request.cpp + src/private/attestation_common_request.hpp src/private/attestation_deserializers_private.cpp + src/private/attestation_deserializers_private.hpp src/private/jsonhelpers.cpp - src/private/attestation_client_private.cpp) - -set( - AZURE_ATTESTATION_SOURCE - ${AZURE_ATTESTATION_SOURCE} - src/private/crypto/openssl/opensslcrypto.cpp - src/private/crypto/openssl/opensslkeys.cpp - src/private/crypto/openssl/opensslcert.cpp) + src/private/jsonhelpers_private.hpp + src/private/package_version.hpp + src/attestation_administration_client.cpp + src/attestation_client.cpp + src/attestation_client_options.cpp +) add_library(azure-security-attestation ${AZURE_ATTESTATION_HEADER} ${AZURE_ATTESTATION_SOURCE}) diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation.hpp index 8b8f6e02b4..17e41da1a8 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation.hpp @@ -10,4 +10,7 @@ #include "attestation/attestation_administration_client.hpp" #include "attestation/attestation_client.hpp" +#include "attestation/attestation_client_models.hpp" +#include "attestation/attestation_client_options.hpp" #include "attestation/dll_import_export.hpp" +#include "attestation/rtti.hpp" diff --git a/sdk/attestation/azure-security-attestation/src/private/package_version.hpp b/sdk/attestation/azure-security-attestation/src/private/package_version.hpp index f762853cea..699f0f0131 100644 --- a/sdk/attestation/azure-security-attestation/src/private/package_version.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/package_version.hpp @@ -11,7 +11,7 @@ #define AZURE_ATTESTATION_VERSION_MAJOR 1 #define AZURE_ATTESTATION_VERSION_MINOR 0 #define AZURE_ATTESTATION_VERSION_PATCH 0 -#define AZURE_ATTESTATION_VERSION_PRERELEASE "beta.2" +#define AZURE_ATTESTATION_VERSION_PRERELEASE "beta.3" #define AZURE_ATTESTATION_VERSION_ITOA_HELPER(i) #i #define AZURE_ATTESTATION_VERSION_ITOA(i) AZURE_ATTESTATION_VERSION_ITOA_HELPER(i) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 3294004d49..ea1d9d9db9 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,13 +1,23 @@ # Release History -## 1.6.0-beta.1 (Unreleased) +## 1.7.0-beta.1 (Unreleased) ### Features Added -- Add `Azure::Core::Http::Request` constructor overload to support payload and non-buffered response. - ### Breaking Changes +- Removed `noexcept` specification from `Azure::Core::Context::IsCancelled()`. + +### Bugs Fixed + +### Other Changes + +## 1.6.0 (2022-05-05) + +### Features Added + +- Add `Azure::Core::Http::Request` constructor overload to support payload and non-buffered response. + ### Bugs Fixed - [[#3537]](https://github.com/Azure/azure-sdk-for-cpp/issues/3537) Updated field type `CurlTransportOptions.Proxy` from `std::string` to `Azure::Nullable`. This allows libcurl to ignore the proxy settings from the environment when the string is empty. @@ -16,6 +26,13 @@ ### Other Changes - [[#3581]](https://github.com/Azure/azure-sdk-for-cpp/issues/3581) Update log level in retry policy from warning to informational. +- Updated the MD5 Hash implementation to work on top of OpenSSL 3.0. (A community contribution, courtesy of _[jepio](https://github.com/jepio)_) + +### Acknowledgments + +Thank you to our developer community members who helped to make Azure Core better with their contributions to this release: + +- Jeremi Piotrowski _([GitHub](https://github.com/jepio))_ ## 1.5.0 (2022-03-31) diff --git a/sdk/core/azure-core/CMakeLists.txt b/sdk/core/azure-core/CMakeLists.txt index 54698b777c..8a00fd083d 100644 --- a/sdk/core/azure-core/CMakeLists.txt +++ b/sdk/core/azure-core/CMakeLists.txt @@ -60,25 +60,25 @@ set( inc/azure/core/credentials/token_credential_options.hpp inc/azure/core/cryptography/hash.hpp inc/azure/core/diagnostics/logger.hpp - inc/azure/core/http/http_status_code.hpp + inc/azure/core/http/policies/policy.hpp inc/azure/core/http/http.hpp + inc/azure/core/http/http_status_code.hpp inc/azure/core/http/raw_response.hpp - inc/azure/core/http/policies/policy.hpp inc/azure/core/http/transport.hpp - inc/azure/core/internal/client_options.hpp - inc/azure/core/internal/contract.hpp inc/azure/core/internal/cryptography/sha_hash.hpp inc/azure/core/internal/diagnostics/log.hpp - inc/azure/core/internal/environment.hpp - inc/azure/core/internal/extendable_enumeration.hpp inc/azure/core/internal/http/pipeline.hpp inc/azure/core/internal/io/null_body_stream.hpp - inc/azure/core/internal/json/json_serializable.hpp inc/azure/core/internal/json/json.hpp + inc/azure/core/internal/json/json_optional.hpp + inc/azure/core/internal/json/json_serializable.hpp + inc/azure/core/internal/client_options.hpp + inc/azure/core/internal/contract.hpp + inc/azure/core/internal/environment.hpp + inc/azure/core/internal/extendable_enumeration.hpp inc/azure/core/internal/strings.hpp inc/azure/core/io/body_stream.hpp inc/azure/core/azure_assert.hpp - inc/azure/core/rtti.hpp inc/azure/core/base64.hpp inc/azure/core/case_insensitive_containers.hpp inc/azure/core/context.hpp @@ -90,10 +90,11 @@ set( inc/azure/core/modified_conditions.hpp inc/azure/core/nullable.hpp inc/azure/core/operation.hpp - inc/azure/core/paged_response.hpp inc/azure/core/operation_status.hpp + inc/azure/core/paged_response.hpp inc/azure/core/platform.hpp inc/azure/core/response.hpp + inc/azure/core/rtti.hpp inc/azure/core/url.hpp inc/azure/core/uuid.hpp inc/azure/core.hpp @@ -103,7 +104,6 @@ set( AZURE_CORE_SOURCE ${CURL_TRANSPORT_ADAPTER_SRC} ${WIN_TRANSPORT_ADAPTER_SRC} - src/azure_assert.cpp src/cryptography/md5.cpp src/cryptography/sha_hash.cpp src/http/bearer_token_authentication_policy.cpp @@ -120,6 +120,7 @@ set( src/io/random_access_file_body_stream.cpp src/private/environment_log_level_listener.hpp src/private/package_version.hpp + src/azure_assert.cpp src/base64.cpp src/context.cpp src/datetime.cpp diff --git a/sdk/core/azure-core/inc/azure/core.hpp b/sdk/core/azure-core/inc/azure/core.hpp index 9f18d88a2b..6b7e1aa75e 100644 --- a/sdk/core/azure-core/inc/azure/core.hpp +++ b/sdk/core/azure-core/inc/azure/core.hpp @@ -25,6 +25,7 @@ #include "azure/core/paged_response.hpp" #include "azure/core/platform.hpp" #include "azure/core/response.hpp" +#include "azure/core/rtti.hpp" #include "azure/core/url.hpp" #include "azure/core/uuid.hpp" diff --git a/sdk/core/azure-core/inc/azure/core/context.hpp b/sdk/core/azure-core/inc/azure/core/context.hpp index 2e72f85d49..a4ae92e631 100644 --- a/sdk/core/azure-core/inc/azure/core/context.hpp +++ b/sdk/core/azure-core/inc/azure/core/context.hpp @@ -233,7 +233,7 @@ namespace Azure { namespace Core { * @brief Checks if the context is cancelled. * @return `true` if this context is cancelled; otherwise, `false`. */ - bool IsCancelled() const noexcept { return GetDeadline() < std::chrono::system_clock::now(); } + bool IsCancelled() const { return GetDeadline() < std::chrono::system_clock::now(); } /** * @brief Checks if the context is cancelled. diff --git a/sdk/core/azure-core/src/private/package_version.hpp b/sdk/core/azure-core/src/private/package_version.hpp index b0f9e28ebe..34dade0671 100644 --- a/sdk/core/azure-core/src/private/package_version.hpp +++ b/sdk/core/azure-core/src/private/package_version.hpp @@ -11,7 +11,7 @@ #include #define AZURE_CORE_VERSION_MAJOR 1 -#define AZURE_CORE_VERSION_MINOR 6 +#define AZURE_CORE_VERSION_MINOR 7 #define AZURE_CORE_VERSION_PATCH 0 #define AZURE_CORE_VERSION_PRERELEASE "beta.1" diff --git a/sdk/identity/azure-identity/CHANGELOG.md b/sdk/identity/azure-identity/CHANGELOG.md index 2dbf4d1b9d..3dddc5d615 100644 --- a/sdk/identity/azure-identity/CHANGELOG.md +++ b/sdk/identity/azure-identity/CHANGELOG.md @@ -1,17 +1,21 @@ # Release History -## 1.3.0-beta.2 (Unreleased) +## 1.3.0-beta.3 (Unreleased) ### Features Added -- Added `ClientCertificateCredential`, and updated `EnvironmentCredential` to support client certificate authentication. - ### Breaking Changes ### Bugs Fixed ### Other Changes +## 1.3.0-beta.2 (2022-05-10) + +### Features Added + +- Added `ClientCertificateCredential`, and updated `EnvironmentCredential` to support client certificate authentication. + ## 1.3.0-beta.1 (2022-04-05) ### Features Added diff --git a/sdk/identity/azure-identity/CMakeLists.txt b/sdk/identity/azure-identity/CMakeLists.txt index f4760cfb01..8cc8f7414c 100644 --- a/sdk/identity/azure-identity/CMakeLists.txt +++ b/sdk/identity/azure-identity/CMakeLists.txt @@ -52,6 +52,7 @@ set( inc/azure/identity/dll_import_export.hpp inc/azure/identity/environment_credential.hpp inc/azure/identity/managed_identity_credential.hpp + inc/azure/identity/rtti.hpp inc/azure/identity.hpp ) @@ -82,7 +83,6 @@ target_include_directories( PUBLIC $ $ - ${azure-core-cpp_INCLUDE_DIRS} ) target_link_libraries(azure-identity PUBLIC Azure::azure-core) diff --git a/sdk/identity/azure-identity/inc/azure/identity.hpp b/sdk/identity/azure-identity/inc/azure/identity.hpp index 0ef5230656..ff96cd03f3 100644 --- a/sdk/identity/azure-identity/inc/azure/identity.hpp +++ b/sdk/identity/azure-identity/inc/azure/identity.hpp @@ -14,3 +14,4 @@ #include "azure/identity/dll_import_export.hpp" #include "azure/identity/environment_credential.hpp" #include "azure/identity/managed_identity_credential.hpp" +#include "azure/identity/rtti.hpp" diff --git a/sdk/identity/azure-identity/samples/CMakeLists.txt b/sdk/identity/azure-identity/samples/CMakeLists.txt index c22e87d3c2..06a1cb0cb1 100644 --- a/sdk/identity/azure-identity/samples/CMakeLists.txt +++ b/sdk/identity/azure-identity/samples/CMakeLists.txt @@ -8,26 +8,26 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED True) add_executable(chained_token_credential_sample chained_token_credential.cpp) -target_link_libraries(chained_token_credential_sample PRIVATE azure-identity) +target_link_libraries(chained_token_credential_sample PRIVATE azure-identity service) target_include_directories(chained_token_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity chained_token_credential_sample) add_executable(client_certificate_credential_sample client_certificate_credential.cpp) -target_link_libraries(client_certificate_credential_sample PRIVATE azure-identity) +target_link_libraries(client_certificate_credential_sample PRIVATE azure-identity service get-env-helper) target_include_directories(client_certificate_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity client_certificate_credential_sample) add_executable(client_secret_credential_sample client_secret_credential.cpp) -target_link_libraries(client_secret_credential_sample PRIVATE azure-identity) +target_link_libraries(client_secret_credential_sample PRIVATE azure-identity service get-env-helper) target_include_directories(client_secret_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity client_secret_credential_sample) add_executable(environment_credential_sample environment_credential.cpp) -target_link_libraries(environment_credential_sample PRIVATE azure-identity) +target_link_libraries(environment_credential_sample PRIVATE azure-identity service) target_include_directories(environment_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity environment_credential_sample) add_executable(managed_identity_credential_sample managed_identity_credential.cpp) -target_link_libraries(managed_identity_credential_sample PRIVATE azure-identity) +target_link_libraries(managed_identity_credential_sample PRIVATE azure-identity service) target_include_directories(managed_identity_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity managed_identity_credential_sample) diff --git a/sdk/identity/azure-identity/samples/client_certificate_credential.cpp b/sdk/identity/azure-identity/samples/client_certificate_credential.cpp index f5a79c389b..2cca8a0780 100644 --- a/sdk/identity/azure-identity/samples/client_certificate_credential.cpp +++ b/sdk/identity/azure-identity/samples/client_certificate_credential.cpp @@ -1,17 +1,21 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT +#include + #include #include #include -// These functions should be getting the real Tenant ID, Client ID, and the Client Certificate to -// authenticate. -std::string GetTenantId() { return std::string(); } -std::string GetClientId() { return std::string(); } -std::string GetClientCertificatePath() { return std::string(); } +// The following environment variables must be set before running the sample. +// * AZURE_TENANT_ID: Tenant ID for the Azure account. +// * AZURE_CLIENT_ID: The Client ID to authenticate the request. +// * AZURE_CLIENT_CERTIFICATE_PATH: The path to a client certificate. +std::string GetTenantId() { return std::getenv("AZURE_TENANT_ID"); } +std::string GetClientId() { return std::getenv("AZURE_CLIENT_ID"); } +std::string GetClientCertificatePath() { return std::getenv("AZURE_CLIENT_CERTIFICATE_PATH"); } int main() { diff --git a/sdk/identity/azure-identity/samples/client_secret_credential.cpp b/sdk/identity/azure-identity/samples/client_secret_credential.cpp index 256ac5d249..3f92f1630c 100644 --- a/sdk/identity/azure-identity/samples/client_secret_credential.cpp +++ b/sdk/identity/azure-identity/samples/client_secret_credential.cpp @@ -1,18 +1,21 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT +#include + #include #include #include -// These functions should be getting the real Tenant ID, Client ID, and the Client Secret to -// authenticate. It is recommended to NOT hardcode the secret in the code, but to get it from the -// environment or read it from a secure location. -std::string GetTenantId() { return std::string(); } -std::string GetClientId() { return std::string(); } -std::string GetClientSecret() { return std::string(); } +// The following environment variables must be set before running the sample. +// * AZURE_TENANT_ID: Tenant ID for the Azure account. +// * AZURE_CLIENT_ID: The Client ID to authenticate the request. +// * AZURE_CLIENT_SECRET: The client secret. +std::string GetTenantId() { return std::getenv("AZURE_TENANT_ID"); } +std::string GetClientId() { return std::getenv("AZURE_CLIENT_ID"); } +std::string GetClientSecret() { return std::getenv("AZURE_CLIENT_SECRET"); } int main() { diff --git a/sdk/identity/azure-identity/src/private/package_version.hpp b/sdk/identity/azure-identity/src/private/package_version.hpp index c456f9cd4b..92ef7abf62 100644 --- a/sdk/identity/azure-identity/src/private/package_version.hpp +++ b/sdk/identity/azure-identity/src/private/package_version.hpp @@ -13,7 +13,7 @@ #define AZURE_IDENTITY_VERSION_MAJOR 1 #define AZURE_IDENTITY_VERSION_MINOR 3 #define AZURE_IDENTITY_VERSION_PATCH 0 -#define AZURE_IDENTITY_VERSION_PRERELEASE "beta.2" +#define AZURE_IDENTITY_VERSION_PRERELEASE "beta.3" #define AZURE_IDENTITY_VERSION_ITOA_HELPER(i) #i #define AZURE_IDENTITY_VERSION_ITOA(i) AZURE_IDENTITY_VERSION_ITOA_HELPER(i) diff --git a/sdk/identity/test-resources-pre.ps1 b/sdk/identity/test-resources-pre.ps1 new file mode 100644 index 0000000000..5c12c39f07 --- /dev/null +++ b/sdk/identity/test-resources-pre.ps1 @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# IMPORTANT: Do not invoke this file directly. Please instead run eng/New-TestResources.ps1 from the repository root. + +#Requires -Version 6.0 +#Requires -PSEdition Core + +using namespace System.Security.Cryptography +using namespace System.Security.Cryptography.X509Certificates + +# Use same parameter names as declared in eng/New-TestResources.ps1 (assume validation therein). +[CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] +param ( + # Captures any arguments from eng/New-TestResources.ps1 not declared here (no parameter errors). + [Parameter(ValueFromRemainingArguments = $true)] + $RemainingArguments +) + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +$certPath = Join-Path $(Get-Location) "azure-identity-test2.pem" + +@" +Bag Attributes + Microsoft Local Key set: + localKeyID: 01 00 00 00 + friendlyName: te-66f5c973-4fc8-4cd3-8acc-64964d79b693 + Microsoft CSP Name: Microsoft Software Key Storage Provider +Key Attributes + X509v3 Key Usage: 90 +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPdm4pukO7ugEx +8wXrmo4VIEoicp7w3QsEJGA2bMx9nHMvwugG54t14QpfqBQYQWLeL1HmpcDeivVD ++15ZXeGLCPVZBHhoY8ZWGibfhAAzqQ0P9Ca1kydjvB4uJcEnF/RYtQv6n6OwmdO1 +wJ22JNcRlMtZqmnb/Q0In2fjXEbdl85/GZlYzMQRdyfI0yriSRBcYV2kg0zeXCxf +mCvB3rb6I1KpoUFHlkeHtkeDwm0VHUEt4Hz8ghcB00tI5eS2fH2rPkINQKc6+0QU +C2KICQC+GzJsYDbwQOao5Vhk80H5LRuM9Ndzv+fU3lLnktYCgXgL9AX4L/R9Z4Pz +tuao/qbRAgMBAAECggEBAMQZIrooiTuZ7uVC3Ja96Y1IjyqOg3QSzAXnSFZJcuVM +i4hayC02khkjVUXjvtLKg2SW/+hvRqZUXM8cfCsm1Tkxh4/T7OhnXyMl5xahU/uA +0IsC8c/xv2rDdxeRskh8mQd8Yk1MtlIIpRgIcEqp+exxY+FmdldtkvNSkcVUBNwQ +nXi+oWPhE2guo2g1BPk2gbF0+3FvSrQ8QwGHg+uQJwrQpJ+SB9TyuQFauGR5/wSq +H93cFH5YC/+v5I7qW6ZQe0f7rEKQDybGVzkBlKJyGCVYmPn7Xa/wJriws+FZIfHz +f3m0kJigxJd/HwTrnKSg+H8oBgng7lZLdBYWHMGJhA0CgYEA48moW7szegvfLuUF +a0sHfyKuNyvOv7Wud4sa0lwdKPHS+atwL6TNUWCAGkomYADEe3qiYgMXDX9U3hlW +6zktYFj03tnRg4iBjp8nchLBVLf3Wd5TPRw1VKu4ZW43y8BRhYWV+3Z4s1nyMEDA +NFbKRmL7LDB05oWHdJMjFK/L6YcCgYEA6ShV4v2RQiXzkW6GHSBZDIVHCeWwvIld +OlEfG7wzZW4e8wNDhfSMtXyJrzfbEyXBtVKoESdP6Nnm9W7ftcynW965S94THuy7 ++ofvHo6JAm8g/0uX70wZ26LU8qhkJMTWmsONBNKLwUzkFT7VGsdaBliam1RLvjeT +URdQgnftIucCgYEA4FYamT0k1W4bv/OOAr1CBNQDABME64ni6Zj2MXbGwSxou7s8 +IbANBbgkcb/VS3d2CqYchqrEaWaeDp6mG8OUDO+POmsLDJ/D+NKF5rLR9L25vahY +EjdVzq3QTRTfnqspnnaR37Yt6XUMMLmUkfdn/yo8dKjEeMPJQ+YlBpqcGMECgYBZ +rmIaxV2yC9b8AX8khOS7pCgG7opkepGZdMp6aJF8WjcdUgwO4lmdFSIAe4OQgd1Y +WUq8Dlr2PZpQnSz/SJC3DZxISksggf5sBw06u6iHfyc6C2GNccAgcylljM+4NN42 ++TCswi9vUpwIb/qYKkW+WyZcyLe5mrbXYhhdlrNn0QKBgDe8aRG+MOSUTTXjAVss +bDY0Us943FN91qBmagNqDyozKAAqDoKvdRxM0IlIDnOptj4AfbpJ1JThNOJDYBpU ++Azo8UoedANgndtZ2n11RSjmlQ6TE/WGlsirHExqr6y/l71znoQm1y3E2cArbsmy +hp0P5v42PKxmAx4pR0EjNKsd +-----END PRIVATE KEY----- +Bag Attributes + localKeyID: 01 00 00 00 + 1.3.6.1.4.1.311.17.3.71: 61 00 6E 00 74 00 6B 00 2D 00 6C 00 61 00 70 00 74 00 6F 00 70 00 00 00 +subject=CN = azure-identity-test + +issuer=CN = azure-identity-test + +-----BEGIN CERTIFICATE----- +MIIDODCCAiCgAwIBAgIQNqa9U3MBxqBF7ksWk+XRkzANBgkqhkiG9w0BAQsFADAe +MRwwGgYDVQQDDBNhenVyZS1pZGVudGl0eS10ZXN0MCAXDTIyMDQyMjE1MDYwNloY +DzIyMjIwMTAxMDcwMDAwWjAeMRwwGgYDVQQDDBNhenVyZS1pZGVudGl0eS10ZXN0 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz3ZuKbpDu7oBMfMF65qO +FSBKInKe8N0LBCRgNmzMfZxzL8LoBueLdeEKX6gUGEFi3i9R5qXA3or1Q/teWV3h +iwj1WQR4aGPGVhom34QAM6kND/QmtZMnY7weLiXBJxf0WLUL+p+jsJnTtcCdtiTX +EZTLWapp2/0NCJ9n41xG3ZfOfxmZWMzEEXcnyNMq4kkQXGFdpINM3lwsX5grwd62 ++iNSqaFBR5ZHh7ZHg8JtFR1BLeB8/IIXAdNLSOXktnx9qz5CDUCnOvtEFAtiiAkA +vhsybGA28EDmqOVYZPNB+S0bjPTXc7/n1N5S55LWAoF4C/QF+C/0fWeD87bmqP6m +0QIDAQABo3AwbjAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwIG +CCsGAQUFBwMBMB4GA1UdEQQXMBWCE2F6dXJlLWlkZW50aXR5LXRlc3QwHQYDVR0O +BBYEFCoJ5tInmafyNuR0tGxZOz522jlWMA0GCSqGSIb3DQEBCwUAA4IBAQBzLXpw +Xmrg1sQTmzMnS24mREKxj9B3YILmgsdBMrHkH07QUROee7IbQ8gfBKeln0dEcfYi +Jyh42jn+fmg9AR17RP80wPthD2eKOt4WYNkNM3H8U4JEo+0ML0jZyswynpR48h/E +m96sm/NUeKUViD5iVTb1uHL4j8mQAN1IbXcunXvrrek1CzFVn5Rpah0Tn+6cYVKd +Jg531i53udzusgZtV1NPZ82tzYkPQG1vxB//D9vd0LzmcfCvT50MKhz0r/c5yJYk +i9q94DBuzMhe+O9j+Ob2pVQt5akVFJVtIVSfBZzRBAd66u9JeADlT4sxwS4QAUHi +RrCsEpJsnJXkx/6O +-----END CERTIFICATE----- +"@ > $certPath + +$EnvironmentVariables["AZURE_CLIENT_CERTIFICATE_PATH"] = $certPath diff --git a/sdk/keyvault/azure-security-keyvault-certificates/CHANGELOG.md b/sdk/keyvault/azure-security-keyvault-certificates/CHANGELOG.md index 08f3f17cca..4648f49e0c 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/CHANGELOG.md +++ b/sdk/keyvault/azure-security-keyvault-certificates/CHANGELOG.md @@ -6,6 +6,8 @@ ### Breaking Changes +- Renamed `keyvault_certificates.hpp` to `certificates.hpp`. + ### Bugs Fixed ### Other Changes diff --git a/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt index 4c999dedd9..205a5e008f 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt @@ -52,15 +52,22 @@ set( inc/azure/keyvault/certificates/certificate_client_models.hpp inc/azure/keyvault/certificates/certificate_client_options.hpp inc/azure/keyvault/certificates/certificate_client_operations.hpp + inc/azure/keyvault/certificates/dll_import_export.hpp + inc/azure/keyvault/certificates/rtti.hpp ) set( AZURE_KEYVAULT_CERTIFICATES_SOURCE + src/private/certificate_constants.hpp + src/private/certificate_key_usage.hpp + src/private/certificate_serializers.hpp + src/private/keyvault_certificates_common_request.hpp + src/private/package_version.hpp src/certificate_client.cpp - src/certificate_serializers.cpp - src/keyvault_certificates_common_request.cpp src/certificate_client_operations.cpp src/certificate_client_paged_response.cpp + src/certificate_serializers.cpp + src/keyvault_certificates_common_request.cpp ) add_library(azure-security-keyvault-certificates @@ -73,11 +80,9 @@ target_include_directories( azure-security-keyvault-certificates PUBLIC $ - $ - $ $ ) -message($ -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/certificate_get_certificates.cpp b/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/certificate_get_certificates.cpp index a9ab513362..da75bf9c9c 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/certificate_get_certificates.cpp +++ b/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/certificate_get_certificates.cpp @@ -19,7 +19,7 @@ #include "get_env.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-import-certificate/certificate_import_certificate.cpp b/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-import-certificate/certificate_import_certificate.cpp index 6eda422347..79a8576ebd 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-import-certificate/certificate_import_certificate.cpp +++ b/sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-import-certificate/certificate_import_certificate.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-certificates/test/ut/certificate_client_base_test.hpp b/sdk/keyvault/azure-security-keyvault-certificates/test/ut/certificate_client_base_test.hpp index 599a5270c5..678cafddca 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/test/ut/certificate_client_base_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-certificates/test/ut/certificate_client_base_test.hpp @@ -12,7 +12,7 @@ #include "../src/private/certificate_serializers.hpp" #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-certificates/test/ut/macro_guard.cpp b/sdk/keyvault/azure-security-keyvault-certificates/test/ut/macro_guard.cpp index 1257993335..d0e5a6e243 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/test/ut/macro_guard.cpp +++ b/sdk/keyvault/azure-security-keyvault-certificates/test/ut/macro_guard.cpp @@ -12,4 +12,4 @@ #define max(x, y) small #define min(x, y) small -#include "azure/keyvault/keyvault_certificates.hpp" +#include "azure/keyvault/certificates.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt index 684ab3c60a..571a1875bb 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt @@ -47,18 +47,22 @@ endif() set( AZURE_KEYVAULT_KEYS_HEADER + inc/azure/keyvault/keys/cryptography/cryptography_client.hpp inc/azure/keyvault/keys/cryptography/cryptography_client_models.hpp inc/azure/keyvault/keys/cryptography/cryptography_client_options.hpp - inc/azure/keyvault/keys/cryptography/cryptography_client.hpp + inc/azure/keyvault/keys/dll_import_export.hpp inc/azure/keyvault/keys/key_client.hpp inc/azure/keyvault/keys/key_client_models.hpp inc/azure/keyvault/keys/key_client_options.hpp + inc/azure/keyvault/keys/rtti.hpp + inc/azure/keyvault/keys.hpp + inc/azure/keyvault/keyvault_keys.hpp ) set( AZURE_KEYVAULT_KEYS_SOURCE - src/cryptography/cryptography_client_options.cpp src/cryptography/cryptography_client.cpp + src/cryptography/cryptography_client_options.cpp src/cryptography/decrypt_parameters.cpp src/cryptography/decrypt_result.cpp src/cryptography/encrypt_parameters.cpp @@ -70,9 +74,9 @@ set( src/cryptography/key_wrap_parameters.cpp src/cryptography/sign_result.cpp src/cryptography/signature_algorithm.cpp - src/cryptography/wrap_result.cpp src/cryptography/unwrap_result.cpp src/cryptography/verify_result.cpp + src/cryptography/wrap_result.cpp src/private/cryptography_internal_access.hpp src/private/cryptography_serializers.hpp src/private/key_backup.hpp @@ -90,8 +94,8 @@ set( src/import_key_options.cpp src/json_web_key.cpp src/key_backup.cpp - src/key_client_options.cpp src/key_client.cpp + src/key_client_options.cpp src/key_client_paged_responses.cpp src/key_curve_name.cpp src/key_operation.cpp @@ -112,8 +116,6 @@ target_include_directories( azure-security-keyvault-keys PUBLIC $ - $ - $ $ ) diff --git a/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keys.hpp b/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keys.hpp new file mode 100644 index 0000000000..0a2c2739ff --- /dev/null +++ b/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keys.hpp @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +/** + * @brief Includes all public headers from Azure Key Vault Keys SDK library. + * + */ + +#pragma once + +#include "azure/keyvault/keys/cryptography/cryptography_client.hpp" +#include "azure/keyvault/keys/cryptography/cryptography_client_models.hpp" +#include "azure/keyvault/keys/cryptography/cryptography_client_options.hpp" +#include "azure/keyvault/keys/dll_import_export.hpp" +#include "azure/keyvault/keys/key_client.hpp" +#include "azure/keyvault/keys/key_client_models.hpp" +#include "azure/keyvault/keys/key_client_options.hpp" +#include "azure/keyvault/keys/rtti.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keyvault_keys.hpp b/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keyvault_keys.hpp index d662199f90..a1a2415109 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keyvault_keys.hpp +++ b/sdk/keyvault/azure-security-keyvault-keys/inc/azure/keyvault/keyvault_keys.hpp @@ -4,14 +4,11 @@ /** * @brief Includes all public headers from Azure Key Vault Keys SDK library. * + * @remark This header file does exist for compatibility with prior stable releases. + * It is recommended to use `keys.hpp` instead. + * */ #pragma once -#include "azure/keyvault/keys/cryptography/cryptography_client.hpp" -#include "azure/keyvault/keys/cryptography/cryptography_client_models.hpp" -#include "azure/keyvault/keys/cryptography/cryptography_client_options.hpp" -#include "azure/keyvault/keys/dll_import_export.hpp" -#include "azure/keyvault/keys/key_client.hpp" -#include "azure/keyvault/keys/key_client_models.hpp" -#include "azure/keyvault/keys/key_client_options.hpp" +#include "azure/keyvault/keys.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/perf/inc/azure/keyvault/keys/test/get_key_test.hpp b/sdk/keyvault/azure-security-keyvault-keys/test/perf/inc/azure/keyvault/keys/test/get_key_test.hpp index 52f4077f14..c75f7a0ef8 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/perf/inc/azure/keyvault/keys/test/get_key_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/perf/inc/azure/keyvault/keys/test/get_key_test.hpp @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/sample1_hello_world.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/sample1_hello_world.cpp index 56e2331cea..c67dfd5203 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/sample1_hello_world.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/sample1_hello_world.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample2-backup-and-restore/sample2_backup_and_restore.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample2-backup-and-restore/sample2_backup_and_restore.cpp index 94e42a7250..db9ab83a31 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample2-backup-and-restore/sample2_backup_and_restore.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample2-backup-and-restore/sample2_backup_and_restore.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/sample3_get_keys.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/sample3_get_keys.cpp index 681f6c0b29..8484e691f9 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/sample3_get_keys.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/sample3_get_keys.cpp @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample4-encrypt-decrypt/sample4_encrypt_decrypt.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample4-encrypt-decrypt/sample4_encrypt_decrypt.cpp index d6e81bcd37..500a58a819 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample4-encrypt-decrypt/sample4_encrypt_decrypt.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample4-encrypt-decrypt/sample4_encrypt_decrypt.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/sample5_sign_verify.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/sample5_sign_verify.cpp index 8749233ac4..85251a18ae 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/sample5_sign_verify.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/sample5_sign_verify.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample6-wrap-unwrap/sample6_wrap_unwrap.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample6-wrap-unwrap/sample6_wrap_unwrap.cpp index 830f4e441c..68092365bc 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample6-wrap-unwrap/sample6_wrap_unwrap.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/samples/sample6-wrap-unwrap/sample6_wrap_unwrap.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_backup_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_backup_test_live.cpp index 00a912ee82..d3536f79f3 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_backup_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_backup_test_live.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_base_test.hpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_base_test.hpp index d7553a6f2f..fbffa95fb9 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_base_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_base_test.hpp @@ -11,8 +11,8 @@ #include #include #include +#include #include -#include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_create_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_create_test_live.cpp index 65ce578886..1efe301349 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_create_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_create_test_live.cpp @@ -5,7 +5,7 @@ #include "key_client_base_test.hpp" -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_delete_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_delete_test_live.cpp index 626987623a..bf13777b2f 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_delete_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_delete_test_live.cpp @@ -5,7 +5,7 @@ #include "key_client_base_test.hpp" -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_get_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_get_test_live.cpp index 7259ff2286..dacc326455 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_get_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_get_test_live.cpp @@ -5,7 +5,7 @@ #include "key_client_base_test.hpp" -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_import_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_import_test_live.cpp index af640c578e..756336584b 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_import_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_import_test_live.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include "key_client_base_test.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_test.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_test.cpp index 1803ab6289..6b87993f12 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_test.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_test.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_update_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_update_test_live.cpp index 2bcb5445cc..691e5793a5 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_update_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_client_update_test_live.cpp @@ -6,7 +6,7 @@ #include "key_client_base_test.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_cryptographic_client_test_live.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_cryptographic_client_test_live.cpp index 1250216b2a..c20df1c2e6 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_cryptographic_client_test_live.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/key_cryptographic_client_test_live.cpp @@ -7,7 +7,7 @@ #include "key_client_base_test.hpp" -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/macro_guard.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/macro_guard.cpp index 35fe152431..304501a03d 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/macro_guard.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/macro_guard.cpp @@ -12,4 +12,4 @@ #define max(x, y) small #define min(x, y) small -#include +#include diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_client_test.cpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_client_test.cpp index d1c37374cf..7587463624 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_client_test.cpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_client_test.cpp @@ -6,7 +6,7 @@ #include "mocked_transport_adapter_test.hpp" #include -#include +#include #include using namespace Azure::Security::KeyVault::Keys; diff --git a/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_transport_adapter_test.hpp b/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_transport_adapter_test.hpp index 97146f8ed6..cd6cf66dc1 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_transport_adapter_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-keys/test/ut/mocked_transport_adapter_test.hpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/CHANGELOG.md b/sdk/keyvault/azure-security-keyvault-secrets/CHANGELOG.md index 01a44320d8..32a1794c1c 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/CHANGELOG.md +++ b/sdk/keyvault/azure-security-keyvault-secrets/CHANGELOG.md @@ -6,6 +6,8 @@ ### Breaking Changes +- Renamed `keyvault_secrets.hpp` to `secrets.hpp`. + ### Bugs Fixed ### Other Changes diff --git a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt index a5f0dc3365..480c5c161f 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt @@ -48,31 +48,32 @@ endif() set( AZURE_SECURITY_KEYVAULT_SECRETS_HEADER inc/azure/keyvault/secrets/dll_import_export.hpp - inc/azure/keyvault/secrets/keyvault_secret.hpp - inc/azure/keyvault/secrets/keyvault_secret_properties.hpp - inc/azure/keyvault/secrets/secret_client.hpp - inc/azure/keyvault/keyvault_secrets.hpp - inc/azure/keyvault/secrets/keyvault_deleted_secret.hpp - inc/azure/keyvault/secrets/keyvault_secret_paged_response.hpp inc/azure/keyvault/secrets/keyvault_backup_secret.hpp + inc/azure/keyvault/secrets/keyvault_deleted_secret.hpp inc/azure/keyvault/secrets/keyvault_operations.hpp inc/azure/keyvault/secrets/keyvault_options.hpp + inc/azure/keyvault/secrets/keyvault_secret.hpp + inc/azure/keyvault/secrets/keyvault_secret_paged_response.hpp + inc/azure/keyvault/secrets/keyvault_secret_properties.hpp + inc/azure/keyvault/secrets/rtti.hpp + inc/azure/keyvault/secrets/secret_client.hpp + inc/azure/keyvault/secrets.hpp ) set( AZURE_SECURITY_KEYVAULT_SECRETS_SOURCE - src/private/package_version.hpp src/private/keyvault_protocol.hpp + src/private/keyvault_secrets_common_request.hpp + src/private/package_version.hpp src/private/secret_constants.hpp src/private/secret_serializers.hpp - src/private/keyvault_secrets_common_request.hpp - src/keyvault_protocol.cpp - src/secret_client.cpp - src/secret_serializers.cpp src/keyvault_operations.cpp + src/keyvault_protocol.cpp src/keyvault_secret_paged_response.cpp src/keyvault_secret_properties.cpp src/keyvault_secrets_common_request.cpp + src/secret_client.cpp + src/secret_serializers.cpp ) add_library(azure-security-keyvault-secrets ${AZURE_SECURITY_KEYVAULT_SECRETS_HEADER} ${AZURE_SECURITY_KEYVAULT_SECRETS_SOURCE}) @@ -83,8 +84,6 @@ target_include_directories( azure-security-keyvault-secrets PUBLIC $ - $ - $ $ ) diff --git a/sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/keyvault_secrets.hpp b/sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/secrets.hpp similarity index 94% rename from sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/keyvault_secrets.hpp rename to sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/secrets.hpp index 775dab5729..cbfc63ebf9 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/keyvault_secrets.hpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/inc/azure/keyvault/secrets.hpp @@ -16,4 +16,5 @@ #include "azure/keyvault/secrets/keyvault_secret.hpp" #include "azure/keyvault/secrets/keyvault_secret_paged_response.hpp" #include "azure/keyvault/secrets/keyvault_secret_properties.hpp" +#include "azure/keyvault/secrets/rtti.hpp" #include "azure/keyvault/secrets/secret_client.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample1-basic-operations/sample1_basic_operations.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample1-basic-operations/sample1_basic_operations.cpp index c4c1cc44e6..4362030ef1 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample1-basic-operations/sample1_basic_operations.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample1-basic-operations/sample1_basic_operations.cpp @@ -16,7 +16,7 @@ #include "get_env.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample2-backup-restore/sample2_backup_restore.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample2-backup-restore/sample2_backup_restore.cpp index 340bb10407..d5c1cab98b 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample2-backup-restore/sample2_backup_restore.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample2-backup-restore/sample2_backup_restore.cpp @@ -16,7 +16,7 @@ #include "get_env.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample3-delete-recover/sample3_delete_recover.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample3-delete-recover/sample3_delete_recover.cpp index 905d67456c..35be06a372 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample3-delete-recover/sample3_delete_recover.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample3-delete-recover/sample3_delete_recover.cpp @@ -16,7 +16,7 @@ #include "get_env.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample4-get-secrets-deleted/sample4_get_secrets_deleted.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample4-get-secrets-deleted/sample4_get_secrets_deleted.cpp index a148760e48..e80c8d4dfc 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample4-get-secrets-deleted/sample4_get_secrets_deleted.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/samples/sample4-get-secrets-deleted/sample4_get_secrets_deleted.cpp @@ -16,7 +16,7 @@ #include "get_env.hpp" #include -#include +#include #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/macro_guard.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/macro_guard.cpp index 29d98b4c10..ee583e8605 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/macro_guard.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/macro_guard.cpp @@ -12,4 +12,4 @@ #define max(x, y) small #define min(x, y) small -#include "azure/keyvault/keyvault_secrets.hpp" +#include "azure/keyvault/secrets.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/test_app.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/test_app.cpp index 7bf83e9dee..972cd41887 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/test_app.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/test-app/test_app.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "azure/keyvault/keyvault_secrets.hpp" +#include "azure/keyvault/secrets.hpp" #include #include diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/ut/macro_guard.cpp b/sdk/keyvault/azure-security-keyvault-secrets/test/ut/macro_guard.cpp index 29d98b4c10..ee583e8605 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/ut/macro_guard.cpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/ut/macro_guard.cpp @@ -12,4 +12,4 @@ #define max(x, y) small #define min(x, y) small -#include "azure/keyvault/keyvault_secrets.hpp" +#include "azure/keyvault/secrets.hpp" diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/ut/secret_client_base_test.hpp b/sdk/keyvault/azure-security-keyvault-secrets/test/ut/secret_client_base_test.hpp index a37bf6ad3b..2d92cb3913 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/test/ut/secret_client_base_test.hpp +++ b/sdk/keyvault/azure-security-keyvault-secrets/test/ut/secret_client_base_test.hpp @@ -10,7 +10,7 @@ #include #include -#include +#include using namespace std::chrono_literals; diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 3ca4d85de9..4e66c0eef0 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -4,6 +4,12 @@ ### Features Added +- Bumped up API version to `2020-10-02`. +- Added new API: `BlockBlobClient::Query()`. +- Added `ContinuationToken` and `PageSizeHint` in `GetPageRangesOptions`. +- Added support for listing system containers. +- Added support for listing deleted root blob with active versions. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 209d582507..2271cd5840 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -52,7 +52,9 @@ set( inc/azure/storage/blobs/dll_import_export.hpp inc/azure/storage/blobs/page_blob_client.hpp inc/azure/storage/blobs/rest_client.hpp + inc/azure/storage/blobs/rtti.hpp inc/azure/storage/blobs.hpp + src/private/avro_parser.hpp src/private/package_version.hpp ) @@ -62,11 +64,13 @@ set( src/blob_client.cpp src/blob_container_client.cpp src/blob_lease_client.cpp + src/blob_options.cpp src/blob_responses.cpp src/blob_sas_builder.cpp src/blob_service_client.cpp src/block_blob_client.cpp src/page_blob_client.cpp + src/private/avro_parser.cpp src/rest_client.cpp ) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp index 7c9ddb1383..03eb6fce23 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs.hpp @@ -8,13 +8,18 @@ #pragma once +#include + #include "azure/storage/blobs/append_blob_client.hpp" #include "azure/storage/blobs/blob_client.hpp" #include "azure/storage/blobs/blob_container_client.hpp" #include "azure/storage/blobs/blob_lease_client.hpp" +#include "azure/storage/blobs/blob_options.hpp" +#include "azure/storage/blobs/blob_responses.hpp" #include "azure/storage/blobs/blob_sas_builder.hpp" #include "azure/storage/blobs/blob_service_client.hpp" #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/dll_import_export.hpp" #include "azure/storage/blobs/page_blob_client.hpp" -#include "azure/storage/common/storage_exception.hpp" +#include "azure/storage/blobs/rest_client.hpp" +#include "azure/storage/blobs/rtti.hpp" diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 944097626c..25dc5ae5aa 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -1042,6 +1042,154 @@ namespace Azure { namespace Storage { namespace Blobs { } AccessConditions; }; + /** + * @brief Blob Query text configuration for input. + */ + class BlobQueryInputTextOptions final { + public: + /** + * @brief Creates CSV text configuration. + * + * @param recordSeparator Record separator. + * @param columnSeparator Column sepeartor. + * @param quotationCharacter Field quote. + * @param escapeCharacter Escape character. + * @param hasHeaders If CSV file has headers. + * @return CSV text configuration. + */ + static BlobQueryInputTextOptions CreateCsvTextOptions( + const std::string& recordSeparator = std::string(), + const std::string& columnSeparator = std::string(), + const std::string& quotationCharacter = std::string(), + const std::string& escapeCharacter = std::string(), + bool hasHeaders = false); + /** + * @brief Creates Json text configuration. + * + * @param recordSeparator Record separator. + * @return Json text configuration. + */ + static BlobQueryInputTextOptions CreateJsonTextOptions( + const std::string& recordSeparator = std::string()); + /** + * @brief Creates Parquet text configuration. + * + * @return Parquet text configuration + */ + static BlobQueryInputTextOptions CreateParquetTextOptions(); + + private: + Models::_detail::QueryFormatType m_format; + std::string m_recordSeparator; + std::string m_columnSeparator; + std::string m_quotationCharacter; + std::string m_escapeCharacter; + bool m_hasHeaders = false; + + friend class BlockBlobClient; + }; + + /** + * @brief Blob Query text configuration for output. + */ + class BlobQueryOutputTextOptions final { + public: + /** + * @brief Creates CSV text configuration. + * + * @param recordSeparator Record separator. + * @param columnSeparator Column sepeartor. + * @param quotationCharacter Field quote. + * @param escapeCharacter Escape character. + * @param hasHeaders If CSV file has headers. + * @return CSV text configuration. + */ + static BlobQueryOutputTextOptions CreateCsvTextOptions( + const std::string& recordSeparator = std::string(), + const std::string& columnSeparator = std::string(), + const std::string& quotationCharacter = std::string(), + const std::string& escapeCharacter = std::string(), + bool hasHeaders = false); + /** + * @brief Creates Json text configuration. + * + * @param recordSeparator Record separator. + * @return Json text configuration. + */ + static BlobQueryOutputTextOptions CreateJsonTextOptions( + const std::string& recordSeparator = std::string()); + /** + * @brief Creates Arrow text configuration. + * + * @param schema A list of fields describing the schema. + * @return Arrow text configuration. + */ + static BlobQueryOutputTextOptions CreateArrowTextOptions( + std::vector schema); + + private: + Models::_detail::QueryFormatType m_format; + std::string m_recordSeparator; + std::string m_columnSeparator; + std::string m_quotationCharacter; + std::string m_escapeCharacter; + bool m_hasHeaders = false; + std::vector m_schema; + + friend class BlockBlobClient; + }; + + /** + * @brief Blob Query Error. + */ + struct BlobQueryError final + { + /** + * @brief Error name. + */ + std::string Name; + /** + * @brief Error description. + */ + std::string Description; + /** + * @brief If the error is a fatal error. + */ + bool IsFatal = false; + /** + * The position of the error.. + */ + int64_t Position; + }; + + /** + * @brief Optional parameters for #Azure::Storage::Blobs::BlockBlobClient::Query. + */ + struct QueryBlobOptions final + { + /** + * @brief Input text configuration. + */ + BlobQueryInputTextOptions InputTextConfiguration; + /** + * @brief Output text configuration. + */ + BlobQueryOutputTextOptions OutputTextConfiguration; + /** + * @brief Optional conditions that must be met to perform this operation. + */ + BlobAccessConditions AccessConditions; + /** + * @brief Callback for progress handling. + */ + std::function ProgressHandler; + /** + * @brief Callback for error handling. If you don't specify one, the default will be used, which + * will ignore all non-fatal errors and throw for fatal errors. + */ + std::function ErrorHandler; + }; + /** * @brief Optional parameters for #Azure::Storage::Blobs::AppendBlobClient::Create. */ @@ -1275,6 +1423,22 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional conditions that must be met to perform this operation. */ BlobAccessConditions AccessConditions; + + /** + * @brief This parameter identifies the portion of the ranges to be returned with the next + * operation. The operation returns a marker value within the response body if the ranges + * returned were not complete. The marker value may then be used in a subsequent call to request + * the next set of ranges.This value is opaque to the client. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief This parameter specifies the maximum number of page ranges to return. If the request + * specifies a value greater than 10000, the server will return up to 10000 items. If there are + * additional results to return, the service returns a continuation token in the NextMarker + * response element. + */ + Azure::Nullable PageSizeHint; }; /** diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp index 1b948dd77a..33b7b39d1b 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/block_blob_client.hpp @@ -241,6 +241,19 @@ namespace Azure { namespace Storage { namespace Blobs { const GetBlockListOptions& options = GetBlockListOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Returns the result of a query against the blob. + * + * @param querySqlExpression The query expression in SQL. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A QueryBlobResult describing the query result. + */ + Azure::Response Query( + const std::string& querySqlExpression, + const QueryBlobOptions& options = QueryBlobOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: explicit BlockBlobClient(BlobClient blobClient); friend class BlobClient; diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index e484896365..316f370646 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -31,7 +31,7 @@ namespace Azure { namespace Storage { namespace Blobs { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2020-08-04"; + constexpr static const char* ApiVersion = "2020-10-02"; } // namespace _detail namespace Models { /** @@ -402,6 +402,10 @@ namespace Azure { namespace Storage { namespace Blobs { * container was deleted. */ Nullable RemainingRetentionDays; + /** + * Indicates if version level worm is enabled on this container. + */ + bool HasImmutableStorageWithVersioning = false; /** * A set of name-value pairs associated with this blob or blob container. */ @@ -410,10 +414,6 @@ namespace Azure { namespace Storage { namespace Blobs { std::string, Core::_internal::StringExtensions::CaseInsensitiveComparator> Metadata; - /** - * Indicates if version level worm is enabled on this container. - */ - bool HasImmutableStorageWithVersioning = false; }; /** * @brief An Azure Storage container. @@ -446,6 +446,7 @@ namespace Azure { namespace Storage { namespace Blobs { None = 0, Metadata = 1, Deleted = 2, + System = 4, }; inline ListBlobContainersIncludeFlags operator|( ListBlobContainersIncludeFlags lhs, @@ -966,6 +967,7 @@ namespace Azure { namespace Storage { namespace Blobs { AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Hot; AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Cool; AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Archive; + AZ_STORAGE_BLOBS_DLLEXPORT const static AccessTier Premium; private: std::string m_value; @@ -1341,6 +1343,10 @@ namespace Azure { namespace Storage { namespace Blobs { * Properties of a blob. */ BlobItemDetails Details; + /** + * Indicates that this root blob has been deleted, but it has versions that are active. + */ + Nullable HasVersionsOnly; /** * Size in bytes. */ @@ -1365,6 +1371,7 @@ namespace Azure { namespace Storage { namespace Blobs { Tags = 64, ImmutabilityPolicy = 128, LegalHold = 256, + DeletedWithVersions = 512, }; inline ListBlobsIncludeFlags operator|(ListBlobsIncludeFlags lhs, ListBlobsIncludeFlags rhs) { @@ -1989,7 +1996,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2027,7 +2034,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2048,7 +2055,7 @@ namespace Azure { namespace Storage { namespace Blobs { */ DateTime LastModified; /** - * Uniquely identifies a blobs's lease. + * Uniquely identifies a blob's lease. */ std::string LeaseId; }; @@ -2200,6 +2207,218 @@ namespace Azure { namespace Storage { namespace Blobs { struct SetBlobAccessTierResult final { }; + namespace _detail { + /** + * @brief Required. The type of the provided query expression. + */ + class QueryRequestQueryType final { + public: + QueryRequestQueryType() = default; + explicit QueryRequestQueryType(std::string value) : m_value(std::move(value)) {} + bool operator==(const QueryRequestQueryType& other) const + { + return m_value == other.m_value; + } + bool operator!=(const QueryRequestQueryType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryRequestQueryType SQL; + + private: + std::string m_value; + }; + /** + * @brief The quick query format type. + */ + class QueryFormatType final { + public: + QueryFormatType() = default; + explicit QueryFormatType(std::string value) : m_value(std::move(value)) {} + bool operator==(const QueryFormatType& other) const { return m_value == other.m_value; } + bool operator!=(const QueryFormatType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Delimited; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Json; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Arrow; + AZ_STORAGE_BLOBS_DLLEXPORT const static QueryFormatType Parquet; + + private: + std::string m_value; + }; + /** + * @brief Groups the settings used for interpreting the blob data if the blob is delimited + * text formatted. + */ + struct DelimitedTextConfiguration final + { + /** + * The string used to separate columns. + */ + std::string ColumnSeparator; + /** + * The string used to quote a specific field. + */ + std::string FieldQuote; + /** + * The string used to separate records. + */ + std::string RecordSeparator; + /** + * The string used as an escape character. + */ + std::string EscapeChar; + /** + * Represents whether the data has headers. + */ + bool HeadersPresent = bool(); + }; + /** + * @brief Json text configuration. + */ + struct JsonTextConfiguration final + { + /** + * The string used to separate records. + */ + std::string RecordSeparator; + }; + } // namespace _detail + /** + * @brief Type of blob query arrow field. + */ + class BlobQueryArrowFieldType final { + public: + BlobQueryArrowFieldType() = default; + explicit BlobQueryArrowFieldType(std::string value) : m_value(std::move(value)) {} + bool operator==(const BlobQueryArrowFieldType& other) const + { + return m_value == other.m_value; + } + bool operator!=(const BlobQueryArrowFieldType& other) const { return !(*this == other); } + const std::string& ToString() const { return m_value; } + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Int64; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Bool; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Timestamp; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType String; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Double; + AZ_STORAGE_BLOBS_DLLEXPORT const static BlobQueryArrowFieldType Decimal; + + private: + std::string m_value; + }; + /** + * @brief Groups settings regarding specific field of an arrow schema. + */ + struct BlobQueryArrowField final + { + /** + * Type of blob query arrow field. + */ + BlobQueryArrowFieldType Type; + Nullable Name; + Nullable Precision; + Nullable Scale; + }; + namespace _detail { + /** + * @brief Groups the settings used for formatting the response if the response should be Arrow + * formatted. + */ + struct ArrowConfiguration final + { + /** + * Array of BlobQueryArrowField. + */ + std::vector Schema; + }; + /** + * @brief Parquet configuration. + */ + struct ParquetConfiguration final + { + }; + struct QueryFormat final + { + /** + * The quick query format type. + */ + QueryFormatType Type; + /** + * Groups the settings used for interpreting the blob data if the blob is delimited text + * formatted. + */ + Nullable<_detail::DelimitedTextConfiguration> DelimitedTextConfiguration; + /** + * Json text configuration. + */ + Nullable<_detail::JsonTextConfiguration> JsonTextConfiguration; + /** + * Groups the settings used for formatting the response if the response should be Arrow + * formatted. + */ + Nullable<_detail::ArrowConfiguration> ArrowConfiguration; + /** + * Parquet configuration. + */ + Nullable ParquetTextConfiguration; + }; + struct QuerySerialization final + { + QueryFormat Format; + }; + /** + * @brief Groups the set of query request settings. + */ + struct QueryRequest final + { + /** + * Required. The type of the provided query expression. + */ + QueryRequestQueryType QueryType; + /** + * The query expression in SQL. The maximum size of the query expression is 256KiB. + */ + std::string Expression; + Nullable InputSerialization; + Nullable OutputSerialization; + }; + } // namespace _detail + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::Query. + */ + struct QueryBlobResult final + { + std::unique_ptr BodyStream; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * blob, including an update of the blob's metadata or properties, changes the last-modified + * time of the blob. + */ + DateTime LastModified; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the + * request version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + Azure::ETag ETag; + /** + * When a blob is leased, specifies whether the lease is of infinite or fixed duration. + */ + Nullable LeaseDuration; + /** + * Lease state of the blob. + */ + Models::LeaseState LeaseState; + /** + * The current lease status of the blob. + */ + Models::LeaseStatus LeaseStatus; + /** + * The value of this header is set to true if the blob data and application metadata are + * completely encrypted using the specified algorithm. Otherwise, the value is set to false + * (when the blob is unencrypted, or if only parts of the blob/application metadata are + * encrypted). + */ + bool IsServerEncrypted = bool(); + }; /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::SetTags. */ @@ -2379,6 +2598,7 @@ namespace Azure { namespace Storage { namespace Blobs { * Array of ClearRange. */ std::vector ClearRanges; + std::string ContinuationToken; }; /** * @brief Response type for #Azure::Storage::Blobs::PageBlobClient::GetPageRangesDiff. @@ -2399,6 +2619,7 @@ namespace Azure { namespace Storage { namespace Blobs { * Array of ClearRange. */ std::vector ClearRanges; + std::string ContinuationToken; }; } // namespace _detail /** @@ -3470,6 +3691,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable ImmutabilityPolicyExpiry; Nullable ImmutabilityPolicyMode; Nullable LegalHold; + Nullable CopySourceAuthorization; Nullable> SourceContentcrc64; }; static Response CopyFromUri( @@ -3501,6 +3723,26 @@ namespace Azure { namespace Storage { namespace Blobs { const Core::Url& url, const SetBlobTierOptions& options, const Core::Context& context); + struct QueryBlobOptions final + { + Models::_detail::QueryRequest QueryRequest; + Nullable Snapshot; + Nullable LeaseId; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; + Nullable IfModifiedSince; + Nullable IfUnmodifiedSince; + ETag IfMatch; + ETag IfNoneMatch; + Nullable IfTags; + Nullable EncryptionScope; + }; + static Response Query( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const QueryBlobOptions& options, + const Core::Context& context); struct GetBlobTagsOptions final { Nullable Snapshot; @@ -3633,6 +3875,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response UploadPagesFromUri( Core::Http::_internal::HttpPipeline& pipeline, @@ -3649,6 +3892,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable Marker; + Nullable MaxResults; }; static Response GetPageRanges( Core::Http::_internal::HttpPipeline& pipeline, @@ -3667,6 +3912,8 @@ namespace Azure { namespace Storage { namespace Blobs { ETag IfMatch; ETag IfNoneMatch; Nullable IfTags; + Nullable Marker; + Nullable MaxResults; }; static Response GetPageRangesDiff( Core::Http::_internal::HttpPipeline& pipeline, @@ -3800,6 +4047,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response AppendBlockFromUri( Core::Http::_internal::HttpPipeline& pipeline, @@ -3885,6 +4133,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable BlobTagsString; std::string CopySource; Nullable CopySourceBlobProperties; + Nullable CopySourceAuthorization; Nullable> SourceContentcrc64; }; static Response UploadFromUri( @@ -3925,6 +4174,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable SourceIfUnmodifiedSince; ETag SourceIfMatch; ETag SourceIfNoneMatch; + Nullable CopySourceAuthorization; }; static Response StageBlockFromUri( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt b/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt index 546a795da2..9f98c022d3 100644 --- a/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/samples/CMakeLists.txt @@ -18,3 +18,7 @@ create_per_service_target_build_for_sample(storage blob-sas) add_executable(transactional-checksum transactional_checksum.cpp) target_link_libraries(transactional-checksum PRIVATE azure-storage-blobs get-env-helper) create_per_service_target_build_for_sample(storage transactional-checksum) + +add_executable(blob-query blob_query.cpp) +target_link_libraries(blob-query PRIVATE azure-storage-blobs get-env-helper) +create_per_service_target_build_for_sample(storage blob-query) diff --git a/sdk/storage/azure-storage-blobs/samples/blob_query.cpp b/sdk/storage/azure-storage-blobs/samples/blob_query.cpp new file mode 100644 index 0000000000..f01c0de632 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/samples/blob_query.cpp @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "get_env.hpp" + +#include + +#include + +std::string GetConnectionString() +{ + const static std::string ConnectionString = ""; + + if (!ConnectionString.empty()) + { + return ConnectionString; + } + const static std::string envConnectionString = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); + if (!envConnectionString.empty()) + { + return envConnectionString; + } + throw std::runtime_error("Cannot find connection string."); +} + +int main() +{ + using namespace Azure::Storage::Blobs; + + const std::string containerName = "sample-container"; + const std::string blobName = "sample-blob"; + + auto containerClient + = BlobContainerClient::CreateFromConnectionString(GetConnectionString(), containerName); + containerClient.CreateIfNotExists(); + BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); + + const std::string blobContent = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": 50} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": 99} +{"id": 104, "name": "clementines", "price": 399} +{"id": 105, "name": "grapes", "price": 150} +{"id": 106, "name": "lemons", "price": 69} +{"id": 107, "name": "pears", "price": 100} +{"id": 108, "name": "cherries", "price": 281} +{"id": 109, "name": "coconut", "price": 178} +{"id": 110, "name": "bananas", "price": 39} +{"id": 111, "name": "peaches", "price": 117} +)json"; + + std::vector buffer(blobContent.begin(), blobContent.end()); + blobClient.UploadFrom(buffer.data(), buffer.size()); + + QueryBlobOptions queryOptions; + // input can be one of csv, json, parquet + queryOptions.InputTextConfiguration = BlobQueryInputTextOptions::CreateJsonTextOptions(); + // output can be one of csv, json, arrow, parquet + queryOptions.OutputTextConfiguration = BlobQueryOutputTextOptions::CreateCsvTextOptions(); + auto queryResponse + = blobClient.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + std::cout << std::string(data.begin(), data.end()); + /* The output is: + * 103,apples,99 + * 106,lemons,69 + * 110,bananas,39 + */ + + return 0; +} diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index da5dd8f808..d934f023f3 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -254,9 +254,16 @@ namespace Azure { namespace Storage { namespace Blobs { { i.Details.IsAccessTierInferred = false; } - if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + if (i.VersionId.HasValue()) { - i.IsCurrentVersion = false; + if (!i.HasVersionsOnly.HasValue()) + { + i.HasVersionsOnly = false; + } + if (!i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } } if (i.BlobType == Models::BlobType::AppendBlob && !i.Details.IsSealed.HasValue()) { @@ -327,9 +334,16 @@ namespace Azure { namespace Storage { namespace Blobs { { i.Details.IsAccessTierInferred = false; } - if (i.VersionId.HasValue() && !i.IsCurrentVersion.HasValue()) + if (i.VersionId.HasValue()) { - i.IsCurrentVersion = false; + if (!i.HasVersionsOnly.HasValue()) + { + i.HasVersionsOnly = false; + } + if (!i.IsCurrentVersion.HasValue()) + { + i.IsCurrentVersion = false; + } } if (i.BlobType == Models::BlobType::AppendBlob && !i.Details.IsSealed.HasValue()) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_options.cpp b/sdk/storage/azure-storage-blobs/src/blob_options.cpp new file mode 100644 index 0000000000..317650c0a6 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/blob_options.cpp @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "azure/storage/blobs/blob_options.hpp" + +namespace Azure { namespace Storage { namespace Blobs { + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateCsvTextOptions( + const std::string& recordSeparator, + const std::string& columnSeparator, + const std::string& quotationCharacter, + const std::string& escapeCharacter, + bool hasHeaders) + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Delimited; + options.m_recordSeparator = recordSeparator; + options.m_columnSeparator = columnSeparator; + options.m_quotationCharacter = quotationCharacter; + options.m_escapeCharacter = escapeCharacter; + options.m_hasHeaders = hasHeaders; + return options; + } + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateJsonTextOptions( + const std::string& recordSeparator) + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Json; + options.m_recordSeparator = recordSeparator; + return options; + } + + BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateParquetTextOptions() + { + BlobQueryInputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Parquet; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateCsvTextOptions( + const std::string& recordSeparator, + const std::string& columnSeparator, + const std::string& quotationCharacter, + const std::string& escapeCharacter, + bool hasHeaders) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Delimited; + options.m_recordSeparator = recordSeparator; + options.m_columnSeparator = columnSeparator; + options.m_quotationCharacter = quotationCharacter; + options.m_escapeCharacter = escapeCharacter; + options.m_hasHeaders = hasHeaders; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateJsonTextOptions( + const std::string& recordSeparator) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Json; + options.m_recordSeparator = recordSeparator; + return options; + } + + BlobQueryOutputTextOptions BlobQueryOutputTextOptions::CreateArrowTextOptions( + std::vector schema) + { + BlobQueryOutputTextOptions options; + options.m_format = Models::_detail::QueryFormatType::Arrow; + options.m_schema = std::move(schema); + return options; + } + +}}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 7a52c5fe9a..a8f5269d65 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -88,11 +88,13 @@ namespace Azure { namespace Storage { namespace Blobs { void GetPageRangesPagedResponse::OnNextPage(const Azure::Core::Context& context) { + m_operationOptions.ContinuationToken = NextPageToken; *this = m_pageBlobClient->GetPageRanges(m_operationOptions, context); } void GetPageRangesDiffPagedResponse::OnNextPage(const Azure::Core::Context& context) { + m_operationOptions.ContinuationToken = NextPageToken; if (m_previousSnapshot.HasValue()) { *this = m_pageBlobClient->GetPageRangesDiff( diff --git a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp index 020269f353..35d7677df0 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_sas_builder.cpp @@ -11,7 +11,7 @@ namespace Azure { namespace Storage { namespace Sas { namespace { - constexpr static const char* SasVersion = "2020-08-04"; + constexpr static const char* SasVersion = Blobs::_detail::ApiVersion; std::string BlobSasResourceToString(BlobSasResource resource) { diff --git a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp index 0bc3fa0e66..9e14727dfd 100644 --- a/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/block_blob_client.cpp @@ -22,6 +22,9 @@ #include #include #include +#include + +#include "private/avro_parser.hpp" namespace Azure { namespace Storage { namespace Blobs { @@ -511,4 +514,147 @@ namespace Azure { namespace Storage { namespace Blobs { *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); } + Azure::Response BlockBlobClient::Query( + const std::string& querySqlExpression, + const QueryBlobOptions& options, + const Azure::Core::Context& context) const + { + _detail::BlobClient::QueryBlobOptions protocolLayerOptions; + protocolLayerOptions.QueryRequest.QueryType = Models::_detail::QueryRequestQueryType::SQL; + protocolLayerOptions.QueryRequest.Expression = querySqlExpression; + if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) + { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.InputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.InputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.InputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.InputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) + { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.InputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) + { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.InputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.InputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format.ToString().empty()) + { + } + else + { + AZURE_UNREACHABLE_CODE(); + } + if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Delimited) + { + Models::_detail::DelimitedTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + c.ColumnSeparator = options.OutputTextConfiguration.m_columnSeparator; + c.FieldQuote = options.OutputTextConfiguration.m_quotationCharacter; + c.EscapeChar = options.OutputTextConfiguration.m_escapeCharacter; + c.HeadersPresent = options.OutputTextConfiguration.m_hasHeaders; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.DelimitedTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Json) + { + Models::_detail::JsonTextConfiguration c; + c.RecordSeparator = options.OutputTextConfiguration.m_recordSeparator; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.JsonTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Parquet) + { + Models::_detail::ParquetConfiguration c; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ParquetTextConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.OutputTextConfiguration.m_format == Models::_detail::QueryFormatType::Arrow) + { + Models::_detail::ArrowConfiguration c; + c.Schema = options.OutputTextConfiguration.m_schema; + Models::_detail::QuerySerialization q; + q.Format.Type = options.OutputTextConfiguration.m_format; + q.Format.ArrowConfiguration = std::move(c); + protocolLayerOptions.QueryRequest.OutputSerialization = std::move(q); + } + else if (options.InputTextConfiguration.m_format.ToString().empty()) + { + } + else + { + AZURE_UNREACHABLE_CODE(); + } + + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + auto response = _detail::BlobClient::Query( + *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + + const auto stautsCode = response.RawResponse->GetStatusCode(); + const auto reasonPhrase = response.RawResponse->GetReasonPhrase(); + const auto requestId + = response.RawResponse->GetHeaders().count(_internal::HttpHeaderRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId) + : std::string(); + + const auto clientRequestId + = response.RawResponse->GetHeaders().count(_internal::HttpHeaderClientRequestId) != 0 + ? response.RawResponse->GetHeaders().at(_internal::HttpHeaderClientRequestId) + : std::string(); + + auto defaultErrorHandler + = [stautsCode, reasonPhrase, requestId, clientRequestId](BlobQueryError e) { + if (e.IsFatal) + { + StorageException exception("Fatal " + e.Name + " at " + std::to_string(e.Position)); + exception.StatusCode = stautsCode; + exception.ReasonPhrase = reasonPhrase; + exception.RequestId = requestId; + exception.ClientRequestId = clientRequestId; + exception.ErrorCode = e.Name; + exception.Message = e.Description; + + throw exception; + } + }; + + response.Value.BodyStream = std::make_unique<_detail::AvroStreamParser>( + std::move(response.Value.BodyStream), + options.ProgressHandler, + options.ErrorHandler ? options.ErrorHandler : defaultErrorHandler); + return response; + } + }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp index c2e4d35de4..4cdaf51a5b 100644 --- a/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/page_blob_client.cpp @@ -322,6 +322,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRanges( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -333,8 +335,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.PageRanges = std::move(response.Value.PageRanges); pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; @@ -363,6 +365,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRangesDiff( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -376,8 +380,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; pagedResponse.m_previousSnapshot = previousSnapshot; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; @@ -406,6 +410,8 @@ namespace Azure { namespace Storage { namespace Blobs { protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.MaxResults = options.PageSizeHint; auto response = _detail::PageBlobClient::GetPageRangesDiff( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); @@ -419,8 +425,8 @@ namespace Azure { namespace Storage { namespace Blobs { pagedResponse.m_pageBlobClient = std::make_shared(*this); pagedResponse.m_operationOptions = options; pagedResponse.m_previousSnapshotUrl = previousSnapshotUrl; - pagedResponse.CurrentPageToken = std::string(); - pagedResponse.NextPageToken = std::string(); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.ContinuationToken; pagedResponse.RawResponse = std::move(response.RawResponse); return pagedResponse; diff --git a/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp b/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp new file mode 100644 index 0000000000..5ff4846023 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/avro_parser.cpp @@ -0,0 +1,680 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "avro_parser.hpp" + +#include +#include + +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + namespace { + int64_t parseInt(AvroStreamReader::ReaderPos& data) + { + uint64_t r = 0; + int nb = 0; + while (true) + { + uint8_t c = (*data.BufferPtr)[data.Offset++]; + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) + { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); + } + + AvroSchema ParseSchemaFromJsonString(const std::string& jsonSchema) + { + const static std::map BuiltinNameSchemaMap = { + {"string", AvroSchema::StringSchema}, + {"bytes", AvroSchema::BytesSchema}, + {"int", AvroSchema::IntSchema}, + {"long", AvroSchema::LongSchema}, + {"float", AvroSchema::FloatSchema}, + {"double", AvroSchema::DoubleSchema}, + {"boolean", AvroSchema::BoolSchema}, + {"null", AvroSchema::NullSchema}, + {"string", AvroSchema::StringSchema}, + }; + std::map nameSchemaMap = BuiltinNameSchemaMap; + + std::function parseSchemaFromJsonObject; + parseSchemaFromJsonObject = [&](const Core::Json::_internal::json& obj) -> AvroSchema { + if (obj.is_string()) + { + auto typeName = obj.get(); + return nameSchemaMap.find(typeName)->second; + } + else if (obj.is_array()) + { + std::vector unionSchemas; + for (const auto& s : obj) + { + unionSchemas.push_back(parseSchemaFromJsonObject(s)); + } + return AvroSchema::UnionSchema(std::move(unionSchemas)); + } + else if (obj.is_object()) + { + if (obj.count("namespace") != 0) + { + throw std::runtime_error("Namespace isn't supported yet in Avro schema."); + } + if (obj.count("aliases") != 0) + { + throw std::runtime_error("Alias isn't supported yet in Avro schema."); + } + auto typeName = obj["type"].get(); + auto i = nameSchemaMap.find(typeName); + if (i != nameSchemaMap.end()) + { + return i->second; + } + if (typeName == "record") + { + std::vector> fieldsSchema; + for (const auto& field : obj["fields"]) + { + fieldsSchema.push_back(std::make_pair( + field["name"].get(), parseSchemaFromJsonObject(field["type"]))); + } + + const std::string recordName = obj["name"].get(); + auto recordSchema = AvroSchema::RecordSchema(recordName, std::move(fieldsSchema)); + nameSchemaMap.insert(std::make_pair(recordName, recordSchema)); + return recordSchema; + } + else if (typeName == "enum") + { + throw std::runtime_error("Enum type isn't supported yet in Avro schema."); + } + else if (typeName == "array") + { + return AvroSchema::ArraySchema(parseSchemaFromJsonObject(obj["items"])); + } + else if (typeName == "map") + { + return AvroSchema::MapSchema(parseSchemaFromJsonObject(obj["items"])); + } + else if (typeName == "fixed") + { + const std::string fixedName = obj["name"].get(); + auto fixedSchema = AvroSchema::FixedSchema(fixedName, obj["size"].get()); + nameSchemaMap.insert(std::make_pair(fixedName, fixedSchema)); + return fixedSchema; + } + else + { + throw std::runtime_error("Unrecognized type " + typeName + " in Avro schema."); + } + } + AZURE_UNREACHABLE_CODE(); + }; + + auto jsonRoot = Core::Json::_internal::json::parse(jsonSchema.begin(), jsonSchema.end()); + return parseSchemaFromJsonObject(jsonRoot); + } + } // namespace + + int64_t AvroStreamReader::ParseInt(const Core::Context& context) + { + uint64_t r = 0; + int nb = 0; + while (true) + { + Preload(1, context); + uint8_t c = m_streambuffer[m_pos.Offset++]; + + r = r | ((static_cast(c) & 0x7f) << (nb * 7)); + if (c & 0x80) + { + ++nb; + continue; + } + break; + } + return static_cast(r >> 1) ^ -static_cast(r & 0x01); + } + + void AvroStreamReader::Advance(size_t n, const Core::Context& context) + { + Preload(n, context); + m_pos.Offset += n; + } + + size_t AvroStreamReader::Preload(size_t n, const Core::Context& context) + { + size_t oldAvailable = AvailableBytes(); + while (true) + { + size_t newAvailable = TryPreload(n, context); + if (newAvailable >= n) + { + return newAvailable; + } + if (oldAvailable == newAvailable) + { + throw std::runtime_error("Unexpected EOF of Avro stream."); + } + oldAvailable = newAvailable; + } + AZURE_UNREACHABLE_CODE(); + } + + size_t AvroStreamReader::TryPreload(size_t n, const Core::Context& context) + { + size_t availableBytes = AvailableBytes(); + if (availableBytes >= n) + { + return availableBytes; + } + const size_t MinRead = 4096; + size_t tryReadSize = std::max(n, MinRead); + size_t currSize = m_streambuffer.size(); + m_streambuffer.resize(m_streambuffer.size() + tryReadSize); + size_t actualReadSize = m_stream->Read(m_streambuffer.data() + currSize, tryReadSize, context); + m_streambuffer.resize(currSize + actualReadSize); + return AvailableBytes(); + } + + void AvroStreamReader::Discard() + { + constexpr size_t MinimumReleaseMemory = 128 * 1024; + if (m_pos.Offset < MinimumReleaseMemory) + { + return; + } + const size_t availableBytes = AvailableBytes(); + std::memmove(&m_streambuffer[0], &m_streambuffer[m_pos.Offset], availableBytes); + m_streambuffer.resize(availableBytes); + m_pos.Offset = 0; + } + + const AvroSchema AvroSchema::StringSchema(AvroDatumType::String); + const AvroSchema AvroSchema::BytesSchema(AvroDatumType::Bytes); + const AvroSchema AvroSchema::IntSchema(AvroDatumType::Int); + const AvroSchema AvroSchema::LongSchema(AvroDatumType::Long); + const AvroSchema AvroSchema::FloatSchema(AvroDatumType::Float); + const AvroSchema AvroSchema::DoubleSchema(AvroDatumType::Double); + const AvroSchema AvroSchema::BoolSchema(AvroDatumType::Bool); + const AvroSchema AvroSchema::NullSchema(AvroDatumType::Null); + + AvroSchema AvroSchema::RecordSchema( + std::string name, + const std::vector>& fieldsSchema) + { + AvroSchema recordSchema(AvroDatumType::Record); + recordSchema.m_name = std::move(name); + recordSchema.m_status = std::make_shared(); + for (auto& i : fieldsSchema) + { + recordSchema.m_status->m_keys.push_back(i.first); + recordSchema.m_status->m_schemas.push_back(i.second); + } + return recordSchema; + } + + AvroSchema AvroSchema::ArraySchema(AvroSchema elementSchema) + { + AvroSchema arraySchema(AvroDatumType::Array); + arraySchema.m_status = std::make_shared(); + arraySchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return arraySchema; + } + + AvroSchema AvroSchema::MapSchema(AvroSchema elementSchema) + { + AvroSchema mapSchema(AvroDatumType::Map); + mapSchema.m_status = std::make_shared(); + mapSchema.m_status->m_schemas.push_back(std::move(elementSchema)); + return mapSchema; + } + + AvroSchema AvroSchema::UnionSchema(std::vector schemas) + { + AvroSchema unionSchema(AvroDatumType::Union); + unionSchema.m_status = std::make_shared(); + unionSchema.m_status->m_schemas = std::move(schemas); + return unionSchema; + } + + AvroSchema AvroSchema::FixedSchema(std::string name, int64_t size) + { + AvroSchema fixedSchema(AvroDatumType::Fixed); + fixedSchema.m_name = std::move(name); + fixedSchema.m_status = std::make_shared(); + fixedSchema.m_status->m_size = size; + return fixedSchema; + } + + void AvroDatum::Fill(AvroStreamReader& reader, const Core::Context& context) + { + m_data = reader.m_pos; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + int64_t stringSize = reader.ParseInt(context); + reader.Advance(static_cast(stringSize), context); + } + else if ( + m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long + || m_schema.Type() == AvroDatumType::Enum) + { + reader.ParseInt(context); + } + else if (m_schema.Type() == AvroDatumType::Float) + { + reader.Advance(4, context); + } + else if (m_schema.Type() == AvroDatumType::Double) + { + reader.Advance(8, context); + } + else if (m_schema.Type() == AvroDatumType::Bool) + { + reader.Advance(1, context); + } + else if (m_schema.Type() == AvroDatumType::Null) + { + reader.Advance(0, context); + } + else if (m_schema.Type() == AvroDatumType::Record) + { + for (const auto& s : m_schema.FieldSchemas()) + { + AvroDatum(s).Fill(reader, context); + } + } + else if (m_schema.Type() == AvroDatumType::Array) + { + while (true) + { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } + else + { + for (auto i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Map) + { + while (true) + { + int64_t numElementsInBlock = reader.ParseInt(context); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = reader.ParseInt(context); + reader.Advance(static_cast(blockSize), context); + } + else + { + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(AvroSchema::StringSchema).Fill(reader, context); + AvroDatum(m_schema.ItemSchema()).Fill(reader, context); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = reader.ParseInt(context); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(reader, context); + } + else if (m_schema.Type() == AvroDatumType::Fixed) + { + reader.Advance(m_schema.Size(), context); + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + + void AvroDatum::Fill(AvroStreamReader::ReaderPos& data) + { + m_data = data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + int64_t stringSize = parseInt(data); + data.Offset += static_cast(stringSize); + } + else if ( + m_schema.Type() == AvroDatumType::Int || m_schema.Type() == AvroDatumType::Long + || m_schema.Type() == AvroDatumType::Enum) + { + parseInt(data); + } + else if (m_schema.Type() == AvroDatumType::Float) + { + data.Offset += 4; + } + else if (m_schema.Type() == AvroDatumType::Double) + { + data.Offset += 8; + } + else if (m_schema.Type() == AvroDatumType::Bool) + { + data.Offset += 1; + } + else if (m_schema.Type() == AvroDatumType::Null) + { + data.Offset += 0; + } + else if (m_schema.Type() == AvroDatumType::Record) + { + for (const auto& s : m_schema.FieldSchemas()) + { + AvroDatum(s).Fill(data); + } + } + else if (m_schema.Type() == AvroDatumType::Array) + { + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } + else + { + for (auto i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Map) + { + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + else if (numElementsInBlock < 0) + { + int64_t blockSize = parseInt(data); + data.Offset += static_cast(blockSize); + } + else + { + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + AvroDatum(AvroSchema::StringSchema).Fill(data); + AvroDatum(m_schema.ItemSchema()).Fill(data); + } + } + } + } + else if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = parseInt(data); + AvroDatum(m_schema.FieldSchemas()[static_cast(i)]).Fill(data); + } + else if (m_schema.Type() == AvroDatumType::Fixed) + { + data.Offset += m_schema.Size(); + } + else + { + AZURE_UNREACHABLE_CODE(); + } + } + + template <> AvroDatum::StringView AvroDatum::Value() const + { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::String || m_schema.Type() == AvroDatumType::Bytes) + { + const int64_t length = parseInt(data); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, static_cast(length)}; + data.Offset += static_cast(length); + return ret; + } + if (m_schema.Type() == AvroDatumType::Fixed) + { + const size_t fixedSize = m_schema.Size(); + const uint8_t* start = &(*data.BufferPtr)[data.Offset]; + StringView ret{start, fixedSize}; + data.Offset += fixedSize; + return ret; + } + AZURE_UNREACHABLE_CODE(); + } + + template <> std::string AvroDatum::Value() const + { + auto stringView = Value(); + return std::string(stringView.Data, stringView.Data + stringView.Length); + } + + template <> std::vector AvroDatum::Value() const + { + auto stringView = Value(); + return std::vector(stringView.Data, stringView.Data + stringView.Length); + } + + template <> int64_t AvroDatum::Value() const + { + auto data = m_data; + return parseInt(data); + } + + template <> int32_t AvroDatum::Value() const { return static_cast(Value()); } + + template <> bool AvroDatum::Value() const { return Value(); } + + template <> std::nullptr_t AvroDatum::Value() const { return nullptr; } + + template <> AvroRecord AvroDatum::Value() const + { + auto data = m_data; + + AvroRecord r; + r.m_keys = &m_schema.FieldNames(); + for (const auto& schema : m_schema.FieldSchemas()) + { + auto datum = AvroDatum(schema); + datum.Fill(data); + r.m_values.push_back(std::move(datum)); + } + + return r; + } + + template <> AvroMap AvroDatum::Value() const + { + auto data = m_data; + + AvroMap m; + while (true) + { + int64_t numElementsInBlock = parseInt(data); + if (numElementsInBlock == 0) + { + break; + } + if (numElementsInBlock < 0) + { + numElementsInBlock = -numElementsInBlock; + parseInt(data); + } + for (int64_t i = 0; i < numElementsInBlock; ++i) + { + auto keyDatum = AvroDatum(AvroSchema::StringSchema); + keyDatum.Fill(data); + auto valueDatum = AvroDatum(m_schema.ItemSchema()); + valueDatum.Fill(data); + m[keyDatum.Value()] = valueDatum; + } + } + return m; + } + + template <> AvroDatum AvroDatum::Value() const + { + auto data = m_data; + if (m_schema.Type() == AvroDatumType::Union) + { + int64_t i = parseInt(data); + auto datum = AvroDatum(m_schema.FieldSchemas()[static_cast(i)]); + datum.Fill(data); + return datum; + } + AZURE_UNREACHABLE_CODE(); + } + + AvroObjectContainerReader::AvroObjectContainerReader(Core::IO::BodyStream& stream) + : m_reader(std::make_unique(stream)) + { + } + + AvroDatum AvroObjectContainerReader::NextImpl( + const AvroSchema* schema, + const Core::Context& context) + { + AZURE_ASSERT_FALSE(m_eof); + static const auto SyncMarkerSchema = AvroSchema::FixedSchema("Sync", 16); + if (!schema) + { + static AvroSchema FileHeaderSchema = []() { + std::vector> fieldsSchema; + fieldsSchema.push_back(std::make_pair("magic", AvroSchema::FixedSchema("Magic", 4))); + fieldsSchema.push_back( + std::make_pair("meta", AvroSchema::MapSchema(AvroSchema::BytesSchema))); + fieldsSchema.push_back(std::make_pair("sync", SyncMarkerSchema)); + return AvroSchema::RecordSchema("org.apache.avro.file.Header", std::move(fieldsSchema)); + }(); + auto fileHeaderDatum = AvroDatum(FileHeaderSchema); + fileHeaderDatum.Fill(*m_reader, context); + auto fileHeader = fileHeaderDatum.Value(); + if (fileHeader.Field("magic").Value() != "Obj\01") + { + throw std::runtime_error("Invalid Avro object container magic."); + } + AvroMap meta = fileHeader.Field("meta").Value(); + std::string objectSchemaJson = meta["avro.schema"].Value(); + std::string codec = "null"; + if (meta.count("avro.codec") != 0) + { + codec = meta["avro.codec"].Value(); + } + if (codec != "null") + { + throw std::runtime_error("Unsupported Avro codec: " + codec); + } + m_syncMarker = fileHeader.Field("sync").Value(); + m_objectSchema = std::make_unique(ParseSchemaFromJsonString(objectSchemaJson)); + schema = m_objectSchema.get(); + } + + if (m_remainingObjectInCurrentBlock == 0) + { + m_reader->Discard(); + m_remainingObjectInCurrentBlock = m_reader->ParseInt(context); + int64_t ObjectsSize = m_reader->ParseInt(context); + m_reader->Preload(static_cast(ObjectsSize), context); + } + + auto objectDatum = AvroDatum(*m_objectSchema); + objectDatum.Fill(*m_reader, context); + if (--m_remainingObjectInCurrentBlock == 0) + { + auto markerDatum = AvroDatum(SyncMarkerSchema); + markerDatum.Fill(*m_reader, context); + auto marker = markerDatum.Value(); + if (marker != m_syncMarker) + { + throw std::runtime_error("Sync marker doesn't match."); + } + m_eof = m_reader->TryPreload(1, context) == 0; + } + return objectDatum; + } + + size_t AvroStreamParser::OnRead( + uint8_t* buffer, + size_t count, + Azure::Core::Context const& context) + { + if (m_parserBuffer.Length != 0) + { + size_t bytesToCopy = std::min(m_parserBuffer.Length, count); + std::memcpy(buffer, m_parserBuffer.Data, bytesToCopy); + m_parserBuffer.Data += bytesToCopy; + m_parserBuffer.Length -= bytesToCopy; + return bytesToCopy; + } + while (!m_parser.End()) + { + auto datum = m_parser.Next(context); + if (datum.Schema().Type() == AvroDatumType::Union) + { + datum = datum.Value(); + } + if (datum.Schema().Type() != AvroDatumType::Record) + { + continue; + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.resultData") + { + auto record = datum.Value(); + auto dataDatum = record.Field("data"); + m_parserBuffer = dataDatum.Value(); + return OnRead(buffer, count, context); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.progress" + && m_progressCallback) + { + auto record = datum.Value(); + auto bytesScanned = record.Field("bytesScanned").Value(); + auto totalBytes = record.Field("totalBytes").Value(); + m_progressCallback(bytesScanned, totalBytes); + } + if (datum.Schema().Name() == "com.microsoft.azure.storage.queryBlobContents.error" + && m_errorCallback) + { + auto record = datum.Value(); + BlobQueryError e; + e.Name = record.Field("name").Value(); + e.Description = record.Field("description").Value(); + e.IsFatal = record.Field("fatal").Value(); + e.Position = record.Field("position").Value(); + m_errorCallback(std::move(e)); + } + } + return 0; + } +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp b/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp new file mode 100644 index 0000000000..0d6554053a --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/avro_parser.hpp @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include + +#include + +#include "azure/storage/blobs/blob_options.hpp" + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + enum class AvroDatumType + { + String, + Bytes, + Int, + Long, + Float, + Double, + Bool, + Null, + Record, + Enum, + Array, + Map, + Union, + Fixed, + }; + + class AvroStreamReader final { + public: + // position of a vector that lives through vector resizing + struct ReaderPos final + { + const std::vector* BufferPtr = nullptr; + size_t Offset = 0; + }; + explicit AvroStreamReader(Core::IO::BodyStream& stream) + : m_stream(&stream), m_pos{&m_streambuffer, 0} + { + } + AvroStreamReader(const AvroStreamReader&) = delete; + AvroStreamReader& operator=(const AvroStreamReader&) = delete; + + int64_t ParseInt(const Core::Context& context); + void Advance(size_t n, const Core::Context& context); + // Read at least n bytes from m_stream and append data to m_streambuffer. Return number of bytes + // available in m_streambuffer; + size_t Preload(size_t n, const Core::Context& context); + size_t TryPreload(size_t n, const Core::Context& context); + // discards data that's before m_pos + void Discard(); + + private: + size_t AvailableBytes() const { return m_streambuffer.size() - m_pos.Offset; } + + private: + Core::IO::BodyStream* m_stream; + std::vector m_streambuffer; + ReaderPos m_pos; + + friend class AvroDatum; + }; + + class AvroSchema final { + public: + static const AvroSchema StringSchema; + static const AvroSchema BytesSchema; + static const AvroSchema IntSchema; + static const AvroSchema LongSchema; + static const AvroSchema FloatSchema; + static const AvroSchema DoubleSchema; + static const AvroSchema BoolSchema; + static const AvroSchema NullSchema; + static AvroSchema RecordSchema( + std::string name, + const std::vector>& fieldsSchema); + static AvroSchema ArraySchema(AvroSchema elementSchema); + static AvroSchema MapSchema(AvroSchema elementSchema); + static AvroSchema UnionSchema(std::vector schemas); + static AvroSchema FixedSchema(std::string name, int64_t size); + + const std::string& Name() const { return m_name; } + AvroDatumType Type() const { return m_type; } + const std::vector& FieldNames() const { return m_status->m_keys; } + AvroSchema ItemSchema() const { return m_status->m_schemas[0]; } + const std::vector& FieldSchemas() const { return m_status->m_schemas; } + size_t Size() const { return static_cast(m_status->m_size); } + + private: + explicit AvroSchema(AvroDatumType type) : m_type(type) {} + + private: + AvroDatumType m_type; + std::string m_name; + + struct SharedStatus + { + std::vector m_keys; + std::vector m_schemas; + int64_t m_size = 0; + }; + std::shared_ptr m_status; + }; + + class AvroDatum; + + class AvroRecord final { + public: + bool HasField(const std::string& key) const { return FindField(key) != m_keys->size(); } + const AvroDatum& Field(const std::string& key) const { return m_values.at(FindField(key)); } + AvroDatum& Field(const std::string& key) { return m_values.at(FindField(key)); } + const AvroDatum& FieldAt(size_t i) const { return m_values.at(i); } + AvroDatum& FieldAt(size_t i) { return m_values.at(i); } + + private: + size_t FindField(const std::string& key) const + { + auto i = find(m_keys->begin(), m_keys->end(), key); + return i - m_keys->begin(); + } + const std::vector* m_keys = nullptr; + std::vector m_values; + + friend class AvroDatum; + }; + + using AvroMap = std::map; + + class AvroDatum final { + public: + AvroDatum() : m_schema(AvroSchema::NullSchema) {} + explicit AvroDatum(AvroSchema schema) : m_schema(std::move(schema)) {} + + void Fill(AvroStreamReader& reader, const Core::Context& context); + void Fill(AvroStreamReader::ReaderPos& data); + + const AvroSchema& Schema() const { return m_schema; } + + template T Value() const; + struct StringView + { + const uint8_t* Data = nullptr; + size_t Length = 0; + }; + + private: + AvroSchema m_schema; + AvroStreamReader::ReaderPos m_data; + }; + + class AvroObjectContainerReader final { + public: + explicit AvroObjectContainerReader(Core::IO::BodyStream& stream); + + bool End() const { return m_eof; } + // Calling Next() will invalidates the previous AvroDatum returned by this function and all + // AvroDatums propagated from there. + AvroDatum Next(const Core::Context& context) { return NextImpl(m_objectSchema.get(), context); } + + private: + AvroDatum NextImpl(const AvroSchema* schema, const Core::Context& context); + + private: + std::unique_ptr m_reader; + std::unique_ptr m_objectSchema; + std::string m_syncMarker; + int64_t m_remainingObjectInCurrentBlock = 0; + bool m_eof = false; + }; + + class AvroStreamParser final : public Core::IO::BodyStream { + public: + explicit AvroStreamParser( + std::unique_ptr inner, + std::function progressCallback, + std::function errorCallback) + : m_inner(std::move(inner)), m_parser(*m_inner), + m_progressCallback(std::move(progressCallback)), m_errorCallback(std::move(errorCallback)) + { + } + + int64_t Length() const override { return -1; } + void Rewind() override { this->m_inner->Rewind(); } + + private: + size_t OnRead(uint8_t* buffer, size_t count, const Azure::Core::Context& context) override; + + private: + std::unique_ptr m_inner; + AvroObjectContainerReader m_parser; + std::function m_progressCallback; + std::function m_errorCallback; + AvroDatum::StringView m_parserBuffer; + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index a2b55c9fef..4fdcad2249 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -33,13 +33,15 @@ std::string ListBlobContainersIncludeFlagsToString( const Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags valueList[] = { Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::Metadata, Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::Deleted, + Azure::Storage::Blobs::Models::ListBlobContainersIncludeFlags::System, }; const char* stringList[] = { "metadata", "deleted", + "system", }; std::string ret; - for (size_t i = 0; i < 2; ++i) + for (size_t i = 0; i < 3; ++i) { if ((val & valueList[i]) == valueList[i]) { @@ -65,6 +67,7 @@ std::string ListBlobsIncludeFlagsToString( Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::Tags, Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy, Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::LegalHold, + Azure::Storage::Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions, }; const char* stringList[] = { "copy", @@ -76,9 +79,10 @@ std::string ListBlobsIncludeFlagsToString( "tags", "immutabilitypolicy", "legalhold", + "deletedwithversions", }; std::string ret; - for (size_t i = 0; i < 9; ++i) + for (size_t i = 0; i < 10; ++i) { if ((val & valueList[i]) == valueList[i]) { @@ -147,6 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { const AccessTier AccessTier::Hot("Hot"); const AccessTier AccessTier::Cool("Cool"); const AccessTier AccessTier::Archive("Archive"); + const AccessTier AccessTier::Premium("Premium"); const ArchiveStatus ArchiveStatus::RehydratePendingToHot("rehydrate-pending-to-hot"); const ArchiveStatus ArchiveStatus::RehydratePendingToCool("rehydrate-pending-to-cool"); const RehydratePriority RehydratePriority::High("High"); @@ -165,6 +170,19 @@ namespace Azure { namespace Storage { namespace Blobs { "RelativeToCreation"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::RelativeToNow("RelativeToNow"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::Absolute("Absolute"); + namespace _detail { + const QueryRequestQueryType QueryRequestQueryType::SQL("SQL"); + const QueryFormatType QueryFormatType::Delimited("delimited"); + const QueryFormatType QueryFormatType::Json("json"); + const QueryFormatType QueryFormatType::Arrow("arrow"); + const QueryFormatType QueryFormatType::Parquet("parquet"); + } // namespace _detail + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Int64("int64"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Bool("bool"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Timestamp("timestamp[ms]"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::String("string"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Double("double"); + const BlobQueryArrowFieldType BlobQueryArrowFieldType::Decimal("decimal"); const SequenceNumberAction SequenceNumberAction::Max("max"); const SequenceNumberAction SequenceNumberAction::Update("update"); const SequenceNumberAction SequenceNumberAction::Increment("increment"); @@ -356,7 +374,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -376,7 +394,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -672,7 +690,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "stats"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -772,7 +790,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobContainersIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -808,8 +826,8 @@ namespace Azure { namespace Storage { namespace Blobs { kDenyEncryptionScopeOverride, kDeletedTime, kRemainingRetentionDays, - kMetadata, kImmutableStorageWithVersioningEnabled, + kMetadata, }; const std::unordered_map XmlTagEnumMap{ {"EnumerationResults", XmlTagEnum::kEnumerationResults}, @@ -833,9 +851,9 @@ namespace Azure { namespace Storage { namespace Blobs { {"DenyEncryptionScopeOverride", XmlTagEnum::kDenyEncryptionScopeOverride}, {"DeletedTime", XmlTagEnum::kDeletedTime}, {"RemainingRetentionDays", XmlTagEnum::kRemainingRetentionDays}, - {"Metadata", XmlTagEnum::kMetadata}, {"ImmutableStorageWithVersioningEnabled", XmlTagEnum::kImmutableStorageWithVersioningEnabled}, + {"Metadata", XmlTagEnum::kMetadata}, }; std::vector xmlPath; Models::BlobContainerItem vectorElement1; @@ -988,18 +1006,18 @@ namespace Azure { namespace Storage { namespace Blobs { else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kContainers && xmlPath[2] == XmlTagEnum::kContainer - && xmlPath[3] == XmlTagEnum::kMetadata) + && xmlPath[3] == XmlTagEnum::kProperties + && xmlPath[4] == XmlTagEnum::kImmutableStorageWithVersioningEnabled) { - mapValue3 = node.Value; + vectorElement1.Details.HasImmutableStorageWithVersioning + = node.Value == std::string("true"); } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kContainers && xmlPath[2] == XmlTagEnum::kContainer - && xmlPath[3] == XmlTagEnum::kProperties - && xmlPath[4] == XmlTagEnum::kImmutableStorageWithVersioningEnabled) + && xmlPath[3] == XmlTagEnum::kMetadata) { - vectorElement1.Details.HasImmutableStorageWithVersioning - = node.Value == std::string("true"); + mapValue3 = node.Value; } } else if (node.Type == _internal::XmlNodeType::Attribute) @@ -1057,7 +1075,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "userdelegationkey"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1172,7 +1190,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "account"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -1202,7 +1220,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1223,7 +1241,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "blobs"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.Where.HasValue() && !options.Where.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1379,7 +1397,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-public-access", options.Access.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.DefaultEncryptionScope.HasValue() && !options.DefaultEncryptionScope.Value().empty()) { @@ -1416,7 +1434,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1494,7 +1512,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1528,7 +1546,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Modified-Since", options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1555,7 +1573,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1724,7 +1742,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1747,7 +1765,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.DeletedContainerName.HasValue() && !options.DeletedContainerName.Value().empty()) { request.SetHeader("x-ms-deleted-container-name", options.DeletedContainerName.Value()); @@ -1777,7 +1795,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "container"); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (!options.SourceContainerName.empty()) { request.SetHeader("x-ms-source-container-name", options.SourceContainerName); @@ -1811,7 +1829,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("Content-Type", options.MultipartContentType); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -1854,7 +1872,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -1895,7 +1913,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1935,7 +1953,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1976,7 +1994,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -2021,7 +2039,7 @@ namespace Azure { namespace Storage { namespace Blobs { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2068,7 +2086,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2138,6 +2156,7 @@ namespace Azure { namespace Storage { namespace Blobs { kOrMetadata, kImmutabilityPolicyUntilDate, kImmutabilityPolicyMode, + kHasVersionsOnly, kContentLength, kBlobType, }; @@ -2197,6 +2216,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"OrMetadata", XmlTagEnum::kOrMetadata}, {"ImmutabilityPolicyUntilDate", XmlTagEnum::kImmutabilityPolicyUntilDate}, {"ImmutabilityPolicyMode", XmlTagEnum::kImmutabilityPolicyMode}, + {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, }; @@ -2614,6 +2634,13 @@ namespace Azure { namespace Storage { namespace Blobs { vectorElement1.Details.ImmutabilityPolicy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kHasVersionsOnly) + { + vectorElement1.HasVersionsOnly = node.Value == std::string("true"); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob @@ -2722,7 +2749,7 @@ namespace Azure { namespace Storage { namespace Blobs { _internal::UrlEncodeQueryParameter( ListBlobsIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2793,6 +2820,7 @@ namespace Azure { namespace Storage { namespace Blobs { kOrMetadata, kImmutabilityPolicyUntilDate, kImmutabilityPolicyMode, + kHasVersionsOnly, kContentLength, kBlobType, kBlobPrefix, @@ -2854,6 +2882,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"OrMetadata", XmlTagEnum::kOrMetadata}, {"ImmutabilityPolicyUntilDate", XmlTagEnum::kImmutabilityPolicyUntilDate}, {"ImmutabilityPolicyMode", XmlTagEnum::kImmutabilityPolicyMode}, + {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, {"BlobPrefix", XmlTagEnum::kBlobPrefix}, @@ -3279,6 +3308,13 @@ namespace Azure { namespace Storage { namespace Blobs { vectorElement1.Details.ImmutabilityPolicy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kHasVersionsOnly) + { + vectorElement1.HasVersionsOnly = node.Value == std::string("true"); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob @@ -3440,7 +3476,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok @@ -3710,7 +3746,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -3966,7 +4002,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -3984,7 +4020,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "undelete"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4003,7 +4039,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "expiry"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (!options.ExpiryOptions.ToString().empty()) { request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.ToString()); @@ -4085,7 +4121,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-blob-content-disposition", options.BlobContentDisposition); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4112,7 +4148,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.IfUnmodifiedSince.HasValue()) { request.SetHeader( @@ -4155,7 +4191,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Delete, url); request.GetUrl().AppendQueryParameter("comp", "immutabilityPolicies"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); (void)options; auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4175,7 +4211,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "legalhold"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); request.SetHeader("x-ms-legal-hold", options.LegalHold ? "true" : "false"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); @@ -4247,7 +4283,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4316,7 +4352,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -4368,7 +4404,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4419,7 +4455,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4475,7 +4511,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -4527,7 +4563,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -4601,7 +4637,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -4702,7 +4738,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -4815,7 +4851,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -4844,6 +4880,12 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-legal-hold", options.LegalHold.Value() ? "true" : "false"); } + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } if (options.SourceContentcrc64.HasValue() && !Core::Convert::Base64Encode(options.SourceContentcrc64.Value()).empty()) { @@ -4901,7 +4943,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::NoContent) @@ -4939,7 +4981,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-rehydrate-priority", options.RehydratePriority.Value().ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4959,6 +5001,317 @@ namespace Azure { namespace Storage { namespace Blobs { return Response( std::move(response), std::move(pRawResponse)); } + Response BlobClient::Query( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const QueryBlobOptions& options, + const Core::Context& context) + { + std::string xmlBody; + { + _internal::XmlWriter writer; + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "QueryRequest"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "QueryType", + options.QueryRequest.QueryType.ToString()}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Expression", options.QueryRequest.Expression}); + if (options.QueryRequest.InputSerialization.HasValue()) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "InputSerialization"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Format"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Type", + options.QueryRequest.InputSerialization.Value().Format.Type.ToString()}); + if (options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "DelimitedTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "ColumnSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .ColumnSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "FieldQuote", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .FieldQuote}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "EscapeChar", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .EscapeChar}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "HasHeaders", + options.QueryRequest.InputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .HeadersPresent + ? "true" + : "false"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value() + .Format.JsonTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "JsonTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.InputSerialization.Value() + .Format.JsonTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value().Format.ArrowConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ArrowConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Schema"}); + for (const auto& i1 : options.QueryRequest.InputSerialization.Value() + .Format.ArrowConfiguration.Value() + .Schema) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Field"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Type", i1.Type.ToString()}); + if (i1.Name.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Name", i1.Name.Value()}); + } + if (i1.Precision.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Precision", + std::to_string(i1.Precision.Value())}); + } + if (i1.Scale.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Scale", std::to_string(i1.Scale.Value())}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.InputSerialization.Value() + .Format.ParquetTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ParquetTextConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.HasValue()) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "OutputSerialization"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Format"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Type", + options.QueryRequest.OutputSerialization.Value().Format.Type.ToString()}); + if (options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "DelimitedTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "ColumnSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .ColumnSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "FieldQuote", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .FieldQuote}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "EscapeChar", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .EscapeChar}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "HasHeaders", + options.QueryRequest.OutputSerialization.Value() + .Format.DelimitedTextConfiguration.Value() + .HeadersPresent + ? "true" + : "false"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value() + .Format.JsonTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "JsonTextConfiguration"}); + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "RecordSeparator", + options.QueryRequest.OutputSerialization.Value() + .Format.JsonTextConfiguration.Value() + .RecordSeparator}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value().Format.ArrowConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ArrowConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Schema"}); + for (const auto& i2 : options.QueryRequest.OutputSerialization.Value() + .Format.ArrowConfiguration.Value() + .Schema) + { + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "Field"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Type", i2.Type.ToString()}); + if (i2.Name.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "Name", i2.Name.Value()}); + } + if (i2.Precision.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, + "Precision", + std::to_string(i2.Precision.Value())}); + } + if (i2.Scale.HasValue()) + { + writer.Write(_internal::XmlNode{ + _internal::XmlNodeType::StartTag, "Scale", std::to_string(i2.Scale.Value())}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + if (options.QueryRequest.OutputSerialization.Value() + .Format.ParquetTextConfiguration.HasValue()) + { + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "ParquetTextConfiguration"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + } + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::End}); + xmlBody = writer.GetDocument(); + } + Core::IO::MemoryBodyStream requestBody( + reinterpret_cast(xmlBody.data()), xmlBody.length()); + auto request = Core::Http::Request(Core::Http::HttpMethod::Post, url, &requestBody, false); + request.SetHeader("Content-Type", "application/xml; charset=UTF-8"); + request.SetHeader("Content-Length", std::to_string(requestBody.Length())); + request.GetUrl().AppendQueryParameter("comp", "query"); + if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "snapshot", _internal::UrlEncodeQueryParameter(options.Snapshot.Value())); + } + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } + if (options.IfModifiedSince.HasValue()) + { + request.SetHeader( + "If-Modified-Since", + options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfUnmodifiedSince.HasValue()) + { + request.SetHeader( + "If-Unmodified-Since", + options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfMatch.HasValue() && !options.IfMatch.ToString().empty()) + { + request.SetHeader("If-Match", options.IfMatch.ToString()); + } + if (options.IfNoneMatch.HasValue() && !options.IfNoneMatch.ToString().empty()) + { + request.SetHeader("If-None-Match", options.IfNoneMatch.ToString()); + } + if (options.IfTags.HasValue() && !options.IfTags.Value().empty()) + { + request.SetHeader("x-ms-if-tags", options.IfTags.Value()); + } + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.EncryptionScope.HasValue() && !options.EncryptionScope.Value().empty()) + { + request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); + } + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok + || httpStatusCode == Core::Http::HttpStatusCode::PartialContent)) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::QueryBlobResult response; + response.BodyStream = pRawResponse->ExtractBodyStream(); + response.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + response.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + response.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + return Response(std::move(response), std::move(pRawResponse)); + } Response> BlobClient::GetTags( Core::Http::_internal::HttpPipeline& pipeline, const Core::Url& url, @@ -4967,7 +5320,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5094,7 +5447,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("Content-Type", "application/xml; charset=UTF-8"); request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("comp", "tags"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.VersionId.HasValue() && !options.VersionId.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -5225,7 +5578,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -5365,7 +5718,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5482,7 +5835,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5616,7 +5969,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -5702,7 +6061,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -5722,6 +6091,7 @@ namespace Azure { namespace Storage { namespace Blobs { kStart, kEnd, kClearRange, + kNextMarker, }; const std::unordered_map XmlTagEnumMap{ {"PageList", XmlTagEnum::kPageList}, @@ -5729,6 +6099,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"Start", XmlTagEnum::kStart}, {"End", XmlTagEnum::kEnd}, {"ClearRange", XmlTagEnum::kClearRange}, + {"NextMarker", XmlTagEnum::kNextMarker}, }; std::vector xmlPath; Core::Http::HttpRange vectorElement1; @@ -5770,6 +6141,12 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement2.Length = std::stoll(node.Value); } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kPageList + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.ContinuationToken = node.Value; + } } else if (node.Type == _internal::XmlNodeType::Attribute) { @@ -5856,7 +6233,17 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -5876,6 +6263,7 @@ namespace Azure { namespace Storage { namespace Blobs { kStart, kEnd, kClearRange, + kNextMarker, }; const std::unordered_map XmlTagEnumMap{ {"PageList", XmlTagEnum::kPageList}, @@ -5883,6 +6271,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"Start", XmlTagEnum::kStart}, {"End", XmlTagEnum::kEnd}, {"ClearRange", XmlTagEnum::kClearRange}, + {"NextMarker", XmlTagEnum::kNextMarker}, }; std::vector xmlPath; Core::Http::HttpRange vectorElement1; @@ -5924,6 +6313,12 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement2.Length = std::stoll(node.Value); } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kPageList + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.ContinuationToken = node.Value; + } } else if (node.Type == _internal::XmlNodeType::Attribute) { @@ -6012,7 +6407,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } request.SetHeader("x-ms-blob-content-length", std::to_string(options.BlobContentLength)); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6072,7 +6467,7 @@ namespace Azure { namespace Storage { namespace Blobs { request.SetHeader( "x-ms-blob-sequence-number", std::to_string(options.BlobSequenceNumber.Value())); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -6124,7 +6519,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-copy-source", options.CopySource); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -6229,7 +6624,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -6357,7 +6752,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6511,7 +6906,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -6561,7 +6962,7 @@ namespace Azure { namespace Storage { namespace Blobs { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "seal"); - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -6700,7 +7101,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -6883,7 +7284,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-tags", options.SourceIfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.SourceContentMD5.HasValue() && !Core::Convert::Base64Encode(options.SourceContentMD5.Value()).empty()) { @@ -6905,6 +7306,12 @@ namespace Azure { namespace Storage { namespace Blobs { "x-ms-copy-source-blob-properties", options.CopySourceBlobProperties.Value() ? "true" : "false"); } + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } if (options.SourceContentcrc64.HasValue() && !Core::Convert::Base64Encode(options.SourceContentcrc64.Value()).empty()) { @@ -7005,7 +7412,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-encryption-scope", options.EncryptionScope.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7119,7 +7526,13 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-source-if-none-match", options.SourceIfNoneMatch.ToString()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); + if (options.CopySourceAuthorization.HasValue() + && !options.CopySourceAuthorization.Value().empty()) + { + request.SetHeader( + "x-ms-copy-source-authorization", options.CopySourceAuthorization.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -7280,7 +7693,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); if (options.BlobTagsString.HasValue() && !options.BlobTagsString.Value().empty()) { request.SetHeader("x-ms-tags", options.BlobTagsString.Value()); @@ -7369,7 +7782,7 @@ namespace Azure { namespace Storage { namespace Blobs { { request.SetHeader("x-ms-if-tags", options.IfTags.Value()); } - request.SetHeader("x-ms-version", "2020-08-04"); + request.SetHeader("x-ms-version", "2020-10-02"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index dc73ddd28e..2a31bd673f 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-blobs namespace: Azure::Storage::Blobs output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Jinming-Hu/azure-storage-api-specs/main/Microsoft.BlobStorage/preview/2020-08-04/blob.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.BlobStorage/preview/2020-10-02/blob.json ``` ## ModelFour Options @@ -75,7 +75,6 @@ directive: delete $["/{filesystem}/{path}?action=setAccessControl&blob"]; delete $["/{filesystem}/{path}?action=getAccessControl&blob"]; delete $["/{filesystem}/{path}?FileRename"]; - delete $["/{containerName}/{blob}?comp=query"]; for (const operation in $) { for (const verb in $[operation]) { @@ -103,7 +102,7 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2020-08-04"], + "enum": ["2020-10-02"], "description": "The version used for the operations to Azure storage services." }; ``` @@ -123,7 +122,7 @@ directive: $["/{containerName}?restype=container&comp=undelete"].put.operationId = "BlobContainer_Undelete"; $["/{containerName}/{blob}?comp=copy"].put.operationId = "Blob_StartCopyFromUri"; $["/{containerName}/{blob}?comp=copy&sync"].put.operationId = "Blob_CopyFromUri"; - $["/{containerName}/{blob}?comp=copy©id={CopyId}"].put.operationId = "Blob_AbortCopyFromUri"; + $["/{containerName}/{blob}?comp=copy©id"].put.operationId = "Blob_AbortCopyFromUri"; $["/{containerName}/{blob}?comp=block&fromURL"].put.operationId = "BlockBlob_StageBlockFromUri"; $["/{containerName}/{blob}?comp=page&update&fromUrl"].put.operationId = "PageBlob_UploadPagesFromUri"; $["/{containerName}/{blob}?comp=appendblock&fromUrl"].put.operationId = "AppendBlob_AppendBlockFromUri"; @@ -270,7 +269,8 @@ directive: {"value": "versions", "name": "Versions"}, {"value": "tags", "name": "Tags"}, {"value": "immutabilitypolicy", "name": "ImmutabilityPolicy"}, - {"value": "legalhold", "name": "LegalHold"} + {"value": "legalhold", "name": "LegalHold"}, + {"value": "deletedwithversions", "name": "DeletedWithVersions"} ]; $.DeleteSnapshots["x-ms-enum"]["name"] = "DeleteSnapshotsOption"; $.DeleteSnapshots["x-ms-enum"]["values"] = [{"value": "include", "name": "IncludeSnapshots"},{"value":"only", "name": "OnlySnapshots"}]; @@ -353,6 +353,9 @@ directive: if (h === "x-ms-meta") { $[h]["x-ms-format"] = "caseinsensitivemap"; } + if (h === "x-ms-lease-id" && $[h].description === "Uniquely identifies a blobs' lease") { + $[h].description = "Uniquely identifies a blob's lease"; + } } - from: swagger-document where: $.parameters @@ -552,8 +555,6 @@ directive: delete $.ContainerItem.properties["Metadata"]; $.ContainerProperties.properties["Metadata"]["x-ms-xml"] = {"name": "../Metadata"}; $.ContainerProperties.properties["DeletedTime"]["x-ms-client-name"] = "DeletedOn"; - $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"] = $.ContainerProperties.properties["VersionLevelWormEnabled"]; - delete $.ContainerProperties.properties["VersionLevelWormEnabled"]; $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"]["x-ms-client-name"] = "HasImmutableStorageWithVersioning "; $.ContainerProperties.properties["ImmutableStorageWithVersioningEnabled"]["x-ms-client-default"] = false; delete $.ContainerProperties.required; @@ -658,8 +659,6 @@ directive: $["x-ms-deny-encryption-scope-override"]["x-nullable"] = true; $["x-ms-deny-encryption-scope-override"]["x-ms-client-default"] = "false"; $["x-ms-meta"].description = "A set of name-value pair associated with this blob container."; - $["x-ms-immutable-storage-with-versioning-enabled"] = $["x-ms-version-level-worm-enabled"]; - delete $["x-ms-version-level-worm-enabled"]; $["x-ms-immutable-storage-with-versioning-enabled"]["x-ms-client-name"] = "HasImmutableStorageWithVersioning"; $["x-ms-immutable-storage-with-versioning-enabled"]["x-ms-client-default"] = false; $["x-ms-immutable-storage-with-versioning-enabled"]["x-nullable"] = true; @@ -762,6 +761,7 @@ directive: $.BlobItemInternal.properties["VersionId"].description = "A string value that uniquely identifies a blob version."; $.BlobItemInternal.properties["IsCurrentVersion"].description = "Indicates if this is the current version of the blob."; $.BlobItemInternal.properties["BlobType"].description = "Type of the blob."; + $.BlobItemInternal.properties["HasVersionsOnly"].description = "Indicates that this root blob has been deleted, but it has versions that are active."; $.BlobPropertiesInternal.properties["Etag"]["x-ms-client-name"] = "ETag"; $.BlobPropertiesInternal["x-ms-client-name"] = "BlobItemDetails"; @@ -1172,9 +1172,71 @@ directive: - from: swagger-document where: $.definitions transform: > + $.BlobQueryArrowFieldType = { + "type": "string", + "enum": ["Int64", "Bool", "Timestamp", "String", "Double", "Decimal"], + "x-ms-enum": { + "name": "BlobQueryArrowFieldType", + "modelAsString": false, + "values": [ + {"value": "int64", "name": "Int64"}, + {"value": "bool", "name": "Bool"}, + {"value": "timestamp[ms]", "name": "Timestamp"}, + {"value": "string", "name": "String"}, + {"value": "double", "name": "Double"}, + {"value": "decimal", "name": "Decimal"} + ] + }, + "description": "Type of blob query arrow field." + }; if ($.ParquetConfiguration) { $.ParquetConfiguration.properties = {"__placeHolder" : { "type": "integer"}}; } + $.QuerySerialization["x-namespace"] = "_detail"; + $.QueryFormat["x-namespace"] = "_detail"; + $.QueryType["x-namespace"] = "_detail"; + $.DelimitedTextConfiguration["x-namespace"] = "_detail"; + $.JsonTextConfiguration["x-namespace"] = "_detail"; + $.ArrowConfiguration["x-namespace"] = "_detail"; + $.ParquetConfiguration["x-namespace"] = "_detail"; + $.QueryRequest["x-namespace"] = "_detail"; + $.QueryRequest.properties["QueryType"]["x-namespace"] = "_detail"; + $.ArrowField["x-ms-client-name"] = "BlobQueryArrowField"; + $.ArrowField.properties["Type"] = {"$ref": "#/definitions/BlobQueryArrowFieldType"}; + $.DelimitedTextConfiguration.properties["HeadersPresent"]["x-ms-xml"] = $.DelimitedTextConfiguration.properties["HeadersPresent"]["xml"]; + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=query"].post.parameters + transform: > + $.push({"$ref": "#/parameters/EncryptionScope"}); + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=query"].post.responses + transform: > + for (const status_code of ["200", "206"]) { + delete $[status_code].headers["x-ms-meta"]; + delete $[status_code].headers["Content-Length"]; + delete $[status_code].headers["Content-Type"]; + delete $[status_code].headers["Content-Range"]; + delete $[status_code].headers["Content-MD5"]; + delete $[status_code].headers["Content-Encoding"]; + delete $[status_code].headers["Cache-Control"]; + delete $[status_code].headers["Content-Disposition"]; + delete $[status_code].headers["Content-Language"]; + delete $[status_code].headers["x-ms-blob-sequence-number"]; + delete $[status_code].headers["x-ms-blob-type"]; + delete $[status_code].headers["x-ms-copy-completion-time"]; + delete $[status_code].headers["x-ms-copy-status-description"]; + delete $[status_code].headers["x-ms-copy-id"]; + delete $[status_code].headers["x-ms-copy-progress"]; + delete $[status_code].headers["x-ms-copy-source"]; + delete $[status_code].headers["x-ms-copy-status"]; + delete $[status_code].headers["Accept-Ranges"]; + delete $[status_code].headers["x-ms-blob-committed-block-count"]; + delete $[status_code].headers["x-ms-encryption-key-sha256"]; + delete $[status_code].headers["x-ms-encryption-scope"]; + delete $[status_code].headers["x-ms-blob-content-md5"]; + delete $[status_code].headers["x-ms-content-crc64"]; + $[status_code].headers["x-ms-lease-duration"]["x-nullable"] = true; + } ``` ### PutBlockList @@ -1300,6 +1362,10 @@ directive: "x-ms-client-name": "ClearRanges", "x-ms-xml": {"name": "PageList"}, "items": {"$ref": "#/definitions/ClearRange"} + }, + "ContinuationToken": { + "type": "string", + "x-ms-xml": {"name": "PageList/NextMarker"} } } } @@ -1333,6 +1399,10 @@ directive: "x-ms-client-name": "ClearRanges", "x-ms-xml": {"name": "PageList"}, "items": {"$ref": "#/definitions/ClearRange"} + }, + "ContinuationToken": { + "type": "string", + "x-ms-xml": {"name": "PageList/NextMarker"} } } } diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index 6e5d30accc..8b5ca8b38b 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable ( append_blob_client_test.hpp blob_container_client_test.cpp blob_container_client_test.hpp + blob_query_test.cpp blob_sas_test.cpp blob_service_client_test.cpp block_blob_client_test.cpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 08aa9c65e1..deb79a8728 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -1274,4 +1274,25 @@ namespace Azure { namespace Storage { namespace Test { blobClient.Delete(); EXPECT_THROW(blobClient.GetProperties(), StorageException); } + + TEST_F(BlobContainerClientTest, ListBlobsDeletedWithActiveVersions) + { + auto client = GetBlobContainerTestClient(); + client.Create(); + + std::string blobName = "blob" + m_containerName; + auto blobClient = client.GetAppendBlobClient(blobName); + blobClient.Create(); + + auto blobItem + = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions); + ASSERT_TRUE(blobItem.HasVersionsOnly.HasValue()); + EXPECT_FALSE(blobItem.HasVersionsOnly.Value()); + + blobClient.Delete(); + + blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::DeletedWithVersions); + ASSERT_TRUE(blobItem.HasVersionsOnly.HasValue()); + EXPECT_TRUE(blobItem.HasVersionsOnly.Value()); + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp new file mode 100644 index 0000000000..7a139f8093 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "block_blob_client_test.hpp" + +#include +#include +#include + +// cspell:ignore sapote + +namespace Azure { namespace Storage { namespace Test { + + const std::string JsonQueryTestData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": 50} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": 99} +{"id": 104, "name": "clementines", "price": 399} +{"id": 105, "name": "grapes", "price": 150} +{"id": 106, "name": "lemons", "price": 69} +{"id": 107, "name": "pears", "price": 100} +{"id": 108, "name": "cherries", "price": 281} +{"id": 109, "name": "coconut", "price": 178} +{"id": 110, "name": "bananas", "price": 39} +{"id": 111, "name": "peaches", "price": 117} +{"id": 112, "name": "sapote,\"mamey", "price": 50} +)json"; + + const std::string CsvQueryTestData = R"csv( +id,name,price +100,oranges,100 +101,limes,50 +102,berries,199 +103,apples,99 +104,clementines,399 +105,grapes,150 +106,lemons,69 +107,pears,100 +108,cherries,281 +109,coconut,178 +110,bananas,39 +111,peaches,117 +112,sapote\,mamey,50 +)csv"; + + const std::vector ParquetQueryTestData = Core::Convert::Base64Decode( + "UEFSMRUAFewBFewBLBUaFQAVBhUIAAACAAAAGgFkAAAAAAAAAGUAAAAAAAAAZgAAAAAAAABnAAAAAAAAAGgAAAAAAAAA" + "aQAAAAAAAABqAAAAAAAAAGsAAAAAAAAAbAAAAAAAAABtAAAAAAAAAG4AAAAAAAAAbwAAAAAAAABwAAAAAAAAAAAAAAAA" + "AAAAFQAVxAIVxAIsFRoVABUGFQgAAAIAAAAaAQcAAABvcmFuZ2VzBQAAAGxpbWVzBwAAAGJlcnJpZXMGAAAAYXBwbGVz" + "CwAAAGNsZW1lbnRpbmVzBgAAAGdyYXBlcwYAAABsZW1vbnMFAAAAcGVhcnMIAAAAY2hlcnJpZXMHAAAAY29jb251dAcA" + "AABiYW5hbmFzBwAAAHBlYWNoZXMOAAAAc2Fwb3RlLCJtYW1leSIAAAAAAAAAABUAFewBFewBLBUaFQAVBhUIAAACAAAA" + "GgFkAAAAAAAAADIAAAAAAAAAxwAAAAAAAABjAAAAAAAAAI8BAAAAAAAAlgAAAAAAAABFAAAAAAAAAGQAAAAAAAAAGQEA" + "AAAAAACyAAAAAAAAACcAAAAAAAAAdQAAAAAAAAAyAAAAAAAAAAAAAAAAAAAAFQIZTEgGc2NoZW1hFQYAFQQVgAEVAhgC" + "aWQAFQwlAhgEbmFtZSUAABUEFYABFQIYBXByaWNlABYaGRwZPCaaAhwVBBkVABkYAmlkFQAWGhaSAhaSAhkAFgg8GAhw" + "AAAAAAAAABgIZAAAAAAAAAAWAAAZHBUAFQAVAgAAACaEBRwVDBkVABkYBG5hbWUVABYaFuoCFuoCGQAWmgI8GA5zYXBv" + "dGUsIm1hbWV5IhgGYXBwbGVzFgAAGRwVABUAFQIAAAAmlgccFQQZFQAZGAVwcmljZRUAFhoWkgIWkgIZABaEBTwYCI8B" + "AAAAAAAAGAgnAAAAAAAAABYAABkcFQAVABUCAAAAFo4HFhoAGRwYBnBhbmRhcxiRBXsiY29sdW1uX2luZGV4ZXMiOiBb" + "eyJmaWVsZF9uYW1lIjogbnVsbCwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiBudWxsLCAibnVtcHlfdHlwZSI6ICJv" + "YmplY3QiLCAicGFuZGFzX3R5cGUiOiAibWl4ZWQtaW50ZWdlciJ9XSwgImNvbHVtbnMiOiBbeyJmaWVsZF9uYW1lIjog" + "ImlkIiwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiAiaWQiLCAibnVtcHlfdHlwZSI6ICJpbnQ2NCIsICJwYW5kYXNf" + "dHlwZSI6ICJpbnQ2NCJ9LCB7ImZpZWxkX25hbWUiOiAibmFtZSIsICJtZXRhZGF0YSI6IG51bGwsICJuYW1lIjogIm5h" + "bWUiLCAibnVtcHlfdHlwZSI6ICJvYmplY3QiLCAicGFuZGFzX3R5cGUiOiAidW5pY29kZSJ9LCB7ImZpZWxkX25hbWUi" + "OiAicHJpY2UiLCAibWV0YWRhdGEiOiBudWxsLCAibmFtZSI6ICJwcmljZSIsICJudW1weV90eXBlIjogImludDY0Iiwg" + "InBhbmRhc190eXBlIjogImludDY0In1dLCAiY3JlYXRvciI6IHsibGlicmFyeSI6ICJmYXN0cGFycXVldCIsICJ2ZXJz" + "aW9uIjogIjAuOC4xIn0sICJpbmRleF9jb2x1bW5zIjogW3sia2luZCI6ICJyYW5nZSIsICJuYW1lIjogbnVsbCwgInN0" + "YXJ0IjogMCwgInN0ZXAiOiAxLCAic3RvcCI6IDEzfV0sICJwYW5kYXNfdmVyc2lvbiI6ICIxLjQuMiIsICJwYXJ0aXRp" + "b25fY29sdW1ucyI6IFtdfQAYKmZhc3RwYXJxdWV0LXB5dGhvbiB2ZXJzaW9uIDAuOC4xIChidWlsZCAwKQDXAwAAUEFS" + "MQ=="); + + TEST_F(BlockBlobClientTest, QueryJsonInputCsvOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(JsonQueryTestData.data()), JsonQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateJsonTextOptions(); + + { + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions(); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103,apples,99 +106,lemons,69 +110,bananas,39 +112,"sapote,""mamey",50 +)csv"); + } + + { + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions("|", ".", "[", "\\", true); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103.apples.99|106.lemons.69|110.bananas.39|112.sapote,"mamey.50|)csv"); + } + } + + TEST_F(BlockBlobClientTest, QueryCsvInputJsonOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions("\n", ",", "\"", "\\", true); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions("|"); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); + } + + TEST_F(BlockBlobClientTest, DISABLED_QueryCsvInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions("\n", ",", "\"", "\\", true); + std::vector fields; + Blobs::Models::BlobQueryArrowField field; + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::Decimal; + field.Name = "price"; + field.Precision = 10; + field.Scale = 2; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+gAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAACAAAAAQAAAAAQAAAC" + "c////AAABBxAAAAAgAAAABAAAAAAAAAAFAAAAcHJpY2UAAAAIAAwABAAIAAgAAAAKAAAAAgAAANT///" + "8AAAEFEAAAABwAAAAEAAAAAAAAAAQAAABuYW1lAAAAAAQABAAEAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEC" + "EAAAABwAAAAEAAAAAAAAAAIAAABpZAAACAAMAAgABwAIAAAAAAAAAUAAAAAAAAAA//////" + "AAAAAUAAAAAAAAAAwAGgAGAAUACAAMAAwAAAAAAwQAHAAAAAgAAAAAAAAAAAAAAAAACgAMAAAABAAIAAoAAACA" + "AAAABAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAQAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAMA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/////" + "4AAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAACYAAAAAAAAAAAACgAYAAwABAAIAAoAAACMAAAAE" + "AAAAAQAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAA" + "AAAAgAAAAAAAAABQAAAAAAAAAOAAAAAAAAAAfAAAAAAAAAFgAAAAAAAAAAAAAAAAAAABYAAAAAAAAAEAAAAAAA" + "AAAAAAAAAMAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAABnAAAAAAA" + "AAGoAAAAAAAAAbgAAAAAAAABwAAAAAAAAAAAAAAAGAAAADAAAABMAAAAfAAAAAAAAAGFwcGxlc2xlbW9uc2Jhb" + "mFuYXNzYXBvdGUsbWFtZXkAYwAAAAAAAAAAAAAAAAAAAEUAAAAAAAAAAAAAAAAAAAAnAAAAAAAAAAAAAAAAAAA" + "AMgAAAAAAAAAAAAAAAAAAAA=="); + EXPECT_EQ(data, expectedData); + } + + TEST_F(BlockBlobClientTest, DISABLED_QueryParquetInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + client.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration + = Blobs::BlobQueryInputTextOptions::CreateParquetTextOptions(); + std::vector fields; + Blobs::Models::BlobQueryArrowField field; + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Blobs::Models::BlobQueryArrowFieldType::Int64; + field.Name = "price"; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAAB4AAAAOAAAAAQAAAC" + "k////AAABAhAAAAAYAAAABAAAAAAAAAAFAAAAcHJpY2UAAACY////AAAAAUAAAADU////" + "AAABBRAAAAAcAAAABAAAAAAAAAAEAAAAbmFtZQAAAAAEAAQABAAAABAAFAAIAAYABwAMAAAAEAAQAAAAAAABAh" + "AAAAAcAAAABAAAAAAAAAACAAAAaWQAAAgADAAIAAcACAAAAAAAAAFAAAAAAAAAAP/////" + "wAAAAFAAAAAAAAAAMABoABgAFAAgADAAMAAAAAAMEABwAAAAIAAAAAAAAAAAAAAAAAAoADAAAAAQACAAKAAAAg" + "AAAAAQAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAADA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAZwAAAAAA" + "AAAAAAAABgAAAGFwcGxlcwAAYwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAagAAAAAA" + "AAAAAAAABgAAAGxlbW9ucwAARQAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABwAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAbgAAAAAA" + "AAAAAAAABwAAAGJhbmFuYXMAJwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAKAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAADgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAcAAAAAAA" + "AAAAAAAADgAAAHNhcG90ZSwibWFtZXkiAAAyAAAAAAAAAP////8AAAAA"); + EXPECT_EQ(data, expectedData); + } + + TEST_F(BlockBlobClientTest, QueryWithError_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + const std::string malformedData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": "aa"} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": "bb"} +{"id": 104, "name": "clementines", "price": 399} +xx +)json"; + client.UploadFrom(reinterpret_cast(malformedData.data()), malformedData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateJsonTextOptions(); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + + try + { + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Core::Http::HttpStatusCode::Ok); + EXPECT_EQ(e.ReasonPhrase, "OK"); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "ParseError"); + EXPECT_FALSE(e.Message.empty()); + EXPECT_FALSE(std::string(e.what()).empty()); + } + + bool progressCallbackCalled = false; + queryOptions.ProgressHandler + = [&malformedData, &progressCallbackCalled](int64_t offset, int64_t totalBytes) { + EXPECT_EQ(totalBytes, static_cast(malformedData.size())); + EXPECT_TRUE(offset >= 0 && offset <= totalBytes); + progressCallbackCalled = true; + }; + int numNonFatalErrors = 0; + int numFatalErrors = 0; + queryOptions.ErrorHandler = [&numNonFatalErrors, &numFatalErrors](Blobs::BlobQueryError e) { + if (e.IsFatal) + { + ++numFatalErrors; + } + else + { + ++numNonFatalErrors; + } + }; + queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + queryResponse.Value.BodyStream->ReadToEnd(); + + EXPECT_EQ(numNonFatalErrors, 2); + EXPECT_EQ(numFatalErrors, 1); + EXPECT_TRUE(progressCallbackCalled); + } + + TEST_F(BlockBlobClientTest, QueryDefaultInputOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + const std::string csvData = "100,oranges,100"; + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + auto queryResponse = client.Query("SELECT * from BlobStorage;"); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + } + + TEST_F(BlockBlobClientTest, QueryLargeBlob_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + + constexpr size_t DataSize = static_cast(32_MB); + + int recordCounter = 0; + std::string csvData; + std::string jsonData; + while (csvData.size() < DataSize) + { + std::string counter = std::to_string(recordCounter++); + std::string record = RandomString(static_cast(RandomInt(1, 3000))); + csvData += counter + "," + record + "\n"; + jsonData += "{\"_1\":\"" + counter + "\",\"_2\":\"" + record + "\"}\n"; + } + + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions(); + queryOptions.OutputTextConfiguration + = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage;", queryOptions); + + size_t comparePos = 0; + std::vector readBuffer(4096); + while (true) + { + auto s = queryResponse.Value.BodyStream->Read(readBuffer.data(), readBuffer.size()); + if (s == 0) + { + break; + } + ASSERT_TRUE(comparePos + s <= jsonData.size()); + ASSERT_EQ( + std::string(readBuffer.begin(), readBuffer.begin() + s), jsonData.substr(comparePos, s)); + comparePos += s; + } + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLeaseId_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + Blobs::BlobLeaseClient leaseClient(client, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + leaseClient.Acquire(Blobs::BlobLeaseClient::InfiniteLeaseDuration); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionTags_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + std::map tags = {{"k1", "value1"}}; + client.SetTags(tags); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.TagConditions = "k1 = 'value1'"; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.TagConditions = "k1 = 'dummy'"; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLastModifiedTime_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + auto lastModifiedTime = client.GetProperties().Value.LastModified; + auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); + auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.IfModifiedSince = timeBeforeStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfModifiedSince = timeAfterStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Blobs::QueryBlobOptions(); + queryOptions.AccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + queryOptions.AccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionETag_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = GetBlockBlobClient(testName); + client.UploadFrom(nullptr, 0); + + auto etag = client.GetProperties().Value.ETag; + + Blobs::QueryBlobOptions queryOptions; + queryOptions.AccessConditions.IfMatch = etag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfMatch = DummyETag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Blobs::QueryBlobOptions(); + queryOptions.AccessConditions.IfNoneMatch = DummyETag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfNoneMatch = etag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + } +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp index 8be9c7d257..b2d28d8075 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp @@ -193,6 +193,28 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(BlobServiceClientTest, ListSystemContainers) + { + const std::string testName = GetTestNameLowerCase(); + auto client = GetClientForTest(testName); + Azure::Storage::Blobs::ListBlobContainersOptions options; + options.Include = Blobs::Models::ListBlobContainersIncludeFlags::System; + std::vector containers; + for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + for (const auto& c : pageResult.BlobContainers) + { + if (c.Name[0] == '$') + { + containers.push_back(c.Name); + } + } + } + + EXPECT_FALSE(containers.empty()); + } + TEST_F(BlobServiceClientTest, GetProperties) { const std::string testName = GetTestName(); diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 0b49b12633..b1bae299eb 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -138,6 +138,33 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(static_cast(clearRanges[0].Length.Value()), 1_KB); } + TEST_F(PageBlobClientTest, GetPageRangesContinuation) + { + auto const testName(GetTestName()); + auto pageBlobClient = GetPageBlobClient(testName); + + std::vector blobContent = std::vector(static_cast(512), 'x'); + + pageBlobClient.Create(8_KB, m_blobUploadOptions); + auto pageContent = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); + pageBlobClient.UploadPages(0, pageContent); + pageContent.Rewind(); + pageBlobClient.UploadPages(1024, pageContent); + pageContent.Rewind(); + pageBlobClient.UploadPages(4096, pageContent); + + Blobs::GetPageRangesOptions options; + options.PageSizeHint = 1; + size_t numRanges = 0; + for (auto pageResult = pageBlobClient.GetPageRanges(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + EXPECT_EQ(pageResult.PageRanges.size(), static_cast(1)); + numRanges += pageResult.PageRanges.size(); + } + EXPECT_EQ(numRanges, static_cast(3)); + } + TEST_F(PageBlobClientTest, UploadFromUri) { auto const testName(GetTestName()); diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json new file mode 100644 index 0000000000..84bac2a971 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobContainerClientTest.ListBlobsDeletedWithActiveVersions.json @@ -0,0 +1,135 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "34530d0a-1ebb-4c9b-5ba3-8fa5df36b8ad", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "etag": "\"0x8DA36E07CE8971C\"", + "last-modified": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "34530d0a-1ebb-4c9b-5ba3-8fa5df36b8ad", + "x-ms-request-id": "d7f9d950-b01e-001e-74c9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "5e3c0bdc-9018-4c50-6e26-eec118c34b35", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "etag": "\"0x8DA36E07D1B2448\"", + "last-modified": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "5e3c0bdc-9018-4c50-6e26-eec118c34b35", + "x-ms-request-id": "d7f9da01-b01e-001e-18c9-68361a000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-05-16T02:05:02.5861463Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions/blobblobcontainerclienttestlistblobsdeletedwithactiveversions" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ff54697b-8d1a-4358-7067-e137517b491a", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "blobblobcontainerclienttestlistblobsdeletedwithactiveversionsblobblobcontainerclienttestlistblobsdeletedwithactiveversions2022-05-16T02:05:02.5861463ZtrueMon, 16 May 2022 02:05:02 GMTMon, 16 May 2022 02:05:02 GMT0x8DA36E07D1B24480application/octet-streamAppendBlobunlockedavailabletrue", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "ff54697b-8d1a-4358-7067-e137517b491a", + "x-ms-request-id": "d7f9dae8-b01e-001e-6fc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?comp=list&include=deletedwithversions&prefix=blobblobcontainerclienttestlistblobsdeletedwithactiveversions&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "57212c10-680b-408d-772d-a4209c2d9aaf", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "57212c10-680b-408d-772d-a4209c2d9aaf", + "x-ms-delete-type-permanent": "false", + "x-ms-request-id": "d7f9dbb7-b01e-001e-2dc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions/blobblobcontainerclienttestlistblobsdeletedwithactiveversions" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "072dc394-f953-4f03-45b9-562e379ac7f2", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "blobblobcontainerclienttestlistblobsdeletedwithactiveversionsblobblobcontainerclienttestlistblobsdeletedwithactiveversionstrueMon, 16 May 2022 02:05:02 GMTMon, 16 May 2022 02:05:02 GMT0x8DA36E07D1B24480application/octet-streamAppendBlobunlockedavailabletrue", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "072dc394-f953-4f03-45b9-562e379ac7f2", + "x-ms-request-id": "d7f9dcbd-b01e-001e-1fc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?comp=list&include=deletedwithversions&prefix=blobblobcontainerclienttestlistblobsdeletedwithactiveversions&restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "f342e63c-21d5-4a16-5017-616ea9f7798c", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:03 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f342e63c-21d5-4a16-5017-616ea9f7798c", + "x-ms-request-id": "d7f9dd35-b01e-001e-0bc9-68361a000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/blobcontainerclienttestlistblobsdeletedwithactiveversions?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json index 666b2b7bfb..429767bcfd 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.SetProperties.json @@ -2,32 +2,32 @@ "networkCallRecords": [ { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "dcd6fe54-0c7e-4e74-5b20-f839a7d7d241", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "35419e25-3165-4f93-51f9-5bab0eaec1ae", + "x-ms-version": "2020-10-02" }, "Method": "GET", "Response": { - "BODY": "1.0truetruetruetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true1falsetrueindex.html404.html2019-12-12", + "BODY": "1.0truetruetruetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-02-10", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Sun, 20 Feb 2022 05:27:42 GMT", + "date": "Mon, 16 May 2022 12:49:31 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", "vary": "Origin", - "x-ms-client-request-id": "dcd6fe54-0c7e-4e74-5b20-f839a7d7d241", - "x-ms-request-id": "9fced8f8-d01e-0015-121a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "35419e25-3165-4f93-51f9-5bab0eaec1ae", + "x-ms-request-id": "da10e09f-c01e-0082-5723-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { "content-type": "application/xml; charset=UTF-8", - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "2f4337b3-0234-44ab-7609-8d9a59404ede", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6944decb-5ada-41b4-6365-2f9b90220ebe", + "x-ms-version": "2020-10-02" }, "Method": "PUT", "Response": { @@ -35,42 +35,42 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Sun, 20 Feb 2022 05:27:43 GMT", + "date": "Mon, 16 May 2022 12:49:33 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "2f4337b3-0234-44ab-7609-8d9a59404ede", - "x-ms-request-id": "9fced97e-d01e-0015-091a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "6944decb-5ada-41b4-6365-2f9b90220ebe", + "x-ms-request-id": "da10e162-c01e-0082-1423-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "8103e424-9d4d-46eb-6c22-2adc1c5a81c4", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "a72f237f-cb76-4e90-52a6-eea437169f93", + "x-ms-version": "2020-10-02" }, "Method": "GET", "Response": { - "BODY": "1.0falsefalsefalsetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-08-04", + "BODY": "1.0falsefalsefalsetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2020-10-02", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Sun, 20 Feb 2022 05:27:53 GMT", + "date": "Mon, 16 May 2022 12:49:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", "vary": "Origin", - "x-ms-client-request-id": "8103e424-9d4d-46eb-6c22-2adc1c5a81c4", - "x-ms-request-id": "9fceee3a-d01e-0015-751a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "a72f237f-cb76-4e90-52a6-eea437169f93", + "x-ms-request-id": "da10fdaf-c01e-0082-1623-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" }, { "Headers": { "content-type": "application/xml; charset=UTF-8", - "user-agent": "azsdk-cpp-storage-blobs/12.4.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "251ae95b-e55f-40ce-7935-edc2ecf28444", - "x-ms-version": "2020-08-04" + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ff874718-2fc2-479f-6308-fe94f0c824c9", + "x-ms-version": "2020-10-02" }, "Method": "PUT", "Response": { @@ -78,11 +78,11 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Sun, 20 Feb 2022 05:27:53 GMT", + "date": "Mon, 16 May 2022 12:49:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "251ae95b-e55f-40ce-7935-edc2ecf28444", - "x-ms-request-id": "9fceeec3-d01e-0015-771a-26dbe6000000", - "x-ms-version": "2020-08-04" + "x-ms-client-request-id": "ff874718-2fc2-479f-6308-fe94f0c824c9", + "x-ms-request-id": "da10fe3e-c01e-0082-1b23-699b7c000000", + "x-ms-version": "2020-10-02" }, "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" } diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json new file mode 100644 index 0000000000..55c2b9d6b0 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/BlobServiceClientTest.listsystemcontainers.json @@ -0,0 +1,26 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "966941db-bfd0-44ac-6a24-e4c192e1c693", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "$blobchangefeedTue, 07 Jul 2020 10:10:36 GMT\"0x8D8225DFE721DFD\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse$logsTue, 07 Jul 2020 10:10:36 GMT\"0x8D8225DFE6A2D4B\"unlockedavailable$account-encryption-keyfalsefalsefalsefalseblobcontainerclienttestblobsastestSat, 19 Feb 2022 15:08:02 GMT\"0x8D9F3B99F86B533\"unlockedavailableblob$account-encryption-keyfalsefalsefalsetrueblockblobclienttestimmutabilitySat, 19 Feb 2022 04:12:29 GMT\"0x8D9F35E0B94285B\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestimmutabilityaccessconditionSat, 19 Feb 2022 06:24:17 GMT\"0x8D9F37074DCCA0C\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestlegalholdSat, 19 Feb 2022 05:37:18 GMT\"0x8D9F369E4FEB44E\"unlockedavailable$account-encryption-keyfalsefalsefalsetrueblockblobclienttestqueryparquetinputparquetoutputSat, 14 May 2022 14:28:57 GMT\"0x8DA35B6149118ED\"unlockedavailable$account-encryption-keyfalsefalsefalsefalseblockblobclienttestuploaddownloadTue, 10 May 2022 15:45:35 GMT\"0x8DA329C1FD1864E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsesample-containerTue, 10 May 2022 10:52:48 GMT\"0x8DA327338F99283\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:05:42 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-client-request-id": "966941db-bfd0-44ac-6a24-e4c192e1c693", + "x-ms-request-id": "b3fadb96-b01e-0053-2ec9-68f9f6000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=list&include=system" + } + ] +} diff --git a/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json b/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json new file mode 100644 index 0000000000..94ea774849 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/recordings/PageBlobClientTest.GetPageRangesContinuation.json @@ -0,0 +1,220 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "3e5a06de-602d-414d-6b45-d25f91578ffa", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:59 GMT", + "etag": "\"0x8DA36E09F89EDFA\"", + "last-modified": "Mon, 16 May 2022 02:06:00 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3e5a06de-602d-414d-6b45-d25f91578ffa", + "x-ms-request-id": "cda9954f-401e-0047-71c9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "b211fc89-cee6-49d2-529d-ee204b60eb53", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:05:59 GMT", + "etag": "\"0x8DA36E09FBD275F\"", + "last-modified": "Mon, 16 May 2022 02:06:00 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "b211fc89-cee6-49d2-529d-ee204b60eb53", + "x-ms-request-id": "cda9961a-401e-0047-20c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02", + "x-ms-version-id": "2022-05-16T02:06:00.6894431Z" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "48c04419-42e4-4417-7e3e-0cc102b855a3", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:00 GMT", + "etag": "\"0x8DA36E0A00374C8\"", + "last-modified": "Mon, 16 May 2022 02:06:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "48c04419-42e4-4417-7e3e-0cc102b855a3", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda99731-401e-0047-22c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ec267c55-a846-4933-60fb-37430b21e4cf", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:00 GMT", + "etag": "\"0x8DA36E0A049E933\"", + "last-modified": "Mon, 16 May 2022 02:06:01 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "ec267c55-a846-4933-60fb-37430b21e4cf", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda998a4-401e-0047-7ac9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "55fb6437-b537-4d3e-6b91-a9f357d54a10", + "x-ms-version": "2020-10-02" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:01 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-blob-sequence-number": "0", + "x-ms-client-request-id": "55fb6437-b537-4d3e-6b91-a9f357d54a10", + "x-ms-content-crc64": "kxclNeFlVMY=", + "x-ms-request-id": "cda999d1-401e-0047-15c9-68b199000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=page" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "ae8b224a-7150-45bf-7e9a-93d441962963", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n05112!16!MDAwMDA0ITEwMjQh", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:02 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "ae8b224a-7150-45bf-7e9a-93d441962963", + "x-ms-request-id": "cda99b34-401e-0047-55c9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "d673efcc-de67-47f7-72fa-94634566af41", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n102415352!16!MDAwMDA0ITQwOTYh", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:02 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "d673efcc-de67-47f7-72fa-94634566af41", + "x-ms-request-id": "cda99e3d-401e-0047-1bc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&marker=2!16!MDAwMDA0ITEwMjQh&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "3cee12a6-f7ed-4e2d-5ada-da0207493646", + "x-ms-version": "2020-10-02" + }, + "Method": "GET", + "Response": { + "BODY": "\n40964607", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 16 May 2022 02:06:03 GMT", + "etag": "\"0x8DA36E0A092A73B\"", + "last-modified": "Mon, 16 May 2022 02:06:02 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "vary": "Origin", + "x-ms-blob-content-length": "8192", + "x-ms-client-request-id": "3cee12a6-f7ed-4e2d-5ada-da0207493646", + "x-ms-request-id": "cda99eee-401e-0047-3dc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation/GetPageRangesContinuation?comp=pagelist&marker=2!16!MDAwMDA0ITQwOTYh&maxresults=1" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.5.0-beta.1 (Windows 10 Pro 6.3 19044 19041.1.amd64fre.vb_release.191206-1406)", + "x-ms-client-request-id": "6340d172-a8eb-46bc-642b-a90af0c9b640", + "x-ms-version": "2020-10-02" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Mon, 16 May 2022 02:06:03 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "6340d172-a8eb-46bc-642b-a90af0c9b640", + "x-ms-request-id": "cda99fa5-401e-0047-5fc9-68b199000000", + "x-ms-version": "2020-10-02" + }, + "Url": "https://REDACTED.blob.core.windows.net/pageblobclienttestgetpagerangescontinuation?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-common/CHANGELOG.md b/sdk/storage/azure-storage-common/CHANGELOG.md index 85fac2ba62..637f3f7eda 100644 --- a/sdk/storage/azure-storage-common/CHANGELOG.md +++ b/sdk/storage/azure-storage-common/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +- Fixed a bug where text of XML element cannot be empty. + ### Other Changes ## 12.2.3 (2022-04-06) diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index 840b85af99..ad8759b9f5 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -55,14 +55,15 @@ set( inc/azure/storage/common/internal/storage_service_version_policy.hpp inc/azure/storage/common/internal/storage_switch_to_secondary_policy.hpp inc/azure/storage/common/internal/xml_wrapper.hpp + inc/azure/storage/common/rtti.hpp inc/azure/storage/common/storage_common.hpp inc/azure/storage/common/storage_credential.hpp inc/azure/storage/common/storage_exception.hpp - src/private/package_version.hpp ) set( AZURE_STORAGE_COMMON_SOURCE + src/private/package_version.hpp src/account_sas_builder.cpp src/crypt.cpp src/file_io.cpp diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/xml_wrapper.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/xml_wrapper.hpp index 6aed398d9d..31c4fb14cb 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/xml_wrapper.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/xml_wrapper.hpp @@ -19,17 +19,20 @@ namespace Azure { namespace Storage { namespace _internal { struct XmlNode final { - explicit XmlNode( - XmlNodeType type, - std::string name = std::string(), - std::string value = std::string()) - : Type(type), Name(std::move(name)), Value(std::move(value)) + explicit XmlNode(XmlNodeType type, std::string name = std::string()) + : Type(type), Name(std::move(name)) + { + } + + explicit XmlNode(XmlNodeType type, std::string name, std::string value) + : Type(type), Name(std::move(name)), Value(std::move(value)), HasValue(true) { } XmlNodeType Type; std::string Name; std::string Value; + bool HasValue = false; }; class XmlReader final { diff --git a/sdk/storage/azure-storage-common/src/xml_wrapper.cpp b/sdk/storage/azure-storage-common/src/xml_wrapper.cpp index f914421b68..4190240c45 100644 --- a/sdk/storage/azure-storage-common/src/xml_wrapper.cpp +++ b/sdk/storage/azure-storage-common/src/xml_wrapper.cpp @@ -203,7 +203,7 @@ namespace Azure { namespace Storage { namespace _internal { } case WS_XML_NODE_TYPE_END_ELEMENT: moveToNext(); - return XmlNode{XmlNodeType::EndTag, std::string()}; + return XmlNode{XmlNodeType::EndTag}; case WS_XML_NODE_TYPE_EOF: return XmlNode{XmlNodeType::End}; case WS_XML_NODE_TYPE_CDATA: @@ -288,7 +288,7 @@ namespace Azure { namespace Storage { namespace _internal { auto context = static_cast(m_context); if (node.Type == XmlNodeType::StartTag) { - if (!node.Value.empty()) + if (node.HasValue) { Write(XmlNode{XmlNodeType::StartTag, std::move(node.Name)}); Write(XmlNode{XmlNodeType::Text, std::string(), std::move(node.Value)}); @@ -576,7 +576,7 @@ namespace Azure { namespace Storage { namespace _internal { xmlTextWriterPtr writer = context->writer; if (node.Type == XmlNodeType::StartTag) { - if (node.Value.empty()) + if (!node.HasValue) { xmlTextWriterStartElement(writer, BadCast(node.Name.data())); } diff --git a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt index 0152ddef4c..f8ed9bc9d3 100644 --- a/sdk/storage/azure-storage-files-datalake/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/CMakeLists.txt @@ -52,13 +52,14 @@ set( inc/azure/storage/files/datalake/datalake_service_client.hpp inc/azure/storage/files/datalake/dll_import_export.hpp inc/azure/storage/files/datalake/rest_client.hpp - src/private/package_version.hpp + inc/azure/storage/files/datalake/rtti.hpp ) set( AZURE_STORAGE_FILES_DATALAKE_SOURCE src/private/datalake_constants.hpp src/private/datalake_utilities.hpp + src/private/package_version.hpp src/datalake_directory_client.cpp src/datalake_file_client.cpp src/datalake_file_system_client.cpp diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake.hpp index a8b861450a..aaa0f7ae82 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake.hpp @@ -8,12 +8,17 @@ #pragma once -#include "azure/storage/common/storage_exception.hpp" +#include + #include "azure/storage/files/datalake/datalake_directory_client.hpp" #include "azure/storage/files/datalake/datalake_file_client.hpp" #include "azure/storage/files/datalake/datalake_file_system_client.hpp" #include "azure/storage/files/datalake/datalake_lease_client.hpp" +#include "azure/storage/files/datalake/datalake_options.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" +#include "azure/storage/files/datalake/datalake_responses.hpp" #include "azure/storage/files/datalake/datalake_sas_builder.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "azure/storage/files/datalake/dll_import_export.hpp" +#include "azure/storage/files/datalake/rest_client.hpp" +#include "azure/storage/files/datalake/rtti.hpp" diff --git a/sdk/storage/azure-storage-files-shares/CMakeLists.txt b/sdk/storage/azure-storage-files-shares/CMakeLists.txt index ceb7649e26..dd8e913086 100644 --- a/sdk/storage/azure-storage-files-shares/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-shares/CMakeLists.txt @@ -42,6 +42,7 @@ set( AZURE_STORAGE_FILES_SHARES_HEADER inc/azure/storage/files/shares/dll_import_export.hpp inc/azure/storage/files/shares/rest_client.hpp + inc/azure/storage/files/shares/rtti.hpp inc/azure/storage/files/shares/share_client.hpp inc/azure/storage/files/shares/share_constants.hpp inc/azure/storage/files/shares/share_directory_client.hpp @@ -52,11 +53,11 @@ set( inc/azure/storage/files/shares/share_sas_builder.hpp inc/azure/storage/files/shares/share_service_client.hpp inc/azure/storage/files/shares.hpp - src/private/package_version.hpp ) set( AZURE_STORAGE_FILES_SHARES_SOURCE + src/private/package_version.hpp src/rest_client.cpp src/share_client.cpp src/share_directory_client.cpp diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares.hpp index 4978566505..fe99ed8c9a 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares.hpp @@ -8,11 +8,17 @@ #pragma once -#include "azure/storage/common/storage_exception.hpp" +#include + #include "azure/storage/files/shares/dll_import_export.hpp" +#include "azure/storage/files/shares/rest_client.hpp" +#include "azure/storage/files/shares/rtti.hpp" #include "azure/storage/files/shares/share_client.hpp" +#include "azure/storage/files/shares/share_constants.hpp" #include "azure/storage/files/shares/share_directory_client.hpp" #include "azure/storage/files/shares/share_file_client.hpp" #include "azure/storage/files/shares/share_lease_client.hpp" +#include "azure/storage/files/shares/share_options.hpp" +#include "azure/storage/files/shares/share_responses.hpp" #include "azure/storage/files/shares/share_sas_builder.hpp" #include "azure/storage/files/shares/share_service_client.hpp" diff --git a/sdk/storage/azure-storage-queues/CMakeLists.txt b/sdk/storage/azure-storage-queues/CMakeLists.txt index ab6b2f4034..ee7d9e7204 100644 --- a/sdk/storage/azure-storage-queues/CMakeLists.txt +++ b/sdk/storage/azure-storage-queues/CMakeLists.txt @@ -48,11 +48,12 @@ set( inc/azure/storage/queues/queue_sas_builder.hpp inc/azure/storage/queues/queue_service_client.hpp inc/azure/storage/queues/rest_client.hpp - src/private/package_version.hpp + inc/azure/storage/queues/rtti.hpp ) set( AZURE_STORAGE_QUEUES_SOURCE + src/private/package_version.hpp src/queue_client.cpp src/queue_options.cpp src/queue_responses.cpp diff --git a/sdk/storage/azure-storage-queues/inc/azure/storage/queues.hpp b/sdk/storage/azure-storage-queues/inc/azure/storage/queues.hpp index 6feb9eb0b4..294cd425c6 100644 --- a/sdk/storage/azure-storage-queues/inc/azure/storage/queues.hpp +++ b/sdk/storage/azure-storage-queues/inc/azure/storage/queues.hpp @@ -8,8 +8,13 @@ #pragma once -#include "azure/storage/common/storage_exception.hpp" +#include + #include "azure/storage/queues/dll_import_export.hpp" #include "azure/storage/queues/queue_client.hpp" +#include "azure/storage/queues/queue_options.hpp" +#include "azure/storage/queues/queue_responses.hpp" #include "azure/storage/queues/queue_sas_builder.hpp" #include "azure/storage/queues/queue_service_client.hpp" +#include "azure/storage/queues/rest_client.hpp" +#include "azure/storage/queues/rtti.hpp" diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 017ab238a9..cebc7b29cf 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -27,8 +27,8 @@ stages: ServiceDirectory: storage # Not running tests on CI but only on LiveTest mode CtestRegex: azure-storage - LineCoverageTarget: 76 - BranchCoverageTarget: 41 + LineCoverageTarget: 73.5305 + BranchCoverageTarget: 40.7738 LiveTestCtestRegex: azure-storage Clouds: Preview SupportedClouds: Preview diff --git a/sdk/template/azure-template/CMakeLists.txt b/sdk/template/azure-template/CMakeLists.txt index c78558f5e9..7d4d3126a4 100644 --- a/sdk/template/azure-template/CMakeLists.txt +++ b/sdk/template/azure-template/CMakeLists.txt @@ -60,12 +60,9 @@ target_include_directories( PUBLIC $ $ - # Disable dependency on azure-core until azure-core has released to vcpkg - # ${azure-core-cpp_INCLUDE_DIRS} ) -# Disable dependency on azure-core until azure-core has released to vcpkg -# target_link_libraries(azure-template PUBLIC Azure::azure-core) +target_link_libraries(azure-template PUBLIC Azure::azure-core) # make sure that users can consume the project as a library. add_library(Azure::azure-template ALIAS azure-template) diff --git a/sdk/template/azure-template/inc/azure/template.hpp b/sdk/template/azure-template/inc/azure/template.hpp index fb47d2120d..05315c0be7 100644 --- a/sdk/template/azure-template/inc/azure/template.hpp +++ b/sdk/template/azure-template/inc/azure/template.hpp @@ -9,4 +9,5 @@ #pragma once #include "azure/template/dll_import_export.hpp" +#include "azure/template/rtti.hpp" #include "azure/template/template_client.hpp"