From 889e0255a2a928a5eaf0fdc40ec53284ab7ee091 Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Tue, 21 Feb 2023 10:52:20 -0800 Subject: [PATCH 01/23] update comment (#4364) * update comment * jghjg * update broken link * Update sdk/keyvault/tools/cleanup/src/cleanup.cpp Co-authored-by: Anton Kolesnyk <41349689+antkmsft@users.noreply.github.com> --------- Co-authored-by: Anton Kolesnyk <41349689+antkmsft@users.noreply.github.com> --- sdk/keyvault/tools/cleanup/src/cleanup.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/keyvault/tools/cleanup/src/cleanup.cpp b/sdk/keyvault/tools/cleanup/src/cleanup.cpp index d9c5ba96a9..c49f9afb84 100644 --- a/sdk/keyvault/tools/cleanup/src/cleanup.cpp +++ b/sdk/keyvault/tools/cleanup/src/cleanup.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: MIT /** - * @brief This sample provides the code implementation to use the Key Vault Secrets SDK client for - * C++ to create, get, update, delete and purge a secret. + * @brief This tool helps cleanup resources (keys, certificates, secrets) existing in a key vault, + * Attempts to restore it as close to original as possible without having to recreate the key vault. * * @remark The following environment variables must be set before running the sample. * - AZURE_KEYVAULT_URL: To the Key Vault account URL. From 9572608a5dd0fba2e5074fd030ed6cc9c31535a0 Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:50:55 -0800 Subject: [PATCH 02/23] Docker comment (#4375) * update comment * add comment about vcpkg * dsfs --- sdk/core/azure-core/test/libcurl-stress-test/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdk/core/azure-core/test/libcurl-stress-test/Dockerfile b/sdk/core/azure-core/test/libcurl-stress-test/Dockerfile index a7a674f38c..8788c90752 100644 --- a/sdk/core/azure-core/test/libcurl-stress-test/Dockerfile +++ b/sdk/core/azure-core/test/libcurl-stress-test/Dockerfile @@ -16,6 +16,13 @@ RUN ln -s /opt/vcpkg/vcpkg /usr/local/bin/vcpkg ADD . /src WORKDIR /build +# During CMake generate step VCPKG runs in manifest mode, as such it will sync the packages to the level +# of the hash specified in src/azure-sdk-for-cpp/cmake-modules/AzureVcpkg.cmake in the VCPKG_COMMIT_STRING +# environment variable thus the packages we run with are not the latest versions but the ones the code +# was developed against. If the builtin-baseline is specified in the vcpkg file then that is the top most +# version of the packages that will be fetched. +# So when building from root we need to match the two values. When not building from root if the vcpkg file +# does not specify a baseline the value set in the cmake file will ensure that we are at the desired level. RUN cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON /src RUN cmake --build . --target azure-core-libcurl-stress-test From 5058f15594058b77ceded8309da7618304a7793a Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 21 Feb 2023 13:32:30 -0800 Subject: [PATCH 03/23] Trigger`keyvault` on proxy changes (#4343) * autotrigger the keyvault and storage CI when a testproxy file is changed --- sdk/keyvault/ci.yml | 1 + sdk/storage/ci.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 86598ada57..730b773766 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -21,6 +21,7 @@ pr: paths: include: - sdk/keyvault + - eng/common/testproxy stages: - template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 9b625242f5..2be69e490e 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -20,6 +20,7 @@ pr: paths: include: - sdk/storage + - eng/common/testproxy stages: - template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml From 7da07d5072b8714974acbf51f4fb738c713d9fe9 Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:35:41 -0500 Subject: [PATCH 04/23] Generate API review for C++ using new parser (#4302) --- .../templates/jobs/archetype-sdk-client.yml | 19 +++++++++++ eng/scripts/Create-APIReview.ps1 | 33 ++++++++++--------- eng/scripts/Language-Settings.ps1 | 2 +- .../azure-template/inc/ApiViewSettings.json | 16 +++++++++ 4 files changed, 53 insertions(+), 17 deletions(-) create mode 100644 sdk/template/azure-template/inc/ApiViewSettings.json diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 5cec0b0f9d..e3cd4dfed7 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -1,3 +1,6 @@ +# cSpell:ignore vsts +# cSpell:ignore parseazuresdkcpp + parameters: - name: Artifacts type: object @@ -84,6 +87,20 @@ jobs: - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + - task: UniversalPackages@0 + displayName: Download ApiView Parser. + inputs: + command: download + vstsFeed: 'internal/AzureSDKForCpp' + vstsFeedPackage: 'parseazuresdkcpp' + downloadDirectory: '$(System.DefaultWorkingDirectory)/parser' + vstsPackageVersion: 0.6.0 + condition: >- + and( + succeeded(), + ne(variables['Skip.CreateApiReview'], 'true'),eq(variables['System.TeamProject'], 'internal') + ) + - task: Powershell@2 inputs: filePath: $(System.DefaultWorkingDirectory)/eng/scripts/Create-APIReview.ps1 @@ -96,6 +113,8 @@ jobs: -SourceBranch $(Build.SourceBranchName) -DefaultBranch $(DefaultBranch) -ConfigFileDir $(Build.ArtifactStagingDirectory)/PackageInfo + -ParserPath $(System.DefaultWorkingDirectory)/parser/RelWithDebInfo/ParseAzureSdkCpp.exe + -SourcePath $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.name }}/inc pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Create API Review for ${{ artifact.name }} diff --git a/eng/scripts/Create-APIReview.ps1 b/eng/scripts/Create-APIReview.ps1 index 0a0d7cd1af..df54fabd50 100644 --- a/eng/scripts/Create-APIReview.ps1 +++ b/eng/scripts/Create-APIReview.ps1 @@ -14,30 +14,31 @@ Param( [Parameter(Mandatory=$True)] [string] $DefaultBranch, [Parameter(Mandatory=$True)] - [string] $ConfigFileDir + [string] $ConfigFileDir, + [Parameter(Mandatory=$True)] + [string] $ParserPath, + [Parameter(Mandatory=$True)] + [string] $SourcePath ) Write-Host "$PSScriptRoot" . (Join-Path $PSScriptRoot .. common scripts common.ps1) $createReviewScript = (Join-Path $PSScriptRoot .. common scripts Create-APIReview.ps1) -Set-Location $PSScriptRoot -Write-Host "Creating API review artifact for $ArtifactName" -New-Item -ItemType Directory -Path $OutPath/$ArtifactName -force +$apiviewSettings = Join-Path $SourcePath "ApiViewSettings.json" +if (!(Test-Path $apiviewSettings)) +{ + Write-Host "ApiViewSettings.json file is not found in $($SourcePath). APIView settings file is required to generate API review file." + exit 1 +} -$gitroot = Join-Path $PSScriptRoot .. .. -Write-Host "Get-ApiViewCommandLine.ps1 $gitroot $ArtifactName" -$cmdLine = & $PSScriptRoot/Get-ApiViewCommandLine.ps1 $gitroot $ArtifactName -Write-Host "Executing clang++ command:" -Write-Host $cmdLine -$cmd, $cmdArgs = $cmdLine -split ' ' -# Get-ApiViewCommandLine.ps1 returns a string representing a clang++ command that needs to be run, e.g. -# clang++ -Xclang -ast-dump -I -# ApiView expects a zip of this ast as the format for a C++ language artifact. -& $cmd $cmdArgs > clangAstOutput +Write-Host "Creating API review artifact for $($ArtifactName)" +New-Item -ItemType Directory -Path $OutPath/$ArtifactName -force +$parentPath = Split-Path $ParserPath -Parent +Write-Host "Contents in $($parentPath)" +Get-ChildItem -Path $parentPath -Recurse -Compress-Archive -Path clangAstOutput -DestinationPath $OutPath/$ArtifactName/$ArtifactName -Rename-Item $OutPath/$ArtifactName/$ArtifactName.zip -NewName "$ArtifactName.cppast" +& $ParserPath -o $OutPath/$ArtifactName/$ArtifactName.json $SourcePath Write-Host "Send request to APIView to create review for $ArtifactName" &($createReviewScript) -ArtifactPath $OutPath -APIViewUri $ApiviewUri -APIKey $ApiKey -APILabel $ApiLabel -PackageName $ArtifactName -SourceBranch $SourceBranch -DefaultBranch $DefaultBranch -ConfigFileDir $ConfigFileDir diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 7dc88bcf89..e6815f1f65 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -101,7 +101,7 @@ function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseD function Find-cpp-Artifacts-For-Apireview($ArtifactPath, $PackageName) { - $artifact = Get-ChildItem -Path (Join-Path $ArtifactPath $PackageName) -Filter "*.cppast" + $artifact = Get-ChildItem -Path (Join-Path $ArtifactPath $PackageName) -Filter "*.json" if ($artifact) { $packages = @{ diff --git a/sdk/template/azure-template/inc/ApiViewSettings.json b/sdk/template/azure-template/inc/ApiViewSettings.json new file mode 100644 index 0000000000..4f51021027 --- /dev/null +++ b/sdk/template/azure-template/inc/ApiViewSettings.json @@ -0,0 +1,16 @@ +{ + "sourceFilesToProcess": null, + "sourceFilesToSkip": [ + ], + "additionalIncludeDirectories": [ + "../../../core/azure-core/inc" + ], + "additionalCompilerSwitches": [], + "allowInternal": false, + "includeDetail": false, + "includePrivate": false, + "filterNamespace": "Azure::", + "reviewName": "Azure Template API Review", + "serviceName": "Azure Template", + "packageName": "azure-template" +} From f45d8f6b2f82734e861455d964d24fc8745f807b Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:13:33 -0500 Subject: [PATCH 05/23] Add vmImage back to common perf.yml - Fixes #5466 - Partially reverts #5456 (#4376) Co-authored-by: Mike Harder --- eng/common/pipelines/templates/jobs/perf.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eng/common/pipelines/templates/jobs/perf.yml b/eng/common/pipelines/templates/jobs/perf.yml index 073e5de641..d1204f284c 100644 --- a/eng/common/pipelines/templates/jobs/perf.yml +++ b/eng/common/pipelines/templates/jobs/perf.yml @@ -8,9 +8,15 @@ parameters: - name: LinuxPool type: string default: 'azsdk-pool-mms-ubuntu-2204-perf' +- name: LinuxVmImage + type: string + default: 'ubuntu-22.04' - name: WindowsPool type: string default: 'azsdk-pool-mms-win-2022-perf' +- name: WindowsVmImage + type: string + default: 'windows-2022' - name: Language type: string default: '' @@ -68,10 +74,12 @@ jobs: ${{ if contains(parameters.OperatingSystems, 'Linux') }}: Linux: Pool: ${{ parameters.LinuxPool }} + OsVmImage: ${{ parameters.LinuxVmImage }} MatrixName: 'Linux' ${{ if contains(parameters.OperatingSystems, 'Windows') }}: Windows: Pool: ${{ parameters.WindowsPool }} + OsVmImage: ${{ parameters.WindowsVmImage }} MatrixName: 'Windows' variables: - ${{ parameters.Variables }} @@ -82,6 +90,7 @@ jobs: value: '' pool: name: $(Pool) + vmImage: $(OSVmImage) steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -100,6 +109,10 @@ jobs: Commitish: ${{ parameters.ToolsRepoCommitish }} WorkingDirectory: $(System.DefaultWorkingDirectory)/azure-sdk-tools + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OSVmImage) + - ${{ parameters.InstallLanguageSteps }} - template: /eng/common/TestResources/deploy-test-resources.yml From 184959758055ff4a4b7486b7cbef752b611716e6 Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:21:17 -0800 Subject: [PATCH 06/23] Fix pipelines path (#4358) * test path * qwq * dsda * asas * dsada * sdsds * Update sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml Co-authored-by: Mike Harder * Update sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml Co-authored-by: Mike Harder * remove warmup --------- Co-authored-by: Mike Harder --- .../azure-security-keyvault-certificates/perf-tests.yml | 2 +- sdk/keyvault/azure-security-keyvault-certificates/perf.yml | 2 +- sdk/keyvault/azure-security-keyvault-keys/perf-tests.yml | 2 +- sdk/keyvault/azure-security-keyvault-keys/perf.yml | 2 +- sdk/keyvault/azure-security-keyvault-secrets/perf-tests.yml | 2 +- sdk/keyvault/azure-security-keyvault-secrets/perf.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml b/sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml index 896c22d51a..0d50896666 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml +++ b/sdk/keyvault/azure-security-keyvault-certificates/perf-tests.yml @@ -14,4 +14,4 @@ Tests: - Test: get-certificate Class: GetCertificate Arguments: - - --parallel 64 + - --parallel 4 diff --git a/sdk/keyvault/azure-security-keyvault-certificates/perf.yml b/sdk/keyvault/azure-security-keyvault-certificates/perf.yml index c20a60b093..7202344f4a 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/perf.yml +++ b/sdk/keyvault/azure-security-keyvault-certificates/perf.yml @@ -27,7 +27,7 @@ extends: - pwsh: | Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" displayName: Set Vcpkg Variables - ServiceDirectory: keyvault + ServiceDirectory: keyvault/azure-security-keyvault-certificates PackageVersions: ${{ parameters.PackageVersions }} Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} diff --git a/sdk/keyvault/azure-security-keyvault-keys/perf-tests.yml b/sdk/keyvault/azure-security-keyvault-keys/perf-tests.yml index cf4d5427e0..6490751eda 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/perf-tests.yml +++ b/sdk/keyvault/azure-security-keyvault-keys/perf-tests.yml @@ -14,4 +14,4 @@ Tests: - Test: get-key Class: GetKey Arguments: - - --parallel 64 + - --parallel 4 diff --git a/sdk/keyvault/azure-security-keyvault-keys/perf.yml b/sdk/keyvault/azure-security-keyvault-keys/perf.yml index 995e746db7..e070b80d5d 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/perf.yml +++ b/sdk/keyvault/azure-security-keyvault-keys/perf.yml @@ -27,7 +27,7 @@ extends: - pwsh: | Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" displayName: Set Vcpkg Variables - ServiceDirectory: keyvault + ServiceDirectory: keyvault/azure-security-keyvault-keys PackageVersions: ${{ parameters.PackageVersions }} Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} diff --git a/sdk/keyvault/azure-security-keyvault-secrets/perf-tests.yml b/sdk/keyvault/azure-security-keyvault-secrets/perf-tests.yml index 683b1e859f..22c2e14bf7 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/perf-tests.yml +++ b/sdk/keyvault/azure-security-keyvault-secrets/perf-tests.yml @@ -14,4 +14,4 @@ Tests: - Test: get-secret Class: GetSecret Arguments: - - --parallel 64 + - --parallel 4 diff --git a/sdk/keyvault/azure-security-keyvault-secrets/perf.yml b/sdk/keyvault/azure-security-keyvault-secrets/perf.yml index 203d9dc09b..2e1b1dda15 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/perf.yml +++ b/sdk/keyvault/azure-security-keyvault-secrets/perf.yml @@ -27,7 +27,7 @@ extends: - pwsh: | Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read" displayName: Set Vcpkg Variables - ServiceDirectory: keyvault + ServiceDirectory: keyvault/azure-security-keyvault-secrets PackageVersions: ${{ parameters.PackageVersions }} Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} From 395e9a0640eecf69ba141e8fc69e591a6c62e0e3 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 22 Feb 2023 09:37:49 +0800 Subject: [PATCH 07/23] Test proxy & storage tests improvements (#4241) --- sdk/attestation/assets.json | 2 +- .../test/ut/policycertmgmt_test.cpp | 4 - .../inc/azure/core/test/network_models.hpp | 113 - .../inc/azure/core/test/test_base.hpp | 39 +- .../azure/core/test/test_proxy_manager.hpp | 12 - sdk/core/azure-core-test/src/test_base.cpp | 1 - .../src/test_proxy_manager.cpp | 52 +- .../azure-core-test/src/test_proxy_policy.cpp | 10 +- sdk/core/azure-core/CHANGELOG.md | 5 +- sdk/core/azure-core/src/http/curl/curl.cpp | 14 +- sdk/storage/assets.json | 2 +- .../inc/azure/storage/blobs/blob_options.hpp | 2 + .../test/ut/CMakeLists.txt | 1 + .../test/ut/append_blob_client_test.cpp | 184 +- .../test/ut/append_blob_client_test.hpp | 43 +- .../test/ut/blob_batch_client_test.cpp | 99 +- .../test/ut/blob_container_client_test.cpp | 430 ++- .../test/ut/blob_container_client_test.hpp | 48 +- .../test/ut/blob_query_test.cpp | 140 +- .../test/ut/blob_sas_test.cpp | 4 +- .../test/ut/blob_service_client_test.cpp | 135 +- .../test/ut/blob_service_client_test.hpp | 25 + .../test/ut/block_blob_client_test.cpp | 2350 ++++++++--------- .../test/ut/block_blob_client_test.hpp | 63 +- .../test/ut/page_blob_client_test.cpp | 183 +- .../test/ut/page_blob_client_test.hpp | 53 +- .../test/ut/bearer_token_test.cpp | 13 +- .../test/ut/crypt_functions_test.cpp | 31 +- .../test/ut/test_base.cpp | 135 +- .../test/ut/test_base.hpp | 155 +- .../ut/datalake_directory_client_test.cpp | 214 +- .../ut/datalake_directory_client_test.hpp | 5 +- .../test/ut/datalake_file_client_test.cpp | 515 ++-- .../test/ut/datalake_file_client_test.hpp | 5 +- .../test/ut/datalake_file_query_test.cpp | 54 +- .../ut/datalake_file_system_client_test.cpp | 864 +++--- .../ut/datalake_file_system_client_test.hpp | 22 +- .../test/ut/datalake_path_client_test.cpp | 236 +- .../test/ut/datalake_path_client_test.hpp | 7 +- .../test/ut/datalake_sas_test.cpp | 2 - .../test/ut/datalake_service_client_test.cpp | 143 +- .../test/ut/datalake_service_client_test.hpp | 31 +- .../test/ut/share_client_test.cpp | 246 +- .../test/ut/share_client_test.hpp | 20 +- .../test/ut/share_directory_client_test.cpp | 352 ++- .../test/ut/share_directory_client_test.hpp | 10 +- .../test/ut/share_file_client_test.cpp | 636 ++--- .../test/ut/share_file_client_test.hpp | 2 - .../test/ut/share_service_client_test.cpp | 148 +- .../test/ut/share_service_client_test.hpp | 16 +- .../test/ut/queue_client_messages_test.cpp | 62 +- .../test/ut/queue_client_test.cpp | 76 +- .../test/ut/queue_client_test.hpp | 11 +- .../test/ut/queue_service_client_test.cpp | 1 - 54 files changed, 3347 insertions(+), 4679 deletions(-) create mode 100644 sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp diff --git a/sdk/attestation/assets.json b/sdk/attestation/assets.json index fc2dbbc442..aabd9cf1b7 100644 --- a/sdk/attestation/assets.json +++ b/sdk/attestation/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/attestation", - "Tag": "cpp/attestation_10abfdc3e0" + "Tag": "cpp/attestation_6398169251" } diff --git a/sdk/attestation/azure-security-attestation/test/ut/policycertmgmt_test.cpp b/sdk/attestation/azure-security-attestation/test/ut/policycertmgmt_test.cpp index 65de53b445..27c3ce57b4 100644 --- a/sdk/attestation/azure-security-attestation/test/ut/policycertmgmt_test.cpp +++ b/sdk/attestation/azure-security-attestation/test/ut/policycertmgmt_test.cpp @@ -171,8 +171,6 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test { TEST_F(CertificateTests, AddPolicyManagementCertificate_LIVEONLY_) { - CHECK_SKIP_TEST() - auto adminClient(CreateClient(ServiceInstanceType::Isolated)); auto isolatedCertificateBase64(GetEnv("ISOLATED_SIGNING_CERTIFICATE")); @@ -233,8 +231,6 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test { TEST_F(CertificateTests, RemovePolicyManagementCertificate_LIVEONLY_) { - CHECK_SKIP_TEST() - auto adminClient(CreateClient(ServiceInstanceType::Isolated)); auto isolatedCertificateBase64(GetEnv("ISOLATED_SIGNING_CERTIFICATE")); diff --git a/sdk/core/azure-core-test/inc/azure/core/test/network_models.hpp b/sdk/core/azure-core-test/inc/azure/core/test/network_models.hpp index 3c292f1a39..246adee5af 100644 --- a/sdk/core/azure-core-test/inc/azure/core/test/network_models.hpp +++ b/sdk/core/azure-core-test/inc/azure/core/test/network_models.hpp @@ -8,14 +8,6 @@ #pragma once -#include -#include - -#include -#include -#include -#include - namespace Azure { namespace Core { namespace Test { /** @@ -29,109 +21,4 @@ namespace Azure { namespace Core { namespace Test { LIVE, }; - /** - * @brief Keeps track of network call records from each unit test session. - * - */ - struct NetworkCallRecord - { - std::string Method; - std::string Url; - std::map Headers; - std::map Response; - }; - - /** - * @brief Keeps track of the network calls and variable names that were made in a test - * session. - * - */ - class RecordedData { - public: - std::list NetworkCallRecords; - std::list Variables; - }; - - /** - * @brief A body stream which holds the memory inside. - * - * @remark The playback http uses this body stream to be returned as part of the raw response so - * the transport policy can read from it. - * - */ - class WithMemoryBodyStream : public Azure::Core::IO::BodyStream { - private: - std::vector m_memory; - Azure::Core::IO::MemoryBodyStream m_streamer; - - size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override - { - return m_streamer.Read(buffer, count, context); - } - - public: - // Forbid constructor for rval so we don't end up storing dangling ptr - WithMemoryBodyStream(std::vector const&&) = delete; - - /** - * @brief Construct using vector of bytes. - * - * @param buffer Vector of bytes with the contents to provide the data from to the readers. - */ - WithMemoryBodyStream(std::vector const& buffer) - : m_memory(buffer), m_streamer(m_memory) - { - } - - int64_t Length() const override { return m_streamer.Length(); } - - void Rewind() override { m_streamer.Rewind(); } - }; - - /** - * @brief Wraps a stream and keep reading bytes from it by rewinding it until some length. - * - * @note Enables to create a stream with huge size by re-using a small buffer (1Mb). - * - */ - class CircularBodyStream : public Azure::Core::IO::BodyStream { - private: - std::unique_ptr> m_buffer; - size_t m_length; - size_t m_totalRead = 0; - Azure::Core::IO::MemoryBodyStream m_memoryStream; - - size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override - { - auto available = m_length - m_totalRead; - if (available == 0) - { - return 0; - } - - auto toRead = std::min(count, available); - auto read = m_memoryStream.Read(buffer, toRead, context); - - // Circurlar implementation. Rewind the stream every time we reach the end - if (read == 0) // No more bytes to read from. - { - m_memoryStream.Rewind(); - read = m_memoryStream.Read(buffer, toRead, context); - } - - m_totalRead += read; - return read; - } - - public: - CircularBodyStream(size_t size, uint8_t fillWith) - : m_buffer(std::make_unique>(1024 * 1024, fillWith)), m_length(size), - m_memoryStream(*m_buffer) - { - } - - int64_t Length() const override { return m_length; } - void Rewind() override { m_totalRead = 0; } - }; - }}} // namespace Azure::Core::Test diff --git a/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp b/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp index f56ff6b735..de0cbc342c 100644 --- a/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp +++ b/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp @@ -22,13 +22,6 @@ #include #include -#define CHECK_SKIP_TEST() \ - std::string const readTestNameAndUpdateTestContext = GetTestName(); \ - if (shouldSkipTest()) \ - { \ - GTEST_SKIP(); \ - } - using namespace std::chrono_literals; namespace Azure { namespace Core { namespace Test { @@ -143,28 +136,6 @@ namespace Azure { namespace Core { namespace Test { } } - inline void ValidateSkippingTest() - { - if (m_wasSkipped) - { - GTEST_SKIP(); - } - } - - bool IsValidTime(const Azure::DateTime& datetime) - { - // Playback won't check dates - if (m_testContext.IsPlaybackMode()) - { - return true; - } - - // We assume datetime within a week is valid. - const auto minTime = std::chrono::system_clock::now() - std::chrono::hours(24 * 7); - const auto maxTime = std::chrono::system_clock::now() + std::chrono::hours(24 * 7); - return datetime > minTime && datetime < maxTime; - } - // Reads the current test instance name. // Name gets also sanitized (special chars are removed) to avoid issues when recording or // creating @@ -236,11 +207,13 @@ namespace Azure { namespace Core { namespace Test { return std::make_unique(url, credential, options); } + template void InitClientOptions(T& options) { PrepareOptions(options); } + template T InitClientOptions() { // Run instrumentation before creating the client T options; - PrepareOptions(options); + InitClientOptions(options); return options; } @@ -395,6 +368,12 @@ namespace Azure { namespace Core { namespace Test { { m_testProxy = std::make_unique(m_testContext); } + + std::string const readTestNameAndUpdateTestContext = GetTestName(); + if (shouldSkipTest()) + { + GTEST_SKIP(); + } } /** diff --git a/sdk/core/azure-core-test/inc/azure/core/test/test_proxy_manager.hpp b/sdk/core/azure-core-test/inc/azure/core/test/test_proxy_manager.hpp index fb1603ade3..6fae5bc816 100644 --- a/sdk/core/azure-core-test/inc/azure/core/test/test_proxy_manager.hpp +++ b/sdk/core/azure-core-test/inc/azure/core/test/test_proxy_manager.hpp @@ -139,18 +139,6 @@ namespace Azure { namespace Core { namespace Test { return std::make_shared(); } - /** - * Sets proxy to stop RECORD test and save the recording file. - * - */ - void SetStopRecordMode(); - - /** - * Sets proxy to stop PLAYBACK test. - * - */ - void SetStopPlaybackMode(); - /** * Gets the test recording ID * diff --git a/sdk/core/azure-core-test/src/test_base.cpp b/sdk/core/azure-core-test/src/test_base.cpp index a2428480fa..48efd8663c 100644 --- a/sdk/core/azure-core-test/src/test_base.cpp +++ b/sdk/core/azure-core-test/src/test_base.cpp @@ -3,7 +3,6 @@ #include -#include "azure/core/test/network_models.hpp" #include "azure/core/test/test_base.hpp" #include diff --git a/sdk/core/azure-core-test/src/test_proxy_manager.cpp b/sdk/core/azure-core-test/src/test_proxy_manager.cpp index b5aa34a144..2c12182317 100644 --- a/sdk/core/azure-core-test/src/test_proxy_manager.cpp +++ b/sdk/core/azure-core-test/src/test_proxy_manager.cpp @@ -94,8 +94,6 @@ void TestProxyManager::StartPlaybackRecord(TestMode testMode) { if (IsPlaybackMode() || IsRecordMode()) { - std::string mode = (IsPlaybackMode() ? "playback" : "record"); - std::cout << "TestProxy already in " + mode + " mode."; return; } @@ -225,12 +223,50 @@ void TestProxyManager::SetProxySanitizer() Azure::Core::Url matcherRequest(m_proxy); matcherRequest.AppendPath("Admin"); matcherRequest.AppendPath("SetMatcher"); - const std::string matcherBody - = "{\"compareBodies\": false ,\"ignoreQueryOrdering\": true , \"ignoredHeaders\": " - "\"x-ms-copy-source,x-ms-proposed-lease-id,x-ms-lease-id,x-ms-file-change-" - "time,x-ms-file-creation-time,x-ms-file-last-write-time,x-ms-destination-" - "lease-id,x-ms-source-lease-id,x-ms-source-content-crc64,x-ms-content-crc64,x-ms-source-" - "content-md5,x-ms-content-md5,Content-MD5\"}"; + std::string matcherBody; + { + auto jsonRoot = Json::_internal::json::object(); + jsonRoot["compareBodies"] = false; + jsonRoot["ignoreQueryOrdering"] = true; + const std::vector excludedHeaders = { + "Expect", + "Connection", + }; + jsonRoot["excludedHeaders"] = std::accumulate( + excludedHeaders.begin(), + excludedHeaders.end(), + std::string(), + [](const std::string& lhs, const std::string& rhs) { + return lhs + (lhs.empty() ? "" : ",") + rhs; + }); + const std::vector ignoredHeaders = { + "x-ms-copy-source", + "x-ms-file-change-time", + "x-ms-file-creation-time", + "x-ms-file-last-write-time", + "x-ms-rename-source", + }; + const std::vector ignoreQueryParameters = { + "st", + "se", + "sig", + }; + jsonRoot["ignoredHeaders"] = std::accumulate( + ignoredHeaders.begin(), + ignoredHeaders.end(), + std::string(), + [](const std::string& lhs, const std::string& rhs) { + return lhs + (lhs.empty() ? "" : ",") + rhs; + }); + jsonRoot["ignoredQueryParameters"] = std::accumulate( + ignoreQueryParameters.begin(), + ignoreQueryParameters.end(), + std::string(), + [](const std::string& lhs, const std::string& rhs) { + return lhs + (lhs.empty() ? "" : ",") + rhs; + }); + matcherBody = jsonRoot.dump(); + } { Azure::Core::IO::MemoryBodyStream payloadStream( diff --git a/sdk/core/azure-core-test/src/test_proxy_policy.cpp b/sdk/core/azure-core-test/src/test_proxy_policy.cpp index b241867c6a..826e8f7a2b 100644 --- a/sdk/core/azure-core-test/src/test_proxy_policy.cpp +++ b/sdk/core/azure-core-test/src/test_proxy_policy.cpp @@ -42,7 +42,10 @@ std::unique_ptr TestProxyPolicy::Send( { // This is a download with keep connection open. Let's switch the request redirectRequest = Azure::Core::Http::Request( - request.GetMethod(), Azure::Core::Url(m_testProxy->GetTestProxy()), false); + request.GetMethod(), + Azure::Core::Url(m_testProxy->GetTestProxy()), + request.GetBodyStream(), + false); } redirectRequest.GetUrl().SetPath(request.GetUrl().GetPath()); @@ -50,7 +53,10 @@ std::unique_ptr TestProxyPolicy::Send( // Copy all headers for (auto& header : request.GetHeaders()) { - redirectRequest.SetHeader(header.first, header.second); + if (header.first != "host") + { + redirectRequest.SetHeader(header.first, header.second); + } } // QP for (auto const& qp : request.GetUrl().GetQueryParameters()) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index a3900fb95e..2a001f176a 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -8,9 +8,12 @@ ### Bugs Fixed +- Fixed a bug where `Host` request header is not set for non-default port (80, 443). + ### Other Changes - [[#4352]](https://github.com/Azure/azure-sdk-for-cpp/pull/4352) Fixed compilation error on Visual Studio 2017. +- Libcurl transport doesn't add `Content-Length` request header for GET/HEAD/DELETE requests anymore. ### Acknowledgments @@ -584,4 +587,4 @@ Thank you to our developer community members who helped to make Azure Core bette ## 1.0.0-beta.1 (2020-09-09) -- Initial release +- Initial release \ No newline at end of file diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index facbe359b9..c061079146 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -413,10 +413,18 @@ CURLcode CurlSession::Perform(Context const& context) if (hostHeader == headers.end()) { Log::Write(Logger::Level::Verbose, LogMsgPrefix + "No Host in request headers. Adding it"); - this->m_request.SetHeader("Host", this->m_request.GetUrl().GetHost()); + std::string hostName = this->m_request.GetUrl().GetHost(); + auto port = this->m_request.GetUrl().GetPort(); + if (port != 0) + { + hostName += ":" + std::to_string(port); + } + this->m_request.SetHeader("Host", hostName); } - auto isContentLengthHeaderInRequest = headers.find("content-length"); - if (isContentLengthHeaderInRequest == headers.end()) + if (this->m_request.GetMethod() != HttpMethod::Get + && this->m_request.GetMethod() != HttpMethod::Head + && this->m_request.GetMethod() != HttpMethod::Delete + && headers.find("content-length") == headers.end()) { Log::Write(Logger::Level::Verbose, LogMsgPrefix + "No content-length in headers. Adding it"); this->m_request.SetHeader( diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index fb19ffabe7..70b4d7a609 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_90ac51538b" + "Tag": "cpp/storage_fc079eacac" } diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 9235eeeb35..a1e4171be2 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -924,6 +924,8 @@ namespace Azure { namespace Storage { namespace Blobs { /** * @brief Optional conditions that source must meet to perform this operation. + * @remarks Azure storage service doesn't support tags access condition for this operation. + * Don't use it. */ struct : public Azure::ModifiedConditions, public Azure::MatchConditions, diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index 8b561f70c8..4bc4fabdbc 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable ( blob_container_client_test.hpp blob_query_test.cpp blob_sas_test.cpp + blob_service_client_test.hpp blob_service_client_test.cpp block_blob_client_test.cpp block_blob_client_test.hpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp index 87d4306479..310c360734 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp @@ -7,32 +7,51 @@ #include #include -namespace Azure { namespace Storage { namespace Test { +namespace Azure { namespace Storage { namespace Blobs { namespace Models { + + bool operator==(const BlobHttpHeaders& lhs, const BlobHttpHeaders& rhs); - void AppendBlobClientTest::SetUp() { BlobContainerClientTest::SetUp(); } +}}}} // namespace Azure::Storage::Blobs::Models + +namespace Azure { namespace Storage { namespace Test { - void AppendBlobClientTest::TearDown() + void AppendBlobClientTest::SetUp() { - if (m_appendBlobClient) + BlobContainerClientTest::SetUp(); + if (shouldSkipTest()) { - m_appendBlobClient->Delete(); + return; } - // Delete container - BlobContainerClientTest::TearDown(); + m_blobName = RandomString(); + m_appendBlobClient = std::make_shared( + m_blobContainerClient->GetAppendBlobClient(m_blobName)); + m_appendBlobClient->Create(); + std::vector blobContent1(static_cast(1_KB), 'x'); + std::vector blobContent2(static_cast(512), 'a'); + auto blobContent = Azure::Core::IO::MemoryBodyStream(blobContent1.data(), blobContent1.size()); + m_appendBlobClient->AppendBlock(blobContent); + blobContent = Azure::Core::IO::MemoryBodyStream(blobContent2.data(), blobContent2.size()); + m_appendBlobClient->AppendBlock(blobContent); + m_blobContent = blobContent1; + m_blobContent.insert(m_blobContent.end(), blobContent2.begin(), blobContent2.end()); } // Requires blob versioning? TEST_F(AppendBlobClientTest, CreateAppendDelete) { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto appendBlobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "1", - InitClientOptions()); - - auto blobContentInfo = appendBlobClient.Create(m_blobUploadOptions); + auto blobClient = *m_appendBlobClient; + + Blobs::CreateAppendBlobOptions createOptions; + createOptions.HttpHeaders.ContentType = "application/x-binary"; + createOptions.HttpHeaders.ContentLanguage = "en-US"; + createOptions.HttpHeaders.ContentDisposition = "attachment"; + createOptions.HttpHeaders.CacheControl = "no-cache"; + createOptions.HttpHeaders.ContentEncoding = "identify"; + createOptions.Metadata = RandomMetadata(); + createOptions.Tags["key1"] = "value1"; + createOptions.Tags["key2"] = "value2"; + createOptions.Tags["key3 +-./:=_"] = "v1 +-./:=_"; + auto blobContentInfo = blobClient.Create(createOptions); EXPECT_TRUE(blobContentInfo.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(blobContentInfo.Value.LastModified)); EXPECT_TRUE(blobContentInfo.Value.VersionId.HasValue()); @@ -40,70 +59,53 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_FALSE(blobContentInfo.Value.EncryptionKeySha256.HasValue()); - auto properties = appendBlobClient.GetProperties().Value; + auto properties = blobClient.GetProperties().Value; EXPECT_TRUE(properties.CommittedBlockCount.HasValue()); EXPECT_EQ(properties.CommittedBlockCount.Value(), 0); EXPECT_EQ(properties.BlobSize, 0); + EXPECT_EQ(properties.Metadata, createOptions.Metadata); + EXPECT_EQ(properties.HttpHeaders, createOptions.HttpHeaders); + EXPECT_EQ(blobClient.GetTags().Value, createOptions.Tags); auto blockContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - auto appendResponse = appendBlobClient.AppendBlock(blockContent); - properties = appendBlobClient.GetProperties().Value; + auto appendResponse = blobClient.AppendBlock(blockContent); + properties = blobClient.GetProperties().Value; EXPECT_EQ(properties.CommittedBlockCount.Value(), 1); EXPECT_EQ(properties.BlobSize, static_cast(m_blobContent.size())); Azure::Storage::Blobs::AppendBlockOptions options; options.AccessConditions.IfAppendPositionEqual = 1_MB; blockContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - EXPECT_THROW(appendBlobClient.AppendBlock(blockContent, options), StorageException); + EXPECT_THROW(blobClient.AppendBlock(blockContent, options), StorageException); options.AccessConditions.IfAppendPositionEqual = properties.BlobSize; blockContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - appendBlobClient.AppendBlock(blockContent, options); + blobClient.AppendBlock(blockContent, options); - properties = appendBlobClient.GetProperties().Value; + properties = blobClient.GetProperties().Value; options = Azure::Storage::Blobs::AppendBlockOptions(); options.AccessConditions.IfMaxSizeLessThanOrEqual = properties.BlobSize + m_blobContent.size() - 1; blockContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - EXPECT_THROW(appendBlobClient.AppendBlock(blockContent, options), StorageException); + EXPECT_THROW(blobClient.AppendBlock(blockContent, options), StorageException); options.AccessConditions.IfMaxSizeLessThanOrEqual = properties.BlobSize + m_blobContent.size(); blockContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - appendBlobClient.AppendBlock(blockContent, options); + blobClient.AppendBlock(blockContent, options); - properties = appendBlobClient.GetProperties().Value; + properties = blobClient.GetProperties().Value; int64_t originalLength = properties.BlobSize; - appendBlobClient.AppendBlockFromUri(client.GetUrl() + GetSas()); - properties = appendBlobClient.GetProperties().Value; - EXPECT_EQ(properties.BlobSize, static_cast(originalLength + m_blobContent.size())); + blobClient.AppendBlockFromUri(blobClient.GetUrl() + GetSas()); + properties = blobClient.GetProperties().Value; + EXPECT_EQ(properties.BlobSize, 2 * originalLength); - auto deleteResponse = appendBlobClient.Delete(); + auto deleteResponse = blobClient.Delete(); EXPECT_TRUE(deleteResponse.Value.Deleted); - EXPECT_THROW(appendBlobClient.Delete(), StorageException); - } - - TEST_F(AppendBlobClientTest, CreateWithTags) - { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto appendBlobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "1", - InitClientOptions()); - - Blobs::CreateAppendBlobOptions options; - options.Tags["key1"] = "value1"; - options.Tags["key2"] = "value2"; - options.Tags["key3 +-./:=_"] = "v1 +-./:=_"; - appendBlobClient.Create(options); - - EXPECT_EQ(appendBlobClient.GetTags().Value, options.Tags); + EXPECT_THROW(blobClient.Delete(), StorageException); } TEST_F(AppendBlobClientTest, AccessConditionLastModifiedTime) { - auto const testName(GetTestName()); - auto appendBlobClient = GetAppendBlobClient(testName); + auto blobClient = *m_appendBlobClient; enum class TimePoint { @@ -118,7 +120,7 @@ namespace Azure { namespace Storage { namespace Test { UnmodifiedSince, }; - auto lastModifiedTime = appendBlobClient.GetProperties().Value.LastModified; + auto lastModifiedTime = blobClient.GetProperties().Value.LastModified; auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(1); auto timeAfterStr = lastModifiedTime + std::chrono::seconds(1); for (auto condition : {Condition::ModifiedSince, Condition::UnmodifiedSince}) @@ -141,11 +143,11 @@ namespace Azure { namespace Storage { namespace Test { || (condition == Condition::UnmodifiedSince && sinceTime == TimePoint::TimeBefore); if (shouldThrow) { - EXPECT_THROW(appendBlobClient.GetProperties(options), StorageException); + EXPECT_THROW(blobClient.GetProperties(options), StorageException); } else { - EXPECT_NO_THROW(appendBlobClient.GetProperties(options)); + EXPECT_NO_THROW(blobClient.GetProperties(options)); } } } @@ -153,20 +155,14 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(AppendBlobClientTest, AccessConditionETag) { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto appendBlobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "1", - InitClientOptions()); + auto blobClient = GetAppendBlobClientForTest(RandomString()); Blobs::CreateAppendBlobOptions createOptions; createOptions.AccessConditions.IfNoneMatch = Azure::ETag::Any(); - EXPECT_NO_THROW(appendBlobClient.Create(createOptions)); - EXPECT_THROW(appendBlobClient.Create(createOptions), StorageException); + EXPECT_NO_THROW(blobClient.Create(createOptions)); + EXPECT_THROW(blobClient.Create(createOptions), StorageException); - Azure::ETag eTag = appendBlobClient.GetProperties().Value.ETag; + Azure::ETag eTag = blobClient.GetProperties().Value.ETag; for (Azure::ETag match : {eTag, DummyETag, Azure::ETag()}) { for (Azure::ETag noneMatch : {eTag, DummyETag, Azure::ETag()}) @@ -183,11 +179,11 @@ namespace Azure { namespace Storage { namespace Test { bool shouldThrow = (match.HasValue() && match != eTag) || noneMatch == eTag; if (shouldThrow) { - EXPECT_THROW(appendBlobClient.GetProperties(options), StorageException); + EXPECT_THROW(blobClient.GetProperties(options), StorageException); } else { - EXPECT_NO_THROW(appendBlobClient.GetProperties(options)); + EXPECT_NO_THROW(blobClient.GetProperties(options)); } } } @@ -195,39 +191,21 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(AppendBlobClientTest, AccessConditionLeaseId) { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto appendBlobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "leaseId", - InitClientOptions()); - appendBlobClient.Create(); + auto blobClient = GetAppendBlobClientForTest(RandomString()); + blobClient.Create(); - std::string leaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - Blobs::BlobLeaseClient leaseClient(appendBlobClient, leaseId); + std::string leaseId = RandomUUID(); + Blobs::BlobLeaseClient leaseClient(blobClient, leaseId); leaseClient.Acquire(std::chrono::seconds(30)); - EXPECT_THROW(appendBlobClient.Delete(), StorageException); + EXPECT_THROW(blobClient.Delete(), StorageException); Blobs::DeleteBlobOptions options; options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(appendBlobClient.Delete(options)); + EXPECT_NO_THROW(blobClient.Delete(options)); } TEST_F(AppendBlobClientTest, Seal) { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto const extraBlobName = testName + "1"; - auto blobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - extraBlobName, - InitClientOptions()); - blobClient.Create(); - - auto blockContent - = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blobClient.AppendBlock(blockContent); + auto blobClient = *m_appendBlobClient; auto downloadResult = blobClient.Download(); EXPECT_TRUE(downloadResult.Value.Details.IsSealed.HasValue()); @@ -237,7 +215,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(getPropertiesResult.Value.IsSealed.HasValue()); EXPECT_FALSE(getPropertiesResult.Value.IsSealed.Value()); - auto blobItem = GetBlobItem(extraBlobName); + auto blobItem = GetBlobItem(m_blobName); EXPECT_TRUE(blobItem.Details.IsSealed.HasValue()); EXPECT_FALSE(blobItem.Details.IsSealed.Value()); @@ -259,16 +237,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(getPropertiesResult.Value.IsSealed.HasValue()); EXPECT_TRUE(getPropertiesResult.Value.IsSealed.Value()); - blobItem = GetBlobItem(extraBlobName); + blobItem = GetBlobItem(m_blobName); EXPECT_TRUE(blobItem.Details.IsSealed.HasValue()); EXPECT_TRUE(blobItem.Details.IsSealed.Value()); - auto const extraBlobName2 = testName + "2"; - auto blobClient2 = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - extraBlobName2, - InitClientOptions()); + auto blobClient2 = GetAppendBlobClientForTest(RandomString()); Blobs::StartBlobCopyFromUriOptions copyOptions; copyOptions.ShouldSealDestination = false; @@ -289,13 +262,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(AppendBlobClientTest, CreateIfNotExists) { - auto const testName(GetTestName()); - auto client = GetAppendBlobClient(testName); - auto blobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "test", - InitClientOptions()); + auto blobClient = GetAppendBlobClientForTest(RandomString()); auto blobClientWithoutAuth = Azure::Storage::Blobs::AppendBlobClient( blobClient.GetUrl(), InitClientOptions()); @@ -317,8 +284,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(AppendBlobClientTest, ContentHash) { - auto const testName(GetTestName()); - auto appendBlobClient = GetAppendBlobClient(testName); + auto appendBlobClient = GetAppendBlobClientForTest(RandomString()); const std::vector blobContent = RandomBuffer(10); const std::vector contentMd5 @@ -330,7 +296,7 @@ namespace Azure { namespace Storage { namespace Test { auto contentStream = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); appendBlobClient.AppendBlock(contentStream); - auto appendBlobClient2 = GetAppendBlobClient(testName + "2"); + auto appendBlobClient2 = GetAppendBlobClientForTest(RandomString()); appendBlobClient2.Create(); Blobs::AppendBlockOptions options1; diff --git a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.hpp index c4e907bc97..797f57bc68 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.hpp @@ -9,42 +9,21 @@ namespace Azure { namespace Storage { namespace Test { class AppendBlobClientTest : public BlobContainerClientTest { - private: - std::shared_ptr m_appendBlobClient; - protected: - Azure::Storage::Blobs::CreateAppendBlobOptions m_blobUploadOptions; - std::vector m_blobContent; + void SetUp() override; - virtual void SetUp() override; - virtual void TearDown() override; - - Azure::Storage::Blobs::AppendBlobClient const& GetAppendBlobClient(std::string const& blobName) + Blobs::AppendBlobClient GetAppendBlobClientForTest( + const std::string& blobName, + Blobs::BlobClientOptions clientOptions = Blobs::BlobClientOptions()) { - // Create container - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - m_appendBlobClient = std::make_unique( - containerClient.GetAppendBlobClient(blobName)); - - m_blobContent = std::vector(100, 'x'); - m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"key2", "Value2"}}; - m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; - m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; - m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; - m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; - m_blobUploadOptions.HttpHeaders.ContentEncoding = "identify"; - m_blobUploadOptions.HttpHeaders.ContentHash.Value.clear(); - m_appendBlobClient->Create(m_blobUploadOptions); - auto blockContent - = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_appendBlobClient->AppendBlock(blockContent); - m_blobUploadOptions.HttpHeaders.ContentHash - = m_appendBlobClient->GetProperties().Value.HttpHeaders.ContentHash; - - return *m_appendBlobClient; + auto containerClient = GetBlobContainerClientForTest(m_containerName, clientOptions); + return containerClient.GetAppendBlobClient(blobName); } + + protected: + std::shared_ptr m_appendBlobClient; + std::string m_blobName; + std::vector m_blobContent; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp index b3fbe63049..26d916603c 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp @@ -3,49 +3,24 @@ #include -#include "test/ut/test_base.hpp" +#include "blob_container_client_test.hpp" namespace Azure { namespace Storage { namespace Test { - class BlobBatchClientTest : public StorageTest { - private: - std::unique_ptr m_client; - - protected: - // Required to rename the test propertly once the test is started. - // We can only know the test instance name until the test instance is run. - Azure::Storage::Blobs::BlobServiceClient const& GetClientForTest(std::string const& testName) - { - // set the interceptor for the current test - m_testContext.RenameTest(testName); - return *m_client; - } - - void SetUp() override - { - StorageTest::SetUp(); - - auto options = InitClientOptions(); - m_client = std::make_unique( - Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), options)); - } - }; - - TEST_F(BlobBatchClientTest, DISABLED_SubmitDeleteBatch) + TEST_F(BlobContainerClientTest, BatchSubmitDelete_LIVEONLY_) { - const std::string testName = GetTestNameLowerCase(); + const std::string containerNamePrefix = LowercaseRandomString(); - const std::string containerName1 = testName + "1"; + const std::string containerName1 = containerNamePrefix + "1"; const std::string blob1Name = "b1"; const std::string blob2Name = "b2"; - const std::string containerName2 = testName + "2"; + const std::string containerName2 = containerNamePrefix + "2"; const std::string blob3Name = "b3"; - auto serviceClient = GetClientForTest(testName); - auto container1Client = serviceClient.GetBlobContainerClient(containerName1); + auto serviceClient = *m_blobServiceClient; + auto container1Client = GetBlobContainerClientForTest(containerName1); container1Client.CreateIfNotExists(); - auto container2Client = serviceClient.GetBlobContainerClient(containerName2); + auto container2Client = GetBlobContainerClientForTest(containerName2); container2Client.CreateIfNotExists(); auto blob1Client = container1Client.GetAppendBlobClient(blob1Name); blob1Client.Create(); @@ -69,19 +44,16 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(blob1Client.GetProperties(), StorageException); EXPECT_THROW(blob2Client.GetProperties(), StorageException); EXPECT_NO_THROW(blob3Client.GetProperties()); - - container1Client.Delete(); - container2Client.Delete(); } - TEST_F(BlobBatchClientTest, DISABLED_SnapshotVersion) + TEST_F(BlobContainerClientTest, BatchSnapshotVersion_LIVEONLY_) { - const std::string testName = GetTestNameLowerCase(); + const std::string containerNamePrefix = LowercaseRandomString(); - const std::string containerName1 = testName + "1"; + const std::string containerName1 = containerNamePrefix + "1"; const std::string blob1Name = "blockblob1"; - auto serviceClient = GetClientForTest(testName); - auto container1Client = serviceClient.GetBlobContainerClient(containerName1); + auto serviceClient = *m_blobServiceClient; + auto container1Client = GetBlobContainerClientForTest(containerName1); container1Client.CreateIfNotExists(); auto blob1Client = container1Client.GetBlockBlobClient(blob1Name); auto versionId = blob1Client.UploadFrom(nullptr, 0).Value.VersionId.Value(); @@ -113,15 +85,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(r4.GetResponse()); EXPECT_THROW(blob1Client.WithVersionId(versionId).GetProperties(), StorageException); EXPECT_THROW(blob1Client.WithSnapshot(snapshotId).GetProperties(), StorageException); - - container1Client.DeleteIfExists(); } - TEST_F(BlobBatchClientTest, SubmitSetTierBatch_LIVEONLY_) + TEST_F(BlobContainerClientTest, BatchSubmitSetTier_LIVEONLY_) { - const std::string testName = GetTestNameLowerCase(); - - const std::string containerName = testName; + const std::string containerName = LowercaseRandomString(); const std::string blob1Name = "b1"; const std::string blob2Name = "b2"; @@ -136,11 +104,13 @@ namespace Azure { namespace Storage { namespace Test { *_internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential); }(); - auto serviceClient = GetClientForTest(testName); - serviceClient.GetBlobContainerClient(containerName).CreateIfNotExists(); + auto serviceClient = *m_blobServiceClient; + GetBlobContainerClientForTest(containerName).CreateIfNotExists(); + Blobs::BlobClientOptions clientOptions; + InitClientOptions(clientOptions); auto containerClient = Blobs::BlobContainerClient( serviceClient.GetBlobContainerClient(containerName).GetUrl() + containerSasToken, - InitClientOptions()); + clientOptions); auto blob1Client = containerClient.GetBlockBlobClient(blob1Name); blob1Client.UploadFrom(nullptr, 0); auto blob2Client = containerClient.GetBlockBlobClient(blob2Name); @@ -157,26 +127,23 @@ namespace Azure { namespace Storage { namespace Test { blob1Client.GetProperties().Value.AccessTier.Value(), Blobs::Models::AccessTier::Cool); EXPECT_EQ( blob2Client.GetProperties().Value.AccessTier.Value(), Blobs::Models::AccessTier::Archive); - - serviceClient.DeleteBlobContainer(containerName); } - TEST_F(BlobBatchClientTest, DISABLED_TokenAuthorization) + TEST_F(BlobContainerClientTest, BatchTokenAuthorization_LIVEONLY_) { - const std::string testName = GetTestNameLowerCase(); - std::shared_ptr credential = std::make_shared( AadTenantId(), AadClientId(), AadClientSecret()); - Blobs::BlobClientOptions options; + Blobs::BlobClientOptions clientOptions; + InitClientOptions(clientOptions); - auto serviceClient = InitTestClient( - GetClientForTest(testName).GetUrl(), credential, options); + auto serviceClient + = Blobs::BlobServiceClient(m_blobServiceClient->GetUrl(), credential, clientOptions); - const std::string containerName = testName; + const std::string containerName = LowercaseRandomString(); const std::string blobName = "b1"; - auto containerClient = serviceClient->GetBlobContainerClient(containerName); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); containerClient.CreateIfNotExists(); auto blobClient = containerClient.GetAppendBlobClient(blobName); blobClient.Create(); @@ -186,19 +153,15 @@ namespace Azure { namespace Storage { namespace Test { auto submitBatchResponse = containerClient.SubmitBatch(batch); EXPECT_TRUE(delete1Response.GetResponse().Value.Deleted); - - containerClient.Delete(); } - TEST_F(BlobBatchClientTest, Exceptions_LIVEONLY_) + TEST_F(BlobContainerClientTest, BatchExceptions_LIVEONLY_) { - const std::string testName = GetTestNameLowerCase(); - - const std::string containerName = testName; + const std::string containerName = LowercaseRandomString(); const std::string blobName = "b1"; - auto serviceClient = GetClientForTest(testName); - auto containerClient = serviceClient.GetBlobContainerClient(containerName); + auto serviceClient = *m_blobServiceClient; + auto containerClient = GetBlobContainerClientForTest(containerName); containerClient.CreateIfNotExists(); auto blobClient = containerClient.GetBlockBlobClient(blobName); blobClient.UploadFrom(nullptr, 0); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index c27785aad8..1716652efd 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -19,6 +19,51 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { + void BlobContainerClientTest::SetUp() + { + BlobServiceClientTest::SetUp(); + if (shouldSkipTest()) + { + return; + } + m_containerName = GetLowercaseIdentifier(); + m_blobContainerClient = std::make_shared( + m_blobServiceClient->GetBlobContainerClient(m_containerName)); + while (true) + { + try + { + m_blobContainerClient->CreateIfNotExists(); + break; + } + catch (StorageException& e) + { + if (e.ErrorCode != "ContainerBeingDeleted") + { + throw; + } + SUCCEED() << "Container is being deleted. Will try again after 3 seconds."; + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + } + + m_resourceCleanupFunctions.push_back( + [blobContainerClient = *m_blobContainerClient]() { blobContainerClient.DeleteIfExists(); }); + } + + Blobs::BlobContainerClient BlobContainerClientTest::GetBlobContainerClientForTest( + const std::string& containerName, + Blobs::BlobClientOptions clientOptions) + { + InitClientOptions(clientOptions); + auto blobContainerClient = Blobs::BlobContainerClient::CreateFromConnectionString( + StandardStorageConnectionString(), containerName, clientOptions); + m_resourceCleanupFunctions.push_back( + [blobContainerClient]() { blobContainerClient.DeleteIfExists(); }); + + return blobContainerClient; + } + std::string BlobContainerClientTest::GetSas() { Sas::BlobSasBuilder sasBuilder; @@ -54,61 +99,60 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, CreateDelete) { - auto container_client = GetBlobContainerTestClient(); + auto containerClient = GetBlobContainerClientForTest(LowercaseRandomString()); Azure::Storage::Blobs::CreateBlobContainerOptions options; Azure::Storage::Metadata metadata; metadata["key1"] = "one"; metadata["key2"] = "TWO"; options.Metadata = metadata; - auto res = container_client.Create(options); + auto res = containerClient.Create(options); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); EXPECT_TRUE(res.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - EXPECT_THROW(container_client.Create(), StorageException); + EXPECT_THROW(containerClient.Create(), StorageException); - auto res2 = container_client.Delete(); + auto res2 = containerClient.Delete(); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); - container_client = GetBlobContainerTestClient("UPPERCASE"); - EXPECT_THROW(container_client.CreateIfNotExists(), StorageException); - container_client = GetBlobContainerTestClient("2"); + containerClient = GetBlobContainerClientForTest("UPPERCASE"); + EXPECT_THROW(containerClient.CreateIfNotExists(), StorageException); + containerClient = GetBlobContainerClientForTest(LowercaseRandomString()); { - auto response = container_client.DeleteIfExists(); + auto response = containerClient.DeleteIfExists(); EXPECT_FALSE(response.Value.Deleted); } { - auto response = container_client.CreateIfNotExists(); + auto response = containerClient.CreateIfNotExists(); EXPECT_TRUE(response.Value.Created); } { - auto response = container_client.CreateIfNotExists(); + auto response = containerClient.CreateIfNotExists(); EXPECT_FALSE(response.Value.Created); } { - auto response = container_client.DeleteIfExists(); + auto response = containerClient.DeleteIfExists(); EXPECT_TRUE(response.Value.Deleted); } } TEST_F(BlobContainerClientTest, Metadata) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; Azure::Storage::Metadata metadata; metadata["key1"] = "one"; metadata["key2"] = "TWO"; - auto res = client.SetMetadata(metadata); + auto res = containerClient.SetMetadata(metadata); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); EXPECT_TRUE(res.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - auto res2 = client.GetProperties(); + auto res2 = containerClient.GetProperties(); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); @@ -118,15 +162,14 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(properties.Metadata, metadata); metadata.clear(); - client.SetMetadata(metadata); - properties = client.GetProperties().Value; + containerClient.SetMetadata(metadata); + properties = containerClient.GetProperties().Value; EXPECT_TRUE(properties.Metadata.empty()); } TEST_F(BlobContainerClientTest, ListBlobsFlat) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; const std::string prefix1 = "prefix1-"; const std::string prefix2 = "prefix2-"; @@ -139,22 +182,22 @@ namespace Azure { namespace Storage { namespace Test { for (int i = 0; i < 5; ++i) { std::string blobName = prefix1 + baseName + std::to_string(i); - auto blobClient = client.GetBlockBlobClient(blobName); + auto blobClient = containerClient.GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::IO::MemoryBodyStream(nullptr, 0); blobClient.Upload(emptyContent); p1Blobs.insert(blobName); p1p2Blobs.insert(blobName); } { - auto appendBlobClient = client.GetAppendBlobClient(m_containerName + "-appendblob"); + auto appendBlobClient = containerClient.GetAppendBlobClient(RandomString() + "-appendblob"); appendBlobClient.Create(); - auto pageBlobClient = client.GetPageBlobClient(m_containerName + "-pageblob"); + auto pageBlobClient = containerClient.GetPageBlobClient(RandomString() + "-pageblob"); pageBlobClient.Create(4096); } for (int i = 0; i < 5; ++i) { std::string blobName = prefix2 + baseName + std::to_string(i); - auto blobClient = client.GetBlockBlobClient(blobName); + auto blobClient = containerClient.GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::IO::MemoryBodyStream(nullptr, 0); blobClient.Upload(emptyContent); p2Blobs.insert(blobName); @@ -164,7 +207,7 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::ListBlobsOptions options; options.PageSizeHint = 4; std::set listBlobs; - for (auto pageResult = client.ListBlobs(options); pageResult.HasPage(); + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { EXPECT_FALSE(pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); @@ -216,7 +259,7 @@ namespace Azure { namespace Storage { namespace Test { options.Prefix = prefix1; listBlobs.clear(); - for (auto pageResult = client.ListBlobs(options); pageResult.HasPage(); + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { for (const auto& blob : pageResult.Blobs) @@ -229,20 +272,19 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, ListBlobsByHierarchy) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; const std::string delimiter = "/"; - const std::string prefix = m_containerName; - const std::string prefix1 = prefix + "-" + m_containerName; - const std::string prefix2 = prefix + "-" + m_containerName; + const std::string prefix = RandomString(); + const std::string prefix1 = prefix + "-" + RandomString(); + const std::string prefix2 = prefix + "-" + RandomString(); std::set blobs; for (const auto& blobNamePrefix : {prefix1, prefix2}) { for (int i = 0; i < 3; ++i) { - std::string blobName = blobNamePrefix + delimiter + m_containerName + std::to_string(i); - auto blobClient = client.GetBlockBlobClient(blobName); + std::string blobName = blobNamePrefix + delimiter + RandomString() + std::to_string(i); + auto blobClient = containerClient.GetBlockBlobClient(blobName); auto emptyContent = Azure::Core::IO::MemoryBodyStream(nullptr, 0); blobClient.Upload(emptyContent); blobs.insert(blobName); @@ -252,7 +294,8 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::ListBlobsOptions options; options.Prefix = prefix; std::set items; - for (auto pageResult = client.ListBlobsByHierarchy(delimiter, options); pageResult.HasPage(); + for (auto pageResult = containerClient.ListBlobsByHierarchy(delimiter, options); + pageResult.HasPage(); pageResult.MoveToNextPage()) { EXPECT_EQ(pageResult.Delimiter, delimiter); @@ -269,7 +312,8 @@ namespace Azure { namespace Storage { namespace Test { for (const auto& p : {prefix1, prefix2}) { options.Prefix = p + delimiter; - for (auto pageResult = client.ListBlobsByHierarchy(delimiter, options); pageResult.HasPage(); + for (auto pageResult = containerClient.ListBlobsByHierarchy(delimiter, options); + pageResult.HasPage(); pageResult.MoveToNextPage()) { EXPECT_EQ(pageResult.Delimiter, delimiter); @@ -284,15 +328,14 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(items, blobs); } - // NOTE: This test Requires storage account with versioning enabled! - // https://docs.microsoft.com/en-us/azure/storage/blobs/versioning-enable?tabs=portal TEST_F(BlobContainerClientTest, ListBlobsOtherStuff) { - auto client = GetBlobContainerTestClient(); - client.Create(); + // NOTE: This test Requires storage account with versioning enabled! + // https://docs.microsoft.com/en-us/azure/storage/blobs/versioning-enable?tabs=portal + auto containerClient = *m_blobContainerClient; std::string blobName = "blob" + m_containerName; - auto blobClient = client.GetAppendBlobClient(blobName); + auto blobClient = containerClient.GetAppendBlobClient(blobName); blobClient.Create(); blobClient.Delete(); blobClient.Create(); @@ -315,7 +358,7 @@ namespace Azure { namespace Storage { namespace Test { bool foundDeleted = false; bool foundMetadata = false; - for (auto pageResult = client.ListBlobs(options); pageResult.HasPage(); + for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { for (const auto& blob : pageResult.Blobs) @@ -361,19 +404,16 @@ namespace Azure { namespace Storage { namespace Test { // Test uses StartsOn and ExpiresOn values based on time now() + minutes // Hence, the test can't be recorded and need to run on live mode always. - TEST_F(BlobContainerClientTest, AccessControlList_LIVEONLY_) + TEST_F(BlobContainerClientTest, AccessControlList) { - // will skip test under some cased where test can't run (usually LIVE only tests) - CHECK_SKIP_TEST() - - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; + containerClient.CreateIfNotExists(); Blobs::SetBlobContainerAccessPolicyOptions options; options.AccessType = Blobs::Models::PublicAccessType::Blob; { Blobs::Models::SignedIdentifier identifier; - identifier.Id = GetStringOfSize(63) + "1"; + identifier.Id = RandomString(63) + "1"; identifier.StartsOn = std::chrono::system_clock::now() - std::chrono::minutes(1); identifier.ExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(1); identifier.Permissions = "r"; @@ -381,7 +421,7 @@ namespace Azure { namespace Storage { namespace Test { } { Blobs::Models::SignedIdentifier identifier; - identifier.Id = GetStringOfSize(63) + "2"; + identifier.Id = RandomString(63) + "2"; identifier.StartsOn = std::chrono::system_clock::now() - std::chrono::minutes(2); identifier.ExpiresOn.Reset(); /* cspell:disable-next-line */ @@ -390,36 +430,45 @@ namespace Azure { namespace Storage { namespace Test { } { Blobs::Models::SignedIdentifier identifier; - identifier.Id = GetStringOfSize(63) + "3"; + identifier.Id = RandomString(63) + "3"; identifier.Permissions = "r"; options.SignedIdentifiers.emplace_back(identifier); } { Blobs::Models::SignedIdentifier identifier; - identifier.Id = GetStringOfSize(63) + "4"; + identifier.Id = RandomString(63) + "4"; identifier.StartsOn = std::chrono::system_clock::now() - std::chrono::minutes(1); identifier.ExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(1); options.SignedIdentifiers.emplace_back(identifier); } - auto ret = client.SetAccessPolicy(options); + auto ret = containerClient.SetAccessPolicy(options); EXPECT_TRUE(ret.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(ret.Value.LastModified)); - auto ret2 = client.GetAccessPolicy(); + auto ret2 = containerClient.GetAccessPolicy(); EXPECT_EQ(ret2.Value.AccessType, options.AccessType); - EXPECT_EQ(ret2.Value.SignedIdentifiers, options.SignedIdentifiers); + if (m_testContext.IsLiveMode()) + { + EXPECT_EQ(ret2.Value.SignedIdentifiers, options.SignedIdentifiers); + } + containerClient.DeleteIfExists(); } - TEST_F(BlobContainerClientTest, Lease_LIVEONLY_) + TEST_F(BlobContainerClientTest, Lease) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; { std::string leaseId1 = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + EXPECT_EQ(leaseId1.length(), leaseId2.length()); + EXPECT_NE(leaseId1, leaseId2); + } + { + std::string leaseId1 = RandomUUID(); auto leaseDuration = std::chrono::seconds(20); - Blobs::BlobLeaseClient leaseClient(client, leaseId1); + Blobs::BlobLeaseClient leaseClient(containerClient, leaseId1); auto aLease = leaseClient.Acquire(leaseDuration).Value; EXPECT_TRUE(aLease.ETag.HasValue()); EXPECT_TRUE(IsValidTime(aLease.LastModified)); @@ -430,7 +479,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(aLease.LastModified)); EXPECT_EQ(aLease.LeaseId, leaseId1); - auto properties = client.GetProperties().Value; + auto properties = containerClient.GetProperties().Value; EXPECT_EQ(properties.LeaseState, Blobs::Models::LeaseState::Leased); EXPECT_EQ(properties.LeaseStatus, Blobs::Models::LeaseStatus::Locked); EXPECT_EQ(properties.LeaseDuration.Value(), Blobs::Models::LeaseDurationType::Fixed); @@ -440,7 +489,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(rLease.LastModified)); EXPECT_EQ(rLease.LeaseId, leaseId1); - std::string leaseId2 = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = RandomUUID(); EXPECT_NE(leaseId1, leaseId2); auto cLease = leaseClient.Change(leaseId2).Value; EXPECT_TRUE(cLease.ETag.HasValue()); @@ -454,9 +503,9 @@ namespace Azure { namespace Storage { namespace Test { } { - Blobs::BlobLeaseClient leaseClient(client, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + Blobs::BlobLeaseClient leaseClient(containerClient, RandomUUID()); auto aLease = leaseClient.Acquire(Blobs::BlobLeaseClient::InfiniteLeaseDuration).Value; - auto properties = client.GetProperties().Value; + auto properties = containerClient.GetProperties().Value; EXPECT_EQ(properties.LeaseDuration.Value(), Blobs::Models::LeaseDurationType::Infinite); auto brokenLease = leaseClient.Break().Value; EXPECT_TRUE(brokenLease.ETag.HasValue()); @@ -464,7 +513,7 @@ namespace Azure { namespace Storage { namespace Test { } { - Blobs::BlobLeaseClient leaseClient(client, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + Blobs::BlobLeaseClient leaseClient(containerClient, RandomUUID()); auto leaseDuration = std::chrono::seconds(20); auto aLease = leaseClient.Acquire(leaseDuration).Value; auto brokenLease = leaseClient.Break().Value; @@ -479,37 +528,35 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, EncryptionScope) { - auto blobContainerClient = GetBlobContainerTestClient(); - blobContainerClient.CreateIfNotExists(); - auto const& testEncryptionScope = GetTestEncryptionScope(); + auto containerClient = *m_blobContainerClient; + const auto& testEncryptionScope = GetTestEncryptionScope(); { - auto properties = blobContainerClient.GetProperties().Value; + auto properties = containerClient.GetProperties().Value; EXPECT_EQ(properties.DefaultEncryptionScope, AccountEncryptionKey); EXPECT_EQ(properties.PreventEncryptionScopeOverride, false); } { - std::string containerName = GetContainerValidName() + "1"; + std::string containerName = GetLowercaseIdentifier() + "1"; std::string blobName = GetTestName() + "1"; - Blobs::BlobClientOptions options = InitClientOptions(); + Blobs::BlobClientOptions options; options.EncryptionScope = testEncryptionScope; - auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), containerName, options); + auto containerClient2 = GetBlobContainerClientForTest(containerName, options); Blobs::CreateBlobContainerOptions createOptions; createOptions.DefaultEncryptionScope = testEncryptionScope; createOptions.PreventEncryptionScopeOverride = true; - EXPECT_NO_THROW(containerClient.Create(createOptions)); - auto properties = containerClient.GetProperties().Value; + EXPECT_NO_THROW(containerClient2.Create(createOptions)); + auto properties = containerClient2.GetProperties().Value; EXPECT_EQ(properties.DefaultEncryptionScope, createOptions.DefaultEncryptionScope.Value()); EXPECT_EQ( properties.PreventEncryptionScopeOverride, createOptions.PreventEncryptionScopeOverride.Value()); - auto appendBlobClient = containerClient.GetAppendBlobClient(blobName); + auto appendBlobClient = containerClient2.GetAppendBlobClient(blobName); auto blobContentInfo = appendBlobClient.Create(); { Blobs::ListBlobsOptions listOptions; listOptions.Prefix = blobName; - for (auto page = containerClient.ListBlobs(listOptions); page.HasPage(); + for (auto page = containerClient2.ListBlobs(listOptions); page.HasPage(); page.MoveToNextPage()) { for (auto& blob : page.Blobs) @@ -526,19 +573,19 @@ namespace Azure { namespace Storage { namespace Test { appendBlobClient.Delete(); EXPECT_TRUE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_EQ(blobContentInfo.Value.EncryptionScope.Value(), testEncryptionScope); - auto appendBlobClientWithoutEncryptionScope = containerClient.GetAppendBlobClient(blobName); + auto appendBlobClientWithoutEncryptionScope = containerClient2.GetAppendBlobClient(blobName); blobContentInfo = appendBlobClientWithoutEncryptionScope.Create(); appendBlobClientWithoutEncryptionScope.Delete(); EXPECT_TRUE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_EQ(blobContentInfo.Value.EncryptionScope.Value(), testEncryptionScope); - containerClient.Delete(); + containerClient2.Delete(); } { std::string blobName = GetTestName() + "2"; - Blobs::BlobClientOptions options = InitClientOptions(); + Blobs::BlobClientOptions options; options.EncryptionScope = testEncryptionScope; - auto appendBlobClient = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blobName, options); + auto appendBlobClient + = GetBlobContainerClientForTest(m_containerName, options).GetAppendBlobClient(blobName); auto blobContentInfo = appendBlobClient.Create(); EXPECT_TRUE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_EQ(blobContentInfo.Value.EncryptionScope.Value(), testEncryptionScope); @@ -554,8 +601,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(appendBlobClient.AppendBlock(bodyStream)); bodyStream.Rewind(); - auto appendBlobClientWithoutEncryptionScope - = blobContainerClient.GetAppendBlobClient(blobName); + auto appendBlobClientWithoutEncryptionScope = containerClient.GetAppendBlobClient(blobName); EXPECT_THROW( appendBlobClientWithoutEncryptionScope.AppendBlock(bodyStream), StorageException); EXPECT_THROW(appendBlobClientWithoutEncryptionScope.CreateSnapshot(), StorageException); @@ -563,19 +609,13 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(BlobContainerClientTest, CustomerProvidedKey_LIVEONLY_) + TEST_F(BlobContainerClientTest, CustomerProvidedKey) { - // will skip test under some cased where test can't run (usually LIVE only tests) - CHECK_SKIP_TEST() - - auto client = GetBlobContainerTestClient(); - client.Create(); + auto sourceContainerClient = *m_blobContainerClient; auto getRandomCustomerProvidedKey = [&]() { Blobs::EncryptionKey key; - std::vector aes256Key; - aes256Key.resize(32); - RandomBuffer(&aes256Key[0], aes256Key.size()); + std::vector aes256Key = RandomBuffer(32); key.Key = Azure::Core::Convert::Base64Encode(aes256Key); key.KeyHash = Azure::Core::Cryptography::_internal::Sha256Hash().Final( aes256Key.data(), aes256Key.size()); @@ -585,17 +625,16 @@ namespace Azure { namespace Storage { namespace Test { Blobs::BlobClientOptions options; options.CustomerProvidedKey = getRandomCustomerProvidedKey(); - auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, options); + auto destContainerClient = GetBlobContainerClientForTest(m_containerName, options); std::vector blobContent(512); Azure::Core::IO::MemoryBodyStream bodyStream(blobContent.data(), blobContent.size()); - auto copySourceBlob = client.GetBlockBlobClient(RandomString()); + auto copySourceBlob = sourceContainerClient.GetBlockBlobClient(RandomString()); copySourceBlob.UploadFrom(blobContent.data(), blobContent.size()); { std::string blockBlobName = RandomString(); - auto blockBlob = containerClient.GetBlockBlobClient(blockBlobName); + auto blockBlob = destContainerClient.GetBlockBlobClient(blockBlobName); bodyStream.Rewind(); EXPECT_NO_THROW(blockBlob.Upload(bodyStream)); std::string blockId1 = Base64EncodeText("1"); @@ -606,18 +645,17 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(blockBlob.CommitBlockList({blockId1, blockId2})); EXPECT_THROW(blockBlob.SetAccessTier(Blobs::Models::AccessTier::Cool), StorageException); - auto appendBlobClientWithoutEncryptionKey - = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blockBlobName); + auto blockBlobClientWithoutEncryptionKey + = m_blobContainerClient->GetBlockBlobClient(blockBlobName); EXPECT_THROW( - appendBlobClientWithoutEncryptionKey.SetAccessTier(Blobs::Models::AccessTier::Cool), + blockBlobClientWithoutEncryptionKey.SetAccessTier(Blobs::Models::AccessTier::Cool), StorageException); - EXPECT_NO_THROW(appendBlobClientWithoutEncryptionKey.GetBlockList()); + EXPECT_NO_THROW(blockBlobClientWithoutEncryptionKey.GetBlockList()); } { std::string appendBlobName = RandomString(); - auto appendBlob = containerClient.GetAppendBlobClient(appendBlobName); + auto appendBlob = destContainerClient.GetAppendBlobClient(appendBlobName); auto blobContentInfo = appendBlob.Create().Value; EXPECT_TRUE(blobContentInfo.IsServerEncrypted); EXPECT_TRUE(blobContentInfo.EncryptionKeySha256.HasValue()); @@ -644,8 +682,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(appendBlob.CreateSnapshot()); auto appendBlobClientWithoutEncryptionKey - = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, appendBlobName); + = m_blobContainerClient->GetAppendBlobClient(appendBlobName); bodyStream.Rewind(); EXPECT_THROW(appendBlobClientWithoutEncryptionKey.AppendBlock(bodyStream), StorageException); EXPECT_THROW( @@ -665,7 +702,7 @@ namespace Azure { namespace Storage { namespace Test { { std::string pageBlobName = RandomString(); - auto pageBlob = containerClient.GetPageBlobClient(pageBlobName); + auto pageBlob = destContainerClient.GetPageBlobClient(pageBlobName); auto blobContentInfo = pageBlob.Create(0).Value; EXPECT_TRUE(blobContentInfo.IsServerEncrypted); EXPECT_TRUE(blobContentInfo.EncryptionKeySha256.HasValue()); @@ -679,8 +716,7 @@ namespace Azure { namespace Storage { namespace Test { 0, copySourceBlob.GetUrl() + GetSas(), {0, static_cast(blobContent.size())})); auto pageBlobClientWithoutEncryptionKey - = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, pageBlobName); + = m_blobContainerClient->GetPageBlobClient(pageBlobName); EXPECT_NO_THROW(pageBlobClientWithoutEncryptionKey.GetPageRanges()); EXPECT_NO_THROW(pageBlobClientWithoutEncryptionKey.Resize(blobContent.size() + 512)); } @@ -688,8 +724,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, AccessConditionLastModifiedTime) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; enum class TimePoint { @@ -708,7 +743,7 @@ namespace Azure { namespace Storage { namespace Test { { for (auto sinceTime : {TimePoint::TimeBefore, TimePoint::TimeAfter}) { - auto lastModifiedTime = client.GetProperties().Value.LastModified; + auto lastModifiedTime = containerClient.GetProperties().Value.LastModified; auto timeBefore = lastModifiedTime - std::chrono::seconds(1); auto timeAfter = lastModifiedTime + std::chrono::seconds(1); @@ -729,68 +764,65 @@ namespace Azure { namespace Storage { namespace Test { || (condition == Condition::UnmodifiedSince && sinceTime == TimePoint::TimeBefore); if (shouldThrow) { - EXPECT_THROW(client.SetAccessPolicy(options), StorageException); + EXPECT_THROW(containerClient.SetAccessPolicy(options), StorageException); } else { - EXPECT_NO_THROW(client.SetAccessPolicy(options)); + EXPECT_NO_THROW(containerClient.SetAccessPolicy(options)); } } } - client.Delete(); } TEST_F(BlobContainerClientTest, AccessConditionLeaseId) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; - const std::string leaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - const std::string dummyLeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - Blobs::BlobLeaseClient leaseClient(client, leaseId); + const std::string leaseId = RandomUUID(); + const std::string dummyLeaseId = RandomUUID(); + Blobs::BlobLeaseClient leaseClient(containerClient, leaseId); leaseClient.Acquire(std::chrono::seconds(30)); { Blobs::GetBlobContainerPropertiesOptions options; options.AccessConditions.LeaseId = dummyLeaseId; - EXPECT_THROW(client.GetProperties(options), StorageException); + EXPECT_THROW(containerClient.GetProperties(options), StorageException); options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(client.GetProperties(options)); + EXPECT_NO_THROW(containerClient.GetProperties(options)); } { Blobs::SetBlobContainerMetadataOptions options; options.AccessConditions.LeaseId = dummyLeaseId; - EXPECT_THROW(client.SetMetadata({}, options), StorageException); + EXPECT_THROW(containerClient.SetMetadata({}, options), StorageException); options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(client.SetMetadata({}, options)); + EXPECT_NO_THROW(containerClient.SetMetadata({}, options)); } { Blobs::GetBlobContainerAccessPolicyOptions options; options.AccessConditions.LeaseId = dummyLeaseId; - EXPECT_THROW(client.GetAccessPolicy(options), StorageException); + EXPECT_THROW(containerClient.GetAccessPolicy(options), StorageException); options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(client.GetAccessPolicy(options)); + EXPECT_NO_THROW(containerClient.GetAccessPolicy(options)); } { Blobs::SetBlobContainerAccessPolicyOptions options; options.AccessConditions.LeaseId = dummyLeaseId; - EXPECT_THROW(client.SetAccessPolicy(options), StorageException); + EXPECT_THROW(containerClient.SetAccessPolicy(options), StorageException); options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(client.SetAccessPolicy(options)); + EXPECT_NO_THROW(containerClient.SetAccessPolicy(options)); } { - EXPECT_THROW(client.Delete(), StorageException); + EXPECT_THROW(containerClient.Delete(), StorageException); Blobs::DeleteBlobContainerOptions options; options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(client.Delete(options)); + EXPECT_NO_THROW(containerClient.Delete(options)); } } TEST_F(BlobContainerClientTest, Tags) { - auto containerClient = GetBlobContainerTestClient(); - containerClient.Create(); + auto containerClient = *m_blobContainerClient; - std::string blobName = "blob" + m_containerName; + std::string blobName = "blob" + RandomString(); auto blobClient = containerClient.GetAppendBlobClient(blobName); blobClient.Create(); @@ -801,12 +833,12 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(downloadRet.Value.Details.TagCount.HasValue()); std::map tags; - std::string c1 = "k" + m_containerName + "1"; - std::string v1 = m_containerName + "2"; - std::string c2 = "k" + m_containerName + "3"; - std::string v2 = m_containerName + "4"; - std::string c3 = "k" + m_containerName + "5"; - std::string v3 = m_containerName + "6"; + std::string c1 = "k" + RandomString() + "1"; + std::string v1 = RandomString() + "2"; + std::string c2 = "k" + RandomString() + "3"; + std::string v2 = RandomString() + "4"; + std::string c3 = "k" + RandomString() + "5"; + std::string v3 = RandomString() + "6"; std::string c4 = "key3 +-./:=_"; std::string v4 = "v1 +-./:=_"; tags[c1] = v1; @@ -841,9 +873,6 @@ namespace Azure { namespace Storage { namespace Test { blobClient1.SetTags(tags); } - auto blobServiceClient = Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), - InitClientOptions()); std::string whereExpression = c1 + " = '" + v1 + "' AND " + c2 + " >= '" + v2 + "' AND " + c3 + " <= '" + v3 + "'"; std::vector findResults; @@ -867,7 +896,7 @@ namespace Azure { namespace Storage { namespace Test { findResults2.emplace_back(item.BlobName); } } - for (auto pageResult = blobServiceClient.FindBlobsByTags(whereExpression); + for (auto pageResult = m_blobContainerClient->FindBlobsByTags(whereExpression); pageResult.HasPage(); pageResult.MoveToNextPage()) { @@ -901,12 +930,11 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, AccessConditionTags) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; std::map tags; - std::string c1 = "k" + m_containerName + "1"; - std::string v1 = m_containerName + "2"; + std::string c1 = "k" + RandomString() + "1"; + std::string v1 = RandomString() + "2"; tags[c1] = v1; std::string successWhereExpression = c1 + " = '" + v1 + "'"; @@ -916,12 +944,8 @@ namespace Azure { namespace Storage { namespace Test { int64_t contentSize = static_cast(contentData.size()); auto content = Azure::Core::IO::MemoryBodyStream(contentData.data(), contentData.size()); - std::string blobName = m_containerName; - auto appendBlobClient = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - blobName, - InitClientOptions()); + std::string blobName = RandomString(); + auto appendBlobClient = m_blobContainerClient->GetAppendBlobClient(blobName); appendBlobClient.Create(); appendBlobClient.SetTags(tags); @@ -997,11 +1021,7 @@ namespace Azure { namespace Storage { namespace Test { std::string url = appendBlobClient.GetUrl() + GetSas(); Blobs::StartBlobCopyFromUriOptions options; - auto blobClient2 = Azure::Storage::Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - m_containerName + "blobClient2", - InitClientOptions()); + auto blobClient2 = m_blobContainerClient->GetAppendBlobClient(RandomString()); options.SourceAccessConditions.TagConditions = failWhereExpression; EXPECT_THROW(blobClient2.StartCopyFromUri(url, options), StorageException); options.SourceAccessConditions.TagConditions = successWhereExpression; @@ -1017,7 +1037,7 @@ namespace Azure { namespace Storage { namespace Test { } { - std::string leaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + std::string leaseId = RandomUUID(); Blobs::AcquireLeaseOptions options; options.AccessConditions.TagConditions = failWhereExpression; Blobs::BlobLeaseClient leaseClient(appendBlobClient, leaseId); @@ -1040,12 +1060,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(appendBlobClient.Delete(options3), StorageException); } - blobName = m_containerName; - auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - blobName + "blob", - InitClientOptions()); + auto pageBlobClient = m_blobContainerClient->GetPageBlobClient(RandomString()); pageBlobClient.Create(contentSize); pageBlobClient.SetTags(tags); @@ -1115,12 +1130,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(pageBlobClient.GetPageRanges(options)); } - blobName = m_containerName; - auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - blobName + "blockBlobClient", - InitClientOptions()); + auto blockBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); blockBlobClient.UploadFrom(contentData.data(), contentData.size()); blockBlobClient.SetTags(tags); @@ -1167,11 +1177,7 @@ namespace Azure { namespace Storage { namespace Test { } { - auto sourceBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - m_containerName + "sourceBlobClient", - InitClientOptions()); + auto sourceBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); std::vector buffer; buffer.resize(1024); sourceBlobClient.UploadFrom(buffer.data(), buffer.size()); @@ -1186,11 +1192,7 @@ namespace Azure { namespace Storage { namespace Test { } { - auto sourceBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - m_containerName + "sourceBlobClient2", - InitClientOptions()); + auto sourceBlobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); std::vector buffer; buffer.resize(1024); sourceBlobClient.UploadFrom(buffer.data(), buffer.size()); @@ -1207,8 +1209,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, SpecialBlobName) { - auto client = GetBlobContainerTestClient(); - client.CreateIfNotExists(); + auto containerClient = *m_blobContainerClient; const std::string non_ascii_word = "\xE6\xB5\x8B\xE8\xAF\x95"; const std::string encoded_non_ascii_word = "%E6%B5%8B%E8%AF%95"; @@ -1217,60 +1218,56 @@ namespace Azure { namespace Storage { namespace Test { const std::string baseBlobName = "a b c / !@#$%^&*(?/<>,.;:'\"[]{}|`~) def" + non_ascii_word; { - const std::string blobName = baseBlobName + m_containerName; - auto blobClient = client.GetAppendBlobClient(blobName); + const std::string blobName = baseBlobName + RandomString(); + auto blobClient = containerClient.GetAppendBlobClient(blobName); EXPECT_NO_THROW(blobClient.Create()); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } { - const std::string blobName = baseBlobName + m_containerName + "1"; - auto blobClient = client.GetPageBlobClient(blobName); + const std::string blobName = baseBlobName + RandomString() + "1"; + auto blobClient = containerClient.GetPageBlobClient(blobName); EXPECT_NO_THROW(blobClient.Create(1024)); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } { - const std::string blobName = baseBlobName + m_containerName + "2"; - auto blobClient = client.GetBlockBlobClient(blobName); + const std::string blobName = baseBlobName + RandomString() + "2"; + auto blobClient = containerClient.GetBlockBlobClient(blobName); EXPECT_NO_THROW(blobClient.UploadFrom(nullptr, 0)); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } - auto clientOptions = InitClientOptions(); { - const std::string blobName = baseBlobName + m_containerName + "3"; - auto blobClient = Blobs::AppendBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blobName, clientOptions); + const std::string blobName = baseBlobName + RandomString() + "3"; + auto blobClient = m_blobContainerClient->GetAppendBlobClient(blobName); EXPECT_NO_THROW(blobClient.Create()); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } { - const std::string blobName = baseBlobName + m_containerName + "4"; - auto blobClient = Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blobName, clientOptions); + const std::string blobName = baseBlobName + RandomString() + "4"; + auto blobClient = m_blobContainerClient->GetPageBlobClient(blobName); EXPECT_NO_THROW(blobClient.Create(1024)); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } { - const std::string blobName = baseBlobName + m_containerName + "5"; - auto blobClient = Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blobName, clientOptions); + const std::string blobName = baseBlobName + RandomString() + "5"; + auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); EXPECT_NO_THROW(blobClient.UploadFrom(nullptr, 0)); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); auto blobItem = GetBlobItem(blobName); EXPECT_EQ(blobItem.Name, blobName); } @@ -1282,15 +1279,14 @@ namespace Azure { namespace Storage { namespace Test { const std::string blobPrefix = std::string("aaaaa\xEF\xBF\xBF") + "bbb/"; // UTF-8 0xEF, 0xBF, 0xBF is UTF-16 0xFFFF const std::string blobName = blobPrefix + "ccc"; - auto blobClient = Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, blobName, clientOptions); + auto blobClient = m_blobContainerClient->GetBlockBlobClient(blobName); EXPECT_NO_THROW(blobClient.UploadFrom(nullptr, 0)); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); Blobs::Models::BlobItem blobItem; Blobs::ListBlobsOptions options; options.Prefix = "aaaaa"; - for (auto page = client.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) + for (auto page = containerClient.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) { for (auto& blob : page.Blobs) { @@ -1302,7 +1298,8 @@ namespace Azure { namespace Storage { namespace Test { } EXPECT_EQ(blobItem.Name, blobName); bool found = false; - for (auto page = client.ListBlobsByHierarchy("/"); page.HasPage(); page.MoveToNextPage()) + for (auto page = containerClient.ListBlobsByHierarchy("/"); page.HasPage(); + page.MoveToNextPage()) { for (auto& p : page.BlobPrefixes) { @@ -1318,23 +1315,21 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, QuestionMarkBlobName) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; std::string blobName = "?"; - auto blobClient = client.GetAppendBlobClient(blobName); + auto blobClient = containerClient.GetAppendBlobClient(blobName); EXPECT_NO_THROW(blobClient.Create()); auto blobUrl = blobClient.GetUrl(); - EXPECT_EQ(blobUrl, client.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); + EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); } TEST_F(BlobContainerClientTest, DeleteBlob) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; - std::string blobName = m_containerName; - auto blobClient = client.GetAppendBlobClient(blobName); + std::string blobName = RandomString(); + auto blobClient = containerClient.GetAppendBlobClient(blobName); blobClient.Create(); EXPECT_NO_THROW(blobClient.GetProperties()); blobClient.Delete(); @@ -1343,11 +1338,10 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, ListBlobsDeletedWithActiveVersions) { - auto client = GetBlobContainerTestClient(); - client.Create(); + auto containerClient = *m_blobContainerClient; - std::string blobName = "blob" + m_containerName; - auto blobClient = client.GetAppendBlobClient(blobName); + std::string blobName = "blob" + RandomString(); + auto blobClient = containerClient.GetAppendBlobClient(blobName); blobClient.Create(); auto blobItem diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.hpp index 740dab9449..3976f0b2f5 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.hpp @@ -3,54 +3,26 @@ #include -#include "test/ut/test_base.hpp" +#include "blob_service_client_test.hpp" namespace Azure { namespace Storage { namespace Test { - class BlobContainerClientTest : public StorageTest { - private: - std::shared_ptr m_blobContainerClient; - + class BlobContainerClientTest : public BlobServiceClientTest { protected: - std::string m_testName; - std::string m_containerName; + void SetUp() override; - virtual void SetUp() override { StorageTest::SetUp(); } - - virtual void TearDown() override - { - if (m_blobContainerClient) - { - m_blobContainerClient->Delete(); - } - - StorageTest::TearDown(); - } - - Azure::Storage::Blobs::BlobContainerClient const& GetBlobContainerTestClient( - std::string const& prefix = std::string("")) - { - // set the interceptor for the current test - if (m_testName.empty()) - { - // Internal state for the test name allows a test to create new container client with any - // name but just the first name is used as the test name for recordings. - m_testName = GetTestName(true); - m_testContext.RenameTest(m_testName); - } - m_containerName = prefix + GetContainerValidName(); - auto options = InitClientOptions(); - m_blobContainerClient = std::make_unique( - Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName, options)); - - return *m_blobContainerClient; - } + Blobs::BlobContainerClient GetBlobContainerClientForTest( + const std::string& containerName, + Blobs::BlobClientOptions clientOptions = Blobs::BlobClientOptions()); std::string GetSas(); Blobs::Models::BlobItem GetBlobItem( const std::string& blobName, Blobs::Models::ListBlobsIncludeFlags include = Blobs::Models::ListBlobsIncludeFlags::None); + + protected: + std::string m_containerName; + std::shared_ptr m_blobContainerClient; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp index cab73b98ef..c56f13e820 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -69,12 +69,11 @@ id,name,price "b25fY29sdW1ucyI6IFtdfQAYKmZhc3RwYXJxdWV0LXB5dGhvbiB2ZXJzaW9uIDAuOC4xIChidWlsZCAwKQDXAwAAUEFS" "MQ=="); - TEST_F(BlockBlobClientTest, QueryJsonInputCsvOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryJsonInputCsvOutput) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - client.UploadFrom( + blobClient.UploadFrom( reinterpret_cast(JsonQueryTestData.data()), JsonQueryTestData.size()); Blobs::QueryBlobOptions queryOptions; @@ -83,8 +82,8 @@ id,name,price { queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions(); - auto queryResponse - = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto queryResponse = blobClient.Query( + "SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); auto data = queryResponse.Value.BodyStream->ReadToEnd(); EXPECT_EQ( std::string(data.begin(), data.end()), @@ -98,8 +97,8 @@ id,name,price { queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateCsvTextOptions("|", ".", "[", "\\", true); - auto queryResponse - = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto queryResponse = blobClient.Query( + "SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); auto data = queryResponse.Value.BodyStream->ReadToEnd(); EXPECT_EQ( @@ -108,12 +107,11 @@ id,name,price } } - TEST_F(BlockBlobClientTest, QueryCsvInputJsonOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryCsvInputJsonOutput) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - client.UploadFrom( + blobClient.UploadFrom( reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); Blobs::QueryBlobOptions queryOptions; @@ -121,8 +119,8 @@ id,name,price = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions("\n", ",", "\"", "\\", true); queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions("|"); - auto queryResponse - = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto queryResponse = blobClient.Query( + "SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); auto data = queryResponse.Value.BodyStream->ReadToEnd(); EXPECT_EQ( @@ -130,12 +128,11 @@ id,name,price R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); } - TEST_F(BlockBlobClientTest, QueryCsvInputArrowOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryCsvInputArrowOutput) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - client.UploadFrom( + blobClient.UploadFrom( reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); Blobs::QueryBlobOptions queryOptions; @@ -156,8 +153,8 @@ id,name,price fields.push_back(field); queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); - auto queryResponse - = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto queryResponse = blobClient.Query( + "SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); auto data = queryResponse.Value.BodyStream->ReadToEnd(); const auto expectedData = Core::Convert::Base64Decode( @@ -180,12 +177,11 @@ id,name,price EXPECT_EQ(data, expectedData); } - TEST_F(BlockBlobClientTest, DISABLED_QueryParquetInputArrowOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryParquetInputArrowOutput) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - client.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); + blobClient.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); Blobs::QueryBlobOptions queryOptions; queryOptions.InputTextConfiguration @@ -203,8 +199,8 @@ id,name,price fields.push_back(field); queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); - auto queryResponse - = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto queryResponse = blobClient.Query( + "SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); auto data = queryResponse.Value.BodyStream->ReadToEnd(); const auto expectedData = Core::Convert::Base64Decode( "/////" @@ -239,10 +235,9 @@ id,name,price EXPECT_EQ(data, expectedData); } - TEST_F(BlockBlobClientTest, QueryWithError_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryWithError) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; const std::string malformedData = R"json( @@ -253,13 +248,15 @@ id,name,price {"id": 104, "name": "clementines", "price": 399} xx )json"; - client.UploadFrom(reinterpret_cast(malformedData.data()), malformedData.size()); + blobClient.UploadFrom( + reinterpret_cast(malformedData.data()), malformedData.size()); Blobs::QueryBlobOptions queryOptions; queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateJsonTextOptions(); queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); - auto queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + auto queryResponse + = blobClient.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); try { @@ -296,7 +293,7 @@ xx ++numNonFatalErrors; } }; - queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + queryResponse = blobClient.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); queryResponse.Value.BodyStream->ReadToEnd(); EXPECT_EQ(numNonFatalErrors, 2); @@ -304,22 +301,20 @@ xx EXPECT_TRUE(progressCallbackCalled); } - TEST_F(BlockBlobClientTest, QueryDefaultInputOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryDefaultInputOutput) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; const std::string csvData = "100,oranges,100"; - client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); - auto queryResponse = client.Query("SELECT * from BlobStorage;"); + blobClient.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + auto queryResponse = blobClient.Query("SELECT * from BlobStorage;"); auto data = queryResponse.Value.BodyStream->ReadToEnd(); } TEST_F(BlockBlobClientTest, QueryLargeBlob_LIVEONLY_) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; constexpr size_t DataSize = static_cast(32_MB); @@ -334,13 +329,13 @@ xx jsonData += "{\"_1\":\"" + counter + "\",\"_2\":\"" + record + "\"}\n"; } - client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + blobClient.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); Blobs::QueryBlobOptions queryOptions; queryOptions.InputTextConfiguration = Blobs::BlobQueryInputTextOptions::CreateCsvTextOptions(); queryOptions.OutputTextConfiguration = Blobs::BlobQueryOutputTextOptions::CreateJsonTextOptions(); - auto queryResponse = client.Query("SELECT * FROM BlobStorage;", queryOptions); + auto queryResponse = blobClient.Query("SELECT * FROM BlobStorage;", queryOptions); size_t comparePos = 0; std::vector readBuffer(4096); @@ -358,80 +353,77 @@ xx } } - TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLeaseId_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLeaseId) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - client.UploadFrom(nullptr, 0); + auto blobClient = *m_blockBlobClient; + + blobClient.UploadFrom(nullptr, 0); - Blobs::BlobLeaseClient leaseClient(client, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + Blobs::BlobLeaseClient leaseClient(blobClient, RandomUUID()); leaseClient.Acquire(Blobs::BlobLeaseClient::InfiniteLeaseDuration); Blobs::QueryBlobOptions queryOptions; - queryOptions.AccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + queryOptions.AccessConditions.LeaseId = RandomUUID(); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); queryOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); } - TEST_F(BlockBlobClientTest, QueryBlobAccessConditionTags_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionTags) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - client.UploadFrom(nullptr, 0); + auto blobClient = *m_blockBlobClient; + blobClient.UploadFrom(nullptr, 0); std::map tags = {{"k1", "value1"}}; - client.SetTags(tags); + blobClient.SetTags(tags); Blobs::QueryBlobOptions queryOptions; queryOptions.AccessConditions.TagConditions = "k1 = 'value1'"; - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); queryOptions.AccessConditions.TagConditions = "k1 = 'dummy'"; - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); } - TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLastModifiedTime_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionLastModifiedTime) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - client.UploadFrom(nullptr, 0); + auto blobClient = *m_blockBlobClient; + blobClient.UploadFrom(nullptr, 0); - auto lastModifiedTime = client.GetProperties().Value.LastModified; + auto lastModifiedTime = blobClient.GetProperties().Value.LastModified; auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); Blobs::QueryBlobOptions queryOptions; queryOptions.AccessConditions.IfModifiedSince = timeBeforeStr; - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); queryOptions.AccessConditions.IfModifiedSince = timeAfterStr; - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); queryOptions = Blobs::QueryBlobOptions(); queryOptions.AccessConditions.IfUnmodifiedSince = timeBeforeStr; - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); queryOptions.AccessConditions.IfUnmodifiedSince = timeAfterStr; - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); } - TEST_F(BlockBlobClientTest, QueryBlobAccessConditionETag_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryBlobAccessConditionETag) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - client.UploadFrom(nullptr, 0); + auto blobClient = *m_blockBlobClient; + blobClient.UploadFrom(nullptr, 0); - auto etag = client.GetProperties().Value.ETag; + auto etag = blobClient.GetProperties().Value.ETag; Blobs::QueryBlobOptions queryOptions; queryOptions.AccessConditions.IfMatch = etag; - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); queryOptions.AccessConditions.IfMatch = DummyETag; - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); queryOptions = Blobs::QueryBlobOptions(); queryOptions.AccessConditions.IfNoneMatch = DummyETag; - EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + EXPECT_NO_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions)); queryOptions.AccessConditions.IfNoneMatch = etag; - EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + EXPECT_THROW(blobClient.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_sas_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_sas_test.cpp index b9f8d3e6f9..f82e26b408 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_sas_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_sas_test.cpp @@ -12,9 +12,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobContainerClientTest, BlobSasTest_LIVEONLY_) { - CHECK_SKIP_TEST(); - - auto blobContainerClient = GetBlobContainerTestClient(); + auto blobContainerClient = GetBlobContainerClientForTest(LowercaseRandomString()); blobContainerClient.CreateIfNotExists(); auto sasStartsOn = std::chrono::system_clock::now() - std::chrono::minutes(5); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp index b2d28d8075..dbb2d17965 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp @@ -6,7 +6,7 @@ #include #include -#include "test/ut/test_base.hpp" +#include "blob_service_client_test.hpp" namespace Azure { namespace Storage { namespace Blobs { namespace Models { @@ -70,49 +70,13 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { - class BlobServiceClientTest : public Azure::Storage::Test::StorageTest { - - private: - std::unique_ptr m_client; - - protected: - // Required to rename the test propertly once the test is started. - // We can only know the test instance name until the test instance is run. - Azure::Storage::Blobs::BlobServiceClient const& GetClientForTest(std::string const& testName) - { - // set the interceptor for the current test - m_testContext.RenameTest(testName); - return *m_client; - } - - Azure::Storage::Blobs::BlobServiceClient const& GetDataLakeClientService() - { - // set the interceptor for the current test - auto options = InitClientOptions(); - m_client = std::make_unique( - Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), options)); - return *m_client; - } - - virtual void SetUp() override - { - StorageTest::SetUp(); - - auto options = InitClientOptions(); - m_client = std::make_unique( - Azure::Storage::Blobs::BlobServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), options)); - } - }; - TEST_F(BlobServiceClientTest, ListContainers) { - const std::string testName = GetTestNameLowerCase(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - const std::string prefix1 = testName + "-prefix1-"; - const std::string prefix2 = testName + "-prefix2-"; + const std::string prefix = LowercaseRandomString(); + const std::string prefix1 = prefix + "-prefix1-"; + const std::string prefix2 = prefix + "-prefix2-"; std::set p1Containers; std::set p2Containers; @@ -121,7 +85,7 @@ namespace Azure { namespace Storage { namespace Test { for (int i = 0; i < 5; ++i) { std::string containerName = prefix1 + std::to_string(i); - auto containerClient = client.GetBlobContainerClient(containerName); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); containerClient.Create(); p1Containers.insert(containerName); p1p2Containers.insert(containerName); @@ -129,7 +93,7 @@ namespace Azure { namespace Storage { namespace Test { for (int i = 0; i < 5; ++i) { std::string containerName = prefix2 + std::to_string(i); - auto containerClient = client.GetBlobContainerClient(containerName); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); containerClient.Create(); p2Containers.insert(containerName); p1p2Containers.insert(containerName); @@ -138,7 +102,7 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::ListBlobContainersOptions options; options.PageSizeHint = 4; std::set listContainers; - for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + for (auto pageResult = serviceClient.ListBlobContainers(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { EXPECT_FALSE(pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); @@ -160,7 +124,7 @@ namespace Azure { namespace Storage { namespace Test { // List with prefix options.Prefix = prefix1; listContainers.clear(); - for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + for (auto pageResult = serviceClient.ListBlobContainers(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { EXPECT_FALSE(pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); @@ -188,19 +152,18 @@ namespace Azure { namespace Storage { namespace Test { // Remove all containers for (const auto& container : p1p2Containers) { - auto container_client = client.GetBlobContainerClient(container); + auto container_client = serviceClient.GetBlobContainerClient(container); container_client.Delete(); } } TEST_F(BlobServiceClientTest, ListSystemContainers) { - const std::string testName = GetTestNameLowerCase(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; Azure::Storage::Blobs::ListBlobContainersOptions options; options.Include = Blobs::Models::ListBlobContainersIncludeFlags::System; std::vector containers; - for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + for (auto pageResult = serviceClient.ListBlobContainers(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { for (const auto& c : pageResult.BlobContainers) @@ -217,10 +180,9 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobServiceClientTest, GetProperties) { - const std::string testName = GetTestName(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - auto ret = client.GetProperties(); + auto ret = serviceClient.GetProperties(); auto properties = ret.Value; auto logging = properties.Logging; EXPECT_FALSE(logging.Version.empty()); @@ -255,10 +217,9 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobServiceClientTest, SetProperties) { - const std::string testName = GetTestName(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - auto getServicePropertiesResult = client.GetProperties().Value; + auto getServicePropertiesResult = serviceClient.GetProperties().Value; Blobs::Models::BlobServiceProperties properties; properties.Logging = getServicePropertiesResult.Logging; properties.HourMetrics = getServicePropertiesResult.HourMetrics; @@ -311,12 +272,12 @@ namespace Azure { namespace Storage { namespace Test { properties.DeleteRetentionPolicy.IsEnabled = true; properties.DeleteRetentionPolicy.Days = 7; - EXPECT_NO_THROW(client.SetProperties(properties)); + EXPECT_NO_THROW(serviceClient.SetProperties(properties)); // It takes some time before the new properties comes into effect. using namespace std::chrono_literals; TestSleep(10s); - auto downloadedProperties = client.GetProperties().Value; + auto downloadedProperties = serviceClient.GetProperties().Value; EXPECT_EQ(downloadedProperties.Logging.Version, properties.Logging.Version); EXPECT_EQ(downloadedProperties.Logging.Delete, properties.Logging.Delete); EXPECT_EQ(downloadedProperties.Logging.Read, properties.Logging.Read); @@ -369,36 +330,30 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(downloadedProperties.DeleteRetentionPolicy, properties.DeleteRetentionPolicy); - auto res = client.SetProperties(originalProperties); + auto res = serviceClient.SetProperties(originalProperties); } TEST_F(BlobServiceClientTest, AccountInfo) { - const std::string testName = GetTestName(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - auto accountInfo = client.GetAccountInfo().Value; + auto accountInfo = serviceClient.GetAccountInfo().Value; EXPECT_FALSE(accountInfo.SkuName.ToString().empty()); EXPECT_FALSE(accountInfo.AccountKind.ToString().empty()); EXPECT_FALSE(accountInfo.IsHierarchicalNamespaceEnabled); - - auto dataLakeServiceClient = GetDataLakeClientService(); - accountInfo = dataLakeServiceClient.GetAccountInfo().Value; - EXPECT_TRUE(accountInfo.IsHierarchicalNamespaceEnabled); } TEST_F(BlobServiceClientTest, Statistics) { - const std::string testName = GetTestName(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - EXPECT_THROW(client.GetStatistics(), StorageException); + EXPECT_THROW(serviceClient.GetStatistics(), StorageException); auto keyCredential = _internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential; auto secondaryServiceClient = Blobs::BlobServiceClient( - InferSecondaryUrl(client.GetUrl()), + InferSecondaryUrl(serviceClient.GetUrl()), keyCredential, InitClientOptions()); @@ -412,24 +367,22 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobServiceClientTest, CreateDeleteBlobContainer) { - const std::string testName = GetTestNameLowerCase(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - std::string containerName = testName; - auto containerClient = client.CreateBlobContainer(containerName); + const std::string containerName = LowercaseRandomString(); + auto containerClient = serviceClient.CreateBlobContainer(containerName); EXPECT_NO_THROW(containerClient.Value.GetProperties()); - client.DeleteBlobContainer(containerName); + serviceClient.DeleteBlobContainer(containerName); EXPECT_THROW(containerClient.Value.GetProperties(), StorageException); } TEST_F(BlobServiceClientTest, UndeleteBlobContainer) { - const std::string testName = GetTestNameLowerCase(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; - std::string containerName = testName; - auto containerClient = client.GetBlobContainerClient(containerName); + const std::string containerName = LowercaseRandomString(); + auto containerClient = serviceClient.GetBlobContainerClient(containerName); containerClient.Create(); containerClient.Delete(); @@ -438,7 +391,7 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::ListBlobContainersOptions options; options.Prefix = containerName; options.Include = Blobs::Models::ListBlobContainersIncludeFlags::Deleted; - for (auto pageResult = client.ListBlobContainers(options); pageResult.HasPage(); + for (auto pageResult = serviceClient.ListBlobContainers(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { for (const auto& container : pageResult.BlobContainers) @@ -464,7 +417,7 @@ namespace Azure { namespace Storage { namespace Test { { try { - client.UndeleteBlobContainer( + serviceClient.UndeleteBlobContainer( deletedContainerItem.Name, deletedContainerItem.VersionId.Value()); break; } @@ -482,12 +435,12 @@ namespace Azure { namespace Storage { namespace Test { } } EXPECT_NO_THROW(containerClient.GetProperties()); + containerClient.DeleteIfExists(); } - TEST_F(BlobServiceClientTest, UserDelegationKey) + TEST_F(BlobServiceClientTest, UserDelegationKey_LIVEONLY_) { - const std::string testName = GetTestName(); - auto client = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; auto sasExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(60); @@ -495,11 +448,11 @@ namespace Azure { namespace Storage { namespace Test { = std::make_shared( AadTenantId(), AadClientId(), AadClientSecret()); Blobs::BlobClientOptions options; + InitClientOptions(options); - auto blobServiceClient1 = InitTestClient( - client.GetUrl(), credential, options); + auto blobServiceClient1 = Blobs::BlobServiceClient(serviceClient.GetUrl(), credential, options); - auto userDelegationKey = blobServiceClient1->GetUserDelegationKey(sasExpiresOn).Value; + auto userDelegationKey = blobServiceClient1.GetUserDelegationKey(sasExpiresOn).Value; EXPECT_FALSE(userDelegationKey.SignedObjectId.empty()); EXPECT_FALSE(userDelegationKey.SignedTenantId.empty()); @@ -512,13 +465,13 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlobServiceClientTest, DISABLED_RenameBlobContainer) { - const std::string testName = GetTestNameLowerCase(); - auto serviceClient = GetClientForTest(testName); + auto serviceClient = *m_blobServiceClient; + const std::string prefix = RandomString(); - const std::string srcContainerName = testName + "src"; + const std::string srcContainerName = prefix + "src"; auto srcContainerClient = serviceClient.CreateBlobContainer(srcContainerName).Value; - const std::string destContainerName = testName + "dest1"; + const std::string destContainerName = prefix + "dest1"; auto destContainerClient = serviceClient.RenameBlobContainer(srcContainerName, destContainerName).Value; @@ -529,7 +482,7 @@ namespace Azure { namespace Storage { namespace Test { destContainerClient, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); leaseClient.Acquire(std::chrono::seconds(60)); - const std::string destContainerName2 = testName + "dest2"; + const std::string destContainerName2 = prefix + "dest2"; Blobs::RenameBlobContainerOptions renameOptions; renameOptions.SourceAccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); EXPECT_THROW( diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp new file mode 100644 index 0000000000..dc1eb88781 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include + +#include "test/ut/test_base.hpp" + +namespace Azure { namespace Storage { namespace Test { + + class BlobServiceClientTest : public StorageTest { + protected: + std::shared_ptr m_blobServiceClient; + + void SetUp() override + { + StorageTest::SetUp(); + + auto options = InitClientOptions(); + m_blobServiceClient = std::make_shared( + Blobs::BlobServiceClient::CreateFromConnectionString( + StandardStorageConnectionString(), options)); + } + }; + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index c27d71df4c..ec2954dd7c 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -30,22 +30,39 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { - void BlockBlobClientTest::SetUp() { BlobContainerClientTest::SetUp(); } - - void BlockBlobClientTest::TearDown() + void BlockBlobClientTest::SetUp() { - // Deleting the container with any blobs in it - BlobContainerClientTest::TearDown(); + BlobContainerClientTest::SetUp(); + if (shouldSkipTest()) + { + return; + } + m_blobName = RandomString(); + m_blockBlobClient = std::make_shared( + m_blobContainerClient->GetBlockBlobClient(m_blobName)); + m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"key2", "Value2"}}; + m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; + m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; + m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; + m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; + m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; + m_blobUploadOptions.HttpHeaders.ContentHash.Value.clear(); + m_blobUploadOptions.AccessTier = Azure::Storage::Blobs::Models::AccessTier::Hot; + m_blobContent = std::vector(static_cast(1_KB), 'x'); + auto blobContent + = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); + m_blockBlobClient->Upload(blobContent, m_blobUploadOptions); + m_blobUploadOptions.HttpHeaders.ContentHash + = m_blockBlobClient->GetProperties().Value.HttpHeaders.ContentHash; } TEST_F(BlockBlobClientTest, CreateDelete) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; auto blobContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - auto blobContentInfo = client.Upload(blobContent, m_blobUploadOptions); + auto blobContentInfo = blobClient.Upload(blobContent, m_blobUploadOptions); EXPECT_TRUE(blobContentInfo.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(blobContentInfo.Value.LastModified)); EXPECT_TRUE(blobContentInfo.Value.VersionId.HasValue()); @@ -53,26 +70,25 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_FALSE(blobContentInfo.Value.EncryptionKeySha256.HasValue()); - client.Delete(); - EXPECT_THROW(client.Delete(), StorageException); + blobClient.Delete(); + EXPECT_THROW(blobClient.Delete(), StorageException); } TEST_F(BlockBlobClientTest, SoftDelete) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + const std::string blobName = m_blobName; + auto blobClient = *m_blockBlobClient; - const std::string blobName(testName); std::vector emptyContent; auto blobContent = Azure::Core::IO::MemoryBodyStream(emptyContent.data(), emptyContent.size()); - client.Upload(blobContent); + blobClient.Upload(blobContent); auto blobItem = GetBlobItem(blobName); EXPECT_FALSE(blobItem.IsDeleted); EXPECT_FALSE(blobItem.Details.DeletedOn.HasValue()); EXPECT_FALSE(blobItem.Details.RemainingRetentionDays.HasValue()); - client.Delete(); + blobClient.Delete(); /* // Soft delete doesn't work in storage account with versioning enabled. @@ -84,14 +100,12 @@ namespace Azure { namespace Storage { namespace Test { */ } - // small default 1Kb upload/download TEST_F(BlockBlobClientTest, SmallUploadDownload) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(); + // small default 1Kb upload/download + auto blobClient = *m_blockBlobClient; - auto res = client.Download(); + auto res = blobClient.Download(); EXPECT_EQ(res.Value.BlobSize, static_cast(m_blobContent.size())); EXPECT_EQ(res.Value.ContentRange.Offset, 0); EXPECT_EQ(res.Value.ContentRange.Length.Value(), static_cast(m_blobContent.size())); @@ -107,7 +121,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); Azure::Storage::Blobs::DownloadBlobOptions options; options.Range = {100, 200}; - res = client.Download(options); + res = blobClient.Download(options); EXPECT_EQ( ReadBodyStream(res.Value.BodyStream), std::vector( @@ -120,15 +134,22 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.Value.BlobSize, static_cast(m_blobContent.size())); } - // big 8Mb upload/download should be LIVE only to avoid big recording files - TEST_F(BlockBlobClientTest, UploadDownload_LIVEONLY_) + TEST_F(BlockBlobClientTest, UploadDownload) { - CHECK_SKIP_TEST(); - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + auto blobClient = *m_blockBlobClient; + m_blobContent = RandomBuffer(100); + { + Azure::Core::Cryptography::Md5Hash md5hash; + md5hash.Append(m_blobContent.data(), m_blobContent.size()); + m_blobUploadOptions.HttpHeaders.ContentHash.Value = md5hash.Final(); + Blobs::UploadBlockBlobOptions options; + options.HttpHeaders = m_blobUploadOptions.HttpHeaders; + options.Metadata = m_blobUploadOptions.Metadata; + Core::IO::MemoryBodyStream bodyStream(m_blobContent.data(), m_blobContent.size()); + blobClient.Upload(bodyStream, options); + } - auto res = client.Download(); + auto res = blobClient.Download(); EXPECT_EQ(res.Value.BlobSize, static_cast(m_blobContent.size())); EXPECT_EQ(res.Value.ContentRange.Offset, 0); EXPECT_EQ(res.Value.ContentRange.Length.Value(), static_cast(m_blobContent.size())); @@ -143,8 +164,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); Azure::Storage::Blobs::DownloadBlobOptions options; - options.Range = {1_MB, 2_MB}; - res = client.Download(options); + options.Range = {10, 20}; + res = blobClient.Download(options); EXPECT_EQ( ReadBodyStream(res.Value.BodyStream), std::vector( @@ -159,22 +180,21 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, UploadWithTags) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; std::map tags; tags["key1"] = "value1"; tags["key2"] = "value2"; tags["key3 +-./:=_"] = "v1 +-./:=_"; - std::vector blobContent(100, 'a'); + std::vector blobContent(10, 'a'); { Blobs::UploadBlockBlobOptions options; options.Tags = tags; auto stream = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); - client.Upload(stream, options); - EXPECT_EQ(client.GetTags().Value, tags); - client.Delete(); + blobClient.Upload(stream, options); + EXPECT_EQ(blobClient.GetTags().Value, tags); + blobClient.Delete(); } { @@ -184,27 +204,26 @@ namespace Azure { namespace Storage { namespace Test { options.Tags = tags; { - client.UploadFrom(blobContent.data(), blobContent.size(), options); - EXPECT_EQ(client.GetTags().Value, tags); - client.Delete(); + blobClient.UploadFrom(blobContent.data(), blobContent.size(), options); + EXPECT_EQ(blobClient.GetTags().Value, tags); + blobClient.Delete(); } { - const std::string tempFilename = "file" + testName; + const std::string tempFilename = "file" + RandomString(); WriteFile(tempFilename, blobContent); - client.UploadFrom(tempFilename, options); - EXPECT_EQ(client.GetTags().Value, tags); - client.Delete(); + blobClient.UploadFrom(tempFilename, options); + EXPECT_EQ(blobClient.GetTags().Value, tags); + blobClient.Delete(); } } } TEST_F(BlockBlobClientTest, DownloadTransactionalHash) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - const std::vector dataPart1(static_cast(4_MB + 1), 'a'); - const std::vector dataPart2(static_cast(4_MB + 1), 'b'); + const std::vector dataPart1 = RandomBuffer(10); + const std::vector dataPart2 = RandomBuffer(20); const std::string blockId1 = Base64EncodeText("0"); const std::string blockId2 = Base64EncodeText("1"); @@ -285,8 +304,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, DISABLED_LastAccessTime) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; { auto res = blobClient.Download(); @@ -299,22 +317,21 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(res.Value.LastAccessedOn.Value())); } { - EXPECT_TRUE(IsValidTime(GetBlobItem(testName).Details.LastAccessedOn.Value())); + EXPECT_TRUE(IsValidTime(GetBlobItem(m_blobName).Details.LastAccessedOn.Value())); } } TEST_F(BlockBlobClientTest, DownloadEmpty) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; std::vector emptyContent; auto blobContent = Azure::Core::IO::MemoryBodyStream(emptyContent.data(), emptyContent.size()); - blockBlobClient.Upload(blobContent); - blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); - blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); + blobClient.Upload(blobContent); + blobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); + blobClient.SetMetadata(m_blobUploadOptions.Metadata); - auto res = blockBlobClient.Download(); + auto res = blobClient.Download(); EXPECT_EQ(res.Value.BodyStream->Length(), 0); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); @@ -325,39 +342,38 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - std::string tempFilename = testName; - EXPECT_NO_THROW(blockBlobClient.DownloadTo(tempFilename)); + std::string tempFilename = RandomString(); + EXPECT_NO_THROW(blobClient.DownloadTo(tempFilename)); EXPECT_TRUE(ReadFile(tempFilename).empty()); DeleteFile(tempFilename); std::vector buff; - EXPECT_NO_THROW(blockBlobClient.DownloadTo(buff.data(), 0)); + EXPECT_NO_THROW(blobClient.DownloadTo(buff.data(), 0)); Azure::Storage::Blobs::DownloadBlobOptions options; options.Range = Core::Http::HttpRange(); options.Range.Value().Offset = 0; - EXPECT_THROW(blockBlobClient.Download(options), StorageException); + EXPECT_THROW(blobClient.Download(options), StorageException); options.Range.Value().Length = 1; - EXPECT_THROW(blockBlobClient.Download(options), StorageException); + EXPECT_THROW(blobClient.Download(options), StorageException); } TEST_F(BlockBlobClientTest, SyncCopyFromUri) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + auto sourceBlobClient = m_blobContainerClient->GetBlockBlobClient("source" + RandomString()); + sourceBlobClient.UploadFrom(m_blobContent.data(), m_blobContent.size()); - const std::string blobName = testName + "blob"; - auto blobClient = GetBlobClient(blobName); + const std::string blobName = "dest" + RandomString(); + auto destBlobClient = m_blobContainerClient->GetBlockBlobClient(blobName); - auto res = blobClient->CopyFromUri(blockBlobClient.GetUrl() + GetSas()); + auto res = destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas()); EXPECT_EQ(res.RawResponse->GetStatusCode(), Azure::Core::Http::HttpStatusCode::Accepted); EXPECT_TRUE(res.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(res.Value.LastModified)); EXPECT_FALSE(res.Value.CopyId.empty()); EXPECT_EQ(res.Value.CopyStatus, Azure::Storage::Blobs::Models::CopyStatus::Success); - auto downloadResult = blobClient->Download(); + auto downloadResult = destBlobClient.Download(); EXPECT_FALSE(downloadResult.Value.Details.CopyId.Value().empty()); EXPECT_FALSE(downloadResult.Value.Details.CopySource.Value().empty()); EXPECT_TRUE( @@ -380,13 +396,12 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, SyncCopyFromUriEncryptionScope) { - auto clientOptions = InitClientOptions(); + Blobs::BlobClientOptions clientOptions; const auto encryptionScope = GetTestEncryptionScope(); clientOptions.EncryptionScope = encryptionScope; - const auto containerName = GetContainerValidName(); + const auto containerName = LowercaseRandomString(); const auto blobName = "b"; - auto containerClient = Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), containerName, clientOptions); + auto containerClient = GetBlobContainerClientForTest(containerName, clientOptions); containerClient.CreateIfNotExists(); auto srcBlobClient = containerClient.GetBlockBlobClient(blobName); uint8_t data; @@ -408,14 +423,14 @@ namespace Azure { namespace Storage { namespace Test { = _internal::ParseConnectionString(StandardStorageConnectionString()).KeyCredential; auto sasToken = builder.GenerateSasToken(*keyCredential); - auto destBlobClient = GetBlockBlobClient(GetTestName()); + auto destBlobClient = *m_blockBlobClient; auto response = destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + sasToken); EXPECT_FALSE(response.Value.EncryptionScope.HasValue()); properties = destBlobClient.GetProperties().Value; EXPECT_FALSE(properties.EncryptionScope.HasValue()); - destBlobClient = containerClient.GetBlockBlobClient(GetTestName()); - response = destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas()); + destBlobClient = containerClient.GetBlockBlobClient(RandomString()); + response = destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + sasToken); ASSERT_TRUE(response.Value.EncryptionScope.HasValue()); EXPECT_EQ(response.Value.EncryptionScope.Value(), encryptionScope); properties = destBlobClient.GetProperties().Value; @@ -426,18 +441,15 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, AsyncCopyFromUri) { + auto sourceBlobClient = *m_blockBlobClient; - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + const std::string blobName = RandomString(); + auto destBlobClient = GetBlockBlobClientForTest(blobName); - const std::string blobName = testName + "blob"; - auto blobClient = GetBlobClient(blobName); - - auto res = blobClient->StartCopyFromUri(blockBlobClient.GetUrl()); + auto res = destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl()); EXPECT_EQ(res.GetRawResponse().GetStatusCode(), Azure::Core::Http::HttpStatusCode::Accepted); res.PollUntilDone(PollInterval()); - auto properties = blobClient->GetProperties().Value; + auto properties = destBlobClient.GetProperties().Value; EXPECT_FALSE(properties.CopyId.Value().empty()); EXPECT_FALSE(properties.CopySource.Value().empty()); EXPECT_TRUE( @@ -448,7 +460,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(properties.IsIncrementalCopy.Value()); EXPECT_FALSE(properties.IncrementalCopyDestinationSnapshot.HasValue()); - auto downloadResult = blobClient->Download(); + auto downloadResult = destBlobClient.Download(); EXPECT_FALSE(downloadResult.Value.Details.CopyId.Value().empty()); EXPECT_FALSE(downloadResult.Value.Details.CopySource.Value().empty()); EXPECT_TRUE( @@ -471,12 +483,10 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, CopyWithTagsMetadataTier) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + auto sourceBlobClient = *m_blockBlobClient; - const std::string blobName = testName + "blob"; - auto blobClient = GetBlobClient(blobName); + const std::string blobName = "dest" + RandomString(); + auto destBlobClient = GetBlockBlobClientForTest(blobName); Blobs::StartBlobCopyFromUriOptions options; options.Tags["key1"] = "value1"; @@ -485,10 +495,10 @@ namespace Azure { namespace Storage { namespace Test { options.Metadata["key1"] = "value1"; options.Metadata["key2"] = "value2"; options.AccessTier = Blobs::Models::AccessTier::Cool; - auto operation = blobClient->StartCopyFromUri(blockBlobClient.GetUrl(), options); + auto operation = destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options); operation.PollUntilDone(std::chrono::seconds(1)); - EXPECT_EQ(blobClient->GetTags().Value, options.Tags); - auto properties = blobClient->GetProperties().Value; + EXPECT_EQ(destBlobClient.GetTags().Value, options.Tags); + auto properties = destBlobClient.GetProperties().Value; EXPECT_EQ(properties.Metadata, options.Metadata); EXPECT_EQ(properties.AccessTier.Value(), options.AccessTier.Value()); @@ -496,26 +506,23 @@ namespace Azure { namespace Storage { namespace Test { options2.Tags = options.Tags; options2.Metadata = options.Metadata; options2.AccessTier = options.AccessTier; - blobClient->CopyFromUri(blockBlobClient.GetUrl() + GetSas(), options2); - EXPECT_EQ(blobClient->GetTags().Value, options2.Tags); - properties = blobClient->GetProperties().Value; + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2); + EXPECT_EQ(destBlobClient.GetTags().Value, options2.Tags); + properties = destBlobClient.GetProperties().Value; EXPECT_EQ(properties.Metadata, options2.Metadata); EXPECT_EQ(properties.AccessTier.Value(), options2.AccessTier.Value()); options2.CopySourceTagsMode = Blobs::Models::BlobCopySourceTagsMode::Copy; options2.Tags.clear(); - blobClient->CopyFromUri(blockBlobClient.GetUrl() + GetSas(), options2); - EXPECT_TRUE(blobClient->GetTags().Value.empty()); + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2); + EXPECT_TRUE(destBlobClient.GetTags().Value.empty()); } TEST_F(BlockBlobClientTest, SnapShotVersions) { + auto blobClient = *m_blockBlobClient; - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); - - auto res = blockBlobClient.CreateSnapshot(); + auto res = blobClient.CreateSnapshot(); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); @@ -524,11 +531,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res.Value.Snapshot.empty()); EXPECT_TRUE(res.Value.VersionId.HasValue()); EXPECT_FALSE(res.Value.VersionId.Value().empty()); - auto snapshotClient = blockBlobClient.WithSnapshot(res.Value.Snapshot); + auto snapshotClient = blobClient.WithSnapshot(res.Value.Snapshot); EXPECT_EQ(ReadBodyStream(snapshotClient.Download().Value.BodyStream), m_blobContent); EXPECT_EQ(snapshotClient.GetProperties().Value.Metadata, m_blobUploadOptions.Metadata); EXPECT_TRUE(snapshotClient.GetProperties().Value.IsServerEncrypted); - auto versionClient = blockBlobClient.WithVersionId(res.Value.VersionId.Value()); + auto versionClient = blobClient.WithVersionId(res.Value.VersionId.Value()); EXPECT_EQ(ReadBodyStream(versionClient.Download().Value.BodyStream), m_blobContent); EXPECT_EQ(versionClient.GetProperties().Value.Metadata, m_blobUploadOptions.Metadata); EXPECT_TRUE(versionClient.GetProperties().Value.IsServerEncrypted); @@ -548,21 +555,20 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Blobs::CreateBlobSnapshotOptions options; options.Metadata = {{"snapshotkey1", "snapshotvalue1"}, {"snapshotkey2", "SNAPSHOTVALUE2"}}; - res = blockBlobClient.CreateSnapshot(options); + res = blobClient.CreateSnapshot(options); EXPECT_FALSE(res.Value.Snapshot.empty()); - auto snapshotClient2 = blockBlobClient.WithSnapshot(res.Value.Snapshot); + auto snapshotClient2 = blobClient.WithSnapshot(res.Value.Snapshot); EXPECT_EQ(snapshotClient2.GetProperties().Value.Metadata, options.Metadata); EXPECT_NO_THROW(snapshotClient.Delete()); EXPECT_NO_THROW(snapshotClient2.Delete()); EXPECT_NO_THROW(versionClient.Delete()); - EXPECT_NO_THROW(blockBlobClient.GetProperties()); + EXPECT_NO_THROW(blobClient.GetProperties()); } TEST_F(BlockBlobClientTest, IsCurrentVersion) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; std::vector emptyContent; blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); @@ -600,7 +606,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(downloadResponse.Value.Details.IsCurrentVersion.Value()); EXPECT_EQ(version1, downloadResponse.Value.Details.VersionId.Value()); - auto blobItem = GetBlobItem(testName, Blobs::Models::ListBlobsIncludeFlags::Versions); + auto blobItem = GetBlobItem(m_blobName, Blobs::Models::ListBlobsIncludeFlags::Versions); ASSERT_TRUE(blobItem.VersionId.HasValue()); ASSERT_TRUE(blobItem.IsCurrentVersion.HasValue()); if (blobItem.VersionId.Value() == latestVersion) @@ -615,17 +621,11 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, Properties) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); + auto blobClient = *m_blockBlobClient; - auto blobContent - = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blockBlobClient.Upload(blobContent); - blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); - blockBlobClient.SetAccessTier(Azure::Storage::Blobs::Models::AccessTier::Cool); - blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); + blobClient.SetAccessTier(Azure::Storage::Blobs::Models::AccessTier::Cool); - auto res = blockBlobClient.GetProperties(); + auto res = blobClient.GetProperties(); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); @@ -642,32 +642,25 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, StageBlock) { - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + auto srcBlobClient = *m_blockBlobClient; + + auto blobClient = GetBlockBlobClientForTest(RandomString()); const std::string blockId1 = Base64EncodeText("0"); const std::string blockId2 = Base64EncodeText("1"); - auto blockBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "extra", - InitClientOptions()); - std::vector block1Content; - block1Content.resize(100); - RandomBuffer(reinterpret_cast(&block1Content[0]), block1Content.size()); + std::vector block1Content = RandomBuffer(100); auto blockContent = Azure::Core::IO::MemoryBodyStream(block1Content.data(), block1Content.size()); - blockBlobClient.StageBlock(blockId1, blockContent); + blobClient.StageBlock(blockId1, blockContent); Azure::Storage::Blobs::CommitBlockListOptions options; options.HttpHeaders = m_blobUploadOptions.HttpHeaders; options.Metadata = m_blobUploadOptions.Metadata; - auto blobContentInfo = blockBlobClient.CommitBlockList({blockId1}, options); + auto blobContentInfo = blobClient.CommitBlockList({blockId1}, options); EXPECT_TRUE(blobContentInfo.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(blobContentInfo.Value.LastModified)); EXPECT_TRUE(blobContentInfo.Value.VersionId.HasValue()); EXPECT_FALSE(blobContentInfo.Value.VersionId.Value().empty()); - auto res = blockBlobClient.GetBlockList(); + auto res = blobClient.GetBlockList(); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); @@ -679,695 +672,652 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(res.Value.CommittedBlocks[0].Size, static_cast(block1Content.size())); EXPECT_TRUE(res.Value.UncommittedBlocks.empty()); - blockBlobClient.StageBlockFromUri(blockId2, client.GetUrl() + GetSas()); + blobClient.StageBlockFromUri(blockId2, srcBlobClient.GetUrl() + GetSas()); Blobs::GetBlockListOptions options2; options2.ListType = Blobs::Models::BlockListType::All; - res = blockBlobClient.GetBlockList(options2); + res = blobClient.GetBlockList(options2); EXPECT_EQ(res.Value.BlobSize, static_cast(block1Content.size())); ASSERT_FALSE(res.Value.UncommittedBlocks.empty()); EXPECT_EQ(res.Value.UncommittedBlocks[0].Name, blockId2); EXPECT_EQ(res.Value.UncommittedBlocks[0].Size, static_cast(m_blobContent.size())); - blockBlobClient.CommitBlockList({blockId1, blockId2}); - res = blockBlobClient.GetBlockList(options2); + blobClient.CommitBlockList({blockId1, blockId2}); + res = blobClient.GetBlockList(options2); EXPECT_EQ( res.Value.BlobSize, static_cast(block1Content.size() + m_blobContent.size())); EXPECT_TRUE(res.Value.UncommittedBlocks.empty()); } - namespace { + TEST_F(BlockBlobClientTest, DeleteIfExists) + { + auto blobClient = GetBlockBlobClientForTest(RandomString()); - struct BlobConcurrentDownloadParameter + auto blobClientWithoutAuth = Azure::Storage::Blobs::BlockBlobClient( + blobClient.GetUrl(), InitClientOptions()); { - int Concurrency; - int64_t DownloadSize; - Azure::Nullable Offset = {}; - Azure::Nullable Length = {}; - Azure::Nullable InitialChunkSize = {}; - Azure::Nullable ChunkSize = {}; - }; + auto response = blobClient.DeleteIfExists(); + EXPECT_FALSE(response.Value.Deleted); + } + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + EXPECT_THROW(blobClientWithoutAuth.DeleteIfExists(), StorageException); + { + auto response = blobClient.DeleteIfExists(); + EXPECT_TRUE(response.Value.Deleted); + } - struct BlobConcurrentUploadParameter + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + auto snapshot = blobClient.CreateSnapshot().Value.Snapshot; + auto blobClientWithSnapshot = blobClient.WithSnapshot(snapshot); { - int Concurrency; - int64_t Size; - }; + auto response = blobClientWithSnapshot.DeleteIfExists(); + EXPECT_TRUE(response.Value.Deleted); + } + { + auto response = blobClientWithSnapshot.DeleteIfExists(); + EXPECT_FALSE(response.Value.Deleted); + } + } - class DownloadBlockBlob - : public BlockBlobClientTest, - public ::testing::WithParamInterface { - }; + TEST_F(BlockBlobClientTest, DeleteSnapshots) + { + auto blobClient = *m_blockBlobClient; - class UploadBlockBlob : public BlockBlobClientTest, - public ::testing::WithParamInterface { - }; + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + auto s1 = blobClient.CreateSnapshot().Value.Snapshot; + Blobs::DeleteBlobOptions deleteOptions; + EXPECT_THROW(blobClient.Delete(deleteOptions), StorageException); + deleteOptions.DeleteSnapshots = Blobs::Models::DeleteSnapshotsOption::OnlySnapshots; + EXPECT_NO_THROW(blobClient.Delete(deleteOptions)); + EXPECT_NO_THROW(blobClient.GetProperties()); + EXPECT_THROW(blobClient.WithSnapshot(s1).GetProperties(), StorageException); + auto s2 = blobClient.CreateSnapshot().Value.Snapshot; + deleteOptions.DeleteSnapshots = Blobs::Models::DeleteSnapshotsOption::IncludeSnapshots; + EXPECT_NO_THROW(blobClient.Delete(deleteOptions)); + EXPECT_THROW(blobClient.GetProperties(), StorageException); + EXPECT_THROW(blobClient.WithSnapshot(s2).GetProperties(), StorageException); + } + + TEST_F(BlockBlobClientTest, SetTier) + { + const std::string blobName = RandomString(); + auto blobClient = GetBlockBlobClientForTest(blobName); + + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + + auto properties = blobClient.GetProperties().Value; + ASSERT_TRUE(properties.AccessTier.HasValue()); + ASSERT_TRUE(properties.IsAccessTierInferred.HasValue()); + EXPECT_TRUE(properties.IsAccessTierInferred.Value()); + EXPECT_FALSE(properties.AccessTierChangedOn.HasValue()); + + auto blobItem = GetBlobItem(blobName); + ASSERT_TRUE(blobItem.Details.AccessTier.HasValue()); + ASSERT_TRUE(blobItem.Details.IsAccessTierInferred.HasValue()); + EXPECT_TRUE(blobItem.Details.IsAccessTierInferred.Value()); + EXPECT_FALSE(blobItem.Details.AccessTierChangedOn.HasValue()); + + // choose a different tier + auto targetTier = properties.AccessTier.Value() == Blobs::Models::AccessTier::Hot + ? Blobs::Models::AccessTier::Cool + : Blobs::Models::AccessTier::Hot; + blobClient.SetAccessTier(targetTier); + + properties = blobClient.GetProperties().Value; + ASSERT_TRUE(properties.AccessTier.HasValue()); + ASSERT_TRUE(properties.IsAccessTierInferred.HasValue()); + EXPECT_FALSE(properties.IsAccessTierInferred.Value()); + EXPECT_TRUE(properties.AccessTierChangedOn.HasValue()); + + blobItem = GetBlobItem(blobName); + ASSERT_TRUE(blobItem.Details.AccessTier.HasValue()); + ASSERT_TRUE(blobItem.Details.IsAccessTierInferred.HasValue()); + EXPECT_FALSE(blobItem.Details.IsAccessTierInferred.Value()); + EXPECT_TRUE(blobItem.Details.AccessTierChangedOn.HasValue()); + + // set to archive, then rehydrate + blobClient.SetAccessTier(Blobs::Models::AccessTier::Archive); + blobClient.SetAccessTier(Blobs::Models::AccessTier::Hot); + properties = blobClient.GetProperties().Value; + ASSERT_TRUE(properties.ArchiveStatus.HasValue()); + EXPECT_EQ( + properties.ArchiveStatus.Value(), Blobs::Models::ArchiveStatus::RehydratePendingToHot); + ASSERT_TRUE(properties.RehydratePriority.HasValue()); + EXPECT_EQ(properties.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard); -#define APPEND_IF_NOT_NULL(value, suffix, destination) \ - if (value) \ - { \ - destination.append(suffix + std::to_string(value.Value())); \ + blobItem = GetBlobItem(blobName); + ASSERT_TRUE(blobItem.Details.ArchiveStatus.HasValue()); + EXPECT_EQ( + blobItem.Details.ArchiveStatus.Value(), + Blobs::Models::ArchiveStatus::RehydratePendingToHot); + ASSERT_TRUE(blobItem.Details.RehydratePriority.HasValue()); + EXPECT_EQ( + blobItem.Details.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard); } - std::string GetDownloadSuffix(const testing::TestParamInfo& info) - { - // Can't use empty spaces or underscores (_) as per google test documentation - // http://google.github.io/googletest/advanced.html#specifying-names-for-value-parameterized-test-parameters - auto const& p = info.param; - std::string suffix( - "c" + std::to_string(p.Concurrency) + "s" + std::to_string(p.DownloadSize)); - APPEND_IF_NOT_NULL(p.Offset, "o", suffix) - APPEND_IF_NOT_NULL(p.Length, "l", suffix) - APPEND_IF_NOT_NULL(p.InitialChunkSize, "ics", suffix) - APPEND_IF_NOT_NULL(p.ChunkSize, "cs", suffix) - return suffix; - } + TEST_F(BlockBlobClientTest, DISABLED_SetTierCold) + { + m_blockBlobClient->SetAccessTier(Blobs::Models::AccessTier::Cold); + auto properties = m_blockBlobClient->GetProperties().Value; + EXPECT_EQ(properties.AccessTier.Value(), Blobs::Models::AccessTier::Cold); + } - std::string GetUploadSuffix(const testing::TestParamInfo& info) - { - // Can't use empty spaces or underscores (_) as per google test documentation - // http://google.github.io/googletest/advanced.html#specifying-names-for-value-parameterized-test-parameters - auto const& p = info.param; - std::string suffix("c" + std::to_string(p.Concurrency) + "s" + std::to_string(p.Size)); - return suffix; - } + TEST_F(BlockBlobClientTest, SetTierWithLeaseId) + { + auto blobClient = *m_blockBlobClient; - std::vector GetDownloadParameters(int64_t const blobSize) - { - std::vector testParametes; - for (int c : {1, 2, 4}) - { - // download whole blob - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, 0})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, 0, blobSize})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, 0, blobSize * 2})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize * 2})); - - // Do offset - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, 0, 1})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, 1, 1})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, blobSize - 1, 1})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, blobSize - 1, 2})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, blobSize, 1})); - testParametes.emplace_back(BlobConcurrentDownloadParameter({c, blobSize, blobSize + 1, 2})); - - // // initial chunk size - testParametes.emplace_back( - BlobConcurrentDownloadParameter({c, blobSize, 0, 1024, 512, 1024})); - testParametes.emplace_back( - BlobConcurrentDownloadParameter({c, blobSize, 0, 1024, 1024, 1024})); - testParametes.emplace_back( - BlobConcurrentDownloadParameter({c, blobSize, 0, 1024, 2048, 1024})); - } - return testParametes; - } + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - std::vector GetUploadParameters() - { - std::vector testParametes; - for (int c : {1, 2, 4}) - { - for (int64_t l : - {0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB}) - { - testParametes.emplace_back(BlobConcurrentUploadParameter({c, l})); - } - } - return testParametes; - } + const std::string leaseId = RandomUUID(); + Blobs::BlobLeaseClient leaseClient(blobClient, leaseId); + leaseClient.Acquire(std::chrono::seconds(30)); + + EXPECT_THROW(blobClient.SetAccessTier(Blobs::Models::AccessTier::Cool), StorageException); - } // namespace + Blobs::SetBlobAccessTierOptions options; + options.AccessConditions.LeaseId = leaseId; + EXPECT_NO_THROW(blobClient.SetAccessTier(Blobs::Models::AccessTier::Cool, options)); + } - TEST_P(DownloadBlockBlob, downloadToBuffer) + TEST_F(BlockBlobClientTest, UncommittedBlob) { - BlobConcurrentDownloadParameter const& p(GetParam()); - auto const testName(GetTestName(true)); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); - - std::vector downloadBuffer; - std::vector expectedData = m_blobContent; - int64_t blobSize = m_blobContent.size(); - int64_t actualDownloadSize = std::min(p.DownloadSize, blobSize); - if (p.Offset.HasValue() && p.Length.HasValue()) - { - actualDownloadSize = std::min(p.Length.Value(), blobSize - p.Offset.Value()); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(p.Offset.Value()), - m_blobContent.begin() + static_cast(p.Offset.Value() + actualDownloadSize)); - } - else - { - expectedData.clear(); - } - } - else if (p.Offset.HasValue()) - { - actualDownloadSize = blobSize - p.Offset.Value(); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(p.Offset.Value()), m_blobContent.end()); - } - else - { - expectedData.clear(); - } - } - downloadBuffer.resize(static_cast(p.DownloadSize), '\x00'); - Blobs::DownloadBlobToOptions options; - options.TransferOptions.Concurrency = p.Concurrency; - if (p.Offset.HasValue() || p.Length.HasValue()) - { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = p.Offset.Value(); - options.Range.Value().Length = p.Length; - } - if (p.InitialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = p.InitialChunkSize.Value(); - } - if (p.ChunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = p.ChunkSize.Value(); - } - if (actualDownloadSize > 0) - { - auto res = client.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options); - EXPECT_EQ(res.Value.BlobSize, blobSize); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - EXPECT_EQ(res.Value.ContentRange.Offset, p.Offset.HasValue() ? p.Offset.Value() : 0); - downloadBuffer.resize(static_cast(res.Value.ContentRange.Length.Value())); - EXPECT_EQ(downloadBuffer, expectedData); - } - else + const std::string blobName = RandomString(); + auto blobClient = GetBlockBlobClientForTest(blobName); + + std::vector buffer(100); + Azure::Core::IO::MemoryBodyStream stream(buffer.data(), buffer.size()); + blobClient.StageBlock("YWJjZA==", stream); + + Blobs::GetBlockListOptions getBlockListOptions; + getBlockListOptions.ListType = Blobs::Models::BlockListType::All; + auto res = blobClient.GetBlockList(getBlockListOptions).Value; + EXPECT_FALSE(res.ETag.HasValue()); + EXPECT_EQ(res.BlobSize, 0); + EXPECT_TRUE(res.CommittedBlocks.empty()); + EXPECT_FALSE(res.UncommittedBlocks.empty()); + + auto blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::UncomittedBlobs); + EXPECT_EQ(blobItem.BlobSize, 0); + } + + TEST_F(BlockBlobClientTest, SourceTagsConditions) + { + auto containerClient = *m_blobContainerClient; + + auto sourceBlobClient = *m_blockBlobClient; + std::map tags; + tags["key1"] = "value1"; + sourceBlobClient.SetTags(tags); + + const std::string successfulTagConditions = "key1 = 'value1'"; + const std::string failedTagConditions = "key1 != 'value1'"; + + auto destBlobClient = containerClient.GetBlockBlobClient("dest" + RandomString()); { + Blobs::StartBlobCopyFromUriOptions options; + options.SourceAccessConditions.TagConditions = successfulTagConditions; + EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); + options.SourceAccessConditions.TagConditions = failedTagConditions; EXPECT_THROW( - client.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options), - StorageException); + destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + + Blobs::UploadBlockBlobFromUriOptions options2; + options2.SourceAccessConditions.TagConditions = successfulTagConditions; + EXPECT_NO_THROW(destBlobClient.UploadFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); + options2.SourceAccessConditions.TagConditions = failedTagConditions; + EXPECT_NO_THROW(destBlobClient.UploadFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); } } - TEST_P(DownloadBlockBlob, downloadToFile) + TEST_F(BlockBlobClientTest, SourceBlobAccessConditions) { - BlobConcurrentDownloadParameter const& p(GetParam()); - auto const testName(GetTestName(true)); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); - - std::string tempFilename(testName); - std::vector expectedData = m_blobContent; - int64_t blobSize = m_blobContent.size(); - int64_t actualDownloadSize = std::min(p.DownloadSize, blobSize); - if (p.Offset.HasValue() && p.Length.HasValue()) + auto containerClient = *m_blobContainerClient; + + auto sourceBlobClient = containerClient.GetBlockBlobClient("source" + RandomString()); + + std::vector buffer; + buffer.resize(1024); + auto createResponse = sourceBlobClient.UploadFrom(buffer.data(), buffer.size()); + Azure::ETag eTag = createResponse.Value.ETag; + auto lastModifiedTime = createResponse.Value.LastModified; + auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); + auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); + + auto destBlobClient = containerClient.GetBlockBlobClient("dest" + RandomString()); + { - actualDownloadSize = std::min(p.Length.Value(), blobSize - p.Offset.Value()); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(p.Offset.Value()), - m_blobContent.begin() + static_cast(p.Offset.Value() + actualDownloadSize)); - } - else - { - expectedData.clear(); - } + Blobs::StartBlobCopyFromUriOptions options; + options.SourceAccessConditions.IfMatch = eTag; + EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); + options.SourceAccessConditions.IfMatch = DummyETag; + EXPECT_THROW( + destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + + Blobs::CopyBlobFromUriOptions options2; + options2.SourceAccessConditions.IfMatch = eTag; + EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); + options2.SourceAccessConditions.IfMatch = DummyETag; + EXPECT_THROW( + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + StorageException); } - else if (p.Offset.HasValue()) { - actualDownloadSize = blobSize - p.Offset.Value(); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(p.Offset.Value()), m_blobContent.end()); - } - else - { - expectedData.clear(); - } + Blobs::StartBlobCopyFromUriOptions options; + options.SourceAccessConditions.IfNoneMatch = DummyETag; + EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); + options.SourceAccessConditions.IfNoneMatch = eTag; + EXPECT_THROW( + destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + + Blobs::CopyBlobFromUriOptions options2; + options2.SourceAccessConditions.IfNoneMatch = DummyETag; + EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); + options2.SourceAccessConditions.IfNoneMatch = eTag; + EXPECT_THROW( + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + StorageException); } - Blobs::DownloadBlobToOptions options; - options.TransferOptions.Concurrency = p.Concurrency; - if (p.Offset.HasValue() || p.Length.HasValue()) { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = p.Offset.Value(); - options.Range.Value().Length = p.Length; - } - if (p.InitialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = p.InitialChunkSize.Value(); - } - if (p.ChunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = p.ChunkSize.Value(); + Blobs::StartBlobCopyFromUriOptions options; + options.SourceAccessConditions.IfModifiedSince = timeBeforeStr; + EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); + options.SourceAccessConditions.IfModifiedSince = timeAfterStr; + EXPECT_THROW( + destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + + sourceBlobClient.GetProperties(); + Blobs::CopyBlobFromUriOptions options2; + options2.SourceAccessConditions.IfModifiedSince = timeBeforeStr; + EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); + options2.SourceAccessConditions.IfModifiedSince = timeAfterStr; + EXPECT_THROW( + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + StorageException); } - if (actualDownloadSize > 0) { - auto res = client.DownloadTo(tempFilename, options); - EXPECT_EQ(res.Value.BlobSize, blobSize); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - EXPECT_EQ(res.Value.ContentRange.Offset, p.Offset.HasValue() ? p.Offset.Value() : 0); - EXPECT_EQ(ReadFile(tempFilename), expectedData); + Blobs::StartBlobCopyFromUriOptions options; + options.SourceAccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); + options.SourceAccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW( + destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + + Blobs::CopyBlobFromUriOptions options2; + options2.SourceAccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); + options2.SourceAccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW( + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + StorageException); } - else + + // lease { - EXPECT_THROW(client.DownloadTo(tempFilename, options), StorageException); + const std::string leaseId = RandomUUID(); + const std::string dummyLeaseId = RandomUUID(); + Blobs::BlobLeaseClient leaseClient(destBlobClient, leaseId); + + leaseClient.Acquire(std::chrono::seconds(60)); + + Blobs::CopyBlobFromUriOptions options; + options.AccessConditions.LeaseId = dummyLeaseId; + EXPECT_THROW( + destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options), + StorageException); + options.AccessConditions.LeaseId = leaseId; + EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options)); + leaseClient.Release(); } - DeleteFile(tempFilename); } - INSTANTIATE_TEST_SUITE_P( - withParam, - DownloadBlockBlob, - testing::ValuesIn(GetDownloadParameters(8_MB)), - GetDownloadSuffix); - - TEST_F(BlockBlobClientTest, ConcurrentDownload_LIVEONLY_) + TEST_F(BlockBlobClientTest, DISABLED_Immutability) { - CHECK_SKIP_TEST(); - auto const testName(GetTestName()); - auto client = GetBlockBlobClient(testName); - UploadBlockBlob(8_MB); + const auto ImmutabilityMaxLength = std::chrono::seconds(5); + const std::string blobName = m_blobName; + auto blobClient = *m_blockBlobClient; - auto testDownloadToBuffer = [&](int concurrency, - int64_t downloadSize, - Azure::Nullable offset = {}, - Azure::Nullable length = {}, - Azure::Nullable initialChunkSize = {}, - Azure::Nullable chunkSize = {}) { - std::vector downloadBuffer; - std::vector expectedData = m_blobContent; - int64_t blobSize = m_blobContent.size(); - int64_t actualDownloadSize = std::min(downloadSize, blobSize); - if (offset.HasValue() && length.HasValue()) - { - actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(offset.Value()), - m_blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); - } - else - { - expectedData.clear(); - } - } - else if (offset.HasValue()) - { - actualDownloadSize = blobSize - offset.Value(); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(offset.Value()), m_blobContent.end()); - } - else - { - expectedData.clear(); - } - } - downloadBuffer.resize(static_cast(downloadSize), '\x00'); - Blobs::DownloadBlobToOptions options; - options.TransferOptions.Concurrency = concurrency; - if (offset.HasValue() || length.HasValue()) - { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = offset.Value(); - options.Range.Value().Length = length; - } - if (initialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); - } - if (chunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = chunkSize.Value(); - } - if (actualDownloadSize > 0) - { - auto res = client.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options); - EXPECT_EQ(res.Value.BlobSize, blobSize); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); - downloadBuffer.resize(static_cast(res.Value.ContentRange.Length.Value())); - EXPECT_EQ(downloadBuffer, expectedData); - } - else - { - EXPECT_THROW( - client.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options), - StorageException); - } - }; + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - auto testDownloadToFile = [&](int concurrency, - int64_t downloadSize, - Azure::Nullable offset = {}, - Azure::Nullable length = {}, - Azure::Nullable initialChunkSize = {}, - Azure::Nullable chunkSize = {}) { - std::string tempFilename = testName + "file" + std::to_string(concurrency); - if (offset) - { - tempFilename.append(std::to_string(offset.Value())); - } - std::vector expectedData = m_blobContent; - int64_t blobSize = m_blobContent.size(); - int64_t actualDownloadSize = std::min(downloadSize, blobSize); - if (offset.HasValue() && length.HasValue()) - { - actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(offset.Value()), - m_blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); - } - else - { - expectedData.clear(); - } - } - else if (offset.HasValue()) - { - actualDownloadSize = blobSize - offset.Value(); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_blobContent.begin() + static_cast(offset.Value()), m_blobContent.end()); - } - else - { - expectedData.clear(); - } - } - Blobs::DownloadBlobToOptions options; - options.TransferOptions.Concurrency = concurrency; - if (offset.HasValue() || length.HasValue()) - { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = offset.Value(); - options.Range.Value().Length = length; - } - if (initialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); - } - if (chunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = chunkSize.Value(); - } - if (actualDownloadSize > 0) - { - auto res = client.DownloadTo(tempFilename, options); - EXPECT_EQ(res.Value.BlobSize, blobSize); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); - EXPECT_EQ(ReadFile(tempFilename), expectedData); - } - else - { - EXPECT_THROW(client.DownloadTo(tempFilename, options), StorageException); - } - DeleteFile(tempFilename); - }; + auto blobContainerClient = *m_blobContainerClient; + ASSERT_TRUE(blobContainerClient.GetProperties().Value.HasImmutableStorageWithVersioning); - const int64_t blobSize = m_blobContent.size(); - std::vector> futures; - for (int c : {1, 2, 4}) - { - // random range - std::mt19937_64 random_generator(std::random_device{}()); - for (int i = 0; i < 16; ++i) - { - std::uniform_int_distribution offsetDistribution(0, m_blobContent.size() - 1); - int64_t offset = offsetDistribution(random_generator); - std::uniform_int_distribution lengthDistribution(1, 64_KB); - int64_t length = lengthDistribution(random_generator); - futures.emplace_back(std::async( - std::launch::async, testDownloadToBuffer, c, blobSize, offset, length, 4_KB, 4_KB)); - futures.emplace_back(std::async( - std::launch::async, testDownloadToFile, c, blobSize, offset, length, 4_KB, 4_KB)); - } + Blobs::Models::BlobImmutabilityPolicy policy; + policy.ExpiresOn = Azure::DateTime::Parse( + Azure::DateTime(std::chrono::system_clock::now() + ImmutabilityMaxLength) + .ToString(Azure::DateTime::DateFormat::Rfc1123), + Azure::DateTime::DateFormat::Rfc1123); + policy.PolicyMode = Blobs::Models::BlobImmutabilityPolicyMode::Unlocked; + auto setPolicyResponse = blobClient.SetImmutabilityPolicy(policy); + EXPECT_EQ(setPolicyResponse.Value.ImmutabilityPolicy, policy); + auto blobProperties = blobClient.GetProperties().Value; + ASSERT_TRUE(blobProperties.ImmutabilityPolicy.HasValue()); + EXPECT_EQ(blobProperties.ImmutabilityPolicy.Value(), policy); + auto downloadResponse = blobClient.Download(); + ASSERT_TRUE(downloadResponse.Value.Details.ImmutabilityPolicy.HasValue()); + EXPECT_EQ(downloadResponse.Value.Details.ImmutabilityPolicy.Value(), policy); + auto blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy); + ASSERT_TRUE(blobItem.Details.ImmutabilityPolicy.HasValue()); + EXPECT_EQ(blobItem.Details.ImmutabilityPolicy.Value(), policy); - // // buffer not big enough - Blobs::DownloadBlobToOptions options; - options.TransferOptions.Concurrency = c; - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = 1; - for (int64_t length : {1ULL, 2ULL, 4_KB, 5_KB, 8_KB, 11_KB, 20_KB}) - { - std::vector downloadBuffer; - downloadBuffer.resize(static_cast(length - 1)); - options.Range.Value().Length = length; - EXPECT_THROW( - client.DownloadTo(downloadBuffer.data(), static_cast(length - 1), options), - std::runtime_error); - } + EXPECT_NO_THROW(blobClient.DeleteImmutabilityPolicy()); + blobProperties = blobClient.GetProperties().Value; + EXPECT_FALSE(blobProperties.ImmutabilityPolicy.HasValue()); + downloadResponse = blobClient.Download(); + ASSERT_FALSE(downloadResponse.Value.Details.ImmutabilityPolicy.HasValue()); + blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy); + ASSERT_FALSE(blobItem.Details.ImmutabilityPolicy.HasValue()); + + auto copySourceBlobClient = GetBlockBlobClientForTest(blobName + "src"); + copySourceBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + { + auto copyDestinationBlobClient = GetBlockBlobClientForTest(blobName + "dest1"); + Blobs::StartBlobCopyFromUriOptions options; + options.ImmutabilityPolicy = policy; + copyDestinationBlobClient.StartCopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options) + .PollUntilDone(std::chrono::seconds(1)); + EXPECT_EQ(copyDestinationBlobClient.GetProperties().Value.ImmutabilityPolicy.Value(), policy); } - for (auto& f : futures) { - f.get(); + auto copyDestinationBlobClient = GetBlockBlobClientForTest(blobName + "dest2"); + Blobs::CopyBlobFromUriOptions options; + options.ImmutabilityPolicy = policy; + copyDestinationBlobClient.CopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options); + EXPECT_EQ(copyDestinationBlobClient.GetProperties().Value.ImmutabilityPolicy.Value(), policy); } - } - - TEST_F(BlockBlobClientTest, ConcurrentUploadFromNonExistingFile) - { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - std::string emptyFilename(testName); - EXPECT_THROW(blockBlobClient.UploadFrom(emptyFilename), std::runtime_error); - EXPECT_THROW(blockBlobClient.Delete(), StorageException); + std::this_thread::sleep_for(ImmutabilityMaxLength); } - TEST_F(BlockBlobClientTest, ConcurrentDownloadNonExistingBlob) + TEST_F(BlockBlobClientTest, DISABLED_ImmutabilityAccessCondition) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); + const auto ImmutabilityMaxLength = std::chrono::seconds(5); - std::vector blobContent(100); - std::string tempFilename(testName); + auto blobClient = *m_blockBlobClient; + std::vector emptyContent; + auto uploadResponse = blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + auto lastModifiedTime = uploadResponse.Value.LastModified; + auto timeBeforeStr = lastModifiedTime - std::chrono::minutes(1); + auto timeAfterStr = lastModifiedTime + std::chrono::minutes(1); - EXPECT_THROW( - blockBlobClient.DownloadTo(blobContent.data(), blobContent.size()), StorageException); - EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename), StorageException); - DeleteFile(tempFilename); + Blobs::Models::BlobImmutabilityPolicy policy; + policy.ExpiresOn = Azure::DateTime::Parse( + Azure::DateTime(std::chrono::system_clock::now() + ImmutabilityMaxLength) + .ToString(Azure::DateTime::DateFormat::Rfc1123), + Azure::DateTime::DateFormat::Rfc1123); + policy.PolicyMode = Blobs::Models::BlobImmutabilityPolicyMode::Unlocked; + + Blobs::SetBlobImmutabilityPolicyOptions options; + options.AccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW(blobClient.SetImmutabilityPolicy(policy, options), StorageException); + options.AccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(blobClient.SetImmutabilityPolicy(policy, options)); + + std::this_thread::sleep_for(ImmutabilityMaxLength); } - TEST_F(BlockBlobClientTest, ConcurrentUploadEmptyBlob) + TEST_F(BlockBlobClientTest, DISABLED_LegalHold) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - + const auto testName = m_blobName; + auto blobClient = *m_blockBlobClient; std::vector emptyContent; - blockBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - EXPECT_NO_THROW(blockBlobClient.Delete()); + auto setLegalHoldResponse = blobClient.SetLegalHold(true); + EXPECT_TRUE(setLegalHoldResponse.Value.HasLegalHold); + auto blobProperties = blobClient.GetProperties().Value; + EXPECT_TRUE(blobProperties.HasLegalHold); + auto downloadResponse = blobClient.Download(); + EXPECT_TRUE(downloadResponse.Value.Details.HasLegalHold); + auto blobItem = GetBlobItem(testName, Blobs::Models::ListBlobsIncludeFlags::LegalHold); + EXPECT_TRUE(blobItem.Details.HasLegalHold); - std::string emptyFilename(testName); - WriteFile(emptyFilename, std::vector{}); - blockBlobClient.UploadFrom(emptyFilename); - EXPECT_NO_THROW(blockBlobClient.Delete()); + setLegalHoldResponse = blobClient.SetLegalHold(false); + EXPECT_FALSE(setLegalHoldResponse.Value.HasLegalHold); - DeleteFile(emptyFilename); + auto copySourceBlobClient = GetBlockBlobClientForTest(RandomString() + "src"); + copySourceBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + { + auto copyDestinationBlobClient = GetBlockBlobClientForTest(RandomString() + "dest1"); + Blobs::StartBlobCopyFromUriOptions options; + options.HasLegalHold = true; + copyDestinationBlobClient.StartCopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options) + .PollUntilDone(std::chrono::seconds(1)); + EXPECT_TRUE(copyDestinationBlobClient.GetProperties().Value.HasLegalHold); + } + { + auto copyDestinationBlobClient = GetBlockBlobClientForTest(RandomString() + "dest2"); + Blobs::CopyBlobFromUriOptions options; + options.HasLegalHold = true; + copyDestinationBlobClient.CopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options); + EXPECT_TRUE(copyDestinationBlobClient.GetProperties().Value.HasLegalHold); + } } - TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob) + TEST_F(BlockBlobClientTest, ContentHash) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - std::string tempFilename(testName); - - std::vector emptyContent; - - auto blobContent = Azure::Core::IO::MemoryBodyStream(emptyContent.data(), emptyContent.size()); - blockBlobClient.Upload(blobContent); - blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); - blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); - - auto res = blockBlobClient.DownloadTo(emptyContent.data(), 0); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - res = blockBlobClient.DownloadTo(tempFilename); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - EXPECT_TRUE(ReadFile(tempFilename).empty()); - DeleteFile(tempFilename); + auto srcBlobClient = *m_blockBlobClient; + std::vector blobContent = RandomBuffer(100); + srcBlobClient.UploadFrom(blobContent.data(), blobContent.size()); + const std::vector contentMd5 + = Azure::Core::Cryptography::Md5Hash().Final(blobContent.data(), blobContent.size()); + const std::vector contentCrc64 + = Azure::Storage::Crc64Hash().Final(blobContent.data(), blobContent.size()); - res = blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB)); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - res = blockBlobClient.DownloadTo(tempFilename); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - EXPECT_TRUE(ReadFile(tempFilename).empty()); - DeleteFile(tempFilename); + Azure::Core::IO::MemoryBodyStream stream(blobContent.data(), blobContent.size()); - for (int c : {1, 2}) { - Azure::Storage::Blobs::DownloadBlobToOptions options; - options.TransferOptions.InitialChunkSize = 10; - options.TransferOptions.ChunkSize = 10; - options.TransferOptions.Concurrency = c; - - res = blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - res = blockBlobClient.DownloadTo(tempFilename, options); - EXPECT_EQ(res.Value.BlobSize, 0); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); - EXPECT_TRUE(res.Value.Details.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); - EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); - EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); - EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); - EXPECT_TRUE(ReadFile(tempFilename).empty()); - DeleteFile(tempFilename); - - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = 0; + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest0"); + Blobs::UploadBlockBlobOptions options; + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); + stream.Rewind(); + EXPECT_THROW(destBlobClient.Upload(stream, options), StorageException); + options.TransactionalContentHash.Value().Value = contentMd5; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.Upload(stream, options)); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; + options.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + stream.Rewind(); + EXPECT_THROW(destBlobClient.Upload(stream, options), StorageException); + options.TransactionalContentHash.Value().Value = contentCrc64; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.Upload(stream, options)); + } + { + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest1"); + Blobs::UploadBlockBlobFromUriOptions options; + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); + stream.Rewind(); EXPECT_THROW( - blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), + destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); - EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - - options.Range.Value().Offset = 1; + options.TransactionalContentHash.Value().Value = contentMd5; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options)); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; + options.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + stream.Rewind(); EXPECT_THROW( - blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), + destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); - EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - - options.Range.Value().Offset = 0; - options.Range.Value().Length = 1; + options.TransactionalContentHash.Value().Value = contentCrc64; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options)); + } + { + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest2"); + Blobs::CopyBlobFromUriOptions options; + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); + stream.Rewind(); EXPECT_THROW( - blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), + destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); + options.TransactionalContentHash.Value().Value = contentMd5; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options)); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; + options.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + stream.Rewind(); + EXPECT_THROW( + destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); + options.TransactionalContentHash.Value().Value = contentCrc64; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options)); + } + { + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest3"); + Blobs::StageBlockOptions options; + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); + stream.Rewind(); + EXPECT_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options), StorageException); + options.TransactionalContentHash.Value().Value = contentMd5; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options)); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; + options.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + stream.Rewind(); + EXPECT_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options), StorageException); + options.TransactionalContentHash.Value().Value = contentCrc64; + stream.Rewind(); + EXPECT_NO_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options)); + } + { + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest4"); + Blobs::StageBlockFromUriOptions options; + options.TransactionalContentHash = ContentHash(); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; + options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); + EXPECT_THROW( + destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options), StorageException); - EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - - options.Range.Value().Offset = 100; - options.Range.Value().Length = 100; + options.TransactionalContentHash.Value().Value = contentMd5; + EXPECT_NO_THROW( + destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options)); + options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; + options.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); EXPECT_THROW( - blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), + destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options), StorageException); - EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - DeleteFile(tempFilename); + options.TransactionalContentHash.Value().Value = contentCrc64; + EXPECT_NO_THROW( + destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options)); } } - TEST_P(UploadBlockBlob, fromBuffer) - { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - SetOptions(); - UploadBlockBlob::ParamType const& p(GetParam()); - auto const blobSize = p.Size; - std::vector blobContent(static_cast(8_MB), 'x'); - - Azure::Storage::Blobs::UploadBlockBlobFromOptions options; - options.TransferOptions.ChunkSize = 1_MB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = m_blobUploadOptions.HttpHeaders; - options.HttpHeaders.ContentHash.Value.clear(); - options.Metadata = m_blobUploadOptions.Metadata; - options.AccessTier = m_blobUploadOptions.AccessTier; - auto res - = blockBlobClient.UploadFrom(blobContent.data(), static_cast(blobSize), options); - EXPECT_TRUE(res.Value.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - auto properties = blockBlobClient.GetProperties().Value; - properties.HttpHeaders.ContentHash.Value.clear(); - EXPECT_EQ(properties.BlobSize, blobSize); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.AccessTier.Value(), options.AccessTier.Value()); - EXPECT_EQ(properties.ETag, res.Value.ETag); - EXPECT_EQ(properties.LastModified, res.Value.LastModified); - std::vector downloadContent(static_cast(blobSize), '\x00'); - blockBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); - EXPECT_EQ( - downloadContent, - std::vector( - blobContent.begin(), blobContent.begin() + static_cast(blobSize))); - } - - TEST_P(UploadBlockBlob, fromFile) + TEST_F(BlockBlobClientTest, UploadFromUri) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); - SetOptions(); - UploadBlockBlob::ParamType const& p(GetParam()); - auto const blobSize = p.Size; - std::vector blobContent(static_cast(p.Size), 'x'); - - Azure::Storage::Blobs::UploadBlockBlobFromOptions options; - options.TransferOptions.ChunkSize = 1_MB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = m_blobUploadOptions.HttpHeaders; - options.HttpHeaders.ContentHash.Value.clear(); - options.Metadata = m_blobUploadOptions.Metadata; - options.AccessTier = m_blobUploadOptions.AccessTier; + auto srcBlobClient = *m_blockBlobClient; + std::vector blobContent(100, 'a'); + srcBlobClient.UploadFrom(blobContent.data(), blobContent.size()); + std::map srcTags; + srcTags["srctags"] = "a1212"; + srcBlobClient.SetTags(srcTags); - std::string tempFilename(testName); - WriteFile(tempFilename, blobContent); - auto res = blockBlobClient.UploadFrom(tempFilename, options); - EXPECT_TRUE(res.Value.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - auto properties = blockBlobClient.GetProperties().Value; - properties.HttpHeaders.ContentHash.Value.clear(); - EXPECT_EQ(properties.BlobSize, blobSize); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.AccessTier.Value(), options.AccessTier.Value()); - EXPECT_EQ(properties.ETag, res.Value.ETag); - EXPECT_EQ(properties.LastModified, res.Value.LastModified); - std::vector downloadContent(static_cast(blobSize), '\x00'); - blockBlobClient.DownloadTo(downloadContent.data(), static_cast(blobSize)); - EXPECT_EQ( - downloadContent, - std::vector( - blobContent.begin(), blobContent.begin() + static_cast(blobSize))); - DeleteFile(tempFilename); + const std::vector blobMd5 + = Azure::Core::Cryptography::Md5Hash().Final(blobContent.data(), blobContent.size()); + const std::vector blobCrc64 + = Azure::Storage::Crc64Hash().Final(blobContent.data(), blobContent.size()); + + auto destBlobClient = GetBlockBlobClientForTest(RandomString() + "dest"); + auto uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas()); + EXPECT_TRUE(uploadFromUriResult.Value.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(uploadFromUriResult.Value.LastModified)); + EXPECT_TRUE(uploadFromUriResult.Value.VersionId.HasValue()); + EXPECT_TRUE(uploadFromUriResult.Value.IsServerEncrypted); + ASSERT_TRUE(uploadFromUriResult.Value.TransactionalContentHash.HasValue()); + if (uploadFromUriResult.Value.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) + { + EXPECT_EQ(uploadFromUriResult.Value.TransactionalContentHash.Value().Value, blobMd5); + } + else if ( + uploadFromUriResult.Value.TransactionalContentHash.Value().Algorithm + == HashAlgorithm::Crc64) + { + EXPECT_EQ(uploadFromUriResult.Value.TransactionalContentHash.Value().Value, blobCrc64); + } + + Blobs::UploadBlockBlobFromUriOptions options; + options.CopySourceBlobProperties = false; + options.HttpHeaders.ContentLanguage = "en-US"; + options.HttpHeaders.ContentType = "application/octet-stream"; + options.Metadata["k"] = "v"; + options.AccessTier = Blobs::Models::AccessTier::Cool; + options.Tags["k1"] = "v1"; + uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options); + auto destBlobProperties = destBlobClient.GetProperties().Value; + destBlobProperties.HttpHeaders.ContentHash.Value.clear(); + EXPECT_EQ(destBlobProperties.HttpHeaders, options.HttpHeaders); + EXPECT_EQ(destBlobProperties.Metadata, options.Metadata); + EXPECT_EQ(destBlobProperties.AccessTier.Value(), options.AccessTier.Value()); + EXPECT_EQ(static_cast(destBlobProperties.TagCount.Value()), options.Tags.size()); + + options.CopySourceTagsMode = Blobs::Models::BlobCopySourceTagsMode::Copy; + options.Tags.clear(); + uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options); + EXPECT_EQ(destBlobClient.GetTags().Value, srcTags); + } + + TEST_F(BlockBlobClientTest, SetGetTagsWithLeaseId) + { + auto blobClient = *m_blockBlobClient; + std::vector emptyContent; + blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + + const std::map tags{{"k", "v"}}; + + Blobs::BlobLeaseClient leaseClient(blobClient, RandomUUID()); + + leaseClient.Acquire(std::chrono::seconds(60)); + + Blobs::SetBlobTagsOptions setTagsOptions; + setTagsOptions.AccessConditions.LeaseId = RandomUUID(); + EXPECT_THROW(blobClient.SetTags(tags, setTagsOptions), StorageException); + Blobs::GetBlobTagsOptions getTagsOptions; + getTagsOptions.AccessConditions.LeaseId = RandomUUID(); + EXPECT_THROW(blobClient.GetTags(getTagsOptions), StorageException); + + setTagsOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); + EXPECT_NO_THROW(blobClient.SetTags(tags, setTagsOptions)); + getTagsOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); + EXPECT_NO_THROW(blobClient.GetTags(getTagsOptions)); + + leaseClient.Release(); } - INSTANTIATE_TEST_SUITE_P( - withParam, - UploadBlockBlob, - testing::ValuesIn(GetUploadParameters()), - GetUploadSuffix); + TEST_F(BlockBlobClientTest, MaximumBlocks) + { + auto blobClient = *m_blockBlobClient; + + const std::vector content(static_cast(1), 'a'); + const std::string blockId = Base64EncodeText(std::string(64, '0')); + auto blockContent = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); + blobClient.StageBlock(blockId, blockContent); + + std::vector blockIds(50000, blockId); + EXPECT_NO_THROW(blobClient.CommitBlockList(blockIds)); + + EXPECT_EQ( + blobClient.GetProperties().Value.BlobSize, + static_cast(blockIds.size() * content.size())); + } TEST_F(BlockBlobClientTest, DownloadError) { - auto const testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); bool exceptionCaught = false; try @@ -1389,641 +1339,423 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, DownloadNonExistingToFile) { - const auto testName(GetTestName()); - auto blockBlobClient = GetBlockBlobClient(testName); + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); - EXPECT_THROW(blockBlobClient.DownloadTo(testName), StorageException); - EXPECT_THROW(ReadFile(testName), std::runtime_error); + const std::string filename = RandomString(); + EXPECT_THROW(blockBlobClient.DownloadTo(filename), StorageException); + EXPECT_THROW(ReadFile(filename), std::runtime_error); } - TEST_F(BlockBlobClientTest, DeleteIfExists) + TEST_F(BlockBlobClientTest, ConcurrentUploadFromNonExistingFile) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - - auto blobClientWithoutAuth = Azure::Storage::Blobs::BlockBlobClient( - blobClient.GetUrl(), InitClientOptions()); - { - auto response = blobClient.DeleteIfExists(); - EXPECT_FALSE(response.Value.Deleted); - } - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - EXPECT_THROW(blobClientWithoutAuth.DeleteIfExists(), StorageException); - { - auto response = blobClient.DeleteIfExists(); - EXPECT_TRUE(response.Value.Deleted); - } + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - auto snapshot = blobClient.CreateSnapshot().Value.Snapshot; - auto blobClientWithSnapshot = blobClient.WithSnapshot(snapshot); - { - auto response = blobClientWithSnapshot.DeleteIfExists(); - EXPECT_TRUE(response.Value.Deleted); - } - { - auto response = blobClientWithSnapshot.DeleteIfExists(); - EXPECT_FALSE(response.Value.Deleted); - } + std::string emptyFilename = RandomString(); + EXPECT_THROW(blockBlobClient.UploadFrom(emptyFilename), std::runtime_error); + EXPECT_THROW(blockBlobClient.Delete(), StorageException); } - TEST_F(BlockBlobClientTest, DeleteSnapshots) + TEST_F(BlockBlobClientTest, ConcurrentDownloadNonExistingBlob) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - auto s1 = blobClient.CreateSnapshot().Value.Snapshot; - Blobs::DeleteBlobOptions deleteOptions; - EXPECT_THROW(blobClient.Delete(deleteOptions), StorageException); - deleteOptions.DeleteSnapshots = Blobs::Models::DeleteSnapshotsOption::OnlySnapshots; - EXPECT_NO_THROW(blobClient.Delete(deleteOptions)); - EXPECT_NO_THROW(blobClient.GetProperties()); - EXPECT_THROW(blobClient.WithSnapshot(s1).GetProperties(), StorageException); - auto s2 = blobClient.CreateSnapshot().Value.Snapshot; - deleteOptions.DeleteSnapshots = Blobs::Models::DeleteSnapshotsOption::IncludeSnapshots; - EXPECT_NO_THROW(blobClient.Delete(deleteOptions)); - EXPECT_THROW(blobClient.GetProperties(), StorageException); - EXPECT_THROW(blobClient.WithSnapshot(s2).GetProperties(), StorageException); + std::vector blobContent(100); + std::string tempFilename = RandomString(); + + EXPECT_THROW( + blockBlobClient.DownloadTo(blobContent.data(), blobContent.size()), StorageException); + EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename), StorageException); } - TEST_F(BlockBlobClientTest, SetTier) + TEST_F(BlockBlobClientTest, ConcurrentUploadEmptyBlob) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); std::vector emptyContent; - std::string blobName(testName); - - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - - auto properties = blobClient.GetProperties().Value; - ASSERT_TRUE(properties.AccessTier.HasValue()); - ASSERT_TRUE(properties.IsAccessTierInferred.HasValue()); - EXPECT_TRUE(properties.IsAccessTierInferred.Value()); - EXPECT_FALSE(properties.AccessTierChangedOn.HasValue()); - - auto blobItem = GetBlobItem(blobName); - ASSERT_TRUE(blobItem.Details.AccessTier.HasValue()); - ASSERT_TRUE(blobItem.Details.IsAccessTierInferred.HasValue()); - EXPECT_TRUE(blobItem.Details.IsAccessTierInferred.Value()); - EXPECT_FALSE(blobItem.Details.AccessTierChangedOn.HasValue()); - - // choose a different tier - auto targetTier = properties.AccessTier.Value() == Blobs::Models::AccessTier::Hot - ? Blobs::Models::AccessTier::Cool - : Blobs::Models::AccessTier::Hot; - blobClient.SetAccessTier(targetTier); - - properties = blobClient.GetProperties().Value; - ASSERT_TRUE(properties.AccessTier.HasValue()); - ASSERT_TRUE(properties.IsAccessTierInferred.HasValue()); - EXPECT_FALSE(properties.IsAccessTierInferred.Value()); - EXPECT_TRUE(properties.AccessTierChangedOn.HasValue()); - blobItem = GetBlobItem(blobName); - ASSERT_TRUE(blobItem.Details.AccessTier.HasValue()); - ASSERT_TRUE(blobItem.Details.IsAccessTierInferred.HasValue()); - EXPECT_FALSE(blobItem.Details.IsAccessTierInferred.Value()); - EXPECT_TRUE(blobItem.Details.AccessTierChangedOn.HasValue()); + blockBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + EXPECT_NO_THROW(blockBlobClient.Delete()); - // set to archive, then rehydrate - blobClient.SetAccessTier(Blobs::Models::AccessTier::Archive); - blobClient.SetAccessTier(Blobs::Models::AccessTier::Hot); - properties = blobClient.GetProperties().Value; - ASSERT_TRUE(properties.ArchiveStatus.HasValue()); - EXPECT_EQ( - properties.ArchiveStatus.Value(), Blobs::Models::ArchiveStatus::RehydratePendingToHot); - ASSERT_TRUE(properties.RehydratePriority.HasValue()); - EXPECT_EQ(properties.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard); + std::string emptyFilename = RandomString(); + WriteFile(emptyFilename, std::vector{}); + blockBlobClient.UploadFrom(emptyFilename); + EXPECT_NO_THROW(blockBlobClient.Delete()); - blobItem = GetBlobItem(blobName); - ASSERT_TRUE(blobItem.Details.ArchiveStatus.HasValue()); - EXPECT_EQ( - blobItem.Details.ArchiveStatus.Value(), - Blobs::Models::ArchiveStatus::RehydratePendingToHot); - ASSERT_TRUE(blobItem.Details.RehydratePriority.HasValue()); - EXPECT_EQ( - blobItem.Details.RehydratePriority.Value(), Blobs::Models::RehydratePriority::Standard); + DeleteFile(emptyFilename); } - TEST_F(BlockBlobClientTest, DISABLED_SetTierCold) + TEST_F(BlockBlobClientTest, ConcurrentDownloadEmptyBlob) { - - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blockBlobClient = GetBlockBlobClientForTest(RandomString()); + std::string tempFilename = RandomString(); std::vector emptyContent; - std::string blobName(testName); - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - - // set to cold - blobClient.SetAccessTier(Blobs::Models::AccessTier::Cold); - auto properties = blobClient.GetProperties().Value; - EXPECT_EQ(properties.AccessTier.Value(), Blobs::Models::AccessTier::Cold); - } - - TEST_F(BlockBlobClientTest, SetTierWithLeaseId) - { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); + auto blobContent = Azure::Core::IO::MemoryBodyStream(emptyContent.data(), emptyContent.size()); + blockBlobClient.Upload(blobContent); + blockBlobClient.SetHttpHeaders(m_blobUploadOptions.HttpHeaders); + blockBlobClient.SetMetadata(m_blobUploadOptions.Metadata); - const std::string leaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - Blobs::BlobLeaseClient leaseClient(blobClient, leaseId); - leaseClient.Acquire(std::chrono::seconds(30)); + auto res = blockBlobClient.DownloadTo(emptyContent.data(), 0); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + res = blockBlobClient.DownloadTo(tempFilename); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); - EXPECT_THROW(blobClient.SetAccessTier(Blobs::Models::AccessTier::Cool), StorageException); + res = blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB)); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + res = blockBlobClient.DownloadTo(tempFilename); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); - Blobs::SetBlobAccessTierOptions options; - options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(blobClient.SetAccessTier(Blobs::Models::AccessTier::Cool, options)); - } + for (int c : {1, 2}) + { + Azure::Storage::Blobs::DownloadBlobToOptions options; + options.TransferOptions.InitialChunkSize = 10; + options.TransferOptions.ChunkSize = 10; + options.TransferOptions.Concurrency = c; - TEST_F(BlockBlobClientTest, UncommittedBlob) - { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - - const std::string blobName(testName); - - std::vector buffer(100); - Azure::Core::IO::MemoryBodyStream stream(buffer.data(), buffer.size()); - blobClient.StageBlock("YWJjZA==", stream); - - Blobs::GetBlockListOptions getBlockListOptions; - getBlockListOptions.ListType = Blobs::Models::BlockListType::All; - auto res = blobClient.GetBlockList(getBlockListOptions).Value; - EXPECT_FALSE(res.ETag.HasValue()); - EXPECT_EQ(res.BlobSize, 0); - EXPECT_TRUE(res.CommittedBlocks.empty()); - EXPECT_FALSE(res.UncommittedBlocks.empty()); - - auto blobItem = GetBlobItem(blobName, Blobs::Models::ListBlobsIncludeFlags::UncomittedBlobs); - EXPECT_EQ(blobItem.BlobSize, 0); - } - - TEST_F(BlobContainerClientTest, SourceTagsConditions) - { - auto const testName(GetTestNameLowerCase()); - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - auto sourceBlobClient = Azure::Storage::Blobs::BlockBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName, - InitClientOptions()); - std::vector buffer; - buffer.resize(1024); - } - - TEST_F(BlobContainerClientTest, SourceBlobAccessConditions) - { - auto const testName(GetTestNameLowerCase()); - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - auto sourceBlobClient = containerClient.GetBlockBlobClient(testName); - - std::vector buffer; - buffer.resize(1024); - auto createResponse = sourceBlobClient.UploadFrom(buffer.data(), buffer.size()); - Azure::ETag eTag = createResponse.Value.ETag; - auto lastModifiedTime = createResponse.Value.LastModified; - auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); - auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); - - auto destBlobClient = containerClient.GetBlockBlobClient(testName + "2"); - - { - Blobs::StartBlobCopyFromUriOptions options; - options.SourceAccessConditions.IfMatch = eTag; - EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); - options.SourceAccessConditions.IfMatch = DummyETag; - EXPECT_THROW( - destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); - - Blobs::CopyBlobFromUriOptions options2; - options2.SourceAccessConditions.IfMatch = eTag; - EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); - options2.SourceAccessConditions.IfMatch = DummyETag; - EXPECT_THROW( - destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), - StorageException); - } - { - Blobs::StartBlobCopyFromUriOptions options; - options.SourceAccessConditions.IfNoneMatch = DummyETag; - EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); - options.SourceAccessConditions.IfNoneMatch = eTag; - EXPECT_THROW( - destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); - - Blobs::CopyBlobFromUriOptions options2; - options2.SourceAccessConditions.IfNoneMatch = DummyETag; - EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); - options2.SourceAccessConditions.IfNoneMatch = eTag; - EXPECT_THROW( - destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), - StorageException); - } - { - Blobs::StartBlobCopyFromUriOptions options; - options.SourceAccessConditions.IfModifiedSince = timeBeforeStr; - EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); - options.SourceAccessConditions.IfModifiedSince = timeAfterStr; - EXPECT_THROW( - destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + res = blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + res = blockBlobClient.DownloadTo(tempFilename, options); + EXPECT_EQ(res.Value.BlobSize, 0); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), 0); + EXPECT_TRUE(res.Value.Details.ETag.HasValue()); + EXPECT_TRUE(IsValidTime(res.Value.Details.LastModified)); + EXPECT_EQ(res.Value.Details.HttpHeaders, m_blobUploadOptions.HttpHeaders); + EXPECT_EQ(res.Value.Details.Metadata, m_blobUploadOptions.Metadata); + EXPECT_EQ(res.Value.BlobType, Azure::Storage::Blobs::Models::BlobType::BlockBlob); + EXPECT_TRUE(ReadFile(tempFilename).empty()); + DeleteFile(tempFilename); - sourceBlobClient.GetProperties(); - Blobs::CopyBlobFromUriOptions options2; - options2.SourceAccessConditions.IfModifiedSince = timeBeforeStr; - EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); - options2.SourceAccessConditions.IfModifiedSince = timeAfterStr; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 0; EXPECT_THROW( - destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), StorageException); - } - { - Blobs::StartBlobCopyFromUriOptions options; - options.SourceAccessConditions.IfUnmodifiedSince = timeAfterStr; - EXPECT_NO_THROW(destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options)); - options.SourceAccessConditions.IfUnmodifiedSince = timeBeforeStr; - EXPECT_THROW( - destBlobClient.StartCopyFromUri(sourceBlobClient.GetUrl(), options), StorageException); + EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - Blobs::CopyBlobFromUriOptions options2; - options2.SourceAccessConditions.IfUnmodifiedSince = timeAfterStr; - EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2)); - options2.SourceAccessConditions.IfUnmodifiedSince = timeBeforeStr; + options.Range.Value().Offset = 1; EXPECT_THROW( - destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options2), + blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), StorageException); - } - - // lease - { - const std::string leaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - const std::string dummyLeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - Blobs::BlobLeaseClient leaseClient(destBlobClient, leaseId); - - leaseClient.Acquire(std::chrono::seconds(60)); + EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - Blobs::CopyBlobFromUriOptions options; - options.AccessConditions.LeaseId = dummyLeaseId; + options.Range.Value().Offset = 0; + options.Range.Value().Length = 1; EXPECT_THROW( - destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options), + blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), StorageException); - options.AccessConditions.LeaseId = leaseId; - EXPECT_NO_THROW(destBlobClient.CopyFromUri(sourceBlobClient.GetUrl() + GetSas(), options)); - leaseClient.Release(); - } - } - - TEST_F(BlockBlobClientTest, DISABLED_Immutability) - { - auto const testName(GetTestName()); - - auto blobClient = GetBlockBlobClient(testName); - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - - auto blobContainerClient = GetBlobContainerTestClient(); - ASSERT_TRUE(blobContainerClient.GetProperties().Value.HasImmutableStorageWithVersioning); - - Blobs::Models::BlobImmutabilityPolicy policy; - policy.ExpiresOn = Azure::DateTime::Parse( - Azure::DateTime(std::chrono::system_clock::now() + std::chrono::hours(24)) - .ToString(Azure::DateTime::DateFormat::Rfc1123), - Azure::DateTime::DateFormat::Rfc1123); - policy.PolicyMode = Blobs::Models::BlobImmutabilityPolicyMode::Unlocked; - auto setPolicyResponse = blobClient.SetImmutabilityPolicy(policy); - EXPECT_EQ(setPolicyResponse.Value.ImmutabilityPolicy, policy); - auto blobProperties = blobClient.GetProperties().Value; - ASSERT_TRUE(blobProperties.ImmutabilityPolicy.HasValue()); - EXPECT_EQ(blobProperties.ImmutabilityPolicy.Value(), policy); - auto downloadResponse = blobClient.Download(); - ASSERT_TRUE(downloadResponse.Value.Details.ImmutabilityPolicy.HasValue()); - EXPECT_EQ(downloadResponse.Value.Details.ImmutabilityPolicy.Value(), policy); - auto blobItem = GetBlobItem(testName, Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy); - ASSERT_TRUE(blobItem.Details.ImmutabilityPolicy.HasValue()); - EXPECT_EQ(blobItem.Details.ImmutabilityPolicy.Value(), policy); - - EXPECT_NO_THROW(blobClient.DeleteImmutabilityPolicy()); - blobProperties = blobClient.GetProperties().Value; - EXPECT_FALSE(blobProperties.ImmutabilityPolicy.HasValue()); - downloadResponse = blobClient.Download(); - ASSERT_FALSE(downloadResponse.Value.Details.ImmutabilityPolicy.HasValue()); - blobItem = GetBlobItem(testName, Blobs::Models::ListBlobsIncludeFlags::ImmutabilityPolicy); - ASSERT_FALSE(blobItem.Details.ImmutabilityPolicy.HasValue()); - - auto copySourceBlobClient = GetBlockBlobClient(testName + "src"); - copySourceBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - { - auto copyDestinationBlobClient = GetBlockBlobClient(testName + "dest1"); - Blobs::StartBlobCopyFromUriOptions options; - options.ImmutabilityPolicy = policy; - copyDestinationBlobClient.StartCopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options) - .PollUntilDone(std::chrono::seconds(1)); - EXPECT_EQ(copyDestinationBlobClient.GetProperties().Value.ImmutabilityPolicy.Value(), policy); - } - { - auto copyDestinationBlobClient = GetBlockBlobClient(testName + "dest2"); - Blobs::CopyBlobFromUriOptions options; - options.ImmutabilityPolicy = policy; - copyDestinationBlobClient.CopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options); - EXPECT_EQ(copyDestinationBlobClient.GetProperties().Value.ImmutabilityPolicy.Value(), policy); - } - } - - TEST_F(BlockBlobClientTest, DISABLED_ImmutabilityAccessCondition) - { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - std::vector emptyContent; - auto uploadResponse = blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - auto lastModifiedTime = uploadResponse.Value.LastModified; - auto timeBeforeStr = lastModifiedTime - std::chrono::minutes(1); - auto timeAfterStr = lastModifiedTime + std::chrono::minutes(1); - - Blobs::Models::BlobImmutabilityPolicy policy; - policy.ExpiresOn = Azure::DateTime::Parse( - Azure::DateTime(std::chrono::system_clock::now() + std::chrono::hours(24)) - .ToString(Azure::DateTime::DateFormat::Rfc1123), - Azure::DateTime::DateFormat::Rfc1123); - policy.PolicyMode = Blobs::Models::BlobImmutabilityPolicyMode::Unlocked; - - Blobs::SetBlobImmutabilityPolicyOptions options; - options.AccessConditions.IfUnmodifiedSince = timeBeforeStr; - EXPECT_THROW(blobClient.SetImmutabilityPolicy(policy, options), StorageException); - options.AccessConditions.IfUnmodifiedSince = timeAfterStr; - EXPECT_NO_THROW(blobClient.SetImmutabilityPolicy(policy, options)); - } - - TEST_F(BlockBlobClientTest, DISABLED_LegalHold) - { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - - auto setLegalHoldResponse = blobClient.SetLegalHold(true); - EXPECT_TRUE(setLegalHoldResponse.Value.HasLegalHold); - auto blobProperties = blobClient.GetProperties().Value; - EXPECT_TRUE(blobProperties.HasLegalHold); - auto downloadResponse = blobClient.Download(); - EXPECT_TRUE(downloadResponse.Value.Details.HasLegalHold); - auto blobItem = GetBlobItem(testName, Blobs::Models::ListBlobsIncludeFlags::LegalHold); - EXPECT_TRUE(blobItem.Details.HasLegalHold); - - setLegalHoldResponse = blobClient.SetLegalHold(false); - EXPECT_FALSE(setLegalHoldResponse.Value.HasLegalHold); - - auto copySourceBlobClient = GetBlockBlobClient(testName + "src"); - copySourceBlobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - { - auto copyDestinationBlobClient = GetBlockBlobClient(testName + "dest1"); - Blobs::StartBlobCopyFromUriOptions options; - options.HasLegalHold = true; - copyDestinationBlobClient.StartCopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options) - .PollUntilDone(std::chrono::seconds(1)); - EXPECT_TRUE(copyDestinationBlobClient.GetProperties().Value.HasLegalHold); - } - { - auto copyDestinationBlobClient = GetBlockBlobClient(testName + "dest2"); - Blobs::CopyBlobFromUriOptions options; - options.HasLegalHold = true; - copyDestinationBlobClient.CopyFromUri(copySourceBlobClient.GetUrl() + GetSas(), options); - EXPECT_TRUE(copyDestinationBlobClient.GetProperties().Value.HasLegalHold); - } - } - - TEST_F(BlockBlobClientTest, ContentHash) - { - auto const testName(GetTestName()); - auto srcBlobClient = GetBlockBlobClient(testName + "src"); - std::vector blobContent = RandomBuffer(100); - srcBlobClient.UploadFrom(blobContent.data(), blobContent.size()); - const std::vector contentMd5 - = Azure::Core::Cryptography::Md5Hash().Final(blobContent.data(), blobContent.size()); - const std::vector contentCrc64 - = Azure::Storage::Crc64Hash().Final(blobContent.data(), blobContent.size()); - - Azure::Core::IO::MemoryBodyStream stream(blobContent.data(), blobContent.size()); + EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); - { - auto destBlobClient = GetBlockBlobClient(testName + "dest0"); - Blobs::UploadBlockBlobOptions options; - options.TransactionalContentHash = ContentHash(); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; - options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); - stream.Rewind(); - EXPECT_THROW(destBlobClient.Upload(stream, options), StorageException); - options.TransactionalContentHash.Value().Value = contentMd5; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.Upload(stream, options)); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; - options.TransactionalContentHash.Value().Value - = Azure::Core::Convert::Base64Decode(DummyCrc64); - stream.Rewind(); - EXPECT_THROW(destBlobClient.Upload(stream, options), StorageException); - options.TransactionalContentHash.Value().Value = contentCrc64; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.Upload(stream, options)); - } - { - auto destBlobClient = GetBlockBlobClient(testName + "dest1"); - Blobs::UploadBlockBlobFromUriOptions options; - options.TransactionalContentHash = ContentHash(); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; - options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); - stream.Rewind(); - EXPECT_THROW( - destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options), - StorageException); - options.TransactionalContentHash.Value().Value = contentMd5; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options)); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; - options.TransactionalContentHash.Value().Value - = Azure::Core::Convert::Base64Decode(DummyCrc64); - stream.Rewind(); - EXPECT_THROW( - destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options), - StorageException); - options.TransactionalContentHash.Value().Value = contentCrc64; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options)); - } - { - auto destBlobClient = GetBlockBlobClient(testName + "dest2"); - Blobs::CopyBlobFromUriOptions options; - options.TransactionalContentHash = ContentHash(); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; - options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); - stream.Rewind(); - EXPECT_THROW( - destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); - options.TransactionalContentHash.Value().Value = contentMd5; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options)); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; - options.TransactionalContentHash.Value().Value - = Azure::Core::Convert::Base64Decode(DummyCrc64); - stream.Rewind(); - EXPECT_THROW( - destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options), StorageException); - options.TransactionalContentHash.Value().Value = contentCrc64; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.CopyFromUri(srcBlobClient.GetUrl() + GetSas(), options)); - } - { - auto destBlobClient = GetBlockBlobClient(testName + "dest3"); - Blobs::StageBlockOptions options; - options.TransactionalContentHash = ContentHash(); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; - options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); - stream.Rewind(); - EXPECT_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options), StorageException); - options.TransactionalContentHash.Value().Value = contentMd5; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options)); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; - options.TransactionalContentHash.Value().Value - = Azure::Core::Convert::Base64Decode(DummyCrc64); - stream.Rewind(); - EXPECT_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options), StorageException); - options.TransactionalContentHash.Value().Value = contentCrc64; - stream.Rewind(); - EXPECT_NO_THROW(destBlobClient.StageBlock("YWJjZA==", stream, options)); - } - { - auto destBlobClient = GetBlockBlobClient(testName + "dest4"); - Blobs::StageBlockFromUriOptions options; - options.TransactionalContentHash = ContentHash(); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Md5; - options.TransactionalContentHash.Value().Value = Azure::Core::Convert::Base64Decode(DummyMd5); - EXPECT_THROW( - destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options), - StorageException); - options.TransactionalContentHash.Value().Value = contentMd5; - EXPECT_NO_THROW( - destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options)); - options.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; - options.TransactionalContentHash.Value().Value - = Azure::Core::Convert::Base64Decode(DummyCrc64); + options.Range.Value().Offset = 100; + options.Range.Value().Length = 100; EXPECT_THROW( - destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options), + blockBlobClient.DownloadTo(emptyContent.data(), static_cast(8_MB), options), StorageException); - options.TransactionalContentHash.Value().Value = contentCrc64; - EXPECT_NO_THROW( - destBlobClient.StageBlockFromUri("YWJjZA==", srcBlobClient.GetUrl() + GetSas(), options)); + EXPECT_THROW(blockBlobClient.DownloadTo(tempFilename, options), StorageException); + DeleteFile(tempFilename); } } - TEST_F(BlockBlobClientTest, UploadFromUri) + TEST_F(BlockBlobClientTest, ConcurrentDownload_LIVEONLY_) { - auto const testName(GetTestName()); - auto srcBlobClient = GetBlockBlobClient(testName + "src"); - std::vector blobContent(100, 'a'); - srcBlobClient.UploadFrom(blobContent.data(), blobContent.size()); - std::map srcTags; - srcTags["srctags"] = "a1212"; - srcBlobClient.SetTags(srcTags); + auto blobClient = *m_blockBlobClient; + const auto blobContent = RandomBuffer(static_cast(8_MB)); + blobClient.UploadFrom(blobContent.data(), blobContent.size()); - const std::vector blobMd5 - = Azure::Core::Cryptography::Md5Hash().Final(blobContent.data(), blobContent.size()); - const std::vector blobCrc64 - = Azure::Storage::Crc64Hash().Final(blobContent.data(), blobContent.size()); + auto testDownloadToBuffer = [&](int concurrency, + int64_t downloadSize, + Azure::Nullable offset = {}, + Azure::Nullable length = {}, + Azure::Nullable initialChunkSize = {}, + Azure::Nullable chunkSize = {}) { + std::vector downloadBuffer; + std::vector expectedData = blobContent; + int64_t blobSize = blobContent.size(); + int64_t actualDownloadSize = std::min(downloadSize, blobSize); + if (offset.HasValue() && length.HasValue()) + { + actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), + blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + } + else + { + expectedData.clear(); + } + } + else if (offset.HasValue()) + { + actualDownloadSize = blobSize - offset.Value(); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), blobContent.end()); + } + else + { + expectedData.clear(); + } + } + downloadBuffer.resize(static_cast(downloadSize), '\x00'); + Blobs::DownloadBlobToOptions options; + options.TransferOptions.Concurrency = concurrency; + if (offset.HasValue() || length.HasValue()) + { + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset.Value(); + options.Range.Value().Length = length; + } + if (initialChunkSize.HasValue()) + { + options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } + if (actualDownloadSize > 0) + { + auto res = blobClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options); + EXPECT_EQ(res.Value.BlobSize, blobSize); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); + EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); + downloadBuffer.resize(static_cast(res.Value.ContentRange.Length.Value())); + EXPECT_EQ(downloadBuffer, expectedData); + } + else + { + EXPECT_THROW( + blobClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options), + StorageException); + } + }; - auto destBlobClient = GetBlockBlobClient(testName + "dest"); - auto uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas()); - EXPECT_TRUE(uploadFromUriResult.Value.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(uploadFromUriResult.Value.LastModified)); - EXPECT_TRUE(uploadFromUriResult.Value.VersionId.HasValue()); - EXPECT_TRUE(uploadFromUriResult.Value.IsServerEncrypted); - ASSERT_TRUE(uploadFromUriResult.Value.TransactionalContentHash.HasValue()); - if (uploadFromUriResult.Value.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) + auto testDownloadToFile = [&](int concurrency, + int64_t downloadSize, + Azure::Nullable offset = {}, + Azure::Nullable length = {}, + Azure::Nullable initialChunkSize = {}, + Azure::Nullable chunkSize = {}) { + std::string tempFilename = RandomString() + "file" + std::to_string(concurrency); + if (offset) + { + tempFilename.append(std::to_string(offset.Value())); + } + std::vector expectedData = blobContent; + int64_t blobSize = blobContent.size(); + int64_t actualDownloadSize = std::min(downloadSize, blobSize); + if (offset.HasValue() && length.HasValue()) + { + actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), + blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + } + else + { + expectedData.clear(); + } + } + else if (offset.HasValue()) + { + actualDownloadSize = blobSize - offset.Value(); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), blobContent.end()); + } + else + { + expectedData.clear(); + } + } + Blobs::DownloadBlobToOptions options; + options.TransferOptions.Concurrency = concurrency; + if (offset.HasValue() || length.HasValue()) + { + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset.Value(); + options.Range.Value().Length = length; + } + if (initialChunkSize.HasValue()) + { + options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } + if (actualDownloadSize > 0) + { + auto res = blobClient.DownloadTo(tempFilename, options); + EXPECT_EQ(res.Value.BlobSize, blobSize); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); + EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); + EXPECT_EQ(ReadFile(tempFilename), expectedData); + } + else + { + EXPECT_THROW(blobClient.DownloadTo(tempFilename, options), StorageException); + } + DeleteFile(tempFilename); + }; + + const int64_t blobSize = blobContent.size(); + std::vector> futures; + for (int c : {1, 2, 4}) { - EXPECT_EQ(uploadFromUriResult.Value.TransactionalContentHash.Value().Value, blobMd5); + // random range + for (int i = 0; i < 16; ++i) + { + int64_t offset = RandomInt(0, blobContent.size() - 1); + int64_t length = RandomInt(1, 64_KB); + futures.emplace_back(std::async( + std::launch::async, testDownloadToBuffer, c, blobSize, offset, length, 8_KB, 4_KB)); + futures.emplace_back(std::async( + std::launch::async, testDownloadToFile, c, blobSize, offset, length, 4_KB, 7_KB)); + } + + // buffer not big enough + Blobs::DownloadBlobToOptions options; + options.TransferOptions.Concurrency = c; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 1; + for (int64_t length : {1ULL, 2ULL, 4_KB, 5_KB, 8_KB, 11_KB, 20_KB}) + { + std::vector downloadBuffer; + downloadBuffer.resize(static_cast(length - 1)); + options.Range.Value().Length = length; + EXPECT_THROW( + blobClient.DownloadTo(downloadBuffer.data(), static_cast(length - 1), options), + std::runtime_error); + } } - else if ( - uploadFromUriResult.Value.TransactionalContentHash.Value().Algorithm - == HashAlgorithm::Crc64) + for (auto& f : futures) { - EXPECT_EQ(uploadFromUriResult.Value.TransactionalContentHash.Value().Value, blobCrc64); + f.get(); } - - Blobs::UploadBlockBlobFromUriOptions options; - options.CopySourceBlobProperties = false; - options.HttpHeaders.ContentLanguage = "en-US"; - options.HttpHeaders.ContentType = "application/octet-stream"; - options.Metadata["k"] = "v"; - options.AccessTier = Blobs::Models::AccessTier::Cool; - options.Tags["k1"] = "v1"; - uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options); - auto destBlobProperties = destBlobClient.GetProperties().Value; - destBlobProperties.HttpHeaders.ContentHash.Value.clear(); - EXPECT_EQ(destBlobProperties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(destBlobProperties.Metadata, options.Metadata); - EXPECT_EQ(destBlobProperties.AccessTier.Value(), options.AccessTier.Value()); - EXPECT_EQ(static_cast(destBlobProperties.TagCount.Value()), options.Tags.size()); - - options.CopySourceTagsMode = Blobs::Models::BlobCopySourceTagsMode::Copy; - options.Tags.clear(); - uploadFromUriResult = destBlobClient.UploadFromUri(srcBlobClient.GetUrl() + GetSas(), options); - EXPECT_EQ(destBlobClient.GetTags().Value, srcTags); } - TEST_F(BlockBlobClientTest, SetGetTagsWithLeaseId) + TEST_F(BlockBlobClientTest, ConcurrentUpload_LIVEONLY_) { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); - std::vector emptyContent; - blobClient.UploadFrom(emptyContent.data(), emptyContent.size()); - - const std::map tags{{"k", "v"}}; - - Blobs::BlobLeaseClient leaseClient(blobClient, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); - leaseClient.Acquire(std::chrono::seconds(60)); - - Blobs::SetBlobTagsOptions setTagsOptions; - setTagsOptions.AccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - EXPECT_THROW(blobClient.SetTags(tags, setTagsOptions), StorageException); - Blobs::GetBlobTagsOptions getTagsOptions; - getTagsOptions.AccessConditions.LeaseId = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); - EXPECT_THROW(blobClient.GetTags(getTagsOptions), StorageException); - - setTagsOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); - EXPECT_NO_THROW(blobClient.SetTags(tags, setTagsOptions)); - getTagsOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); - EXPECT_NO_THROW(blobClient.GetTags(getTagsOptions)); + const auto blobContent = RandomBuffer(static_cast(8_MB)); - leaseClient.Release(); - } + auto testUploadFromBuffer = [&](int concurrency, + int64_t bufferSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Blobs::UploadBlockBlobFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) + { + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } - TEST_F(BlockBlobClientTest, MaximumBlocks) - { - auto const testName(GetTestName()); - auto blobClient = GetBlockBlobClient(testName); + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + EXPECT_NO_THROW( + blobClient.UploadFrom(blobContent.data(), static_cast(bufferSize), options)); + std::vector downloadBuffer(static_cast(bufferSize), '\x00'); + blobClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(bufferSize)); + EXPECT_EQ(downloadBuffer, expectedData); + }; - const std::vector content(static_cast(1), 'a'); - const std::string blockId = Base64EncodeText(std::string(64, '0')); - auto blockContent = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); - blobClient.StageBlock(blockId, blockContent); + auto testUploadFromFile = [&](int concurrency, + int64_t fileSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Blobs::UploadBlockBlobFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) + { + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } - std::vector blockIds(50000, blockId); - EXPECT_NO_THROW(blobClient.CommitBlockList(blockIds)); + const std::string tempFileName = RandomString(); + WriteFile( + tempFileName, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(fileSize))); + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + EXPECT_NO_THROW(blobClient.UploadFrom(tempFileName, options)); + DeleteFile(tempFileName); + std::vector downloadBuffer(static_cast(fileSize), '\x00'); + blobClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(fileSize)); + EXPECT_EQ(downloadBuffer, expectedData); + }; - EXPECT_EQ( - blobClient.GetProperties().Value.BlobSize, - static_cast(blockIds.size() * content.size())); + for (int c : {1, 2, 4}) + { + std::vector> futures; + // random range + for (int i = 0; i < 16; ++i) + { + int64_t fileSize = RandomInt(1, 1_MB); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); + } + for (auto& f : futures) + { + f.get(); + } + } } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.hpp index 955cb5d852..5b15b2a4d0 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.hpp @@ -4,67 +4,26 @@ #include #include "blob_container_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class BlockBlobClientTest : public BlobContainerClientTest { - private: - std::shared_ptr m_blockBlobClient; - std::string m_blobName; - protected: - virtual void SetUp() override; - virtual void TearDown() override; - std::vector m_blobContent; - Azure::Storage::Blobs::UploadBlockBlobOptions m_blobUploadOptions; + void SetUp() override; - Azure::Storage::Blobs::BlockBlobClient const& GetBlockBlobClient(std::string const& blobName) + Blobs::BlockBlobClient GetBlockBlobClientForTest( + const std::string& blobName, + Blobs::BlobClientOptions clientOptions = Blobs::BlobClientOptions()) { - // Create container - auto const testName(GetTestNameLowerCase(true)); - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - m_blockBlobClient = std::make_shared( - containerClient.GetBlockBlobClient(blobName)); - - return *m_blockBlobClient; - } - - std::unique_ptr GetBlobClient(std::string const& blobName) - { - // Create container - auto const testName(GetTestNameLowerCase()); - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - return std::make_unique( - containerClient.GetBlobClient(blobName)); + auto containerClient = GetBlobContainerClientForTest(m_containerName, clientOptions); + return containerClient.GetBlockBlobClient(blobName); } - void SetOptions() - { - m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"key2", "Value2"}}; - m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; - m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; - m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; - m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; - m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; - m_blobUploadOptions.HttpHeaders.ContentHash.Value.clear(); - m_blobUploadOptions.AccessTier = Azure::Storage::Blobs::Models::AccessTier::Hot; - } - - void UploadBlockBlob(unsigned long long blobSize = 1_KB) - { - m_blobContent = std::vector(static_cast(blobSize), 'x'); - SetOptions(); - auto blobContent - = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_blockBlobClient->Upload(blobContent, m_blobUploadOptions); - m_blobUploadOptions.HttpHeaders.ContentHash - = m_blockBlobClient->GetProperties().Value.HttpHeaders.ContentHash; - } + protected: + std::shared_ptr m_blockBlobClient; + std::string m_blobName; + std::vector m_blobContent; + Azure::Storage::Blobs::UploadBlockBlobOptions m_blobUploadOptions; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 5d5374e8a3..1e3e66cc40 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -11,18 +11,47 @@ #include #include -namespace Azure { namespace Storage { namespace Test { +namespace Azure { namespace Storage { namespace Blobs { namespace Models { - void PageBlobClientTest::SetUp() { BlobContainerClientTest::SetUp(); } + bool operator==(const BlobHttpHeaders& lhs, const BlobHttpHeaders& rhs); - void PageBlobClientTest::TearDown() { BlobContainerClientTest::TearDown(); } +}}}} // namespace Azure::Storage::Blobs::Models - TEST_F(PageBlobClientTest, CreateDelete) +namespace Azure { namespace Storage { namespace Test { + + void PageBlobClientTest::SetUp() { - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); + BlobContainerClientTest::SetUp(); + if (shouldSkipTest()) + { + return; + } + m_blobName = RandomString(); + m_pageBlobClient = std::make_shared( + m_blobContainerClient->GetPageBlobClient(m_blobName)); + m_blobContent = std::vector(static_cast(1_KB), 'x'); + auto blobContent + = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); + m_pageBlobClient->Create(2_KB); + m_pageBlobClient->UploadPages(0, blobContent); + m_blobContent.resize(static_cast(2_KB)); + } - auto blobContentInfo = pageBlobClient.Create(0, m_blobUploadOptions); + TEST_F(PageBlobClientTest, CreateDelete) + { + auto pageBlobClient = *m_pageBlobClient; + + Blobs::CreatePageBlobOptions createOptions; + createOptions.HttpHeaders.ContentType = "application/x-binary"; + createOptions.HttpHeaders.ContentLanguage = "en-US"; + createOptions.HttpHeaders.ContentDisposition = "attachment"; + createOptions.HttpHeaders.CacheControl = "no-cache"; + createOptions.HttpHeaders.ContentEncoding = "identify"; + createOptions.Metadata = RandomMetadata(); + createOptions.Tags["key1"] = "value1"; + createOptions.Tags["key2"] = "value2"; + createOptions.Tags["key3 +-./:=_"] = "v1 +-./:=_"; + auto blobContentInfo = pageBlobClient.Create(0, createOptions); EXPECT_TRUE(blobContentInfo.Value.ETag.HasValue()); EXPECT_TRUE(IsValidTime(blobContentInfo.Value.LastModified)); EXPECT_TRUE(blobContentInfo.Value.VersionId.HasValue()); @@ -30,30 +59,20 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(blobContentInfo.Value.EncryptionScope.HasValue()); EXPECT_FALSE(blobContentInfo.Value.EncryptionKeySha256.HasValue()); + auto properties = pageBlobClient.GetProperties().Value; + EXPECT_EQ(properties.Metadata, createOptions.Metadata); + EXPECT_EQ(properties.HttpHeaders, createOptions.HttpHeaders); + EXPECT_EQ(pageBlobClient.GetTags().Value, createOptions.Tags); + pageBlobClient.Delete(); EXPECT_THROW(pageBlobClient.Delete(), StorageException); } - TEST_F(PageBlobClientTest, CreateWithTags) - { - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); - - Blobs::CreatePageBlobOptions options; - options.Tags["key1"] = "value1"; - options.Tags["key2"] = "value2"; - options.Tags["key3 +-./:=_"] = "v1 +-./:=_"; - pageBlobClient.Create(512, options); - - EXPECT_EQ(pageBlobClient.GetTags().Value, options.Tags); - } - TEST_F(PageBlobClientTest, Resize) { - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); + auto pageBlobClient = *m_pageBlobClient; - pageBlobClient.Create(0, m_blobUploadOptions); + pageBlobClient.Create(0); EXPECT_EQ(pageBlobClient.GetProperties().Value.BlobSize, 0); pageBlobClient.Resize(static_cast(2_KB)); @@ -62,15 +81,13 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(static_cast(pageBlobClient.GetProperties().Value.BlobSize), 1_KB); } - TEST_F(PageBlobClientTest, UploadClear_LIVEONLY_) + TEST_F(PageBlobClientTest, UploadClear) { - CHECK_SKIP_TEST(); - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); + auto pageBlobClient = *m_pageBlobClient; - std::vector blobContent = std::vector(static_cast(4_KB), 'x'); + std::vector blobContent = RandomBuffer(static_cast(4_KB)); - pageBlobClient.Create(8_KB, m_blobUploadOptions); + pageBlobClient.Create(8_KB); auto pageContent = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); pageBlobClient.UploadPages(2_KB, pageContent); // |_|_|x|x| |x|x|_|_| @@ -140,12 +157,11 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, GetPageRangesContinuation) { - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); + auto pageBlobClient = *m_pageBlobClient; - std::vector blobContent = std::vector(static_cast(512), 'x'); + std::vector blobContent = RandomBuffer(512); - pageBlobClient.Create(8_KB, m_blobUploadOptions); + pageBlobClient.Create(8_KB); auto pageContent = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); pageBlobClient.UploadPages(0, pageContent); pageContent.Rewind(); @@ -167,36 +183,28 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, UploadFromUri) { - auto const testName(GetTestName()); - auto testPageBlobClient = GetPageBlobClient(testName); - UploadPage(); - - auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "2", - InitClientOptions()); - pageBlobClient.Create(m_blobContent.size(), m_blobUploadOptions); - pageBlobClient.UploadPagesFromUri( - 0, testPageBlobClient.GetUrl() + GetSas(), {0, static_cast(m_blobContent.size())}); + auto pageBlobClient = *m_pageBlobClient; + + auto pageBlobClient2 = GetPageBlobClientTestForTest(RandomString()); + pageBlobClient2.Create(m_blobContent.size()); + pageBlobClient2.UploadPagesFromUri( + 0, pageBlobClient.GetUrl() + GetSas(), {0, static_cast(m_blobContent.size())}); + EXPECT_EQ( + pageBlobClient2.Download().Value.BodyStream->ReadToEnd(), + pageBlobClient.Download().Value.BodyStream->ReadToEnd()); } TEST_F(PageBlobClientTest, StartCopyIncremental) { - auto const testName(GetTestName()); - auto testPageBlobClient = GetPageBlobClient(testName); - UploadPage(); - - const std::string blobName(testName + "2"); - auto pageBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - blobName, - InitClientOptions()); - - std::string snapshot = testPageBlobClient.CreateSnapshot().Value.Snapshot; - Azure::Core::Url sourceUri(testPageBlobClient.WithSnapshot(snapshot).GetUrl()); - auto copyInfo = pageBlobClient.StartCopyIncremental(AppendQueryParameters(sourceUri, GetSas())); + auto pageBlobClient = *m_pageBlobClient; + + const std::string blobName = RandomString(); + auto pageBlobClient2 = GetPageBlobClientTestForTest(blobName); + + std::string snapshot = pageBlobClient.CreateSnapshot().Value.Snapshot; + Azure::Core::Url sourceUri(pageBlobClient.WithSnapshot(snapshot).GetUrl()); + auto copyInfo + = pageBlobClient2.StartCopyIncremental(AppendQueryParameters(sourceUri, GetSas())); EXPECT_EQ( copyInfo.GetRawResponse().GetStatusCode(), Azure::Core::Http::HttpStatusCode::Accepted); auto getPropertiesResult = copyInfo.PollUntilDone(PollInterval()); @@ -222,16 +230,14 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(blobItem.Details.IncrementalCopyDestinationSnapshot.Value().empty()); } - TEST_F(PageBlobClientTest, Lease_LIVEONLY_) + TEST_F(PageBlobClientTest, Lease) { - auto const testName(GetTestName()); - auto testPageBlobClient = GetPageBlobClient(testName); - UploadPage(); + auto pageBlobClient = *m_pageBlobClient; { - std::string leaseId1 = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + std::string leaseId1 = RandomUUID(); auto leaseDuration = std::chrono::seconds(20); - Blobs::BlobLeaseClient leaseClient(testPageBlobClient, leaseId1); + Blobs::BlobLeaseClient leaseClient(pageBlobClient, leaseId1); auto aLease = leaseClient.Acquire(leaseDuration).Value; EXPECT_TRUE(aLease.ETag.HasValue()); EXPECT_TRUE(IsValidTime(aLease.LastModified)); @@ -242,7 +248,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(aLease.LastModified)); EXPECT_EQ(aLease.LeaseId, leaseId1); - auto properties = testPageBlobClient.GetProperties().Value; + auto properties = pageBlobClient.GetProperties().Value; EXPECT_EQ(properties.LeaseState.Value(), Blobs::Models::LeaseState::Leased); EXPECT_EQ(properties.LeaseStatus.Value(), Blobs::Models::LeaseStatus::Locked); EXPECT_EQ(properties.LeaseDuration.Value(), Blobs::Models::LeaseDurationType::Fixed); @@ -252,7 +258,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(rLease.LastModified)); EXPECT_EQ(rLease.LeaseId, leaseId1); - std::string leaseId2 = Blobs::BlobLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = RandomUUID(); EXPECT_NE(leaseId1, leaseId2); auto cLease = leaseClient.Change(leaseId2).Value; EXPECT_TRUE(cLease.ETag.HasValue()); @@ -266,10 +272,9 @@ namespace Azure { namespace Storage { namespace Test { } { - Blobs::BlobLeaseClient leaseClient( - testPageBlobClient, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + Blobs::BlobLeaseClient leaseClient(pageBlobClient, RandomUUID()); auto aLease = leaseClient.Acquire(Blobs::BlobLeaseClient::InfiniteLeaseDuration).Value; - auto properties = testPageBlobClient.GetProperties().Value; + auto properties = pageBlobClient.GetProperties().Value; EXPECT_EQ(properties.LeaseDuration.Value(), Blobs::Models::LeaseDurationType::Infinite); auto brokenLease = leaseClient.Break().Value; EXPECT_TRUE(brokenLease.ETag.HasValue()); @@ -277,8 +282,7 @@ namespace Azure { namespace Storage { namespace Test { } { - Blobs::BlobLeaseClient leaseClient( - testPageBlobClient, Blobs::BlobLeaseClient::CreateUniqueLeaseId()); + Blobs::BlobLeaseClient leaseClient(pageBlobClient, RandomUUID()); auto leaseDuration = std::chrono::seconds(20); auto aLease = leaseClient.Acquire(leaseDuration).Value; auto brokenLease = leaseClient.Break().Value; @@ -293,8 +297,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, ContentHash) { - auto const testName(GetTestName()); - auto pageBlobClient = GetPageBlobClient(testName); + auto pageBlobClient = *m_pageBlobClient; std::vector blobContent = RandomBuffer(static_cast(4_KB)); const std::vector contentMd5 @@ -306,7 +309,7 @@ namespace Azure { namespace Storage { namespace Test { auto contentStream = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); pageBlobClient.UploadPages(0, contentStream); - auto pageBlobClient2 = GetPageBlobClient(testName + "2"); + auto pageBlobClient2 = GetPageBlobClientTestForTest(RandomString()); pageBlobClient2.Create(blobContent.size()); Blobs::UploadPagesOptions options1; @@ -355,32 +358,30 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, CreateIfNotExists) { - auto const testName(GetTestName()); - auto blobClient = GetPageBlobClient(testName); + auto pageBlobClient = GetPageBlobClientTestForTest(RandomString()); auto blobClientWithoutAuth = Azure::Storage::Blobs::PageBlobClient( - blobClient.GetUrl(), InitClientOptions()); + pageBlobClient.GetUrl(), InitClientOptions()); EXPECT_THROW(blobClientWithoutAuth.CreateIfNotExists(m_blobContent.size()), StorageException); { - auto response = blobClient.CreateIfNotExists(m_blobContent.size()); + auto response = pageBlobClient.CreateIfNotExists(m_blobContent.size()); EXPECT_TRUE(response.Value.Created); } auto blobContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - blobClient.UploadPages(0, blobContent); + pageBlobClient.UploadPages(0, blobContent); { - auto response = blobClient.CreateIfNotExists(m_blobContent.size()); + auto response = pageBlobClient.CreateIfNotExists(m_blobContent.size()); EXPECT_FALSE(response.Value.Created); } - auto downloadStream = std::move(blobClient.Download().Value.BodyStream); + auto downloadStream = std::move(pageBlobClient.Download().Value.BodyStream); EXPECT_EQ(downloadStream->ReadToEnd(Azure::Core::Context()), m_blobContent); } TEST_F(PageBlobClientTest, SourceBlobAccessConditions) { - auto const testName(GetTestName()); - auto sourceBlobClient = GetPageBlobClient(testName); + auto sourceBlobClient = GetPageBlobClientTestForTest("source" + RandomString()); const std::string url = sourceBlobClient.GetUrl() + GetSas(); @@ -391,11 +392,7 @@ namespace Azure { namespace Storage { namespace Test { auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(1); auto timeAfterStr = lastModifiedTime + std::chrono::seconds(1); - auto destBlobClient = Azure::Storage::Blobs::PageBlobClient::CreateFromConnectionString( - StandardStorageConnectionString(), - m_containerName, - testName + "2", - InitClientOptions()); + auto destBlobClient = GetPageBlobClientTestForTest("dest" + RandomString()); destBlobClient.Create(blobSize); { @@ -434,8 +431,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, UpdateSequenceNumber) { - auto const testName(GetTestName()); - auto blobClient = GetPageBlobClient(testName); + auto blobClient = *m_pageBlobClient; blobClient.Create(512); @@ -476,8 +472,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(PageBlobClientTest, PageBlobAccessConditions) { - auto const testName(GetTestName()); - auto blobClient = GetPageBlobClient(testName); + auto blobClient = *m_pageBlobClient; blobClient.Create(1024); Blobs::UpdatePageBlobSequenceNumberOptions updateSequenceNumberOptions; diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.hpp index 985bbbe3e4..a9fb009c99 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.hpp @@ -4,58 +4,25 @@ #include #include "blob_container_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class PageBlobClientTest : public BlobContainerClientTest { - private: - std::shared_ptr m_pageBlobClient; - protected: - std::string m_blobName; - Azure::Storage::Blobs::CreatePageBlobOptions m_blobUploadOptions; - std::vector m_blobContent; - - virtual void SetUp() override; - virtual void TearDown() override; + void SetUp() override; - Azure::Storage::Blobs::PageBlobClient const& GetPageBlobClient(std::string const& blobName) + Blobs::PageBlobClient GetPageBlobClientTestForTest( + const std::string& blobName, + Blobs::BlobClientOptions clientOptions = Blobs::BlobClientOptions()) { - // Create container - auto containerClient = GetBlobContainerTestClient(); - containerClient.CreateIfNotExists(); - - m_blobContent = std::vector(static_cast(1_KB), 'x'); - m_pageBlobClient = std::make_unique( - containerClient.GetPageBlobClient(blobName)); - - return *m_pageBlobClient; + auto containerClient = GetBlobContainerClientForTest(m_containerName, clientOptions); + return containerClient.GetPageBlobClient(blobName); } - void SetOptions() - { - m_blobUploadOptions.Metadata = {{"key1", "V1"}, {"key2", "Value2"}}; - m_blobUploadOptions.HttpHeaders.ContentType = "application/x-binary"; - m_blobUploadOptions.HttpHeaders.ContentLanguage = "en-US"; - m_blobUploadOptions.HttpHeaders.ContentDisposition = "attachment"; - m_blobUploadOptions.HttpHeaders.CacheControl = "no-cache"; - m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; - m_blobUploadOptions.HttpHeaders.ContentHash.Value.clear(); - } - - void UploadPage(unsigned long long blobSize = 1_KB) - { - SetOptions(); - - m_blobContent = std::vector(static_cast(blobSize), 'x'); - m_pageBlobClient->Create(m_blobContent.size(), m_blobUploadOptions); - auto pageContent - = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); - m_pageBlobClient->UploadPages(0, pageContent); - m_blobUploadOptions.HttpHeaders.ContentHash - = m_pageBlobClient->GetProperties().Value.HttpHeaders.ContentHash; - } + protected: + std::shared_ptr m_pageBlobClient; + std::string m_blobName; + std::vector m_blobContent; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp b/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp index a247ae3d5c..8fc594ee94 100644 --- a/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp +++ b/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp @@ -5,9 +5,18 @@ namespace Azure { namespace Storage { namespace Test { - TEST_F(ClientSecretCredentialTest, ClientSecretCredentialWorks_LIVEONLY_) + TEST_F(StorageTest, ClientSecretCredentialWorks_LIVEONLY_) { - auto containerClient = GetClientForTest(GetTestName()); + const std::string containerName = LowercaseRandomString(); + auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + StandardStorageConnectionString(), containerName); + auto credential = std::make_shared( + AadTenantId(), + AadClientId(), + AadClientSecret(), + InitClientOptions()); + containerClient = Blobs::BlobContainerClient( + containerClient.GetUrl(), credential, InitClientOptions()); EXPECT_NO_THROW(containerClient.Create()); EXPECT_NO_THROW(containerClient.Delete()); diff --git a/sdk/storage/azure-storage-common/test/ut/crypt_functions_test.cpp b/sdk/storage/azure-storage-common/test/ut/crypt_functions_test.cpp index dc0d1cbea5..c01c23e3a7 100644 --- a/sdk/storage/azure-storage-common/test/ut/crypt_functions_test.cpp +++ b/sdk/storage/azure-storage-common/test/ut/crypt_functions_test.cpp @@ -9,6 +9,9 @@ namespace Azure { namespace Storage { namespace Test { + class CryptFunctionsTest : public StorageTest { + }; + std::vector ToBinaryVector(const char* text) { const uint8_t* start = reinterpret_cast(text); @@ -111,34 +114,6 @@ namespace Azure { namespace Storage { namespace Test { crc64Single.Final(reinterpret_cast(allData.data()), allData.size())); } - TEST_F(CryptFunctionsTest, Crc64Hash_ExpectThrow) - { - std::string data = ""; - const uint8_t* ptr = reinterpret_cast(data.data()); - Crc64Hash instance; - -#if GTEST_HAS_DEATH_TEST - ASSERT_DEATH(instance.Final(nullptr, 1), ""); - ASSERT_DEATH(instance.Append(nullptr, 1), ""); -#endif - - EXPECT_EQ( - Azure::Core::Convert::Base64Encode(instance.Final(ptr, data.length())), "AAAAAAAAAAA="); - -#if GTEST_HAS_DEATH_TEST -#if defined(NDEBUG) - // Release build won't provide assert msg - ASSERT_DEATH(instance.Final(), ""); - ASSERT_DEATH(instance.Final(ptr, data.length()), ""); - ASSERT_DEATH(instance.Append(ptr, data.length()), ""); -#else - ASSERT_DEATH(instance.Final(), "Cannot call Final"); - ASSERT_DEATH(instance.Final(ptr, data.length()), "Cannot call Final"); - ASSERT_DEATH(instance.Append(ptr, data.length()), "Cannot call Append after calling Final"); -#endif -#endif - } - TEST_F(CryptFunctionsTest, Crc64Hash_CtorDtor) { { diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.cpp b/sdk/storage/azure-storage-common/test/ut/test_base.cpp index 7515c5a7b1..f8a3206313 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.cpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.cpp @@ -43,6 +43,37 @@ namespace Azure { namespace Storage { namespace Test { constexpr static const char* AadClientIdValue = ""; constexpr static const char* AadClientSecretValue = ""; + void StorageTest::SetUp() + { + Azure::Core::Test::TestBase::SetUpTestBase(AZURE_TEST_RECORDING_DIR); + + if (m_testContext.IsLiveMode()) + { + m_randomGenerator.seed(std::random_device{}()); + } + else + { + auto seedStr = GetIdentifier(); + std::seed_seq seedSeq(seedStr.begin(), seedStr.end()); + m_randomGenerator.seed(seedSeq); + } + } + + void StorageTest::TearDown() + { + for (auto& f : m_resourceCleanupFunctions) + { + try + { + f(); + } + catch (...) + { + } + } + TestBase::TearDown(); + } + const std::string& StorageTest::StandardStorageConnectionString() { const static std::string connectionString = [&]() -> std::string { @@ -176,53 +207,27 @@ namespace Azure { namespace Storage { namespace Test { return absoluteUrl; } - static thread_local std::mt19937_64 random_generator(std::random_device{}()); - uint64_t StorageTest::RandomInt(uint64_t minNumber, uint64_t maxNumber) { - std::uniform_int_distribution distribution(minNumber, maxNumber); - return distribution(random_generator); + uint64_t val = m_randomGenerator(); + if (minNumber == 0 && maxNumber == std::numeric_limits::max()) + { + return val; + } + return minNumber + val % (maxNumber - minNumber + 1); } - static char RandomChar() + char StorageTest::RandomChar() { const char charset[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; - std::uniform_int_distribution distribution(0, sizeof(charset) - 2); - return charset[distribution(random_generator)]; + return charset[RandomInt(0, sizeof(charset) - 2)]; } std::string StorageTest::RandomString(size_t size) { std::string str; str.resize(size); - std::generate(str.begin(), str.end(), RandomChar); - return str; - } - - std::string StorageTest::GetStringOfSize(size_t size, bool lowercase) - { - auto const testName = GetTestName(); - auto const testNameSize = testName.size(); - auto duplicationTimes = size / testNameSize; - auto leftToFill = size % testNameSize; - std::string str; - - while (duplicationTimes != 0) - { - str.append(testName); - duplicationTimes -= 1; - } - while (leftToFill != 0) - { - // do % 10 to use only one digit per appending - str.append(std::to_string(leftToFill % 10)); - leftToFill -= 1; - } - - if (lowercase) - { - return Azure::Core::_internal::StringExtensions::ToLower(str); - } + std::generate(str.begin(), str.end(), [this]() { return RandomChar(); }); return str; } @@ -231,12 +236,12 @@ namespace Azure { namespace Storage { namespace Test { return Azure::Core::_internal::StringExtensions::ToLower(RandomString(size)); } - Storage::Metadata StorageTest::GetMetadata(size_t size) + Storage::Metadata StorageTest::RandomMetadata(size_t size) { Storage::Metadata result; for (size_t i = 0; i < size; ++i) { - result["meta" + std::to_string(i % 10)] = "value"; + result["meta" + LowercaseRandomString(5)] = RandomString(10); } return result; } @@ -253,11 +258,10 @@ namespace Azure { namespace Storage { namespace Test { *(start_addr++) = RandomChar(); } - std::uniform_int_distribution distribution( - 0ULL, std::numeric_limits::max()); while (start_addr + rand_int_size <= end_addr) { - *reinterpret_cast(start_addr) = distribution(random_generator); + *reinterpret_cast(start_addr) + = RandomInt(0ULL, std::numeric_limits::max()); start_addr += rand_int_size; } while (start_addr < end_addr) @@ -266,6 +270,46 @@ namespace Azure { namespace Storage { namespace Test { } } + std::vector StorageTest::RandomBuffer(size_t length) + { + std::vector result(length); + if (length != 0) + { + char* dataPtr = reinterpret_cast(&result[0]); + RandomBuffer(dataPtr, length); + } + return result; + } + + std::string StorageTest::RandomUUID() + { + std::vector randomNum = RandomBuffer(16); + char buffer[37]; + + std::snprintf( + buffer, + sizeof(buffer), + "%2.2x%2.2x%2.2x%2.2x-%2.2x%2.2x-%2.2x%2.2x-%2.2x%2.2x-%2.2x%2.2x%2.2x%2.2x%2.2x%2.2x", + randomNum[0], + randomNum[1], + randomNum[2], + randomNum[3], + randomNum[4], + randomNum[5], + randomNum[6], + randomNum[7], + randomNum[8], + randomNum[9], + randomNum[10], + randomNum[11], + randomNum[12], + randomNum[13], + randomNum[14], + randomNum[15]); + + return std::string(buffer); + } + std::vector StorageTest::ReadFile(const std::string& filename) { #if defined(_MSC_VER) @@ -302,17 +346,6 @@ namespace Azure { namespace Storage { namespace Test { void StorageTest::DeleteFile(const std::string& filename) { std::remove(filename.data()); } - std::vector StorageTest::RandomBuffer(size_t length) - { - std::vector result(length); - if (length != 0) - { - char* dataPtr = reinterpret_cast(&result[0]); - RandomBuffer(dataPtr, length); - } - return result; - } - std::string StorageTest::InferSecondaryUrl(const std::string primaryUrl) { Azure::Core::Url secondaryUri(primaryUrl); diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.hpp b/sdk/storage/azure-storage-common/test/ut/test_base.hpp index 05f28531a1..da26d99ee3 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.hpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,13 @@ namespace Azure { namespace Storage { public: StorageTest() { TestBase::SetUpTestSuiteLocal(AZURE_TEST_ASSETS_DIR); } + const static Azure::ETag DummyETag; + const static Azure::ETag DummyETag2; + /* cspell:disable-next-line */ + constexpr static const char* DummyMd5 = "tQbD1aMPeB+LiPffUwFQJQ=="; + /* cspell:disable-next-line */ + constexpr static const char* DummyCrc64 = "+DNR5PON4EM="; + protected: const std::string& StandardStorageConnectionString(); const std::string& PremiumStorageConnectionString(); @@ -39,40 +47,47 @@ namespace Azure { namespace Storage { const std::string& AadClientId(); const std::string& AadClientSecret(); - std::string GetContainerValidName() const + void SetUp() override; + void TearDown() override; + + /** + * @brief Retruns a string related to test suite name and test case name. + */ + std::string GetIdentifier() const { - std::string name(m_testContext.GetTestSuiteName() + m_testContext.GetTestName()); - // Make sure the name is less than 63 characters long - auto const nameSize = name.size(); - size_t const maxContainerNameSize = 63; - if (nameSize > maxContainerNameSize) + size_t MaxLength = 63; + std::string name = m_testContext.GetTestSuiteName() + m_testContext.GetTestName(); + if (name[0] == '-') { - name = std::string(name.begin() + nameSize - maxContainerNameSize, name.end()); + name = name.substr(1); } - // Check name won't start with `-` - if (name[0] == '-') + if (name.length() > MaxLength) { - name = std::string(name.begin() + 1, name.end()); + name.resize(MaxLength); } - return Azure::Core::_internal::StringExtensions::ToLower(name); + return name; } - std::string GetFileSystemValidName() const + /** + * @brief Retruns a lowercase string related to test suite name and test case name. + */ + std::string GetLowercaseIdentifier() const { - std::string name(m_testContext.GetTestSuiteName() + m_testContext.GetTestName()); - // Make sure the name is less than 63 characters long - auto const nameSize = name.size(); - size_t const maxContainerNameSize = 63; - if (nameSize > maxContainerNameSize) - { - name = std::string(name.begin() + nameSize - maxContainerNameSize, name.end()); - } - // Check name won't start with `-` - if (name[0] == '-') + return Azure::Core::_internal::StringExtensions::ToLower(GetIdentifier()); + } + + bool IsValidTime(const Azure::DateTime& datetime) const + { + // Playback won't check dates + if (m_testContext.IsPlaybackMode()) { - name = std::string(name.begin() + 1, name.end()); + return true; } - return Azure::Core::_internal::StringExtensions::ToLower(name); + + // We assume datetime within a week is valid. + const auto minTime = std::chrono::system_clock::now() - std::chrono::hours(24 * 7); + const auto maxTime = std::chrono::system_clock::now() + std::chrono::hours(24 * 7); + return datetime > minTime && datetime < maxTime; } static std::string GetTestEncryptionScope() @@ -85,29 +100,31 @@ namespace Azure { namespace Storage { const Azure::Core::Url& url, const std::string& queryParameters); - /* cspell:disable-next-line */ - constexpr static const char* DummyMd5 = "tQbD1aMPeB+LiPffUwFQJQ=="; - /* cspell:disable-next-line */ - constexpr static const char* DummyCrc64 = "+DNR5PON4EM="; - - static uint64_t RandomInt( + /** + * Random functions below are not thread-safe. You must NOT call them from multiple threads. + * + * To make record-playback testing work, you have to call these functions in a determined way, + * e.g. always in the same order, the same times and with the same parameters. + * + * Note that in C++, evaluation order of function parameters is undefined. So you CANNOT do: + * `auto ret = RandomInt() + RandomInt();` + * or + * `auto ret = function1(RandomInt()) + function2(RandomInt());` + */ + uint64_t RandomInt( uint64_t minNumber = std::numeric_limits::min(), uint64_t maxNumber = std::numeric_limits::max()); - - static std::string RandomString(size_t size = 10); - - std::string GetStringOfSize(size_t size = 10, bool lowercase = false); - - static std::string LowercaseRandomString(size_t size = 10); - - static Storage::Metadata GetMetadata(size_t size = 5); - - static void RandomBuffer(char* buffer, size_t length); - static void RandomBuffer(uint8_t* buffer, size_t length) + char RandomChar(); + std::string RandomString(size_t size = 10); + std::string LowercaseRandomString(size_t size = 10); + Storage::Metadata RandomMetadata(size_t size = 5); + void RandomBuffer(char* buffer, size_t length); + void RandomBuffer(uint8_t* buffer, size_t length) { RandomBuffer(reinterpret_cast(buffer), length); } - static std::vector RandomBuffer(size_t length); + std::vector RandomBuffer(size_t length); + std::string RandomUUID(); static std::vector ReadBodyStream( std::unique_ptr& stream) @@ -135,14 +152,11 @@ namespace Azure { namespace Storage { return Azure::Core::Convert::Base64Encode(std::vector(text.begin(), text.end())); } - void SetUp() override - { - Azure::Core::Test::TestBase::SetUpTestBase(AZURE_TEST_RECORDING_DIR); - } + protected: + std::vector> m_resourceCleanupFunctions; - public: - const static Azure::ETag DummyETag; - const static Azure::ETag DummyETag2; + private: + std::mt19937_64 m_randomGenerator; }; constexpr inline unsigned long long operator""_KB(unsigned long long x) { return x * 1024; } @@ -159,49 +173,6 @@ namespace Azure { namespace Storage { return x * 1024 * 1024 * 1024 * 1024; } - class CryptFunctionsTest : public StorageTest { - }; - - class ClientSecretCredentialTest : public StorageTest { - - private: - std::unique_ptr m_client; - - protected: - std::shared_ptr m_credential; - std::string m_containerName; - - // Required to rename the test propertly once the test is started. - // We can only know the test instance name until the test instance is run. - Azure::Storage::Blobs::BlobContainerClient const& GetClientForTest( - std::string const& testName) - { - // set the interceptor for the current test - m_testContext.RenameTest(testName); - return *m_client; - } - - void SetUp() override - { - StorageTest::SetUp(); - m_containerName = Azure::Core::_internal::StringExtensions::ToLower(GetTestName()); - - m_credential = std::make_shared( - AadTenantId(), AadClientId(), AadClientSecret()); - - Azure::Storage::Blobs::BlobClientOptions options; - - m_client = InitTestClient< - Azure::Storage::Blobs::BlobContainerClient, - Azure::Storage::Blobs::BlobClientOptions>( - Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_containerName) - .GetUrl(), - m_credential, - options); - } - }; - } // namespace Test namespace Files { namespace DataLake { namespace _detail { diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp index 84f77e1787..43aabe43e2 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp @@ -14,30 +14,25 @@ namespace Azure { namespace Storage { namespace Test { void DataLakeDirectoryClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); - CHECK_SKIP_TEST(); - - m_directoryName = GetFileSystemValidName(); + if (shouldSkipTest()) + { + return; + } + m_directoryName = RandomString(); m_directoryClient = std::make_shared( m_fileSystemClient->GetDirectoryClient(m_directoryName)); m_fileSystemClient->GetFileClient(m_directoryName).Create(); } - void DataLakeDirectoryClientTest::TearDown() - { - CHECK_SKIP_TEST(); - m_fileSystemClient->GetFileClient(m_directoryName).Delete(); - DataLakeFileSystemClientTest::TearDown(); - } - TEST_F(DataLakeDirectoryClientTest, CreateDeleteDirectory) { - const std::string testName(GetTestName()); + const std::string baseName = RandomString(); { // Normal create/delete. std::vector directoryClient; for (int32_t i = 0; i < 5; ++i) { - auto client = m_fileSystemClient->GetDirectoryClient(testName + std::to_string(i)); + auto client = m_fileSystemClient->GetDirectoryClient(baseName + std::to_string(i)); EXPECT_NO_THROW(client.Create()); directoryClient.emplace_back(std::move(client)); } @@ -51,7 +46,7 @@ namespace Azure { namespace Storage { namespace Test { std::vector directoryClient; for (int32_t i = 0; i < 2; ++i) { - auto client = m_fileSystemClient->GetDirectoryClient(testName + "2" + std::to_string(i)); + auto client = m_fileSystemClient->GetDirectoryClient(baseName + "2" + std::to_string(i)); EXPECT_NO_THROW(client.Create()); directoryClient.emplace_back(std::move(client)); } @@ -72,7 +67,7 @@ namespace Azure { namespace Storage { namespace Test { std::vector directoryClient; for (int32_t i = 0; i < 2; ++i) { - auto client = m_fileSystemClient->GetDirectoryClient(testName + "3" + std::to_string(i)); + auto client = m_fileSystemClient->GetDirectoryClient(baseName + "3" + std::to_string(i)); EXPECT_NO_THROW(client.Create()); directoryClient.emplace_back(std::move(client)); } @@ -92,7 +87,7 @@ namespace Azure { namespace Storage { namespace Test { { // Recursive delete works. std::vector directoryClient; - auto rootDir = testName + "root"; + auto rootDir = baseName + "root"; auto rootDirClient = m_fileSystemClient->GetDirectoryClient(rootDir); EXPECT_NO_THROW(rootDirClient.Create()); for (int32_t i = 0; i < 5; ++i) @@ -108,9 +103,9 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, CreateDeleteIfExistsDirectory) { - std::string const testName(GetTestName()); + std::string const baseName = RandomString(); { - auto client = m_fileSystemClient->GetDirectoryClient(testName + "1"); + auto client = m_fileSystemClient->GetDirectoryClient(baseName + "1"); bool created = false; bool deleted = false; EXPECT_NO_THROW(created = client.Create().Value.Created); @@ -123,38 +118,34 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(deleted); } { - auto client = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - GetTestNameLowerCase(), - testName + "2", - InitClientOptions()); + auto dirClient = m_fileSystemClient->GetDirectoryClient(RandomString()); bool deleted = false; - EXPECT_NO_THROW(deleted = client.DeleteEmptyIfExists().Value.Deleted); + EXPECT_NO_THROW(deleted = dirClient.DeleteEmptyIfExists().Value.Deleted); EXPECT_FALSE(deleted); } } TEST_F(DataLakeDirectoryClientTest, RenameFile) { - const std::string testName(GetTestName()); - const std::string baseDirectoryName = testName + "1"; + const std::string baseName = RandomString(); + const std::string baseDirectoryName = baseName + "1"; auto baseDirectoryClient = m_fileSystemClient->GetDirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); - const std::string oldFilename = testName + "2"; + const std::string oldFilename = baseName + "2"; auto oldFileClient = baseDirectoryClient.GetSubdirectoryClient(oldFilename); oldFileClient.Create(); - const std::string newFilename = testName + "3"; + const std::string newFilename = baseName + "3"; auto newFileClient = baseDirectoryClient.RenameFile(oldFilename, baseDirectoryName + "/" + newFilename).Value; EXPECT_NO_THROW(newFileClient.GetProperties()); EXPECT_NO_THROW(baseDirectoryClient.GetSubdirectoryClient(newFilename).GetProperties()); EXPECT_THROW(oldFileClient.GetProperties(), StorageException); - const std::string newFileSystemName = GetTestNameLowerCase() + "1"; - const std::string newFilename2 = testName + "4"; + const std::string newFileSystemName = LowercaseRandomString(); + const std::string newFilename2 = baseName + "4"; - auto newFileSystem = m_dataLakeServiceClient->GetFileSystemClient(newFileSystemName); + auto newFileSystem = GetFileSystemClientForTest(newFileSystemName); newFileSystem.Create(); Files::DataLake::RenameFileOptions options; @@ -167,25 +158,26 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(newFileClient.GetProperties(), StorageException); } - TEST_F(DataLakeDirectoryClientTest, RenameFileSasAuthentication_LIVEONLY_) + TEST_F(DataLakeDirectoryClientTest, RenameFileSasAuthentication) { - const std::string testName(GetTestName()); - const std::string sourceFilename = testName + "1"; - const std::string destinationFilename = testName + "2"; + const std::string baseName = RandomString(); + const std::string sourceFilename = baseName + "1"; + const std::string destinationFilename = baseName + "2"; auto baseDirectoryClient = m_fileSystemClient->GetDirectoryClient("based"); baseDirectoryClient.Create(); auto fileClient = baseDirectoryClient.GetFileClient(sourceFilename); fileClient.CreateIfNotExists(); Files::DataLake::DataLakeDirectoryClient directoryClientSas( - Files::DataLake::_detail::GetDfsUrlFromUrl(baseDirectoryClient.GetUrl()) + GetSas()); + Files::DataLake::_detail::GetDfsUrlFromUrl(baseDirectoryClient.GetUrl()) + GetSas(), + InitClientOptions()); directoryClientSas.RenameFile(sourceFilename, destinationFilename); EXPECT_THROW( baseDirectoryClient.GetFileClient(sourceFilename).GetProperties(), StorageException); EXPECT_NO_THROW(m_fileSystemClient->GetFileClient(destinationFilename).GetProperties()); - const std::string sourceDirectoryName = testName + "3"; - const std::string destinationDirectoryName = testName + "4"; + const std::string sourceDirectoryName = baseName + "3"; + const std::string destinationDirectoryName = baseName + "4"; auto directoryClient = baseDirectoryClient.GetSubdirectoryClient(sourceDirectoryName); directoryClient.CreateIfNotExists(); @@ -199,16 +191,16 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, RenameFileAccessCondition) { - const std::string testName(GetTestName()); + const std::string baseName = RandomString(); - const std::string baseDirectoryName = testName + "1"; + const std::string baseDirectoryName = baseName + "1"; auto baseDirectoryClient = m_fileSystemClient->GetDirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); - const std::string oldFilename = testName + "2"; + const std::string oldFilename = baseName + "2"; auto oldFileClient = baseDirectoryClient.GetSubdirectoryClient(oldFilename); oldFileClient.Create(); - const std::string newFilename = testName + "3"; + const std::string newFilename = baseName + "3"; Files::DataLake::RenameFileOptions options; options.SourceAccessConditions.IfModifiedSince @@ -238,16 +230,16 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, RenameDirectory) { - const std::string testName(GetTestName()); + const std::string baseName = RandomString(); - const std::string baseDirectoryName = testName + "1"; + const std::string baseDirectoryName = baseName + "1"; auto baseDirectoryClient = m_fileSystemClient->GetDirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); - const std::string oldDirectoryName = testName + "2"; + const std::string oldDirectoryName = baseName + "2"; auto oldDirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldDirectoryName); oldDirectoryClient.Create(); - const std::string newDirectoryName = testName + "3"; + const std::string newDirectoryName = baseName + "3"; auto newDirectoryClient = baseDirectoryClient .RenameSubdirectory(oldDirectoryName, baseDirectoryName + "/" + newDirectoryName) @@ -256,10 +248,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(baseDirectoryClient.GetSubdirectoryClient(newDirectoryName).GetProperties()); EXPECT_THROW(oldDirectoryClient.GetProperties(), StorageException); - const std::string newFileSystemName = GetTestNameLowerCase(); - const std::string newDirectoryName2 = testName + "4"; + const std::string newFileSystemName = LowercaseRandomString(); + const std::string newDirectoryName2 = baseName + "4"; - auto newFileSystem = m_dataLakeServiceClient->GetFileSystemClient(newFileSystemName); + auto newFileSystem = GetFileSystemClientForTest(newFileSystemName); newFileSystem.Create(); Files::DataLake::RenameDirectoryOptions options; @@ -276,16 +268,16 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, RenameDirectoryAccessCondition) { - const std::string testName(GetTestName()); + const std::string baseName = RandomString(); - const std::string baseDirectoryName = testName + "1"; + const std::string baseDirectoryName = baseName + "1"; auto baseDirectoryClient = m_fileSystemClient->GetDirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); - const std::string oldDirectoryName = testName + "2"; + const std::string oldDirectoryName = baseName + "2"; auto oldDirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldDirectoryName); oldDirectoryClient.Create(); - const std::string newDirectoryName = testName + "3"; + const std::string newDirectoryName = baseName + "3"; Files::DataLake::RenameDirectoryOptions options; options.SourceAccessConditions.IfModifiedSince @@ -319,8 +311,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, DirectoryMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata1)); @@ -333,9 +325,9 @@ namespace Azure { namespace Storage { namespace Test { { // Create path with metadata works - const std::string testName(GetTestName()); - auto client1 = m_fileSystemClient->GetDirectoryClient(testName + "1"); - auto client2 = m_fileSystemClient->GetDirectoryClient(testName + "2"); + const std::string baseName = RandomString(); + auto client1 = m_fileSystemClient->GetDirectoryClient(baseName + "1"); + auto client2 = m_fileSystemClient->GetDirectoryClient(baseName + "2"); Files::DataLake::CreatePathOptions options1; Files::DataLake::CreatePathOptions options2; options1.Metadata = metadata1; @@ -356,8 +348,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, DirectoryProperties) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Get Metadata via properties works EXPECT_NO_THROW(m_directoryClient->SetMetadata(metadata1)); @@ -385,24 +377,30 @@ namespace Azure { namespace Storage { namespace Test { { // HTTP headers works. - auto httpHeader = GetInterestingHttpHeaders(); + auto httpHeaders = Files::DataLake::Models::PathHttpHeaders(); + httpHeaders.ContentType = "application/x-binary"; + httpHeaders.ContentLanguage = "en-US"; + httpHeaders.ContentDisposition = "attachment"; + httpHeaders.CacheControl = "no-cache"; + httpHeaders.ContentEncoding = "identity"; std::vector directoryClient; - const std::string testName(GetTestName()); + const std::string baseName = RandomString(); for (int32_t i = 0; i < 2; ++i) { - auto client = m_fileSystemClient->GetDirectoryClient(testName + std::to_string(i)); + auto client = m_fileSystemClient->GetDirectoryClient(baseName + std::to_string(i)); Files::DataLake::CreatePathOptions options; - options.HttpHeaders = httpHeader; + options.HttpHeaders = httpHeaders; EXPECT_NO_THROW(client.Create(options)); directoryClient.emplace_back(std::move(client)); } for (const auto& client : directoryClient) { auto result = client.GetProperties(); - EXPECT_EQ(httpHeader.CacheControl, result.Value.HttpHeaders.CacheControl); - EXPECT_EQ(httpHeader.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); - EXPECT_EQ(httpHeader.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); - EXPECT_EQ(httpHeader.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); + EXPECT_EQ(httpHeaders.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); + EXPECT_EQ(httpHeaders.CacheControl, result.Value.HttpHeaders.CacheControl); + EXPECT_EQ(httpHeaders.ContentEncoding, result.Value.HttpHeaders.ContentEncoding); EXPECT_NO_THROW(client.DeleteEmpty()); } } @@ -411,10 +409,10 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeDirectoryClientTest, DirectoryAccessControlRecursive) { // Setup directories. - const std::string testName(GetTestName()); - auto rootDirectoryName = testName + "1"; - auto directoryName1 = testName + "2"; - auto directoryName2 = testName + "3"; + const std::string baseName = RandomString(); + auto rootDirectoryName = baseName + "1"; + auto directoryName1 = baseName + "2"; + auto directoryName2 = baseName + "3"; auto rootDirectoryClient = m_fileSystemClient->GetDirectoryClient(rootDirectoryName); rootDirectoryClient.Create(); auto directoryClient1 @@ -426,7 +424,7 @@ namespace Azure { namespace Storage { namespace Test { { // Set Acls recursive. - std::vector acls = GetValidAcls(); + std::vector acls = GetAclsForTesting(); EXPECT_NO_THROW(rootDirectoryClient.SetAccessControlListRecursive(acls)); std::vector resultAcls1; std::vector resultAcls2; @@ -447,7 +445,7 @@ namespace Azure { namespace Storage { namespace Test { } { // Update Acls recursive. - std::vector originalAcls = GetValidAcls(); + std::vector originalAcls = GetAclsForTesting(); Files::DataLake::Models::Acl newAcl; newAcl.Type = "group"; newAcl.Id = ""; @@ -525,7 +523,7 @@ namespace Azure { namespace Storage { namespace Test { } { // Remove Acls recursive. - std::vector originalAcls = GetValidAcls(); + std::vector originalAcls = GetAclsForTesting(); Files::DataLake::Models::Acl removeAcl; removeAcl.Type = "user"; removeAcl.Id = "72a3f86f-271f-439e-b031-25678907d381"; @@ -652,7 +650,7 @@ namespace Azure { namespace Storage { namespace Test { } { // verify user has only one entry - std::vector originalAcls = GetValidAcls(); + std::vector originalAcls = GetAclsForTesting(); auto userFinder = [&originalAcls](const Files::DataLake::Models::Acl& targetAcl) { return targetAcl.Type == "user" && targetAcl.Id == originalAcls[0].Id; }; @@ -677,70 +675,4 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(DataLakeDirectoryClientTest, ConstructorsWorks) - { - { - // Create from connection string validates static creator function and shared key - // constructor. - auto directoryName = GetTestName() + "1"; - auto connectionStringClient - = Azure::Storage::Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - directoryName, - InitClientOptions()); - EXPECT_NO_THROW(connectionStringClient.Create()); - EXPECT_NO_THROW(connectionStringClient.DeleteRecursive()); - } - - { - // Create from client secret credential. - std::shared_ptr credential - = std::make_shared( - AadTenantId(), AadClientId(), AadClientSecret()); - - Azure::Storage::Files::DataLake::DataLakeClientOptions options; - - auto clientSecretClient = InitTestClient< - Azure::Storage::Files::DataLake::DataLakeDirectoryClient, - Azure::Storage::Files::DataLake::DataLakeClientOptions>( - Azure::Storage::Files::DataLake::_detail::GetDfsUrlFromUrl( - Azure::Storage::Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, GetTestName() + "a") - .GetUrl()), - credential, - options); - - EXPECT_NO_THROW(clientSecretClient->Create()); - EXPECT_NO_THROW(clientSecretClient->DeleteRecursive()); - } - - { - // Create from Anonymous credential. - auto objectName = GetTestName() + "2"; - auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - InitClientOptions()); - Azure::Storage::Blobs::SetBlobContainerAccessPolicyOptions options; - options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::BlobContainer; - containerClient.SetAccessPolicy(options); - - auto directoryClient - = Azure::Storage::Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - objectName, - InitClientOptions()); - EXPECT_NO_THROW(directoryClient.Create()); - - auto anonymousClient = Azure::Storage::Files::DataLake::DataLakeDirectoryClient( - directoryClient.GetUrl(), - InitClientOptions()); - - TestSleep(std::chrono::seconds(30)); - - EXPECT_NO_THROW(anonymousClient.GetProperties()); - } - } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.hpp index dfdcb0d033..c80ab4c3c7 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.hpp @@ -4,15 +4,14 @@ #include #include "datalake_path_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class DataLakeDirectoryClientTest : public DataLakePathClientTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; + protected: std::shared_ptr m_directoryClient; std::string m_directoryName; }; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index d7e1f28780..1503bdc874 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -29,20 +29,16 @@ namespace Azure { namespace Storage { namespace Test { void DataLakeFileClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); - CHECK_SKIP_TEST(); - m_fileName = GetFileSystemValidName(); + if (shouldSkipTest()) + { + return; + } + m_fileName = RandomString(); m_fileClient = std::make_shared( m_fileSystemClient->GetFileClient(m_fileName)); m_fileClient->CreateIfNotExists(); } - void DataLakeFileClientTest::TearDown() - { - CHECK_SKIP_TEST(); - m_fileSystemClient->GetFileClient(m_fileName).Delete(); - DataLakeFileSystemClientTest::TearDown(); - } - TEST_F(DataLakeFileClientTest, CreateDeleteFiles) { { @@ -119,22 +115,17 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(deleted); } { - std::string testName(GetTestNameLowerCase()); - auto client = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - testName + "a", - testName + "b", - InitClientOptions()); + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); bool deleted = false; - EXPECT_NO_THROW(deleted = client.DeleteIfExists().Value.Deleted); + EXPECT_NO_THROW(deleted = fileClient.DeleteIfExists().Value.Deleted); EXPECT_FALSE(deleted); } } TEST_F(DataLakeFileClientTest, FileMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_fileClient->SetMetadata(metadata1)); @@ -167,8 +158,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, FileProperties) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Get Metadata via properties works EXPECT_NO_THROW(m_fileClient->SetMetadata(metadata1)); @@ -196,23 +187,29 @@ namespace Azure { namespace Storage { namespace Test { { // HTTP headers works. - auto httpHeader = GetInterestingHttpHeaders(); + auto httpHeaders = Files::DataLake::Models::PathHttpHeaders(); + httpHeaders.ContentType = "application/x-binary"; + httpHeaders.ContentLanguage = "en-US"; + httpHeaders.ContentDisposition = "attachment"; + httpHeaders.CacheControl = "no-cache"; + httpHeaders.ContentEncoding = "identity"; std::vector fileClient; for (int32_t i = 0; i < 2; ++i) { auto client = m_fileSystemClient->GetFileClient("client" + std::to_string(i)); Files::DataLake::CreateFileOptions options; - options.HttpHeaders = httpHeader; + options.HttpHeaders = httpHeaders; EXPECT_NO_THROW(client.Create(options)); fileClient.emplace_back(std::move(client)); } for (const auto& client : fileClient) { auto result = client.GetProperties(); - EXPECT_EQ(httpHeader.CacheControl, result.Value.HttpHeaders.CacheControl); - EXPECT_EQ(httpHeader.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); - EXPECT_EQ(httpHeader.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); - EXPECT_EQ(httpHeader.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); + EXPECT_EQ(httpHeaders.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); + EXPECT_EQ(httpHeaders.CacheControl, result.Value.HttpHeaders.CacheControl); + EXPECT_EQ(httpHeaders.ContentEncoding, result.Value.HttpHeaders.ContentEncoding); EXPECT_NO_THROW(client.Delete()); } } @@ -247,14 +244,14 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, AppendFileWithFlush) { - const int32_t bufferSize = 4 * 1024; // 4KB data size - std::vector buffer(bufferSize, 'x'); + const int32_t bufferSize = 1; + auto buffer = RandomBuffer(bufferSize); auto bufferStream = std::make_unique( Azure::Core::IO::MemoryBodyStream(buffer)); // Append with flush option { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_flush_true"); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.Create(); auto properties1 = client.GetProperties(); Files::DataLake::AppendFileOptions options; @@ -266,7 +263,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(bufferSize, properties2.Value.FileSize); } { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_flush_false"); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.Create(); auto properties1 = client.GetProperties(); Files::DataLake::AppendFileOptions options; @@ -281,18 +278,18 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, AppendFileWithLease) { - const int32_t bufferSize = 4 * 1024; // 4KB data size - std::vector buffer(bufferSize, 'x'); + const int32_t bufferSize = 1; + auto buffer = RandomBuffer(bufferSize); auto bufferStream = std::make_unique( Azure::Core::IO::MemoryBodyStream(buffer)); // Append Lease Acquire { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acquire"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_acquire"); client.Create(); Files::DataLake::AppendFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::Acquire; - options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseId = RandomUUID(); options.LeaseDuration = std::chrono::seconds(20); bufferStream->Rewind(); client.Append(*bufferStream, 0, options); @@ -308,9 +305,9 @@ namespace Azure { namespace Storage { namespace Test { } // Append Lease AutoRenew { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_auto_renew"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_auto_renew"); client.Create(); - const std::string leaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + const std::string leaseId = RandomUUID(); Files::DataLake::DataLakeLeaseClient leaseClient(client, leaseId); leaseClient.Acquire(std::chrono::seconds(20)); Files::DataLake::AppendFileOptions options; @@ -331,9 +328,9 @@ namespace Azure { namespace Storage { namespace Test { } // Append Lease Release { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_release"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_release"); client.Create(); - const std::string leaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + const std::string leaseId = RandomUUID(); Files::DataLake::DataLakeLeaseClient leaseClient(client, leaseId); leaseClient.Acquire(std::chrono::seconds(20)); Files::DataLake::AppendFileOptions options; @@ -352,11 +349,11 @@ namespace Azure { namespace Storage { namespace Test { } // Append Lease AcquireRelease { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acquire_release"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_acquire_release"); client.Create(); Files::DataLake::AppendFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::AcquireRelease; - options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseId = RandomUUID(); options.LeaseDuration = std::chrono::seconds(20); options.Flush = true; bufferStream->Rewind(); @@ -373,20 +370,20 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, FlushFileWithLease) { - const int32_t bufferSize = 4 * 1024; // 4KB data size - std::vector buffer(bufferSize, 'x'); + const int32_t bufferSize = 1; + auto buffer = RandomBuffer(bufferSize); auto bufferStream = std::make_unique( Azure::Core::IO::MemoryBodyStream(buffer)); // Flush Lease Acquire { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acquire"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_acquire"); client.Create(); bufferStream->Rewind(); client.Append(*bufferStream, 0); Files::DataLake::FlushFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::Acquire; - options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseId = RandomUUID(); options.LeaseDuration = std::chrono::seconds(20); client.Flush(bufferSize, options); auto properties = client.GetProperties(); @@ -401,9 +398,9 @@ namespace Azure { namespace Storage { namespace Test { } // Flush Lease AutoRenew { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_auto_renew"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_auto_renew"); client.Create(); - const std::string leaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + const std::string leaseId = RandomUUID(); Files::DataLake::AppendFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::Acquire; options.LeaseId = leaseId; @@ -427,9 +424,9 @@ namespace Azure { namespace Storage { namespace Test { } // Flush Lease Release { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_release"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_release"); client.Create(); - const std::string leaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + const std::string leaseId = RandomUUID(); Files::DataLake::AppendFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::Acquire; options.LeaseId = leaseId; @@ -450,13 +447,13 @@ namespace Azure { namespace Storage { namespace Test { } // Flush Lease AcquireRelease { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acquire_release"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_acquire_release"); client.Create(); bufferStream->Rewind(); client.Append(*bufferStream, 0); Files::DataLake::FlushFileOptions options; options.LeaseAction = Files::DataLake::Models::LeaseAction::AcquireRelease; - options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseId = RandomUUID(); options.LeaseDuration = std::chrono::seconds(20); client.Flush(bufferSize, options); auto properties = client.GetProperties(); @@ -559,13 +556,13 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, ReadEmptyFile) { - auto fileClient = m_fileSystemClient->GetFileClient(GetTestNameLowerCase()); + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); fileClient.Create(); auto res = fileClient.Download(); EXPECT_EQ(res.Value.Body->Length(), 0); - std::string tempFilename(GetTestNameLowerCase()); + std::string tempFilename(RandomString()); EXPECT_NO_THROW(fileClient.DownloadTo(tempFilename)); EXPECT_TRUE(ReadFile(tempFilename).empty()); DeleteFile(tempFilename); @@ -577,7 +574,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, DownloadNonExistingToFile) { - const auto testName(GetTestName()); + const auto testName = RandomString(); auto fileClient = m_fileSystemClient->GetFileClient(testName); EXPECT_THROW(fileClient.DownloadTo(testName), StorageException); @@ -587,7 +584,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, ScheduleForDeletion) { { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase()); + auto client = m_fileSystemClient->GetFileClient(RandomString()); auto createResponse = client.Create(); auto scheduleDeletionResponse = client.ScheduleDeletion(Files::DataLake::ScheduleFileExpiryOriginType::NeverExpire); @@ -595,7 +592,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(scheduleDeletionResponse.Value.LastModified, createResponse.Value.LastModified); } { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "1"); + auto client = m_fileSystemClient->GetFileClient(RandomString()); EXPECT_NO_THROW(client.Create()); Files::DataLake::ScheduleFileDeletionOptions options; EXPECT_THROW( @@ -607,7 +604,7 @@ namespace Azure { namespace Storage { namespace Test { Files::DataLake::ScheduleFileExpiryOriginType::RelativeToNow, options)); } { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "2"); + auto client = m_fileSystemClient->GetFileClient(RandomString()); EXPECT_NO_THROW(client.Create()); Files::DataLake::ScheduleFileDeletionOptions options; EXPECT_THROW( @@ -625,181 +622,271 @@ namespace Azure { namespace Storage { namespace Test { } } - namespace { - - struct FileConcurrentUploadParameter - { - int Concurrency; - int64_t FileSize; + TEST_F(DataLakeFileClientTest, ConcurrentDownload_LIVEONLY_) + { + auto fileClient = *m_fileClient; + const auto blobContent = RandomBuffer(static_cast(8_MB)); + fileClient.UploadFrom(blobContent.data(), blobContent.size()); + + auto testDownloadToBuffer = [&](int concurrency, + int64_t downloadSize, + Azure::Nullable offset = {}, + Azure::Nullable length = {}, + Azure::Nullable initialChunkSize = {}, + Azure::Nullable chunkSize = {}) { + std::vector downloadBuffer; + std::vector expectedData = blobContent; + int64_t blobSize = blobContent.size(); + int64_t actualDownloadSize = std::min(downloadSize, blobSize); + if (offset.HasValue() && length.HasValue()) + { + actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), + blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + } + else + { + expectedData.clear(); + } + } + else if (offset.HasValue()) + { + actualDownloadSize = blobSize - offset.Value(); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), blobContent.end()); + } + else + { + expectedData.clear(); + } + } + downloadBuffer.resize(static_cast(downloadSize), '\x00'); + Files::DataLake::DownloadFileToOptions options; + options.TransferOptions.Concurrency = concurrency; + if (offset.HasValue() || length.HasValue()) + { + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset.Value(); + options.Range.Value().Length = length; + } + if (initialChunkSize.HasValue()) + { + options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } + if (actualDownloadSize > 0) + { + auto res = fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options); + EXPECT_EQ(res.Value.FileSize, blobSize); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); + EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); + downloadBuffer.resize(static_cast(res.Value.ContentRange.Length.Value())); + EXPECT_EQ(downloadBuffer, expectedData); + } + else + { + EXPECT_THROW( + fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options), + StorageException); + } }; - class UploadFile : public DataLakeFileClientTest, - public ::testing::WithParamInterface { + auto testDownloadToFile = [&](int concurrency, + int64_t downloadSize, + Azure::Nullable offset = {}, + Azure::Nullable length = {}, + Azure::Nullable initialChunkSize = {}, + Azure::Nullable chunkSize = {}) { + std::string tempFilename = RandomString() + "file" + std::to_string(concurrency); + if (offset) + { + tempFilename.append(std::to_string(offset.Value())); + } + std::vector expectedData = blobContent; + int64_t blobSize = blobContent.size(); + int64_t actualDownloadSize = std::min(downloadSize, blobSize); + if (offset.HasValue() && length.HasValue()) + { + actualDownloadSize = std::min(length.Value(), blobSize - offset.Value()); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), + blobContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + } + else + { + expectedData.clear(); + } + } + else if (offset.HasValue()) + { + actualDownloadSize = blobSize - offset.Value(); + if (actualDownloadSize >= 0) + { + expectedData.assign( + blobContent.begin() + static_cast(offset.Value()), blobContent.end()); + } + else + { + expectedData.clear(); + } + } + Files::DataLake::DownloadFileToOptions options; + options.TransferOptions.Concurrency = concurrency; + if (offset.HasValue() || length.HasValue()) + { + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset.Value(); + options.Range.Value().Length = length; + } + if (initialChunkSize.HasValue()) + { + options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } + if (actualDownloadSize > 0) + { + auto res = fileClient.DownloadTo(tempFilename, options); + EXPECT_EQ(res.Value.FileSize, blobSize); + EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); + EXPECT_EQ(res.Value.ContentRange.Offset, offset.HasValue() ? offset.Value() : 0); + EXPECT_EQ(ReadFile(tempFilename), expectedData); + } + else + { + EXPECT_THROW(fileClient.DownloadTo(tempFilename, options), StorageException); + } + DeleteFile(tempFilename); }; - std::string GetUploadSuffix(const testing::TestParamInfo& info) + const int64_t blobSize = blobContent.size(); + for (int c : {1, 2, 4}) { - // Can't use empty spaces or underscores (_) as per google test documentation - // http://google.github.io/googletest/advanced.html#specifying-names-for-value-parameterized-test-parameters - auto const& p = info.param; - std::string suffix("c" + std::to_string(p.Concurrency) + "s" + std::to_string(p.FileSize)); - return suffix; - } + std::vector> futures; + // random range + for (int i = 0; i < 16; ++i) + { + int64_t offset = RandomInt(0, blobContent.size() - 1); + int64_t length = RandomInt(1, 64_KB); + futures.emplace_back(std::async( + std::launch::async, testDownloadToBuffer, c, blobSize, offset, length, 8_KB, 4_KB)); + futures.emplace_back(std::async( + std::launch::async, testDownloadToFile, c, blobSize, offset, length, 4_KB, 7_KB)); + } - std::vector GetUploadParameters() - { - std::vector testParametes; - for (int c : {1, 2, 5}) + // buffer not big enough + Files::DataLake::DownloadFileToOptions options; + options.TransferOptions.Concurrency = c; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 1; + for (int64_t length : {1ULL, 2ULL, 4_KB, 5_KB, 8_KB, 11_KB, 20_KB}) { - for (int64_t l : - {0ULL, 1ULL, 2ULL, 2_KB, 4_KB, 999_KB, 1_MB, 2_MB - 1, 3_MB, 5_MB, 8_MB - 1234, 8_MB}) - { - testParametes.emplace_back(FileConcurrentUploadParameter({c, l})); - } + std::vector downloadBuffer; + downloadBuffer.resize(static_cast(length - 1)); + options.Range.Value().Length = length; + EXPECT_THROW( + fileClient.DownloadTo(downloadBuffer.data(), static_cast(length - 1), options), + std::runtime_error); + } + for (auto& f : futures) + { + f.get(); } - return testParametes; } - } // namespace - - TEST_P(UploadFile, fromBuffer) - { - UploadFile::ParamType const& p(GetParam()); - std::vector fileContent(static_cast(8_MB), 'x'); - auto fileClient = m_fileSystemClient->GetFileClient(GetTestNameLowerCase()); - - Azure::Storage::Files::DataLake::UploadFileFromOptions options; - options.TransferOptions.ChunkSize = 1_MB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = GetInterestingHttpHeaders(); - options.Metadata = GetMetadata(); - auto res = fileClient.UploadFrom(fileContent.data(), static_cast(p.FileSize), options); - auto lastModified = fileClient.GetProperties().Value.LastModified; - EXPECT_TRUE(res.Value.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - EXPECT_EQ(res.Value.LastModified, lastModified); - auto properties = fileClient.GetProperties().Value; - EXPECT_EQ(properties.FileSize, p.FileSize); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.ETag, res.Value.ETag); - EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - EXPECT_EQ(properties.LastModified, res.Value.LastModified); - std::vector downloadContent(static_cast(p.FileSize), '\x00'); - fileClient.DownloadTo(downloadContent.data(), static_cast(p.FileSize)); - EXPECT_EQ( - downloadContent, - std::vector( - fileContent.begin(), fileContent.begin() + static_cast(p.FileSize))); - fileClient.Delete(); } - TEST_P(UploadFile, fromFile) + TEST_F(DataLakeFileClientTest, ConcurrentUpload_LIVEONLY_) { - UploadFile::ParamType const& p(GetParam()); - std::vector fileContent(static_cast(p.FileSize), 'x'); - auto fileClient = m_fileSystemClient->GetFileClient(GetTestNameLowerCase()); - - Azure::Storage::Files::DataLake::UploadFileFromOptions options; - options.TransferOptions.ChunkSize = 1_MB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = GetInterestingHttpHeaders(); - options.Metadata = GetMetadata(); - - std::string tempFilename = GetTestNameLowerCase(); - WriteFile(tempFilename, fileContent); - auto res = fileClient.UploadFrom(tempFilename, options); - auto lastModified = fileClient.GetProperties().Value.LastModified; - EXPECT_TRUE(res.Value.ETag.HasValue()); - EXPECT_TRUE(IsValidTime(res.Value.LastModified)); - EXPECT_EQ(res.Value.LastModified, lastModified); - auto properties = fileClient.GetProperties().Value; - EXPECT_EQ(properties.FileSize, p.FileSize); - EXPECT_EQ(properties.HttpHeaders, options.HttpHeaders); - EXPECT_EQ(properties.Metadata, options.Metadata); - EXPECT_EQ(properties.ETag, res.Value.ETag); - EXPECT_EQ(properties.LastModified, res.Value.LastModified); - std::vector downloadContent(static_cast(p.FileSize), '\x00'); - fileClient.DownloadTo(downloadContent.data(), static_cast(p.FileSize)); - EXPECT_EQ( - downloadContent, - std::vector( - fileContent.begin(), fileContent.begin() + static_cast(p.FileSize))); - std::string tempFileDestinationName = RandomString(); - fileClient.DownloadTo(tempFileDestinationName); - EXPECT_EQ(ReadFile(tempFileDestinationName), fileContent); - DeleteFile(tempFileDestinationName); - DeleteFile(tempFilename); - fileClient.Delete(); - } + const auto blobContent = RandomBuffer(static_cast(8_MB)); + + auto testUploadFromBuffer = [&](int concurrency, + int64_t bufferSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Files::DataLake::UploadFileFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) + { + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } - INSTANTIATE_TEST_SUITE_P( - withParam, - UploadFile, - testing::ValuesIn(GetUploadParameters()), - GetUploadSuffix); + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); + EXPECT_NO_THROW( + fileClient.UploadFrom(blobContent.data(), static_cast(bufferSize), options)); + std::vector downloadBuffer(static_cast(bufferSize), '\x00'); + fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(bufferSize)); + EXPECT_EQ(downloadBuffer, expectedData); + }; - TEST_F(DataLakeFileClientTest, ConstructorsWorks) - { - { - // Create from connection string validates static creator function and shared key - // constructor. - auto fileName = GetTestNameLowerCase(); - auto connectionStringClient - = Azure::Storage::Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - fileName, - InitClientOptions()); - EXPECT_NO_THROW(connectionStringClient.Create()); - EXPECT_NO_THROW(connectionStringClient.Delete()); - } + auto testUploadFromFile = [&](int concurrency, + int64_t fileSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Files::DataLake::UploadFileFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) + { + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); + } + if (chunkSize.HasValue()) + { + options.TransferOptions.ChunkSize = chunkSize.Value(); + } - { - // Create from client secret credential. - std::shared_ptr credential - = std::make_shared( - AadTenantId(), AadClientId(), AadClientSecret()); - Azure::Storage::Files::DataLake::DataLakeClientOptions options; - - auto clientSecretClient = InitTestClient< - Azure::Storage::Files::DataLake::DataLakeFileClient, - Azure::Storage::Files::DataLake::DataLakeClientOptions>( - Azure::Storage::Files::DataLake::_detail::GetDfsUrlFromUrl( - Azure::Storage::Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, "credential") - .GetUrl()), - credential, - options); - - EXPECT_NO_THROW(clientSecretClient->Create()); - EXPECT_NO_THROW(clientSecretClient->Delete()); - } + const std::string tempFileName = RandomString(); + WriteFile( + tempFileName, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(fileSize))); + auto fileClient = m_fileSystemClient->GetFileClient(RandomString()); + EXPECT_NO_THROW(fileClient.UploadFrom(tempFileName, options)); + DeleteFile(tempFileName); + std::vector downloadBuffer(static_cast(fileSize), '\x00'); + fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(fileSize)); + EXPECT_EQ(downloadBuffer, expectedData); + }; + for (int c : {1, 2, 4}) { - // Create from Anonymous credential. - std::vector blobContent(static_cast(1_MB), 'x'); - - auto objectName = "testObject"; - auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - InitClientOptions()); - Azure::Storage::Blobs::SetBlobContainerAccessPolicyOptions options; - options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::Blob; - containerClient.SetAccessPolicy(options); - auto blobClient = containerClient.GetBlockBlobClient(objectName); - auto memoryStream = Azure::Core::IO::MemoryBodyStream(blobContent.data(), blobContent.size()); - EXPECT_NO_THROW(blobClient.Upload(memoryStream)); - - auto anonymousClient = Azure::Storage::Files::DataLake::DataLakeFileClient( - Azure::Storage::Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, objectName) - .GetUrl(), - InitClientOptions()); - - TestSleep(std::chrono::seconds(30)); - - EXPECT_NO_THROW(anonymousClient.Download()); + std::vector> futures; + // random range + for (int i = 0; i < 16; ++i) + { + int64_t fileSize = RandomInt(1, 1_MB); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); + } + for (auto& f : futures) + { + f.get(); + } } } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.hpp index e9482acb3e..9a57c2276f 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.hpp @@ -4,15 +4,14 @@ #include #include "datalake_file_system_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class DataLakeFileClientTest : public DataLakeFileSystemClientTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; + protected: std::shared_ptr m_fileClient; std::string m_fileName; }; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp index 1aa6a8c3d8..e5686af9f0 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp @@ -69,10 +69,9 @@ id,name,price "b25fY29sdW1ucyI6IFtdfQAYKmZhc3RwYXJxdWV0LXB5dGhvbiB2ZXJzaW9uIDAuOC4xIChidWlsZCAwKQDXAwAAUEFS" "MQ=="); - TEST_F(DataLakeFileClientTest, QueryJsonInputCsvOutput_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryJsonInputCsvOutput) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom( reinterpret_cast(JsonQueryTestData.data()), JsonQueryTestData.size()); @@ -110,10 +109,9 @@ id,name,price } } - TEST_F(DataLakeFileClientTest, QueryCsvInputJsonOutput_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryCsvInputJsonOutput) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom( reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); @@ -133,10 +131,9 @@ id,name,price R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); } - TEST_F(DataLakeFileClientTest, QueryCsvInputArrowOutput_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryCsvInputArrowOutput) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom( reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); @@ -184,10 +181,9 @@ id,name,price EXPECT_EQ(data, expectedData); } - TEST_F(DataLakeFileClientTest, QueryParquetInputArrowOutput_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryParquetInputArrowOutput) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); @@ -243,10 +239,9 @@ id,name,price EXPECT_EQ(data, expectedData); } - TEST_F(DataLakeFileClientTest, QueryWithError_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryWithError) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); const std::string malformedData = R"json( @@ -310,10 +305,9 @@ xx EXPECT_TRUE(progressCallbackCalled); } - TEST_F(DataLakeFileClientTest, QueryDefaultInputOutput_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryDefaultInputOutput) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); const std::string csvData = "100,oranges,100"; client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); @@ -324,8 +318,7 @@ xx TEST_F(DataLakeFileClientTest, QueryLargeBlob_LIVEONLY_) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); constexpr size_t DataSize = static_cast(32_MB); @@ -365,29 +358,25 @@ xx } } - TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLeaseId_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLeaseId) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom(nullptr, 0); - Files::DataLake::DataLakeLeaseClient leaseClient( - client, Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId()); + Files::DataLake::DataLakeLeaseClient leaseClient(client, RandomUUID()); leaseClient.Acquire(Files::DataLake::DataLakeLeaseClient::InfiniteLeaseDuration); Files::DataLake::QueryFileOptions queryOptions; - queryOptions.AccessConditions.LeaseId - = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + queryOptions.AccessConditions.LeaseId = RandomUUID(); EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); queryOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); } - TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLastModifiedTime_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLastModifiedTime) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom(nullptr, 0); auto lastModifiedTime = client.GetProperties().Value.LastModified; @@ -407,10 +396,9 @@ xx EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); } - TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionETag_LIVEONLY_) + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionETag) { - auto const testName(GetTestName()); - auto client = m_fileSystemClient->GetFileClient(testName); + auto client = m_fileSystemClient->GetFileClient(RandomString()); client.UploadFrom(nullptr, 0); auto etag = client.GetProperties().Value.ETag; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index c08fb3015a..43009ca13b 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -17,8 +17,6 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { - const size_t PathTestSize = 5; - std::string DataLakeFileSystemClientTest::GetSas() { Sas::DataLakeSasBuilder sasBuilder; @@ -31,304 +29,206 @@ namespace Azure { namespace Storage { namespace Test { *_internal::ParseConnectionString(AdlsGen2ConnectionString()).KeyCredential); } - void DataLakeFileSystemClientTest::CreateDirectoryList() - { - std::string const directoryName(GetFileSystemValidName()); - std::string const prefix(directoryName.begin(), directoryName.end() - 2); - m_directoryA = prefix + "a"; - m_directoryB = prefix + "b"; - m_pathNameSetA.clear(); - m_pathNameSetB.clear(); - for (size_t i = 0; i < PathTestSize; ++i) - { - { - auto name = m_directoryA + "/" + std::to_string(i); - m_fileSystemClient->GetFileClient(name).Create(); - m_pathNameSetA.emplace_back(std::move(name)); - } - { - auto name = m_directoryB + "/" + std::to_string(i); - m_fileSystemClient->GetFileClient(name).Create(); - m_pathNameSetB.emplace_back(std::move(name)); - } - } - } - void DataLakeFileSystemClientTest::SetUp() { DataLakeServiceClientTest::SetUp(); - CHECK_SKIP_TEST(); - - m_fileSystemName = GetFileSystemValidName(); + if (shouldSkipTest()) + { + return; + } + m_fileSystemName = GetLowercaseIdentifier(); m_fileSystemClient = std::make_shared( m_dataLakeServiceClient->GetFileSystemClient(m_fileSystemName)); - m_fileSystemClient->CreateIfNotExists(); - } - - void DataLakeFileSystemClientTest::TearDown() - { - CHECK_SKIP_TEST(); - m_fileSystemClient->Delete(); - DataLakeServiceClientTest::TearDown(); - } - - std::vector DataLakeFileSystemClientTest::ListAllPaths( - bool recursive, - const std::string& directory) - { - std::vector result; - std::string continuation; - Files::DataLake::ListPathsOptions options; - if (directory.empty()) + while (true) { - for (auto pageResult = m_fileSystemClient->ListPaths(recursive, options); - pageResult.HasPage(); - pageResult.MoveToNextPage()) + try { - result.insert(result.end(), pageResult.Paths.begin(), pageResult.Paths.end()); + m_fileSystemClient->CreateIfNotExists(); + break; } - } - else - { - auto directoryClient = m_fileSystemClient->GetDirectoryClient(directory); - for (auto pageResult = directoryClient.ListPaths(recursive, options); pageResult.HasPage(); - pageResult.MoveToNextPage()) + catch (StorageException& e) { - result.insert(result.end(), pageResult.Paths.begin(), pageResult.Paths.end()); + if (e.ErrorCode != "ContainerBeingDeleted") + { + throw; + } + SUCCEED() << "Container is being deleted. Will try again after 3 seconds."; + std::this_thread::sleep_for(std::chrono::seconds(3)); } } - return result; + m_resourceCleanupFunctions.push_back( + [fileSystemClient = *m_fileSystemClient]() { fileSystemClient.DeleteIfExists(); }); } - Files::DataLake::Models::PathHttpHeaders DataLakeFileSystemClientTest::GetInterestingHttpHeaders() + Files::DataLake::DataLakeFileSystemClient + DataLakeFileSystemClientTest::GetFileSystemClientForTest( + const std::string& fileSystemName, + Files::DataLake::DataLakeClientOptions clientOptions) { - static Files::DataLake::Models::PathHttpHeaders result = []() { - Files::DataLake::Models::PathHttpHeaders ret; - ret.CacheControl = std::string("no-cache"); - ret.ContentDisposition = std::string("attachment"); - ret.ContentEncoding = std::string("deflate"); - ret.ContentLanguage = std::string("en-US"); - ret.ContentType = std::string("application/octet-stream"); - return ret; - }(); - return result; + InitClientOptions(clientOptions); + auto fsClient = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), fileSystemName, clientOptions); + m_resourceCleanupFunctions.push_back([fsClient]() { fsClient.DeleteIfExists(); }); + return fsClient; } TEST_F(DataLakeFileSystemClientTest, CreateDeleteFileSystems) { - std::string fileSystemName("prefix"); - { - // Normal create/delete. - std::vector fileSystemClient; - for (int32_t i = 0; i < 5; ++i) - { - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemName + std::to_string(i), - InitClientOptions()); - EXPECT_NO_THROW(client.Create()); - fileSystemClient.emplace_back(std::move(client)); - } - for (const auto& client : fileSystemClient) - { - EXPECT_NO_THROW(client.Delete()); - } - } + auto fsClient = GetFileSystemClientForTest(LowercaseRandomString()); + EXPECT_THROW(fsClient.Delete(), StorageException); + EXPECT_NO_THROW(fsClient.Create()); + EXPECT_NO_THROW(fsClient.CreateIfNotExists()); + EXPECT_THROW(fsClient.Create(), StorageException); + EXPECT_NO_THROW(fsClient.Delete()); + EXPECT_NO_THROW(fsClient.DeleteIfExists()); + } + + TEST_F(DataLakeFileSystemClientTest, CreateDeleteFileSystemsWithAccessCondition) + { { - std::string fileSystemNameAccessCondition(fileSystemName + "a"); - // Normal delete with access condition. - std::vector fileSystemClient; - for (int32_t i = 0; i < 5; ++i) - { - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemNameAccessCondition + std::to_string(i), - InitClientOptions()); - EXPECT_NO_THROW(client.Create()); - fileSystemClient.emplace_back(std::move(client)); - } - for (const auto& client : fileSystemClient) - { - auto response = client.GetProperties(); - Files::DataLake::DeleteFileSystemOptions options1; - options1.AccessConditions.IfModifiedSince = response.Value.LastModified; - EXPECT_THROW(client.Delete(options1), StorageException); - Files::DataLake::DeleteFileSystemOptions options2; - options2.AccessConditions.IfUnmodifiedSince = response.Value.LastModified; - EXPECT_NO_THROW(client.Delete(options2)); - } - } - { - // CreateIfNotExists & DeleteIfExists. - { - std::string fileSystemNameCreate(fileSystemName + "c"); - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemNameCreate, - InitClientOptions()); - EXPECT_NO_THROW(client.Create()); - EXPECT_NO_THROW(client.CreateIfNotExists()); - EXPECT_NO_THROW(client.Delete()); - EXPECT_NO_THROW(client.DeleteIfExists()); - } - { - std::string fileSystemNameCreateIf(fileSystemName + "ci"); - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemNameCreateIf, - InitClientOptions()); - EXPECT_NO_THROW(client.CreateIfNotExists()); - EXPECT_THROW(client.Create(), StorageException); - EXPECT_NO_THROW(client.DeleteIfExists()); - } - { - std::string fileSystemNameCreateIfNot(fileSystemName + "cid"); - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemNameCreateIfNot, - InitClientOptions()); - auto created = client.Create().Value.Created; - EXPECT_TRUE(created); - auto createResult = client.CreateIfNotExists(); - EXPECT_FALSE(createResult.Value.Created); - EXPECT_FALSE(createResult.Value.ETag.HasValue()); - EXPECT_EQ(DateTime(), createResult.Value.LastModified); - auto deleted = client.Delete().Value.Deleted; - EXPECT_TRUE(deleted); - } - { - std::string fileSystemNameDelete(fileSystemName + "d"); - auto client = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - fileSystemNameDelete, - InitClientOptions()); - auto deleteResult = client.DeleteIfExists(); - EXPECT_FALSE(deleteResult.Value.Deleted); - } + auto fsClient = GetFileSystemClientForTest(LowercaseRandomString()); + fsClient.Create(); + auto properties = fsClient.GetProperties().Value; + + Files::DataLake::DeleteFileSystemOptions deleteOptions; + deleteOptions.AccessConditions.IfModifiedSince + = properties.LastModified + std::chrono::seconds(5); + EXPECT_THROW(fsClient.Delete(deleteOptions), StorageException); + deleteOptions.AccessConditions.IfModifiedSince + = properties.LastModified - std::chrono::seconds(5); + EXPECT_NO_THROW(fsClient.Delete(deleteOptions)); + } + { + auto fsClient = GetFileSystemClientForTest(LowercaseRandomString()); + fsClient.Create(); + auto properties = fsClient.GetProperties().Value; + + Files::DataLake::DeleteFileSystemOptions deleteOptions; + deleteOptions.AccessConditions.IfUnmodifiedSince + = properties.LastModified - std::chrono::seconds(5); + EXPECT_THROW(fsClient.Delete(deleteOptions), StorageException); + deleteOptions.AccessConditions.IfUnmodifiedSince + = properties.LastModified + std::chrono::seconds(5); + EXPECT_NO_THROW(fsClient.Delete(deleteOptions)); + } + { + auto leaseId = RandomUUID(); + auto dummyLeaseId = RandomUUID(); + auto fsClient = GetFileSystemClientForTest(LowercaseRandomString()); + fsClient.Create(); + + Files::DataLake::DataLakeLeaseClient leaseClient(fsClient, leaseId); + leaseClient.Acquire(std::chrono::seconds(30)); + EXPECT_THROW(fsClient.Delete(), StorageException); + Files::DataLake::DeleteFileSystemOptions deleteOptions; + deleteOptions.AccessConditions.LeaseId = dummyLeaseId; + EXPECT_THROW(fsClient.Delete(deleteOptions), StorageException); + deleteOptions.AccessConditions.LeaseId = leaseId; + EXPECT_NO_THROW(fsClient.Delete(deleteOptions)); } } TEST_F(DataLakeFileSystemClientTest, FileSystemMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); { + auto metadata = RandomMetadata(); // Set/Get Metadata works - EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata1)); - auto result = m_fileSystemClient->GetProperties().Value.Metadata; - EXPECT_EQ(metadata1, result); - EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata2)); - result = m_fileSystemClient->GetProperties().Value.Metadata; - EXPECT_EQ(metadata2, result); + EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata)); + EXPECT_EQ(metadata, m_fileSystemClient->GetProperties().Value.Metadata); + EXPECT_NO_THROW(m_fileSystemClient->SetMetadata({})); + EXPECT_TRUE(m_fileSystemClient->GetProperties().Value.Metadata.empty()); } { - // Create file system with metadata works - auto options = InitClientOptions(); - auto client1 = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName + "1", options); - auto client2 = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName + "2", options); - Files::DataLake::CreateFileSystemOptions options1; - Files::DataLake::CreateFileSystemOptions options2; - options1.Metadata = metadata1; - options2.Metadata = metadata2; - - EXPECT_NO_THROW(client1.Create(options1)); - EXPECT_NO_THROW(client2.Create(options2)); - auto result = client1.GetProperties().Value.Metadata; - EXPECT_EQ(metadata1, result); - result = client2.GetProperties().Value.Metadata; - EXPECT_EQ(metadata2, result); - client1.DeleteIfExists(); - client2.DeleteIfExists(); + auto fsClient = GetFileSystemClientForTest(LowercaseRandomString()); + Files::DataLake::CreateFileSystemOptions options; + options.Metadata = RandomMetadata(); + fsClient.Create(options); + EXPECT_EQ(fsClient.GetProperties().Value.Metadata, options.Metadata); } } TEST_F(DataLakeFileSystemClientTest, GetDataLakeFileSystemPropertiesResult) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); - { - // Get Metadata via properties works - EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata1)); - auto result = m_fileSystemClient->GetProperties(); - EXPECT_EQ(metadata1, result.Value.Metadata); - EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata2)); - result = m_fileSystemClient->GetProperties(); - EXPECT_EQ(metadata2, result.Value.Metadata); - } + auto metadata1 = RandomMetadata(); + // Get Metadata via properties works + EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata1)); + auto properties = m_fileSystemClient->GetProperties().Value; + EXPECT_EQ(metadata1, properties.Metadata); + EXPECT_TRUE(IsValidTime(properties.LastModified)); + EXPECT_TRUE(properties.ETag.HasValue()); + EXPECT_FALSE(properties.DefaultEncryptionScope.empty()); + EXPECT_FALSE(properties.PreventEncryptionScopeOverride); + } - { - // Last modified Etag works. - auto properties1 = m_fileSystemClient->GetProperties(); - auto properties2 = m_fileSystemClient->GetProperties(); - EXPECT_EQ(properties1.Value.ETag, properties2.Value.ETag); - EXPECT_EQ(properties1.Value.LastModified, properties2.Value.LastModified); + TEST_F(DataLakeFileSystemClientTest, ListPaths) + { + std::set paths; + const std::string dir1 = RandomString(); + const std::string dir2 = RandomString(); - // This operation changes ETag/LastModified. - EXPECT_NO_THROW(m_fileSystemClient->SetMetadata(metadata1)); + std::set rootPaths; + rootPaths.emplace(dir1); + rootPaths.emplace(dir2); - auto properties3 = m_fileSystemClient->GetProperties(); - EXPECT_NE(properties1.Value.ETag, properties3.Value.ETag); + { + auto dirClient = m_fileSystemClient->GetDirectoryClient(dir1); + for (int i = 0; i < 3; ++i) + { + std::string filename = RandomString(); + auto fileClient = dirClient.GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(dir1 + "/" + filename); + } + + dirClient = m_fileSystemClient->GetDirectoryClient(dir2); + for (int i = 0; i < 4; ++i) + { + std::string filename = RandomString(); + auto fileClient = dirClient.GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(dir2 + "/" + filename); + } + std::string filename = RandomString(); + auto fileClient = m_fileSystemClient->GetFileClient(filename); + fileClient.CreateIfNotExists(); + paths.emplace(filename); + rootPaths.emplace(filename); } - } - TEST_F(DataLakeFileSystemClientTest, ListPaths) - { - CreateDirectoryList(); { // Normal list recursively. - auto result = ListAllPaths(true); - for (const auto& name : m_pathNameSetA) + std::set results; + for (auto page = m_fileSystemClient->ListPaths(true); page.HasPage(); page.MoveToNextPage()) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::DataLake::Models::PathItem& path) { - return path.Name == name; - }); - EXPECT_NE(result.end(), iter); - EXPECT_EQ(iter->Name, name); - EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); - EXPECT_TRUE(iter->CreatedOn.HasValue()); - EXPECT_FALSE(iter->ExpiresOn.HasValue()); + for (auto& path : page.Paths) + { + results.insert(path.Name); + } } - for (const auto& name : m_pathNameSetB) + + for (const auto& path : paths) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::DataLake::Models::PathItem& path) { - return path.Name == name; - }); - EXPECT_NE(result.end(), iter); - EXPECT_EQ(iter->Name, name); - EXPECT_EQ(iter->Name.substr(0U, m_directoryB.size()), m_directoryB); - EXPECT_TRUE(iter->CreatedOn.HasValue()); - EXPECT_FALSE(iter->ExpiresOn.HasValue()); + EXPECT_NE(results.find(path), results.end()); } } { - // List with directory. - auto result = ListAllPaths(true, m_directoryA); - for (const auto& name : m_pathNameSetA) + // non-recursive + std::set results; + for (auto page = m_fileSystemClient->ListPaths(false); page.HasPage(); page.MoveToNextPage()) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::DataLake::Models::PathItem& path) { - return path.Name == name; - }); - EXPECT_NE(result.end(), iter); - EXPECT_EQ(iter->Name, name); - EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); - EXPECT_TRUE(iter->CreatedOn.HasValue()); - EXPECT_FALSE(iter->ExpiresOn.HasValue()); + for (auto& path : page.Paths) + { + results.insert(path.Name); + } } - for (const auto& name : m_pathNameSetB) + + for (const auto& path : rootPaths) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::DataLake::Models::PathItem& path) { - return path.Name == name; - }); - EXPECT_EQ(result.end(), iter); + EXPECT_NE(results.find(path), results.end()); } + EXPECT_LT(results.size(), paths.size()); } { // List max result @@ -337,23 +237,6 @@ namespace Azure { namespace Storage { namespace Test { auto response = m_fileSystemClient->ListPaths(true, options); EXPECT_LE(2U, response.Paths.size()); } - { - // check expiry time - const std::string filename = GetTestNameLowerCase() + "check_expiry"; - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "check_expiry"); - Files::DataLake::CreateFileOptions createOptions; - createOptions.ScheduleDeletionOptions.ExpiresOn = Azure::DateTime::Parse( - "Wed, 29 Sep 2100 09:53:03 GMT", Azure::DateTime::DateFormat::Rfc1123); - client.Create(createOptions); - - auto result = ListAllPaths(false); - auto iter = std::find_if( - result.begin(), result.end(), [&filename](const Files::DataLake::Models::PathItem& path) { - return path.Name == filename; - }); - EXPECT_TRUE(iter->ExpiresOn.HasValue()); - EXPECT_EQ(createOptions.ScheduleDeletionOptions.ExpiresOn.Value(), iter->ExpiresOn.Value()); - } } TEST_F(DataLakeFileSystemClientTest, UnencodedPathDirectoryFileNameWorks) @@ -362,14 +245,14 @@ namespace Azure { namespace Storage { namespace Test { const std::string encoded_non_ascii_word = "%E6%B5%8B%E8%AF%95"; std::string baseName = "a b c / !@#$%^&*(?/<>,.;:'\"[]{}|`~\\) def" + non_ascii_word; { - std::string pathName = baseName + GetTestNameLowerCase(); + std::string pathName = baseName + RandomString(); auto fileClient = m_fileSystemClient->GetFileClient(pathName); EXPECT_NO_THROW(fileClient.Create()); auto fileUrl = fileClient.GetUrl(); EXPECT_EQ(fileUrl, m_fileSystemClient->GetUrl() + "/" + _internal::UrlEncodePath(pathName)); } { - std::string directoryName = baseName + GetTestNameLowerCase() + "1"; + std::string directoryName = baseName + RandomString() + "1"; auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName); EXPECT_NO_THROW(directoryClient.Create()); auto directoryUrl = directoryClient.GetUrl(); @@ -378,7 +261,7 @@ namespace Azure { namespace Storage { namespace Test { m_fileSystemClient->GetUrl() + "/" + _internal::UrlEncodePath(directoryName)); } { - std::string fileName = baseName + GetTestNameLowerCase() + "2"; + std::string fileName = baseName + RandomString() + "2"; auto fileClient = m_fileSystemClient->GetFileClient(fileName); EXPECT_NO_THROW(fileClient.Create()); auto fileUrl = fileClient.GetUrl(); @@ -390,7 +273,7 @@ namespace Azure { namespace Storage { namespace Test { { { // Create from connection string validates static creator function and shared key constructor. - auto fileSystemName = GetTestNameLowerCase() + "1"; + auto fileSystemName = LowercaseRandomString() + "1"; auto connectionStringClient = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( AdlsGen2ConnectionString(), @@ -411,7 +294,7 @@ namespace Azure { namespace Storage { namespace Test { Azure::Storage::Files::DataLake::DataLakeFileSystemClient, Azure::Storage::Files::DataLake::DataLakeClientOptions>( Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), GetTestNameLowerCase()) + AdlsGen2ConnectionString(), LowercaseRandomString()) .GetUrl(), credential, options); @@ -421,14 +304,11 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey_LIVEONLY_) + TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey) { - auto getRandomCustomerProvidedKey = [&]() { Files::DataLake::EncryptionKey key; - std::vector aes256Key; - aes256Key.resize(32); - RandomBuffer(&aes256Key[0], aes256Key.size()); + std::vector aes256Key = RandomBuffer(32); key.Key = Azure::Core::Convert::Base64Encode(aes256Key); key.KeyHash = Azure::Core::Cryptography::_internal::Sha256Hash().Final( aes256Key.data(), aes256Key.size()); @@ -436,74 +316,64 @@ namespace Azure { namespace Storage { namespace Test { return key; }; - const int32_t bufferSize = 1024; // 1KB data size - auto buffer = std::make_shared>(bufferSize, 'x'); - Azure::Core::IO::MemoryBodyStream bodyStream(buffer->data(), buffer->size()); + auto buffer = RandomBuffer(10); + Azure::Core::IO::MemoryBodyStream bodyStream(buffer.data(), buffer.size()); - auto customerProvidedKey - = std::make_shared(getRandomCustomerProvidedKey()); - Files::DataLake::DataLakeClientOptions options; - options.CustomerProvidedKey = *customerProvidedKey; - auto fileServiceClient = std::make_shared( - Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), options)); - auto fileSystemClient = std::make_shared( - fileServiceClient->GetFileSystemClient(m_fileSystemName)); + auto customerProvidedKey = getRandomCustomerProvidedKey(); + Files::DataLake::DataLakeClientOptions clientOptionsWithCPK; + clientOptionsWithCPK.CustomerProvidedKey = customerProvidedKey; + auto fileSystemClientWithCPK + = GetFileSystemClientForTest(m_fileSystemName, clientOptionsWithCPK); + auto fileSystemClientWithoutCPK = GetFileSystemClientForTest(m_fileSystemName); // fileSystem works { - auto fileSystemClientWithoutEncryptionKey - = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName); // Rename File - const std::string filename1 = GetTestName() + "file1"; - const std::string filename2 = GetTestName() + "file2"; - const std::string filename3 = GetTestName() + "file3"; - const std::string filename4 = GetTestName() + "file4"; - - auto oldFileClient = fileSystemClient->GetFileClient(filename1); - oldFileClient.Create(); - auto newFileClient = fileSystemClient->RenameFile(filename1, filename2).Value; - auto properties = std::make_shared( - newFileClient.GetProperties().Value); - EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + const std::string filename1 = RandomString() + "file1"; + const std::string filename2 = RandomString() + "file2"; + const std::string filename3 = RandomString() + "file3"; + const std::string filename4 = RandomString() + "file4"; + + auto oldFileClient = std::make_shared( + fileSystemClientWithCPK.GetFileClient(filename1)); + oldFileClient->Create(); + auto newFileClient = std::make_shared( + fileSystemClientWithCPK.RenameFile(filename1, filename2).Value); + auto properties = newFileClient->GetProperties().Value; + EXPECT_EQ(customerProvidedKey.KeyHash, properties.EncryptionKeySha256.Value()); auto newFileClientWithoutEncryptionKey - = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, filename2); - EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); + = std::make_shared( + fileSystemClientWithoutCPK.GetFileClient(filename2)); + EXPECT_THROW(newFileClientWithoutEncryptionKey->GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutCPK.RenameFile(filename2, filename3)); // Rename Directory - const std::string testName(GetTestName()); - const std::string oldDirectoryName = testName + "dir1"; - const std::string newDirectoryName = testName + "dir2"; - const std::string newDirectoryName2 = testName + "dir3"; - - auto oldDirectoryClient = fileSystemClient->GetDirectoryClient(oldDirectoryName); - oldDirectoryClient.Create(); - oldDirectoryClient.GetFileClient(testName + "file3").Create(); - oldDirectoryClient.GetSubdirectoryClient(testName + "dir4").Create(); - - auto newDirectoryClient - = fileSystemClient->RenameDirectory(oldDirectoryName, newDirectoryName).Value; - properties = std::make_shared( - newDirectoryClient.GetProperties().Value); - EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + const std::string baseName = RandomString(); + const std::string oldDirectoryName = baseName + "dir1"; + const std::string newDirectoryName = baseName + "dir2"; + const std::string newDirectoryName2 = baseName + "dir3"; + + auto oldDirectoryClient = std::make_shared( + fileSystemClientWithCPK.GetDirectoryClient(oldDirectoryName)); + oldDirectoryClient->Create(); + oldDirectoryClient->GetFileClient(baseName + "file3").Create(); + oldDirectoryClient->GetSubdirectoryClient(baseName + "dir4").Create(); + + auto newDirectoryClient = std::make_shared( + fileSystemClientWithCPK.RenameDirectory(oldDirectoryName, newDirectoryName).Value); + properties = newDirectoryClient->GetProperties().Value; + EXPECT_TRUE(properties.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.EncryptionKeySha256.Value()); auto newDirectoryClientWithoutEncryptionKey - = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); - EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( - newDirectoryName, newDirectoryName2)); + = std::make_shared( + fileSystemClientWithoutCPK.GetDirectoryClient(newDirectoryName)); + EXPECT_THROW(newDirectoryClientWithoutEncryptionKey->GetProperties(), StorageException); + EXPECT_NO_THROW( + fileSystemClientWithoutCPK.RenameDirectory(newDirectoryName, newDirectoryName2)); - auto fileSystemClientWithEncryptionKey - = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, options); - auto created = std::make_shared( - fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value); - EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + auto createResult = fileSystemClientWithCPK.GetFileClient(filename4).Create().Value; + EXPECT_TRUE(createResult.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, createResult.EncryptionKeySha256.Value()); } // path works @@ -511,74 +381,70 @@ namespace Azure { namespace Storage { namespace Test { const std::string pathName = "path"; const std::string pathName2 = "path2"; - auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); - EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); - EXPECT_NO_THROW(pathClient.SetMetadata(GetMetadata())); - auto properties = std::make_shared( - pathClient.GetProperties().Value); - EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); - auto pathClientWithoutEncryptionKey - = Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, pathName); - EXPECT_THROW(pathClientWithoutEncryptionKey.SetMetadata(GetMetadata()), StorageException); - EXPECT_THROW(pathClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(pathClientWithoutEncryptionKey.GetAccessControlList()); - EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetHttpHeaders( - Files::DataLake::Models::PathHttpHeaders())); - EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetPermissions("rwxrw-rw-")); - - auto pathClientWithEncryptionKey - = Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, pathName2, options); - auto created - = pathClientWithEncryptionKey.Create(Files::DataLake::Models::PathResourceType::File) - .Value; - EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, created.EncryptionKeySha256.Value()); + auto pathClientWithCPK = std::make_shared( + fileSystemClientWithCPK.GetFileClient(pathName)); + auto pathClientWithoutCPK = std::make_shared( + fileSystemClientWithoutCPK.GetFileClient(pathName)); + auto pathClient2WithCPK = std::make_shared( + fileSystemClientWithCPK.GetFileClient(pathName2)); + + EXPECT_NO_THROW(pathClientWithCPK->Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(pathClientWithCPK->SetMetadata(RandomMetadata())); + auto properties = pathClientWithCPK->GetProperties().Value; + EXPECT_TRUE(properties.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.EncryptionKeySha256.Value()); + + EXPECT_THROW(pathClientWithoutCPK->SetMetadata(RandomMetadata()), StorageException); + EXPECT_THROW(pathClientWithoutCPK->GetProperties(), StorageException); + EXPECT_NO_THROW(pathClientWithoutCPK->GetAccessControlList()); + EXPECT_NO_THROW( + pathClientWithoutCPK->SetHttpHeaders(Files::DataLake::Models::PathHttpHeaders())); + EXPECT_NO_THROW(pathClientWithoutCPK->SetPermissions("rwxrw-rw-")); + + auto createResult + = pathClient2WithCPK->Create(Files::DataLake::Models::PathResourceType::File).Value; + EXPECT_TRUE(createResult.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, createResult.EncryptionKeySha256.Value()); } // file works { const std::string fileName = "file"; const std::string fileName2 = "file2"; - auto fileClient = fileSystemClient->GetFileClient(fileName); - auto fileClientWithoutEncryptionKey - = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, fileName); + auto fileClientWithCPK = std::make_shared( + fileSystemClientWithCPK.GetFileClient(fileName)); + auto fileClientWithoutCPK = std::make_shared( + fileSystemClientWithoutCPK.GetFileClient(fileName)); + auto fileClient2WithCPK = std::make_shared( + fileSystemClientWithCPK.GetFileClient(fileName2)); + // upload test - EXPECT_NO_THROW(fileClient.Create()); - EXPECT_NO_THROW(fileClient.UploadFrom(buffer->data(), bufferSize)); - auto result = fileClient.Download(); - auto downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); - EXPECT_EQ(*buffer, *downloaded); - EXPECT_NO_THROW(fileClient.Delete()); + EXPECT_NO_THROW(fileClientWithCPK->Create()); + EXPECT_NO_THROW(fileClientWithCPK->UploadFrom(buffer.data(), buffer.size())); + auto result = fileClientWithCPK->Download(); + auto downloaded = ReadBodyStream(result.Value.Body); + EXPECT_EQ(buffer, downloaded); + EXPECT_NO_THROW(fileClientWithCPK->Delete()); // append test - EXPECT_NO_THROW(fileClient.Create()); + EXPECT_NO_THROW(fileClientWithCPK->Create()); bodyStream.Rewind(); - EXPECT_NO_THROW(fileClient.Append(bodyStream, 0)); + EXPECT_NO_THROW(fileClientWithCPK->Append(bodyStream, 0)); bodyStream.Rewind(); - EXPECT_THROW(fileClientWithoutEncryptionKey.Append(bodyStream, bufferSize), StorageException); - EXPECT_NO_THROW(fileClient.Flush(bufferSize)); - result = fileClient.Download(); - downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); - EXPECT_EQ(*buffer, *downloaded); - EXPECT_NO_THROW(fileClient.SetMetadata(GetMetadata())); - auto properties = std::make_shared( - fileClient.GetProperties().Value); - EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); - EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); - EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); - - auto fileClientWithEncryptionKey - = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, fileName2, options); - auto created = std::make_shared( - fileClientWithEncryptionKey.Create().Value); - EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + EXPECT_THROW(fileClientWithoutCPK->Append(bodyStream, buffer.size()), StorageException); + EXPECT_NO_THROW(fileClientWithCPK->Flush(buffer.size())); + result = fileClientWithCPK->Download(); + downloaded = ReadBodyStream(result.Value.Body); + EXPECT_EQ(buffer, downloaded); + EXPECT_NO_THROW(fileClientWithCPK->SetMetadata(RandomMetadata())); + auto properties = fileClientWithCPK->GetProperties().Value; + EXPECT_TRUE(properties.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.EncryptionKeySha256.Value()); + EXPECT_THROW(fileClientWithoutCPK->Flush(buffer.size()), StorageException); + EXPECT_THROW(fileClientWithoutCPK->Download(), StorageException); + + auto createResult = fileClient2WithCPK->Create().Value; + EXPECT_TRUE(createResult.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, createResult.EncryptionKeySha256.Value()); } // directory works { @@ -590,59 +456,55 @@ namespace Azure { namespace Storage { namespace Test { const std::string fileName1 = "file1"; const std::string fileName2 = "file2"; const std::string fileName3 = "file3"; - auto directoryClient = fileSystemClient->GetDirectoryClient(directoryName); - auto directoryClientWithoutEncryptionKey - = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, directoryName); + + auto directoryClientWithCPK = std::make_shared( + fileSystemClientWithCPK.GetDirectoryClient(directoryName)); + auto directoryClientWithoutCPK = std::make_shared( + fileSystemClientWithoutCPK.GetDirectoryClient(directoryName)); + // create subdirectory/file - EXPECT_NO_THROW(directoryClient.Create()); - auto subdirectoryClient = directoryClient.GetSubdirectoryClient(subdirectoryName1); - EXPECT_NO_THROW(subdirectoryClient.Create()); - auto fileClient = directoryClient.GetFileClient(fileName1); - EXPECT_NO_THROW(fileClient.Create()); - auto subdirectoryProperties = std::make_shared( - subdirectoryClient.GetProperties().Value); - EXPECT_EQ(customerProvidedKey->KeyHash, subdirectoryProperties->EncryptionKeySha256.Value()); - auto fileProperties = fileClient.GetProperties(); - EXPECT_EQ(customerProvidedKey->KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); + EXPECT_NO_THROW(directoryClientWithCPK->Create()); + auto subdirectoryClientWithCPK = std::make_shared( + directoryClientWithCPK->GetSubdirectoryClient(subdirectoryName1)); + EXPECT_NO_THROW(subdirectoryClientWithCPK->Create()); + auto fileClientWithCPK = std::make_shared( + directoryClientWithCPK->GetFileClient(fileName1)); + EXPECT_NO_THROW(fileClientWithCPK->Create()); + auto subdirectoryProperties = subdirectoryClientWithCPK->GetProperties().Value; + EXPECT_EQ(customerProvidedKey.KeyHash, subdirectoryProperties.EncryptionKeySha256.Value()); + auto fileProperties = fileClientWithCPK->GetProperties(); + EXPECT_EQ(customerProvidedKey.KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); // rename file auto newFileClient - = directoryClient.RenameFile(fileName1, directoryName + "/" + fileName2).Value; - auto newFileProperties = std::make_shared( - newFileClient.GetProperties().Value); - EXPECT_EQ(customerProvidedKey->KeyHash, newFileProperties->EncryptionKeySha256.Value()); - auto newFileClientWithoutEncryptionKey - = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, directoryName + "/" + fileName2); - EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameFile( - fileName2, directoryName + "/" + fileName3)); - - auto newSubdirectoryClient - = directoryClient - .RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) + = directoryClientWithCPK->RenameFile(fileName1, directoryName + "/" + fileName2).Value; + auto newFileProperties = newFileClient.GetProperties().Value; + EXPECT_EQ(customerProvidedKey.KeyHash, newFileProperties.EncryptionKeySha256.Value()); + auto newFileClientWithoutCPK = std::make_shared( + fileSystemClientWithoutCPK.GetFileClient(directoryName + "/" + fileName2)); + EXPECT_THROW(newFileClientWithoutCPK->GetProperties(), StorageException); + EXPECT_NO_THROW( + directoryClientWithoutCPK->RenameFile(fileName2, directoryName + "/" + fileName3)); + + auto newSubdirectoryClientWithCPK + = directoryClientWithCPK + ->RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) .Value; - auto newSubdirectoryProperties = std::make_shared( - newSubdirectoryClient.GetProperties().Value); - EXPECT_EQ( - customerProvidedKey->KeyHash, newSubdirectoryProperties->EncryptionKeySha256.Value()); - auto newsubdirectoryClientWithoutEncryptionKey - = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - directoryName + "/" + subdirectoryName2); - EXPECT_THROW(newsubdirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameSubdirectory( + auto newSubdirectoryProperties = newSubdirectoryClientWithCPK.GetProperties().Value; + EXPECT_EQ(customerProvidedKey.KeyHash, newSubdirectoryProperties.EncryptionKeySha256.Value()); + auto newsubdirectoryClientWithoutCPK + = std::make_shared( + fileSystemClientWithoutCPK.GetDirectoryClient( + directoryName + "/" + subdirectoryName2)); + EXPECT_THROW(newsubdirectoryClientWithoutCPK->GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutCPK->RenameSubdirectory( subdirectoryName2, directoryName + "/" + subdirectoryName3)); - auto directoryClientWithEncryptionKey - = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, directoryName2, options); - auto created = std::make_shared( - directoryClientWithEncryptionKey.Create().Value); - EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + auto directoryClient2WithCPK = std::make_shared( + fileSystemClientWithCPK.GetDirectoryClient(directoryName2)); + auto createResult = directoryClient2WithCPK->Create().Value; + EXPECT_TRUE(createResult.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, createResult.EncryptionKeySha256.Value()); } } @@ -657,9 +519,9 @@ namespace Azure { namespace Storage { namespace Test { } // with EncryptionScope { - std::string fileSystemName = GetFileSystemValidName() + "1"; - std::string pathName = GetTestName() + "1"; - auto fileSystemClient = m_dataLakeServiceClient->GetFileSystemClient(fileSystemName); + std::string fileSystemName = LowercaseRandomString() + "1"; + std::string pathName = RandomString() + "1"; + auto fileSystemClient = GetFileSystemClientForTest(fileSystemName); Files::DataLake::CreateFileSystemOptions createOptions; createOptions.DefaultEncryptionScope = testEncryptionScope; createOptions.PreventEncryptionScopeOverride = true; @@ -685,40 +547,17 @@ namespace Azure { namespace Storage { namespace Test { createOptions.PreventEncryptionScopeOverride.Value()); } } - auto fileClient = fileSystemClient.GetFileClient(pathName); - fileClient.Create(); - Files::DataLake::ListPathsOptions listOptions; - for (auto page = fileSystemClient.ListPaths(false, listOptions); page.HasPage(); - page.MoveToNextPage()) - { - for (auto& path : page.Paths) - { - if (path.Name == pathName) - { - EXPECT_TRUE(path.EncryptionScope.HasValue()); - EXPECT_EQ(path.EncryptionScope.Value(), testEncryptionScope); - } - } - } - auto fileProperties = fileClient.GetProperties().Value; - EXPECT_TRUE(fileProperties.EncryptionScope.HasValue()); - EXPECT_EQ(fileProperties.EncryptionScope.Value(), testEncryptionScope); - fileSystemClient.Delete(); } } - TEST_F(DataLakeFileSystemClientTest, GetSetAccessPolicy_LIVEONLY_) + TEST_F(DataLakeFileSystemClientTest, GetSetAccessPolicy) { - CHECK_SKIP_TEST(); { - auto fileSystem = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - GetTestNameLowerCase(), - InitClientOptions()); - fileSystem.Create(); + auto fileSystem = GetFileSystemClientForTest(LowercaseRandomString()); + fileSystem.CreateIfNotExists(); Files::DataLake::SetFileSystemAccessPolicyOptions options; - options.AccessType = Files::DataLake::Models::PublicAccessType::Path; + options.AccessType = Files::DataLake::Models::PublicAccessType::None; { Files::DataLake::Models::SignedIdentifier identifier; identifier.Id = std::string(64, 'a'); @@ -757,9 +596,9 @@ namespace Azure { namespace Storage { namespace Test { auto ret2 = fileSystem.GetAccessPolicy(); EXPECT_EQ(ret2.Value.AccessType, options.AccessType); ASSERT_EQ(ret2.Value.SignedIdentifiers.size(), options.SignedIdentifiers.size()); - for (size_t i = 0; i < ret2.Value.SignedIdentifiers.size(); ++i) + if (m_testContext.IsLiveMode()) { - EXPECT_EQ(ret2.Value.SignedIdentifiers[i], options.SignedIdentifiers[i]); + EXPECT_EQ(ret2.Value.SignedIdentifiers, options.SignedIdentifiers); } options.AccessType = Files::DataLake::Models::PublicAccessType::FileSystem; @@ -771,51 +610,37 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(fileSystem.SetAccessPolicy(options)); ret2 = fileSystem.GetAccessPolicy(); EXPECT_EQ(ret2.Value.AccessType, options.AccessType); - - fileSystem.Delete(); } { - auto fileSystem = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - GetTestNameLowerCase() + "1", - InitClientOptions()); + auto fileSystem = GetFileSystemClientForTest(LowercaseRandomString()); Files::DataLake::CreateFileSystemOptions options; options.AccessType = Files::DataLake::Models::PublicAccessType::FileSystem; fileSystem.Create(options); auto ret = fileSystem.GetAccessPolicy(); EXPECT_EQ(Files::DataLake::Models::PublicAccessType::FileSystem, ret.Value.AccessType); - fileSystem.Delete(); } { - auto fileSystem = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - GetTestNameLowerCase() + "2", - InitClientOptions()); + auto fileSystem = GetFileSystemClientForTest(LowercaseRandomString()); Files::DataLake::CreateFileSystemOptions options; options.AccessType = Files::DataLake::Models::PublicAccessType::Path; fileSystem.Create(options); auto ret = fileSystem.GetAccessPolicy(); EXPECT_EQ(Files::DataLake::Models::PublicAccessType::Path, ret.Value.AccessType); - fileSystem.Delete(); } { - auto fileSystem = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - GetTestNameLowerCase() + "3", - InitClientOptions()); + auto fileSystem = GetFileSystemClientForTest(LowercaseRandomString()); Files::DataLake::CreateFileSystemOptions options; options.AccessType = Files::DataLake::Models::PublicAccessType::Path; fileSystem.Create(options); auto ret = fileSystem.GetAccessPolicy(); EXPECT_EQ(Files::DataLake::Models::PublicAccessType::Path, ret.Value.AccessType); - fileSystem.Delete(); } } TEST_F(DataLakeFileSystemClientTest, RenameFile) { - const std::string oldFilename = GetTestName() + "1"; - const std::string newFilename = GetTestName() + "2"; + const std::string oldFilename = RandomString() + "1"; + const std::string newFilename = RandomString() + "2"; auto oldFileClient = m_fileSystemClient->GetFileClient(oldFilename); oldFileClient.Create(); @@ -826,10 +651,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(m_fileSystemClient->GetFileClient(newFilename).GetProperties()); EXPECT_THROW(oldFileClient.GetProperties(), StorageException); - const std::string newFileSystemName = GetTestNameLowerCase() + "1"; - const std::string newFilename2 = GetTestName() + "3"; + const std::string newFileSystemName = LowercaseRandomString() + "1"; + const std::string newFilename2 = LowercaseRandomString() + "3"; - auto newFileSystem = m_dataLakeServiceClient->GetFileSystemClient(newFileSystemName); + auto newFileSystem = GetFileSystemClientForTest(newFileSystemName); newFileSystem.Create(); Files::DataLake::RenameFileOptions options; @@ -844,14 +669,14 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileSystemClientTest, RenameDirectory) { - const std::string testName(GetTestName()); - const std::string oldDirectoryName = testName + "1"; - const std::string newDirectoryName = testName + "2"; + const std::string baseName = RandomString(); + const std::string oldDirectoryName = baseName + "1"; + const std::string newDirectoryName = baseName + "2"; auto oldDirectoryClient = m_fileSystemClient->GetDirectoryClient(oldDirectoryName); oldDirectoryClient.Create(); - oldDirectoryClient.GetFileClient(testName + "3").Create(); - oldDirectoryClient.GetSubdirectoryClient(testName + "4").Create(); + oldDirectoryClient.GetFileClient(baseName + "3").Create(); + oldDirectoryClient.GetSubdirectoryClient(baseName + "4").Create(); auto newDirectoryClient = m_fileSystemClient->RenameDirectory(oldDirectoryName, newDirectoryName).Value; @@ -860,10 +685,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(m_fileSystemClient->GetDirectoryClient(newDirectoryName).GetProperties()); EXPECT_THROW(oldDirectoryClient.GetProperties(), StorageException); - const std::string newFileSystemName = GetTestNameLowerCase(); - const std::string newDirectoryName2 = testName + "5"; + const std::string newFileSystemName = LowercaseRandomString(); + const std::string newDirectoryName2 = baseName + "5"; - auto newFileSystem = m_dataLakeServiceClient->GetFileSystemClient(newFileSystemName); + auto newFileSystem = GetFileSystemClientForTest(newFileSystemName); newFileSystem.Create(); Files::DataLake::RenameDirectoryOptions options; @@ -877,23 +702,26 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(newDirectoryClient.GetProperties(), StorageException); } - TEST_F(DataLakeFileSystemClientTest, RenameFileSasAuthentication_LIVEONLY_) + TEST_F(DataLakeFileSystemClientTest, RenameFileSasAuthentication) { - const std::string testName(GetTestName()); - const std::string sourceFilename = testName + "1"; - const std::string destinationFilename = testName + "2"; + const std::string baseName = RandomString(); + const std::string sourceFilename = baseName + "1"; + const std::string destinationFilename = baseName + "2"; auto fileClient = m_fileSystemClient->GetFileClient(sourceFilename); fileClient.CreateIfNotExists(); + Files::DataLake::DataLakeClientOptions options; + InitClientOptions(options); Files::DataLake::DataLakeFileSystemClient fileSystemClientSas( - Files::DataLake::_detail::GetDfsUrlFromUrl(m_fileSystemClient->GetUrl()) + GetSas()); + Files::DataLake::_detail::GetDfsUrlFromUrl(m_fileSystemClient->GetUrl()) + GetSas(), + options); fileSystemClientSas.RenameFile(sourceFilename, destinationFilename); EXPECT_THROW( m_fileSystemClient->GetFileClient(sourceFilename).GetProperties(), StorageException); EXPECT_NO_THROW(m_fileSystemClient->GetFileClient(destinationFilename).GetProperties()); - const std::string sourceDirectoryName = testName + "3"; - const std::string destinationDirectoryName = testName + "4"; + const std::string sourceDirectoryName = baseName + "3"; + const std::string destinationDirectoryName = baseName + "4"; auto directoryClient = m_fileSystemClient->GetDirectoryClient(sourceDirectoryName); directoryClient.CreateIfNotExists(); @@ -907,10 +735,10 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileSystemClientTest, ListDeletedPaths) { - const std::string deletedFilename = GetTestNameLowerCase() + "_file_deleted"; - const std::string nonDeletedFilename = GetTestNameLowerCase() + "_file"; - const std::string deletedDirectoryName = GetTestNameLowerCase() + "_dir_deleted"; - const std::string nonDeletedDirectoryName = GetTestNameLowerCase() + "_dir"; + const std::string deletedFilename = RandomString() + "_file_deleted"; + const std::string nonDeletedFilename = RandomString() + "_file"; + const std::string deletedDirectoryName = RandomString() + "_dir_deleted"; + const std::string nonDeletedDirectoryName = RandomString() + "_dir"; auto deletedFileClient = m_fileSystemClient->GetFileClient(deletedFilename); auto nonDeletedFileClient = m_fileSystemClient->GetFileClient(nonDeletedFilename); @@ -926,15 +754,17 @@ namespace Azure { namespace Storage { namespace Test { nonDeletedDirectoryClient.Create(); { - std::vector paths; + std::set paths; for (auto pageResult = m_fileSystemClient->ListDeletedPaths(); pageResult.HasPage(); pageResult.MoveToNextPage()) { - paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); + for (const auto& p : pageResult.DeletedPaths) + { + paths.insert(p.Name); + } } - EXPECT_EQ(2, paths.size()); - EXPECT_EQ(deletedDirectoryName, paths[0].Name); - EXPECT_EQ(deletedFilename, paths[1].Name); + EXPECT_NE(paths.find(deletedDirectoryName), paths.end()); + EXPECT_NE(paths.find(deletedFilename), paths.end()); } // List max result { @@ -951,7 +781,7 @@ namespace Azure { namespace Storage { namespace Test { } // prefix works { - const std::string directoryName = GetTestNameLowerCase() + "_prefix"; + const std::string directoryName = RandomString() + "_prefix"; const std::string filename = "file"; auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName); @@ -975,7 +805,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileSystemClientTest, Undelete) { - const std::string directoryName = GetTestNameLowerCase() + "_dir"; + const std::string directoryName = RandomString() + "_dir"; const std::string subFileName = "sub_file"; auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.hpp index fe8b045bfa..359e38784a 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.hpp @@ -4,31 +4,23 @@ #include #include "datalake_service_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class DataLakeFileSystemClientTest : public DataLakeServiceClientTest { protected: - void SetUp(); - void TearDown(); - std::string GetSas(); - void CreateDirectoryList(); + void SetUp() override; - std::vector ListAllPaths( - bool recursive, - const std::string& directory = std::string()); + std::string GetSas(); - Files::DataLake::Models::PathHttpHeaders GetInterestingHttpHeaders(); + Files::DataLake::DataLakeFileSystemClient GetFileSystemClientForTest( + const std::string& fileSystemName, + Files::DataLake::DataLakeClientOptions clientOptions + = Files::DataLake::DataLakeClientOptions()); + protected: std::shared_ptr m_fileSystemClient; std::string m_fileSystemName; - - // Path related - std::vector m_pathNameSetA; - std::string m_directoryA; - std::vector m_pathNameSetB; - std::string m_directoryB; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp index 0dd105a694..137b6c51cc 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp @@ -13,22 +13,17 @@ namespace Azure { namespace Storage { namespace Test { void DataLakePathClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); - CHECK_SKIP_TEST(); - - m_pathName = GetFileSystemValidName(); + if (shouldSkipTest()) + { + return; + } + m_pathName = RandomString(); m_pathClient = std::make_shared( m_fileSystemClient->GetFileClient(m_pathName)); m_fileSystemClient->GetFileClient(m_pathName).Create(); } - void DataLakePathClientTest::TearDown() - { - CHECK_SKIP_TEST(); - m_pathClient->Delete(); - DataLakeFileSystemClientTest::TearDown(); - } - - std::vector DataLakePathClientTest::GetValidAcls() + std::vector DataLakePathClientTest::GetAclsForTesting() { static std::vector result = []() { std::vector ret; @@ -61,7 +56,7 @@ namespace Azure { namespace Storage { namespace Test { { // owner&group { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "owner_group"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "owner_group"); Files::DataLake::CreateFileOptions options; options.Group = "$superuser"; options.Owner = "$superuser"; @@ -72,9 +67,9 @@ namespace Azure { namespace Storage { namespace Test { } // acl { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acl"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_acl"); Files::DataLake::CreateFileOptions options; - auto acls = GetValidAcls(); + auto acls = GetAclsForTesting(); options.Acls = acls; EXPECT_NO_THROW(client.Create(options)); auto resultAcls = client.GetAccessControlList().Value.Acls; @@ -94,9 +89,9 @@ namespace Azure { namespace Storage { namespace Test { } // lease { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_lease"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_lease"); Files::DataLake::CreateFileOptions options; - options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseId = RandomUUID(); options.LeaseDuration = std::chrono::seconds(20); EXPECT_NO_THROW(client.Create(options)); auto response = client.GetProperties(); @@ -110,7 +105,7 @@ namespace Azure { namespace Storage { namespace Test { } // relative expiry { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_relative_expiry"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_relative_expiry"); Files::DataLake::CreateFileOptions options; options.ScheduleDeletionOptions.TimeToExpire = std::chrono::hours(1); EXPECT_NO_THROW(client.Create(options)); @@ -126,7 +121,7 @@ namespace Azure { namespace Storage { namespace Test { } // absolute expiry { - auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_absolute_expiry"); + auto client = m_fileSystemClient->GetFileClient(RandomString() + "_absolute_expiry"); Files::DataLake::CreateFileOptions options; options.ScheduleDeletionOptions.ExpiresOn = Azure::DateTime::Parse( "Wed, 29 Sep 2100 09:53:03 GMT", Azure::DateTime::DateFormat::Rfc1123); @@ -140,8 +135,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakePathClientTest, PathMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_pathClient->SetMetadata(metadata1)); @@ -154,9 +149,9 @@ namespace Azure { namespace Storage { namespace Test { { // Create path with metadata works - std::string const testName(GetTestName()); - auto client1 = m_fileSystemClient->GetFileClient(testName + "1"); - auto client2 = m_fileSystemClient->GetFileClient(testName + "1"); + std::string const baseName = RandomString(); + auto client1 = m_fileSystemClient->GetFileClient(baseName + "1"); + auto client2 = m_fileSystemClient->GetFileClient(baseName + "2"); Files::DataLake::CreatePathOptions options1; Files::DataLake::CreatePathOptions options2; options1.Metadata = metadata1; @@ -173,8 +168,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakePathClientTest, GetDataLakePathPropertiesResult) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Get Metadata via properties works EXPECT_NO_THROW(m_pathClient->SetMetadata(metadata1)); @@ -203,48 +198,55 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakePathClientTest, PathHttpHeaders) { - const std::string testName(GetTestName()); + auto httpHeaders = Files::DataLake::Models::PathHttpHeaders(); + httpHeaders.ContentType = "application/x-binary"; + httpHeaders.ContentLanguage = "en-US"; + httpHeaders.ContentDisposition = "attachment"; + httpHeaders.CacheControl = "no-cache"; + httpHeaders.ContentEncoding = "identity"; + + const std::string baseName = RandomString(); { // HTTP headers works with create. - auto httpHeader = GetInterestingHttpHeaders(); std::vector pathClient; for (int32_t i = 0; i < 2; ++i) { - auto client = m_fileSystemClient->GetFileClient(testName + std::to_string(i)); + auto client = m_fileSystemClient->GetFileClient(baseName + std::to_string(i)); Files::DataLake::CreatePathOptions options; - options.HttpHeaders = httpHeader; + options.HttpHeaders = httpHeaders; EXPECT_NO_THROW(client.Create(options)); pathClient.emplace_back(std::move(client)); } for (const auto& client : pathClient) { auto result = client.GetProperties(); - EXPECT_EQ(httpHeader.CacheControl, result.Value.HttpHeaders.CacheControl); - EXPECT_EQ(httpHeader.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); - EXPECT_EQ(httpHeader.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); - EXPECT_EQ(httpHeader.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.CacheControl, result.Value.HttpHeaders.CacheControl); + EXPECT_EQ(httpHeaders.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); + EXPECT_EQ(httpHeaders.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); + EXPECT_EQ(httpHeaders.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentEncoding, result.Value.HttpHeaders.ContentEncoding); client.Delete(); } } { // HTTP headers works with SetHttpHeaders. - auto httpHeader = GetInterestingHttpHeaders(); std::vector pathClient; for (int32_t i = 0; i < 2; ++i) { - auto client = m_fileSystemClient->GetFileClient(testName + "2" + std::to_string(i)); + auto client = m_fileSystemClient->GetFileClient(baseName + "2" + std::to_string(i)); EXPECT_NO_THROW(client.Create()); - EXPECT_NO_THROW(client.SetHttpHeaders(httpHeader)); + EXPECT_NO_THROW(client.SetHttpHeaders(httpHeaders)); pathClient.emplace_back(std::move(client)); } for (const auto& client : pathClient) { auto result = client.GetProperties(); - EXPECT_EQ(httpHeader.CacheControl, result.Value.HttpHeaders.CacheControl); - EXPECT_EQ(httpHeader.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); - EXPECT_EQ(httpHeader.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); - EXPECT_EQ(httpHeader.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.CacheControl, result.Value.HttpHeaders.CacheControl); + EXPECT_EQ(httpHeaders.ContentDisposition, result.Value.HttpHeaders.ContentDisposition); + EXPECT_EQ(httpHeaders.ContentLanguage, result.Value.HttpHeaders.ContentLanguage); + EXPECT_EQ(httpHeaders.ContentType, result.Value.HttpHeaders.ContentType); + EXPECT_EQ(httpHeaders.ContentEncoding, result.Value.HttpHeaders.ContentEncoding); client.Delete(); } } @@ -254,11 +256,10 @@ namespace Azure { namespace Storage { namespace Test { auto response = m_pathClient->GetProperties(); Files::DataLake::SetPathHttpHeadersOptions options1; options1.AccessConditions.IfModifiedSince = response.Value.LastModified; - EXPECT_THROW( - m_pathClient->SetHttpHeaders(GetInterestingHttpHeaders(), options1), StorageException); + EXPECT_THROW(m_pathClient->SetHttpHeaders(httpHeaders, options1), StorageException); Files::DataLake::SetPathHttpHeadersOptions options2; options2.AccessConditions.IfUnmodifiedSince = response.Value.LastModified; - EXPECT_NO_THROW(m_pathClient->SetHttpHeaders(GetInterestingHttpHeaders(), options2)); + EXPECT_NO_THROW(m_pathClient->SetHttpHeaders(httpHeaders, options2)); } { @@ -266,11 +267,10 @@ namespace Azure { namespace Storage { namespace Test { auto response = m_pathClient->GetProperties(); Files::DataLake::SetPathHttpHeadersOptions options1; options1.AccessConditions.IfNoneMatch = response.Value.ETag; - EXPECT_THROW( - m_pathClient->SetHttpHeaders(GetInterestingHttpHeaders(), options1), StorageException); + EXPECT_THROW(m_pathClient->SetHttpHeaders(httpHeaders, options1), StorageException); Files::DataLake::SetPathHttpHeadersOptions options2; options2.AccessConditions.IfMatch = response.Value.ETag; - EXPECT_NO_THROW(m_pathClient->SetHttpHeaders(GetInterestingHttpHeaders(), options2)); + EXPECT_NO_THROW(m_pathClient->SetHttpHeaders(httpHeaders, options2)); } } @@ -278,7 +278,7 @@ namespace Azure { namespace Storage { namespace Test { { { // Set/Get Acls works. - std::vector acls = GetValidAcls(); + std::vector acls = GetAclsForTesting(); EXPECT_NO_THROW(m_pathClient->SetAccessControlList(acls)); std::vector resultAcls; EXPECT_NO_THROW(resultAcls = m_pathClient->GetAccessControlList().Value.Acls); @@ -299,7 +299,7 @@ namespace Azure { namespace Storage { namespace Test { { // Set/Get Acls works with last modified access condition. - std::vector acls = GetValidAcls(); + std::vector acls = GetAclsForTesting(); auto response = m_pathClient->GetProperties(); Files::DataLake::SetPathAccessControlListOptions options1; @@ -312,7 +312,7 @@ namespace Azure { namespace Storage { namespace Test { { // Set/Get Acls works with if match access condition. - std::vector acls = GetValidAcls(); + std::vector acls = GetAclsForTesting(); auto response = m_pathClient->GetProperties(); Files::DataLake::SetPathAccessControlListOptions options1; options1.AccessConditions.IfNoneMatch = response.Value.ETag; @@ -325,12 +325,12 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakePathClientTest, PathSetPermissions) { - std::string const testName(GetTestName()); + std::string const baseName = RandomString(); { auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, - testName + "1", + baseName + "1", InitClientOptions()); pathClient.Create(Files::DataLake::Models::PathResourceType::File); std::string pathPermissions = "rwxrw-rw-"; @@ -352,7 +352,7 @@ namespace Azure { namespace Storage { namespace Test { auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, - testName + "2", + baseName + "2", InitClientOptions()); auto response = pathClient.Create(Files::DataLake::Models::PathResourceType::File); Files::DataLake::SetPathPermissionsOptions options1, options2; @@ -367,7 +367,7 @@ namespace Azure { namespace Storage { namespace Test { auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, - testName + "3", + baseName + "3", InitClientOptions()); auto response = pathClient.Create(Files::DataLake::Models::PathResourceType::File); Files::DataLake::SetPathPermissionsOptions options1, options2; @@ -378,135 +378,5 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(pathClient.SetPermissions(pathPermissions, options1)); } } -#if 0 - TEST_F(DataLakePathClientTest, LeaseRelated) - { - std::string leaseId1 = CreateUniqueLeaseId(); - int32_t leaseDuration = 20; - auto lastModified = m_pathClient->GetProperties().Value.LastModified; - auto aLease = *m_pathClient->AcquireLease(leaseId1, leaseDuration); - EXPECT_FALSE(aLease.ETag.empty()); - EXPECT_FALSE(aLease.LastModified > lastModified); - EXPECT_EQ(aLease.LeaseId, leaseId1); - aLease = *m_pathClient->AcquireLease(leaseId1, leaseDuration); - EXPECT_FALSE(aLease.ETag.empty()); - EXPECT_FALSE(aLease.LastModified > lastModified); - EXPECT_EQ(aLease.LeaseId, leaseId1); - - auto properties = *m_pathClient->GetProperties(); - EXPECT_EQ(properties.LeaseState.Value(), - Files::DataLake::Models::LeaseStateType::Leased); - EXPECT_EQ(properties.LeaseStatus.Value(), - Files::DataLake::Models::LeaseStatusType::Locked); - EXPECT_FALSE(properties.LeaseDuration.Value().empty()); - - lastModified = properties.LastModified; - auto rLease = *m_pathClient->RenewLease(leaseId1); - EXPECT_FALSE(rLease.ETag.empty()); - EXPECT_FALSE(rLease.LastModified > lastModified); - EXPECT_EQ(rLease.LeaseId, leaseId1); - - std::string leaseId2 = CreateUniqueLeaseId(); - EXPECT_NE(leaseId1, leaseId2); - lastModified = m_pathClient->GetProperties().Value.LastModified; - auto cLease = *m_pathClient->ChangeLease(leaseId1, leaseId2); - EXPECT_FALSE(cLease.ETag.empty()); - EXPECT_FALSE(cLease.LastModified > lastModified); - EXPECT_EQ(cLease.LeaseId, leaseId2); - - lastModified = m_pathClient->GetProperties().Value.LastModified; - auto pathInfo = *m_pathClient->ReleaseLease(leaseId2); - EXPECT_FALSE(pathInfo.ETag.empty()); - EXPECT_FALSE(pathInfo.LastModified > lastModified); - - lastModified = m_pathClient->GetProperties().Value.LastModified; - aLease = *m_pathClient->AcquireLease(CreateUniqueLeaseId(), InfiniteLeaseDuration); - properties = *m_pathClient->GetProperties(); - EXPECT_FALSE(properties.LeaseDuration.Value().empty()); - auto brokenLease = *m_pathClient->BreakLease(); - EXPECT_FALSE(brokenLease.ETag.empty()); - EXPECT_FALSE(brokenLease.LastModified > lastModified); - EXPECT_EQ(brokenLease.LeaseTime, 0); - - aLease = *m_pathClient->AcquireLease(CreateUniqueLeaseId(), leaseDuration); - Files::DataLake::BreakDataLakePathLeaseOptions breakOptions; - breakOptions.BreakPeriod = 30; - lastModified = m_pathClient->GetProperties().Value.LastModified; - brokenLease = *m_pathClient->BreakLease(breakOptions); - EXPECT_FALSE(brokenLease.ETag.empty()); - EXPECT_FALSE(brokenLease.LastModified > lastModified); - EXPECT_NE(brokenLease.LeaseTime, 0); - - Files::DataLake::BreakDataLakePathLeaseOptions options; - options.BreakPeriod = 0; - m_pathClient->BreakLease(options); - } -#endif - TEST_F(DataLakePathClientTest, ConstructorsWorks) - { - { - // Create from connection string validates static creator function and shared key constructor. - auto pathName = GetTestName(); - auto connectionStringClient - = Azure::Storage::Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - pathName, - InitClientOptions()); - EXPECT_NO_THROW( - connectionStringClient.Create(Files::DataLake::Models::PathResourceType::File)); - EXPECT_NO_THROW(connectionStringClient.Delete()); - } - - { - // Create from client secret credential. - std::shared_ptr credential - = std::make_shared( - AadTenantId(), AadClientId(), AadClientSecret()); - Files::DataLake::DataLakeClientOptions options; - - auto clientSecretClient = InitTestClient< - Azure::Storage::Files::DataLake::DataLakePathClient, - Files::DataLake::DataLakeClientOptions>( - Files::DataLake::_detail::GetDfsUrlFromUrl( - Azure::Storage::Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - GetTestName() + "withSecret", - InitClientOptions()) - .GetUrl()), - credential, - options); - - EXPECT_NO_THROW(clientSecretClient->Create(Files::DataLake::Models::PathResourceType::File)); - EXPECT_NO_THROW(clientSecretClient->Delete()); - } - - { - // Create from Anonymous credential. - auto objectName = "objectName"; - auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - InitClientOptions()); - Azure::Storage::Blobs::SetBlobContainerAccessPolicyOptions options; - options.AccessType = Azure::Storage::Blobs::Models::PublicAccessType::BlobContainer; - containerClient.SetAccessPolicy(options); - - auto pathClient - = Azure::Storage::Files::DataLake::DataLakePathClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - m_fileSystemName, - objectName, - InitClientOptions()); - EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); - auto anonymousClient = Azure::Storage::Files::DataLake::DataLakePathClient( - pathClient.GetUrl(), InitClientOptions()); - - TestSleep(std::chrono::seconds(30)); - - EXPECT_NO_THROW(anonymousClient.GetProperties()); - } - } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.hpp index 84be4e583f..004e8aea41 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.hpp @@ -4,17 +4,16 @@ #include #include "datalake_file_system_client_test.hpp" -#include "test/ut/test_base.hpp" namespace Azure { namespace Storage { namespace Test { class DataLakePathClientTest : public DataLakeFileSystemClientTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; - std::vector GetValidAcls(); + std::vector GetAclsForTesting(); + protected: std::shared_ptr m_pathClient; std::string m_pathName; }; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp index eaa3e7616d..234d54cd09 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp @@ -13,8 +13,6 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileSystemClientTest, DataLakeSasTest_LIVEONLY_) { - CHECK_SKIP_TEST(); - auto sasStartsOn = std::chrono::system_clock::now() - std::chrono::minutes(5); auto sasExpiredOn = std::chrono::system_clock::now() - std::chrono::minutes(1); auto sasExpiresOn = std::chrono::system_clock::now() + std::chrono::minutes(60); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp index 97f68e9dd2..c922e88e70 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp @@ -68,115 +68,57 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { - const size_t FileSystemTestSize = 5; - - void DataLakeServiceClientTest::CreateFileSystemList() + TEST_F(DataLakeServiceClientTest, ListFileSystemsSegment) { - std::string const fileSystemName(GetFileSystemValidName()); - std::string const prefix(fileSystemName.begin(), fileSystemName.end() - 2); - m_fileSystemPrefixA = prefix + "a"; - m_fileSystemPrefixB = prefix + "b"; - m_fileSystemNameSetA.clear(); - m_fileSystemNameSetB.clear(); - for (size_t i = 0; i < FileSystemTestSize; ++i) + const std::string prefix1 = LowercaseRandomString(); + const std::string prefix2 = LowercaseRandomString(); + std::set filesystemSet1; + std::set filesystemSet2; + for (int i = 0; i < 5; ++i) { - { - auto name = m_fileSystemPrefixA + std::to_string(i); - m_dataLakeServiceClient->GetFileSystemClient(name).Create(); - m_fileSystemNameSetA.emplace_back(std::move(name)); - } - { - auto name = m_fileSystemPrefixB + std::to_string(i); - m_dataLakeServiceClient->GetFileSystemClient(name).Create(); - m_fileSystemNameSetB.emplace_back(std::move(name)); - } - } - } - - void DataLakeServiceClientTest::SetUp() - { - StorageTest::SetUp(); - - CHECK_SKIP_TEST(); + std::string fsName = prefix1 + LowercaseRandomString(); - m_dataLakeServiceClient = std::make_shared( - Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), - InitClientOptions())); - } - - std::vector - DataLakeServiceClientTest::ListAllFileSystems(const std::string& prefix) - { - std::vector result; - std::string continuation; - Files::DataLake::ListFileSystemsOptions options; - if (!prefix.empty()) - { - options.Prefix = prefix; - } - for (auto pageResult = m_dataLakeServiceClient->ListFileSystems(options); pageResult.HasPage(); - pageResult.MoveToNextPage()) - { - result.insert(result.end(), pageResult.FileSystems.begin(), pageResult.FileSystems.end()); + m_dataLakeServiceClient->GetFileSystemClient(fsName).Create(); + filesystemSet1.insert(fsName); + fsName = prefix2 + LowercaseRandomString(); + m_dataLakeServiceClient->GetFileSystemClient(fsName).Create(); + filesystemSet2.insert(fsName); } - return result; - } - - TEST_F(DataLakeServiceClientTest, ListFileSystemsSegment) - { - CreateFileSystemList(); { // Normal list without prefix. - auto result = ListAllFileSystems(); - for (const auto& name : m_fileSystemNameSetA) + std::set result; + for (auto page = m_dataLakeServiceClient->ListFileSystems(); page.HasPage(); + page.MoveToNextPage()) + { + for (auto fs : page.FileSystems) + { + result.insert(fs.Name); + } + } + for (const auto& f : filesystemSet1) { - auto iter = std::find_if( - result.begin(), - result.end(), - [&name](const Files::DataLake::Models::FileSystemItem& fileSystem) { - return fileSystem.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_fileSystemPrefixA.size()), m_fileSystemPrefixA); - EXPECT_NE(result.end(), iter); + EXPECT_NE(result.find(f), result.end()); } - for (const auto& name : m_fileSystemNameSetB) + for (const auto& f : filesystemSet2) { - auto iter = std::find_if( - result.begin(), - result.end(), - [&name](const Files::DataLake::Models::FileSystemItem& fileSystem) { - return fileSystem.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_fileSystemPrefixB.size()), m_fileSystemPrefixB); - EXPECT_NE(result.end(), iter); + EXPECT_NE(result.find(f), result.end()); } } { - // List prefix. - auto result = ListAllFileSystems(m_fileSystemPrefixA); - for (const auto& name : m_fileSystemNameSetA) + std::set result; + Files::DataLake::ListFileSystemsOptions listOptions; + listOptions.Prefix = prefix1; + for (auto page = m_dataLakeServiceClient->ListFileSystems(listOptions); page.HasPage(); + page.MoveToNextPage()) { - auto iter = std::find_if( - result.begin(), - result.end(), - [&name](const Files::DataLake::Models::FileSystemItem& fileSystem) { - return fileSystem.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_fileSystemPrefixA.size()), m_fileSystemPrefixA); - EXPECT_EQ(iter->Details.DefaultEncryptionScope, AccountEncryptionKey); - EXPECT_FALSE(iter->Details.PreventEncryptionScopeOverride); - EXPECT_NE(result.end(), iter); + for (auto fs : page.FileSystems) + { + result.insert(fs.Name); + } } - for (const auto& name : m_fileSystemNameSetB) + for (const auto& f : filesystemSet1) { - auto iter = std::find_if( - result.begin(), - result.end(), - [&name](const Files::DataLake::Models::FileSystemItem& fileSystem) { - return fileSystem.Name == name; - }); - EXPECT_EQ(result.end(), iter); + EXPECT_NE(result.find(f), result.end()); } } { @@ -186,6 +128,14 @@ namespace Azure { namespace Storage { namespace Test { auto response = m_dataLakeServiceClient->ListFileSystems(options); EXPECT_LE(2U, response.FileSystems.size()); } + for (const auto& fsName : filesystemSet1) + { + m_dataLakeServiceClient->GetFileSystemClient(fsName).DeleteIfExists(); + } + for (const auto& fsName : filesystemSet2) + { + m_dataLakeServiceClient->GetFileSystemClient(fsName).DeleteIfExists(); + } } TEST_F(DataLakeServiceClientTest, DISABLED_ListSystemFileSystems) @@ -209,10 +159,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(fileSystems.empty()); } - TEST_F(DataLakeServiceClientTest, AnonymousConstructorsWorks_LIVEONLY_) + TEST_F(DataLakeServiceClientTest, AnonymousConstructorsWorks) { - CHECK_SKIP_TEST(); - auto keyCredential = Azure::Storage::_internal::ParseConnectionString(AdlsGen2ConnectionString()) .KeyCredential; @@ -388,4 +336,5 @@ namespace Azure { namespace Storage { namespace Test { auto res = m_dataLakeServiceClient->SetProperties(originalProperties); } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp index de564448a2..a19ce04bf5 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp @@ -9,32 +9,17 @@ namespace Azure { namespace Storage { namespace Test { class DataLakeServiceClientTest : public Azure::Storage::Test::StorageTest { protected: - std::shared_ptr m_dataLakeServiceClient; - std::vector m_fileSystemNameSetA; - std::string m_fileSystemPrefixA; - std::vector m_fileSystemNameSetB; - std::string m_fileSystemPrefixB; - - virtual void SetUp() override; - - void CreateFileSystemList(); - - virtual void TearDown() override + void SetUp() override { - for (const auto& name : m_fileSystemNameSetA) - { - m_dataLakeServiceClient->GetFileSystemClient(name).Delete(); - } - for (const auto& name : m_fileSystemNameSetB) - { - m_dataLakeServiceClient->GetFileSystemClient(name).Delete(); - } - - StorageTest::TearDown(); + StorageTest::SetUp(); + auto options = InitClientOptions(); + m_dataLakeServiceClient = std::make_shared( + Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), options)); } - std::vector ListAllFileSystems( - const std::string& prefix = std::string()); + protected: + std::shared_ptr m_dataLakeServiceClient; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp index 05731619a6..1c83a1925d 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp @@ -30,95 +30,104 @@ namespace Azure { namespace Storage { namespace Test { void FileShareClientTest::SetUp() { - StorageTest::SetUp(); - CHECK_SKIP_TEST(); - - m_shareName = GetFileSystemValidName(); - m_testName = GetTestName(); - m_testNameLowercase = GetTestNameLowerCase(); - m_options = InitClientOptions(); + FileShareServiceClientTest::SetUp(); + if (shouldSkipTest()) + { + return; + } + m_shareName = GetLowercaseIdentifier(); m_shareClient = std::make_shared( - Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_shareName, m_options)); - m_shareClient->CreateIfNotExists(); + m_shareServiceClient->GetShareClient(m_shareName)); + while (true) + { + try + { + m_shareClient->CreateIfNotExists(); + break; + } + catch (StorageException& e) + { + if (e.ErrorCode != "ShareBeingDeleted") + { + throw; + } + SUCCEED() << "Share is being deleted. Will try again after 3 seconds."; + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + } + + m_resourceCleanupFunctions.push_back([shareClient = *m_shareClient]() { + Files::Shares::DeleteShareOptions options; + options.DeleteSnapshots = true; + shareClient.DeleteIfExists(options); + }); } - void FileShareClientTest::TearDown() + Files::Shares::ShareClient FileShareClientTest::GetShareClientForTest( + const std::string& shareName, + Files::Shares::ShareClientOptions clientOptions) { - CHECK_SKIP_TEST(); - - auto deleteOptions = Files::Shares::DeleteShareOptions(); - deleteOptions.DeleteSnapshots = true; - m_shareClient->DeleteIfExists(deleteOptions); - - StorageTest::TearDown(); + InitClientOptions(clientOptions); + auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( + StandardStorageConnectionString(), shareName, clientOptions); + m_resourceCleanupFunctions.push_back([shareClient]() { + Files::Shares::DeleteShareOptions options; + options.DeleteSnapshots = true; + shareClient.DeleteIfExists(options); + }); + + return shareClient; } - Files::Shares::Models::FileHttpHeaders FileShareClientTest::GetInterestingHttpHeaders() + Files::Shares::ShareClient FileShareClientTest::GetPremiumShareClientForTest( + const std::string& shareName, + Files::Shares::ShareClientOptions clientOptions) { - static Files::Shares::Models::FileHttpHeaders result = []() { - Files::Shares::Models::FileHttpHeaders ret; - ret.CacheControl = std::string("no-cache"); - ret.ContentDisposition = std::string("attachment"); - ret.ContentEncoding = std::string("deflate"); - ret.ContentLanguage = std::string("en-US"); - ret.ContentType = std::string("application/octet-stream"); - return ret; - }(); - return result; + InitClientOptions(clientOptions); + auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( + PremiumFileConnectionString(), shareName, clientOptions); + m_resourceCleanupFunctions.push_back([shareClient]() { shareClient.DeleteIfExists(); }); + + return shareClient; } TEST_F(FileShareClientTest, CreateDeleteShares) { - { // Normal create/delete. - std::vector shareClients; - for (int32_t i = 0; i < 5; ++i) - { - auto client = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + std::to_string(i), m_options); - EXPECT_NO_THROW(client.Create()); - shareClients.emplace_back(std::move(client)); - } - for (const auto& client : shareClients) - { - EXPECT_NO_THROW(client.Delete()); - } + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + EXPECT_NO_THROW(shareClient.Create()); + EXPECT_NO_THROW(shareClient.Delete()); } { // CreateIfNotExists & DeleteIfExists. { - auto client = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "if", m_options); - EXPECT_NO_THROW(client.Create()); - EXPECT_NO_THROW(client.CreateIfNotExists()); - EXPECT_NO_THROW(client.Delete()); - EXPECT_NO_THROW(client.DeleteIfExists()); + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + EXPECT_NO_THROW(shareClient.Create()); + EXPECT_NO_THROW(shareClient.CreateIfNotExists()); + EXPECT_NO_THROW(shareClient.Delete()); + EXPECT_NO_THROW(shareClient.DeleteIfExists()); } { - auto client = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "if2", m_options); - EXPECT_NO_THROW(client.CreateIfNotExists()); - EXPECT_THROW(client.Create(), StorageException); - EXPECT_NO_THROW(client.DeleteIfExists()); + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + EXPECT_NO_THROW(shareClient.CreateIfNotExists()); + EXPECT_THROW(shareClient.Create(), StorageException); + EXPECT_NO_THROW(shareClient.DeleteIfExists()); } { - auto client = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "if3", m_options); - auto created = client.Create().Value.Created; + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + auto created = shareClient.Create().Value.Created; EXPECT_TRUE(created); - auto createResult = client.CreateIfNotExists(); + auto createResult = shareClient.CreateIfNotExists(); EXPECT_FALSE(createResult.Value.Created); EXPECT_FALSE(createResult.Value.ETag.HasValue()); EXPECT_EQ(DateTime(), createResult.Value.LastModified); - auto deleted = client.Delete().Value.Deleted; + auto deleted = shareClient.Delete().Value.Deleted; EXPECT_TRUE(deleted); } { - auto client = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "if4", m_options); - auto deleteResult = client.DeleteIfExists(); + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + auto deleteResult = shareClient.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } } @@ -126,8 +135,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareClientTest, ShareMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_shareClient->SetMetadata(metadata1)); @@ -141,10 +150,8 @@ namespace Azure { namespace Storage { namespace Test { { // Create share with metadata works - auto client1 = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "1", m_options); - auto client2 = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "2", m_options); + auto client1 = GetShareClientForTest(LowercaseRandomString()); + auto client2 = GetShareClientForTest(LowercaseRandomString()); Files::Shares::CreateShareOptions options1; Files::Shares::CreateShareOptions options2; options1.Metadata = metadata1; @@ -183,10 +190,8 @@ namespace Azure { namespace Storage { namespace Test { { // Create share with quota works - auto client1 = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "1", m_options); - auto client2 = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "2", m_options); + auto client1 = GetShareClientForTest(LowercaseRandomString()); + auto client2 = GetShareClientForTest(LowercaseRandomString()); Files::Shares::CreateShareOptions options1; Files::Shares::CreateShareOptions options2; options1.ShareQuotaInGiB = quota32GB; @@ -212,10 +217,8 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareClientTest, ShareAccessPolicy_LIVEONLY_) + TEST_F(FileShareClientTest, ShareAccessPolicy) { - CHECK_SKIP_TEST(); - std::vector identifiers; for (unsigned i = 0; i < 3; ++i) { @@ -231,13 +234,14 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(ret.Value.LastModified)); auto ret2 = m_shareClient->GetAccessPolicy(); - EXPECT_EQ(ret2.Value.SignedIdentifiers, identifiers); + if (m_testContext.IsLiveMode()) + { + EXPECT_EQ(ret2.Value.SignedIdentifiers, identifiers); + } } - TEST_F(FileShareClientTest, ShareAccessPolicyNullable_LIVEONLY_) + TEST_F(FileShareClientTest, ShareAccessPolicyNullable) { - CHECK_SKIP_TEST(); - std::vector identifiers; { Files::Shares::Models::SignedIdentifier identifier; @@ -264,7 +268,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(IsValidTime(ret.Value.LastModified)); auto ret2 = m_shareClient->GetAccessPolicy(); - EXPECT_EQ(ret2.Value.SignedIdentifiers, identifiers); + if (m_testContext.IsLiveMode()) + { + EXPECT_EQ(ret2.Value.SignedIdentifiers, identifiers); + } } TEST_F(FileShareClientTest, SharePermissions) @@ -282,10 +289,13 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(expectedPermission, ret2.Value); } - TEST_F(FileShareClientTest, Lease_LIVEONLY_) + TEST_F(FileShareClientTest, Lease) { + EXPECT_NE( + Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(), + Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); { - std::string leaseId1 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId1 = RandomUUID(); auto lastModified = m_shareClient->GetProperties().Value.LastModified; std::chrono::seconds leaseDuration(20); Files::Shares::ShareLeaseClient leaseClient(*m_shareClient, leaseId1); @@ -310,7 +320,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(rLease.LeaseId, leaseId1); lastModified = m_shareClient->GetProperties().Value.LastModified; - std::string leaseId2 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = RandomUUID(); EXPECT_NE(leaseId1, leaseId2); auto cLease = leaseClient.Change(leaseId2).Value; EXPECT_TRUE(cLease.ETag.HasValue()); @@ -325,8 +335,7 @@ namespace Azure { namespace Storage { namespace Test { } { - Files::Shares::ShareLeaseClient leaseClient( - *m_shareClient, Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); + Files::Shares::ShareLeaseClient leaseClient(*m_shareClient, RandomUUID()); auto aLease = leaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration).Value; auto properties = m_shareClient->GetProperties().Value; @@ -338,12 +347,12 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareClientTest, SnapshotLease_LIVEONLY_) + TEST_F(FileShareClientTest, SnapshotLease) { auto snapshotResult = m_shareClient->CreateSnapshot(); auto shareSnapshot = m_shareClient->WithSnapshot(snapshotResult.Value.Snapshot); { - std::string leaseId1 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId1 = RandomUUID(); auto lastModified = m_shareClient->GetProperties().Value.LastModified; std::chrono::seconds leaseDuration(20); Files::Shares::ShareLeaseClient shareSnapshotLeaseClient(shareSnapshot, leaseId1); @@ -370,7 +379,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(rLease.LeaseId, leaseId1); lastModified = shareSnapshot.GetProperties().Value.LastModified; - std::string leaseId2 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = RandomUUID(); EXPECT_NE(leaseId1, leaseId2); auto cLease = shareSnapshotLeaseClient.Change(leaseId2).Value; EXPECT_TRUE(cLease.ETag.HasValue()); @@ -385,8 +394,7 @@ namespace Azure { namespace Storage { namespace Test { } { - Files::Shares::ShareLeaseClient shareSnapshotLeaseClient( - shareSnapshot, Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); + Files::Shares::ShareLeaseClient shareSnapshotLeaseClient(shareSnapshot, RandomUUID()); auto aLease = shareSnapshotLeaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration) .Value; @@ -409,7 +417,7 @@ namespace Azure { namespace Storage { namespace Test { std::string baseName = "a b c !@#$%^&(,.;'[]{}`~) def" + non_ascii_word; { - std::string directoryName = baseName + m_testNameLowercase + "1"; + std::string directoryName = baseName + LowercaseRandomString() + "1"; auto directoryClient = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(directoryName); EXPECT_NO_THROW(directoryClient.Create()); @@ -418,7 +426,7 @@ namespace Azure { namespace Storage { namespace Test { directoryUrl, m_shareClient->GetUrl() + "/" + _internal::UrlEncodePath(directoryName)); } { - std::string fileName = baseName + m_testNameLowercase + "2"; + std::string fileName = baseName + LowercaseRandomString() + "2"; auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(fileName); EXPECT_NO_THROW(fileClient.Create(1024)); auto fileUrl = fileClient.GetUrl(); @@ -433,9 +441,8 @@ namespace Azure { namespace Storage { namespace Test { std::string prefix = "prefix"; Files::Shares::Models::ShareProperties properties; { - auto shareName = prefix + GetStringOfSize(5, true) + "1"; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), shareName, m_options); + auto shareName = prefix + LowercaseRandomString() + "1"; + auto shareClient = GetShareClientForTest(shareName); auto options = Files::Shares::CreateShareOptions(); options.AccessTier = Files::Shares::Models::AccessTier::TransactionOptimized; EXPECT_NO_THROW(shareClient.Create(options)); @@ -447,9 +454,8 @@ namespace Azure { namespace Storage { namespace Test { shareClients.emplace(std::move(shareName), std::move(shareClient)); } { - auto shareName = prefix + GetStringOfSize(5, true) + "2"; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), shareName, m_options); + auto shareName = prefix + LowercaseRandomString() + "2"; + auto shareClient = GetShareClientForTest(shareName); auto options = Files::Shares::CreateShareOptions(); options.AccessTier = Files::Shares::Models::AccessTier::Hot; EXPECT_NO_THROW(shareClient.Create(options)); @@ -460,9 +466,8 @@ namespace Azure { namespace Storage { namespace Test { shareClients.emplace(std::move(shareName), std::move(shareClient)); } { - auto shareName = prefix + GetStringOfSize(5, true) + "3"; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), shareName, m_options); + auto shareName = prefix + LowercaseRandomString() + "3"; + auto shareClient = GetShareClientForTest(shareName); auto options = Files::Shares::CreateShareOptions(); options.AccessTier = Files::Shares::Models::AccessTier::Cool; EXPECT_NO_THROW(shareClient.Create(options)); @@ -475,8 +480,7 @@ namespace Azure { namespace Storage { namespace Test { // Set properties works { - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "1", m_options); + auto shareClient = GetShareClientForTest(LowercaseRandomString()); auto options = Files::Shares::CreateShareOptions(); options.AccessTier = Files::Shares::Models::AccessTier::Cool; EXPECT_NO_THROW(shareClient.Create(options)); @@ -503,10 +507,7 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::ListSharesOptions listOptions; listOptions.Prefix = prefix; std::vector shareItems; - for (auto pageResult = Files::Shares::ShareServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_options) - .ListShares(listOptions); - pageResult.HasPage(); + for (auto pageResult = m_shareServiceClient->ListShares(listOptions); pageResult.HasPage(); pageResult.MoveToNextPage()) { shareItems.insert(shareItems.end(), pageResult.Shares.begin(), pageResult.Shares.end()); @@ -530,10 +531,12 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareClientTest, PremiumShare) { + auto shareClientOptions = InitClientOptions(); + auto shareServiceClient = Files::Shares::ShareServiceClient::CreateFromConnectionString( + PremiumFileConnectionString(), shareClientOptions); { - auto shareName = m_testNameLowercase; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - PremiumFileConnectionString(), shareName, m_options); + auto shareName = LowercaseRandomString(); + auto shareClient = GetPremiumShareClientForTest(shareName); // create works EXPECT_NO_THROW(shareClient.Create()); Files::Shares::Models::ShareProperties properties; @@ -547,10 +550,7 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::ListSharesOptions listOptions; listOptions.Prefix = shareName; std::vector shareItems; - for (auto pageResult = Files::Shares::ShareServiceClient::CreateFromConnectionString( - PremiumFileConnectionString(), m_options) - .ListShares(listOptions); - pageResult.HasPage(); + for (auto pageResult = shareServiceClient.ListShares(listOptions); pageResult.HasPage(); pageResult.MoveToNextPage()) { shareItems.insert(shareItems.end(), pageResult.Shares.begin(), pageResult.Shares.end()); @@ -580,9 +580,8 @@ namespace Azure { namespace Storage { namespace Test { } // nfs protocol works { - auto shareName = m_testNameLowercase + "1"; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - PremiumFileConnectionString(), shareName, m_options); + auto shareName = LowercaseRandomString(); + auto shareClient = GetPremiumShareClientForTest(shareName); // create works Files::Shares::CreateShareOptions options; options.EnabledProtocols = Files::Shares::Models::ShareProtocols::Nfs; @@ -597,10 +596,7 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::ListSharesOptions listOptions; listOptions.Prefix = shareName; std::vector shareItems; - for (auto pageResult = Files::Shares::ShareServiceClient::CreateFromConnectionString( - PremiumFileConnectionString(), m_options) - .ListShares(listOptions); - pageResult.HasPage(); + for (auto pageResult = shareServiceClient.ListShares(listOptions); pageResult.HasPage(); pageResult.MoveToNextPage()) { shareItems.insert(shareItems.end(), pageResult.Shares.begin(), pageResult.Shares.end()); @@ -619,9 +615,8 @@ namespace Azure { namespace Storage { namespace Test { } // smb protocol works { - auto shareName = m_testNameLowercase + "2"; - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - PremiumFileConnectionString(), shareName, m_options); + auto shareName = LowercaseRandomString(); + auto shareClient = GetPremiumShareClientForTest(shareName); // create works Files::Shares::CreateShareOptions options; options.EnabledProtocols = Files::Shares::Models::ShareProtocols::Smb; @@ -634,10 +629,7 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::ListSharesOptions listOptions; listOptions.Prefix = shareName; std::vector shareItems; - for (auto pageResult = Files::Shares::ShareServiceClient::CreateFromConnectionString( - PremiumFileConnectionString(), m_options) - .ListShares(listOptions); - pageResult.HasPage(); + for (auto pageResult = shareServiceClient.ListShares(listOptions); pageResult.HasPage(); pageResult.MoveToNextPage()) { shareItems.insert(shareItems.end(), pageResult.Shares.begin(), pageResult.Shares.end()); diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.hpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.hpp index 161512f610..c54d80b301 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.hpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.hpp @@ -1,24 +1,26 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -#include "test/ut/test_base.hpp" - #include +#include "share_service_client_test.hpp" + namespace Azure { namespace Storage { namespace Test { - class FileShareClientTest : public Azure::Storage::Test::StorageTest { + class FileShareClientTest : public FileShareServiceClientTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; - Files::Shares::Models::FileHttpHeaders GetInterestingHttpHeaders(); + Files::Shares::ShareClient GetShareClientForTest( + const std::string& shareName, + Files::Shares::ShareClientOptions clientOptions = Files::Shares::ShareClientOptions()); + Files::Shares::ShareClient GetPremiumShareClientForTest( + const std::string& shareName, + Files::Shares::ShareClientOptions clientOptions = Files::Shares::ShareClientOptions()); + protected: std::shared_ptr m_shareClient; std::string m_shareName; - Files::Shares::ShareClientOptions m_options; - std::string m_testName; - std::string m_testNameLowercase; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.cpp index 6e278e1d07..445a179c09 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.cpp @@ -11,100 +11,50 @@ namespace Azure { namespace Storage { namespace Test { void FileShareDirectoryClientTest::SetUp() { FileShareClientTest::SetUp(); - CHECK_SKIP_TEST(); - - m_directoryName = GetTestName() + "base"; + if (shouldSkipTest()) + { + return; + } + m_directoryName = RandomString(); m_fileShareDirectoryClient = std::make_shared( m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_directoryName)); m_fileShareDirectoryClient->Create(); } - void FileShareDirectoryClientTest::TearDown() - { - CHECK_SKIP_TEST(); - m_shareClient->DeleteIfExists(); - FileShareClientTest::TearDown(); - } - - std::pair< - std::vector, - std::vector> - FileShareDirectoryClientTest::ListAllFilesAndDirectories( - const std::string& directoryPath, - const std::string& prefix) - { - std::vector directoryResult; - std::vector fileResult; - Files::Shares::ListFilesAndDirectoriesOptions options; - if (!prefix.empty()) - { - options.Prefix = prefix; - } - auto directoryClient - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(directoryPath); - for (auto pageResult = directoryClient.ListFilesAndDirectories(options); pageResult.HasPage(); - pageResult.MoveToNextPage()) - { - directoryResult.insert( - directoryResult.end(), pageResult.Directories.begin(), pageResult.Directories.end()); - fileResult.insert(fileResult.end(), pageResult.Files.begin(), pageResult.Files.end()); - } - return std::make_pair< - std::vector, - std::vector>( - std::move(fileResult), std::move(directoryResult)); - } - TEST_F(FileShareDirectoryClientTest, CreateDeleteDirectories) { { // Normal create/delete. - std::vector directoryClients; - for (int32_t i = 0; i < 5; ++i) - { - Files::Shares::ShareDirectoryClient client - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient( - m_testName + std::to_string(i)); - EXPECT_NO_THROW(client.Create()); - directoryClients.emplace_back(std::move(client)); - } - for (const auto& client : directoryClients) - { - EXPECT_NO_THROW(client.Delete()); - } + Files::Shares::ShareDirectoryClient client + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); + EXPECT_NO_THROW(client.Create()); + EXPECT_NO_THROW(client.Delete()); } { // Create directory that already exist throws. - for (int32_t i = 0; i < 5; ++i) - { - Files::Shares::ShareDirectoryClient client - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient( - m_testName + "e" + std::to_string(i)); - EXPECT_NO_THROW(client.Create()); - EXPECT_THROW(client.Create(), StorageException); - } + Files::Shares::ShareDirectoryClient client + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); + EXPECT_NO_THROW(client.Create()); + EXPECT_THROW(client.Create(), StorageException); } { // CreateIfNotExists & DeleteIfExists. { - auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient( - m_testName + "CreateIfNotExists"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); EXPECT_NO_THROW(client.Create()); EXPECT_NO_THROW(client.CreateIfNotExists()); EXPECT_NO_THROW(client.Delete()); EXPECT_NO_THROW(client.DeleteIfExists()); } { - auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient( - m_testName + "StorageException"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); EXPECT_NO_THROW(client.CreateIfNotExists()); EXPECT_THROW(client.Create(), StorageException); EXPECT_NO_THROW(client.DeleteIfExists()); } { - auto client - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "delete"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); auto created = client.Create().Value.Created; EXPECT_TRUE(created); auto createResult = client.CreateIfNotExists(); @@ -115,23 +65,18 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(deleted); } { - auto client - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "deleted"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } { - auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "1", m_options); - auto client - = shareClient.GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "new"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } { - auto client = m_shareClient->GetRootDirectoryClient() - .GetSubdirectoryClient(m_testName + "s") - .GetSubdirectoryClient(m_testName + "s1"); + auto client = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); + client = client.GetSubdirectoryClient(RandomString()); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } @@ -140,18 +85,18 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareDirectoryClientTest, RenameFile) { - const std::string testName(GetTestName()); - const std::string baseDirectoryName = testName + "1"; + const std::string baseName = RandomString(); + const std::string baseDirectoryName = baseName + "1"; auto rootDirectoryClient = m_shareClient->GetRootDirectoryClient(); auto baseDirectoryClient = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); // base test { - const std::string oldFilename = testName + "2"; + const std::string oldFilename = baseName + "2"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string newFilename = testName + "3"; + const std::string newFilename = baseName + "3"; auto newFileClient = baseDirectoryClient.RenameFile(oldFilename, baseDirectoryName + "/" + newFilename) .Value; @@ -161,10 +106,10 @@ namespace Azure { namespace Storage { namespace Test { // overwrite { - const std::string oldFilename = testName + "4"; + const std::string oldFilename = baseName + "4"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string newFilename = testName + "5"; + const std::string newFilename = baseName + "5"; auto newFileClient = baseDirectoryClient.GetFileClient(newFilename); newFileClient.Create(512); EXPECT_THROW( @@ -183,10 +128,10 @@ namespace Azure { namespace Storage { namespace Test { } // overwrite readOnly { - const std::string oldFilename = testName + "6"; + const std::string oldFilename = baseName + "6"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string newFilename = testName + "7"; + const std::string newFilename = baseName + "7"; Files::Shares::CreateFileOptions createOptions; Files::Shares::Models::FileSmbProperties properties; properties.Attributes = Files::Shares::Models::FileAttributes::ReadOnly; @@ -216,12 +161,12 @@ namespace Azure { namespace Storage { namespace Test { "2127521184-1604012920-1887927527-513D:AI(A;;FA;;;SY)(A;;FA;;;BA)(A;;" "0x1200a9;;;S-1-5-21-397955417-626881126-188441444-3053964)"; - const std::string oldFilename = testName + "8"; + const std::string oldFilename = baseName + "8"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string newFilename = testName + "9"; + const std::string newFilename = baseName + "9"; Files::Shares::RenameFileOptions renameOptions; - renameOptions.Metadata = GetMetadata(); + renameOptions.Metadata = RandomMetadata(); renameOptions.FilePermission = permission; Files::Shares::Models::FileSmbProperties properties; properties.ChangedOn = std::chrono::system_clock::now(); @@ -242,17 +187,17 @@ namespace Azure { namespace Storage { namespace Test { // diff directory { - const std::string oldSubdirectoryName = testName + "10"; + const std::string oldSubdirectoryName = baseName + "10"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - const std::string oldFilename = testName + "11"; + const std::string oldFilename = baseName + "11"; auto oldFileClient = oldSubdirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string otherDirectoryname = testName + "12"; + const std::string otherDirectoryname = baseName + "12"; auto otherDirectoryClient = rootDirectoryClient.GetSubdirectoryClient(otherDirectoryname); otherDirectoryClient.Create(); - const std::string newFilename = testName + "13"; + const std::string newFilename = baseName + "13"; auto newFileClient = baseDirectoryClient .RenameFile( @@ -263,26 +208,26 @@ namespace Azure { namespace Storage { namespace Test { } // root directory { - const std::string oldFilename = testName + "14"; + const std::string oldFilename = baseName + "14"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string newFilename = testName + "15"; + const std::string newFilename = baseName + "15"; auto newFileClient = baseDirectoryClient.RenameFile(oldFilename, newFilename).Value; EXPECT_NO_THROW(newFileClient.GetProperties()); EXPECT_THROW(oldFileClient.GetProperties(), StorageException); } // lease { - const std::string oldFilename = testName + "16"; + const std::string oldFilename = baseName + "16"; auto oldFileClient = baseDirectoryClient.GetFileClient(oldFilename); oldFileClient.Create(512); - const std::string oldLeaseId = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + const std::string oldLeaseId = RandomUUID(); Files::Shares::ShareLeaseClient oldLeaseClient(oldFileClient, oldLeaseId); oldLeaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration); - const std::string newFilename = testName + "17"; + const std::string newFilename = baseName + "17"; auto newFileClient = baseDirectoryClient.GetFileClient(newFilename); newFileClient.Create(512); - const std::string newLeaseId = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + const std::string newLeaseId = RandomUUID(); Files::Shares::ShareLeaseClient newLeaseClient(newFileClient, newLeaseId); newLeaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration); @@ -303,19 +248,19 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareDirectoryClientTest, RenameSubdirectory) { - const std::string testName(GetTestName()); - const std::string baseDirectoryName = testName + "1"; + const std::string baseName = RandomString(); + const std::string baseDirectoryName = baseName + "1"; auto rootDirectoryClient = m_shareClient->GetRootDirectoryClient(); auto baseDirectoryClient = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(baseDirectoryName); baseDirectoryClient.Create(); // base test { - const std::string oldSubdirectoryName = testName + "2"; + const std::string oldSubdirectoryName = baseName + "2"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - oldSubdirectoryClient.GetFileClient(testName + "File1").Create(512); - const std::string newSubdirectoryName = testName + "3"; + oldSubdirectoryClient.GetFileClient(baseName + "File1").Create(512); + const std::string newSubdirectoryName = baseName + "3"; auto newSubdirectoryClient = baseDirectoryClient .RenameSubdirectory( @@ -327,10 +272,10 @@ namespace Azure { namespace Storage { namespace Test { // overwrite { - const std::string oldSubdirectoryName = testName + "4"; + const std::string oldSubdirectoryName = baseName + "4"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - const std::string existFilename = testName + "5"; + const std::string existFilename = baseName + "5"; auto existFileClient = baseDirectoryClient.GetFileClient(existFilename); existFileClient.Create(512); EXPECT_THROW( @@ -346,10 +291,10 @@ namespace Azure { namespace Storage { namespace Test { } // overwrite readOnly { - const std::string oldSubdirectoryName = testName + "6"; + const std::string oldSubdirectoryName = baseName + "6"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - const std::string existFilename = testName + "7"; + const std::string existFilename = baseName + "7"; Files::Shares::CreateFileOptions createOptions; Files::Shares::Models::FileSmbProperties properties; properties.Attributes = Files::Shares::Models::FileAttributes::ReadOnly; @@ -375,12 +320,12 @@ namespace Azure { namespace Storage { namespace Test { "2127521184-1604012920-1887927527-513D:AI(A;;FA;;;SY)(A;;FA;;;BA)(A;;" "0x1200a9;;;S-1-5-21-397955417-626881126-188441444-3053964)"; - const std::string oldSubdirectoryName = testName + "8"; + const std::string oldSubdirectoryName = baseName + "8"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - const std::string newSubdirectoryName = testName + "9"; + const std::string newSubdirectoryName = baseName + "9"; Files::Shares::RenameDirectoryOptions renameOptions; - renameOptions.Metadata = GetMetadata(); + renameOptions.Metadata = RandomMetadata(); renameOptions.FilePermission = permission; Files::Shares::Models::FileSmbProperties properties; properties.ChangedOn = std::chrono::system_clock::now(); @@ -401,19 +346,19 @@ namespace Azure { namespace Storage { namespace Test { // diff directory { - const std::string oldMiddleDirectoryName = testName + "10"; + const std::string oldMiddleDirectoryName = baseName + "10"; auto oldMiddleDirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldMiddleDirectoryName); oldMiddleDirectoryClient.Create(); - const std::string oldSubdirectoryName = testName + "11"; + const std::string oldSubdirectoryName = baseName + "11"; auto oldSubdirectoryClient = oldMiddleDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - const std::string otherDirectoryName = testName + "12"; + const std::string otherDirectoryName = baseName + "12"; auto otherDirectoryClient = rootDirectoryClient.GetSubdirectoryClient(otherDirectoryName); otherDirectoryClient.Create(); - const std::string newSubdirectoryName = testName + "13"; + const std::string newSubdirectoryName = baseName + "13"; auto newSubdirectoryClient = baseDirectoryClient .RenameSubdirectory( oldMiddleDirectoryName + "/" + oldSubdirectoryName, @@ -424,11 +369,11 @@ namespace Azure { namespace Storage { namespace Test { } // root directory { - const std::string oldSubdirectoryName = testName + "14"; + const std::string oldSubdirectoryName = baseName + "14"; auto oldSubdirectoryClient = baseDirectoryClient.GetSubdirectoryClient(oldSubdirectoryName); oldSubdirectoryClient.Create(); - oldSubdirectoryClient.GetFileClient(testName + "File1").Create(512); - const std::string newSubdirectoryName = testName + "15"; + oldSubdirectoryClient.GetFileClient(baseName + "File1").Create(512); + const std::string newSubdirectoryName = baseName + "15"; auto newSubdirectoryClient = baseDirectoryClient.RenameSubdirectory(oldSubdirectoryName, newSubdirectoryName).Value; EXPECT_NO_THROW(newSubdirectoryClient.GetProperties()); @@ -438,8 +383,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareDirectoryClientTest, DirectoryMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_fileShareDirectoryClient->SetMetadata(metadata1)); @@ -453,9 +398,9 @@ namespace Azure { namespace Storage { namespace Test { { // Create directory with metadata works auto client1 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "meta1"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "meta1"); auto client2 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "meta2"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "meta2"); Files::Shares::CreateDirectoryOptions options1; Files::Shares::CreateDirectoryOptions options2; options1.Metadata = metadata1; @@ -481,9 +426,9 @@ namespace Azure { namespace Storage { namespace Test { { // Create directory with permission/permission key works auto client1 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "1"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "1"); auto client2 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "2"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "2"); Files::Shares::CreateDirectoryOptions options1; Files::Shares::CreateDirectoryOptions options2; options1.DirectoryPermission = permission; @@ -496,7 +441,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(result1.Value(), result2.Value()); auto client3 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "3"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "3"); Files::Shares::CreateDirectoryOptions options3; options3.SmbProperties.PermissionKey = result1; EXPECT_NO_THROW(client3.Create(options3)); @@ -513,9 +458,9 @@ namespace Azure { namespace Storage { namespace Test { properties.LastWrittenOn = std::chrono::system_clock::now(); properties.PermissionKey = ""; auto client1 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "4"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "4"); auto client2 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "5"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "5"); EXPECT_NO_THROW(client1.Create()); EXPECT_NO_THROW(client2.Create()); @@ -530,7 +475,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(result1.Value(), result2.Value()); auto client3 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "6"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "6"); Files::Shares::CreateDirectoryOptions options3; options3.SmbProperties.PermissionKey = result1; std::string permissionKey; @@ -554,9 +499,9 @@ namespace Azure { namespace Storage { namespace Test { { // Create directory with SmbProperties works auto client1 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "1"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "1"); auto client2 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "2"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "2"); Files::Shares::CreateDirectoryOptions options1; Files::Shares::CreateDirectoryOptions options2; options1.SmbProperties = properties; @@ -583,9 +528,9 @@ namespace Azure { namespace Storage { namespace Test { { // SetProperties works auto client1 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "3"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "3"); auto client2 - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName + "4"); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "4"); EXPECT_NO_THROW(client1.Create()); EXPECT_NO_THROW(client2.Create()); @@ -611,7 +556,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareDirectoryClientTest, SmbPropertiesDefaultValue) { auto directoryClient - = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(m_testName); + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString()); directoryClient.Create(); auto smbProperties = directoryClient.GetProperties().Value.SmbProperties; EXPECT_EQ(smbProperties.Attributes, Files::Shares::Models::FileAttributes::Directory); @@ -641,6 +586,7 @@ namespace Azure { namespace Storage { namespace Test { std::vector directoryNameSetB; std::vector fileNameSetA; std::vector fileNameSetB; + std::string prefix = RandomString(); auto clientA = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(directoryNameA); clientA.Create(); auto clientB = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(directoryNameB); @@ -648,93 +594,115 @@ namespace Azure { namespace Storage { namespace Test { for (size_t i = 0; i < 5; ++i) { { - auto directoryName = m_testName + std::to_string(i); - auto fileName = "file" + std::to_string(i); + auto directoryName = RandomString() + std::to_string(i); + auto fileName = RandomString() + std::to_string(i); EXPECT_NO_THROW(clientA.GetSubdirectoryClient(directoryName).Create()); EXPECT_NO_THROW(clientA.GetFileClient(fileName).Create(1024)); directoryNameSetA.emplace_back(std::move(directoryName)); fileNameSetA.emplace_back(std::move(fileName)); } { - auto directoryName = m_testName + "B" + std::to_string(i); - auto fileName = "fileB" + std::to_string(i); + auto directoryName = RandomString() + std::to_string(i); + auto fileName = RandomString() + std::to_string(i); EXPECT_NO_THROW(clientB.GetSubdirectoryClient(directoryName).Create()); EXPECT_NO_THROW(clientB.GetFileClient(fileName).Create(1024)); directoryNameSetB.emplace_back(std::move(directoryName)); fileNameSetB.emplace_back(std::move(fileName)); } } + { + auto dirName = prefix + RandomString(); + auto filename = prefix + RandomString(); + clientB.GetSubdirectoryClient(dirName).Create(); + clientB.GetFileClient(filename).Create(1024); + directoryNameSetB.emplace_back(dirName); + fileNameSetB.emplace_back(filename); + } { // Normal root share list. - auto result = ListAllFilesAndDirectories(); - EXPECT_TRUE(result.first.empty()); - EXPECT_TRUE(result.second.size() >= 2); - auto iter = std::find_if( - result.second.begin(), - result.second.end(), - [&directoryNameA](const Files::Shares::Models::DirectoryItem& item) { - return item.Name == directoryNameA; - }); - EXPECT_EQ(iter->Name, directoryNameA); - EXPECT_NE(result.second.end(), iter); - iter = std::find_if( - result.second.begin(), - result.second.end(), - [&directoryNameB](const Files::Shares::Models::DirectoryItem& item) { - return item.Name == directoryNameB; - }); - EXPECT_EQ(iter->Name, directoryNameB); - EXPECT_NE(result.second.end(), iter); - } - { - // List with directory. - auto result = ListAllFilesAndDirectories(directoryNameA, std::string()); - for (const auto& name : directoryNameSetA) + std::set files; + std::set dirs; + for (auto page = m_shareClient->GetRootDirectoryClient().ListFilesAndDirectories(); + page.HasPage(); + page.MoveToNextPage()) { - auto iter = std::find_if( - result.second.begin(), - result.second.end(), - [&name](const Files::Shares::Models::DirectoryItem& item) { - return item.Name == name; - }); - EXPECT_EQ(iter->Name, name); - EXPECT_NE(result.second.end(), iter); + for (const auto& f : page.Files) + { + files.insert(f.Name); + } + for (const auto& d : page.Directories) + { + dirs.insert(d.Name); + } } - for (const auto& name : fileNameSetA) + EXPECT_TRUE(files.empty()); + EXPECT_FALSE(dirs.empty()); + EXPECT_NE(dirs.find(directoryNameA), dirs.end()); + EXPECT_NE(dirs.find(directoryNameB), dirs.end()); + } + { + std::set files; + std::set dirs; + for (auto page = clientA.ListFilesAndDirectories(); page.HasPage(); page.MoveToNextPage()) { - auto iter = std::find_if( - result.first.begin(), - result.first.end(), - [&name](const Files::Shares::Models::FileItem& item) { return item.Name == name; }); - EXPECT_EQ(iter->Name, name); - EXPECT_EQ(1024, iter->Details.FileSize); - EXPECT_NE(result.first.end(), iter); + for (const auto& f : page.Files) + { + files.insert(f.Name); + } + for (const auto& d : page.Directories) + { + dirs.insert(d.Name); + } } - for (const auto& name : directoryNameSetB) + for (const auto& d : directoryNameSetA) { - auto iter = std::find_if( - result.second.begin(), - result.second.end(), - [&name](const Files::Shares::Models::DirectoryItem& item) { - return item.Name == name; - }); - EXPECT_EQ(result.second.end(), iter); + EXPECT_NE(dirs.find(d), dirs.end()); } - for (const auto& name : fileNameSetB) + for (const auto& f : fileNameSetA) { - auto iter = std::find_if( - result.first.begin(), - result.first.end(), - [&name](const Files::Shares::Models::FileItem& item) { return item.Name == name; }); - EXPECT_EQ(result.first.end(), iter); + EXPECT_NE(files.find(f), files.end()); } } { // List with prefix. - auto result = ListAllFilesAndDirectories(std::string(), directoryNameA); - EXPECT_TRUE(result.first.empty()); - EXPECT_EQ(result.second.size(), size_t(1)); - EXPECT_EQ(directoryNameA, result.second[0].Name); + std::set files; + std::set dirs; + Files::Shares::ListFilesAndDirectoriesOptions listOptions; + listOptions.Prefix = prefix; + for (auto page = clientB.ListFilesAndDirectories(listOptions); page.HasPage(); + page.MoveToNextPage()) + { + for (const auto& f : page.Files) + { + files.insert(f.Name); + } + for (const auto& d : page.Directories) + { + dirs.insert(d.Name); + } + } + for (const auto& f : fileNameSetB) + { + if (f.substr(0, prefix.length()) == prefix) + { + EXPECT_NE(files.find(f), files.end()); + } + else + { + EXPECT_EQ(files.find(f), files.end()); + } + } + for (const auto& d : directoryNameSetB) + { + if (d.substr(0, prefix.length()) == prefix) + { + EXPECT_NE(dirs.find(d), dirs.end()); + } + else + { + EXPECT_EQ(dirs.find(d), dirs.end()); + } + } } { // List max result diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.hpp b/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.hpp index 2e7d7abd23..1ab9fdccbb 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.hpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_directory_client_test.hpp @@ -10,15 +10,7 @@ namespace Azure { namespace Storage { namespace Test { class FileShareDirectoryClientTest : public FileShareClientTest { protected: - void SetUp(); - void TearDown(); - - std::pair< - std::vector, - std::vector> - ListAllFilesAndDirectories( - const std::string& directoryPath = std::string(), - const std::string& prefix = std::string()); + void SetUp() override; std::shared_ptr m_fileShareDirectoryClient; std::string m_directoryName; diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp index 1e7cacd7ce..3a7366bb9d 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp @@ -18,76 +18,56 @@ namespace Azure { namespace Storage { namespace Test { void FileShareFileClientTest::SetUp() { FileShareDirectoryClientTest::SetUp(); - CHECK_SKIP_TEST(); - - m_fileName = m_testName + "basefile"; + if (shouldSkipTest()) + { + return; + } + m_fileName = RandomString(); m_fileClient = std::make_shared( m_fileShareDirectoryClient->GetFileClient(m_fileName)); m_fileClient->Create(1024); } - void FileShareFileClientTest::TearDown() - { - CHECK_SKIP_TEST(); - - Files::Shares::DeleteShareOptions options; - options.DeleteSnapshots = true; - m_shareClient->Delete(options); - - FileShareDirectoryClientTest::TearDown(); - } - TEST_F(FileShareFileClientTest, CreateDeleteFiles) { { // Normal create/delete. - std::vector fileClients; - for (int32_t i = 0; i < 5; ++i) - { - auto fileName = m_testName + std::to_string(i); - Files::Shares::ShareFileClient client = m_fileShareDirectoryClient->GetFileClient(fileName); - EXPECT_NO_THROW(client.Create(1024)); - fileClients.emplace_back(std::move(client)); - } - for (const auto& client : fileClients) - { - EXPECT_NO_THROW(client.Delete()); - } + auto fileName = RandomString(); + Files::Shares::ShareFileClient client = m_fileShareDirectoryClient->GetFileClient(fileName); + EXPECT_NO_THROW(client.Create(1024)); + EXPECT_NO_THROW(client.Delete()); } { // Create file that already exist overwrites. - for (int32_t i = 0; i < 5; ++i) - { - auto fileName = m_testName + "a" + std::to_string(i); - Files::Shares::ShareFileClient client = m_fileShareDirectoryClient->GetFileClient(fileName); - EXPECT_NO_THROW(client.Create(1024)); - EXPECT_NO_THROW(client.Create(1024)); - } + auto fileName = RandomString(); + Files::Shares::ShareFileClient client = m_fileShareDirectoryClient->GetFileClient(fileName); + EXPECT_NO_THROW(client.Create(1024)); + EXPECT_NO_THROW(client.Create(1024)); } { // DeleteIfExists. { - auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "1"); + auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "1"); EXPECT_NO_THROW(client.Create(1024)); EXPECT_NO_THROW(client.Delete()); EXPECT_NO_THROW(client.DeleteIfExists()); } { - auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "2"); + auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "2"); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } { auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( StandardStorageConnectionString(), LowercaseRandomString()); - auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "3"); + auto client = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "3"); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } { - auto client = m_shareClient->GetRootDirectoryClient() - .GetSubdirectoryClient(m_testName + "4") - .GetFileClient(m_testName + "5"); + auto subdirClient + = m_shareClient->GetRootDirectoryClient().GetSubdirectoryClient(RandomString() + "4"); + auto client = subdirClient.GetFileClient(RandomString() + "5"); auto deleteResult = client.DeleteIfExists(); EXPECT_FALSE(deleteResult.Value.Deleted); } @@ -96,13 +76,13 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, DownloadEmptyFile) { - auto fileClient = m_fileShareDirectoryClient->GetFileClient(m_testName); + auto fileClient = m_fileShareDirectoryClient->GetFileClient(RandomString()); fileClient.Create(0); auto res = fileClient.Download(); EXPECT_EQ(res.Value.BodyStream->Length(), 0); - std::string tempFilename = m_testName + "1"; + std::string tempFilename = RandomString() + "1"; EXPECT_NO_THROW(fileClient.DownloadTo(tempFilename)); EXPECT_TRUE(ReadFile(tempFilename).empty()); DeleteFile(tempFilename); @@ -113,17 +93,18 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, DownloadNonExistingToFile) { - const auto testName(GetTestName()); - auto fileClient = m_fileShareDirectoryClient->GetFileClient(m_testName); + const auto tempFilename = RandomString(); + auto fileClient = m_fileShareDirectoryClient->GetFileClient(RandomString()); - EXPECT_THROW(fileClient.DownloadTo(testName), StorageException); - EXPECT_THROW(ReadFile(testName), std::runtime_error); + EXPECT_THROW(fileClient.DownloadTo(tempFilename), StorageException); + EXPECT_THROW(ReadFile(tempFilename), std::runtime_error); + DeleteFile(tempFilename); } TEST_F(FileShareFileClientTest, FileMetadata) { - auto metadata1 = GetMetadata(); - auto metadata2 = GetMetadata(); + auto metadata1 = RandomMetadata(); + auto metadata2 = RandomMetadata(); { // Set/Get Metadata works EXPECT_NO_THROW(m_fileClient->SetMetadata(metadata1)); @@ -136,8 +117,8 @@ namespace Azure { namespace Storage { namespace Test { { // Create directory with metadata works - auto client1 = m_fileShareDirectoryClient->GetFileClient(m_testName + "1"); - auto client2 = m_fileShareDirectoryClient->GetFileClient(m_testName + "2"); + auto client1 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "1"); + auto client2 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "2"); Files::Shares::CreateFileOptions options1; Files::Shares::CreateFileOptions options2; options1.Metadata = metadata1; @@ -160,8 +141,8 @@ namespace Azure { namespace Storage { namespace Test { { // Create directory with permission/permission key works - auto client1 = m_fileShareDirectoryClient->GetFileClient(m_testName + "d1"); - auto client2 = m_fileShareDirectoryClient->GetFileClient(m_testName + "d2"); + auto client1 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "d1"); + auto client2 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "d2"); Files::Shares::CreateFileOptions options1; Files::Shares::CreateFileOptions options2; options1.Permission = permission; @@ -175,7 +156,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(result2.HasValue()); EXPECT_EQ(result1.Value(), result2.Value()); - auto client3 = m_fileShareDirectoryClient->GetFileClient(m_testName + "d3"); + auto client3 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "d3"); Files::Shares::CreateFileOptions options3; options3.SmbProperties.PermissionKey = result1; EXPECT_NO_THROW(client3.Create(1024, options3)); @@ -186,14 +167,21 @@ namespace Azure { namespace Storage { namespace Test { { // Set permission with SetProperties works + Files::Shares::Models::FileHttpHeaders httpHeaders; + httpHeaders.ContentType = "application/x-binary"; + httpHeaders.ContentLanguage = "en-US"; + httpHeaders.ContentDisposition = "attachment"; + httpHeaders.CacheControl = "no-cache"; + httpHeaders.ContentEncoding = "identity"; + Files::Shares::Models::FileSmbProperties properties; properties.Attributes = Files::Shares::Models::FileAttributes::System | Files::Shares::Models::FileAttributes::NotContentIndexed; properties.CreatedOn = std::chrono::system_clock::now(); properties.LastWrittenOn = std::chrono::system_clock::now(); properties.PermissionKey = ""; - auto client1 = m_fileShareDirectoryClient->GetFileClient(m_testName + "a1"); - auto client2 = m_fileShareDirectoryClient->GetFileClient(m_testName + "a2"); + auto client1 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "a1"); + auto client2 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "a2"); EXPECT_NO_THROW(client1.Create(1024)); EXPECT_NO_THROW(client2.Create(1024)); @@ -201,15 +189,15 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::SetFilePropertiesOptions options2; options1.Permission = permission; options2.Permission = permission; - EXPECT_NO_THROW(client1.SetProperties(GetInterestingHttpHeaders(), properties, options1)); - EXPECT_NO_THROW(client2.SetProperties(GetInterestingHttpHeaders(), properties, options2)); + EXPECT_NO_THROW(client1.SetProperties(httpHeaders, properties, options1)); + EXPECT_NO_THROW(client2.SetProperties(httpHeaders, properties, options2)); auto result1 = client1.GetProperties().Value.SmbProperties.PermissionKey; auto result2 = client1.GetProperties().Value.SmbProperties.PermissionKey; EXPECT_TRUE(result1.HasValue()); EXPECT_TRUE(result2.HasValue()); EXPECT_EQ(result1.Value(), result2.Value()); - auto client3 = m_fileShareDirectoryClient->GetFileClient(m_testName + "a3"); + auto client3 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "a3"); Files::Shares::CreateFileOptions options3; options3.SmbProperties.PermissionKey = result1; std::string permissionKey; @@ -232,8 +220,8 @@ namespace Azure { namespace Storage { namespace Test { properties.PermissionKey = m_fileClient->GetProperties().Value.SmbProperties.PermissionKey; { // Create directory with SmbProperties works - auto client1 = m_fileShareDirectoryClient->GetFileClient(m_testName + "1"); - auto client2 = m_fileShareDirectoryClient->GetFileClient(m_testName + "2"); + auto client1 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "1"); + auto client2 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "2"); Files::Shares::CreateFileOptions options1; Files::Shares::CreateFileOptions options2; options1.SmbProperties = properties; @@ -259,13 +247,20 @@ namespace Azure { namespace Storage { namespace Test { { // SetProperties works - auto client1 = m_fileShareDirectoryClient->GetFileClient(m_testName + "3"); - auto client2 = m_fileShareDirectoryClient->GetFileClient(m_testName + "4"); + Files::Shares::Models::FileHttpHeaders httpHeaders; + httpHeaders.ContentType = "application/x-binary"; + httpHeaders.ContentLanguage = "en-US"; + httpHeaders.ContentDisposition = "attachment"; + httpHeaders.CacheControl = "no-cache"; + httpHeaders.ContentEncoding = "identity"; + + auto client1 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "3"); + auto client2 = m_fileShareDirectoryClient->GetFileClient(RandomString() + "4"); EXPECT_NO_THROW(client1.Create(1024)); EXPECT_NO_THROW(client2.Create(1024)); - EXPECT_NO_THROW(client1.SetProperties(GetInterestingHttpHeaders(), properties)); - EXPECT_NO_THROW(client2.SetProperties(GetInterestingHttpHeaders(), properties)); + EXPECT_NO_THROW(client1.SetProperties(httpHeaders, properties)); + EXPECT_NO_THROW(client2.SetProperties(httpHeaders, properties)); auto directoryProperties1 = client1.GetProperties(); auto directoryProperties2 = client2.GetProperties(); EXPECT_EQ( @@ -285,7 +280,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, SmbPropertiesDefaultValue) { - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); fileClient.Create(1024); auto smbProperties = fileClient.GetProperties().Value.SmbProperties; EXPECT_EQ(smbProperties.Attributes, Files::Shares::Models::FileAttributes::Archive); @@ -326,10 +321,10 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareFileClientTest, LeaseRelated_LIVEONLY_) + TEST_F(FileShareFileClientTest, LeaseRelated) { { - std::string leaseId1 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId1 = RandomUUID(); auto lastModified = m_fileClient->GetProperties().Value.LastModified; Files::Shares::ShareLeaseClient leaseClient(*m_fileClient, leaseId1); auto aLease @@ -347,7 +342,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(properties.LeaseState.Value(), Files::Shares::Models::LeaseState::Leased); EXPECT_EQ(properties.LeaseStatus.Value(), Files::Shares::Models::LeaseStatus::Locked); - std::string leaseId2 = Files::Shares::ShareLeaseClient::CreateUniqueLeaseId(); + std::string leaseId2 = RandomUUID(); EXPECT_NE(leaseId1, leaseId2); lastModified = m_fileClient->GetProperties().Value.LastModified; auto cLease = leaseClient.Change(leaseId2).Value; @@ -364,8 +359,7 @@ namespace Azure { namespace Storage { namespace Test { { - Files::Shares::ShareLeaseClient leaseClient( - *m_fileClient, Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); + Files::Shares::ShareLeaseClient leaseClient(*m_fileClient, RandomUUID()); auto aLease = leaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration).Value; auto lastModified = m_fileClient->GetProperties().Value.LastModified; @@ -375,324 +369,92 @@ namespace Azure { namespace Storage { namespace Test { } } - namespace { - struct ShareConcurrentUploadParameter - { - int Concurrency; - int64_t FileSize; - }; - - class UploadShare : public FileShareFileClientTest, - public ::testing::WithParamInterface { - }; - - std::string GetUploadSuffix(const testing::TestParamInfo& info) - { - // Can't use empty spaces or underscores (_) as per google test documentation - // http://google.github.io/googletest/advanced.html#specifying-names-for-value-parameterized-test-parameters - auto const& p = info.param; - std::string suffix("c" + std::to_string(p.Concurrency) + "s" + std::to_string(p.FileSize)); - return suffix; - } - - std::vector GetUploadParameters() - { - std::vector testParametes; - for (int c : {1, 2, 5}) - { - for (int64_t l : {0ULL, 512ULL, 1_KB, 4_KB, 1_MB, 4_MB + 512}) - { - testParametes.emplace_back(ShareConcurrentUploadParameter({c, l})); - } - } - return testParametes; - } - } // namespace - - TEST_P(UploadShare, fromBuffer) + TEST_F(FileShareFileClientTest, ConcurrentUpload_LIVEONLY_) { - UploadShare::ParamType const& p(GetParam()); - auto fileClient = m_fileShareDirectoryClient->GetFileClient(m_testName); - std::vector fileContent(static_cast(p.FileSize), 'x'); - - Files::Shares::UploadFileFromOptions options; - options.TransferOptions.ChunkSize = 512_KB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = GetInterestingHttpHeaders(); - options.Metadata = GetMetadata(); - - auto res = fileClient.UploadFrom(fileContent.data(), static_cast(p.FileSize), options); - - auto properties = fileClient.GetProperties().Value; - EXPECT_EQ(properties.FileSize, p.FileSize); - EXPECT_EQ(properties.Metadata, options.Metadata); - std::vector downloadContent(static_cast(p.FileSize), '\x00'); - fileClient.DownloadTo(downloadContent.data(), static_cast(p.FileSize)); - EXPECT_EQ(downloadContent, fileContent); - } + const auto blobContent = RandomBuffer(static_cast(8_MB)); - TEST_P(UploadShare, fromFile) - { - - UploadShare::ParamType const& p(GetParam()); - auto fileClient = m_fileShareDirectoryClient->GetFileClient(m_testName); - std::vector fileContent = std::vector(static_cast(p.FileSize), 'x'); - - Files::Shares::UploadFileFromOptions options; - options.TransferOptions.ChunkSize = 512_KB; - options.TransferOptions.Concurrency = p.Concurrency; - options.HttpHeaders = GetInterestingHttpHeaders(); - options.Metadata = GetMetadata(); - - std::string tempFilename(m_testName); - WriteFile(tempFilename, fileContent); - - auto res = fileClient.UploadFrom(tempFilename, options); - - auto properties = fileClient.GetProperties().Value; - EXPECT_EQ(properties.FileSize, p.FileSize); - EXPECT_EQ(properties.Metadata, options.Metadata); - std::vector downloadContent(static_cast(p.FileSize), '\x00'); - fileClient.DownloadTo(downloadContent.data(), static_cast(p.FileSize)); - EXPECT_EQ( - downloadContent, - std::vector( - fileContent.begin(), fileContent.begin() + static_cast(p.FileSize))); - - DeleteFile(tempFilename); - } - - INSTANTIATE_TEST_SUITE_P( - withParam, - UploadShare, - testing::ValuesIn(GetUploadParameters()), - GetUploadSuffix); - - namespace { - struct ShareConcurrentDownloadParameter - { - int Concurrency; - int64_t DownloadSize; - Azure::Nullable Offset = {}; - Azure::Nullable Length = {}; - Azure::Nullable InitialChunkSize = {}; - Azure::Nullable ChunkSize = {}; - }; - - class DowloadShare : public FileShareFileClientTest, - public ::testing::WithParamInterface { - }; - -#if !defined(APPEND_IF_NOT_NULL) -#define APPEND_IF_NOT_NULL(value, suffix, destination) \ - if (value) \ - { \ - destination.append(suffix + std::to_string(value.Value())); \ - } -#endif - - std::string GetDownloadSuffix(const testing::TestParamInfo& info) - { - // Can't use empty spaces or underscores (_) as per google test documentation - // http://google.github.io/googletest/advanced.html#specifying-names-for-value-parameterized-test-parameters - auto const& p = info.param; - std::string suffix( - "c" + std::to_string(p.Concurrency) + "s" + std::to_string(p.DownloadSize)); - APPEND_IF_NOT_NULL(p.Offset, "o", suffix) - APPEND_IF_NOT_NULL(p.Length, "l", suffix) - APPEND_IF_NOT_NULL(p.InitialChunkSize, "ics", suffix) - APPEND_IF_NOT_NULL(p.ChunkSize, "cs", suffix) - return suffix; - } - - std::vector GetDownloadParameters(int64_t const fileSize) - { - std::vector testParametes; - for (int c : {1, 2, 4}) - { - // download whole file - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize})); - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize, 0})); - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize, 0, fileSize})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, 0, fileSize * 2})); - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize * 2})); - - // Do offset - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize, 0, 1})); - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize, 1, 1})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, fileSize - 1, 1})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, fileSize - 1, 2})); - testParametes.emplace_back(ShareConcurrentDownloadParameter({c, fileSize, fileSize, 1})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, fileSize + 1, 2})); - - // // initial chunk size - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, 0, 1024, 512, 1024})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, 0, 1024, 1024, 1024})); - testParametes.emplace_back( - ShareConcurrentDownloadParameter({c, fileSize, 0, 1024, 2048, 1024})); - } - return testParametes; - } - } // namespace - - TEST_P(DowloadShare, fromBuffer) - { - ShareConcurrentDownloadParameter const& p(GetParam()); - m_fileContent = std::vector(static_cast(8_MB), 'x'); - m_fileClient->UploadFrom(m_fileContent.data(), m_fileContent.size()); - - std::vector downloadBuffer; - std::vector expectedData = m_fileContent; - int64_t fileSize = m_fileContent.size(); - int64_t actualDownloadSize = std::min(p.DownloadSize, fileSize); - auto length = p.Length; - auto chunkSize = p.ChunkSize; - auto concurrency = p.Concurrency; - auto initialChunkSize = p.InitialChunkSize; - if (p.Offset.HasValue() && length.HasValue()) - { - actualDownloadSize = std::min(length.Value(), fileSize - p.Offset.Value()); - if (actualDownloadSize >= 0) - { - expectedData.assign( - m_fileContent.begin() + static_cast(p.Offset.Value()), - m_fileContent.begin() + static_cast(p.Offset.Value() + actualDownloadSize)); - } - else - { - expectedData.clear(); - } - } - else if (p.Offset.HasValue()) - { - actualDownloadSize = fileSize - p.Offset.Value(); - if (actualDownloadSize >= 0) + auto testUploadFromBuffer = [&](int concurrency, + int64_t bufferSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Files::Shares::UploadFileFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) { - expectedData.assign( - m_fileContent.begin() + static_cast(p.Offset.Value()), m_fileContent.end()); + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); } - else + if (chunkSize.HasValue()) { - expectedData.clear(); + options.TransferOptions.ChunkSize = chunkSize.Value(); } - } - downloadBuffer.resize(static_cast(p.DownloadSize), '\x00'); - Files::Shares::DownloadFileToOptions options; - options.TransferOptions.Concurrency = concurrency; - if (p.Offset.HasValue()) - { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = p.Offset.Value(); - options.Range.Value().Length = length; - } - if (initialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = initialChunkSize.Value(); - } - if (chunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = chunkSize.Value(); - } - if (actualDownloadSize > 0) - { - auto res = m_fileClient->DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - downloadBuffer.resize(static_cast(res.Value.ContentRange.Length.Value())); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); + EXPECT_NO_THROW( + fileClient.UploadFrom(blobContent.data(), static_cast(bufferSize), options)); + std::vector downloadBuffer(static_cast(bufferSize), '\x00'); + fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(bufferSize)); EXPECT_EQ(downloadBuffer, expectedData); - } - else - { - EXPECT_THROW( - m_fileClient->DownloadTo(downloadBuffer.data(), downloadBuffer.size(), options), - StorageException); - } - } + }; - TEST_P(DowloadShare, fromFile) - { - ShareConcurrentDownloadParameter const& p(GetParam()); - m_fileContent = std::vector(static_cast(8_MB), 'x'); - m_fileClient->UploadFrom(m_fileContent.data(), m_fileContent.size()); - - std::string tempFilename = RandomString(); - std::vector expectedData = m_fileContent; - int64_t fileSize = m_fileContent.size(); - int64_t actualDownloadSize = std::min(p.DownloadSize, fileSize); - if (p.Offset.HasValue() && p.Length.HasValue()) - { - actualDownloadSize = std::min(p.Length.Value(), fileSize - p.Offset.Value()); - if (actualDownloadSize >= 0) + auto testUploadFromFile = [&](int concurrency, + int64_t fileSize, + Azure::Nullable singleUploadThreshold = {}, + Azure::Nullable chunkSize = {}) { + Files::Shares::UploadFileFromOptions options; + options.TransferOptions.Concurrency = concurrency; + if (singleUploadThreshold.HasValue()) { - expectedData.assign( - m_fileContent.begin() + static_cast(p.Offset.Value()), - m_fileContent.begin() + static_cast(p.Offset.Value() + actualDownloadSize)); + options.TransferOptions.SingleUploadThreshold = singleUploadThreshold.Value(); } - else + if (chunkSize.HasValue()) { - expectedData.clear(); + options.TransferOptions.ChunkSize = chunkSize.Value(); } - } - else if (p.Offset.HasValue()) + + const std::string tempFileName = RandomString(); + WriteFile( + tempFileName, + std::vector( + blobContent.begin(), blobContent.begin() + static_cast(fileSize))); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); + EXPECT_NO_THROW(fileClient.UploadFrom(tempFileName, options)); + DeleteFile(tempFileName); + std::vector downloadBuffer(static_cast(fileSize), '\x00'); + fileClient.DownloadTo(downloadBuffer.data(), downloadBuffer.size()); + std::vector expectedData( + blobContent.begin(), blobContent.begin() + static_cast(fileSize)); + EXPECT_EQ(downloadBuffer, expectedData); + }; + + for (int c : {1, 2, 4}) { - actualDownloadSize = fileSize - p.Offset.Value(); - if (actualDownloadSize >= 0) + std::vector> futures; + // random range + for (int i = 0; i < 16; ++i) { - expectedData.assign( - m_fileContent.begin() + static_cast(p.Offset.Value()), m_fileContent.end()); + int64_t fileSize = RandomInt(1, 1_MB); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + futures.emplace_back( + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); } - else + for (auto& f : futures) { - expectedData.clear(); + f.get(); } } - Files::Shares::DownloadFileToOptions options; - options.TransferOptions.Concurrency = p.Concurrency; - if (p.Offset.HasValue()) - { - options.Range = Core::Http::HttpRange(); - options.Range.Value().Offset = p.Offset.Value(); - options.Range.Value().Length = p.Length; - } - if (p.InitialChunkSize.HasValue()) - { - options.TransferOptions.InitialChunkSize = p.InitialChunkSize.Value(); - } - if (p.ChunkSize.HasValue()) - { - options.TransferOptions.ChunkSize = p.ChunkSize.Value(); - } - if (actualDownloadSize > 0) - { - auto res = m_fileClient->DownloadTo(tempFilename, options); - EXPECT_EQ(res.Value.ContentRange.Length.Value(), actualDownloadSize); - EXPECT_EQ(ReadFile(tempFilename), expectedData); - } - else - { - EXPECT_THROW(m_fileClient->DownloadTo(tempFilename, options), StorageException); - } - DeleteFile(tempFilename); } - INSTANTIATE_TEST_SUITE_P( - withParam, - DowloadShare, - testing::ValuesIn(GetDownloadParameters(8_MB)), - GetDownloadSuffix); - TEST_F(FileShareFileClientTest, ConcurrentDownload_LIVEONLY_) { - CHECK_SKIP_TEST(); - - m_fileContent = RandomBuffer(8 * 1024 * 1024); - m_fileClient->UploadFrom(m_fileContent.data(), 8 * 1024 * 1024); + auto fileContent = RandomBuffer(8 * 1024 * 1024); + m_fileClient->UploadFrom(fileContent.data(), 8 * 1024 * 1024); auto testDownloadToBuffer = [&](int concurrency, int64_t downloadSize, Azure::Nullable offset = {}, @@ -700,8 +462,8 @@ namespace Azure { namespace Storage { namespace Test { Azure::Nullable initialChunkSize = {}, Azure::Nullable chunkSize = {}) { std::vector downloadBuffer; - std::vector expectedData = m_fileContent; - int64_t fileSize = m_fileContent.size(); + std::vector expectedData = fileContent; + int64_t fileSize = fileContent.size(); int64_t actualDownloadSize = std::min(downloadSize, fileSize); if (offset.HasValue() && length.HasValue()) { @@ -709,8 +471,8 @@ namespace Azure { namespace Storage { namespace Test { if (actualDownloadSize >= 0) { expectedData.assign( - m_fileContent.begin() + static_cast(offset.Value()), - m_fileContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + fileContent.begin() + static_cast(offset.Value()), + fileContent.begin() + static_cast(offset.Value() + actualDownloadSize)); } else { @@ -723,7 +485,7 @@ namespace Azure { namespace Storage { namespace Test { if (actualDownloadSize >= 0) { expectedData.assign( - m_fileContent.begin() + static_cast(offset.Value()), m_fileContent.end()); + fileContent.begin() + static_cast(offset.Value()), fileContent.end()); } else { @@ -769,8 +531,8 @@ namespace Azure { namespace Storage { namespace Test { Azure::Nullable initialChunkSize = {}, Azure::Nullable chunkSize = {}) { std::string tempFilename = RandomString(); - std::vector expectedData = m_fileContent; - int64_t fileSize = m_fileContent.size(); + std::vector expectedData = fileContent; + int64_t fileSize = fileContent.size(); int64_t actualDownloadSize = std::min(downloadSize, fileSize); if (offset.HasValue() && length.HasValue()) { @@ -778,8 +540,8 @@ namespace Azure { namespace Storage { namespace Test { if (actualDownloadSize >= 0) { expectedData.assign( - m_fileContent.begin() + static_cast(offset.Value()), - m_fileContent.begin() + static_cast(offset.Value() + actualDownloadSize)); + fileContent.begin() + static_cast(offset.Value()), + fileContent.begin() + static_cast(offset.Value() + actualDownloadSize)); } else { @@ -792,7 +554,7 @@ namespace Azure { namespace Storage { namespace Test { if (actualDownloadSize >= 0) { expectedData.assign( - m_fileContent.begin() + static_cast(offset.Value()), m_fileContent.end()); + fileContent.begin() + static_cast(offset.Value()), fileContent.end()); } else { @@ -828,7 +590,7 @@ namespace Azure { namespace Storage { namespace Test { DeleteFile(tempFilename); }; - const int64_t fileSize = m_fileContent.size(); + const int64_t fileSize = fileContent.size(); std::vector> futures; for (int c : {1, 2, 4}) { @@ -836,7 +598,7 @@ namespace Azure { namespace Storage { namespace Test { std::mt19937_64 random_generator(std::random_device{}()); for (int i = 0; i < 16; ++i) { - std::uniform_int_distribution offsetDistribution(0, m_fileContent.size() - 1); + std::uniform_int_distribution offsetDistribution(0, fileContent.size() - 1); int64_t offset = offsetDistribution(random_generator); std::uniform_int_distribution lengthDistribution(1, 64_KB); int64_t length = lengthDistribution(random_generator); @@ -870,13 +632,13 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, RangeUploadDownload) { - auto rangeSize = 1 * 1024 * 1024; + auto rangeSize = 128; auto numOfChunks = 3; - std::vector rangeContent(rangeSize, 'x'); + std::vector rangeContent = RandomBuffer(rangeSize); auto memBodyStream = Core::IO::MemoryBodyStream(rangeContent); { // Simple upload/download. - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); fileClient.Create(static_cast(numOfChunks) * rangeSize); for (int32_t i = 0; i < numOfChunks; ++i) { @@ -902,7 +664,7 @@ namespace Azure { namespace Storage { namespace Test { // last write time { memBodyStream.Rewind(); - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); fileClient.Create(static_cast(numOfChunks) * rangeSize); auto lastWriteTimeBeforeUpload = fileClient.GetProperties().Value.SmbProperties.LastWrittenOn.Value(); @@ -916,7 +678,7 @@ namespace Azure { namespace Storage { namespace Test { } { memBodyStream.Rewind(); - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); fileClient.Create(static_cast(numOfChunks) * rangeSize); auto lastWriteTimeBeforeUpload = fileClient.GetProperties().Value.SmbProperties.LastWrittenOn.Value(); @@ -933,7 +695,7 @@ namespace Azure { namespace Storage { namespace Test { memBodyStream.Rewind(); Azure::Core::Cryptography::Md5Hash instance; auto md5 = instance.Final(rangeContent.data(), rangeContent.size()); - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "2"); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "2"); Files::Shares::UploadFileRangeOptions uploadOptions; fileClient.Create(static_cast(numOfChunks) * rangeSize); ContentHash hash; @@ -950,15 +712,16 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, CopyRelated) { - size_t fileSize = 1 * 1024 * 1024; - std::vector fileContent(fileSize, 'x'); + size_t fileSize = 128; + std::vector fileContent = RandomBuffer(fileSize); auto memBodyStream = Core::IO::MemoryBodyStream(fileContent); { // Simple copy works. - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "1"); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "1"); fileClient.Create(fileSize); - auto destFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "2"); + auto destFileClient + = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "2"); auto copyOperation = destFileClient.StartCopy(fileClient.GetUrl()); EXPECT_EQ( copyOperation.GetRawResponse().GetStatusCode(), @@ -969,22 +732,17 @@ namespace Azure { namespace Storage { namespace Test { { // Copy mode with override and empty permission throws error.. - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "3"); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "3"); fileClient.Create(fileSize); - auto destFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "4"); + auto destFileClient + = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "4"); } } - TEST_F(FileShareFileClientTest, RangeRelated_LIVEONLY_) + TEST_F(FileShareFileClientTest, RangeRelated) { - // Test uploads a file with some content. - // Then the content is updated in the server (second half is cleared) - // Then all file is downloaded and we checked the downloaded content - // Test is LIVE only as there's no support for this behavior from the test recorded. - CHECK_SKIP_TEST(); - - size_t fileSize = 1 * 1024 * 1024; + size_t fileSize = 1024 * 3; auto fileContent = RandomBuffer(fileSize); auto memBodyStream = Core::IO::MemoryBodyStream(fileContent); auto halfContent @@ -1010,21 +768,15 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(static_cast(fileSize / 2) - 1024, result.Ranges[1].Length.Value()); } - TEST_F(FileShareFileClientTest, PreviousRangeWithSnapshot_LIVEONLY_) + TEST_F(FileShareFileClientTest, PreviousRangeWithSnapshot) { - // Test uploads a file with some content. - // Then the content is updated in the server (second half is cleared) - // Then all file is downloaded and we checked the downloaded content - // Test is LIVE only as there's no support for this behavior from the test recorded. - CHECK_SKIP_TEST(); - - size_t fileSize = 1 * 1024 * 1024; - std::vector fileContent(fileSize, 'x'); + size_t fileSize = 1024 * 10; + std::vector fileContent = RandomBuffer(fileSize); auto memBodyStream = Core::IO::MemoryBodyStream(fileContent); auto halfContent = std::vector(fileContent.begin(), fileContent.begin() + fileSize / 2); halfContent.resize(fileSize); - auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName); + auto fileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString()); fileClient.Create(fileSize); EXPECT_NO_THROW(fileClient.UploadRange(0, memBodyStream)); EXPECT_NO_THROW(fileClient.ClearRange(fileSize / 2, fileSize / 2)); @@ -1094,7 +846,7 @@ namespace Azure { namespace Storage { namespace Test { }; options.PerOperationPolicies.emplace_back(std::make_unique()); auto fileClient = Azure::Storage::Files::Shares::ShareFileClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_shareName, m_testName, options); + StandardStorageConnectionString(), m_shareName, RandomString(), options); try { fileClient.Create(1024); @@ -1115,15 +867,16 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, UploadRangeFromUri) { - size_t fileSize = 1 * 1024 * 1024; - std::string fileName = m_testName + "file"; - std::vector fileContent(fileSize, 'x'); + size_t fileSize = 1 * 1024; + std::string fileName = RandomString() + "file"; + std::vector fileContent = RandomBuffer(fileSize); auto memBodyStream = Core::IO::MemoryBodyStream(fileContent); auto sourceFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(fileName); sourceFileClient.Create(fileSize); EXPECT_NO_THROW(sourceFileClient.UploadRange(0, memBodyStream)); - auto destFileClient = m_shareClient->GetRootDirectoryClient().GetFileClient(m_testName + "f2"); + auto destFileClient + = m_shareClient->GetRootDirectoryClient().GetFileClient(RandomString() + "f2"); destFileClient.Create(fileSize * 4); Azure::Core::Http::HttpRange sourceRange; Azure::Core::Http::HttpRange destRange; @@ -1201,10 +954,6 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(lastWriteTimeBeforeUpload, lastWriteTimeAfterUpload); } // source access condition works. - std::vector invalidCrc64( - uploadResult.TransactionalContentHash.Value.begin(), - uploadResult.TransactionalContentHash.Value.begin() - + uploadResult.TransactionalContentHash.Value.size() / 2); { Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; uploadRangeOptions.SourceAccessCondition.IfNoneMatchContentHash @@ -1218,13 +967,17 @@ namespace Azure { namespace Storage { namespace Test { uploadRangeOptions) .Value, StorageException); - // Below code seems to be triggering a server bug. Uncomment when server resolves the issue. - // uploadRangeOptions.SourceAccessCondition.IfNoneMatchContentHash.Value().Value - // = invalidCrc64; - // EXPECT_NO_THROW( - // uploadResult = *destFileClient.UploadRangeFromUri( - // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, - // uploadRangeOptions)); + uploadRangeOptions.SourceAccessCondition.IfNoneMatchContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + + EXPECT_NO_THROW( + uploadResult = destFileClient + .UploadRangeFromUri( + destRange.Offset, + sourceFileClient.GetUrl() + sourceSas, + sourceRange, + uploadRangeOptions) + .Value); } { Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; @@ -1238,14 +991,19 @@ namespace Azure { namespace Storage { namespace Test { sourceRange, uploadRangeOptions) .Value); - // Below code seems to be triggering a server high latency. Uncomment when server resolves - // the issue. uploadRangeOptions.SourceAccessCondition.IfMatchContentHash.Value().Value = - // invalidCrc64; - // EXPECT_THROW( - // uploadResult = *destFileClient.UploadRangeFromUri( - // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, - // uploadRangeOptions), - // StorageException); + + uploadRangeOptions.SourceAccessCondition.IfMatchContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + EXPECT_THROW( + uploadResult = destFileClient + .UploadRangeFromUri( + destRange.Offset, + sourceFileClient.GetUrl() + sourceSas, + sourceRange, + + uploadRangeOptions) + .Value, + StorageException); } { Files::Shares::UploadFileRangeFromUriOptions uploadRangeOptions; @@ -1258,13 +1016,17 @@ namespace Azure { namespace Storage { namespace Test { sourceRange, uploadRangeOptions) .Value); - // Below code seems to be triggering a server high latency. Uncomment when server resolves - // the issue. uploadRangeOptions.SourceContentHash.Value().Value = invalidCrc64; - // EXPECT_THROW( - // uploadResult = *destFileClient.UploadRangeFromUri( - // sourceFileClient.GetUrl() + sourceSas, sourceRange, destRange, - // uploadRangeOptions), - // StorageException); + uploadRangeOptions.TransactionalContentHash.Value().Value + = Azure::Core::Convert::Base64Decode(DummyCrc64); + EXPECT_THROW( + uploadResult = destFileClient + .UploadRangeFromUri( + destRange.Offset, + sourceFileClient.GetUrl() + sourceSas, + sourceRange, + uploadRangeOptions) + .Value, + StorageException); } } diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.hpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.hpp index 19ad857645..9c0451cb4b 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.hpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.hpp @@ -11,11 +11,9 @@ namespace Azure { namespace Storage { namespace Test { class FileShareFileClientTest : public FileShareDirectoryClientTest { protected: void SetUp(); - void TearDown(); std::shared_ptr m_fileClient; std::string m_fileName; - std::vector m_fileContent; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp index 8d12c487ce..6ba1eec5f6 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp @@ -16,126 +16,94 @@ namespace Azure { namespace Storage { namespace Test { } } // namespace - const size_t ShareTestSize = 5; - void FileShareServiceClientTest::SetUp() { StorageTest::SetUp(); - CHECK_SKIP_TEST(); - m_options = InitClientOptions(); - m_fileShareServiceClient = std::make_shared( + auto options = InitClientOptions(); + m_shareServiceClient = std::make_shared( Files::Shares::ShareServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_options)); - m_testName = GetTestName(); - m_testNameLowercase = GetTestNameLowerCase(); - m_sharePrefixA = m_testNameLowercase + "a"; - m_sharePrefixB = m_testNameLowercase + "b"; - m_shareNameSetA.clear(); - m_shareNameSetB.clear(); - for (size_t i = 0; i < ShareTestSize; ++i) - { - { - auto name = m_sharePrefixA + "a" + std::to_string(i); - m_fileShareServiceClient->GetShareClient(name).Create(); - m_shareNameSetA.emplace_back(std::move(name)); - } - { - auto name = m_sharePrefixB + "b" + std::to_string(i); - m_fileShareServiceClient->GetShareClient(name).Create(); - m_shareNameSetB.emplace_back(std::move(name)); - } - } + StandardStorageConnectionString(), options)); } - void FileShareServiceClientTest::TearDown() - { - CHECK_SKIP_TEST(); - for (const auto& name : m_shareNameSetA) - { - m_fileShareServiceClient->GetShareClient(name).Delete(); - } - for (const auto& name : m_shareNameSetB) - { - m_fileShareServiceClient->GetShareClient(name).Delete(); - } - StorageTest::TearDown(); - } - - std::vector FileShareServiceClientTest::ListAllShares( - const std::string& prefix) + TEST_F(FileShareServiceClientTest, ListShares) { - std::vector result; - Files::Shares::ListSharesOptions options; - if (!prefix.empty()) + std::string prefix1 = LowercaseRandomString(); + std::string prefix2 = LowercaseRandomString(); + std::set shareSet1; + std::set shareSet2; + for (int i = 0; i < 5; ++i) { - options.Prefix = prefix; + auto shareName = prefix1 + LowercaseRandomString(); + auto shareClient = m_shareServiceClient->GetShareClient(shareName); + shareClient.Create(); + shareSet1.emplace(shareName); + shareName = prefix2 + LowercaseRandomString(); + shareClient = m_shareServiceClient->GetShareClient(shareName); + shareClient.Create(); + shareSet2.emplace(shareName); } - for (auto pageResult = m_fileShareServiceClient->ListShares(options); pageResult.HasPage(); - pageResult.MoveToNextPage()) - { - result.insert(result.end(), pageResult.Shares.begin(), pageResult.Shares.end()); - } - return result; - } - - TEST_F(FileShareServiceClientTest, ListShares) - { { // Normal list without prefix. - auto result = ListAllShares(); - for (const auto& name : m_shareNameSetA) + std::set result; + for (auto page = m_shareServiceClient->ListShares(); page.HasPage(); page.MoveToNextPage()) + { + for (const auto& share : page.Shares) + { + result.insert(share.Name); + } + } + for (const auto& name : shareSet1) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::Shares::Models::ShareItem& share) { - return share.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_sharePrefixA.size()), m_sharePrefixA); - EXPECT_NE(result.end(), iter); + EXPECT_NE(result.find(name), result.end()); } - for (const auto& name : m_shareNameSetB) + for (const auto& name : shareSet2) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::Shares::Models::ShareItem& share) { - return share.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_sharePrefixB.size()), m_sharePrefixB); - EXPECT_NE(result.end(), iter); + EXPECT_NE(result.find(name), result.end()); } } { // List prefix. - auto result = ListAllShares(m_sharePrefixA); - for (const auto& name : m_shareNameSetA) + std::set result; + Files::Shares::ListSharesOptions options; + options.Prefix = prefix1; + for (auto page = m_shareServiceClient->ListShares(options); page.HasPage(); + page.MoveToNextPage()) + { + for (const auto& share : page.Shares) + { + result.insert(share.Name); + } + } + for (const auto& name : shareSet1) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::Shares::Models::ShareItem& share) { - return share.Name == name; - }); - EXPECT_EQ(iter->Name.substr(0U, m_sharePrefixA.size()), m_sharePrefixA); - EXPECT_NE(result.end(), iter); + EXPECT_NE(result.find(name), result.end()); } - for (const auto& name : m_shareNameSetB) + for (const auto& name : shareSet2) { - auto iter = std::find_if( - result.begin(), result.end(), [&name](const Files::Shares::Models::ShareItem& share) { - return share.Name == name; - }); - EXPECT_EQ(result.end(), iter); + EXPECT_EQ(result.find(name), result.end()); } } { // List max result Files::Shares::ListSharesOptions options; options.PageSizeHint = 2; - auto response = m_fileShareServiceClient->ListShares(options); + auto response = m_shareServiceClient->ListShares(options); EXPECT_LE(2U, response.Shares.size()); } + for (const auto& shareName : shareSet1) + { + m_shareServiceClient->GetShareClient(shareName).DeleteIfExists(); + } + for (const auto& shareName : shareSet2) + { + m_shareServiceClient->GetShareClient(shareName).DeleteIfExists(); + } } TEST_F(FileShareServiceClientTest, GetProperties) { - auto ret = m_fileShareServiceClient->GetProperties(); + auto ret = m_shareServiceClient->GetProperties(); auto properties = ret.Value; auto hourMetrics = properties.HourMetrics; if (hourMetrics.Enabled) @@ -151,7 +119,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareServiceClientTest, SetProperties) { - auto properties = m_fileShareServiceClient->GetProperties().Value; + auto properties = m_shareServiceClient->GetProperties().Value; auto originalProperties = properties; properties.HourMetrics.Enabled = true; @@ -179,11 +147,11 @@ namespace Azure { namespace Storage { namespace Test { corsRule.MaxAgeInSeconds = 20; properties.Cors.emplace_back(corsRule); - EXPECT_NO_THROW(m_fileShareServiceClient->SetProperties(properties)); + EXPECT_NO_THROW(m_shareServiceClient->SetProperties(properties)); // It takes some time before the new properties comes into effect. using namespace std::chrono_literals; TestSleep(10s); - auto downloadedProperties = m_fileShareServiceClient->GetProperties().Value; + auto downloadedProperties = m_shareServiceClient->GetProperties().Value; EXPECT_EQ(downloadedProperties.HourMetrics.Version, properties.HourMetrics.Version); EXPECT_EQ(downloadedProperties.HourMetrics.Enabled, properties.HourMetrics.Enabled); @@ -235,7 +203,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(properties.Cors.end(), iter); } - m_fileShareServiceClient->SetProperties(originalProperties); + m_shareServiceClient->SetProperties(originalProperties); } TEST_F(FileShareServiceClientTest, DISABLED_SetPremiumFileProperties) diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.hpp b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.hpp index d11ff7a8ad..ab042aabb1 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.hpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.hpp @@ -9,20 +9,10 @@ namespace Azure { namespace Storage { namespace Test { class FileShareServiceClientTest : public Azure::Storage::Test::StorageTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; - std::vector ListAllShares( - const std::string& prefix = std::string()); - - std::shared_ptr m_fileShareServiceClient; - std::vector m_shareNameSetA; - std::string m_sharePrefixA; - std::vector m_shareNameSetB; - std::string m_sharePrefixB; - Files::Shares::ShareClientOptions m_options; - std::string m_testName; - std::string m_testNameLowercase; + protected: + std::shared_ptr m_shareServiceClient; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_client_messages_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_client_messages_test.cpp index 17f906c0f5..93f8dbb45b 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_client_messages_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_client_messages_test.cpp @@ -10,9 +10,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(QueueClientTest, EnqueueMessage) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content."; auto res = queueClient.EnqueueMessage(message).Value; @@ -28,15 +26,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(peekedMessage.MessageId, res.MessageId); EXPECT_EQ(peekedMessage.InsertedOn, res.InsertedOn); EXPECT_EQ(peekedMessage.ExpiresOn, res.ExpiresOn); - - queueClient.Delete(); } TEST_F(QueueClientTest, EnqueueMessageTTL) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content."; Queues::EnqueueMessageOptions enqueueOptions; @@ -64,15 +58,11 @@ namespace Azure { namespace Storage { namespace Test { auto receiveRes = queueClient.ReceiveMessages(); ASSERT_FALSE(receiveRes.Value.Messages.empty()); EXPECT_EQ(receiveRes.Value.Messages[0].ExpiresOn, neverExpireDateTime); - - queueClient.Delete(); } TEST_F(QueueClientTest, ReceiveMessage) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; EXPECT_TRUE(queueClient.ReceiveMessages().Value.Messages.empty()); @@ -94,15 +84,11 @@ namespace Azure { namespace Storage { namespace Test { TestSleep(std::chrono::milliseconds(1200)); receivedMessage = queueClient.ReceiveMessages().Value.Messages[0]; EXPECT_EQ(receivedMessage.DequeueCount, 2); - - queueClient.Delete(); } TEST_F(QueueClientTest, ReceiveMessages) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; Queues::ReceiveMessagesOptions receiveOptions; receiveOptions.MaxMessages = 1; @@ -135,15 +121,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(receivedMessages.size(), size_t(2)); EXPECT_EQ(receivedMessages[0].MessageText, message3); EXPECT_EQ(receivedMessages[1].MessageText, message4); - - queueClient.Delete(); } - TEST_F(QueueClientTest, PeekMessage_LIVEONLY_) + TEST_F(QueueClientTest, PeekSingleMessage) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; EXPECT_TRUE(queueClient.PeekMessages().Value.Messages.empty()); @@ -157,15 +139,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(peekedMessage.InsertedOn, res.InsertedOn); EXPECT_EQ(peekedMessage.ExpiresOn, res.ExpiresOn); EXPECT_EQ(peekedMessage.DequeueCount, 0); - - queueClient.Delete(); } TEST_F(QueueClientTest, PeekMessages) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; Queues::PeekMessagesOptions peekOptions; peekOptions.MaxMessages = 1; @@ -200,15 +178,11 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(peekedMessages[1].MessageText, message2); EXPECT_EQ(peekedMessages[2].MessageText, message3); EXPECT_EQ(peekedMessages[3].MessageText, message4); - - queueClient.Delete(); } TEST_F(QueueClientTest, UpdateMessage) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content."; const std::string updatedMessage = "MESSAGE CONTENT2"; @@ -231,45 +205,33 @@ namespace Azure { namespace Storage { namespace Test { TestSleep(std::chrono::milliseconds(1200)); peekedMessage = queueClient.PeekMessages().Value.Messages[0]; EXPECT_EQ(peekedMessage.MessageText, updatedMessage); - - queueClient.Delete(); } TEST_F(QueueClientTest, DeleteMessage) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content."; auto res = queueClient.EnqueueMessage(message).Value; EXPECT_NO_THROW(queueClient.DeleteMessage(res.MessageId, res.PopReceipt)); EXPECT_TRUE(queueClient.PeekMessages().Value.Messages.empty()); - - queueClient.Delete(); } TEST_F(QueueClientTest, ClearMessages) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content."; queueClient.EnqueueMessage(message); EXPECT_NO_THROW(queueClient.ClearMessages()); EXPECT_TRUE(queueClient.PeekMessages().Value.Messages.empty()); - - queueClient.Delete(); } TEST_F(QueueClientTest, MessageSpecialCharacters) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); - queueClient.Create(); + auto queueClient = *m_queueClient; const std::string message = "message content`~!@#$%^&*()-=_+[]{}\\|;':\",.<>/?"; @@ -278,8 +240,6 @@ namespace Azure { namespace Storage { namespace Test { auto peekedMessage = queueClient.PeekMessages().Value.Messages[0]; EXPECT_EQ(peekedMessage.MessageText, message); - - queueClient.Delete(); } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp index f4b8e9d35b..791c6e79a5 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp @@ -24,37 +24,58 @@ namespace Azure { namespace Storage { namespace Test { void QueueClientTest::SetUp() { StorageTest::SetUp(); - CHECK_SKIP_TEST(); - - m_options = InitClientOptions(); + if (shouldSkipTest()) + { + return; + } + auto options = InitClientOptions(); m_queueServiceClient = std::make_shared( Queues::QueueServiceClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_options)); - m_testName = GetTestName(); - m_testNameLowercase = GetTestNameLowerCase(); + StandardStorageConnectionString(), options)); - m_queueName = m_testNameLowercase + "base"; + m_queueName = GetLowercaseIdentifier(); m_queueClient = std::make_shared(m_queueServiceClient->GetQueueClient(m_queueName)); - m_queueClient->Create(); + + while (true) + { + try + { + m_queueClient->Create(); + break; + } + catch (StorageException& e) + { + if (e.ErrorCode != "QueueBeingDeleted") + { + throw; + } + SUCCEED() << "Queue is being deleted. Will try again after 3 seconds."; + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + } + + m_resourceCleanupFunctions.push_back( + [queueClient = *m_queueClient]() { queueClient.Delete(); }); } - void QueueClientTest::TearDown() + Queues::QueueClient QueueClientTest::GetQueueClientForTest( + const std::string& queueName, + Queues::QueueClientOptions clientOptions) { - CHECK_SKIP_TEST(); - m_queueClient->Delete(); - StorageTest::TearDown(); + InitClientOptions(clientOptions); + auto queueClient = Queues::QueueClient::CreateFromConnectionString( + StandardStorageConnectionString(), queueName, clientOptions); + m_resourceCleanupFunctions.push_back([queueClient]() { queueClient.Delete(); }); + + return queueClient; } TEST_F(QueueClientTest, CreateDelete) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase, m_options); + auto queueClient = GetQueueClientForTest(LowercaseRandomString()); Azure::Storage::Queues::CreateQueueOptions options; - Azure::Storage::Metadata metadata; - metadata["key1"] = "one"; - metadata["key2"] = "TWO"; - options.Metadata = metadata; + options.Metadata = RandomMetadata(); auto res = queueClient.Create(options); EXPECT_TRUE(res.Value.Created); EXPECT_FALSE(res.RawResponse->GetHeaders().at(_internal::HttpHeaderRequestId).empty()); @@ -70,11 +91,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE(res2.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); - queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "UPPERCASE", m_options); + queueClient = GetQueueClientForTest(LowercaseRandomString() + "UPPERCASE"); EXPECT_THROW(queueClient.Create(), StorageException); - queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), m_testNameLowercase + "2", m_options); + + queueClient = GetQueueClientForTest(LowercaseRandomString()); { auto response = queueClient.Delete(); EXPECT_FALSE(response.Value.Deleted); @@ -127,11 +147,9 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_TRUE(properties.Metadata.empty()); } - TEST_F(QueueClientTest, AccessControlList_LIVEONLY_) + TEST_F(QueueClientTest, AccessControlList) { - auto queueClient = Azure::Storage::Queues::QueueClient::CreateFromConnectionString( - StandardStorageConnectionString(), LowercaseRandomString()); - queueClient.Create(); + auto queueClient = *m_queueClient; std::vector signedIdentifiers; { @@ -170,8 +188,10 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(queueClient.SetAccessPolicy(accessPolicy)); auto ret = queueClient.GetAccessPolicy(); - EXPECT_EQ(ret.Value.SignedIdentifiers, signedIdentifiers); - + if (m_testContext.IsLiveMode()) + { + EXPECT_EQ(ret.Value.SignedIdentifiers, signedIdentifiers); + } queueClient.Delete(); } diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.hpp b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.hpp index 5eb9e1e5b5..f08cfe468b 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.hpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.hpp @@ -9,15 +9,16 @@ namespace Azure { namespace Storage { namespace Test { class QueueClientTest : public Azure::Storage::Test::StorageTest { protected: - void SetUp(); - void TearDown(); + void SetUp() override; + Queues::QueueClient GetQueueClientForTest( + const std::string& queueName, + Queues::QueueClientOptions clientOptions = Queues::QueueClientOptions()); + + protected: std::shared_ptr m_queueServiceClient; std::shared_ptr m_queueClient; std::string m_queueName; - Queues::QueueClientOptions m_options; - std::string m_testName; - std::string m_testNameLowercase; }; }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp index 0fcbfad6e7..e4870cf960 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp @@ -43,7 +43,6 @@ namespace Azure { namespace Storage { namespace Test { void SetUp() { StorageTest::SetUp(); - CHECK_SKIP_TEST(); m_options = InitClientOptions(); m_testName = GetTestName(); From 78eabdc1af9b55e686935770119f3df72d681b03 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 21 Feb 2023 22:36:08 -0500 Subject: [PATCH 08/23] Temporarily pin Node 18 to 18.13.0 - Fixes #5536 (#4378) Co-authored-by: Mike Harder --- eng/common/pipelines/templates/steps/check-spelling.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/pipelines/templates/steps/check-spelling.yml b/eng/common/pipelines/templates/steps/check-spelling.yml index a25fd94441..0c489f0893 100644 --- a/eng/common/pipelines/templates/steps/check-spelling.yml +++ b/eng/common/pipelines/templates/steps/check-spelling.yml @@ -18,8 +18,8 @@ steps: - task: NodeTool@0 condition: and(succeededOrFailed(), ne(variables['Skip.SpellCheck'],'true')) inputs: - versionSpec: 18.x - displayName: Use Node.js 18.x + versionSpec: 18.13.0 + displayName: Use Node.js 18.13.0 - task: PowerShell@2 displayName: Check spelling (cspell) From 737037d1af078337d3b4cd6bdd6ebb8a9bb4b23a Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 23 Feb 2023 11:10:27 +0800 Subject: [PATCH 09/23] Storage tests improvement (#4382) --- sdk/storage/assets.json | 2 +- .../test/ut/append_blob_client_test.cpp | 4 ++-- .../test/ut/blob_container_client_test.cpp | 4 ++-- .../test/ut/block_blob_client_test.cpp | 8 ++++---- .../azure-storage-blobs/test/ut/page_blob_client_test.cpp | 2 +- .../azure-storage-blobs/test/ut/storage_timeout_test.cpp | 4 +--- .../test/ut/datalake_file_client_test.cpp | 8 ++++---- .../test/ut/queue_service_client_test.cpp | 6 +----- 8 files changed, 16 insertions(+), 22 deletions(-) diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index 70b4d7a609..dc46939212 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_fc079eacac" + "Tag": "cpp/storage_925cef0316" } diff --git a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp index 310c360734..6e0d1c5b65 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp @@ -26,8 +26,8 @@ namespace Azure { namespace Storage { namespace Test { m_appendBlobClient = std::make_shared( m_blobContainerClient->GetAppendBlobClient(m_blobName)); m_appendBlobClient->Create(); - std::vector blobContent1(static_cast(1_KB), 'x'); - std::vector blobContent2(static_cast(512), 'a'); + std::vector blobContent1 = RandomBuffer(static_cast(1_KB)); + std::vector blobContent2 = RandomBuffer(512); auto blobContent = Azure::Core::IO::MemoryBodyStream(blobContent1.data(), blobContent1.size()); m_appendBlobClient->AppendBlock(blobContent); blobContent = Azure::Core::IO::MemoryBodyStream(blobContent2.data(), blobContent2.size()); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 1716652efd..8be8f3d052 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -538,7 +538,7 @@ namespace Azure { namespace Storage { namespace Test { } { std::string containerName = GetLowercaseIdentifier() + "1"; - std::string blobName = GetTestName() + "1"; + std::string blobName = RandomString() + "1"; Blobs::BlobClientOptions options; options.EncryptionScope = testEncryptionScope; auto containerClient2 = GetBlobContainerClientForTest(containerName, options); @@ -581,7 +581,7 @@ namespace Azure { namespace Storage { namespace Test { containerClient2.Delete(); } { - std::string blobName = GetTestName() + "2"; + std::string blobName = RandomString() + "2"; Blobs::BlobClientOptions options; options.EncryptionScope = testEncryptionScope; auto appendBlobClient diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index ec2954dd7c..9ad68c461b 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -48,7 +48,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobUploadOptions.HttpHeaders.ContentEncoding = "identity"; m_blobUploadOptions.HttpHeaders.ContentHash.Value.clear(); m_blobUploadOptions.AccessTier = Azure::Storage::Blobs::Models::AccessTier::Hot; - m_blobContent = std::vector(static_cast(1_KB), 'x'); + m_blobContent = RandomBuffer(static_cast(1_KB)); auto blobContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); m_blockBlobClient->Upload(blobContent, m_blobUploadOptions); @@ -187,7 +187,7 @@ namespace Azure { namespace Storage { namespace Test { tags["key2"] = "value2"; tags["key3 +-./:=_"] = "v1 +-./:=_"; - std::vector blobContent(10, 'a'); + std::vector blobContent = RandomBuffer(10); { Blobs::UploadBlockBlobOptions options; options.Tags = tags; @@ -1221,7 +1221,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(BlockBlobClientTest, UploadFromUri) { auto srcBlobClient = *m_blockBlobClient; - std::vector blobContent(100, 'a'); + std::vector blobContent = RandomBuffer(100); srcBlobClient.UploadFrom(blobContent.data(), blobContent.size()); std::map srcTags; srcTags["srctags"] = "a1212"; @@ -1302,7 +1302,7 @@ namespace Azure { namespace Storage { namespace Test { { auto blobClient = *m_blockBlobClient; - const std::vector content(static_cast(1), 'a'); + const std::vector content = RandomBuffer(1); const std::string blockId = Base64EncodeText(std::string(64, '0')); auto blockContent = Azure::Core::IO::MemoryBodyStream(content.data(), content.size()); blobClient.StageBlock(blockId, blockContent); diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index 1e3e66cc40..f9aa17d594 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -29,7 +29,7 @@ namespace Azure { namespace Storage { namespace Test { m_blobName = RandomString(); m_pageBlobClient = std::make_shared( m_blobContainerClient->GetPageBlobClient(m_blobName)); - m_blobContent = std::vector(static_cast(1_KB), 'x'); + m_blobContent = RandomBuffer(static_cast(1_KB)); auto blobContent = Azure::Core::IO::MemoryBodyStream(m_blobContent.data(), m_blobContent.size()); m_pageBlobClient->Create(2_KB); diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp index 8e09de5ef6..34fd18ccf9 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp @@ -52,13 +52,11 @@ namespace Azure { namespace Storage { namespace Test { } }; - m_testContext.RenameTest(GetTestName()); - auto peekPolicyPtr = std::make_unique(callback); Blobs::BlobClientOptions clientOptions = InitClientOptions(); clientOptions.PerRetryPolicies.emplace_back(std::move(peekPolicyPtr)); auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( - StandardStorageConnectionString(), GetTestNameLowerCase(), clientOptions); + StandardStorageConnectionString(), LowercaseRandomString(), clientOptions); containerClient.DeleteIfExists(); EXPECT_FALSE(timeout.HasValue()); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index 1503bdc874..e6acc9e09c 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -217,8 +217,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, FileDataActions) { - const int32_t bufferSize = 4 * 1024; // 4KB data size - std::vector buffer(bufferSize, 'x'); + const int32_t bufferSize = 10; + std::vector buffer = RandomBuffer(bufferSize); auto bufferStream = std::make_unique( Azure::Core::IO::MemoryBodyStream(buffer)); auto properties1 = m_fileClient->GetProperties(); @@ -468,8 +468,8 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileClientTest, FileReadReturns) { - const int32_t bufferSize = 4 * 1024; // 4KB data size - std::vector buffer(bufferSize, 'x'); + const int32_t bufferSize = 20; + std::vector buffer = RandomBuffer(bufferSize); auto bufferStream = std::make_unique( Azure::Core::IO::MemoryBodyStream(buffer)); std::string newFileName("fileForTest"); diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp index e4870cf960..b04874688b 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp @@ -45,8 +45,6 @@ namespace Azure { namespace Storage { namespace Test { StorageTest::SetUp(); m_options = InitClientOptions(); - m_testName = GetTestName(); - m_testNameLowercase = GetTestNameLowerCase(); m_queueServiceClient = std::make_shared( Queues::QueueServiceClient::CreateFromConnectionString( StandardStorageConnectionString(), m_options)); @@ -55,8 +53,6 @@ namespace Azure { namespace Storage { namespace Test { std::shared_ptr m_queueServiceClient; Queues::QueueClientOptions m_options; - std::string m_testName; - std::string m_testNameLowercase; }; TEST_F(QueueServiceClientTest, ListQueues) @@ -268,7 +264,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(QueueServiceClientTest, CreateDeleteQueue) { - std::string queueName = m_testNameLowercase; + std::string queueName = LowercaseRandomString(); auto queueClient = m_queueServiceClient->CreateQueue(queueName); EXPECT_NO_THROW(queueClient.Value.GetProperties()); From 365966de73f4b278b481c36f35c9dabaf9400bc8 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 23 Feb 2023 12:44:49 +0800 Subject: [PATCH 10/23] Update CODEOWNERS (#4380) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 66f75ac193..bc52d464a2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,7 +29,7 @@ /sdk/keyvault/ @vhvb1989 @gearama @antkmsft @rickwinter # PRLabel: %Storage -/sdk/storage/ @vinjiang @katmsft @Jinming-Hu @EmmaZhu @antkmsft @vhvb1989 @gearama @LarryOsterman @microzchang +/sdk/storage/ @vinjiang @Jinming-Hu @EmmaZhu @antkmsft @vhvb1989 @gearama @LarryOsterman @microzchang # PRLabel: %EngSys /sdk/template/ @danieljurek @weshaggard @LarryOsterman @RickWinter From 8cd4ecbb79b4da454570de60fd02f522733aaf24 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 23 Feb 2023 14:57:49 +0800 Subject: [PATCH 11/23] show headers and query parameters in storage CI pipeline (#4379) --- .../test/ut/append_blob_client_test.cpp | 2 +- .../test/ut/blob_batch_client_test.cpp | 4 +- .../test/ut/blob_container_client_test.cpp | 2 +- .../test/ut/blob_service_client_test.cpp | 4 +- .../test/ut/blob_service_client_test.hpp | 2 +- .../test/ut/block_blob_client_test.cpp | 2 +- .../test/ut/page_blob_client_test.cpp | 3 +- .../test/ut/storage_timeout_test.cpp | 2 +- .../test/ut/bearer_token_test.cpp | 4 +- .../test/ut/test_base.cpp | 184 ++++++++++++++++++ .../test/ut/test_base.hpp | 16 ++ .../ut/datalake_directory_client_test.cpp | 2 +- .../ut/datalake_file_system_client_test.cpp | 6 +- .../test/ut/datalake_path_client_test.cpp | 6 +- .../test/ut/datalake_service_client_test.cpp | 2 +- .../test/ut/datalake_service_client_test.hpp | 2 +- .../test/ut/share_client_test.cpp | 6 +- .../test/ut/share_file_client_test.cpp | 2 +- .../test/ut/share_service_client_test.cpp | 2 +- .../test/ut/queue_client_test.cpp | 4 +- .../test/ut/queue_service_client_test.cpp | 3 +- 21 files changed, 230 insertions(+), 30 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp index 6e0d1c5b65..eed379673a 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/append_blob_client_test.cpp @@ -265,7 +265,7 @@ namespace Azure { namespace Storage { namespace Test { auto blobClient = GetAppendBlobClientForTest(RandomString()); auto blobClientWithoutAuth = Azure::Storage::Blobs::AppendBlobClient( - blobClient.GetUrl(), InitClientOptions()); + blobClient.GetUrl(), InitStorageClientOptions()); EXPECT_THROW(blobClientWithoutAuth.CreateIfNotExists(), StorageException); { auto response = blobClient.CreateIfNotExists(); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp index 26d916603c..0814f4de8d 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_batch_client_test.cpp @@ -107,7 +107,7 @@ namespace Azure { namespace Storage { namespace Test { auto serviceClient = *m_blobServiceClient; GetBlobContainerClientForTest(containerName).CreateIfNotExists(); Blobs::BlobClientOptions clientOptions; - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto containerClient = Blobs::BlobContainerClient( serviceClient.GetBlobContainerClient(containerName).GetUrl() + containerSasToken, clientOptions); @@ -135,7 +135,7 @@ namespace Azure { namespace Storage { namespace Test { = std::make_shared( AadTenantId(), AadClientId(), AadClientSecret()); Blobs::BlobClientOptions clientOptions; - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto serviceClient = Blobs::BlobServiceClient(m_blobServiceClient->GetUrl(), credential, clientOptions); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 8be8f3d052..1dc17f88a8 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -55,7 +55,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string& containerName, Blobs::BlobClientOptions clientOptions) { - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto blobContainerClient = Blobs::BlobContainerClient::CreateFromConnectionString( StandardStorageConnectionString(), containerName, clientOptions); m_resourceCleanupFunctions.push_back( diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp index dbb2d17965..0bd9e5d921 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.cpp @@ -355,7 +355,7 @@ namespace Azure { namespace Storage { namespace Test { auto secondaryServiceClient = Blobs::BlobServiceClient( InferSecondaryUrl(serviceClient.GetUrl()), keyCredential, - InitClientOptions()); + InitStorageClientOptions()); auto serviceStatistics = secondaryServiceClient.GetStatistics().Value; EXPECT_FALSE(serviceStatistics.GeoReplication.Status.ToString().empty()); @@ -448,7 +448,7 @@ namespace Azure { namespace Storage { namespace Test { = std::make_shared( AadTenantId(), AadClientId(), AadClientSecret()); Blobs::BlobClientOptions options; - InitClientOptions(options); + InitStorageClientOptions(options); auto blobServiceClient1 = Blobs::BlobServiceClient(serviceClient.GetUrl(), credential, options); diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp index dc1eb88781..f3e30e6576 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_service_client_test.hpp @@ -15,7 +15,7 @@ namespace Azure { namespace Storage { namespace Test { { StorageTest::SetUp(); - auto options = InitClientOptions(); + auto options = InitStorageClientOptions(); m_blobServiceClient = std::make_shared( Blobs::BlobServiceClient::CreateFromConnectionString( StandardStorageConnectionString(), options)); diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index 9ad68c461b..1b226bbcbc 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -693,7 +693,7 @@ namespace Azure { namespace Storage { namespace Test { auto blobClient = GetBlockBlobClientForTest(RandomString()); auto blobClientWithoutAuth = Azure::Storage::Blobs::BlockBlobClient( - blobClient.GetUrl(), InitClientOptions()); + blobClient.GetUrl(), InitStorageClientOptions()); { auto response = blobClient.DeleteIfExists(); EXPECT_FALSE(response.Value.Deleted); diff --git a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp index f9aa17d594..0401e4dd51 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/page_blob_client_test.cpp @@ -361,7 +361,8 @@ namespace Azure { namespace Storage { namespace Test { auto pageBlobClient = GetPageBlobClientTestForTest(RandomString()); auto blobClientWithoutAuth = Azure::Storage::Blobs::PageBlobClient( - pageBlobClient.GetUrl(), InitClientOptions()); + pageBlobClient.GetUrl(), + InitStorageClientOptions()); EXPECT_THROW(blobClientWithoutAuth.CreateIfNotExists(m_blobContent.size()), StorageException); { auto response = pageBlobClient.CreateIfNotExists(m_blobContent.size()); diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp index 34fd18ccf9..13c7d08026 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp @@ -53,7 +53,7 @@ namespace Azure { namespace Storage { namespace Test { }; auto peekPolicyPtr = std::make_unique(callback); - Blobs::BlobClientOptions clientOptions = InitClientOptions(); + Blobs::BlobClientOptions clientOptions = InitStorageClientOptions(); clientOptions.PerRetryPolicies.emplace_back(std::move(peekPolicyPtr)); auto containerClient = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( StandardStorageConnectionString(), LowercaseRandomString(), clientOptions); diff --git a/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp b/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp index 8fc594ee94..5c26892f1f 100644 --- a/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp +++ b/sdk/storage/azure-storage-common/test/ut/bearer_token_test.cpp @@ -14,9 +14,9 @@ namespace Azure { namespace Storage { namespace Test { AadTenantId(), AadClientId(), AadClientSecret(), - InitClientOptions()); + InitStorageClientOptions()); containerClient = Blobs::BlobContainerClient( - containerClient.GetUrl(), credential, InitClientOptions()); + containerClient.GetUrl(), credential, InitStorageClientOptions()); EXPECT_NO_THROW(containerClient.Create()); EXPECT_NO_THROW(containerClient.Delete()); diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.cpp b/sdk/storage/azure-storage-common/test/ut/test_base.cpp index f8a3206313..98c2b08cc6 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.cpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.cpp @@ -357,6 +357,190 @@ namespace Azure { namespace Storage { namespace Test { return secondaryUri.GetAbsoluteUrl(); } + void StorageTest::InitLoggingOptions(Azure::Core::_internal::ClientOptions& options) + { + // cspell:ignore mibps, numofmessages, rscc, rscd, rsce, rscl, rsct + const std::set allowedHttpHeaders = { + "x-ms-version", + "x-ms-write", + "x-ms-version-id", + "x-ms-type", + "x-ms-time-next-visible", + "x-ms-tags", + "x-ms-tag-count", + "x-ms-source-range", + "x-ms-source-lease-id", + "x-ms-source-if-unmodified-since", + "x-ms-source-if-tags", + "x-ms-source-if-none-match-crc64", + "x-ms-source-if-none-match", + "x-ms-source-if-modified-since", + "x-ms-source-if-match-crc64", + "x-ms-source-if-match", + "x-ms-source-content-md5", + "x-ms-source-content-crc64", + "x-ms-snapshot", + "x-ms-sku-name", + "x-ms-share-quota", + "x-ms-request-server-encrypted", + "x-ms-requires-sync", + "x-ms-resource-type", + "x-ms-root-squash", + "x-ms-seal-blob", + "x-ms-sequence-number-action", + "x-ms-server-encrypted", + "x-ms-share-next-allowed-quota-downgrade-time", + "x-ms-share-provisioned-bandwidth-mibps", + "x-ms-share-provisioned-egress-mbps", + "x-ms-share-provisioned-ingress-mbps", + "x-ms-share-provisioned-iops", + "x-ms-page-write", + "x-ms-permissions", + "x-ms-popreceipt", + "x-ms-properties", + "x-ms-proposed-lease-id", + "x-ms-range", + "x-ms-range-get-content-crc64", + "x-ms-range-get-content-md5", + "x-ms-rehydrate-priority", + "x-ms-meta-*", + "x-ms-namespace-enabled", + "x-ms-number-of-handles-closed", + "x-ms-number-of-handles-failed", + "x-ms-has-immutability-policy", + "x-ms-has-legal-hold", + "x-ms-if-sequence-number-eq", + "x-ms-if-sequence-number-le", + "x-ms-if-sequence-number-lt", + "x-ms-if-tags", + "x-ms-immutable-storage-with-versioning-enabled", + "x-ms-incremental-copy", + "x-ms-is-current-version", + "x-ms-is-hns-enabled", + "x-ms-is-soft-deleted", + "x-ms-lease-action", + "x-ms-lease-break-period", + "x-ms-lease-duration", + "x-ms-lease-id", + "x-ms-lease-renewed", + "x-ms-lease-state", + "x-ms-lease-status", + "x-ms-lease-time", + "accept-ranges", + "content-disposition", + "content-encoding", + "content-language", + "content-md5", + "content-range", + "x-ms-access-tier", + "x-ms-access-tier-change-time", + "x-ms-access-tier-inferred", + "x-ms-account-kind", + "x-ms-approximate-messages-count", + "x-ms-archive-status", + "x-ms-blob-append-offset", + "x-ms-blob-cache-control", + "x-ms-blob-committed-block-count", + "x-ms-blob-condition-appendpos", + "x-ms-blob-condition-maxsize", + "x-ms-blob-content-disposition", + "x-ms-blob-content-encoding", + "x-ms-blob-content-language", + "x-ms-blob-content-length", + "x-ms-blob-content-md5", + "x-ms-blob-content-type", + "x-ms-blob-public-access", + "x-ms-blob-sealed", + "x-ms-blob-sequence-number", + "x-ms-blob-type", + "x-ms-cache-control", + "x-ms-content-crc64", + "x-ms-content-disposition", + "x-ms-content-encoding", + "x-ms-content-language", + "x-ms-content-length", + "x-ms-content-type", + "x-ms-copy-completion-time", + "x-ms-copy-destination-snapshot", + "x-ms-copy-id", + "x-ms-copy-progress", + "x-ms-copy-source-blob-properties", + "x-ms-copy-source-tag-option", + "x-ms-copy-status", + "x-ms-creation-time", + "x-ms-date", + "x-ms-delete-snapshots", + "x-ms-delete-type-permanent", + "x-ms-deleted-container-name", + "x-ms-deleted-container-version", + "x-ms-deletion-id", + "x-ms-deny-encryption-scope-override", + "x-ms-destination-lease-id", + "x-ms-enabled-protocols", + "x-ms-encryption-algorithm", + "x-ms-error-code", + "x-ms-existing-resource-type", + "x-ms-expiry-option", + "x-ms-expiry-time", + "x-ms-file-attributes", + "x-ms-file-change-time", + "x-ms-file-creation-time", + "x-ms-file-last-write-time", + "x-ms-file-permission-copy-mode", + "x-ms-file-rename-ignore-readonly", + "x-ms-file-rename-replace-if-exists", + }; + const std::set allowedQueryParameters = { + "comp", + "blockid", + "restype", + "versionid", + "snapshot", + "sv", + "sr", + "sp", + "spr", + "se", + "where", + "prefix", + "maxresults", + "delimiter", + "include", + "blocklisttype", + "ss", + "st", + "srt", + "popreceipt", + "visibilitytimeout", + "peekonly", + "numofmessages", + "messagettl", + "rscc", + "rscd", + "rsce", + "rscl", + "rsct", + "resource", + "action", + "recursive", + "timeout", + "position", + "mode", + "showonly", + "flush", + "maxResults", + "ske", + "sks", + "skv", + "skt", + "sdd", + "directory", + }; + options.Log.AllowedHttpHeaders.insert(allowedHttpHeaders.begin(), allowedHttpHeaders.end()); + options.Log.AllowedHttpQueryParameters.insert( + allowedQueryParameters.begin(), allowedQueryParameters.end()); + } + const Azure::ETag StorageTest::DummyETag("0x8D83B58BDF51D75"); const Azure::ETag StorageTest::DummyETag2("0x8D812645BFB0CDE"); diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.hpp b/sdk/storage/azure-storage-common/test/ut/test_base.hpp index da26d99ee3..3955b896ca 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.hpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.hpp @@ -152,9 +152,25 @@ namespace Azure { namespace Storage { return Azure::Core::Convert::Base64Encode(std::vector(text.begin(), text.end())); } + template T InitStorageClientOptions() + { + T options; + InitStorageClientOptions(options); + return options; + } + template void InitStorageClientOptions(T& options) + { + InitClientOptions(options); + InitLoggingOptions(options); + } + protected: std::vector> m_resourceCleanupFunctions; + private: + void InitLoggingOptions(Azure::Core::_internal::ClientOptions& options); + using TestBase::InitClientOptions; + private: std::mt19937_64 m_randomGenerator; }; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp index 43aabe43e2..b891792a4b 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_directory_client_test.cpp @@ -170,7 +170,7 @@ namespace Azure { namespace Storage { namespace Test { Files::DataLake::DataLakeDirectoryClient directoryClientSas( Files::DataLake::_detail::GetDfsUrlFromUrl(baseDirectoryClient.GetUrl()) + GetSas(), - InitClientOptions()); + InitStorageClientOptions()); directoryClientSas.RenameFile(sourceFilename, destinationFilename); EXPECT_THROW( baseDirectoryClient.GetFileClient(sourceFilename).GetProperties(), StorageException); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 43009ca13b..9b6f255704 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -66,7 +66,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string& fileSystemName, Files::DataLake::DataLakeClientOptions clientOptions) { - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto fsClient = Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( AdlsGen2ConnectionString(), fileSystemName, clientOptions); m_resourceCleanupFunctions.push_back([fsClient]() { fsClient.DeleteIfExists(); }); @@ -278,7 +278,7 @@ namespace Azure { namespace Storage { namespace Test { = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( AdlsGen2ConnectionString(), fileSystemName, - InitClientOptions()); + InitStorageClientOptions()); EXPECT_NO_THROW(connectionStringClient.Create()); EXPECT_NO_THROW(connectionStringClient.Delete()); } @@ -711,7 +711,7 @@ namespace Azure { namespace Storage { namespace Test { fileClient.CreateIfNotExists(); Files::DataLake::DataLakeClientOptions options; - InitClientOptions(options); + InitStorageClientOptions(options); Files::DataLake::DataLakeFileSystemClient fileSystemClientSas( Files::DataLake::_detail::GetDfsUrlFromUrl(m_fileSystemClient->GetUrl()) + GetSas(), options); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp index 137b6c51cc..16a88bd0d1 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp @@ -331,7 +331,7 @@ namespace Azure { namespace Storage { namespace Test { AdlsGen2ConnectionString(), m_fileSystemName, baseName + "1", - InitClientOptions()); + InitStorageClientOptions()); pathClient.Create(Files::DataLake::Models::PathResourceType::File); std::string pathPermissions = "rwxrw-rw-"; EXPECT_NO_THROW(pathClient.SetPermissions(pathPermissions)); @@ -353,7 +353,7 @@ namespace Azure { namespace Storage { namespace Test { AdlsGen2ConnectionString(), m_fileSystemName, baseName + "2", - InitClientOptions()); + InitStorageClientOptions()); auto response = pathClient.Create(Files::DataLake::Models::PathResourceType::File); Files::DataLake::SetPathPermissionsOptions options1, options2; options1.AccessConditions.IfUnmodifiedSince = response.Value.LastModified; @@ -368,7 +368,7 @@ namespace Azure { namespace Storage { namespace Test { AdlsGen2ConnectionString(), m_fileSystemName, baseName + "3", - InitClientOptions()); + InitStorageClientOptions()); auto response = pathClient.Create(Files::DataLake::Models::PathResourceType::File); Files::DataLake::SetPathPermissionsOptions options1, options2; options1.AccessConditions.IfMatch = response.Value.ETag; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp index c922e88e70..04b28787fb 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp @@ -180,7 +180,7 @@ namespace Azure { namespace Storage { namespace Test { .GetUrl(); auto datalakeServiceClient = Azure::Storage::Files::DataLake::DataLakeServiceClient( datalakeServiceUrl + sasToken, - InitClientOptions()); + InitStorageClientOptions()); EXPECT_NO_THROW(datalakeServiceClient.ListFileSystems()); } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp index a19ce04bf5..408e426166 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.hpp @@ -12,7 +12,7 @@ namespace Azure { namespace Storage { namespace Test { void SetUp() override { StorageTest::SetUp(); - auto options = InitClientOptions(); + auto options = InitStorageClientOptions(); m_dataLakeServiceClient = std::make_shared( Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( AdlsGen2ConnectionString(), options)); diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp index 1c83a1925d..101a4af7e5 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp @@ -67,7 +67,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string& shareName, Files::Shares::ShareClientOptions clientOptions) { - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( StandardStorageConnectionString(), shareName, clientOptions); m_resourceCleanupFunctions.push_back([shareClient]() { @@ -83,7 +83,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string& shareName, Files::Shares::ShareClientOptions clientOptions) { - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto shareClient = Files::Shares::ShareClient::CreateFromConnectionString( PremiumFileConnectionString(), shareName, clientOptions); m_resourceCleanupFunctions.push_back([shareClient]() { shareClient.DeleteIfExists(); }); @@ -531,7 +531,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareClientTest, PremiumShare) { - auto shareClientOptions = InitClientOptions(); + auto shareClientOptions = InitStorageClientOptions(); auto shareServiceClient = Files::Shares::ShareServiceClient::CreateFromConnectionString( PremiumFileConnectionString(), shareClientOptions); { diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp index 3a7366bb9d..2025212f5b 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp @@ -825,7 +825,7 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(FileShareFileClientTest, StorageExceptionAdditionalInfo) { - auto options = InitClientOptions(); + auto options = InitStorageClientOptions(); class InvalidQueryParameterPolicy final : public Azure::Core::Http::Policies::HttpPolicy { public: ~InvalidQueryParameterPolicy() override {} diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp index 6ba1eec5f6..12bf9f3e6b 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp @@ -20,7 +20,7 @@ namespace Azure { namespace Storage { namespace Test { { StorageTest::SetUp(); - auto options = InitClientOptions(); + auto options = InitStorageClientOptions(); m_shareServiceClient = std::make_shared( Files::Shares::ShareServiceClient::CreateFromConnectionString( StandardStorageConnectionString(), options)); diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp index 791c6e79a5..f81236826f 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_client_test.cpp @@ -28,7 +28,7 @@ namespace Azure { namespace Storage { namespace Test { { return; } - auto options = InitClientOptions(); + auto options = InitStorageClientOptions(); m_queueServiceClient = std::make_shared( Queues::QueueServiceClient::CreateFromConnectionString( StandardStorageConnectionString(), options)); @@ -63,7 +63,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string& queueName, Queues::QueueClientOptions clientOptions) { - InitClientOptions(clientOptions); + InitStorageClientOptions(clientOptions); auto queueClient = Queues::QueueClient::CreateFromConnectionString( StandardStorageConnectionString(), queueName, clientOptions); m_resourceCleanupFunctions.push_back([queueClient]() { queueClient.Delete(); }); diff --git a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp index b04874688b..1be6664b5a 100644 --- a/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp +++ b/sdk/storage/azure-storage-queues/test/ut/queue_service_client_test.cpp @@ -43,8 +43,7 @@ namespace Azure { namespace Storage { namespace Test { void SetUp() { StorageTest::SetUp(); - - m_options = InitClientOptions(); + m_options = InitStorageClientOptions(); m_queueServiceClient = std::make_shared( Queues::QueueServiceClient::CreateFromConnectionString( StandardStorageConnectionString(), m_options)); From 77a849ffac6b734476f5b821753c692f015f7693 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 23 Feb 2023 16:54:22 -0500 Subject: [PATCH 12/23] Sync eng/common directory with azure-sdk-tools for PR 5431 #2501 Co-authored-by: Konrad Jamrozik --- eng/common/scripts/get-codeowners.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 index 5c2bb9b50a..afe993d50b 100644 --- a/eng/common/scripts/get-codeowners.ps1 +++ b/eng/common/scripts/get-codeowners.ps1 @@ -168,7 +168,7 @@ if ($Test) { $azSdkToolsCodeowners = (Resolve-Path "$PSScriptRoot/../../../.github/CODEOWNERS") TestGetCodeowners -targetPath "eng/common/scripts/get-codeowners.ps1" -codeownersFileLocation $azSdkToolsCodeowners -includeNonUserAliases $true -expectReturn @("konrad-jamrozik", "weshaggard", "benbp") - $testCodeowners = (Resolve-Path "$PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/glob_path_CODEOWNERS") + $testCodeowners = (Resolve-Path "$PSScriptRoot/../../../tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/test_CODEOWNERS") TestGetCodeowners -targetPath "tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners.Tests/TestData/InputDir/a.txt" -codeownersFileLocation $testCodeowners -includeNonUserAliases $true -expectReturn @("2star") exit 0 } From 1df70cb1044a37a73cb77dcb4954aaecaf1d8241 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 23 Feb 2023 20:49:29 -0500 Subject: [PATCH 13/23] Sync eng/common directory with azure-sdk-tools for PR 5562 (#4384) * Add todos to update packages to pick up the newest CODEOWNERS interpreter * update --------- Co-authored-by: Konrad Jamrozik --- eng/common/scripts/get-codeowners.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/get-codeowners.ps1 b/eng/common/scripts/get-codeowners.ps1 index afe993d50b..a4a18ecea4 100644 --- a/eng/common/scripts/get-codeowners.ps1 +++ b/eng/common/scripts/get-codeowners.ps1 @@ -62,7 +62,7 @@ param ( # but not this one: # Remove the obsolete, prefix-based CODEOWNERS matcher & related tests # https://github.com/Azure/azure-sdk-tools/pull/5431 - [string]$ToolVersion = "1.0.0-dev.20230214.3", + [string]$ToolVersion = "1.0.0-dev.20230223.4", [string]$ToolPath = (Join-Path ([System.IO.Path]::GetTempPath()) "codeowners-tool-path"), [string]$DevOpsFeed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json", [string]$VsoVariable = "", From 71a7cfb0c1ea7657406a796266a6fdfd4e9aff3a Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Thu, 23 Feb 2023 18:14:41 -0800 Subject: [PATCH 14/23] Fix Share Client failure #4377 (#4381) * Fix Share Client failure #4377 --- sdk/storage/assets.json | 2 +- .../azure-storage-files-shares/src/rest_client.cpp | 12 +++++++++--- .../azure-storage-files-shares/swagger/README.md | 4 ++++ .../test/ut/share_client_test.cpp | 2 ++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index dc46939212..bca7a91d0d 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_925cef0316" + "Tag": "cpp/storage_92d8f38118" } diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index 746db51fe0..b09969a7a6 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -1652,9 +1652,15 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { } } } - response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); - response.LastModified = DateTime::Parse( - pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + if (pRawResponse->GetHeaders().count("ETag") != 0) + { + response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + } + if (pRawResponse->GetHeaders().count("Last-Modified") != 0) + { + response.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + } return Response(std::move(response), std::move(pRawResponse)); } Response DirectoryClient::Create( diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index d35b2a83fb..a90f9023ae 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -378,6 +378,10 @@ directive: - from: swagger-document where: $["x-ms-paths"]["/{shareName}?restype=share&comp=stats"].get.responses["200"] transform: > + $.headers["ETag"]["x-ms-client-default"] = ""; + $.headers["ETag"]["x-nullable"] = true; + $.headers["Last-Modified"]["x-ms-client-default"] = ""; + $.headers["Last-Modified"]["x-nullable"] = true; $.schema = { "description": "Stats for the share.", "type": "object", diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp index 101a4af7e5..63475b85b3 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp @@ -529,6 +529,8 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(FileShareClientTest, GetStatistics) { EXPECT_NO_THROW(m_shareClient->GetStatistics()); } + TEST_F(FileShareClientTest, PremiumShare) { auto shareClientOptions = InitStorageClientOptions(); From 5059347098c511b5de11c9d112eb11f7a330870c Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Fri, 24 Feb 2023 17:19:08 +0800 Subject: [PATCH 15/23] decrease request count to avoid throttling errors for storage tests (#4385) --- .../test/ut/block_blob_client_test.cpp | 8 ++++---- .../test/ut/datalake_file_client_test.cpp | 8 ++++---- .../test/ut/share_file_client_test.cpp | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index 1b226bbcbc..4ee6a5cfae 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -1743,13 +1743,13 @@ namespace Azure { namespace Storage { namespace Test { { int64_t fileSize = RandomInt(1, 1_MB); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 47_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 185_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 117_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 259_KB)); } for (auto& f : futures) { diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index e6acc9e09c..eaeb1d9c3c 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -874,13 +874,13 @@ namespace Azure { namespace Storage { namespace Test { { int64_t fileSize = RandomInt(1, 1_MB); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 56_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 172_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 109_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 256_KB)); } for (auto& f : futures) { diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp index 2025212f5b..26d7f50849 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp @@ -436,13 +436,13 @@ namespace Azure { namespace Storage { namespace Test { { int64_t fileSize = RandomInt(1, 1_MB); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 4_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 4_KB, 40_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 16_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 2_KB, 162_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 12_KB)); + std::async(std::launch::async, testUploadFromBuffer, c, fileSize, 0, 127_KB)); futures.emplace_back( - std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 25_KB)); + std::async(std::launch::async, testUploadFromFile, c, fileSize, 0, 253_KB)); } for (auto& f : futures) { From 127073119c9973d7c7ca9019a819ddc34977dd4c Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:36:00 -0500 Subject: [PATCH 16/23] Sync eng/common directory with azure-sdk-tools for PR 5568 (#4387) * we encourage folks to place their assets.jsons at the package level * update generate-assets-json.ps1 to only include src/**/session-records so as to avoid picking up the duplicated 'target' sessionrecords --------- Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/common/testproxy/transition-scripts/README.md | 8 ++++---- .../testproxy/transition-scripts/generate-assets-json.ps1 | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/common/testproxy/transition-scripts/README.md b/eng/common/testproxy/transition-scripts/README.md index 862d22ed6d..06290bd0c7 100644 --- a/eng/common/testproxy/transition-scripts/README.md +++ b/eng/common/testproxy/transition-scripts/README.md @@ -60,12 +60,12 @@ You will not be able to clean them up however. There exists [planned work](https - `language` repo - An individual language repository eg. azure-sdk-for-python or azure-sdk-for-net etc. - `assets` repo - The repository where assets are being moved to. -The `test-proxy` tool is integrated with the ability to automatically restore these assets. This process is kick-started by the presence of an `assets.json` alongside a dev's actual code. This means that while assets will be cloned down externally, the _map_ to those assets will be stored alongside the tests. Normally, it is recommended to create an `assets.json` under the path `sdk/`. However, more granular storage is also possible. +The `test-proxy` tool is integrated with the ability to automatically restore these assets. This process is kick-started by the presence of an `assets.json` alongside a dev's actual code. This means that while assets will be cloned down externally, the _map_ to those assets will be stored alongside the tests. Normally, it is recommended to create an `assets.json` under the path `sdk//`. More granular storage than on an individual package level is possible, but each language's test framework would need to support that on a case-by-case basis. -Service/Package-Level examples: +Examples of current assets.json locations: -- `sdk/storage/assets.json` -- `sdk/storage/azure-storage-file-datalake/assets.json` +- [`sdk/data/aztables/assets.json`](https://github.com/Azure/azure-sdk-for-go/blob/main/sdk/data/aztables/assets.json) +- [`sdk/keyvault/azure-keyvault-keys/assets.json`](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/keyvault/azure-keyvault-keys/assets.json) The location of the actual test code is referred to as the `language repo`. diff --git a/eng/common/testproxy/transition-scripts/generate-assets-json.ps1 b/eng/common/testproxy/transition-scripts/generate-assets-json.ps1 index 2b24bdcb1a..3f43c7d3ad 100644 --- a/eng/common/testproxy/transition-scripts/generate-assets-json.ps1 +++ b/eng/common/testproxy/transition-scripts/generate-assets-json.ps1 @@ -72,7 +72,7 @@ if ($UseTestRepo) { # 3. ios $LangRecordingDirs = @{"cpp" = "recordings"; "go" = "recordings"; - "java" = "session-records"; + "java" = "src.*?session-records"; "js" = "recordings"; "net" = "SessionRecords"; "python" = "recordings"; @@ -321,8 +321,8 @@ Function Move-AssetsFromLangRepo { ) $filter = $LangRecordingDirs[$language] Write-Host "Language recording directory name=$filter" - Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { `$_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" - $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } + Write-Host "Get-ChildItem -Recurse -Filter ""*.json"" | Where-Object { if ($filter.Contains(""*"")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains ""$filter"" }" + $filesToMove = Get-ChildItem -Recurse -Filter "*.json" | Where-Object { if ($filter.Contains("*")) { $_.DirectoryName -match $filter } else { $_.DirectoryName.Split([IO.Path]::DirectorySeparatorChar) -contains "$filter" } } [string] $currentDir = Get-Location foreach ($fromFile in $filesToMove) { From e2a1b99a0625d6309e22474332fb3c342b4a2dc6 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Sat, 25 Feb 2023 11:12:08 +0800 Subject: [PATCH 17/23] Add support to ignore invalid cert common name (#4361) --- sdk/core/azure-core/CHANGELOG.md | 3 +++ .../inc/azure/core/http/policies/policy.hpp | 13 +++++++++++++ .../inc/azure/core/http/win_http_transport.hpp | 5 +++++ sdk/core/azure-core/src/http/curl/curl.cpp | 1 + .../azure-core/src/http/transport_policy.cpp | 10 +++++----- .../src/http/winhttp/win_http_transport.cpp | 16 ++++++++++++++++ 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 2a001f176a..145796da0a 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -4,6 +4,9 @@ ### Features Added +- Added the ability to ignore invalid certificate common name for TLS connections in WinHTTP transport. +- Added `DisableTlsCertificateValidation` in `TransportOptions`. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp b/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp index bbd6db90a0..f23c6ef82e 100644 --- a/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp @@ -173,6 +173,19 @@ namespace Azure { namespace Core { namespace Http { namespace Policies { */ bool EnableCertificateRevocationListCheck{false}; + /** + * @brief Disable SSL/TLS certificate verification. This option allows transport layer to + * perform insecure SSL/TLS connections and skip SSL/TLS certificate checks while still having + * SSL/TLS-encrypted communications. + * + * @remark Disabling TLS security is generally a bad idea because it allows malicious actors to + * spoof the target server and should never be enabled in production code. + * + * @remark This field is only used if the customer has not specified a default transport + * adapter. If the customer has set a Transport adapter, this option is ignored. + */ + bool DisableTlsCertificateValidation{false}; + /** * @brief Base64 encoded DER representation of an X.509 certificate expected in the certificate * chain used in TLS connections. diff --git a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp index 9ce13dd502..1efdbc17b8 100644 --- a/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/win_http_transport.hpp @@ -70,6 +70,11 @@ namespace Azure { namespace Core { */ bool IgnoreUnknownCertificateAuthority{false}; + /** + * @brief When `true`, allows an invalid common name in a certificate. + */ + bool IgnoreInvalidCertificateCommonName{false}; + /** * Proxy information. */ diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp index c061079146..01edbcc107 100644 --- a/sdk/core/azure-core/src/http/curl/curl.cpp +++ b/sdk/core/azure-core/src/http/curl/curl.cpp @@ -311,6 +311,7 @@ Azure::Core::Http::CurlTransportOptions CurlTransportOptionsFromTransportOptions curlOptions.SslOptions.PemEncodedExpectedRootCertificates = PemEncodeFromBase64(transportOptions.ExpectedTlsRootCertificate, "CERTIFICATE"); } + curlOptions.SslVerifyPeer = !transportOptions.DisableTlsCertificateValidation; return curlOptions; } diff --git a/sdk/core/azure-core/src/http/transport_policy.cpp b/sdk/core/azure-core/src/http/transport_policy.cpp index 119dc31683..ef001d07e8 100644 --- a/sdk/core/azure-core/src/http/transport_policy.cpp +++ b/sdk/core/azure-core/src/http/transport_policy.cpp @@ -28,11 +28,11 @@ namespace Azure { namespace Core { namespace Http { namespace Policies { namespa */ bool AreAnyTransportOptionsSpecified(TransportOptions const& transportOptions) { - return ( - transportOptions.HttpProxy.HasValue() || transportOptions.ProxyPassword.HasValue() - || transportOptions.ProxyUserName.HasValue() - || transportOptions.EnableCertificateRevocationListCheck - || !transportOptions.ExpectedTlsRootCertificate.empty()); + return (transportOptions.HttpProxy.HasValue() || transportOptions.ProxyPassword.HasValue() + || transportOptions.ProxyUserName.HasValue() + || transportOptions.EnableCertificateRevocationListCheck + || !transportOptions.ExpectedTlsRootCertificate.empty()) + || transportOptions.DisableTlsCertificateValidation; } } // namespace diff --git a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp index 236b921479..26481b88d7 100644 --- a/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp +++ b/sdk/core/azure-core/src/http/winhttp/win_http_transport.cpp @@ -765,6 +765,12 @@ WinHttpTransportOptions WinHttpTransportOptionsFromTransportOptions( httpOptions.IgnoreUnknownCertificateAuthority = true; } + if (transportOptions.DisableTlsCertificateValidation) + { + httpOptions.IgnoreUnknownCertificateAuthority = true; + httpOptions.IgnoreInvalidCertificateCommonName = true; + } + return httpOptions; } } // namespace @@ -918,6 +924,16 @@ _detail::WinHttpRequest::WinHttpRequest( } } + if (options.IgnoreInvalidCertificateCommonName) + { + auto option = SECURITY_FLAG_IGNORE_CERT_CN_INVALID; + if (!WinHttpSetOption( + m_requestHandle.get(), WINHTTP_OPTION_SECURITY_FLAGS, &option, sizeof(option))) + { + GetErrorAndThrow("Error while setting ignore invalid certificate common name."); + } + } + if (options.EnableCertificateRevocationListCheck) { DWORD value = WINHTTP_ENABLE_SSL_REVOCATION; From e48cf927042ab828bee5bdd14ed509a8f3388300 Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Mon, 27 Feb 2023 12:03:57 -0800 Subject: [PATCH 18/23] update to version 7.4 for admin. update tests (#4388) --- sdk/keyvault/assets.json | 2 +- .../azure/keyvault/administration/settings_client_options.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/keyvault/assets.json b/sdk/keyvault/assets.json index 9377449c75..a76b5bafdb 100644 --- a/sdk/keyvault/assets.json +++ b/sdk/keyvault/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/keyvault", - "Tag": "cpp/keyvault_ea82152bd3" + "Tag": "cpp/keyvault_408d8544f1" } diff --git a/sdk/keyvault/azure-security-keyvault-administration/inc/azure/keyvault/administration/settings_client_options.hpp b/sdk/keyvault/azure-security-keyvault-administration/inc/azure/keyvault/administration/settings_client_options.hpp index 21ac0e7caa..fef1f468c5 100644 --- a/sdk/keyvault/azure-security-keyvault-administration/inc/azure/keyvault/administration/settings_client_options.hpp +++ b/sdk/keyvault/azure-security-keyvault-administration/inc/azure/keyvault/administration/settings_client_options.hpp @@ -18,7 +18,7 @@ namespace Azure { namespace Security { namespace KeyVault { namespace Administration { /** - * @brief Define the options to create a Keyvault Administration client. + * @brief Define the options to create a Key Vault Administration client. * */ struct SettingsClientOptions final : public Azure::Core::_internal::ClientOptions @@ -27,7 +27,7 @@ namespace Azure { namespace Security { namespace KeyVault { namespace Administra * @brief Service Version used. * */ - const std::string ApiVersion{"7.4-preview.1"}; + const std::string ApiVersion{"7.4"}; }; }}}} // namespace Azure::Security::KeyVault::Administration From 4ab6a41cf1f385f52a7522c5b0fc4092f665bd87 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:28:22 -0800 Subject: [PATCH 19/23] update proxy version to include Info/Active against individual sessions + allow delayed response (#4391) Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/common/testproxy/target_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index cbd1d1b89a..e25bd79aa5 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20230201.10 +1.0.0-dev.20230224.5 From 4c22ce6ab0f4f92b88c183fd3659c9fd01c063d9 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:54:03 -0800 Subject: [PATCH 20/23] Sync eng/common directory with azure-sdk-tools for PR 5540 (#4396) * add parameter to set cadl emitter options * remove emitter name in the additional parameter --------- Co-authored-by: chunyu3 --- eng/common/scripts/Cadl-Project-Generate.ps1 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Cadl-Project-Generate.ps1 b/eng/common/scripts/Cadl-Project-Generate.ps1 index d6adc59cef..3e7ee781b0 100644 --- a/eng/common/scripts/Cadl-Project-Generate.ps1 +++ b/eng/common/scripts/Cadl-Project-Generate.ps1 @@ -4,7 +4,9 @@ param ( [Parameter(Position=0)] [ValidateNotNullOrEmpty()] - [string] $ProjectDirectory + [string] $ProjectDirectory, + [Parameter(Position=1)] + [string] $CadlAdditionalOptions ## addtional cadl emitter options, separated by semicolon if more than one, e.g. option1=value1;option2=value2 ) $ErrorActionPreference = "Stop" @@ -78,6 +80,12 @@ try { } } $cadlCompileCommand = "npx cadl compile $mainCadlFile --emit $emitterName$emitterAdditionalOptions" + if ($CadlAdditionalOptions) { + $options = $CadlAdditionalOptions.Split(";"); + foreach ($option in $options) { + $cadlCompileCommand += " --option $emitterName.$option" + } + } Write-Host($cadlCompileCommand) Invoke-Expression $cadlCompileCommand From c8b028dce071cd3ae582075a5063c875a4b49bad Mon Sep 17 00:00:00 2001 From: Ahson Khan Date: Thu, 2 Mar 2023 12:06:00 -0800 Subject: [PATCH 21/23] Follow-up to update changelog to reflect community contribution (#4393) * Follow-up to update changelog to reflect community contribution * Upgrade cspell version from 0.1 to 0.2 --- .vscode/cspell.json | 2 +- sdk/core/azure-core/CHANGELOG.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 0164a40f2f..c76dbcf56e 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -1,5 +1,5 @@ { - "version": "0.1", + "version": "0.2", "language": "en", "languageId": "cpp", "dictionaries": [ diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 145796da0a..4ecb1b52a6 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -15,7 +15,7 @@ ### Other Changes -- [[#4352]](https://github.com/Azure/azure-sdk-for-cpp/pull/4352) Fixed compilation error on Visual Studio 2017. +- [[#4352]](https://github.com/Azure/azure-sdk-for-cpp/pull/4352) Fixed compilation error on Visual Studio 2017. (A community contribution, courtesy of _[jorgen](https://github.com/jorgen)_) - Libcurl transport doesn't add `Content-Length` request header for GET/HEAD/DELETE requests anymore. ### Acknowledgments @@ -590,4 +590,4 @@ Thank you to our developer community members who helped to make Azure Core bette ## 1.0.0-beta.1 (2020-09-09) -- Initial release \ No newline at end of file +- Initial release From ecb5f3db1c3b74121a788a945e180178be96820c Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 2 Mar 2023 14:21:41 -0800 Subject: [PATCH 22/23] logging api post request body (#4404) Co-authored-by: Albert Cheng --- eng/common/scripts/artifact-metadata-parsing.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 1fc388d1d8..bfdf6edfd1 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -34,6 +34,9 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { body = $releaseNotes } + Write-Host "Post Request Body:" + Write-Host $body + $headers = @{ "Content-Type" = "application/json" "Authorization" = "token $($env:GH_TOKEN)" From ab7b238197f68a638bea9ec5af76a81703e91924 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 3 Mar 2023 07:28:46 -0800 Subject: [PATCH 23/23] Sync eng/common directory with azure-sdk-tools for PR 5595 (#4400) * Use "npm ci" to install cspell and respect package-lock.json * Review feedback * Pipe npm ci output to Write-Host --------- Co-authored-by: Daniel Jurek --- eng/common/spelling/Invoke-Cspell.ps1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/common/spelling/Invoke-Cspell.ps1 b/eng/common/spelling/Invoke-Cspell.ps1 index 79205ed906..9fc5dec9b9 100644 --- a/eng/common/spelling/Invoke-Cspell.ps1 +++ b/eng/common/spelling/Invoke-Cspell.ps1 @@ -25,7 +25,7 @@ created in the temp folder, package*.json files will be placed in that folder. .PARAMETER LeavePackageInstallCache If set the PackageInstallCache will not be deleted. Use if there are multiple calls to Invoke-Cspell.ps1 to prevent creating multiple working directories and -redundant calls `npm install`. +redundant calls `npm ci`. .PARAMETER Test Run test functions against the script logic @@ -167,9 +167,7 @@ $originalLocation = Get-Location try { Set-Location $PackageInstallCache - npm install npx | Out-Null - npm install cspell | Out-Null - npm install | Out-Null + npm ci | Write-Host # Use the mutated configuration file when calling cspell $command = "npx cspell $JobType --config $CSpellConfigPath --no-must-find-files --root $SpellCheckRoot --relative"