From deb847e59f908b46719b8a275c5fb3fb55924f4a Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Tue, 13 Sep 2022 08:50:38 -0700 Subject: [PATCH 01/18] Add `$schema` to `cgmanifest.json` (#3918) Co-authored-by: Jamie Magee --- .../cgmanifest.json | 1 + sdk/core/azure-core-test/cgmanifest.json | 45 +++++++------ .../cgmanifest.json | 1 + sdk/core/azure-core/cgmanifest.json | 3 +- sdk/core/perf/cgmanifest.json | 1 + sdk/identity/azure-identity/cgmanifest.json | 4 +- .../cgmanifest.json | 67 ++++++++++--------- .../cgmanifest.json | 67 ++++++++++--------- .../cgmanifest.json | 67 ++++++++++--------- .../azure-storage-blobs/cgmanifest.json | 1 + .../azure-storage-common/cgmanifest.json | 1 + .../cgmanifest.json | 1 + .../cgmanifest.json | 1 + .../azure-storage-queues/cgmanifest.json | 1 + sdk/template/azure-template/cgmanifest.json | 67 ++++++++++--------- 15 files changed, 171 insertions(+), 157 deletions(-) diff --git a/sdk/attestation/azure-security-attestation/cgmanifest.json b/sdk/attestation/azure-security-attestation/cgmanifest.json index d4b0875a5c..648f67628c 100644 --- a/sdk/attestation/azure-security-attestation/cgmanifest.json +++ b/sdk/attestation/azure-security-attestation/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/core/azure-core-test/cgmanifest.json b/sdk/core/azure-core-test/cgmanifest.json index 7348d592f0..00dd915e65 100644 --- a/sdk/core/azure-core-test/cgmanifest.json +++ b/sdk/core/azure-core-test/cgmanifest.json @@ -1,25 +1,26 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json b/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json index 698c2a9558..1e4219e5c9 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json +++ b/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/core/azure-core/cgmanifest.json b/sdk/core/azure-core/cgmanifest.json index 5e418d475b..a84431c793 100644 --- a/sdk/core/azure-core/cgmanifest.json +++ b/sdk/core/azure-core/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { @@ -94,4 +95,4 @@ "DevelopmentDependency": true } ] -} \ No newline at end of file +} diff --git a/sdk/core/perf/cgmanifest.json b/sdk/core/perf/cgmanifest.json index 244dcec184..7b765a74ad 100644 --- a/sdk/core/perf/cgmanifest.json +++ b/sdk/core/perf/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/identity/azure-identity/cgmanifest.json b/sdk/identity/azure-identity/cgmanifest.json index 819cdc37e6..f16d4d85ac 100644 --- a/sdk/identity/azure-identity/cgmanifest.json +++ b/sdk/identity/azure-identity/cgmanifest.json @@ -1,4 +1,4 @@ { - "Registrations": [ - ] + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [] } diff --git a/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/storage/azure-storage-blobs/cgmanifest.json b/sdk/storage/azure-storage-blobs/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-blobs/cgmanifest.json +++ b/sdk/storage/azure-storage-blobs/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-common/cgmanifest.json b/sdk/storage/azure-storage-common/cgmanifest.json index 568ceec412..5a5f148d9c 100644 --- a/sdk/storage/azure-storage-common/cgmanifest.json +++ b/sdk/storage/azure-storage-common/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-files-datalake/cgmanifest.json b/sdk/storage/azure-storage-files-datalake/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-files-datalake/cgmanifest.json +++ b/sdk/storage/azure-storage-files-datalake/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-files-shares/cgmanifest.json b/sdk/storage/azure-storage-files-shares/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-files-shares/cgmanifest.json +++ b/sdk/storage/azure-storage-files-shares/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-queues/cgmanifest.json b/sdk/storage/azure-storage-queues/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-queues/cgmanifest.json +++ b/sdk/storage/azure-storage-queues/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/template/azure-template/cgmanifest.json b/sdk/template/azure-template/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/template/azure-template/cgmanifest.json +++ b/sdk/template/azure-template/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } From 3fec13f7a65a0cdfd9a82bc001c6e598228dc3c2 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Wed, 14 Sep 2022 16:39:10 -0400 Subject: [PATCH 02/18] Remove SkipDefaultCheckout parameter (#3930) Co-authored-by: Ben Broderick Phillips --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 39d5a7bff1..cee1fbb2d5 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -8,16 +8,12 @@ parameters: - Name: $(Build.Repository.Name) Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory) - # NOTE: SkipDefaultCheckout is being deprecated in favor of SkipCheckoutNone - - name: SkipDefaultCheckout - type: boolean - default: false - name: SkipCheckoutNone type: boolean default: false steps: - - ${{ if and(not(parameters.SkipDefaultCheckout), not(parameters.SkipCheckoutNone)) }}: + - ${{ if not(parameters.SkipCheckoutNone) }}: - checkout: none - task: PowerShell@2 From 0e00a3a52c0153574310d01b6eca82282e507a7c Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Thu, 15 Sep 2022 14:54:24 +0800 Subject: [PATCH 03/18] Compatibility improvement for striped blob (#3932) --- .../azure-storage-blobs/src/rest_client.cpp | 21 +++++++++++++------ .../azure-storage-blobs/swagger/README.md | 20 ++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 1678ac9b72..22277f0397 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -6023,8 +6023,11 @@ namespace Azure { namespace Storage { namespace Blobs { = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64")); response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } response.IsServerEncrypted = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) @@ -6132,8 +6135,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.LastModified = DateTime::Parse( pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } return Response(std::move(response), std::move(pRawResponse)); } Response PageBlobClient::UploadPagesFromUri( @@ -6292,8 +6298,11 @@ namespace Azure { namespace Storage { namespace Blobs { = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64")); response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } response.IsServerEncrypted = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 2b34011ca2..66b8270fd4 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -515,6 +515,26 @@ directive: }); } } + - from: swagger-document + where: $ + transform: > + const operations = [ + "PageBlob_UploadPages", + "PageBlob_ClearPages", + "PageBlob_UploadPagesFromUri", + ]; + for (const url in $["x-ms-paths"]) { + for (const verb in $["x-ms-paths"][url]) { + if (!operations.includes($["x-ms-paths"][url][verb].operationId)) continue; + const operation = $["x-ms-paths"][url][verb]; + + const status_codes = Object.keys(operation.responses).filter(s => s !== "default"); + status_codes.forEach((status_code, i) => { + operation.responses[status_code].headers["x-ms-blob-sequence-number"]["x-ms-client-default"] = "int64_t()"; + operation.responses[status_code].headers["x-ms-blob-sequence-number"]["x-nullable"] = true; + }); + } + } ``` ### GetBlobServiceProperties From 17ca68720c271b614ab8791974c1ab78b8dba48a Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 15 Sep 2022 13:13:32 -0400 Subject: [PATCH 04/18] add a parameter 'condition' to the test-proxy ci invocations. (#3931) Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/common/testproxy/test-proxy-docker.yml | 6 ++++++ eng/common/testproxy/test-proxy-tool.yml | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 52a7c807a3..71a03d1be3 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -2,24 +2,30 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' targetVersion: '' templateRoot: '$(Build.SourcesDirectory)' + condition: true steps: - pwsh: | ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | docker info displayName: 'Dump active docker information' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | ${{ parameters.templateRoot }}/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" -VersionOverride="${{ parameters.targetVersion }}" displayName: 'Run the docker container' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | docker container ls -a displayName: Check running container + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" displayName: 'Set PROXY_MANUAL_START' + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 679ad2108d..293848da12 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -3,11 +3,13 @@ parameters: runProxy: true targetVersion: '' templateRoot: '$(Build.SourcesDirectory)' + condition: true steps: - pwsh: | ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | $version = $(Get-Content "${{ parameters.templateRoot }}/eng/common/testproxy/target_version.txt" -Raw).Trim() @@ -23,6 +25,7 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version $version displayName: "Install test-proxy" + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy" @@ -34,19 +37,20 @@ steps: Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` -ArgumentList "--storage-location ${{ parameters.rootFolder }}" ` -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.templateRoot }}/test-proxy.log displayName: 'Run the testproxy - windows' - condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) # nohup does NOT continue beyond the current session if you use it within powershell - bash: | nohup $(Build.BinariesDirectory)/test-proxy/test-proxy > ${{ parameters.templateRoot }}/test-proxy.log & displayName: "Run the testproxy - linux/mac" - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | @@ -62,4 +66,4 @@ steps: Write-Error "Could not connect to test proxy." exit 1 displayName: Test Proxy IsAlive - + condition: and(succeeded(), ${{ parameters.condition }}) From a511ef60544a6517bf699a709bcc4007ae539838 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Fri, 16 Sep 2022 11:02:19 +0800 Subject: [PATCH 05/18] storage ad-hoc release (#3934) --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 10 +++------- .../src/private/package_version.hpp | 6 +++--- sdk/storage/azure-storage-blobs/vcpkg.json | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 218ae12165..e76b31fc49 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -1,14 +1,10 @@ # Release History -## 12.7.0-beta.1 (Unreleased) +## 12.6.1 (2022-09-16) -### Features Added - -### Breaking Changes - -### Bugs Fixed +### Other changes -### Other Changes +- No public changes in this release. ## 12.6.0 (2022-09-06) diff --git a/sdk/storage/azure-storage-blobs/src/private/package_version.hpp b/sdk/storage/azure-storage-blobs/src/private/package_version.hpp index 6f21bf905b..05e75d177e 100644 --- a/sdk/storage/azure-storage-blobs/src/private/package_version.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/package_version.hpp @@ -9,9 +9,9 @@ #pragma once #define AZURE_STORAGE_BLOBS_VERSION_MAJOR 12 -#define AZURE_STORAGE_BLOBS_VERSION_MINOR 7 -#define AZURE_STORAGE_BLOBS_VERSION_PATCH 0 -#define AZURE_STORAGE_BLOBS_VERSION_PRERELEASE "beta.1" +#define AZURE_STORAGE_BLOBS_VERSION_MINOR 6 +#define AZURE_STORAGE_BLOBS_VERSION_PATCH 1 +#define AZURE_STORAGE_BLOBS_VERSION_PRERELEASE "" #define AZURE_STORAGE_BLOBS_VERSION_ITOA_HELPER(i) #i #define AZURE_STORAGE_BLOBS_VERSION_ITOA(i) AZURE_STORAGE_BLOBS_VERSION_ITOA_HELPER(i) diff --git a/sdk/storage/azure-storage-blobs/vcpkg.json b/sdk/storage/azure-storage-blobs/vcpkg.json index 6d375116db..ae37d635b5 100644 --- a/sdk/storage/azure-storage-blobs/vcpkg.json +++ b/sdk/storage/azure-storage-blobs/vcpkg.json @@ -1,6 +1,6 @@ { "name": "azure-storage-blobs-cpp", - "version-semver": "12.6.0", + "version-semver": "12.6.1", "description": [ "Microsoft Azure Storage Blobs SDK for C++", "This library provides Azure Storage Blobs SDK." From 9b29538f8a608d185186eff35a16ebfccad9672b Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:17:39 -0400 Subject: [PATCH 06/18] Increment package version after release of azure-storage-blobs (#3935) --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 10 ++++++++++ .../src/private/package_version.hpp | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index e76b31fc49..bbc3e87f8a 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -1,5 +1,15 @@ # Release History +## 12.7.0-beta.1 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +### Other Changes + ## 12.6.1 (2022-09-16) ### Other changes diff --git a/sdk/storage/azure-storage-blobs/src/private/package_version.hpp b/sdk/storage/azure-storage-blobs/src/private/package_version.hpp index 05e75d177e..6f21bf905b 100644 --- a/sdk/storage/azure-storage-blobs/src/private/package_version.hpp +++ b/sdk/storage/azure-storage-blobs/src/private/package_version.hpp @@ -9,9 +9,9 @@ #pragma once #define AZURE_STORAGE_BLOBS_VERSION_MAJOR 12 -#define AZURE_STORAGE_BLOBS_VERSION_MINOR 6 -#define AZURE_STORAGE_BLOBS_VERSION_PATCH 1 -#define AZURE_STORAGE_BLOBS_VERSION_PRERELEASE "" +#define AZURE_STORAGE_BLOBS_VERSION_MINOR 7 +#define AZURE_STORAGE_BLOBS_VERSION_PATCH 0 +#define AZURE_STORAGE_BLOBS_VERSION_PRERELEASE "beta.1" #define AZURE_STORAGE_BLOBS_VERSION_ITOA_HELPER(i) #i #define AZURE_STORAGE_BLOBS_VERSION_ITOA(i) AZURE_STORAGE_BLOBS_VERSION_ITOA_HELPER(i) From 52f9437adb36dfe6affc04e9e1032c85eb6bc30b Mon Sep 17 00:00:00 2001 From: Peng Li <86324823+penglimsft@users.noreply.github.com> Date: Mon, 19 Sep 2022 16:01:44 -0700 Subject: [PATCH 07/18] Update `AttestationClient::AttestTpm` API to match existing `AttestOpenEnclave` and `AttestSgxmEnclave` (#3928) * Fix broken link and typo in contributing.md * Use vector for attest instead of strings * remove options * fix comments * update release version * remove versionig * revert changelog * add the change * update comment * Update sdk/attestation/azure-security-attestation/CHANGELOG.md Co-authored-by: Larry Osterman * fix formatting * address pr comment * fix formating * update a comment * remove the attest tpm comment Co-authored-by: Peng Li Co-authored-by: Larry Osterman --- CONTRIBUTING.md | 6 +++--- .../azure-security-attestation/CHANGELOG.md | 5 +++++ .../azure/attestation/attestation_client.hpp | 20 +++++++------------ .../attestation/attestation_client_models.hpp | 5 +++-- .../attestation_client_options.hpp | 6 ------ .../src/attestation_client.cpp | 7 ++++--- .../attestation_deserializers_private.cpp | 12 +++++------ .../attestation_deserializers_private.hpp | 7 ++++--- .../test/ut/tpmattestation_test.cpp | 3 ++- 9 files changed, 34 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6eab9f5cab..e35adf304c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Thank you for your interest in contributing to Azure SDK for C++. - **DO** submit all code changes via pull requests (PRs) rather than through a direct commit. PRs will be reviewed and potentially merged by the repo maintainers after a peer review that includes at least one maintainer. - **DO** review your own PR to make sure there aren't any unintended changes or commits before submitting it. - **DO NOT** submit "work in progress" PRs. A PR should only be submitted when it is considered ready for review and subsequent merging by the contributor. - - If the change is work-in-progress or an experiment, **DO** start if off as a temporary draft PR. + - If the change is work-in-progress or an experiment, **DO** start it off as a temporary draft PR. - **DO** give PRs short-but-descriptive names (e.g. "Improve code coverage for Azure.Core by 10%", not "Fix #1234") and add a description which explains why the change is being made. - **DO** refer to any relevant issues, and include [keywords](https://docs.github.com/articles/closing-issues-via-commit-messages/) that automatically close issues when the PR is merged. - **DO** tag any users that should know about and/or review the change. @@ -48,13 +48,13 @@ Codespaces is new technology that allows you to use a container as your developm ### GitHub Codespaces 1. From the Azure SDK GitHub repo, click on the "Code -> Open with Codespaces" button. -1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing). +1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project). ### VS Code Codespaces 1. Install the [VS Code Remote Extension Pack](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) 1. When you open the Azure SDK for C++ repo in VS Code, it will prompt you to open the project in the Dev Container. If it does not prompt you, then hit CTRL+P, and select "Remote-Containers: Open Folder in Container..." -1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing). +1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project). ## Full Local Setup diff --git a/sdk/attestation/azure-security-attestation/CHANGELOG.md b/sdk/attestation/azure-security-attestation/CHANGELOG.md index b032e5a8d2..e5c537269e 100644 --- a/sdk/attestation/azure-security-attestation/CHANGELOG.md +++ b/sdk/attestation/azure-security-attestation/CHANGELOG.md @@ -6,6 +6,11 @@ ### Breaking Changes +- Changed `AttestationClient::AttestTpm` to match `AttestOpenEnclave` and `AttestSgxEnclave` + - Added `std::vector` dataToAttest parameter. + - Removed `PayLoad` in `TpmAttestationOptions` + - Changed `TpmResult` in `TpmAttestationResult` to type `std::vector` + ### Bugs Fixed ### Other Changes diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp index 4aa60061ed..d7c44d4941 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp @@ -223,26 +223,20 @@ namespace Azure { namespace Security { namespace Attestation { Azure::Core::Context const& context = Azure::Core::Context{}) const; /** - * @brief Perform a single leg - * - * Processes attestation evidence from a VBS enclave, producing an attestation result. - * + * @brief Sends TPM-based attestation data to the service. * The TPM attestation protocol is defined * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') * - * Unlike OpenEnclave reports and SGX enclave quotes, TPM attestation is implemented using - * JSON encoded strings. * - * The client formats a string serialized JSON request to the - * service, which responds with a JSON response. The serialized JSON object exchange continues - * until the service responds with a JSON string with a property named {@code "report"}, whose - * value will be an attestation result token. + * @param dataToAttest - Attestation request data. + * @param options - Options to the attestation request. + * @param context - Context for the operation. * - * @param options sent to the service for Trusted Platform Module (TPM) attestation. - * @return attestation response for Trusted Platform Module (TPM) attestation. + * @return Response - The result of the attestation operation */ Response AttestTpm( - AttestTpmOptions const& options, + std::vector const& dataToAttest, + AttestTpmOptions const& options = AttestTpmOptions{}, Azure::Core::Context const& context = Azure::Core::Context{}) const; private: diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp index 2fb15bcc51..c4db47e2ae 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp @@ -448,12 +448,13 @@ namespace Azure { namespace Security { namespace Attestation { namespace Models */ struct TpmAttestationResult final { - /** @brief The JSON encoded value returned from TPM attestation. + /** @brief Attestation response data. + * * The TPM attestation protocol is defined * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') * */ - std::string TpmResult; + std::vector TpmResult; }; /** diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp index a94cf6092e..b9f1a2450a 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp @@ -261,12 +261,6 @@ namespace Azure { namespace Security { namespace Attestation { */ struct AttestTpmOptions final { - /** - * @brief JSON Data to send to the attestation service for TPM attestation. - * @details The TPM attestation protocol is defined - * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') - */ - std::string Payload; }; /** @brief The AttestationSigningKey represents a tuple of asymmetric private cryptographic key diff --git a/sdk/attestation/azure-security-attestation/src/attestation_client.cpp b/sdk/attestation/azure-security-attestation/src/attestation_client.cpp index 1e0077adc3..8786466e1d 100644 --- a/sdk/attestation/azure-security-attestation/src/attestation_client.cpp +++ b/sdk/attestation/azure-security-attestation/src/attestation_client.cpp @@ -196,13 +196,14 @@ Azure::Response> AttestationClient::AttestOp } Azure::Response AttestationClient::AttestTpm( - AttestTpmOptions const& attestTpmOptions, + std::vector const& dataToAttest, + AttestTpmOptions const&, Azure::Core::Context const& context) const { auto tracingContext(m_tracingFactory.CreateTracingContext("AttestTpm", context)); try { - std::string jsonToSend = TpmDataSerializer::Serialize(attestTpmOptions.Payload); + std::string jsonToSend = TpmDataSerializer::Serialize(dataToAttest); auto encodedVector = std::vector(jsonToSend.begin(), jsonToSend.end()); Azure::Core::IO::MemoryBodyStream stream(encodedVector); @@ -212,7 +213,7 @@ Azure::Response AttestationClient::AttestTpm( // Send the request to the service. auto response = AttestationCommonRequest::SendRequest(*m_pipeline, request, tracingContext.Context); - std::string returnedBody(TpmDataSerializer::Deserialize(response)); + std::vector returnedBody{TpmDataSerializer::Deserialize(response)}; return Response(TpmAttestationResult{returnedBody}, std::move(response)); } catch (std::runtime_error const& ex) diff --git a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp index 580fcdf503..684cec9432 100644 --- a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp +++ b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp @@ -397,21 +397,21 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail returnValue.CertificateThumbprint, jsonResult, "x-ms-certificate-thumbprint"); return returnValue; } - std::string TpmDataSerializer::Serialize(std::string const& tpmData) + std::string TpmDataSerializer::Serialize(std::vector const& tpmData) { Azure::Core::Json::_internal::json jsonData; - jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode( - std::vector(tpmData.begin(), tpmData.end())); + jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode(tpmData); return jsonData.dump(); } - std::string TpmDataSerializer::Deserialize(Azure::Core::Json::_internal::json const& jsonData) + std::vector TpmDataSerializer::Deserialize( + Azure::Core::Json::_internal::json const& jsonData) { std::vector returnValue; JsonOptional::SetIfExists>( returnValue, jsonData, "data", Azure::Core::_internal::Base64Url::Base64UrlDecode); - return std::string(returnValue.begin(), returnValue.end()); + return returnValue; } - std::string TpmDataSerializer::Deserialize( + std::vector TpmDataSerializer::Deserialize( std::unique_ptr const& response) { return TpmDataSerializer::Deserialize( diff --git a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp index 6cc7d1279b..9b72f35fac 100644 --- a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp @@ -136,9 +136,10 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail struct TpmDataSerializer { - static std::string Serialize(std::string const& tpmData); - static std::string Deserialize(Azure::Core::Json::_internal::json const& jsonData); - static std::string Deserialize(std::unique_ptr const& response); + static std::string Serialize(std::vector const& tpmData); + static std::vector Deserialize(Azure::Core::Json::_internal::json const& jsonData); + static std::vector Deserialize( + std::unique_ptr const& response); }; }}}} // namespace Azure::Security::Attestation::_detail diff --git a/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp b/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp index dad0c78f55..2d0737e12c 100644 --- a/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp +++ b/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp @@ -118,7 +118,8 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test { { auto client(CreateClient(InstanceType::AAD)); - auto response(client.AttestTpm(AttestTpmOptions{R"({"payload": { "type": "aikcert" } })"})); + std::string tpmQuote = R"({"payload": { "type": "aikcert" } })"; + auto response(client.AttestTpm(std::vector(tpmQuote.begin(), tpmQuote.end()))); Azure::Core::Json::_internal::json parsedResponse( Azure::Core::Json::_internal::json::parse(response.Value.TpmResult)); From 237c03a9d93f7e40ff7a1baeed030c70d4350dc4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 19 Sep 2022 19:23:50 -0400 Subject: [PATCH 08/18] Update check enforcer yaml permissions and name (#3944) --- .github/workflows/event.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/event.yml b/.github/workflows/event.yml index 3a88082782..76c184c71c 100644 --- a/.github/workflows/event.yml +++ b/.github/workflows/event.yml @@ -1,4 +1,7 @@ -name: GitHub Event Handler +# NOTE: currently azure-sdk-actions only hosts check enforcer code. +# If further functionality is added, this name should be updated to reflect +# the more generic behavior +name: Check Enforcer on: check_suite: @@ -6,8 +9,14 @@ on: issue_comment: types: [created] +permissions: {} + jobs: event-handler: + permissions: + statuses: write # to set status (azure/azure-sdk-actions) + pull-requests: read # to read pull requests (azure/azure-sdk-actions) + checks: read # to read check status (azure/azure-sdk-actions) name: Handle ${{ github.event_name }} ${{ github.event.action }} event runs-on: ubuntu-latest steps: From 92957377fc2b3fc7bb506b78a4ad0b38f24a68cb Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 20 Sep 2022 10:20:06 -0700 Subject: [PATCH 09/18] Set VCPKG_ROOT (#3938) * Set VCPKG_ROOT * Update eng/pipelines/templates/steps/vcpkg-clone.yml Co-authored-by: Ben Broderick Phillips Co-authored-by: Ben Broderick Phillips --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 1 + eng/pipelines/templates/steps/vcpkg-clone.yml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index e4070400a6..aaf9eeb9f1 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -144,6 +144,7 @@ stages: parameters: RepoOwner: Azure RepoName: azure-sdk-vcpkg-betas + SetVcpkgRoot: false - template: /eng/pipelines/templates/steps/vcpkg-publish.yml parameters: diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index b374bee15d..eb0f1dc8ab 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -3,6 +3,7 @@ parameters: RepoOwner: not-set RepoName: vcpkg PRBranchName: not-set + SetVcpkgRoot: true steps: - pwsh: | @@ -14,6 +15,13 @@ steps: } displayName: Clone vcpkg (${{ parameters.RepoOwner }}/${{ parameters.RepoName }}) + - ${{ if eq(parameters.SetVcpkgRoot, true) }}: + - pwsh: | + $vcpkgRoot = Resolve-Path "${{ parameters.Workspace }}/${{ parameters.RepoName }}" + Write-Host "Set VCPKG_ROOT: $vcpkgRoot" + Write-Host "##vso[task.setvariable variable=VCPKG_ROOT]$vcpkgRoot" + displayName: Set VCPKG_ROOT + # Check out the PR branch if it's already in remote. Ignore failures. - pwsh: | $ErrorActionPreference = "Continue" From b809bf9883fe884e78c8aca0488c624cda9da12d Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 20 Sep 2022 11:04:28 -0700 Subject: [PATCH 10/18] Sync the devops helper scripts between azure-sdk repo and tools repo (#3949) This should fix https://github.com/Azure/azure-sdk-tools/issues/3538 as it will now upgrade the devops extension to the latest version which has the auth fix needed. Co-authored-by: Wes Haggard --- .../Helpers/DevOps-WorkItem-Helpers.ps1 | 26 ++++++++++++++++--- .../Update-DevOps-Release-WorkItem.ps1 | 4 +++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 index 07bd6f8848..ba299856b2 100644 --- a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 +++ b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 @@ -108,8 +108,8 @@ function LoginToAzureDevops([string]$devops_pat) if (!$devops_pat) { return } - $azCmdStr = "'$devops_pat' | az devops login $($ReleaseDevOpsOrgParameters -join ' ')" - Invoke-Expression $azCmdStr + # based on the docs at https://aka.ms/azure-devops-cli-auth the recommendation is to set this env variable to login + $env:AZURE_DEVOPS_EXT_PAT = $devops_pat } function BuildHashKeyNoNull() @@ -234,6 +234,7 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr $fields += "Custom.PackagePatchVersions" $fields += "Custom.Generated" $fields += "Custom.RoadmapState" + $fields += "Microsoft.VSTS.Common.StateChangeDate" $fieldList = ($fields | ForEach-Object { "[$_]"}) -join ", " $query = "SELECT ${fieldList} FROM WorkItems WHERE [Work Item Type] = 'Package'" @@ -466,7 +467,7 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte if ($pkgName -ne $existingItem.fields["Custom.Package"]) { $changedField = "Custom.Package" } if ($verMajorMinor -ne $existingItem.fields["Custom.PackageVersionMajorMinor"]) { $changedField = "Custom.PackageVersionMajorMinor" } if ($pkgDisplayName -ne $existingItem.fields["Custom.PackageDisplayName"]) { $changedField = "Custom.PackageDisplayName" } - if ($pkgType -ne $existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" } + if ($pkgType -ne [string]$existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" } if ($pkgNewLibrary -ne $existingItem.fields["Custom.PackageTypeNewLibrary"]) { $changedField = "Custom.PackageTypeNewLibrary" } if ($pkgRepoPath -ne $existingItem.fields["Custom.PackageRepoPath"]) { $changedField = "Custom.PackageRepoPath" } if ($serviceName -ne $existingItem.fields["Custom.ServiceName"]) { $changedField = "Custom.ServiceName" } @@ -884,6 +885,25 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions) "value": "$shippedPackages" } "@ + + # If we shipped a version after we set "In Release" state then reset the state to "Next Release Unknown" + if ($pkgWorkItem.fields["System.State"] -eq "In Release") + { + $lastShippedDate = [DateTime]$newShippedVersions[0].Date + $markedInReleaseDate = ([DateTime]$pkgWorkItem.fields["Microsoft.VSTS.Common.StateChangeDate"]) + + # We just shipped so lets set the state to "Next Release Unknown" + if ($markedInReleaseDate -le $lastShippedDate) + { + $fieldUpdates += @' +{ + "op": "replace", + "path": "/fields/State", + "value": "Next Release Unknown" +} +'@ + } + } } # Full merged version set diff --git a/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 b/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 index 34c7540062..cb96691b83 100644 --- a/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 +++ b/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 @@ -33,6 +33,10 @@ az extension show -n azure-devops *> $null if (!$?){ Write-Host 'Installing azure-devops extension' az extension add --name azure-devops +} else { + # Force update the extension to the latest version if it was already installed + # this is needed to ensure we have the authentication issue fixed from earlier versions + az extension update -n azure-devops *> $null } . (Join-Path $PSScriptRoot SemVer.ps1) From 14e4f52b9179fa05c4e150f61b3a437f8ad76863 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 21 Sep 2022 09:31:34 +0800 Subject: [PATCH 11/18] Add Win32FileTimeConverter (#3941) --- .../azure-core/inc/azure/core/datetime.hpp | 47 ++++++++++++++++++- sdk/core/azure-core/test/ut/datetime_test.cpp | 41 ++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/inc/azure/core/datetime.hpp b/sdk/core/azure-core/inc/azure/core/datetime.hpp index 4ed531473c..53eecb5f9f 100644 --- a/sdk/core/azure-core/inc/azure/core/datetime.hpp +++ b/sdk/core/azure-core/inc/azure/core/datetime.hpp @@ -269,7 +269,7 @@ inline bool operator>=(std::chrono::system_clock::time_point const& tp, DateTime namespace Core { namespace _internal { /** - * @brief Provides convertion methods for POSIX time to an #Azure::DateTime. + * @brief Provides conversion methods for POSIX time to an #Azure::DateTime. * */ class PosixTimeConverter final { @@ -311,6 +311,51 @@ namespace Core { namespace _internal { */ ~PosixTimeConverter() = delete; }; + + /** + * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. + * + */ + class Win32FileTimeConverter final { + public: + /** + * @brief Converts Win32 FILETIME to an #Azure::DateTime. + * + * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. + * @return Calculated #Azure::DateTime. + */ + static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) + { + auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); + return DateTime(t); + } + + /** + * @brief Converts a DateTime to Win32 FILETIME. + * + * @param dateTime The `%DateTime` to convert. + * @return The number of 100-nanoseconds since 1601-01-01. + */ + static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) + { + return std::chrono::duration_cast(dateTime - DateTime(1601)) + .count(); + } + + private: + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be created. + * + */ + Win32FileTimeConverter() = delete; + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no + * instance can be created. + * + */ + ~Win32FileTimeConverter() = delete; + }; }} // namespace Core::_internal } // namespace Azure diff --git a/sdk/core/azure-core/test/ut/datetime_test.cpp b/sdk/core/azure-core/test/ut/datetime_test.cpp index 2496819925..02534134f2 100644 --- a/sdk/core/azure-core/test/ut/datetime_test.cpp +++ b/sdk/core/azure-core/test/ut/datetime_test.cpp @@ -895,3 +895,44 @@ TEST(DateTime, LeapYear) EXPECT_NO_THROW(static_cast(DateTime(2021, 2, 28))); EXPECT_THROW(static_cast(DateTime(2021, 2, 29)), std::invalid_argument); } + +TEST(DateTime, Win32FileTimeConverter) +{ + using namespace Azure::Core::_internal; + + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(0)), + 0); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(1)), + 1); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412)), + 133080315699856412); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569)), + 2650467743995784569); + + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(0).ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "1601-01-01T00:00:00.0000000Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(1).ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "1601-01-01T00:00:00.0000001Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412) + .ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "2022-09-19T03:26:09.9856412Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569) + .ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "9999-12-31T23:59:59.5784569Z"); +} From 65ea32f33be9c1cc8470e3444ba46acb187eb9a8 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Wed, 21 Sep 2022 19:00:41 -0700 Subject: [PATCH 12/18] Sync eng/common directory with azure-sdk-tools for PR 4169 (#3950) * pin newest version of proxy Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/common/testproxy/target_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index 0ea1143fba..aef19a2487 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220810.2 +1.0.0-dev.20220921.2 From 769b6c5b9025c6b6d365fc76bb86e0fea7d4f333 Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Thu, 22 Sep 2022 12:28:58 +0800 Subject: [PATCH 13/18] Enable Query() test in blob SDK (#3960) --- sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp index 7a139f8093..cab73b98ef 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -130,7 +130,7 @@ id,name,price R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); } - TEST_F(BlockBlobClientTest, DISABLED_QueryCsvInputArrowOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryCsvInputArrowOutput_LIVEONLY_) { auto const testName(GetTestName()); auto client = GetBlockBlobClient(testName); From 68eea9a91f683850ea9a8ac92f5351948f67c57b Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 22 Sep 2022 16:25:13 -0700 Subject: [PATCH 14/18] Sync eng/common directory with azure-sdk-tools for PR 4212 (#3962) * version including lock fixes * target the correct proxy version Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/common/testproxy/target_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index aef19a2487..dedc530f84 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220921.2 +1.0.0-dev.20220922.1 From c8d79610e0204aeb31e13d94593a9c37cf8181ee Mon Sep 17 00:00:00 2001 From: George Arama <50641385+gearama@users.noreply.github.com> Date: Mon, 26 Sep 2022 10:34:11 -0700 Subject: [PATCH 15/18] Pipeline core (#3953) * test1 * glob * quick cleanup * ddasda * ddsa * envs * dsds * Revert "ddsa" This reverts commit 6d9e385284ba37736fb5bc41e56065943abd7937. * wq * comment out issues * valid test * maybe * qwewqewq * lang steps * Vcpkg pre-steps in the place Daniel should've recommend they go * RepoOwner * Correct repo owner name * Remove vcpkg.yml (we don't need to write) * PR trigger * small change * Update eng/pipelines/templates/jobs/perf.yml Co-authored-by: Mike Harder * Update eng/pipelines/templates/jobs/perf.yml Co-authored-by: Mike Harder * Update sdk/core/perf.yml Co-authored-by: Mike Harder Co-authored-by: Daniel Jurek Co-authored-by: Mike Harder --- eng/pipelines/templates/jobs/perf.yml | 44 +++++++++++++++++++ eng/pipelines/templates/variables/globals.yml | 22 ++++++++++ sdk/core/perf.yml | 32 ++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 eng/pipelines/templates/jobs/perf.yml create mode 100644 eng/pipelines/templates/variables/globals.yml create mode 100644 sdk/core/perf.yml diff --git a/eng/pipelines/templates/jobs/perf.yml b/eng/pipelines/templates/jobs/perf.yml new file mode 100644 index 0000000000..4e1d328449 --- /dev/null +++ b/eng/pipelines/templates/jobs/perf.yml @@ -0,0 +1,44 @@ +parameters: +- name: ServiceDirectory + type: string + default: '' +- name: Services + type: string + default: '' +- name: PackageVersions + type: string + default: '.*' +- name: Tests + type: string + default: '.*' +- name: Arguments + type: string + default: '.*' +- name: Iterations + type: number + default: '5' +- name: AdditionalArguments + type: string + default: '' +- name: EnvVars + type: object + default: [] + +extends: + template: /eng/common/pipelines/templates/jobs/perf.yml + parameters: + Variables: + - template: /eng/pipelines/templates/variables/globals.yml + Language: Cpp + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Services: ${{ parameters.Services }} + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + EnvVars: ${{ parameters.EnvVars}} + InstallLanguageSteps: + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: Microsoft diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml new file mode 100644 index 0000000000..b782b88239 --- /dev/null +++ b/eng/pipelines/templates/variables/globals.yml @@ -0,0 +1,22 @@ +variables: + # True if 'Enable system diagnostics' is checked when running a pipeline manually + IsDebug: $[coalesce(variables['System.Debug'], 'false')] + + AdditionalOptions: '' + + # Exists if needed in coalesce situations. + DefaultTestGoals: 'surefire:test' + # This will be overwritten by the test matrix, if configured. + TestGoals: $(DefaultTestGoals) + + # This will be overwritten by the test matrix, if configured. + TestOptions: '' + # TestFromSource is one of the cache keys but isn't set until the test matrix + # has been processed. Without a default value it'll be treated as a string literal + # "$(TestFromSource)" instead of true/false. It'll be overwritten when the test + # matrix has been processed + TestFromSource: false + + skipComponentGovernanceDetection: true + DisableDockerDetector: true + Package.EnableSBOMSigning: true diff --git a/sdk/core/perf.yml b/sdk/core/perf.yml new file mode 100644 index 0000000000..a8494272b1 --- /dev/null +++ b/sdk/core/perf.yml @@ -0,0 +1,32 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: 'source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(extendedOptions)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: core + Services: "^core$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} From 53797c82ce411a6d1ee67ca83ab88fe9a9e5225c Mon Sep 17 00:00:00 2001 From: JinmingHu Date: Wed, 28 Sep 2022 10:50:18 +0800 Subject: [PATCH 16/18] Win32FileTimeConverter (#3967) --- .../src/private/datalake_utilities.hpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp index a8e4e3adf6..d0d006882d 100644 --- a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -28,4 +29,48 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam Blobs::BlobClientOptions GetBlobClientOptions(const DataLakeClientOptions& options); + /** + * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. + * + */ + class Win32FileTimeConverter final { + public: + /** + * @brief Converts Win32 FILETIME to an #Azure::DateTime. + * + * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. + * @return Calculated #Azure::DateTime. + */ + static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) + { + auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); + return DateTime(t); + } + + /** + * @brief Converts a DateTime to Win32 FILETIME. + * + * @param dateTime The `%DateTime` to convert. + * @return The number of 100-nanoseconds since 1601-01-01. + */ + static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) + { + return std::chrono::duration_cast(dateTime - DateTime(1601)) + .count(); + } + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be created. + * + */ + Win32FileTimeConverter() = delete; + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no + * instance can be created. + * + */ + ~Win32FileTimeConverter() = delete; + }; + }}}}} // namespace Azure::Storage::Files::DataLake::_detail From 607cab7965fae1f847ea44ce5a4804650b7b144d Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 28 Sep 2022 12:01:46 +0800 Subject: [PATCH 17/18] change win32filetimeconverter --- .../files/datalake/datalake_responses.hpp | 21 ++++++++++++ .../storage/files/datalake/rest_client.hpp | 34 +++++++++---------- .../src/datalake_directory_client.cpp | 27 +++++++++++++-- .../src/datalake_file_system_client.cpp | 28 +++++++++++---- .../src/rest_client.cpp | 10 ++---- .../swagger/README.md | 5 +-- .../ut/datalake_file_system_client_test.cpp | 2 ++ 7 files changed, 92 insertions(+), 35 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index ef15e55df3..89da98cb0f 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -193,6 +193,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using RehydratePriority = Blobs::Models::RehydratePriority; using ArchiveStatus = Blobs::Models::ArchiveStatus; + /** + * @brief The path item returned when listing the paths. + */ + struct PathItem final + { + std::string Name; + bool IsDirectory = false; + DateTime LastModified; + int64_t FileSize = int64_t(); + std::string Owner; + std::string Group; + std::string Permissions; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + Nullable CreatedOn; + Nullable ExpiresOn; + std::string ETag; + }; + /** * @brief The properties of the path. */ diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index d4c0afcb5c..a2d73be699 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -83,24 +83,24 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { private: std::string m_value; }; - struct PathItem final - { - std::string Name; - bool IsDirectory = false; - DateTime LastModified; - int64_t FileSize = int64_t(); - std::string Owner; - std::string Group; - std::string Permissions; - /** - * The name of the encryption scope under which the blob is encrypted. - */ - Nullable EncryptionScope; - Nullable CreatedOn; - Nullable ExpiresOn; - std::string ETag; - }; namespace _detail { + struct PathItem final + { + std::string Name; + bool IsDirectory = false; + DateTime LastModified; + int64_t FileSize = int64_t(); + std::string Owner; + std::string Group; + std::string Permissions; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + Nullable CreatedOn; + Nullable ExpiresOn; + std::string ETag; + }; /** * @brief Response type for #Azure::Storage::Files::DataLake::FileSystemClient::ListPaths. */ diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 7e81aa9f0b..3023758b84 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -247,8 +247,31 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { _internal::WithReplicaStatus(context)); ListPathsPagedResponse pagedResponse; - - pagedResponse.Paths = std::move(response.Value.Paths); + const std::string emptyExpiresOnString = "0"; + for (auto& path : response.Value.Paths) + { + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) + { + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); + } + pagedResponse.Paths.push_back(std::move(item)); + } pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c4e292127f..a6dddbfc15 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -276,18 +276,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptionsCopy, _internal::WithReplicaStatus(context)); - const auto emptyExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(0); + ListPathsPagedResponse pagedResponse; + const std::string emptyExpiresOnString = "0"; for (auto& path : response.Value.Paths) { - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() == emptyExpiresOn) + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) { - path.ExpiresOn.Reset(); + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); } + pagedResponse.Paths.push_back(std::move(item)); } - - ListPathsPagedResponse pagedResponse; - pagedResponse.Paths = std::move(response.Value.Paths); pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 47c7d56686..c105e1428a 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -94,7 +94,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ? jsonRoot["paths"] : Core::Json::_internal::json::array()) { - Models::PathItem vectorElement2; + Models::_detail::PathItem vectorElement2; vectorElement2.Name = var0["name"].get(); if (var0.count("isDirectory") != 0) { @@ -116,15 +116,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } if (var0.count("creationTime") != 0) { - vectorElement2.CreatedOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["creationTime"].get())); + vectorElement2.CreatedOn = var0["creationTime"].get(); } if (var0.count("expiryTime") != 0) { - vectorElement2.ExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["expiryTime"].get())); + vectorElement2.ExpiresOn = var0["expiryTime"].get(); } if (var0.count("etag") != 0) { diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 9626429b02..b7c915b212 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -229,12 +229,13 @@ directive: where: $.definitions transform: > $.Path["x-ms-client-name"] = "PathItem"; + $.Path["x-namespace"] = "_detail"; $.Path.properties["lastModified"]["format"] = "date-time-rfc1123"; $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize"; $.Path.properties["isDirectory"]["x-ms-client-default"] = false; $.Path.properties["EncryptionScope"]["x-nullable"] = true; - $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true}; - $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; + $.Path.properties["creationTime"] = {"type": "string", "x-ms-client-name": "CreatedOn", "x-nullable": true}; + $.Path.properties["expiryTime"] = {"type": "string", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"}; delete $.Path.properties["eTag"]; $.PathList["x-namespace"] = "_detail"; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 2cbf43567a..24f88449a1 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -318,6 +318,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(result.end(), iter); EXPECT_EQ(iter->Name, name); EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); + EXPECT_TRUE(iter->CreatedOn.HasValue()); + EXPECT_FALSE(iter->ExpiresOn.HasValue()); } for (const auto& name : m_pathNameSetB) { From 4d02558b0cc166ee6ef07c3705547420c6da62e8 Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 28 Sep 2022 13:20:25 +0800 Subject: [PATCH 18/18] add doxygen docs. --- .../files/datalake/datalake_responses.hpp | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 89da98cb0f..36a75bcd69 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -198,19 +198,59 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { */ struct PathItem final { + /** + * The name of the path. + */ std::string Name; + + /** + * Indicates whether this path is a directory. + */ bool IsDirectory = false; + + /** + * The data and time the path was last modified. + */ DateTime LastModified; + + /** + * The size of the file. + */ int64_t FileSize = int64_t(); + + /** + * The owner of the path. + */ std::string Owner; + + /** + * The group of the path. + */ std::string Group; + + /** + * The permission of the path. + */ std::string Permissions; + /** * The name of the encryption scope under which the blob is encrypted. */ Nullable EncryptionScope; + + /** + * The creation time of the path. + */ Nullable CreatedOn; + + /** + * The expiry time of the path. + */ Nullable ExpiresOn; + + /** + * An HTTP entity tag associated with the path. + */ std::string ETag; };