From 820728022827d5ef88f750e93bc38a802da054c8 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 19 Mar 2024 16:14:52 -0700 Subject: [PATCH 01/14] convert partner release to be called through 1es redirect --- eng/pipelines/partner-release.yml | 125 +++++++++++++++--------------- 1 file changed, 64 insertions(+), 61 deletions(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 1d542b3e6c16..e0c525ff7424 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -11,73 +11,76 @@ parameters: type: boolean default: true -resources: - repositories: - - repository: azure-sdk-build-tools - type: git - name: internal/azure-sdk-build-tools - ref: refs/tags/azure-sdk-build-tools_20230829.1 +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - stage: + displayName: 'Partner Release' + variables: + - name: BuildToolScripts + value: $(Pipeline.Workspace)/azure-sdk-build-tools/scripts + - name: Artifacts + value: $(Pipeline.Workspace)/artifacts + - name: EsrpArtifacts + value: $(Pipeline.Workspace)/packages-esrp + - template: /eng/common/pipelines/templates/variables.yml + jobs: + - job: Signing + pool: + name: $(WINDOWSPOOL) + vmImage: $(WINDOWSVMIMAGE) + os: windows + steps: + - checkout: azure-sdk-build-tools + path: azure-sdk-build-tools -variables: - BuildToolScripts: $(Pipeline.Workspace)/azure-sdk-build-tools/scripts - Artifacts: $(Pipeline.Workspace)/artifacts - EsrpArtifacts: $(Pipeline.Workspace)/packages-esrp + - task: PowerShell@2 + displayName: 'Download packages from blob storage' + inputs: + targetType: filePath + filePath: '$(BuildToolScripts)/copy-from-azuresdkpartnerdrops.ps1' + arguments: '$(Artifacts) ${{ parameters.BlobPath }} $(azuresdkpartnerdrops-access-key)' -jobs: -- job: Signing - pool: - name: $(WINDOWSPOOL) - vmImage: $(WINDOWSVMIMAGE) - os: windows - steps: - - checkout: azure-sdk-build-tools - path: azure-sdk-build-tools + - template: tools/java-esrp-signing/java-esrp-signing.yml@azure-sdk-build-tools + parameters: + ArtifactDirectory: $(Artifacts) - - task: PowerShell@2 - displayName: 'Download packages from blob storage' - inputs: - targetType: filePath - filePath: '$(BuildToolScripts)/copy-from-azuresdkpartnerdrops.ps1' - arguments: '$(Artifacts) ${{ parameters.BlobPath }} $(azuresdkpartnerdrops-access-key)' + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactName: packages-signed + ArtifactPath: $(Artifacts) - - template: tools/java-esrp-signing/java-esrp-signing.yml@azure-sdk-build-tools - parameters: - ArtifactDirectory: $(Artifacts) + - job: Release + dependsOn: Signing + pool: + name: $(WINDOWSPOOL) + vmImage: $(WINDOWSVMIMAGE) + os: windows + steps: + - checkout: self + path: azure-sdk-for-java - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactName: packages-signed - ArtifactPath: $(Artifacts) -- job: Release - dependsOn: Signing - pool: - name: $(WINDOWSPOOL) - vmImage: $(WINDOWSVMIMAGE) - os: windows - steps: - - checkout: self - path: azure-sdk-for-java + - checkout: azure-sdk-build-tools + path: azure-sdk-build-tools - - checkout: azure-sdk-build-tools - path: azure-sdk-build-tools + - download: current + displayName: Download Signed Artifacts + artifact: packages-signed - - download: current - displayName: Download Signed Artifacts - artifact: packages-signed + - template: tools/gpg/gpg.yml@azure-sdk-build-tools - - template: tools/gpg/gpg.yml@azure-sdk-build-tools + - template: /eng/pipelines/templates/steps/java-publishing.yml + parameters: + ArtifactDirectory: $(Pipeline.Workspace)/packages-signed + OutputDirectory: $(EsrpArtifacts) + Target: EsrpRelease + BuildToolsPath: $(Pipeline.Workspace)/azure-sdk-build-tools + JavaRepoRoot: $(Pipeline.Workspace)/azure-sdk-for-java + ShouldPublish: ${{ parameters.ShouldPublish }} + StageOnly: false - - template: /eng/pipelines/templates/steps/java-publishing.yml - parameters: - ArtifactDirectory: $(Pipeline.Workspace)/packages-signed - OutputDirectory: $(EsrpArtifacts) - Target: EsrpRelease - BuildToolsPath: $(Pipeline.Workspace)/azure-sdk-build-tools - JavaRepoRoot: $(Pipeline.Workspace)/azure-sdk-for-java - ShouldPublish: ${{ parameters.ShouldPublish }} - StageOnly: false - - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactName: packages-esrp-$(System.JobAttempt) - ArtifactPath: $(EsrpArtifacts) + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactName: packages-esrp-$(System.JobAttempt) + ArtifactPath: $(EsrpArtifacts) From e91f4ec256e4f54d30f5543309005cad5ee7e942 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 19 Mar 2024 16:16:23 -0700 Subject: [PATCH 02/14] fix path to image.yml --- eng/pipelines/partner-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index e0c525ff7424..9b72eabd6e36 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -24,7 +24,7 @@ extends: value: $(Pipeline.Workspace)/artifacts - name: EsrpArtifacts value: $(Pipeline.Workspace)/packages-esrp - - template: /eng/common/pipelines/templates/variables.yml + - template: /eng/pipelines/templates/variables/image.yml jobs: - job: Signing pool: From 2aa0010074f5906ffe524b82305f0f4fc895b579 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 19 Mar 2024 16:17:22 -0700 Subject: [PATCH 03/14] publish-artifact.yml -> publish-1es-artifact.yml --- eng/pipelines/partner-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 9b72eabd6e36..f07225ac72e8 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -80,7 +80,7 @@ extends: ShouldPublish: ${{ parameters.ShouldPublish }} StageOnly: false - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactName: packages-esrp-$(System.JobAttempt) ArtifactPath: $(EsrpArtifacts) From 5390c87898bfd6dc021e5ba284d74c974101007f Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 19 Mar 2024 16:20:15 -0700 Subject: [PATCH 04/14] vmImage -> image --- eng/pipelines/partner-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index f07225ac72e8..7d0d3666d28f 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -29,7 +29,7 @@ extends: - job: Signing pool: name: $(WINDOWSPOOL) - vmImage: $(WINDOWSVMIMAGE) + image: $(WINDOWSVMIMAGE) os: windows steps: - checkout: azure-sdk-build-tools @@ -55,7 +55,7 @@ extends: dependsOn: Signing pool: name: $(WINDOWSPOOL) - vmImage: $(WINDOWSVMIMAGE) + image: $(WINDOWSVMIMAGE) os: windows steps: - checkout: self From 0e00d8bf157ffafac3e88c931d6642243d5da39f Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 19 Mar 2024 16:30:24 -0700 Subject: [PATCH 05/14] ensure that the credscan suppressions are present right after checkiout. this ensures that the injected credscan can pass successfully --- eng/pipelines/partner-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 7d0d3666d28f..354dff6321f7 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -35,6 +35,8 @@ extends: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools + - template: /eng/pipelines/templates/steps/download-credscan-suppressions.yml + - task: PowerShell@2 displayName: 'Download packages from blob storage' inputs: @@ -64,6 +66,8 @@ extends: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools + - template: /eng/pipelines/templates/steps/download-credscan-suppressions.yml + - download: current displayName: Download Signed Artifacts artifact: packages-signed From 65d387d070a765e867146296f8870264365b73ae Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 20 Mar 2024 11:40:24 -0700 Subject: [PATCH 06/14] try a different bin skim pattern that includes backslash? --- eng/pipelines/templates/stages/1es-redirect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index e10ad86b5f28..96917d6c8d99 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -39,7 +39,7 @@ extends: binskim: # Exclude imported azure-sdk-build-tools gpg/azcopy binaries # See https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1378/Glob-Format - analyzeTargetGlob: +:file|**/*.jar;+:file|**/*.exe;-:f|**/tools/gpg/**/*.dll;-:f|**/tools/gpg/**/*.exe;-:f|**/tools/azcopy/**/*.exe;-:f|**/tools/azcopy/**/*.dll + analyzeTargetGlob: +:file|**/*.jar;+:file|**\*.jar;+:file|**/*.exe;-:f|**/tools/gpg/**/*.dll;-:f|**/tools/gpg/**/*.exe;-:f|**/tools/azcopy/**/*.exe;-:f|**/tools/azcopy/**/*.dll eslint: enabled: false justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' From 6e2b28b8809dfdfdd3482c0889df452553dcc382 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 20 Mar 2024 12:18:11 -0700 Subject: [PATCH 07/14] add an output of the folder that we're publishing to confirm that binskim should be picking up data --- eng/pipelines/partner-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 354dff6321f7..cad4be9230ba 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -84,6 +84,10 @@ extends: ShouldPublish: ${{ parameters.ShouldPublish }} StageOnly: false + - pwsh: | + Get-ChildItem -R -Path $(EsrpArtifacts) + displayName: Dump contents of ESRP Artifacts + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactName: packages-esrp-$(System.JobAttempt) From 5f8b50e11f24ef6ebd1ee11f171ed763c62058eb Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 14:23:00 -0700 Subject: [PATCH 08/14] maybe if we copy the original artifacts into the folder the tool will be able to see realize that the files are definitely there --- eng/pipelines/templates/stages/1es-redirect.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 96917d6c8d99..a51d503d24b1 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -7,7 +7,7 @@ resources: - repository: azure-sdk-build-tools type: git name: internal/azure-sdk-build-tools - ref: refs/tags/azure-sdk-build-tools_20230829.1 + ref: refs/tags/azure-sdk-build-tools_20240320.1 parameters: - name: stages @@ -39,7 +39,7 @@ extends: binskim: # Exclude imported azure-sdk-build-tools gpg/azcopy binaries # See https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1378/Glob-Format - analyzeTargetGlob: +:file|**/*.jar;+:file|**\*.jar;+:file|**/*.exe;-:f|**/tools/gpg/**/*.dll;-:f|**/tools/gpg/**/*.exe;-:f|**/tools/azcopy/**/*.exe;-:f|**/tools/azcopy/**/*.dll + analyzeTargetGlob: +:file|**/*.jar;+:file|**/*.exe;-:f|**/tools/gpg/**/*.dll;-:f|**/tools/gpg/**/*.exe;-:f|**/tools/azcopy/**/*.exe;-:f|**/tools/azcopy/**/*.dll eslint: enabled: false justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' From 194f47037d4579a573d113c871ea757a72e095a3 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 14:46:20 -0700 Subject: [PATCH 09/14] add fodder for the esrp tool to consume --- eng/pipelines/partner-release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index cad4be9230ba..964127b6637d 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -85,8 +85,11 @@ extends: StageOnly: false - pwsh: | + $DestinationFolder = Join-Path -Path $(EsrpArtifacts) -ChildPath "original" + New-Item -Path $DestinationFolder -ItemType Directory -Force + Copy-Item -Path packages-signed/* -Destination $DestinationFolder -Recurse -Force Get-ChildItem -R -Path $(EsrpArtifacts) - displayName: Dump contents of ESRP Artifacts + displayName: Copy Elements + Dump ESRP Folder - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: From 5751835081962b848fce3dde3aa93d09ee433d68 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 15:14:41 -0700 Subject: [PATCH 10/14] copy from the correct location --- eng/pipelines/partner-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 964127b6637d..50ccbc5c5afd 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -87,7 +87,7 @@ extends: - pwsh: | $DestinationFolder = Join-Path -Path $(EsrpArtifacts) -ChildPath "original" New-Item -Path $DestinationFolder -ItemType Directory -Force - Copy-Item -Path packages-signed/* -Destination $DestinationFolder -Recurse -Force + Copy-Item -Path $(Pipeline.Workspace)/packages-signed/* -Destination $DestinationFolder -Recurse -Force Get-ChildItem -R -Path $(EsrpArtifacts) displayName: Copy Elements + Dump ESRP Folder From 1b128826ddfbc34c01dff8cc290a6a82caf674b0 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 15:56:05 -0700 Subject: [PATCH 11/14] stable path direction --- eng/pipelines/partner-release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 50ccbc5c5afd..28e76dcf89df 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -89,7 +89,10 @@ extends: New-Item -Path $DestinationFolder -ItemType Directory -Force Copy-Item -Path $(Pipeline.Workspace)/packages-signed/* -Destination $DestinationFolder -Recurse -Force Get-ChildItem -R -Path $(EsrpArtifacts) - displayName: Copy Elements + Dump ESRP Folder + + $finalPlace = Resolve-Path -Path $(EsrpArtifacts) + Write-Host "##vso[task.setvariable variable=EsrpArtifacts]$finalPlace" + displayName: Copy Elements + Dump ESRP Folder + Resolve Path - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: From 0bf1960dd4a539ab4ac18679acd67208f6f976b1 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 16:22:03 -0700 Subject: [PATCH 12/14] yet another attempt at binskim working properly --- eng/CredScanSuppression.json | 2 +- eng/pipelines/partner-release.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json index e592df534d02..7402c236042d 100644 --- a/eng/CredScanSuppression.json +++ b/eng/CredScanSuppression.json @@ -37,7 +37,7 @@ }, { "file": [ - "test-proxy.log", + "test-proxy.log" ], "_justification": "Transient test file that is locked by test-proxy and should not be scanned" }, diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 28e76dcf89df..d26b932782fa 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -92,9 +92,10 @@ extends: $finalPlace = Resolve-Path -Path $(EsrpArtifacts) Write-Host "##vso[task.setvariable variable=EsrpArtifacts]$finalPlace" + Copy-Item -R -Path $(EsrpArtifacts)/* -Destination $(Build.SourcesDirectory)/packages-esrp -Force displayName: Copy Elements + Dump ESRP Folder + Resolve Path - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactName: packages-esrp-$(System.JobAttempt) - ArtifactPath: $(EsrpArtifacts) + ArtifactPath: $(Build.SourcesDirectory)/packages-esrp From 848b2c7dd88a21d42be200326fd41dea3d88ae9d Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 25 Mar 2024 17:24:49 -0700 Subject: [PATCH 13/14] change the target path --- eng/pipelines/partner-release.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index d26b932782fa..82bea6d78e3d 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -23,7 +23,7 @@ extends: - name: Artifacts value: $(Pipeline.Workspace)/artifacts - name: EsrpArtifacts - value: $(Pipeline.Workspace)/packages-esrp + value: $(Build.SourcesDirectory)/packages-esrp - template: /eng/pipelines/templates/variables/image.yml jobs: - job: Signing @@ -85,17 +85,10 @@ extends: StageOnly: false - pwsh: | - $DestinationFolder = Join-Path -Path $(EsrpArtifacts) -ChildPath "original" - New-Item -Path $DestinationFolder -ItemType Directory -Force - Copy-Item -Path $(Pipeline.Workspace)/packages-signed/* -Destination $DestinationFolder -Recurse -Force Get-ChildItem -R -Path $(EsrpArtifacts) - - $finalPlace = Resolve-Path -Path $(EsrpArtifacts) - Write-Host "##vso[task.setvariable variable=EsrpArtifacts]$finalPlace" - Copy-Item -R -Path $(EsrpArtifacts)/* -Destination $(Build.SourcesDirectory)/packages-esrp -Force displayName: Copy Elements + Dump ESRP Folder + Resolve Path - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactName: packages-esrp-$(System.JobAttempt) - ArtifactPath: $(Build.SourcesDirectory)/packages-esrp + ArtifactPath: $(EsrpArtifacts) From a152d276f1def25e309bee159f15d8a3eafd7a02 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 25 Mar 2024 18:12:56 -0700 Subject: [PATCH 14/14] remove debug dump! --- eng/pipelines/partner-release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/partner-release.yml b/eng/pipelines/partner-release.yml index 82bea6d78e3d..8e6e5e544a6f 100644 --- a/eng/pipelines/partner-release.yml +++ b/eng/pipelines/partner-release.yml @@ -84,10 +84,6 @@ extends: ShouldPublish: ${{ parameters.ShouldPublish }} StageOnly: false - - pwsh: | - Get-ChildItem -R -Path $(EsrpArtifacts) - displayName: Copy Elements + Dump ESRP Folder + Resolve Path - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactName: packages-esrp-$(System.JobAttempt)