From a889e78e305eb9ee61874aee60528d49e8b1a575 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Tue, 3 Sep 2024 12:03:50 -0700 Subject: [PATCH 1/7] Refactor prepare-pipelines --- .../templates/jobs/prepare-pipelines.yml | 340 ++++++++---------- 1 file changed, 157 insertions(+), 183 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index f2cbb2e93..1687a4b98 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -2,8 +2,10 @@ parameters: - name: Repository type: string default: $(Build.Repository.Name) +## TODO Remove these convention options once all language repos are updated to remove the usages - name: Prefix type: string + default: '' - name: CIConventionOptions type: string default: '' @@ -17,14 +19,59 @@ parameters: type: boolean default: false +# Standard variable group ids for the Azure SDK repositories +# AzureSDK_Maven_Release_Pipeline_Secrets: 1 +# AzureSDK_Nuget_Release_Pipeline_Secrets: 13 +# AzureSDK_CocoaPods_Release_Pipeline_Secrets: 98 +# NPM_Registry_Authentication: 24 +# Release_Secrets_for_GitHub: 58 +# Secrets_for_Resource_Provisioner: 64 +# APIReview_AutoCreate_Configurations: 93 + jobs: -- job: PreparePipelines +- job: + displayName: 'Prepare pipelines for ${{ parameters.Repository }}' pool: - name: azsdk-pool-mms-ubuntu-2204-general - vmImage: ubuntu-22.04 + name: $(LINUXPOOL) + vmImage: $(LINUXVMIMAGE) + + variables: + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-java') }}: + Prefix: java + InternalVariableGroups: '1 58 93' + TestVariableGroups: 64 + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-android') }}: + Prefix: android + InternalVariableGroups: '1 58 93' + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-js') }}: + Prefix: js + InternalVariableGroups: '24 58 93' + TestVariableGroups: 64 + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-python') }}: + Prefix: python + InternalVariableGroups: '58 93' + TestVariableGroups: 64 + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-net') }}: + Prefix: net + InternalVariableGroups: '13 58 93' + TestVariableGroups: 64 + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-cpp') }}: + Prefix: cpp + InternalVariableGroups: '58 93 64' + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-ios') }}: + Prefix: ios + InternalVariableGroups: '58 93 64 98' + ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-go') }}: + Prefix: go + InternalVariableGroups: '58 93 64' + GenerateUnifiedWeekly: true + steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: + Repositories: + - Name: ${{ parameters.Repository }} + WorkingDirectory: $(System.DefaultWorkingDirectory) Paths: - 'sdk/**/*.yml' - '!sdk/**/test-recordings/*' @@ -33,183 +80,110 @@ jobs: - '!sdk/**/session-records/*' - template: /eng/common/pipelines/templates/steps/install-pipeline-generation.yml - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - # This covers our public repos. - - ${{ if not(endsWith(parameters.Repository, '-pr'))}}: - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project public - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention ci - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.CIConventionOptions}} - displayName: Create CI Pipelines for Public Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention up - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.UPConventionOptions}} - displayName: Create UP Pipelines for Public Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention tests - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.TestsConventionOptions}} - displayName: Create Live Test Pipelines for Public Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention testsweekly - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.TestsConventionOptions}} - displayName: Create Weekly (Multi-Cloud) Live Test Pipelines for Public Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention upweekly - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.UPConventionOptions}} - displayName: Create Weekly (Multi-Cloud) Unified Test Pipelines for Public Repository - condition: and(succeeded(), eq(${{parameters.GenerateUnifiedWeekly}},true)) - - # This covers our -pr repositories. - - ${{ if endsWith(parameters.Repository, '-pr')}}: - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention ci - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.CIConventionOptions}} - displayName: Create CI Pipelines for Private Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention up - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.UPConventionOptions}} - displayName: Create UP Pipelines for Private Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention tests - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.TestsConventionOptions}} - displayName: Create Live Test Pipelines for Private Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project public + --prefix $(Prefix) + --devopspath "\$(Prefix)" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention ci + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + displayName: 'Generate public pipelines for: ${{ parameters.Repository }}' + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "\$(Prefix)" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention up + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(InternalVariableGroups) $(TestVariableGroups) + displayName: 'Generate internal pipelines for: ${{ parameters.Repository }}' + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "\$(Prefix)" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention tests + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(TestVariableGroups) + displayName: 'Generate test pipelines for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "\$(Prefix)" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention testsweekly + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(TestVariableGroups) + displayName: 'Generate weekly test pipelines (multi-cloud) for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "\$(Prefix)" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention upweekly + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(InternalVariableGroups) $(TestVariableGroups) + displayName: 'Generate weekly unified test pipelines (multi-cloud) for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['GenerateUnifiedWeekly'],'')) From 092936608e16e5b90bd8b22cd08cda6059f77d78 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Wed, 4 Sep 2024 15:12:51 -0700 Subject: [PATCH 2/7] Convert pipeling generation single to use template --- .../templates/jobs/prepare-pipelines.yml | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 1687a4b98..0a784043a 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -2,6 +2,9 @@ parameters: - name: Repository type: string default: $(Build.Repository.Name) + - name: RepositoryPath + type: string + default: 'sdk' ## TODO Remove these convention options once all language repos are updated to remove the usages - name: Prefix type: string @@ -36,32 +39,34 @@ jobs: vmImage: $(LINUXVMIMAGE) variables: - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-java') }}: + RepositoryPath: "$(System.DefaultWorkingDirectory)/${{ parameters.RepositoryPath }}" + ProjectForPRValidation: 'public' + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-java') }}: Prefix: java InternalVariableGroups: '1 58 93' TestVariableGroups: 64 - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-android') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-android') }}: Prefix: android InternalVariableGroups: '1 58 93' - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-js') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-js') }}: Prefix: js InternalVariableGroups: '24 58 93' TestVariableGroups: 64 - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-python') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-python') }}: Prefix: python InternalVariableGroups: '58 93' TestVariableGroups: 64 - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-net') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-net') }}: Prefix: net InternalVariableGroups: '13 58 93' TestVariableGroups: 64 - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-cpp') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-cpp') }}: Prefix: cpp InternalVariableGroups: '58 93 64' - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-ios') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-ios') }}: Prefix: ios InternalVariableGroups: '58 93 64 98' - ${{ if eq(parameters.Repository, 'Azure/azure-sdk-for-go') }}: + ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-go') }}: Prefix: go InternalVariableGroups: '58 93 64' GenerateUnifiedWeekly: true @@ -80,6 +85,18 @@ jobs: - '!sdk/**/session-records/*' - template: /eng/common/pipelines/templates/steps/install-pipeline-generation.yml - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + + - ${{ if endsWith(parameters.Repository, '-pr') }}: + - pwsh: | + Write-Host "Update variables because we are targeting the private repo:" + Write-Host "ProjectForPRValidation = internal" + Write-Host "DevOpsPath = $(DevOpsPath)/pr" + Write-Host "Prefix = $(Prefix)-pr" + Write-Host "##vso[task.setvariable variable=ProjectForPRValidation]internal" + Write-Host "##vso[task.setvariable variable=DevOpsPath]$(DevOpsPath)/pr" + Write-Host "##vso[task.setvariable variable=Prefix]$(Prefix)-pr" + displayName: Update variables for private repp + - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -88,10 +105,10 @@ jobs: inlineScript: > $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate --organization azure-sdk - --project public + --project $(ProjectForPRValidation) --prefix $(Prefix) - --devopspath "\$(Prefix)" - --path $(System.DefaultWorkingDirectory)/sdk + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) --endpoint Azure --repository ${{ parameters.Repository }} --convention ci @@ -99,7 +116,7 @@ jobs: --branch refs/heads/$(DefaultBranch) --set-managed-variables --debug - displayName: 'Generate public pipelines for: ${{ parameters.Repository }}' + displayName: 'Generate ci pipelines for: ${{ parameters.Repository }}' - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -110,8 +127,8 @@ jobs: --organization azure-sdk --project internal --prefix $(Prefix) - --devopspath "\$(Prefix)" - --path $(System.DefaultWorkingDirectory)/sdk + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) --endpoint Azure --repository ${{ parameters.Repository }} --convention up @@ -120,7 +137,7 @@ jobs: --set-managed-variables --debug --variablegroups $(InternalVariableGroups) $(TestVariableGroups) - displayName: 'Generate internal pipelines for: ${{ parameters.Repository }}' + displayName: 'Generate release pipelines for: ${{ parameters.Repository }}' - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -131,8 +148,8 @@ jobs: --organization azure-sdk --project internal --prefix $(Prefix) - --devopspath "\$(Prefix)" - --path $(System.DefaultWorkingDirectory)/sdk + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) --endpoint Azure --repository ${{ parameters.Repository }} --convention tests @@ -153,8 +170,8 @@ jobs: --organization azure-sdk --project internal --prefix $(Prefix) - --devopspath "\$(Prefix)" - --path $(System.DefaultWorkingDirectory)/sdk + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) --endpoint Azure --repository ${{ parameters.Repository }} --convention testsweekly @@ -175,8 +192,8 @@ jobs: --organization azure-sdk --project internal --prefix $(Prefix) - --devopspath "\$(Prefix)" - --path $(System.DefaultWorkingDirectory)/sdk + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) --endpoint Azure --repository ${{ parameters.Repository }} --convention upweekly From 0c3a38f8d3fd206daf33f8c1d5e8b6133bfdb22b Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 5 Sep 2024 15:40:57 -0700 Subject: [PATCH 3/7] Convert to pwsh for setting the variables --- .../templates/jobs/prepare-pipelines.yml | 127 +++++++++++------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 0a784043a..8e11335a4 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -22,15 +22,6 @@ parameters: type: boolean default: false -# Standard variable group ids for the Azure SDK repositories -# AzureSDK_Maven_Release_Pipeline_Secrets: 1 -# AzureSDK_Nuget_Release_Pipeline_Secrets: 13 -# AzureSDK_CocoaPods_Release_Pipeline_Secrets: 98 -# NPM_Registry_Authentication: 24 -# Release_Secrets_for_GitHub: 58 -# Secrets_for_Resource_Provisioner: 64 -# APIReview_AutoCreate_Configurations: 93 - jobs: - job: displayName: 'Prepare pipelines for ${{ parameters.Repository }}' @@ -41,35 +32,20 @@ jobs: variables: RepositoryPath: "$(System.DefaultWorkingDirectory)/${{ parameters.RepositoryPath }}" ProjectForPRValidation: 'public' - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-java') }}: - Prefix: java - InternalVariableGroups: '1 58 93' - TestVariableGroups: 64 - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-android') }}: - Prefix: android - InternalVariableGroups: '1 58 93' - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-js') }}: - Prefix: js - InternalVariableGroups: '24 58 93' - TestVariableGroups: 64 - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-python') }}: - Prefix: python - InternalVariableGroups: '58 93' - TestVariableGroups: 64 - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-net') }}: - Prefix: net - InternalVariableGroups: '13 58 93' - TestVariableGroups: 64 - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-cpp') }}: - Prefix: cpp - InternalVariableGroups: '58 93 64' - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-ios') }}: - Prefix: ios - InternalVariableGroups: '58 93 64 98' - ${{ if startsWith(parameters.Repository, 'Azure/azure-sdk-for-go') }}: - Prefix: go - InternalVariableGroups: '58 93 64' - GenerateUnifiedWeekly: true + Prefix: 'na' + DevOpsPath: 'na' + InternalVariableGroups: '' + GenerateUnifiedWeekly: '' + TestVariableGroups: '' + + # Standard set of variable groups for the Azure SDK repositories + AzureSDK_Maven_Release_Pipeline_Secrets: 1 + AzureSDK_Nuget_Release_Pipeline_Secrets: 13 + AzureSDK_CocoaPods_Release_Pipeline_Secrets: 98 + NPM_Registry_Authentication: 24 + Release_Secrets_for_GitHub: 58 + Secrets_for_Resource_Provisioner: 64 + APIReview_AutoCreate_Configurations: 93 steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -86,16 +62,71 @@ jobs: - template: /eng/common/pipelines/templates/steps/install-pipeline-generation.yml - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - - ${{ if endsWith(parameters.Repository, '-pr') }}: - - pwsh: | - Write-Host "Update variables because we are targeting the private repo:" - Write-Host "ProjectForPRValidation = internal" - Write-Host "DevOpsPath = $(DevOpsPath)/pr" - Write-Host "Prefix = $(Prefix)-pr" - Write-Host "##vso[task.setvariable variable=ProjectForPRValidation]internal" - Write-Host "##vso[task.setvariable variable=DevOpsPath]$(DevOpsPath)/pr" - Write-Host "##vso[task.setvariable variable=Prefix]$(Prefix)-pr" - displayName: Update variables for private repp + - pwsh: | + Write-Host "Setting up pipeline variables" + if ("${{ parameters.Repository }}" -match "Azure/azure-sdk-for-(?[^-]*)(?-pr)?") { + $prefix = $matches['prefix'] + $devOpsPath = $prefix + if ($matches['pr']) { + $devOpsPath = "${devOpsPath}\pr" + $prefix = "${prefix}-pr" + Write-Host "##vso[task.setvariable variable=ProjectForPRValidation]internal" + } + Write-Host "Prefix = $prefix" + Write-Host "DevOpsPath = $devOpsPath" + Write-Host "##vso[task.setvariable variable=Prefix]$prefix" + Write-Host "##vso[task.setvariable variable=DevOpsPath]$devOpsPath" + } + else { + Write-Error "Repository name '${{ parameters.Repository }}' is not in the expected format." + exit 1 + } + + # Map the language prefix to the appropriate variable groups + switch $prefix + { + "java" { + $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "android" { + $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + } + "js" { + $internalVariableGroups = '$(NPM_Registry_Authentication) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "python" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "net" { + $internalVariableGroups = '$(AzureSDK_Nuget_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "cpp" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)' + } + "ios" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner) $(AzureSDK_CocoaPods_Release_Pipeline_Secrets)' + } + "go" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)' + $generateUnifiedWeekly = 'true' + } + default { + Write-Error "Prefix '$prefix' is not recognized." + exit 1 + } + } + + Write-Host "InternalVariableGroups = $internalVariableGroups" + Write-Host "TestVariableGroups = $testVariableGroups" + Write-Host "GenerateUnifiedWeekly = $generateUnifiedWeekly" + Write-Host "##vso[task.setvariable variable=InternalVariableGroups]$internalVariableGroups" + Write-Host "##vso[task.setvariable variable=TestVariableGroups]$testVariableGroups" + Write-Host "##vso[task.setvariable variable=GenerateUnifiedWeekly]$generateUnifiedWeekly" + displayName: Setup pipeline generation variables - task: AzureCLI@2 inputs: From 363e81eba064eee5d1b5a5443e391813b1763179 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 5 Sep 2024 15:46:40 -0700 Subject: [PATCH 4/7] Fix switch expression --- eng/common/pipelines/templates/jobs/prepare-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 8e11335a4..361dac74c 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -83,7 +83,7 @@ jobs: } # Map the language prefix to the appropriate variable groups - switch $prefix + switch ($prefix) { "java" { $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' From 5242ffa64dfeba2e8dc8638cf897ba6e1d1e2f2b Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 5 Sep 2024 16:05:19 -0700 Subject: [PATCH 5/7] Add path separator in front of devopspath --- eng/common/pipelines/templates/jobs/prepare-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 361dac74c..16a4eb93d 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -66,7 +66,7 @@ jobs: Write-Host "Setting up pipeline variables" if ("${{ parameters.Repository }}" -match "Azure/azure-sdk-for-(?[^-]*)(?-pr)?") { $prefix = $matches['prefix'] - $devOpsPath = $prefix + $devOpsPath = "\$prefix" if ($matches['pr']) { $devOpsPath = "${devOpsPath}\pr" $prefix = "${prefix}-pr" From 07794ed196345eb733b9a3b54ec13b9410e7e4b9 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 5 Sep 2024 16:17:05 -0700 Subject: [PATCH 6/7] Continue on error for individual step failures --- eng/common/pipelines/templates/jobs/prepare-pipelines.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 16a4eb93d..478bad42d 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -148,6 +148,7 @@ jobs: --set-managed-variables --debug displayName: 'Generate ci pipelines for: ${{ parameters.Repository }}' + continueOnError: true - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -169,6 +170,7 @@ jobs: --debug --variablegroups $(InternalVariableGroups) $(TestVariableGroups) displayName: 'Generate release pipelines for: ${{ parameters.Repository }}' + continueOnError: true - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -191,6 +193,7 @@ jobs: --variablegroups $(TestVariableGroups) displayName: 'Generate test pipelines for: ${{ parameters.Repository }}' condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + continueOnError: true - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -213,6 +216,7 @@ jobs: --variablegroups $(TestVariableGroups) displayName: 'Generate weekly test pipelines (multi-cloud) for: ${{ parameters.Repository }}' condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + continueOnError: true - task: AzureCLI@2 inputs: azureSubscription: 'opensource-api-connection' @@ -235,3 +239,4 @@ jobs: --variablegroups $(InternalVariableGroups) $(TestVariableGroups) displayName: 'Generate weekly unified test pipelines (multi-cloud) for: ${{ parameters.Repository }}' condition: and(succeeded(), ne(variables['GenerateUnifiedWeekly'],'')) + continueOnError: true From a158437711e386eec3d7cff06513cddea493063e Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Mon, 9 Sep 2024 09:22:51 -0700 Subject: [PATCH 7/7] Remove unnecessary parameters --- .../templates/jobs/prepare-pipelines.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index 478bad42d..2387c9624 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -5,22 +5,6 @@ parameters: - name: RepositoryPath type: string default: 'sdk' -## TODO Remove these convention options once all language repos are updated to remove the usages - - name: Prefix - type: string - default: '' - - name: CIConventionOptions - type: string - default: '' - - name: UPConventionOptions - type: string - default: '' - - name: TestsConventionOptions - type: string - default: '' - - name: GenerateUnifiedWeekly - type: boolean - default: false jobs: - job: