diff --git a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml index f2cbb2e931e5..2387c9624568 100644 --- a/eng/common/pipelines/templates/jobs/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/jobs/prepare-pipelines.yml @@ -2,29 +2,41 @@ parameters: - name: Repository type: string default: $(Build.Repository.Name) - - name: Prefix + - name: RepositoryPath type: string - - name: CIConventionOptions - type: string - default: '' - - name: UPConventionOptions - type: string - default: '' - - name: TestsConventionOptions - type: string - default: '' - - name: GenerateUnifiedWeekly - type: boolean - default: false + default: 'sdk' jobs: -- job: PreparePipelines +- job: + displayName: 'Prepare pipelines for ${{ parameters.Repository }}' pool: - name: azsdk-pool-mms-ubuntu-2204-general - vmImage: ubuntu-22.04 + name: $(LINUXPOOL) + vmImage: $(LINUXVMIMAGE) + + variables: + RepositoryPath: "$(System.DefaultWorkingDirectory)/${{ parameters.RepositoryPath }}" + ProjectForPRValidation: 'public' + Prefix: 'na' + DevOpsPath: 'na' + InternalVariableGroups: '' + GenerateUnifiedWeekly: '' + TestVariableGroups: '' + + # Standard set of variable groups for the Azure SDK repositories + AzureSDK_Maven_Release_Pipeline_Secrets: 1 + AzureSDK_Nuget_Release_Pipeline_Secrets: 13 + AzureSDK_CocoaPods_Release_Pipeline_Secrets: 98 + NPM_Registry_Authentication: 24 + Release_Secrets_for_GitHub: 58 + Secrets_for_Resource_Provisioner: 64 + APIReview_AutoCreate_Configurations: 93 + steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: + Repositories: + - Name: ${{ parameters.Repository }} + WorkingDirectory: $(System.DefaultWorkingDirectory) Paths: - 'sdk/**/*.yml' - '!sdk/**/test-recordings/*' @@ -33,183 +45,182 @@ jobs: - '!sdk/**/session-records/*' - template: /eng/common/pipelines/templates/steps/install-pipeline-generation.yml - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - # This covers our public repos. - - ${{ if not(endsWith(parameters.Repository, '-pr'))}}: - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project public - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention ci - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.CIConventionOptions}} - displayName: Create CI Pipelines for Public Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention up - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.UPConventionOptions}} - displayName: Create UP Pipelines for Public Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention tests - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.TestsConventionOptions}} - displayName: Create Live Test Pipelines for Public Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention testsweekly - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.TestsConventionOptions}} - displayName: Create Weekly (Multi-Cloud) Live Test Pipelines for Public Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}} - --devopspath "\${{parameters.Prefix}}" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention upweekly - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - ${{parameters.UPConventionOptions}} - displayName: Create Weekly (Multi-Cloud) Unified Test Pipelines for Public Repository - condition: and(succeeded(), eq(${{parameters.GenerateUnifiedWeekly}},true)) - # This covers our -pr repositories. - - ${{ if endsWith(parameters.Repository, '-pr')}}: - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention ci - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.CIConventionOptions}} - displayName: Create CI Pipelines for Private Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention up - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.UPConventionOptions}} - displayName: Create UP Pipelines for Private Repository - - task: AzureCLI@2 - inputs: - azureSubscription: 'opensource-api-connection' - scriptType: pscore - scriptLocation: inlineScript - inlineScript: > - $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate - --organization azure-sdk - --project internal - --prefix ${{parameters.Prefix}}-pr - --devopspath "\${{parameters.Prefix}}\pr" - --path $(System.DefaultWorkingDirectory)/sdk - --endpoint Azure - --repository ${{parameters.Repository}} - --convention tests - --agentpool Hosted - --branch refs/heads/$(DefaultBranch) - --set-managed-variables - --debug - --no-schedule - ${{parameters.TestsConventionOptions}} - displayName: Create Live Test Pipelines for Private Repository - condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) + - pwsh: | + Write-Host "Setting up pipeline variables" + if ("${{ parameters.Repository }}" -match "Azure/azure-sdk-for-(?[^-]*)(?-pr)?") { + $prefix = $matches['prefix'] + $devOpsPath = "\$prefix" + if ($matches['pr']) { + $devOpsPath = "${devOpsPath}\pr" + $prefix = "${prefix}-pr" + Write-Host "##vso[task.setvariable variable=ProjectForPRValidation]internal" + } + Write-Host "Prefix = $prefix" + Write-Host "DevOpsPath = $devOpsPath" + Write-Host "##vso[task.setvariable variable=Prefix]$prefix" + Write-Host "##vso[task.setvariable variable=DevOpsPath]$devOpsPath" + } + else { + Write-Error "Repository name '${{ parameters.Repository }}' is not in the expected format." + exit 1 + } + + # Map the language prefix to the appropriate variable groups + switch ($prefix) + { + "java" { + $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "android" { + $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + } + "js" { + $internalVariableGroups = '$(NPM_Registry_Authentication) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "python" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "net" { + $internalVariableGroups = '$(AzureSDK_Nuget_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)' + $testVariableGroups = '$(Secrets_for_Resource_Provisioner)' + } + "cpp" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)' + } + "ios" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner) $(AzureSDK_CocoaPods_Release_Pipeline_Secrets)' + } + "go" { + $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)' + $generateUnifiedWeekly = 'true' + } + default { + Write-Error "Prefix '$prefix' is not recognized." + exit 1 + } + } + + Write-Host "InternalVariableGroups = $internalVariableGroups" + Write-Host "TestVariableGroups = $testVariableGroups" + Write-Host "GenerateUnifiedWeekly = $generateUnifiedWeekly" + Write-Host "##vso[task.setvariable variable=InternalVariableGroups]$internalVariableGroups" + Write-Host "##vso[task.setvariable variable=TestVariableGroups]$testVariableGroups" + Write-Host "##vso[task.setvariable variable=GenerateUnifiedWeekly]$generateUnifiedWeekly" + displayName: Setup pipeline generation variables + + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project $(ProjectForPRValidation) + --prefix $(Prefix) + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention ci + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + displayName: 'Generate ci pipelines for: ${{ parameters.Repository }}' + continueOnError: true + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention up + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(InternalVariableGroups) $(TestVariableGroups) + displayName: 'Generate release pipelines for: ${{ parameters.Repository }}' + continueOnError: true + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention tests + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(TestVariableGroups) + displayName: 'Generate test pipelines for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + continueOnError: true + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention testsweekly + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(TestVariableGroups) + displayName: 'Generate weekly test pipelines (multi-cloud) for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['TestVariableGroups'],'')) + continueOnError: true + - task: AzureCLI@2 + inputs: + azureSubscription: 'opensource-api-connection' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate + --organization azure-sdk + --project internal + --prefix $(Prefix) + --devopspath "$(DevOpsPath)" + --path $(RepositoryPath) + --endpoint Azure + --repository ${{ parameters.Repository }} + --convention upweekly + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --set-managed-variables + --debug + --variablegroups $(InternalVariableGroups) $(TestVariableGroups) + displayName: 'Generate weekly unified test pipelines (multi-cloud) for: ${{ parameters.Repository }}' + condition: and(succeeded(), ne(variables['GenerateUnifiedWeekly'],'')) + continueOnError: true