From 331da63f61cbfb17af494ad3793e7770b26256cc Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Fri, 11 Oct 2024 15:12:03 -0700 Subject: [PATCH 1/4] Changes required for sdk-repo based emitter pipelines --- .../stages/archetype-typespec-emitter.yml | 452 ++++++++++++++++++ .../steps/create-authenticated-npmrc.yml | 23 + .../Helpers/CommandInvocation-Helpers.ps1 | 26 +- eng/common/scripts/New-RegenerateMatrix.ps1 | 11 +- eng/common/scripts/common.ps1 | 1 + .../typespec/New-EmitterPackageJson.ps1 | 6 +- 6 files changed, 510 insertions(+), 9 deletions(-) create mode 100644 eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml create mode 100644 eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml new file mode 100644 index 000000000000..f50b94688cff --- /dev/null +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -0,0 +1,452 @@ +parameters: +# Path to the emitter package. This is used as the base path for script invocations. +- name: EmitterPackagePath + type: string + +# Pool to use for the pipeline stages +- name: Pool + type: object + +# Whether to build alpha versions of the packages. This is passed as a flag to the build script. +- name: BuildPrereleaseVersion + type: boolean + default: true + +# Whether to use the `next` version of TypeSpec. This is passed as a flag to the initialize script. +- name: UseTypeSpecNext + type: boolean + default: false + +# Custom steps to run after the repository is cloned but before other job steps. +- name: InitializationSteps + type: stepList + default: [] + +# Indicates the build matrix to use for post-build autorest validation +- name: TestMatrix + type: object + default: {} + +# Whether to run the publish and regeneration stages. If false, only the build stage will run. +- name: ShouldPublish + type: boolean + default: false + +# List of packages to publish. Each package is an object with the following properties: +# name: The name of the package. This is used to determine the name of the file to publish. +# type: The type of package. Currently supported values are 'npm' and 'nuget'. +# file: The path to the file to publish. This is relative to the packages directory in the build artifacts directory. +- name: Packages + type: object + default: [] + +# Whether to publish to the internal feed. +- name: PublishInternal + type: boolean + default: true + +# Indicates if the Publish stage should depend on the Test stage +- name: PublishDependsOnTest + type: boolean + default: false + +# Whether to publish to the internal feed. +- name: ShouldRegenrate + type: boolean + default: false + +# Number of jobs to generate. This is the maximum number of jobs that will be generated. The actual number of jobs will be reduced if it would result in fewer than MinimumPerJob packages per job. +- name: RegenerationJobCount + type: number + default: 10 + +# Minimum number of packages to generate per job. +- name: MinimumPerJob + type: number + default: 10 + +# Indicates if regenration matrix should only contain folders with typespec files +- name: OnlyGenerateTypespec + type: boolean + default: false + +stages: + +# Build stage +# Responsible for building the autorest generator and typespec emitter packages +# Produces the artifact `build_artifacts` which contains the following: +# package-versions.json: Contains a map of package name to version for the packages that were built +# overrides.json: Contains npm package version overrides for the emitter and generator +# packages/: Contains the packages to publish +# lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions +- stage: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Build + steps: + # Validate parameters and fail early if invalid + - ${{ each package in parameters.Packages }}: + - ${{ if notIn(package.type, 'npm', 'nuget') }}: + - script: | + echo "Package ${{ package.name }} has unsupported type: ${{ package.type }}" + exit 1 + displayName: 'Unsupported package type' + condition: always() + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + workingDirectory: ${{ parameters.EmitterPackagePath }} + + - task: PowerShell@2 + displayName: 'Run build script' + name: ci_build + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Build-Packages.ps1 + arguments: > + -BuildNumber "$(Build.BuildNumber)" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -PublishInternal:$${{ parameters.PublishInternal }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} + + - pwsh: | + $sourceBranch = '$(Build.SourceBranch)' + $buildReason = '$(Build.Reason)' + $buildNumber = '$(Build.BuildNumber)' + + if ($buildReason -eq 'Schedule') { + $branchName = 'validate-typespec-scheduled' + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $branchName = "validate-typespec-pr-$($Matches[1])" + } else { + $branchName = "validate-typespec-$buildNumber" + } + + Write-Host "Setting variable 'branchName' to '$branchName'" + Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" + displayName: Set branch name + name: set_branch_name + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + +# Publish stage +# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` +# Produces the artifact `publish_artifacts` which contains the following: +# emitter-package.json: Created using the package json from the build step. +# emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` +- ${{ if parameters.ShouldPublish }}: + - stage: Publish + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + pool: ${{ parameters.Pool }} + jobs: + - job: Publish + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing + - ${{ if eq(parameters.PublishInternal, 'true') }}: + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + - ${{ else }}: + - pwsh: | + "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' + displayName: Authenticate .npmrc for npmjs.org + workingDirectory: $(buildArtifactsPath)/packages + + # per package, publishing using appropriate tool + - ${{ each package in parameters.Packages }}: + - ${{ if eq(package.type, 'npm') }}: + - pwsh: | + $file = Resolve-Path "${{ package.file }}" + Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages" + npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages + displayName: Publish ${{ package.name }} + workingDirectory: $(buildArtifactsPath)/packages + - ${{ elseif eq(package.type, 'nuget') }}: + - task: NuGetCommand@2 + displayName: Publish ${{ package.name }} + inputs: + command: 'push' + packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} + # Nuget packages are always published to the same internal feed https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net + nuGetFeedType: 'internal' + publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' + -OverridesPath '$(buildArtifactsPath)/overrides.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + arguments: > + -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + +# Regenerate stage +# Responsible for regenerating the SDK code using the emitter package and the generation matrix. +- ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: + - stage: Regenerate + dependsOn: + - Build + - Publish + variables: + pullRequestTargetBranch: 'main' + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] + pool: ${{ parameters.Pool }} + jobs: + - job: Initialize + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - pwsh: | + Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force + + Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force + displayName: Copy emitter-package json files + + - ${{ parameters.InitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch + PushArgs: --force + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + workingDirectory: $(Build.SourcesDirectory) + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $emitterVersion = '$(emitterVersion)' + $collectionUri = '$(System.CollectionUri)' + $project = '$(System.TeamProject)' + $definitionName = '$(Build.DefinitionName)' + $repoUrl = '$(Build.Repository.Uri)' + $repoName = '$(Build.Repository.Name)' + $sourceBranch = '$(Build.SourceBranch)' + $reason = '$(Build.Reason)' + $buildId = '$(Build.BuildId)' + $buildNumber = '$(Build.BuildNumber)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" + + if ($sourceBranch -match "^refs/heads/(.+)$") { + $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" + } else { + $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" + } + + if ($reason -eq 'Schedule') { + $prTitle = "Scheduled code regeneration test" + } else { + if ($preRelease) { + $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + } else { + $prTitle = "Update typespec emitter version to $emitterVersion" + } + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + + $repoNameParts = $repoName.Split('/') + if ($repoNameParts.Length -eq 2) { + Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" + Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" + + Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" + Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" + } else { + Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" + } + displayName: Get PR title and body + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: ./eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner '$(RepoOwner)' + -RepoName '$(RepoName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' + workingDirectory: $(Build.SourcesDirectory) + +# Test stage +# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. +# Will only run if the parameter `TestMatrix` is not empty. +# The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. +- ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Test-Packages.ps1 + arguments: > + $(TestArguments) + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml new file mode 100644 index 000000000000..4b6f08359bd9 --- /dev/null +++ b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml @@ -0,0 +1,23 @@ +parameters: + - name: npmrcPath + type: string + - name: registryUrl + type: string + +steps: +- pwsh: | + Write-Host "Creating .npmrc file ${{ parameters.npmrcPath }} for registry ${{ parameters.registryUrl }}" + $parentFolder = Split-Path -Path '${{ parameters.npmrcPath }}' -Parent + + if (!(Test-Path $parentFolder)) { + Write-Host "Creating folder $parentFolder" + New-Item -Path $parentFolder -ItemType Directory | Out-Null + } + + $content = "registry=${{ parameters.registryUrl }}`n`nalways-auth=true" + $content | Out-File '${{ parameters.npmrcPath }}' + displayName: 'Create .npmrc' +- task: npmAuthenticate@0 + displayName: Authenticate .npmrc + inputs: + workingFile: ${{ parameters.npmrcPath }} diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 index 5dc0c8c7da1a..0b9f810b83aa 100644 --- a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 +++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 @@ -1,5 +1,14 @@ -function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) +function Invoke-LoggedCommand { + [CmdletBinding()] + param + ( + [string] $Command, + [string] $ExecutePath, + [switch] $GroupOutput, + [int[]] $AllowedExitCodes = @(0) + ) + $pipelineBuild = !!$env:TF_BUILD $startTime = Get-Date @@ -22,7 +31,7 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) Write-Host "##[endgroup]" } - if($LastExitCode -ne 0) + if($LastExitCode -notin $AllowedExitCodes) { if($pipelineBuild) { Write-Error "##[error]Command failed to execute ($duration): $Command`n" @@ -40,3 +49,16 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) } } } + +function Set-ConsoleEncoding +{ + [CmdletBinding()] + param + ( + [string] $Encoding = 'utf-8' + ) + + $outputEncoding = [System.Text.Encoding]::GetEncoding($Encoding) + [Console]::OutputEncoding = $outputEncoding + [Console]::InputEncoding = $outputEncoding +} diff --git a/eng/common/scripts/New-RegenerateMatrix.ps1 b/eng/common/scripts/New-RegenerateMatrix.ps1 index 1df97420c25d..9176c23e4f53 100644 --- a/eng/common/scripts/New-RegenerateMatrix.ps1 +++ b/eng/common/scripts/New-RegenerateMatrix.ps1 @@ -14,7 +14,7 @@ param ( [int]$MinimumPerJob = 10, [Parameter()] - [string]$OnlyTypespec + [string]$OnlyTypeSpec ) . (Join-Path $PSScriptRoot common.ps1) @@ -57,14 +57,13 @@ function Split-Items([array]$Items) { New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null if (Test-Path "Function:$GetDirectoriesForGenerationFn") { - $directoriesForGeneration = &$GetDirectoriesForGenerationFn + $directoriesForGeneration = &$GetDirectoriesForGenerationFn -OnlyTypeSpec $OnlyTypespec } else { $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory -} - -if ($OnlyTypespec) { - $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + if ($OnlyTypespec) { + $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + } } [array]$packageDirectories = $directoriesForGeneration diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 831b4719f88a..a8c38d0a0126 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -16,6 +16,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts" . (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" git-helpers.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" Package-Helpers.ps1) +. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1) # Setting expected from common languages settings $Language = "Unknown" diff --git a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 index d9e45038b879..24c6c50a9668 100644 --- a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 +++ b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 @@ -1,3 +1,4 @@ +#Requires -Version 7.0 [CmdletBinding()] param ( [parameter(Mandatory = $true)] @@ -35,9 +36,12 @@ if ($OverridesPath) { $devDependencies = [ordered]@{} -foreach ($package in $packageJson.peerDependencies.Keys | Sort-Object) { +$possiblyPinnedPackages = $packageJson['azure-sdk/emitter-package-json-pinning'] ?? $packageJson.peerDependencies.Keys; + +foreach ($package in $possiblyPinnedPackages | Sort-Object) { $pinnedVersion = $packageJson.devDependencies[$package] if ($pinnedVersion -and -not $overrides[$package]) { + #We have a dev pinned version that isn't overridden by the overrides.json file Write-Host "Pinning $package to $pinnedVersion" $devDependencies[$package] = $pinnedVersion } From a128610bc26280d26eb26b4835ea3f932b87a1c2 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Tue, 15 Oct 2024 14:32:18 -0700 Subject: [PATCH 2/4] Rename template --- .../stages/archetype-typespec-emitter.yml | 82 +++++++++---------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml index f50b94688cff..c3e6bc3ac86e 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -40,8 +40,8 @@ parameters: type: object default: [] -# Whether to publish to the internal feed. -- name: PublishInternal +# Whether to publish to npmjs.org. +- name: PublishPublic type: boolean default: true @@ -50,7 +50,7 @@ parameters: type: boolean default: false -# Whether to publish to the internal feed. +# Whether to regenerate sdk clients using the new emitter. - name: ShouldRegenrate type: boolean default: false @@ -84,15 +84,6 @@ stages: jobs: - job: Build steps: - # Validate parameters and fail early if invalid - - ${{ each package in parameters.Packages }}: - - ${{ if notIn(package.type, 'npm', 'nuget') }}: - - script: | - echo "Package ${{ package.name }} has unsupported type: ${{ package.type }}" - exit 1 - displayName: 'Unsupported package type' - condition: always() - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - ${{ parameters.InitializationSteps }} @@ -114,7 +105,7 @@ stages: arguments: > -BuildNumber "$(Build.BuildNumber)" -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -PublishInternal:$${{ parameters.PublishInternal }} + -TargetNpmJsFeed:$${{ parameters.PublishPublic }} -Prerelease:$${{ parameters.BuildPrereleaseVersion }} - pwsh: | @@ -163,36 +154,41 @@ stages: artifact: build_artifacts displayName: Download build artifacts - # Create authenticated .npmrc file for publishing - - ${{ if eq(parameters.PublishInternal, 'true') }}: - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - parameters: - npmrcPath: $(buildArtifactsPath)/packages/.npmrc - registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ - - ${{ else }}: - - pwsh: | - "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' - displayName: Authenticate .npmrc for npmjs.org - workingDirectory: $(buildArtifactsPath)/packages - - # per package, publishing using appropriate tool - - ${{ each package in parameters.Packages }}: - - ${{ if eq(package.type, 'npm') }}: - - pwsh: | - $file = Resolve-Path "${{ package.file }}" - Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages" - npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages - displayName: Publish ${{ package.name }} - workingDirectory: $(buildArtifactsPath)/packages - - ${{ elseif eq(package.type, 'nuget') }}: - - task: NuGetCommand@2 - displayName: Publish ${{ package.name }} - inputs: - command: 'push' - packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} - # Nuget packages are always published to the same internal feed https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net - nuGetFeedType: 'internal' - publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + # Create authenticated .npmrc file for publishing to devops + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + + # publish to devops feed + - pwsh: | + $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' + foreach ($file in $packageFiles.Name) { + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + } + displayName: Publish npm to Azure DevOps feed + workingDirectory: $(buildArtifactsPath)/packages + + - ${{ if parameters.PublishPublic }}: + # publish to npmjs.org using ESRP + - task: EsrpRelease@7 + inputs: + displayName: 'Publish to npmjs.org' + ConnectedServiceName: 'Azure SDK Engineering System' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + KeyVaultName: 'AzureSDKEngKeyVault' + AuthCertName: 'azure-sdk-esrp-release-auth-certificate' + SignCertName: 'azure-sdk-esrp-release-sign-certificate' + Intent: 'PackageDistribution' + ContentType: 'npm' + FolderLocation: $(buildArtifactsPath)/packages + Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMANTEST' + DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + productstate: ${{parameters.Tag}} - task: PowerShell@2 displayName: Create emitter-package.json From acb0ae4280c7a22eed8f867fc5f638a8a04ba16d Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 16 Oct 2024 13:22:53 -0700 Subject: [PATCH 3/4] Convert emitter archetype to pipeline template --- .../stages/archetype-typespec-emitter.yml | 745 +++++++++--------- 1 file changed, 369 insertions(+), 376 deletions(-) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml index c3e6bc3ac86e..99848e9eedc6 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -1,11 +1,11 @@ parameters: -# Path to the emitter package. This is used as the base path for script invocations. -- name: EmitterPackagePath - type: string - # Pool to use for the pipeline stages - name: Pool type: object + default: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux # Whether to build alpha versions of the packages. This is passed as a flag to the build script. - name: BuildPrereleaseVersion @@ -32,14 +32,6 @@ parameters: type: boolean default: false -# List of packages to publish. Each package is an object with the following properties: -# name: The name of the package. This is used to determine the name of the file to publish. -# type: The type of package. Currently supported values are 'npm' and 'nuget'. -# file: The path to the file to publish. This is relative to the packages directory in the build artifacts directory. -- name: Packages - type: object - default: [] - # Whether to publish to npmjs.org. - name: PublishPublic type: boolean @@ -70,379 +62,380 @@ parameters: type: boolean default: false -stages: - -# Build stage -# Responsible for building the autorest generator and typespec emitter packages -# Produces the artifact `build_artifacts` which contains the following: -# package-versions.json: Contains a map of package name to version for the packages that were built -# overrides.json: Contains npm package version overrides for the emitter and generator -# packages/: Contains the packages to publish -# lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions -- stage: Build - pool: ${{ parameters.Pool }} - jobs: - - job: Build - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: 'Run initialize script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 - arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - workingDirectory: ${{ parameters.EmitterPackagePath }} - - - task: PowerShell@2 - displayName: 'Run build script' - name: ci_build - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Build-Packages.ps1 - arguments: > - -BuildNumber "$(Build.BuildNumber)" - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -TargetNpmJsFeed:$${{ parameters.PublishPublic }} - -Prerelease:$${{ parameters.BuildPrereleaseVersion }} - - - pwsh: | - $sourceBranch = '$(Build.SourceBranch)' - $buildReason = '$(Build.Reason)' - $buildNumber = '$(Build.BuildNumber)' - - if ($buildReason -eq 'Schedule') { - $branchName = 'validate-typespec-scheduled' - } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { - $branchName = "validate-typespec-pr-$($Matches[1])" - } else { - $branchName = "validate-typespec-$buildNumber" - } - - Write-Host "Setting variable 'branchName' to '$branchName'" - Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" - displayName: Set branch name - name: set_branch_name - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: build_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - -# Publish stage -# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` -# Produces the artifact `publish_artifacts` which contains the following: -# emitter-package.json: Created using the package json from the build step. -# emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` -- ${{ if parameters.ShouldPublish }}: - - stage: Publish - dependsOn: - - Build - - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: - - Test - variables: - buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts - pool: ${{ parameters.Pool }} - jobs: - - job: Publish - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - artifact: build_artifacts - displayName: Download build artifacts - - # Create authenticated .npmrc file for publishing to devops - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - parameters: - npmrcPath: $(buildArtifactsPath)/packages/.npmrc - registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ - - # publish to devops feed - - pwsh: | - $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' - foreach ($file in $packageFiles.Name) { - Write-Host "npm publish $file --verbose --access public" - npm publish $file --verbose --access public - } - displayName: Publish npm to Azure DevOps feed - workingDirectory: $(buildArtifactsPath)/packages - - - ${{ if parameters.PublishPublic }}: - # publish to npmjs.org using ESRP - - task: EsrpRelease@7 - inputs: - displayName: 'Publish to npmjs.org' - ConnectedServiceName: 'Azure SDK Engineering System' - ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' - KeyVaultName: 'AzureSDKEngKeyVault' - AuthCertName: 'azure-sdk-esrp-release-auth-certificate' - SignCertName: 'azure-sdk-esrp-release-sign-certificate' - Intent: 'PackageDistribution' - ContentType: 'npm' - FolderLocation: $(buildArtifactsPath)/packages - Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMANTEST' - DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - productstate: ${{parameters.Tag}} +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + + # Build stage + # Responsible for building the autorest generator and typespec emitter packages + # Produces the artifact `build_artifacts` which contains the following: + # package-versions.json: Contains a map of package name to version for the packages that were built + # overrides.json: Contains npm package version overrides for the emitter and generator + # packages/: Contains the packages to publish + # lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions + - stage: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Build + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - ${{ parameters.InitializationSteps }} - task: PowerShell@2 - displayName: Create emitter-package.json + displayName: 'Run initialize script' inputs: pwsh: true - filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 - arguments: > - -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' - -OverridesPath '$(buildArtifactsPath)/overrides.json' - -OutputDirectory '$(Build.ArtifactStagingDirectory)' - workingDirectory: $(Build.SourcesDirectory) + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - task: PowerShell@2 - displayName: Create emitter-package-lock.json + displayName: 'Run build script' + name: ci_build inputs: pwsh: true - filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Build-Emitter.ps1 arguments: > - -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' - -OutputDirectory '$(Build.ArtifactStagingDirectory)' - workingDirectory: $(Build.SourcesDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: publish_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - -# Regenerate stage -# Responsible for regenerating the SDK code using the emitter package and the generation matrix. -- ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: - - stage: Regenerate - dependsOn: - - Build - - Publish - variables: - pullRequestTargetBranch: 'main' - publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts - branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] - pool: ${{ parameters.Pool }} - jobs: - - job: Initialize - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - "/*" - - "!SessionRecords" - - - download: current - displayName: Download pipeline artifacts - - - pwsh: | - Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" - Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force - - Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" - Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force - displayName: Copy emitter-package json files - - - ${{ parameters.InitializationSteps }} - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoOwner: azure-sdk - TargetRepoName: $(Build.Repository.Name) - BaseRepoBranch: $(branchName) - CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) - WorkingDirectory: $(Build.SourcesDirectory) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch - PushArgs: --force - - - task: PowerShell@2 - displayName: Get generation job matrix - name: generate_matrix - inputs: - pwsh: true - filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 - arguments: > - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -OutputVariableName matrix - -JobCount ${{ parameters.RegenerationJobCount }} - -MinimumPerJob ${{ parameters.MinimumPerJob }} - -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} - workingDirectory: $(Build.SourcesDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: matrix_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - - - job: Generate - dependsOn: Initialize - strategy: - matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] - variables: - matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - "/*" - - "!SessionRecords" - - - download: current - displayName: Download pipeline artifacts - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: Call regeneration script - inputs: - pwsh: true - filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 - arguments: > - -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" - workingDirectory: $(Build.SourcesDirectory) - continueOnError: true - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoOwner: azure-sdk - TargetRepoName: $(Build.Repository.Name) - BaseRepoBranch: $(branchName) - CommitMsg: Update SDK code $(JobKey) - WorkingDirectory: $(Build.SourcesDirectory) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - - - job: Create_PR - displayName: Create PR - dependsOn: - - Generate - variables: - generateJobResult: $[dependencies.Generate.result] - emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - pwsh: | - $generateJobResult = '$(generateJobResult)' - $emitterVersion = '$(emitterVersion)' - $collectionUri = '$(System.CollectionUri)' - $project = '$(System.TeamProject)' - $definitionName = '$(Build.DefinitionName)' - $repoUrl = '$(Build.Repository.Uri)' - $repoName = '$(Build.Repository.Name)' - $sourceBranch = '$(Build.SourceBranch)' - $reason = '$(Build.Reason)' - $buildId = '$(Build.BuildId)' - $buildNumber = '$(Build.BuildNumber)' - $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' - - $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" - - if ($sourceBranch -match "^refs/heads/(.+)$") { - $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" - } elseif ($sourceBranch -match "^refs/tags/(.+)$") { - $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" - } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { - $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" - } else { - $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" - } - - if ($reason -eq 'Schedule') { - $prTitle = "Scheduled code regeneration test" - } else { - if ($preRelease) { - $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + -BuildNumber "$(Build.BuildNumber)" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -TargetNpmJsFeed:$${{ parameters.PublishPublic }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} + + - pwsh: | + $sourceBranch = '$(Build.SourceBranch)' + $buildReason = '$(Build.Reason)' + $buildNumber = '$(Build.BuildNumber)' + + if ($buildReason -eq 'Schedule') { + $branchName = 'validate-typespec-scheduled' + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $branchName = "validate-typespec-pr-$($Matches[1])" } else { - $prTitle = "Update typespec emitter version to $emitterVersion" + $branchName = "validate-typespec-$buildNumber" } - if ($generateJobResult -ne 'Succeeded') { - $prTitle = "Failed: $prTitle" - } - } - - Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" - Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" - - Write-Host "Setting variable 'PullRequestBody' to '$prBody'" - Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" - - $repoNameParts = $repoName.Split('/') - if ($repoNameParts.Length -eq 2) { - Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" - Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" - - Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" - Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" - } else { - Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" - } - displayName: Get PR title and body - - - task: PowerShell@2 - displayName: Create pull request - inputs: - pwsh: true - filePath: ./eng/common/scripts/Submit-PullRequest.ps1 - arguments: > - -RepoOwner '$(RepoOwner)' - -RepoName '$(RepoName)' - -BaseBranch '$(pullRequestTargetBranch)' - -PROwner 'azure-sdk' - -PRBranch '$(branchName)' - -AuthToken '$(azuresdk-github-pat)' - -PRTitle '$(PullRequestTitle)' - -PRBody '$(PullRequestBody)' - -OpenAsDraft $true - -PRLabels 'Do Not Merge' - workingDirectory: $(Build.SourcesDirectory) - -# Test stage -# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. -# Will only run if the parameter `TestMatrix` is not empty. -# The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. -- ${{ if ne(length(parameters.TestMatrix), 0) }}: - - stage: Test - dependsOn: Build - pool: ${{ parameters.Pool }} - jobs: - - job: Test - strategy: - matrix: ${{ parameters.TestMatrix }} - variables: - buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - artifact: build_artifacts - displayName: Download build artifacts - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: 'Run initialize script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 - arguments: -BuildArtifactsPath '$(buildArtifactsPath)' - - - task: PowerShell@2 - displayName: 'Run test script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Test-Packages.ps1 - arguments: > - $(TestArguments) - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: test_artifacts_$(System.JobName) - artifactPath: $(Build.ArtifactStagingDirectory) + Write-Host "Setting variable 'branchName' to '$branchName'" + Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" + displayName: Set branch name + name: set_branch_name + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Publish stage + # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` + # Produces the artifact `publish_artifacts` which contains the following: + # emitter-package.json: Created using the package json from the build step. + # emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` + - ${{ if parameters.ShouldPublish }}: + - stage: Publish + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + pool: ${{ parameters.Pool }} + jobs: + - job: Publish + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing to devops + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + + # publish to devops feed + - pwsh: | + $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' + foreach ($file in $packageFiles.Name) { + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + } + displayName: Publish to DevOps feed + workingDirectory: $(buildArtifactsPath)/packages + + - ${{ if parameters.PublishPublic }}: + # publish to npmjs.org using ESRP + - task: EsrpRelease@7 + inputs: + displayName: Publish to npmjs.org + ConnectedServiceName: Azure SDK Engineering System + ClientId: 5f81938c-2544-4f1f-9251-dd9de5b8a81b + KeyVaultName: AzureSDKEngKeyVault + AuthCertName: azure-sdk-esrp-release-auth-certificate + SignCertName: azure-sdk-esrp-release-sign-certificate + Intent: PackageDistribution + ContentType: npm + FolderLocation: $(buildArtifactsPath)/packages + Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + ServiceEndpointUrl: https://api.esrp.microsoft.com + MainPublisher: ESRPRELPACMANTEST + DomainTenantId: 72f988bf-86f1-41af-91ab-2d7cd011db47 + + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' + -OverridesPath '$(buildArtifactsPath)/overrides.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + arguments: > + -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Regenerate stage + # Responsible for regenerating the SDK code using the emitter package and the generation matrix. + - ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: + - stage: Regenerate + dependsOn: + - Build + - Publish + variables: + pullRequestTargetBranch: 'main' + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] + pool: ${{ parameters.Pool }} + jobs: + - job: Initialize + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - pwsh: | + Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force + + Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force + displayName: Copy emitter-package json files + + - ${{ parameters.InitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch + PushArgs: --force + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + workingDirectory: $(Build.SourcesDirectory) + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $emitterVersion = '$(emitterVersion)' + $collectionUri = '$(System.CollectionUri)' + $project = '$(System.TeamProject)' + $definitionName = '$(Build.DefinitionName)' + $repoUrl = '$(Build.Repository.Uri)' + $repoName = '$(Build.Repository.Name)' + $sourceBranch = '$(Build.SourceBranch)' + $reason = '$(Build.Reason)' + $buildId = '$(Build.BuildId)' + $buildNumber = '$(Build.BuildNumber)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" + + if ($sourceBranch -match "^refs/heads/(.+)$") { + $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" + } else { + $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" + } + + if ($reason -eq 'Schedule') { + $prTitle = "Scheduled code regeneration test" + } else { + if ($preRelease) { + $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + } else { + $prTitle = "Update typespec emitter version to $emitterVersion" + } + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + + $repoNameParts = $repoName.Split('/') + if ($repoNameParts.Length -eq 2) { + Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" + Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" + + Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" + Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" + } else { + Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" + } + displayName: Get PR title and body + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: ./eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner '$(RepoOwner)' + -RepoName '$(RepoName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' + workingDirectory: $(Build.SourcesDirectory) + + # Test stage + # Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. + # Will only run if the parameter `TestMatrix` is not empty. + # The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. + - ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Test-Emitter.ps1 + arguments: > + $(TestArguments) + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) From 384b19a8b105684f2c9a3a87a2db095b85c4eeff Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 16 Oct 2024 13:26:46 -0700 Subject: [PATCH 4/4] Move the archetype template out of the stages folder --- .../templates/{stages => }/archetype-typespec-emitter.yml | 1 - 1 file changed, 1 deletion(-) rename eng/common/pipelines/templates/{stages => }/archetype-typespec-emitter.yml (99%) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/archetype-typespec-emitter.yml similarity index 99% rename from eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml rename to eng/common/pipelines/templates/archetype-typespec-emitter.yml index 99848e9eedc6..71d16c908fcf 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/archetype-typespec-emitter.yml @@ -66,7 +66,6 @@ extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: stages: - # Build stage # Responsible for building the autorest generator and typespec emitter packages # Produces the artifact `build_artifacts` which contains the following: