diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8bbc4efd790d..fccce7ef4df5 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -82,7 +82,33 @@ jobs: parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} - - template: ../steps/build-artifacts.yml + - template: ../steps/build-package-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + + - job: 'Build_Extended' + displayName: Build Extended + dependsOn: + - 'Build' + timeoutInMinutes: 90 + variables: + Codeql.Enabled: true + Codeql.BuildIdentifier: ${{ parameters.ServiceDirectory }} + Codeql.SkipTaskAutoInjection: false + + pool: + name: azsdk-pool-mms-ubuntu-2004-general + vmImage: MMSUbuntu20.04 + + steps: + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + + - template: ../steps/build-extended-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} @@ -102,7 +128,7 @@ jobs: timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} dependsOn: - - 'Build' + - 'Build_Extended' pool: name: azsdk-pool-mms-ubuntu-2004-general diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index ebf2259401fd..673991678a5d 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -120,7 +120,7 @@ stages: Artifacts: ${{ parameters.Artifacts }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true - ArtifactName: packages + ArtifactName: packages_extended DocArtifact: documentation TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} TargetDocRepoName: ${{ parameters.TargetDocRepoName }} diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index f813dd1c5c56..b2088d37ffa0 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -113,7 +113,7 @@ steps: - task: DownloadPipelineArtifact@2 condition: succeededOrFailed() inputs: - artifactName: 'packages' + artifactName: 'packages_extended' targetPath: $(Build.ArtifactStagingDirectory) - template: ../steps/run_bandit.yml @@ -137,10 +137,12 @@ steps: parameters: Artifacts: ${{ parameters.Artifacts }} GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} + ArtifactName: "packages_extended" - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml parameters: Artifacts: ${{ parameters.Artifacts }} + ArtifactName: "packages_extended" - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml new file mode 100644 index 000000000000..e65533eb5ce2 --- /dev/null +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -0,0 +1,105 @@ +parameters: + - name: BeforePublishSteps + type: object + default: [] + - name: TestPipeline + type: boolean + default: false + - name: ServiceDirectory + type: string + default: '' + - name: BuildDocs + type: boolean + default: true + - name: Artifacts + type: object + default: [] + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' +# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: +# eng/pipelines/templates/jobs/ci.tests.yml +# eng/pipelines/templates/jobs/ci.yml +# eng/pipelines/templates/jobs/live.test.yml + +# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages' + targetPath: $(Build.ArtifactStagingDirectory) + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + + - script: | + echo "##vso[build.addbuildtag]Scheduled" + displayName: 'Tag scheduled builds' + condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) + + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - script: | + python -m pip install setuptools==58.3.0 + python -m pip install -r eng/ci_tools.txt + displayName: 'Prep Environment' + + - template: set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - template: auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - task: PythonScript@0 + displayName: 'Generate Docs' + condition: and(succeededOrFailed(), ${{parameters.BuildDocs}}) + inputs: + scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' + arguments: >- + "$(TargetingString)" + --service="${{ parameters.ServiceDirectory }}" + --toxenv=sphinx + + - pwsh: | + Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" + displayName: Reset PIP Index For APIStubGen + + - template: /eng/pipelines/templates/steps/run_apistub.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + + - ${{ parameters.BeforePublishSteps }} + + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'packages_extended' + + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.SourcesDirectory)/_docs' + CustomCondition: ${{ parameters.BuildDocs }} + ArtifactName: 'documentation' + + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate BOM' + condition: succeededOrFailed() + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' + ArtifactName: 'extended_manifest' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml similarity index 77% rename from eng/pipelines/templates/steps/build-artifacts.yml rename to eng/pipelines/templates/steps/build-package-artifacts.yml index 10fba4632273..e536c5f89be0 100644 --- a/eng/pipelines/templates/steps/build-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -8,9 +8,6 @@ parameters: - name: ServiceDirectory type: string default: '' - - name: BuildDocs - type: boolean - default: true - name: Artifacts type: object default: [] @@ -47,7 +44,7 @@ steps: python -m pip install -r eng/ci_tools.txt displayName: 'Prep Environment' - - template: ../steps/set-dev-build.yml + - template: set-dev-build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -74,28 +71,10 @@ steps: displayName: 'Verify Readme' - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - template: ../steps/auth-dev-feed.yml + - template: auth-dev-feed.yml parameters: DevFeedName: ${{ parameters.DevFeedName }} - - task: PythonScript@0 - displayName: 'Generate Docs' - condition: and(succeededOrFailed(), ${{parameters.BuildDocs}}) - inputs: - scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' - arguments: >- - "$(TargetingString)" - --service="${{ parameters.ServiceDirectory }}" - --toxenv=sphinx - - - pwsh: | - Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" - displayName: Reset PIP Index For APIStubGen - - - template: /eng/pipelines/templates/steps/run_apistub.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - ${{ parameters.BeforePublishSteps }} - template: /eng/common/pipelines/templates/steps/publish-artifact.yml @@ -103,12 +82,6 @@ steps: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages' - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/_docs' - CustomCondition: ${{ parameters.BuildDocs }} - ArtifactName: 'documentation' - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/.logs' @@ -124,4 +97,4 @@ steps: - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' - ArtifactName: 'manifest' \ No newline at end of file + ArtifactName: 'packages_manifest' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/create-apireview.yml b/eng/pipelines/templates/steps/create-apireview.yml index e3637470a7f2..c8715cd50d56 100644 --- a/eng/pipelines/templates/steps/create-apireview.yml +++ b/eng/pipelines/templates/steps/create-apireview.yml @@ -5,6 +5,7 @@ parameters: Language: 'Python' APIViewUri: 'https://apiview.dev/AutoReview/CreateApiReview' GenerateApiReviewForManualOnly: false + ArtifactName: 'packages_extended' steps: # ideally this should be done as initial step of a job in caller template @@ -28,6 +29,7 @@ steps: -BuildId $(Build.BuildId) -RepoName $(Build.Repository.Name) -Language ${{ parameters.Language }} + -ArtifactName ${{ parameters.ArtifactName }} pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Create Automatic API Review diff --git a/eng/scripts/Create-ApiReview.ps1 b/eng/scripts/Create-ApiReview.ps1 index 5a48ffc4a0a4..7c133e4dce60 100644 --- a/eng/scripts/Create-ApiReview.ps1 +++ b/eng/scripts/Create-ApiReview.ps1 @@ -19,7 +19,9 @@ Param ( [Parameter(Mandatory=$True)] [string] $repoName, [Parameter(Mandatory=$True)] - [string] $Language + [string] $Language, + [Parameter(Mandatory=$False)] + [string] $ArtifactName = "packages" ) Set-StrictMode -Version 3 @@ -28,7 +30,7 @@ Set-StrictMode -Version 3 # Submit API review request and return status whether current revision is approved or pending or failed to create review function Submit-APIReview($packageArtifactname, $apiLabel, $releaseStatus, $reviewFileName) { - $params = "buildId=$buildId&artifactName=packages&originalFilePath=$packageArtifactname&reviewFilePath=$reviewFileName" + $params = "buildId=$buildId&artifactName=$ArtifactName&originalFilePath=$packageArtifactname&reviewFilePath=$reviewFileName" $params += "&label=$apiLabel&repoName=$repoName&packageName=$PackageName&project=internal" $uri = "$($APIViewUri)?$params"