From a40371975c28fdf833059733c4b1908da309a5c6 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 17:38:10 -0700 Subject: [PATCH 1/8] fix attestation tests --- sdk/attestation/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/attestation/tests.yml b/sdk/attestation/tests.yml index 43e89bb4f1c8..747436285417 100644 --- a/sdk/attestation/tests.yml +++ b/sdk/attestation/tests.yml @@ -1,6 +1,6 @@ trigger: none -extend: +extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: attestation From 38880d2e89a06bb3bb32d4603106fe723fc55dd9 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 17:48:12 -0700 Subject: [PATCH 2/8] attempt to use 1es-redirect to abstract python-analyze-weekly.yml --- .../templates/stages/1es-redirect.yml | 45 +++++++++++++++++++ .../python-analyze-weekly-standalone.yml | 19 ++++++++ sdk/ai/tests.yml | 4 +- 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 eng/pipelines/templates/stages/1es-redirect.yml create mode 100644 eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml new file mode 100644 index 000000000000..b57cdfec9565 --- /dev/null +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -0,0 +1,45 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: azure-sdk-build-tools + type: git + name: internal/azure-sdk-build-tools + ref: refs/tags/azure-sdk-build-tools_20230829.1 + +parameters: +- name: stages + type: stageList + default: [] +- name: Use1ESOfficial + type: boolean + default: true + +extends: + ${{ if and(parameters.Use1ESOfficial, eq(variables['System.TeamProject'], 'internal')) }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + sourceRepositoriesToScan: + exclude: + - repository: azure-sdk-build-tools + credscan: + suppressionsFile: '$(Build.SourcesDirectory)/eng/CredScanSuppression.json' + toolVersion: '2.3.12.23' + baselineFiles: $(Build.SourcesDirectory)/eng/python.gdnbaselines + psscriptanalyzer: + compiled: true + break: true + policy: M365 + + stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml new file mode 100644 index 000000000000..62177864fcbc --- /dev/null +++ b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml @@ -0,0 +1,19 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: BuildTargetingString + type: string + default: 'azure-*' + - name: JobName + type: string + default: 'Test' + +extends: /eng/pipelines/templates/stages/1es-redirect.yml +parameters: + stages: + - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml@self + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + JobName: ${{ parameters.JobName }} \ No newline at end of file diff --git a/sdk/ai/tests.yml b/sdk/ai/tests.yml index c6061bae0acc..960b65f98853 100644 --- a/sdk/ai/tests.yml +++ b/sdk/ai/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: ai From 0db053beb97fb00280a838d3a722e910601f671d Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 17:51:23 -0700 Subject: [PATCH 3/8] can I place a template reference in a parameter construct? --- .../stages/python-analyze-weekly-standalone.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml index 62177864fcbc..7b4e82127b55 100644 --- a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml +++ b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml @@ -9,11 +9,12 @@ parameters: type: string default: 'Test' -extends: /eng/pipelines/templates/stages/1es-redirect.yml -parameters: - stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml@self - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - JobName: ${{ parameters.JobName }} \ No newline at end of file +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml@self + parameters: + stages: + - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml@self + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + JobName: ${{ parameters.JobName }} \ No newline at end of file From 06367076f36cf5f8a7464ba7a9741bbec1655a3e Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 17:52:13 -0700 Subject: [PATCH 4/8] copy the template in --- .../python-analyze-weekly-standalone.yml | 106 +++++++++++++++++- 1 file changed, 101 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml index 7b4e82127b55..9990789a7270 100644 --- a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml +++ b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml @@ -13,8 +13,104 @@ extends: template: /eng/pipelines/templates/stages/1es-redirect.yml@self parameters: stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml@self - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - JobName: ${{ parameters.JobName }} \ No newline at end of file + - ${{ if contains(variables['Build.DefinitionName'], 'tests-weekly') }}: + - stage: + displayName: 'Analyze_${{ parameters.JobName }}' + dependsOn: [] + jobs: + - job: 'Analyze' + timeoutInMinutes: 90 + + variables: + - template: /eng/pipelines/templates/variables/image.yml + - name: PythonVersion + value: '3.8' + + pool: + name: $(LINUXPOOL) + vmImage: $(LINUXVMIMAGE) + os: linux + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.8' + inputs: + versionSpec: '3.8' + - script: | + python -m pip install setuptools==58.3.0 + python -m pip install -r eng/ci_tools.txt + displayName: 'Prep Environment' + - task: PythonScript@0 + displayName: 'Run Pylint Next' + continueOnError: true + inputs: + scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + arguments: >- + ${{ parameters.BuildTargetingString }} + --service="${{ parameters.ServiceDirectory }}" + --toxenv="next-pylint" + --disablecov + --filter-type="Omit_management" + env: + GH_TOKEN: $(azuresdk-github-pat) + + - task: PythonScript@0 + displayName: 'Run MyPy Next' + continueOnError: true + inputs: + scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + arguments: >- + ${{ parameters.BuildTargetingString }} + --service="${{ parameters.ServiceDirectory }}" + --toxenv="next-mypy" + --disablecov + env: + GH_TOKEN: $(azuresdk-github-pat) + + - task: PythonScript@0 + displayName: 'Run Pyright Next' + continueOnError: true + inputs: + scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + arguments: >- + ${{ parameters.BuildTargetingString }} + --service="${{ parameters.ServiceDirectory }}" + --toxenv="next-pyright" + --disablecov + env: + GH_TOKEN: $(azuresdk-github-pat) + + - task: PythonScript@0 + displayName: 'Run Ruff' + continueOnError: true + inputs: + scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + arguments: >- + ${{ parameters.BuildTargetingString }} + --service="${{ parameters.ServiceDirectory }}" + --toxenv="ruff" + --disablecov + env: + GH_TOKEN: $(azuresdk-github-pat) + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11 for docs generation' + inputs: + versionSpec: '3.11' + + - script: | + python -m pip install setuptools==58.3.0 + python -m pip install -r eng/ci_tools.txt + displayName: 'Prep Environment' + + - task: PythonScript@0 + displayName: 'Generate Docs' + continueOnError: true + inputs: + scriptPath: 'scripts/devops_tasks/dispatch_tox.py' + arguments: >- + ${{ parameters.BuildTargetingString }} + --service="${{ parameters.ServiceDirectory }}" + --toxenv="strict-sphinx" + env: + GH_TOKEN: $(azuresdk-github-pat) From f0777013397a44ec7e9be3b32262ec5c129a984e Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 18:04:46 -0700 Subject: [PATCH 5/8] correct indentation of stages --- eng/pipelines/templates/stages/1es-redirect.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index b57cdfec9565..9dc7afb73bef 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -41,5 +41,4 @@ extends: compiled: true break: true policy: M365 - - stages: ${{ parameters.stages }} \ No newline at end of file + stages: ${{ parameters.stages }} From ddf069dae67e563d3d6e4178ee7a907927ff4069 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 18:10:41 -0700 Subject: [PATCH 6/8] update how we reference the build pools --- .../templates/stages/python-analyze-weekly-standalone.yml | 5 +++-- eng/pipelines/templates/stages/python-analyze-weekly.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml index 9990789a7270..db0ef167764d 100644 --- a/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml +++ b/eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml @@ -16,19 +16,20 @@ extends: - ${{ if contains(variables['Build.DefinitionName'], 'tests-weekly') }}: - stage: displayName: 'Analyze_${{ parameters.JobName }}' + variables: + - template: /eng/pipelines/templates/variables/image.yml dependsOn: [] jobs: - job: 'Analyze' timeoutInMinutes: 90 variables: - - template: /eng/pipelines/templates/variables/image.yml - name: PythonVersion value: '3.8' pool: name: $(LINUXPOOL) - vmImage: $(LINUXVMIMAGE) + image: $(LINUXVMIMAGE) os: linux steps: diff --git a/eng/pipelines/templates/stages/python-analyze-weekly.yml b/eng/pipelines/templates/stages/python-analyze-weekly.yml index aafcdd8ec38b..fe9d50fbc02b 100644 --- a/eng/pipelines/templates/stages/python-analyze-weekly.yml +++ b/eng/pipelines/templates/stages/python-analyze-weekly.yml @@ -13,19 +13,20 @@ stages: - ${{ if contains(variables['Build.DefinitionName'], 'tests-weekly') }}: - stage: displayName: 'Analyze_${{ parameters.JobName }}' + variables: + - template: /eng/pipelines/templates/variables/image.yml dependsOn: [] jobs: - job: 'Analyze' timeoutInMinutes: 90 variables: - - template: /eng/pipelines/templates/variables/image.yml - name: PythonVersion value: '3.8' pool: name: $(LINUXPOOL) - vmImage: $(LINUXVMIMAGE) + image: $(LINUXVMIMAGE) os: linux steps: From be73f778dbfb42bde75814f958033d46ca62e34a Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 18:11:14 -0700 Subject: [PATCH 7/8] resolve remaining weekly tests --- sdk/anomalydetector/tests.yml | 4 ++-- sdk/batch/tests.yml | 4 ++-- sdk/contentsafety/tests.yml | 4 ++-- sdk/devcenter/tests.yml | 4 ++-- sdk/deviceupdate/tests.yml | 4 ++-- sdk/easm/tests.yml | 4 ++-- sdk/iothub/tests.yml | 4 ++-- sdk/loadtesting/tests.yml | 4 ++-- sdk/purview/tests.yml | 4 ++-- sdk/servicefabric/tests.yml | 2 +- sdk/synapse/tests.yml | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sdk/anomalydetector/tests.yml b/sdk/anomalydetector/tests.yml index 98eeea9b5f70..a1003b38d2ba 100644 --- a/sdk/anomalydetector/tests.yml +++ b/sdk/anomalydetector/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: anomalydetector diff --git a/sdk/batch/tests.yml b/sdk/batch/tests.yml index 2f19868d7f55..2c3305a3e135 100644 --- a/sdk/batch/tests.yml +++ b/sdk/batch/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: batch diff --git a/sdk/contentsafety/tests.yml b/sdk/contentsafety/tests.yml index 0acaf50154d6..5d1a6dc55332 100644 --- a/sdk/contentsafety/tests.yml +++ b/sdk/contentsafety/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: contentsafety diff --git a/sdk/devcenter/tests.yml b/sdk/devcenter/tests.yml index 2042339b1175..1b3232bde697 100644 --- a/sdk/devcenter/tests.yml +++ b/sdk/devcenter/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: devcenter diff --git a/sdk/deviceupdate/tests.yml b/sdk/deviceupdate/tests.yml index c2a59498e6fc..4c6c10d0a16a 100644 --- a/sdk/deviceupdate/tests.yml +++ b/sdk/deviceupdate/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: deviceupdate diff --git a/sdk/easm/tests.yml b/sdk/easm/tests.yml index bc054120ab9e..79895d7ea907 100644 --- a/sdk/easm/tests.yml +++ b/sdk/easm/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: easm diff --git a/sdk/iothub/tests.yml b/sdk/iothub/tests.yml index 4fc6f631f6ef..2564f68d16d4 100644 --- a/sdk/iothub/tests.yml +++ b/sdk/iothub/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: iothub diff --git a/sdk/loadtesting/tests.yml b/sdk/loadtesting/tests.yml index 54b4aae9828e..810047004048 100644 --- a/sdk/loadtesting/tests.yml +++ b/sdk/loadtesting/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: loadtesting diff --git a/sdk/purview/tests.yml b/sdk/purview/tests.yml index 97f537b0cb30..7ff4c54648a0 100644 --- a/sdk/purview/tests.yml +++ b/sdk/purview/tests.yml @@ -1,7 +1,7 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. -stages: - - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: purview diff --git a/sdk/servicefabric/tests.yml b/sdk/servicefabric/tests.yml index be3e7d8286b2..3fbd2e3dcacb 100644 --- a/sdk/servicefabric/tests.yml +++ b/sdk/servicefabric/tests.yml @@ -2,6 +2,6 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. extends: - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: servicefabric diff --git a/sdk/synapse/tests.yml b/sdk/synapse/tests.yml index 080c276b64f8..83cef81d8759 100644 --- a/sdk/synapse/tests.yml +++ b/sdk/synapse/tests.yml @@ -2,6 +2,6 @@ trigger: none # NOTE: Service live tests are NOT enabled. This file only enables the analyze stage currently. extends: - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml + template: /eng/pipelines/templates/stages/python-analyze-weekly-standalone.yml parameters: ServiceDirectory: synapse From 9b31e8f02a49affa6b0702bded30fb140c4441c1 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 18 Mar 2024 18:27:09 -0700 Subject: [PATCH 8/8] disable eslint --- eng/pipelines/templates/stages/1es-redirect.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 9dc7afb73bef..4c2f341173a2 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -37,6 +37,9 @@ extends: suppressionsFile: '$(Build.SourcesDirectory)/eng/CredScanSuppression.json' toolVersion: '2.3.12.23' baselineFiles: $(Build.SourcesDirectory)/eng/python.gdnbaselines + eslint: + enabled: false + justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3556850" psscriptanalyzer: compiled: true break: true