diff --git a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml index e89eeda58c27..dc7825daacdd 100644 --- a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml @@ -17,13 +17,15 @@ resources: - release/*.0.1xx* - internal/release/*.0.1xx* + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + pr: none trigger: none -pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - parameters: - name: dotnetDotnetRunId displayName: 'Specific dotnet-dotnet run ID number (e.g `2108850`)' @@ -36,46 +38,60 @@ variables: # GH access token for SB bot - BotAccount-dotnet-sb-bot-pat - group: Dotnet-SourceBuild-Secrets -jobs: -- template: templates/jobs/sdk-diff-tests.yml +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} - targetRid: ${{ variables.centOSStreamX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} - publishTestResultsPr: true + pool: + name: NetCore1ESPool-Svc-Internal + image: 1es-ubuntu-2204 + os: linux + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022 + os: windows + stages: + - stage: stage + jobs: + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} + targetRid: ${{ variables.centOSStreamX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + publishTestResultsPr: true -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} - targetRid: ${{ variables.almaLinuxX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} + targetRid: ${{ variables.almaLinuxX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.alpineName) }} - targetRid: ${{ variables.alpineX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.alpineName) }} + targetRid: ${{ variables.alpineX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} - targetRid: ${{ variables.fedoraX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} + targetRid: ${{ variables.fedoraX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuArm64Rid }} - architecture: arm64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuArm64Rid }} + architecture: arm64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} diff --git a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml index ced30310a324..0fdda6a946b5 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml @@ -18,11 +18,15 @@ parameters: jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 variables: - template: ../variables/pipelines.yml + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) steps: - script: | dotnet_dotnet_build='${{ replace(parameters.dotnetDotnetRunId, ' ', '') }}' @@ -152,12 +156,6 @@ jobs: continueOnError: true condition: succeededOrFailed() - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() diff --git a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml index 96f6662088b9..b04b44e2bf83 100644 --- a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml @@ -24,152 +24,173 @@ variables: - name: installerRoot value: '$(Build.SourcesDirectory)/src/installer' -jobs: -- job: Setup - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - steps: - - script: | - vmrSrcDir="$(Build.SourcesDirectory)/src" - - # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, - # allowing a job to be run for each src repo. - matrix="" - - # Trim leading/trailing spaces from the repo name - specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') - - # If the repo name is provided, only scan that repo. - if [ ! -z "$specificRepoName" ]; then - matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" - else - for dir in $vmrSrcDir/*/ - do - if [ ! -z "$matrix" ]; then - matrix="$matrix," - fi - repoName=$(basename $dir) - matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" - done - fi - - matrix="{ $matrix }" - - echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" - name: GetMatrix - displayName: Get Matrix - -- job: LicenseScan - dependsOn: Setup - container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net8.0-source-build-test-amd64 - options: '--memory=6g' - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - timeoutInMinutes: 420 - strategy: - matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] - steps: - - - script: > - dotnet test - $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj - --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" - --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' - --logger:'console;verbosity=detailed' - -c Release - -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog - -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log - -clp:v=m - -e SMOKE_TESTS_LICENSE_SCAN_PATH=$(repoPath) - -e SMOKE_TESTS_RUNNING_IN_CI=true - -e SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS=false - -e SMOKE_TESTS_TARGET_RID=linux-x64 - -e SMOKE_TESTS_PORTABLE_RID=linux-x64 - displayName: Run Tests - workingDirectory: $(Build.SourcesDirectory) - - - script: | - set -x - targetFolder=$(Build.StagingDirectory)/BuildLogs/ - mkdir -p ${targetFolder} - cd "$(Build.SourcesDirectory)" - find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; - echo "Updated:" - find test/ -type f -name "Updated*.json" - find test/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "Results:" - find test/ -type f -name "scancode-results*.json" - find test/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "All:" - ls -R test/ - echo "BuildLogs:" - ls -R ${targetFolder} - displayName: Prepare BuildLogs staging directory - continueOnError: true - condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: '*.trx' - searchFolder: $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults - mergeTestResults: true - publishRunAttachments: true - testRunTitle: $(Agent.JobName) - -- job: CreateBaselineUpdatePR - dependsOn: LicenseScan - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - variables: - - template: templates/variables/pipelines.yml - steps: - - - script: | - source ./eng/common/tools.sh - InitializeDotNetCli true - echo "##vso[task.setvariable variable=DotNetPath]$_InitializeDotNetCli" - displayName: Install .NET SDK - workingDirectory: $(Build.SourcesDirectory) - - - task: DownloadPipelineArtifact@2 - inputs: - buildType: specific - buildVersionToDownload: specific - project: internal - buildId: $(Build.BuildId) - artifact: '' - patterns: '**/Updated*' - allowPartiallySucceededBuilds: true - allowFailedBuilds: true - downloadPath: $(Pipeline.Workspace)/Artifacts - checkDownloadedFiles: true - displayName: Download Updated Baselines - - - script: | - find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; - displayName: Move Artifacts to root - - - template: templates/steps/create-baseline-update-pr.yml - parameters: - dotnetPath: $(DotNetPath) - pipeline: license - repo: dotnet/installer - originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/licenses - updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts - pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: NetCore1ESPool-Svc-Internal + image: 1es-ubuntu-2204 + os: linux + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022 + os: windows + stages: + - stage: stage + jobs: + - job: Setup + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2204 + steps: + - script: | + vmrSrcDir="$(Build.SourcesDirectory)/src" + + # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, + # allowing a job to be run for each src repo. + matrix="" + + # Trim leading/trailing spaces from the repo name + specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') + + # If the repo name is provided, only scan that repo. + if [ ! -z "$specificRepoName" ]; then + matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" + else + for dir in $vmrSrcDir/*/ + do + if [ ! -z "$matrix" ]; then + matrix="$matrix," + fi + repoName=$(basename $dir) + matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" + done + fi + + matrix="{ $matrix }" + + echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + name: GetMatrix + displayName: Get Matrix + + - job: LicenseScan + dependsOn: Setup + container: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net8.0-source-build-test-amd64 + options: '--memory=6g' + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2204 + timeoutInMinutes: 420 + strategy: + matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] + steps: + + - script: > + dotnet test + $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj + --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" + --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' + --logger:'console;verbosity=detailed' + -c Release + -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog + -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log + -clp:v=m + -e SMOKE_TESTS_LICENSE_SCAN_PATH=$(repoPath) + -e SMOKE_TESTS_RUNNING_IN_CI=true + -e SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS=false + -e SMOKE_TESTS_TARGET_RID=linux-x64 + -e SMOKE_TESTS_PORTABLE_RID=linux-x64 + displayName: Run Tests + workingDirectory: $(Build.SourcesDirectory) + + - script: | + set -x + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + cd "$(Build.SourcesDirectory)" + find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; + echo "Updated:" + find test/ -type f -name "Updated*.json" + find test/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "Results:" + find test/ -type f -name "scancode-results*.json" + find test/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "All:" + ls -R test/ + echo "BuildLogs:" + ls -R ${targetFolder} + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - publish: '$(Build.StagingDirectory)/BuildLogs' + artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + displayName: Publish BuildLogs + continueOnError: true + condition: succeededOrFailed() + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '*.trx' + searchFolder: $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(Agent.JobName) + + - job: CreateBaselineUpdatePR + dependsOn: LicenseScan + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + - template: templates/variables/pipelines.yml + steps: + + - script: | + source ./eng/common/tools.sh + InitializeDotNetCli true + echo "##vso[task.setvariable variable=DotNetPath]$_InitializeDotNetCli" + displayName: Install .NET SDK + workingDirectory: $(Build.SourcesDirectory) + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: specific + buildVersionToDownload: specific + project: internal + buildId: $(Build.BuildId) + artifact: '' + patterns: '**/Updated*' + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + downloadPath: $(Pipeline.Workspace)/Artifacts + checkDownloadedFiles: true + displayName: Download Updated Baselines + + - script: | + find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; + displayName: Move Artifacts to root + + - template: templates/steps/create-baseline-update-pr.yml + parameters: + dotnetPath: $(DotNetPath) + pipeline: license + repo: dotnet/installer + originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/licenses + updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts + pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions