diff --git a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml index 7008c8bb080c..97ce66882e53 100644 --- a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml @@ -20,13 +20,15 @@ resources: - refs/heads/release/*.0.1xx-preview* - refs/heads/internal/release/*.0.1xx* + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + pr: none trigger: none -pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - parameters: - name: dotnetDotnetRunId displayName: 'Specific dotnet-dotnet run ID number (e.g `2108850`)' @@ -39,46 +41,60 @@ variables: # GH access token for SB bot - BotAccount-dotnet-sb-bot-pat - group: Dotnet-SourceBuild-Secrets -jobs: -- template: templates/jobs/sdk-diff-tests.yml +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} - targetRid: ${{ variables.centOSStreamX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} - publishTestResultsPr: true + pool: + name: NetCore1ESPool-Svc-Internal + image: 1es-ubuntu-2204 + os: linux + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022 + os: windows + stages: + - stage: stage + jobs: + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} + targetRid: ${{ variables.centOSStreamX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + publishTestResultsPr: true -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} - targetRid: ${{ variables.almaLinuxX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} + targetRid: ${{ variables.almaLinuxX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.alpineName) }} - targetRid: ${{ variables.alpineX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.alpineName) }} + targetRid: ${{ variables.alpineX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} - targetRid: ${{ variables.fedoraX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} + targetRid: ${{ variables.fedoraX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuArm64Rid }} - architecture: arm64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: /eng/pipelines/templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuArm64Rid }} + architecture: arm64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} diff --git a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml index 9bd63a32fe0f..cd13ef57fb89 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml @@ -18,11 +18,15 @@ parameters: jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 variables: - template: ../variables/pipelines.yml + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) steps: - script: | dotnet_dotnet_build='${{ replace(parameters.dotnetDotnetRunId, ' ', '') }}' @@ -151,12 +155,6 @@ jobs: continueOnError: true condition: succeededOrFailed() - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() diff --git a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml index 1eaaa7d2a07e..657ded1a4aa2 100644 --- a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml @@ -25,6 +25,13 @@ trigger: - test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs - test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseScanTests +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: # Provides a way to scan a specific repo. If not provided, all repos of the VMR will be scanned. - name: specificRepoName @@ -38,148 +45,159 @@ variables: - name: sdkRoot value: '$(Build.SourcesDirectory)/src/sdk' -jobs: -- job: Setup - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - steps: - - script: | - vmrSrcDir="$(Build.SourcesDirectory)/src" - # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, - # allowing a job to be run for each src repo. - matrix="" - - # Trim leading/trailing spaces from the repo name - specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') - - # If the repo name is provided, only scan that repo. - if [ ! -z "$specificRepoName" ]; then - matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" - else - for dir in $vmrSrcDir/*/ - do - if [ ! -z "$matrix" ]; then - matrix="$matrix," - fi - repoName=$(basename $dir) - matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" - done - fi - - matrix="{ $matrix }" - - echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" - name: GetMatrix - displayName: Get Matrix - -- job: LicenseScan - dependsOn: Setup - container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-source-build-test-amd64 - options: '--memory=6g' - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - timeoutInMinutes: 420 - strategy: - matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] - steps: - - - script: > - dotnet test - $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj - --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" - --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' - --logger:'console;verbosity=detailed' - -c Release - -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog - -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log - -clp:v=m - /p:SmokeTestsLicenseScanPath=$(repoPath) - /p:SmokeTestsWarnOnLicenseScanDiffs=false - /p:TargetRid=linux-x64 - /p:PortableRid=linux-x64 - /p:SkipPrepareSdkArchive=true - displayName: Run Tests - workingDirectory: $(Build.SourcesDirectory) - - - script: | - set -x - targetFolder=$(Build.StagingDirectory)/BuildLogs/ - mkdir -p ${targetFolder} - cd "$(Build.SourcesDirectory)" - find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - echo "Updated:" - find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" - find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "Updated*.json" - find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "Results:" - find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "All:" - ls -R artifacts/TestResults/ - echo "BuildLogs:" - ls -R ${targetFolder} - displayName: Prepare BuildLogs staging directory - continueOnError: true - condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: '**/*.trx' - searchFolder: $(Build.SourcesDirectory)/artifacts/TestResults - mergeTestResults: true - publishRunAttachments: true - testRunTitle: $(Agent.JobName) - -- job: Publish_Test_Results_PR - dependsOn: LicenseScan - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2204 - variables: - - template: templates/variables/pipelines.yml - steps: - - - script: | - source ./eng/common/tools.sh - InitializeDotNetCli true - echo "##vso[task.setvariable variable=DotNetPath]$_InitializeDotNetCli" - displayName: Install .NET SDK - workingDirectory: $(Build.SourcesDirectory) - - - template: templates/steps/download-pipeline-artifact.yml - parameters: - pipeline: $(SOURCE_BUILD_LICENSE_SCAN_PIPELINE_ID) - buildId: $(Build.BuildId) - artifact: '' - patterns: '**/Updated*' - displayName: Download Updated Test Files - - - script: | - find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; - displayName: Move Artifacts to root - - - template: templates/steps/create-baseline-update-pr.yml - parameters: - dotnetPath: $(DotNetPath) - pipeline: license - repo: dotnet/sdk - originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseScanTests - updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts - pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: NetCore1ESPool-Svc-Internal + image: 1es-ubuntu-2204 + os: linux + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022 + os: windows + stages: + - stage: stage + jobs: + - job: Setup + steps: + - script: | + vmrSrcDir="$(Build.SourcesDirectory)/src" + # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, + # allowing a job to be run for each src repo. + matrix="" + + # Trim leading/trailing spaces from the repo name + specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') + + # If the repo name is provided, only scan that repo. + if [ ! -z "$specificRepoName" ]; then + matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" + else + for dir in $vmrSrcDir/*/ + do + if [ ! -z "$matrix" ]; then + matrix="$matrix," + fi + repoName=$(basename $dir) + matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" + done + fi + + matrix="{ $matrix }" + + echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + name: GetMatrix + displayName: Get Matrix + + - job: LicenseScan + dependsOn: Setup + container: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-source-build-test-amd64 + options: '--memory=6g' + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2204 + timeoutInMinutes: 420 + strategy: + matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] + steps: + + - script: > + dotnet test + $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj + --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" + --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' + --logger:'console;verbosity=detailed' + -c Release + -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog + -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log + -clp:v=m + /p:SmokeTestsLicenseScanPath=$(repoPath) + /p:SmokeTestsWarnOnLicenseScanDiffs=false + /p:TargetRid=linux-x64 + /p:PortableRid=linux-x64 + /p:SkipPrepareSdkArchive=true + displayName: Run Tests + workingDirectory: $(Build.SourcesDirectory) + + - script: | + set -x + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + cd "$(Build.SourcesDirectory)" + find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + echo "Updated:" + find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" + find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "Updated*.json" + find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "Results:" + find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "All:" + ls -R artifacts/TestResults/ + echo "BuildLogs:" + ls -R ${targetFolder} + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - publish: '$(Build.StagingDirectory)/BuildLogs' + artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + displayName: Publish BuildLogs + continueOnError: true + condition: succeededOrFailed() + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Build.SourcesDirectory)/artifacts/TestResults + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(Agent.JobName) + + - job: Publish_Test_Results_PR + dependsOn: LicenseScan + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + - template: templates/variables/pipelines.yml + steps: + + - script: | + source ./eng/common/tools.sh + InitializeDotNetCli true + echo "##vso[task.setvariable variable=DotNetPath]$_InitializeDotNetCli" + displayName: Install .NET SDK + workingDirectory: $(Build.SourcesDirectory) + + - template: templates/steps/download-pipeline-artifact.yml + parameters: + pipeline: $(SOURCE_BUILD_LICENSE_SCAN_PIPELINE_ID) + buildId: $(Build.BuildId) + artifact: '' + patterns: '**/Updated*' + displayName: Download Updated Test Files + + - script: | + find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; + displayName: Move Artifacts to root + + - template: templates/steps/create-baseline-update-pr.yml + parameters: + dotnetPath: $(DotNetPath) + pipeline: license + repo: dotnet/sdk + originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseScanTests + updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts + pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions