From cb5ea71c8cc90ec026e06f1f52ffae55d8aef4e8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 26 Aug 2026 12:17:46 -0700 Subject: [PATCH] ci: publish Azure Pipelines code coverage --- .azure/pipelines/templates/build.yaml | 79 +++++++++++++++++++-- .azure/pipelines/templates/vars.yaml | 1 + .github/actions/dotnet-test/action.yml | 3 +- .github/coverage.config.xml | 3 +- .github/coverage.static.config.xml | 3 +- .github/scripts/setup-coverage.ps1 | 18 ++++- .github/scripts/test-summarize-coverage.ps1 | 69 +++++++++++++++--- 7 files changed, 153 insertions(+), 23 deletions(-) diff --git a/.azure/pipelines/templates/build.yaml b/.azure/pipelines/templates/build.yaml index 9300bf8fbdc..71535ae69a9 100644 --- a/.azure/pipelines/templates/build.yaml +++ b/.azure/pipelines/templates/build.yaml @@ -177,7 +177,7 @@ jobs: - ${{ if and(eq(parameters.skip_test, false), ne(variables.runCodeQL3000, 'true')) }}: - ${{ each suite in parameters.suites }}: - ${{ each framework in parameters.frameworks }}: - - job: + - job: Test_${{suite}}_${{ replace(framework, '.', '_') }} displayName: ${{suite}} on ${{framework}} timeoutInMinutes: 120 dependsOn: Build @@ -233,7 +233,7 @@ jobs: inputs: version: ${{variables.testFrameworkToInstall}} displayName: 'Install .NET Core sdk used in tests' - - task: DotNetCoreCLI@2 + - task: PowerShell@2 displayName: Test env: ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -242,11 +242,51 @@ jobs: SERVICE_CONNECTION_ID: $(serviceConnectionId) SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: - command: 'test' - projects: '$(solution)' - testRunTitle: ${{suite}} on ${{framework}} - arguments: '--no-build --report-trx --report-trx-filename "testresults-${{framework}}-${{suite}}-{asm}_{tfm}_{arch}.trx" --framework ${{framework}} --configuration "${{parameters.build_configuration}}" --filter-query "/[(Provider=None)&(Suite=${{suite}})]" --minimum-expected-tests 1 --crashdump --crashdump-type Full --hangdump --hangdump-timeout 10m --hangdump-type Full --max-parallel-test-modules 1' - publishTestResults: false # The next step publishes all module-specific TRX files as one test run. + pwsh: true + targetType: inline + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $coverageDirectory = '$(Build.ArtifactStagingDirectory)/test_outputs_${{suite}}_${{framework}}_$(Build.BuildId)' + New-Item -ItemType Directory -Force -Path $coverageDirectory | Out-Null + + $toolPath = Join-Path '$(Agent.TempDirectory)' 'orleans-dotnet-coverage-$(System.JobId)' + ./.github/scripts/setup-coverage.ps1 -InstallPath $toolPath + + $command = @( + 'dotnet' + 'test' + '--solution' + '$(solution)' + '--no-build' + '--report-trx' + '--report-trx-filename' + 'testresults-${{framework}}-${{suite}}-{asm}_{tfm}_{arch}.trx' + '--framework' + '${{framework}}' + '--configuration' + '${{parameters.build_configuration}}' + '--filter-query' + '/[(Provider=None)&(Suite=${{suite}})]' + '--minimum-expected-tests' + '1' + '--crashdump' + '--crashdump-type' + 'Full' + '--hangdump' + '--hangdump-timeout' + '10m' + '--hangdump-type' + 'Full' + '--max-parallel-test-modules' + '1' + ) + + ./.github/scripts/invoke-coverage.ps1 ` + -Settings ./.github/coverage.config.xml ` + -Output "$coverageDirectory/coverage-${{suite}}-${{framework}}.cobertura.xml" ` + -RetryLogFile "$coverageDirectory/coverage-${{suite}}-${{framework}}.retry.log" ` + -CoverageCommand (Join-Path $toolPath 'dotnet-coverage.exe') ` + -Command $command - task: PublishTestResults@2 displayName: Publishing test results condition: succeededOrFailed() @@ -269,3 +309,28 @@ jobs: Contents: '**\*.dmp' TargetFolder: '$(Build.ArtifactStagingDirectory)/test_outputs_${{suite}}_${{framework}}_$(Build.BuildId)' OverWrite: true + - job: PublishCodeCoverage + displayName: Publish code coverage + dependsOn: + - ${{ each suite in parameters.suites }}: + - ${{ each framework in parameters.frameworks }}: + - Test_${{suite}}_${{ replace(framework, '.', '_') }} + condition: succeededOrFailed() + steps: + - checkout: self + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + - task: DownloadPipelineArtifact@2 + displayName: Download code coverage + inputs: + buildType: current + itemPattern: '**/*.cobertura.xml' + targetPath: '$(Pipeline.Workspace)/coverage' + - task: PublishCodeCoverageResults@2 + displayName: Publish code coverage + inputs: + summaryFileLocation: '$(Pipeline.Workspace)/coverage/**/*.cobertura.xml' + pathToSources: '$(Build.SourcesDirectory)' + failIfCoverageEmpty: true diff --git a/.azure/pipelines/templates/vars.yaml b/.azure/pipelines/templates/vars.yaml index 54b4808ab3a..cded22d2418 100644 --- a/.azure/pipelines/templates/vars.yaml +++ b/.azure/pipelines/templates/vars.yaml @@ -6,6 +6,7 @@ variables: codesign_runtime: '2.1.x' GDN_SUPPRESS_FORKED_BUILD_WARNING: true # Avoid warning "Guardian is not supported for builds from forked GitHub repositories" MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + DOTNET_COVERAGE_VERSION: 18.5.2 # Auto-injection is not necessary because the tasks are explicitly included where they're enabled. Codeql.SkipTaskAutoInjection: true ${{ if eq(variables['System.TeamProject'], 'GitHub - PR Builds') }}: diff --git a/.github/actions/dotnet-test/action.yml b/.github/actions/dotnet-test/action.yml index 5c12000b0fe..7ca3e404708 100644 --- a/.github/actions/dotnet-test/action.yml +++ b/.github/actions/dotnet-test/action.yml @@ -29,7 +29,7 @@ runs: - name: Build and discover tests for static coverage if: github.event_name == 'pull_request' && (runner.os == 'macOS' || inputs.static-instrumentation == 'true') shell: pwsh - run: dotnet test --solution Orleans.slnx --framework "${{ inputs.framework }}" --filter-query "${{ inputs.filter-query }}" --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1 -p:ContinuousIntegrationBuild=false + run: dotnet test --solution Orleans.slnx --framework "${{ inputs.framework }}" --filter-query "${{ inputs.filter-query }}" --list-tests --minimum-expected-tests 1 --max-parallel-test-modules 1 - name: Test if: github.event_name != 'pull_request' shell: pwsh @@ -62,7 +62,6 @@ runs: 'test_results_${{ inputs.result-id }}_{asm}_{tfm}_{arch}.trx' '--max-parallel-test-modules' '1' - '-p:ContinuousIntegrationBuild=false' ) ./.github/scripts/invoke-coverage.ps1 -Settings ./.github/coverage.config.xml -Output "TestResults/${{ inputs.coverage-id }}.cobertura.xml" -RetryLogFile "logs/${{ inputs.coverage-id }}.retry.log" -Command $command - name: Test with static coverage diff --git a/.github/coverage.config.xml b/.github/coverage.config.xml index 477e84d6397..e49e0c09314 100644 --- a/.github/coverage.config.xml +++ b/.github/coverage.config.xml @@ -1,7 +1,8 @@ + True False - MissingAll + None diff --git a/.github/coverage.static.config.xml b/.github/coverage.static.config.xml index 485a7ddb054..1ab578923b7 100644 --- a/.github/coverage.static.config.xml +++ b/.github/coverage.static.config.xml @@ -1,7 +1,8 @@ + True False - MissingAll + None diff --git a/.github/scripts/setup-coverage.ps1 b/.github/scripts/setup-coverage.ps1 index e0b01270107..52b146d81ce 100644 --- a/.github/scripts/setup-coverage.ps1 +++ b/.github/scripts/setup-coverage.ps1 @@ -1,6 +1,8 @@ [CmdletBinding()] param( - [string] $Version = $env:DOTNET_COVERAGE_VERSION + [string] $Version = $env:DOTNET_COVERAGE_VERSION, + + [string] $InstallPath ) Set-StrictMode -Version Latest @@ -24,7 +26,15 @@ if ([string]::IsNullOrWhiteSpace($Version)) { throw 'DOTNET_COVERAGE_VERSION must specify the dotnet-coverage tool version' } -$toolPath = Join-Path $env:GITHUB_WORKSPACE '.tools' +$toolPath = $InstallPath +if ([string]::IsNullOrWhiteSpace($toolPath)) { + if ([string]::IsNullOrWhiteSpace($env:GITHUB_WORKSPACE)) { + throw 'InstallPath must be specified outside GitHub Actions' + } + + $toolPath = Join-Path $env:GITHUB_WORKSPACE '.tools' +} + Assert-NotReparsePoint $toolPath dotnet tool install --tool-path $toolPath dotnet-coverage --version $Version if ($LASTEXITCODE -ne 0) { @@ -32,4 +42,6 @@ if ($LASTEXITCODE -ne 0) { } Assert-NotReparsePoint $toolPath -$toolPath >> $env:GITHUB_PATH +if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_PATH)) { + $toolPath >> $env:GITHUB_PATH +} diff --git a/.github/scripts/test-summarize-coverage.ps1 b/.github/scripts/test-summarize-coverage.ps1 index cb03d7fc587..196b9c1ac57 100644 --- a/.github/scripts/test-summarize-coverage.ps1 +++ b/.github/scripts/test-summarize-coverage.ps1 @@ -5,8 +5,12 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $scriptPath = Join-Path $PSScriptRoot 'summarize-coverage.ps1' +$coverageConfigPath = Join-Path $PSScriptRoot '../coverage.config.xml' $coverageReportScriptPath = Join-Path $PSScriptRoot 'coverage-report.ps1' +$coverageStaticConfigPath = Join-Path $PSScriptRoot '../coverage.static.config.xml' $archiveTestResultsActionPath = Join-Path $PSScriptRoot '../actions/archive-test-results/action.yml' +$azureBuildTemplatePath = Join-Path $PSScriptRoot '../../.azure/pipelines/templates/build.yaml' +$azureVariablesPath = Join-Path $PSScriptRoot '../../.azure/pipelines/templates/vars.yaml' $dotnetTestActionPath = Join-Path $PSScriptRoot '../actions/dotnet-test/action.yml' $invokeCoverageScriptPath = Join-Path $PSScriptRoot 'invoke-coverage.ps1' $runTestsActionPath = Join-Path $PSScriptRoot '../actions/run-tests/action.yml' @@ -276,14 +280,28 @@ try { Invoke-Test 'uses external coverage collection for CI builds' { $dotnetTestAction = Get-Content -Raw -LiteralPath $dotnetTestActionPath $coverageReportScript = Get-Content -Raw -LiteralPath $coverageReportScriptPath + $coverageConfigs = @( + Get-Content -Raw -LiteralPath $coverageConfigPath + Get-Content -Raw -LiteralPath $coverageStaticConfigPath + ) Assert-Matches ` $dotnetTestAction ` 'invoke-coverage\.ps1' ` - 'Coverage must use the external collector with ContinuousIntegrationBuild.' - Assert-Matches ` - $dotnetTestAction ` - '-p:ContinuousIntegrationBuild=false' ` - 'Coverage builds must disable deterministic CI instrumentation.' + 'Coverage must use the external collector during CI.' + Assert-Equal ` + 0 ` + ([regex]::Matches($dotnetTestAction, 'ContinuousIntegrationBuild=false')).Count ` + 'GitHub coverage must preserve continuous integration build semantics.' + foreach ($coverageConfig in $coverageConfigs) { + Assert-Matches ` + $coverageConfig ` + 'True' ` + 'Coverage reports must preserve deterministic source paths.' + Assert-Matches ` + $coverageConfig ` + 'None' ` + 'Coverage collection must retain symbol-bearing assemblies with deterministic sources.' + } Assert-Matches ` $dotnetTestAction ` '-IncludeFiles' ` @@ -373,11 +391,11 @@ exit 0 -Output $coverageOutput ` -RetryLogFile $retryLog ` -CoverageCommand $fakeCollector ` - -Command @('dotnet', 'test', '-p:ContinuousIntegrationBuild=false') + -Command @('dotnet', 'test', '--forwarded-argument') Assert-Equal 0 $LASTEXITCODE 'The retry should succeed.' Assert-Equal 2 ([int] (Get-Content -Raw -LiteralPath $attemptFile)) 'The collector attempt count differs.' $retryArguments = Get-Content -LiteralPath "$collectorArguments.2.txt" - Assert-Equal $true ($retryArguments -contains '-p:ContinuousIntegrationBuild=false') 'The inner MSBuild property must be forwarded.' + Assert-Equal $true ($retryArguments -contains '--forwarded-argument') 'The inner test argument must be forwarded.' Assert-Equal $true ($retryArguments -contains '--log-file') 'The retry must capture a collector log.' Assert-Equal $true ($retryArguments -contains 'Verbose') 'The retry collector log must be verbose.' @@ -388,7 +406,7 @@ exit 0 -Output $coverageOutput ` -RetryLogFile $retryLog ` -CoverageCommand $fakeCollector ` - -Command @('dotnet', 'test', '-p:ContinuousIntegrationBuild=false') + -Command @('dotnet', 'test', '--forwarded-argument') Assert-Equal 1 $LASTEXITCODE 'An unrelated test failure should be preserved.' Assert-Equal 1 ([int] (Get-Content -Raw -LiteralPath $attemptFile)) 'An unrelated failure must not be retried.' @@ -399,7 +417,7 @@ exit 0 -Output $coverageOutput ` -RetryLogFile $retryLog ` -CoverageCommand $fakeCollector ` - -Command @('dotnet', 'test', '-p:ContinuousIntegrationBuild=false') + -Command @('dotnet', 'test', '--forwarded-argument') Assert-Equal 1 $LASTEXITCODE 'A persistent collector failure should be preserved.' Assert-Equal 2 ([int] (Get-Content -Raw -LiteralPath $attemptFile)) 'The collector must retry only once.' } finally { @@ -484,12 +502,45 @@ exit 0 Assert-Equal 0 ([regex]::Matches($dotnetTestAction, '--project|--test-modules')).Count 'Native test action must discover projects from the solution.' } + Invoke-Test 'publishes Azure Pipelines coverage' { + $azureBuildTemplate = Get-Content -Raw -LiteralPath $azureBuildTemplatePath + $azureVariables = Get-Content -Raw -LiteralPath $azureVariablesPath + Assert-Matches ` + $azureVariables ` + 'DOTNET_COVERAGE_VERSION:\s*\d+\.\d+\.\d+' ` + 'Azure Pipelines must pin the coverage collector version.' + Assert-Matches ` + $azureBuildTemplate ` + '(?s)setup-coverage\.ps1.*?invoke-coverage\.ps1.*?coverage-\$\{\{suite\}\}-\$\{\{framework\}\}\.cobertura\.xml' ` + 'Azure Pipelines must use the shared coverage scripts to collect a distinct report from every test job.' + Assert-Equal ` + 0 ` + ([regex]::Matches($azureBuildTemplate, 'ContinuousIntegrationBuild=false')).Count ` + 'Azure Pipelines coverage must preserve continuous integration build semantics.' + Assert-Matches ` + $azureBuildTemplate ` + '(?s)job: PublishCodeCoverage.*?dependsOn:.*?Test_\$\{\{suite\}\}_\$\{\{ replace\(framework.*?DownloadPipelineArtifact@2.*?itemPattern: ''\*\*/\*\.cobertura\.xml''' ` + 'Azure Pipelines must aggregate coverage after every test matrix job.' + Assert-Matches ` + $azureBuildTemplate ` + '(?s)PublishCodeCoverageResults@2.*?summaryFileLocation:.*?\*\*/\*\.cobertura\.xml.*?failIfCoverageEmpty: true' ` + 'Azure Pipelines must publish the aggregated coverage and require results.' + } + Invoke-Test 'validates the coverage tool version' { $setupCoverageScript = Get-Content -Raw -LiteralPath $setupCoverageScriptPath Assert-Matches ` $setupCoverageScript ` 'DOTNET_COVERAGE_VERSION must specify' ` 'Coverage setup must reject a missing tool version.' + Assert-Matches ` + $setupCoverageScript ` + 'InstallPath must be specified outside GitHub Actions' ` + 'Coverage setup must require an explicit installation path in other CI systems.' + Assert-Matches ` + $setupCoverageScript ` + 'if \(-not \[string\]::IsNullOrWhiteSpace\(\$env:GITHUB_PATH\)\)' ` + 'Coverage setup must continue adding the tool to the GitHub Actions path.' Assert-Equal 2 ([regex]::Matches($setupCoverageScript, 'Assert-NotReparsePoint \$toolPath')).Count 'Coverage tool path validation count differs.' }