Skip to content

[6.1] Fix code coverage job disk full #3813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulmedynski merged 1 commit into from
Dec 1, 2025
Merged

[6.1] Fix code coverage job disk full #3813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
322 changes: 178 additions & 144 deletions eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,160 +3,194 @@
# The .NET Foundation licenses this file to you under the MIT license. #
# See the LICENSE file in the project root for more information. #
#################################################################################

# This job processes code coverage reports generated during test runs,
# merges them, generates code coverage reports, publishes them to the
# pipeline, and uploads them to CodeCov.

parameters:

# True to include debug steps.
- name: debug
type: boolean
default: false

- name: upload
# The pool image to use.
- name: image
type: string
default: $(ci_var_uploadTestResult)

- name: poolName
# The agent pool name.
- name: pool
type: string
default: $(defaultHostedPoolName)

- name: image
type: string
default: 'windows-2022'

- name: downloadArtifactsSteps
type: stepList
default: []
# Array of target frameworks to process code coverage for:
#
# e.g. [net462, net8.0, net9.0]
#
- name: targetFrameworks
type: object

# True to upload code coverage results to CodeCov.
- name: upload
type: boolean
default: true

jobs:
- job: CodeCoverage
displayName: 'Merge Code Coverage'

variables:
uploadTestResult: ${{ parameters.upload }}

pool:
name: '${{ parameters.poolName }}'
vmImage: ${{ parameters.image }}

steps:
- ${{if eq(parameters.debug, true)}}:
- powershell: |
Get-ChildItem env: | Sort-Object Name
displayName: 'List Environment Variables [debug]'
- task: NuGetAuthenticate@1
displayName: 'NuGet Authenticate'

- template: ../steps/ensure-dotnet-version.yml@self
parameters:
packageType: 'sdk'
version: '8.0'

- ${{ parameters.downloadArtifactsSteps }}

- ${{ if eq(parameters.debug, true)}}:
- powershell: |
Get-ChildItem $(Build.SourcesDirectory)\coverageNetFx\ -Recurse -File -Filter *.coverage
displayName: 'List coverageNetFx files [debug]'
- powershell: |
Get-ChildItem $(Build.SourcesDirectory)\coverageNetCore\ -Recurse -File -Filter *.coverage
displayName: 'List coverageNetCore files [debug]'
- pwsh: |
dotnet tool install --global dotnet-coverage
function MergeFiles {
param(
[string]$InputDirectoryPath,
[string]$OutputDirectoryName
)
$files = Get-ChildItem $InputDirectoryPath -Recurse -File -Filter *.coverage
# echo $files
mkdir $OutputDirectoryName
$counter=0
$toProcess = @()
foreach ($file in $files) {
$toProcess += @{
File = $file.FullName
OutputFile = "$OutputDirectoryName\$counter.coveragexml"
- job: CodeCoverage
displayName: Publish Code Coverage

pool:
name: ${{ parameters.pool }}
${{ if eq(parameters.pool, 'Azure Pipelines') }}:
vmImage: ${{ parameters.image }}
${{ else }}:
demands:
- imageOverride -equals ${{ parameters.image }}

variables:
netFxDir: $(Build.SourcesDirectory)\coverageNetFx
netCoreDir: $(Build.SourcesDirectory)\coverageNetCore

steps:
- ${{if eq(parameters.debug, true)}}:
- pwsh: |
Get-ChildItem env: | Sort-Object Name
displayName: 'List Environment Variables [debug]'
- pwsh: Get-Volume
displayName: '[Debug] Show Disk Usage'

- pwsh: 'Get-ChildItem env: | Sort-Object Name'
displayName: '[Debug] List Environment Variables'

- template: ../steps/ensure-dotnet-version.yml@self
parameters:
packageType: sdk
version: '9.0'

- pwsh: |
dotnet tool install --global dotnet-coverage
dotnet tool install --global dotnet-reportgenerator-globaltool
displayName: Install dotnet tools
- ${{ each targetFramework in parameters.targetFrameworks }}:
- task: DownloadPipelineArtifact@2
displayName: 'Download Coverage Reports [${{ targetFramework }}]'
inputs:
itemPattern: '**\${{ targetFramework }}*'
${{ if startsWith(targetFramework, 'net4') }}:
targetPath: $(netFxDir)
${{ else }}:
targetPath: $(netCoreDir)

- ${{if eq(parameters.debug, true)}}:
- pwsh: Get-Volume
displayName: '[Debug] Show Disk Usage'

- pwsh: Get-ChildItem $(netFxDir) -Recurse -File -Filter *.coverage
displayName: '[Debug] List coverageNetFx files'

- pwsh: Get-ChildItem $(netCoreDir) -Recurse -File -Filter *.coverage
displayName: '[Debug] List coverageNetCore files'

- pwsh: |
function MergeFiles {
param(
[string]$InputDirectoryPath,
[string]$OutputDirectoryName
)
$files = Get-ChildItem $InputDirectoryPath -Recurse -File -Filter *.coverage
# echo $files
mkdir $OutputDirectoryName
$counter=0
$toProcess = @()
foreach ($file in $files) {
$toProcess += @{
File = $file.FullName
OutputFile = "$OutputDirectoryName\$counter.coveragexml"
}
$counter++
}
$counter++
}
$jobs = @()
foreach ($file in $toProcess){
$jobs += Start-ThreadJob -ScriptBlock {
$params = $using:file
& dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml
}
}
Write-Host "Merging started..."
Wait-Job -Job $jobs
foreach ($job in $jobs) {
Receive-Job -Job $job -Wait -AutoRemoveJob
}
}
MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetFx\" -OutputDirectoryName "coverageNetFxXml"
MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetCore\" -OutputDirectoryName "coverageNetCoreXml"
# dir coverageNetFxXml\
# dir coverageNetCoreXml\
Write-Host "Clean up disk ... [removing coverageNetFx & coverageNetCore]"
Remove-Item $(Build.SourcesDirectory)\coverageNetFx -Recurse -Force
Remove-Item $(Build.SourcesDirectory)\coverageNetCore -Recurse -Force
displayName: 'Convert coverage files to xml'
- ${{ if eq(parameters.debug, true)}}:
- powershell: |
dir coverageNetFxXml\
dir coverageNetCoreXml\
displayName: 'List converted files [debug]'
- pwsh: |
dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools
$jobs = @()
$jobs += Start-ThreadJob -ScriptBlock {
& tools\reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
}
$jobs += Start-ThreadJob -ScriptBlock {
& tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
}
$jobs += Start-ThreadJob -ScriptBlock {
& tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
}
Write-Host "Running ReportGenerator..."
Wait-Job -Job $jobs
foreach ($job in $jobs) {
Receive-Job -Job $job -Wait -AutoRemoveJob
}
displayName: 'Run ReportGenerator'
- task: PublishCodeCoverageResults@2
displayName: 'Publish code coverage from netcore'
inputs:
summaryFileLocation: '*\Cobertura.xml'

- powershell: |
#download Codecov CLI
$ProgressPreference = 'SilentlyContinue'
Invoke-WebRequest -Uri https://cli.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetFx\Cobertura.xml" -F netfx
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetCore\Cobertura.xml" -F netcore
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportAddOns\Cobertura.xml" -F addons
displayName: 'Upload to CodeCov'
condition: and(succeeded(), eq(variables['uploadTestResult'], 'true'))
$jobs = @()
foreach ($file in $toProcess){
$jobs += Start-ThreadJob -ScriptBlock {
$params = $using:file
& dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml
}
}
Write-Host "Merging started..."
Wait-Job -Job $jobs
foreach ($job in $jobs) {
Receive-Job -Job $job -Wait -AutoRemoveJob
}
}
MergeFiles -InputDirectoryPath "$(netFxDir)" -OutputDirectoryName "coverageNetFxXml"
MergeFiles -InputDirectoryPath "$(netCoreDir)" -OutputDirectoryName "coverageNetCoreXml"
Write-Host "Removing original coverage files..."
Remove-Item $(netFxDir) -Recurse -Force -ErrorAction SilentlyContinue
Remove-Item $(netCoreDir) -Recurse -Force -ErrorAction SilentlyContinue
displayName: Convert coverage files to xml
- ${{if eq(parameters.debug, true)}}:
- pwsh: Get-Volume
displayName: '[Debug] Show Disk Usage'

- pwsh: |
dir coverageNetFxXml\
dir coverageNetCoreXml\
displayName: '[Debug] List converted files'
- pwsh: |
$jobs = @()
$jobs += Start-ThreadJob -ScriptBlock {
& reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
}
$jobs += Start-ThreadJob -ScriptBlock {
& reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
}
$jobs += Start-ThreadJob -ScriptBlock {
& reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
}
Write-Host "Running ReportGenerator..."
Wait-Job -Job $jobs
foreach ($job in $jobs) {
Receive-Job -Job $job -Wait -AutoRemoveJob
}
Write-Host "Removing merged XML files..."
Remove-Item coverageNetFxXml -Recurse -Force -ErrorAction SilentlyContinue
Remove-Item coverageNetCoreXml -Recurse -Force -ErrorAction SilentlyContinue
displayName: Run ReportGenerator
- ${{if eq(parameters.debug, true)}}:
- pwsh: Get-Volume
displayName: '[Debug] Show Disk Usage'

- task: PublishCodeCoverageResults@2
displayName: Publish code coverage results
inputs:
summaryFileLocation: '*\Cobertura.xml'

- ${{if eq(parameters.upload, true)}}:
- pwsh: |
#download Codecov CLI
$ProgressPreference = 'SilentlyContinue'
Invoke-WebRequest -Uri https://cli.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetFx\Cobertura.xml" -F netfx
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportNetCore\Cobertura.xml" -F netcore
./codecov --verbose upload-process --fail-on-error -t $(CODECOV_TOKEN) -f "coveragereportAddOns\Cobertura.xml" -F addons
displayName: Upload to CodeCov
18 changes: 4 additions & 14 deletions eng/pipelines/dotnet-sqlclient-ci-core.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,6 @@ variables:
- name: artifactName
value: Artifacts

- name: defaultHostedPoolName
value: 'Azure Pipelines'

stages:
- stage: build_nugets
displayName: 'Build NuGet Packages'
Expand Down Expand Up @@ -152,16 +149,9 @@ stages:
- template: common/templates/jobs/ci-code-coverage-job.yml@self
parameters:
debug: ${{ parameters.debug }}
downloadArtifactsSteps:
- ${{ each targetFramework in parameters.codeCovTargetFrameworks }}:
- task: DownloadPipelineArtifact@2
displayName: 'Download Coverage Reports [${{ targetFramework }}]'
inputs:
itemPattern: '**\${{ targetFramework }}*'
${{ if contains(targetFramework, 'net4') }}:
targetPath: '$(Build.SourcesDirectory)\coverageNetFx'
${{ else }}:
targetPath: '$(Build.SourcesDirectory)\coverageNetCore'
image: ADO-MMS22-CodeCov
pool: ${{ parameters.defaultPoolName }}
targetFrameworks: ${{ parameters.codeCovTargetFrameworks }}

# test stages configurations
# self hosted SQL Server on Windows
Expand Down Expand Up @@ -510,7 +500,7 @@ stages:

# Self hosted SQL Server on Mac
mac_sql_22:
pool: $(defaultHostedPoolName)
pool: Azure Pipelines
hostedPool: true
images:
MacOSLatest_Sql22: macos-latest
Expand Down
Loading