From 0214f2841783ef9e7b8afd72a33123a683ec8e30 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 31 Aug 2026 14:38:50 -0500 Subject: [PATCH 01/70] code coverage migration in TheRock --- .../multi_arch_build_portable_linux.yml | 115 +++++++ ...ti_arch_build_portable_linux_artifacts.yml | 18 +- .github/workflows/multi_arch_ci_coverage.yml | 172 +++++++++++ .../multi_arch_ci_coverage_linux.yml | 268 ++++++++++++++++ .../multi_arch_ci_coverage_nightly.yml | 163 ++++++++++ .github/workflows/test_component.yml | 38 +++ CMakeLists.txt | 51 ++++ build_tools/artifact_manager.py | 34 ++- .../github_actions/configure_coverage_ci.py | 288 ++++++++++++++++++ .../github_actions/merge_coverage_report.py | 186 +++++++++++ .../tests/configure_coverage_ci_test.py | 222 ++++++++++++++ .../tests/merge_coverage_report_test.py | 213 +++++++++++++ .../tests/artifact_manager_tool_test.py | 62 ++++ cmake/therock_subproject.cmake | 9 + compiler/CMakeLists.txt | 8 + docs/development/README.md | 1 + docs/development/ci_overview.md | 9 + docs/development/code_coverage.md | 145 +++++++++ docs/development/development_guide.md | 19 ++ 19 files changed, 2014 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/multi_arch_ci_coverage.yml create mode 100644 .github/workflows/multi_arch_ci_coverage_linux.yml create mode 100644 .github/workflows/multi_arch_ci_coverage_nightly.yml create mode 100644 build_tools/github_actions/configure_coverage_ci.py create mode 100644 build_tools/github_actions/merge_coverage_report.py create mode 100644 build_tools/github_actions/tests/configure_coverage_ci_test.py create mode 100644 build_tools/github_actions/tests/merge_coverage_report_test.py create mode 100644 docs/development/code_coverage.md diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index 32fdafe641f..f296098c05e 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -41,6 +41,24 @@ on: type: string default: "" description: "Comma-separated build stages to skip (artifacts already copied by the orchestrator)" + build_stages: + type: string + default: "" + description: >- + Comma-separated allowlist of build stages to run. Empty runs every + stage. Variant pipelines such as coverage use this to build only the + stages they need. + stage_projects: + type: string + default: "" + description: >- + Space-separated project names to narrow per-arch stages down to + (e.g. "hiprand"). Empty builds every project in the stage. + extra_cmake_options: + type: string + default: "" + description: >- + Extra -D options appended to each stage's configure command. skip_stages: type: string default: "" @@ -96,6 +114,13 @@ jobs: # STAGE: compiler-runtime (generic) # ========================================================================== compiler-runtime: + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'compiler-runtime') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',compiler-runtime,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') && !contains(inputs.skip_stages, 'compiler-runtime') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -106,6 +131,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -121,6 +148,13 @@ jobs: # ========================================================================== runtime-tests: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'runtime-tests') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',runtime-tests,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') && !contains(inputs.skip_stages, 'runtime-tests') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -131,6 +165,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -146,6 +182,13 @@ jobs: # ========================================================================== wsl-rocdxg: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',wsl-rocdxg,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && !contains(inputs.skip_stages, 'wsl-rocdxg') }} uses: ./.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml secrets: inherit @@ -170,6 +213,13 @@ jobs: # ========================================================================== math-libs: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'math-libs') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',math-libs,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') && !contains(inputs.skip_stages, 'math-libs') }} strategy: # `fail-fast: false` lets all families complete even if one fails. @@ -190,6 +240,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -208,6 +260,13 @@ jobs: # ========================================================================== comm-libs: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'comm-libs') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',comm-libs,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'comm-libs') && !contains(inputs.skip_stages, 'comm-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -219,6 +278,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -235,6 +296,13 @@ jobs: # ========================================================================== storage-libs: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'storage-libs') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',storage-libs,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'storage-libs') && !contains(inputs.skip_stages, 'storage-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -246,6 +314,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -261,6 +331,13 @@ jobs: # ========================================================================== debug-tools: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'debug-tools') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',debug-tools,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') && !contains(inputs.skip_stages, 'debug-tools') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -271,6 +348,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -286,6 +365,13 @@ jobs: # ========================================================================== dctools-core: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'dctools-core') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',dctools-core,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'dctools-core') && !contains(inputs.skip_stages, 'dctools-core') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -296,6 +382,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -311,6 +399,13 @@ jobs: # ========================================================================== profiler-apps: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'profiler-apps') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',profiler-apps,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'profiler-apps') && !contains(inputs.skip_stages, 'profiler-apps') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -321,6 +416,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -336,6 +433,13 @@ jobs: # ========================================================================== cv-libs: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'cv-libs') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',cv-libs,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'cv-libs') && !contains(inputs.skip_stages, 'cv-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -346,6 +450,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -361,6 +467,13 @@ jobs: # ========================================================================== media-libs: needs: compiler-runtime + if: >- + ${{ + !cancelled() && !failure() && + !contains(inputs.prebuilt_stages, 'media-libs') && + (inputs.build_stages == '' || + contains(format(',{0},', inputs.build_stages), ',media-libs,')) + }} if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'media-libs') && !contains(inputs.skip_stages, 'media-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit @@ -371,6 +484,8 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} + stage_projects: ${{ inputs.stage_projects }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 45118192fe3..282120ca42e 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -49,6 +49,20 @@ on: required: false default: "" description: "CMake preset name for this build variant (optional)" + stage_projects: + type: string + required: false + default: "" + description: >- + Space-separated project names to narrow the stage down to (e.g. + "hiprand"). Empty builds every project in the stage. + extra_cmake_options: + type: string + required: false + default: "" + description: >- + Extra -D options appended to the configure command, after the + stage-derived arguments. Used by variant builds such as coverage. release_type: description: 'Release type; controls artifact bucket and IAM role' type: string @@ -173,6 +187,7 @@ jobs: --stage="${STAGE_NAME}" \ --amdgpu-families="${{ inputs.amdgpu_family }}" \ --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ + ${{ inputs.stage_projects && format('--projects {0}', inputs.stage_projects) || '' }} \ --manylinux \ --gha-output @@ -189,7 +204,8 @@ jobs: ${{ inputs.build_variant_cmake_preset && format('--preset={0}', inputs.build_variant_cmake_preset) }} \ ${{ inputs.external_repo_config != '' && format('-DTHEROCK_{0}_SOURCE_DIR={1}', fromJSON(inputs.external_repo_config).source_package, fromJSON(inputs.external_repo_config).checkout_path) || '' }} \ ${{ inputs.external_repo_config != '' && fromJSON(inputs.external_repo_config).extra_cmake_options || '' }} \ - ${{ steps.stage_config.outputs.cmake_args }} + ${{ steps.stage_config.outputs.cmake_args }} \ + ${{ inputs.extra_cmake_options }} - name: Build stage run: | diff --git a/.github/workflows/multi_arch_ci_coverage.yml b/.github/workflows/multi_arch_ci_coverage.yml new file mode 100644 index 00000000000..3f2224a47b6 --- /dev/null +++ b/.github/workflows/multi_arch_ci_coverage.yml @@ -0,0 +1,172 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Multi-Arch CI Coverage +# +# Entry point for source based code coverage CI. Coverage builds are expensive +# and only meaningful for projects whose test suites report something useful, +# so this does not run on every PR: dispatch it manually, or add the +# `ci:coverage` label to a pull request. +# +# The instrumented compiler-runtime stage is built once and shared, then each +# selected project rebuilds only its own stage with instrumentation enabled. + +name: Multi-Arch CI Coverage + +on: + workflow_dispatch: + inputs: + projects_to_test: + type: string + description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." + default: "" + amdgpu_families: + type: string + description: "Comma-separated GPU families to collect coverage on." + default: "gfx94X-dcgpu" + coverage_config_source: + type: string + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" + test_type: + type: string + description: "Test type to run against the instrumented build." + default: "quick" + pull_request: + types: + - labeled + - opened + - synchronize + +permissions: + contents: read + +concurrency: + # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency + # + # `github.workflow`: each workflow gets its own concurrency group + # `github.head_ref`: the branch name (only defined for pull_request events) + # `github.run_id` : fallback, unique for each run + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + setup_coverage_matrix: + name: Setup Coverage Matrix + # Coverage on a PR is opt-in: without the label this workflow does nothing. + if: >- + ${{ + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'ci:coverage') + }} + runs-on: ubuntu-24.04 + outputs: + coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} + dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} + families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} + rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} + steps: + - name: Checking out repository + id: checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Generate coverage matrix + id: matrix + env: + PROJECTS_TO_TEST: ${{ inputs.projects_to_test || '' }} + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} + COVERAGE_CONFIG_SOURCE: ${{ inputs.coverage_config_source || 'ROCm/rocm-libraries@main' }} + run: python ./build_tools/github_actions/configure_coverage_ci.py + + - name: Compute package version + id: rocm_package_version + run: | + python ./build_tools/compute_rocm_package_version.py \ + --release-type=ci \ + --override-git-sha=${{ steps.checkout.outputs.commit }} + + # Instrumented device code needs the ROCm profiling runtime, which lives in + # compiler-rt. Building this stage once keeps the per-project jobs cheap and + # avoids several jobs racing to publish the same compiler artifacts. + build_instrumented_compiler_runtime: + name: Build Instrumented Compiler Runtime + needs: setup_coverage_matrix + if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} + uses: ./.github/workflows/multi_arch_build_portable_linux.yml + secrets: inherit + with: + artifact_group: multi-arch-coverage + matrix_per_family_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + build_variant_label: coverage + build_variant_cmake_preset: "" + build_variant_suffix: coverage + build_stages: compiler-runtime + extra_cmake_options: >- + -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON + -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + release_type: ci + permissions: + contents: read + id-token: write + + coverage_linux: + name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) + needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + # `!failure() && !cancelled()` alone would let this run when + # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so + # the matrix has to be checked explicitly. + if: >- + ${{ + !failure() && !cancelled() && + needs.setup_coverage_matrix.result == 'success' && + needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' + }} + strategy: + fail-fast: false + matrix: + coverage: ${{ fromJSON(needs.setup_coverage_matrix.outputs.coverage_matrix) }} + uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml + secrets: inherit + with: + project_name: ${{ matrix.coverage.project_name }} + coverage_flag: ${{ matrix.coverage.coverage_flag }} + build_stage: ${{ matrix.coverage.build_stage }} + stage_project: ${{ matrix.coverage.stage_project }} + test_component: ${{ matrix.coverage.test_component }} + object_globs: ${{ matrix.coverage.object_globs }} + fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} + codecov_flag: ${{ matrix.coverage.codecov_flag }} + amdgpu_families: ${{ matrix.coverage.amdgpu_families }} + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + test_type: ${{ inputs.test_type || 'quick' }} + release_type: ci + permissions: + contents: read + id-token: write + + ci_summary: + name: Coverage CI Summary + if: always() + needs: + - setup_coverage_matrix + - build_instrumented_compiler_runtime + - coverage_linux + runs-on: ubuntu-24.04 + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + - name: Evaluate workflow results + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python build_tools/github_actions/workflow_summary.py \ + --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml new file mode 100644 index 00000000000..6d89808f1be --- /dev/null +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -0,0 +1,268 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Multi-Arch CI Coverage - Linux +# +# Runs the coverage pipeline for a single project on a single GPU family: +# +# build (instrumented) -> test (collect profraw) -> merge -> report +# +# The compiler-runtime stage is expected to have been built already, by the +# caller, with the ROCm profiling runtime enabled. This workflow rebuilds only +# the stage that owns the target project, so a coverage run costs roughly one +# extra library build per project rather than a full instrumented stack. + +name: Multi-Arch CI Coverage - Linux + +on: + workflow_call: + inputs: + project_name: + type: string + description: "Coverage project key (e.g. hiprand)" + build_enabled: + type: boolean + default: true + description: >- + Build the instrumented project. Set to false when the caller already + built a full instrumented stack in this run (nightly coverage) and + only the test and report halves are needed. + coverage_flag: + type: string + description: "Per-project CMake coverage flag (e.g. HIPRAND_ENABLE_COVERAGE)" + build_stage: + type: string + default: math-libs + description: "Build stage that produces the instrumented project" + stage_project: + type: string + description: "Project name passed to configure_stage.py --projects" + test_component: + type: string + description: "Test matrix key for the project's tests" + object_globs: + type: string + description: "Comma-separated globs matching the instrumented binaries" + fetch_artifact_args: + type: string + default: "" + description: "Extra install_rocm_from_artifacts.py args for the report job" + codecov_flag: + type: string + default: "" + description: "Codecov flag to file the report under" + amdgpu_families: + type: string + description: "GPU family to build and test coverage for" + extra_cmake_options: + type: string + default: "" + description: "Additional CMake options for the instrumented build" + rocm_package_version: + type: string + test_type: + type: string + default: quick + release_type: + type: string + default: ci + repository: + description: "Repository to checkout. Defaults to github.repository." + type: string + default: "" + ref: + description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." + type: string + default: "" + +permissions: + contents: read + +jobs: + build_coverage: + name: Build Coverage (${{ inputs.project_name }}) + if: ${{ inputs.build_enabled }} + uses: ./.github/workflows/multi_arch_build_portable_linux.yml + secrets: inherit + with: + artifact_group: multi-arch-coverage + matrix_per_family_json: >- + [{"amdgpu_family": "${{ inputs.amdgpu_families }}"}] + dist_amdgpu_families: ${{ inputs.amdgpu_families }} + build_variant_label: coverage + build_variant_cmake_preset: "" + build_variant_suffix: coverage + build_stages: ${{ inputs.build_stage }} + stage_projects: ${{ inputs.stage_project }} + # KPACK artifact splitting is disabled so the instrumented device code + # stays in the library that llvm-cov is pointed at. + extra_cmake_options: >- + -D${{ inputs.coverage_flag }}=ON + -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON + -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + ${{ inputs.extra_cmake_options }} + rocm_package_version: ${{ inputs.rocm_package_version }} + release_type: ${{ inputs.release_type }} + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + permissions: + contents: read + id-token: write + + configure_test_matrix: + name: Configure Coverage Tests (${{ inputs.project_name }}) + needs: build_coverage + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-24.04 + outputs: + components: ${{ steps.configure.outputs.components }} + platform: ${{ steps.configure.outputs.platform }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} + + - name: Checkout CI config + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ROCm/therock-ci-config + ref: main + path: ci-config + continue-on-error: true + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Configuring coverage test options + id: configure + env: + ARTIFACT_GROUP: multi-arch-coverage + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} + TEST_TYPE: ${{ inputs.test_type }} + TEST_LABELS: "[]" + # Only the instrumented project was rebuilt, so only its tests can + # contribute coverage. + PROJECTS_TO_TEST: ${{ inputs.test_component }} + CI_CONFIG_PATH: ci-config + BUILD_VARIANT: coverage + run: | + python ./build_tools/github_actions/fetch_test_configurations.py \ + --platform=linux + + test_coverage: + name: Test Coverage (${{ inputs.project_name }}) + needs: [build_coverage, configure_test_matrix] + if: >- + ${{ + !failure() && !cancelled() && + needs.configure_test_matrix.outputs.components != '' && + needs.configure_test_matrix.outputs.components != '[]' + }} + strategy: + fail-fast: false + matrix: + components: ${{ fromJSON(needs.configure_test_matrix.outputs.components || '[]') }} + uses: ./.github/workflows/test_component.yml + secrets: inherit + with: + artifact_group: multi-arch-coverage + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ matrix.components.test_runner }} + platform: ${{ needs.configure_test_matrix.outputs.platform }} + component: ${{ toJSON(matrix.components) }} + release_type: ${{ inputs.release_type }} + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + build_variant: coverage + coverage_enabled: true + coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + + aggregate_coverage: + name: Coverage Report (${{ inputs.project_name }}) + needs: [configure_test_matrix, test_coverage] + # Runs even when some shards failed: a partial report is more useful than + # none, and a failing test still exercises code. + if: >- + ${{ + !cancelled() && + needs.configure_test_matrix.result == 'success' && + needs.configure_test_matrix.outputs.components != '[]' + }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build + PROFRAW_DIR: ${{ github.workspace }}/coverage-profraw + REPORT_DIR: ${{ github.workspace }}/coverage-report + HAS_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN != '' }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Download profraw files from all shards + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }}-* + path: ${{ env.PROFRAW_DIR }} + continue-on-error: true + + - name: Install python deps + run: pip install -r requirements.txt + + - name: Download instrumented artifacts + env: + GITHUB_TOKEN: ${{ github.token }} + RELEASE_TYPE: ${{ inputs.release_type }} + FETCH_ARTIFACT_ARGS: ${{ inputs.fetch_artifact_args }} + run: | + python ./build_tools/install_rocm_from_artifacts.py \ + --run-id=${{ github.run_id }} \ + --artifact-group=${{ inputs.amdgpu_families }} \ + --output-dir=${OUTPUT_ARTIFACTS_DIR} \ + ${FETCH_ARTIFACT_ARGS} + + - name: Merge profraw and generate lcov report + run: | + python ./build_tools/github_actions/merge_coverage_report.py \ + --profraw-dir="${PROFRAW_DIR}" \ + --rocm-dir="${OUTPUT_ARTIFACTS_DIR}" \ + --object-globs="${{ inputs.object_globs }}" \ + --profdata-output="${REPORT_DIR}/coverage.profdata" \ + --lcov-output="${REPORT_DIR}/coverage.info" + + - name: Upload coverage report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-report-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + path: ${{ env.REPORT_DIR }}/coverage.info + if-no-files-found: warn + + # Skipped rather than failed when the repository has no Codecov token, + # so forks and first-time setups still get the lcov artifact above. + - name: Upload report to Codecov + if: ${{ env.HAS_CODECOV_TOKEN == 'true' }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ env.REPORT_DIR }}/coverage.info + flags: ${{ inputs.codecov_flag || inputs.project_name }} + name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + fail_ci_if_error: false diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml new file mode 100644 index 00000000000..9861c97687b --- /dev/null +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -0,0 +1,163 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Multi-Arch CI Coverage - Nightly +# +# Builds one fully instrumented stack covering every coverage-enabled project, +# then runs each project's tests against it and reports separately. Compared to +# the per-PR coverage workflow this pays for a single large build instead of one +# build per project, which is the right trade-off once several projects are +# enabled. +# +# Phase 1 of RFC0014 runs every enabled component on the default architecture. +# Change-based selection and multi-architecture expansion come later. + +name: Multi-Arch CI Coverage Nightly + +on: + schedule: + # 3 AM UTC, after the regular nightly builds have settled. + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + projects_to_test: + type: string + description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." + default: "" + amdgpu_families: + type: string + description: "Comma-separated GPU families to collect coverage on." + default: "gfx94X-dcgpu" + coverage_config_source: + type: string + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" + test_type: + type: string + description: "Test type to run against the instrumented build." + default: "standard" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.run_id }} + cancel-in-progress: false + +jobs: + setup_coverage_matrix: + name: Setup Coverage Matrix + runs-on: ubuntu-24.04 + outputs: + coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} + dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} + families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} + coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} + build_stages: ${{ steps.matrix.outputs.build_stages }} + rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} + steps: + - name: Checking out repository + id: checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Generate coverage matrix + id: matrix + env: + PROJECTS_TO_TEST: ${{ inputs.projects_to_test || '' }} + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} + COVERAGE_CONFIG_SOURCE: ${{ inputs.coverage_config_source || 'ROCm/rocm-libraries@main' }} + run: python ./build_tools/github_actions/configure_coverage_ci.py + + - name: Compute package version + id: rocm_package_version + run: | + python ./build_tools/compute_rocm_package_version.py \ + --release-type=ci \ + --override-git-sha=${{ steps.checkout.outputs.commit }} + + build_instrumented_stack: + name: Build Full Instrumented Stack + needs: setup_coverage_matrix + if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} + uses: ./.github/workflows/multi_arch_build_portable_linux.yml + secrets: inherit + with: + artifact_group: multi-arch-coverage + matrix_per_family_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + build_variant_label: coverage + build_variant_cmake_preset: "" + build_variant_suffix: coverage + build_stages: ${{ needs.setup_coverage_matrix.outputs.build_stages }} + extra_cmake_options: >- + ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON + -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + release_type: ci + permissions: + contents: read + id-token: write + + # One test + report job per project, all consuming the single instrumented + # stack built above (same run id, so the artifacts are already published). + coverage_report: + name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) + needs: [setup_coverage_matrix, build_instrumented_stack] + # `!failure() && !cancelled()` alone would let this run when + # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so + # the matrix has to be checked explicitly. + if: >- + ${{ + !failure() && !cancelled() && + needs.setup_coverage_matrix.result == 'success' && + needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' + }} + strategy: + fail-fast: false + matrix: + coverage: ${{ fromJSON(needs.setup_coverage_matrix.outputs.coverage_matrix) }} + uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml + secrets: inherit + with: + build_enabled: false + project_name: ${{ matrix.coverage.project_name }} + coverage_flag: ${{ matrix.coverage.coverage_flag }} + build_stage: ${{ matrix.coverage.build_stage }} + stage_project: ${{ matrix.coverage.stage_project }} + test_component: ${{ matrix.coverage.test_component }} + object_globs: ${{ matrix.coverage.object_globs }} + fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} + codecov_flag: ${{ matrix.coverage.codecov_flag }} + amdgpu_families: ${{ matrix.coverage.amdgpu_families }} + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + test_type: ${{ inputs.test_type || 'standard' }} + release_type: ci + permissions: + contents: read + id-token: write + + ci_summary: + name: Nightly Coverage Summary + if: always() + needs: + - setup_coverage_matrix + - build_instrumented_stack + - coverage_report + runs-on: ubuntu-24.04 + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + - name: Evaluate workflow results + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python build_tools/github_actions/workflow_summary.py \ + --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index 183b518815b..c231cf0f84d 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -35,6 +35,20 @@ on: description: "Build variant (e.g., release, asan). Used to skip tests incompatible with certain builds." type: string default: "release" + coverage_enabled: + description: >- + Collect llvm profraw files from the test run and upload them as a + workflow artifact. Only meaningful against a coverage instrumented + build. + type: boolean + default: false + coverage_artifact_prefix: + description: >- + Prefix for the uploaded profraw artifact name. A coverage run tests + several projects in one workflow run, so the caller sets this to + something project-specific to keep each project's profiles apart. + type: string + default: "coverage-profraw" default_container_image: type: string default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" @@ -178,6 +192,18 @@ jobs: echo "HSA_XNACK=1" >> $GITHUB_ENV echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV + # Instrumented binaries write one profraw per process. %p (pid) and %m + # (binary signature) keep the files distinct so a shard that forks or + # loads several instrumented libraries does not overwrite itself. + - name: Prepare coverage profile directory + if: ${{ inputs.coverage_enabled }} + env: + TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }} + SHARD_INDEX: ${{ matrix.shard }} + run: | + mkdir -p "${GITHUB_WORKSPACE}/coverage-report/profraw" + echo "LLVM_PROFILE_FILE=${GITHUB_WORKSPACE}/coverage-report/profraw/${TEST_COMPONENT}-shard${SHARD_INDEX}-%p-%m.profraw" >> "${GITHUB_ENV}" + # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription. # Must be done at runtime since container env vars aren't in GitHub's env context. - name: Set thread limits @@ -219,6 +245,18 @@ jobs: run: | ${{ fromJSON(inputs.component).test_script }} + # Uploaded even when the test step failed: partial coverage from a failing + # shard is still useful, and the aggregation job tolerates missing shards. + - name: Upload coverage profraw files + if: ${{ always() && inputs.coverage_enabled }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: >- + ${{ inputs.coverage_artifact_prefix }}-${{ fromJSON(inputs.component).job_name }}-shard${{ matrix.shard }} + path: ${{ github.workspace }}/coverage-report/profraw/*.profraw + if-no-files-found: warn + retention-days: 3 + - name: Print test reproduction command if: ${{ failure() && steps.test.outcome == 'failure' }} run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 727f549ecde..8ca4655d235 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -212,6 +212,57 @@ option(THEROCK_COMPILER_RT_DEBUG "Enables compiler-rt debug build" OFF) option(THEROCK_QUIET_INSTALL "Enable quiet install logging (install logs only go to the logfile)" ON) option(THEROCK_USE_SAFE_DEPENDENCY_PROVIDER "Enable the safe dependency provider, performing strict package checks" ON) +################################################################################ +# Coverage Options +# Enable code-coverage instrumentation for a whole component group by setting the +# per-project _ENABLE_COVERAGE flags that therock_subproject.cmake +# forwards to each subproject. The per-group project lists are generated from the +# coverage registry (COVERAGE_PROJECTS in +# build_tools/github_actions/configure_coverage_ci.py) at configure time. +################################################################################ + +option(THEROCK_COVERAGE_ROCM_LIBRARIES_ALL + "Enable coverage instrumentation for all rocm-libraries projects" OFF) +option(THEROCK_COVERAGE_ROCM_SYSTEMS_ALL + "Enable coverage instrumentation for all rocm-systems projects" OFF) +option(THEROCK_COVERAGE_ALL + "Enable coverage instrumentation for all coverage-enabled projects" OFF) + +# Emit the per-group project lists from COVERAGE_PROJECTS. CMAKE_CONFIGURE_DEPENDS +# re-runs configure when that file changes. +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/build_tools/github_actions/configure_coverage_ci.py") +file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/cmake") +execute_process( + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/build_tools/github_actions/configure_coverage_ci.py" + "--emit-cmake" "${CMAKE_CURRENT_BINARY_DIR}/cmake/therock_coverage_projects.cmake" + COMMAND_ERROR_IS_FATAL ANY) +include("${CMAKE_CURRENT_BINARY_DIR}/cmake/therock_coverage_projects.cmake") + +set(_therock_coverage_selected "") +if(THEROCK_COVERAGE_ALL OR THEROCK_COVERAGE_ROCM_LIBRARIES_ALL) + list(APPEND _therock_coverage_selected ${THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS}) +endif() +if(THEROCK_COVERAGE_ALL OR THEROCK_COVERAGE_ROCM_SYSTEMS_ALL) + list(APPEND _therock_coverage_selected ${THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS}) +endif() + +if(THEROCK_COVERAGE_ROCM_SYSTEMS_ALL AND NOT THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS) + message(WARNING + "THEROCK_COVERAGE_ROCM_SYSTEMS_ALL is ON but no rocm-systems projects are " + "onboarded to coverage yet; nothing will be instrumented for that group.") +endif() + +foreach(_proj IN LISTS _therock_coverage_selected) + string(TOUPPER "${_proj}" _proj_upper) + # Respect an explicit -D_ENABLE_COVERAGE=OFF from the command line. + if(NOT DEFINED ${_proj_upper}_ENABLE_COVERAGE) + set(${_proj_upper}_ENABLE_COVERAGE ON) + message(STATUS "Coverage: enabling ${_proj_upper}_ENABLE_COVERAGE") + endif() +endforeach() + ################################################################################ # Feature selection # Each feature added via therock_add_feature produces a boolean cache variable diff --git a/build_tools/artifact_manager.py b/build_tools/artifact_manager.py index 8fd55e6fdc8..41a011f52d9 100644 --- a/build_tools/artifact_manager.py +++ b/build_tools/artifact_manager.py @@ -236,11 +236,12 @@ def parse_excluded_components(raw_components: str) -> set[str]: return components -def parse_excluded_artifacts( +def parse_artifact_names( raw_artifacts: str, valid_artifacts: set[str], + option_name: str, ) -> set[str]: - """Parse and validate artifact names passed to --exclude-artifacts.""" + """Parse and validate a comma- or semicolon-separated artifact name list.""" artifacts = { artifact.strip() for artifact in raw_artifacts.replace(";", ",").split(",") @@ -249,7 +250,7 @@ def parse_excluded_artifacts( invalid_artifacts = artifacts - valid_artifacts if invalid_artifacts: raise ValueError( - "Invalid artifact name(s): " + f"Invalid artifact name(s) for {option_name}: " f"{', '.join(sorted(invalid_artifacts))}. " f"Valid artifacts are: {', '.join(sorted(valid_artifacts))}" ) @@ -469,9 +470,23 @@ def do_fetch(args: argparse.Namespace): ) target_families = parse_target_families(args) - excluded_artifacts = parse_excluded_artifacts( - args.exclude_artifacts, - set(topology.artifacts.keys()), + valid_artifacts = set(topology.artifacts.keys()) + + # An include list narrows the stage's inbound set; hybrid fetches use it to + # pull one instrumented artifact from a coverage run and everything else + # from a regular run. + included_artifacts = parse_artifact_names( + args.artifact_names, valid_artifacts, "--artifact-names" + ) + if included_artifacts: + log(f"Limiting to artifacts: {', '.join(sorted(included_artifacts))}") + inbound &= included_artifacts + if not inbound: + log("No requested artifacts are inbound for this stage") + return + + excluded_artifacts = parse_artifact_names( + args.exclude_artifacts, valid_artifacts, "--exclude-artifacts" ) if excluded_artifacts: log(f"Excluding artifacts: {', '.join(sorted(excluded_artifacts))}") @@ -1331,6 +1346,13 @@ def main(argv: Optional[List[str]] = None): help="Comma- or semicolon-separated artifact components to exclude " f"when fetching. Valid components: {', '.join(ARTIFACT_COMPONENTS)}", ) + fetch_parser.add_argument( + "--artifact-names", + type=str, + default="", + help="Comma- or semicolon-separated artifact names to fetch. Empty " + "(the default) fetches every artifact inbound to the stage.", + ) fetch_parser.add_argument( "--exclude-artifacts", type=str, diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py new file mode 100644 index 00000000000..eb1968aca79 --- /dev/null +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -0,0 +1,288 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Builds the job matrix for TheRock's code coverage CI workflows. + +Coverage is opt-in per project: instrumenting a library slows it down +substantially and only pays off for projects whose test suites are good enough +to produce a meaningful signal. `COVERAGE_PROJECTS` below is the allowlist, and +everything the coverage workflows need to know about a project lives there. + +Environment variables: + - PROJECTS_TO_TEST: comma-separated project keys; empty selects every + project in the allowlist. + - AMDGPU_FAMILIES: comma-separated GPU families to build coverage for. + - COVERAGE_CONFIG_SOURCE: "/@" holding the per-project + coverage metadata files (currently ROCm/rocm-libraries). + +Outputs (GITHUB_OUTPUT): + - coverage_matrix: JSON array of per-project job configurations. + - dist_amdgpu_families: semicolon-separated families, as CMake expects them. + - families_matrix_json: JSON array of {amdgpu_family} objects for the + per-arch stages of multi_arch_build_portable_linux.yml. + - coverage_cmake_options: every selected project's coverage flag, for the + nightly full-stack instrumented build. + - build_stages: comma-separated stages the selected projects live in. +""" + +import argparse +import json +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +sys.path.insert(0, os.fspath(Path(__file__).resolve().parent)) + +from github_actions_api import gha_set_output + +DEFAULT_AMDGPU_FAMILIES = "gfx94X-dcgpu" +DEFAULT_COVERAGE_CONFIG_SOURCE = "ROCm/rocm-libraries@main" + +# Values for CoverageProject.source_repo: the repo a project ships from. Drives +# the group aliases (rocm_libraries_all / rocm_systems_all) below. +ROCM_LIBRARIES = "rocm-libraries" +ROCM_SYSTEMS = "rocm-systems" + + +@dataclass(frozen=True) +class CoverageProject: + """Everything the coverage pipeline needs to know about one project. + + Attributes: + cmake_target: TheRock subproject name. Upper cased to form the + `_ENABLE_COVERAGE` flag that therock_subproject.cmake + forwards to the subproject. + stage: Build stage that produces the project. + stage_project: Name passed to configure_stage.py --projects, which + narrows the stage down so only this project is rebuilt. + test_component: Key in fetch_test_configurations.py's test matrix. + coverage_config: Per-project coverage metadata file, relative to the + root of the repository named by COVERAGE_CONFIG_SOURCE. + object_globs: Globs, relative to the extracted artifact directory, + matching the instrumented binaries handed to `llvm-cov`. + fetch_artifact_args: Arguments to install_rocm_from_artifacts.py that + pull the instrumented libraries into the report generation job. + codecov_flag: Flag the report is filed under in Codecov. + source_repo: The repo this project ships from (ROCM_LIBRARIES or + ROCM_SYSTEMS); drives the group aliases. + """ + + cmake_target: str + stage: str + stage_project: str + test_component: str + coverage_config: str + object_globs: list[str] = field(default_factory=list) + fetch_artifact_args: str = "" + codecov_flag: str = "" + source_repo: str = ROCM_LIBRARIES + + +# Projects that participate in coverage CI. Start small (RFC0014 phase 1) and +# grow the list as component test suites become good enough to report on. +COVERAGE_PROJECTS: dict[str, CoverageProject] = { + "hiprand": CoverageProject( + cmake_target="hipRAND", + stage="math-libs", + stage_project="hiprand", + test_component="hiprand", + coverage_config="projects/hiprand/test_categories_coverage.yaml", + object_globs=["lib/libhiprand.so*"], + fetch_artifact_args="--rand", + codecov_flag="hipRAND", + ), +} + +_VALID_SOURCE_REPOS = frozenset({ROCM_LIBRARIES, ROCM_SYSTEMS}) +for _key, _proj in COVERAGE_PROJECTS.items(): + assert ( + _proj.source_repo in _VALID_SOURCE_REPOS + ), f"{_key}: source_repo must be one of {sorted(_VALID_SOURCE_REPOS)}" + +ROCM_LIBRARIES_PROJECTS: frozenset[str] = frozenset( + k for k, v in COVERAGE_PROJECTS.items() if v.source_repo == ROCM_LIBRARIES +) +ROCM_SYSTEMS_PROJECTS: frozenset[str] = frozenset( + k for k, v in COVERAGE_PROJECTS.items() if v.source_repo == ROCM_SYSTEMS +) + +# Group-alias tokens accepted by PROJECTS_TO_TEST (and the projects_to_test CI +# input) that expand to a whole component group. Matched case-insensitively. +_GROUP_ALIASES: dict[str, frozenset[str]] = { + "rocm_libraries_all": ROCM_LIBRARIES_PROJECTS, + "rocm_systems_all": ROCM_SYSTEMS_PROJECTS, + "all": frozenset(COVERAGE_PROJECTS), +} + + +def parse_projects(raw_projects: str) -> list[str]: + """Resolves PROJECTS_TO_TEST into an ordered list of known project keys. + + Recognizes the group aliases 'rocm_libraries_all', 'rocm_systems_all', and + 'all', which expand to a whole component group and may be mixed with + explicit project names. Matching is case-insensitive. + """ + requested = [p.strip().lower() for p in raw_projects.split(",") if p.strip()] + if not requested: + return sorted(COVERAGE_PROJECTS) + + expanded: list[str] = [] + unknown: list[str] = [] + for token in requested: + if token in _GROUP_ALIASES: + group = _GROUP_ALIASES[token] + if not group: + raise ValueError( + f"Group alias '{token}' selected no projects: no such " + "projects are onboarded to coverage yet." + ) + expanded.extend(sorted(group)) + elif token in COVERAGE_PROJECTS: + expanded.append(token) + else: + unknown.append(token) + + if unknown: + raise ValueError( + f"Unknown coverage project(s): {', '.join(sorted(unknown))}. " + f"Coverage-enabled projects are: {', '.join(sorted(COVERAGE_PROJECTS))}. " + f"Group aliases: {', '.join(sorted(_GROUP_ALIASES))}." + ) + # Preserve first-seen order, drop duplicates (explicit + alias overlap). + return list(dict.fromkeys(expanded)) + + +def parse_amdgpu_families(raw_families: str) -> list[str]: + families = [f.strip() for f in raw_families.split(",") if f.strip()] + return families or [DEFAULT_AMDGPU_FAMILIES] + + +def parse_config_source(raw_source: str) -> tuple[str, str]: + """Splits "/@" into its repository and ref.""" + source = raw_source.strip() or DEFAULT_COVERAGE_CONFIG_SOURCE + repository, separator, ref = source.partition("@") + if not separator or not repository or not ref: + raise ValueError( + f"Invalid coverage config source '{raw_source}'. " + "Expected the form '/@'." + ) + return repository, ref + + +def build_coverage_matrix( + project_keys: list[str], + amdgpu_families: list[str], + config_repository: str, + config_ref: str, +) -> list[dict]: + """Produces one job configuration per (project, GPU family) pair.""" + matrix = [] + for project_key in project_keys: + project = COVERAGE_PROJECTS[project_key] + for family in amdgpu_families: + matrix.append( + { + "project_name": project_key, + # therock_subproject.cmake only honors the upper case + # spelling of the per-project coverage flag. + "coverage_flag": f"{project.cmake_target.upper()}_ENABLE_COVERAGE", + "cmake_target": project.cmake_target, + "build_stage": project.stage, + "stage_project": project.stage_project, + "test_component": project.test_component, + "coverage_config": project.coverage_config, + "coverage_config_repository": config_repository, + "coverage_config_ref": config_ref, + "object_globs": ",".join(project.object_globs), + "fetch_artifact_args": project.fetch_artifact_args, + "codecov_flag": project.codecov_flag or project_key, + "amdgpu_families": family, + } + ) + return matrix + + +def emit_cmake(output_path: Path) -> None: + """Writes the coverage group-membership lists as a CMake include. + + Groups COVERAGE_PROJECTS by source_repo and emits one set() per group, using + each project's cmake_target (upper cased by CMake to form the + _ENABLE_COVERAGE flag). CMakeLists.txt reads this so it does not + hardcode the project names. + """ + + def targets(repo: str) -> list[str]: + return sorted( + ( + p.cmake_target + for p in COVERAGE_PROJECTS.values() + if p.source_repo == repo + ), + key=str.lower, + ) + + output_path.write_text( + "# Generated by configure_coverage_ci.py --emit-cmake; do not edit.\n" + "# Edit COVERAGE_PROJECTS in\n" + "# build_tools/github_actions/configure_coverage_ci.py instead.\n" + f"set(THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS {' '.join(targets(ROCM_LIBRARIES))})\n" + f"set(THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS {' '.join(targets(ROCM_SYSTEMS))})\n" + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--print", + action="store_true", + dest="print_matrix", + help="Print the matrix to stdout instead of writing GITHUB_OUTPUT", + ) + parser.add_argument( + "--emit-cmake", + type=Path, + default=None, + metavar="PATH", + help="Write the coverage group lists as a CMake include to PATH, then exit.", + ) + args = parser.parse_args(argv) + + if args.emit_cmake is not None: + emit_cmake(args.emit_cmake) + return 0 + + project_keys = parse_projects(os.getenv("PROJECTS_TO_TEST", "")) + amdgpu_families = parse_amdgpu_families(os.getenv("AMDGPU_FAMILIES", "")) + config_repository, config_ref = parse_config_source( + os.getenv("COVERAGE_CONFIG_SOURCE", "") + ) + + matrix = build_coverage_matrix( + project_keys, amdgpu_families, config_repository, config_ref + ) + coverage_flags = [ + f"-D{COVERAGE_PROJECTS[key].cmake_target.upper()}_ENABLE_COVERAGE=ON" + for key in project_keys + ] + build_stages = sorted({COVERAGE_PROJECTS[key].stage for key in project_keys}) + outputs = { + "coverage_matrix": json.dumps(matrix), + "dist_amdgpu_families": ";".join(amdgpu_families), + "families_matrix_json": json.dumps( + [{"amdgpu_family": family} for family in amdgpu_families] + ), + "coverage_cmake_options": " ".join(coverage_flags), + "build_stages": ",".join(["compiler-runtime"] + build_stages), + } + + if args.print_matrix: + print(json.dumps(outputs, indent=2)) + else: + gha_set_output(outputs) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py new file mode 100644 index 00000000000..e5f4b6eea2a --- /dev/null +++ b/build_tools/github_actions/merge_coverage_report.py @@ -0,0 +1,186 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Turns the profraw files produced by a coverage test run into an lcov report. + +Instrumented binaries emit one `.profraw` per process, so a sharded test job +produces many of them spread across downloaded artifact directories. This +merges them into a single `.profdata` index and exports lcov for the +instrumented objects. + +The `llvm-profdata` and `llvm-cov` binaries must come from the same compiler +that built the instrumented objects; a version mismatch produces an +unhelpfully generic "malformed instrumentation profile data" error. Both live +under `lib/llvm/bin` of an installed ROCm distribution. +""" + +import argparse +import logging +import shutil +import subprocess +import sys +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(message)s") + +EXECUTABLE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +def find_llvm_tool(llvm_bin_dir: Path, tool_name: str) -> Path: + """Locates an LLVM tool, preferring the ROCm distribution's own copy.""" + candidate = llvm_bin_dir / f"{tool_name}{EXECUTABLE_SUFFIX}" + if candidate.is_file(): + return candidate + + fallback = shutil.which(tool_name) + if fallback: + logging.warning( + "%s not found in %s, falling back to %s. A version mismatch with " + "the compiler that produced the profiles may cause failures.", + tool_name, + llvm_bin_dir, + fallback, + ) + return Path(fallback) + + raise FileNotFoundError( + f"Could not find '{tool_name}' in '{llvm_bin_dir}' or on PATH." + ) + + +def find_profraw_files(profraw_dir: Path) -> list[Path]: + return sorted(profraw_dir.rglob("*.profraw")) + + +def resolve_objects(rocm_dir: Path, object_globs: list[str]) -> list[Path]: + """Expands the per-project object globs into concrete instrumented files. + + Globs typically match a versioned family of symlinks (libfoo.so, + libfoo.so.1, ...) that all resolve to one file, so results are deduplicated + by real path to avoid handing llvm-cov the same object several times. + """ + objects: dict[Path, Path] = {} + for pattern in object_globs: + matches = sorted(rocm_dir.glob(pattern)) + if not matches: + logging.warning( + "No files matched object glob '%s' under %s", pattern, rocm_dir + ) + for match in matches: + if match.is_dir(): + continue + objects.setdefault(match.resolve(), match) + return sorted(objects.values()) + + +def merge_profraw(llvm_profdata: Path, profraw_files: list[Path], output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + command = [ + str(llvm_profdata), + "merge", + "-sparse", + "-o", + str(output), + *[str(f) for f in profraw_files], + ] + logging.info("Merging %d profraw file(s) into %s", len(profraw_files), output) + subprocess.run(command, check=True) + + +def export_lcov( + llvm_cov: Path, profdata: Path, objects: list[Path], output: Path +) -> None: + # llvm-cov takes the first object positionally and the rest via -object. + command = [str(llvm_cov), "export", str(objects[0])] + for obj in objects[1:]: + command.extend(["-object", str(obj)]) + command.extend([f"-instr-profile={profdata}", "--format=lcov"]) + + output.parent.mkdir(parents=True, exist_ok=True) + logging.info("Exporting lcov for %d object(s) to %s", len(objects), output) + with open(output, "w") as lcov_file: + subprocess.run(command, check=True, stdout=lcov_file) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profraw-dir", + type=Path, + required=True, + help="Directory searched recursively for .profraw files", + ) + parser.add_argument( + "--rocm-dir", + type=Path, + required=True, + help="Directory holding the installed ROCm artifacts under test", + ) + parser.add_argument( + "--object-globs", + type=str, + required=True, + help="Comma-separated globs, relative to --rocm-dir, matching the " + "instrumented binaries to report on (e.g. 'lib/libhiprand.so*')", + ) + parser.add_argument( + "--llvm-bin-dir", + type=Path, + default=None, + help="Directory holding llvm-profdata and llvm-cov " + "(default: /lib/llvm/bin)", + ) + parser.add_argument( + "--profdata-output", + type=Path, + default=Path("coverage-report/coverage.profdata"), + help="Path for the merged profdata index", + ) + parser.add_argument( + "--lcov-output", + type=Path, + default=Path("coverage-report/coverage.info"), + help="Path for the generated lcov report", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="Exit successfully when no profraw files were collected", + ) + args = parser.parse_args(argv) + + profraw_files = find_profraw_files(args.profraw_dir) + if not profraw_files: + message = f"No .profraw files found under {args.profraw_dir}" + if args.allow_empty: + logging.warning("%s, skipping report generation", message) + return 0 + logging.error( + "%s. The tests either never ran or the instrumented libraries were " + "not the ones loaded at runtime.", + message, + ) + return 1 + + llvm_bin_dir = args.llvm_bin_dir or (args.rocm_dir / "lib" / "llvm" / "bin") + llvm_profdata = find_llvm_tool(llvm_bin_dir, "llvm-profdata") + llvm_cov = find_llvm_tool(llvm_bin_dir, "llvm-cov") + + object_globs = [g.strip() for g in args.object_globs.split(",") if g.strip()] + objects = resolve_objects(args.rocm_dir, object_globs) + if not objects: + logging.error( + "None of the object globs (%s) matched anything under %s", + ", ".join(object_globs), + args.rocm_dir, + ) + return 1 + + merge_profraw(llvm_profdata, profraw_files, args.profdata_output) + export_lcov(llvm_cov, args.profdata_output, objects, args.lcov_output) + logging.info("Wrote coverage report to %s", args.lcov_output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py new file mode 100644 index 00000000000..5a48621cb2a --- /dev/null +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -0,0 +1,222 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +# Add repo root to PYTHONPATH +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +import configure_coverage_ci + + +class ParseProjectsTest(unittest.TestCase): + def test_empty_selects_every_coverage_project(self): + self.assertEqual( + configure_coverage_ci.parse_projects(""), + sorted(configure_coverage_ci.COVERAGE_PROJECTS), + ) + + def test_whitespace_and_case_are_normalized(self): + self.assertEqual(configure_coverage_ci.parse_projects(" HipRand "), ["hiprand"]) + + def test_duplicates_are_dropped_but_order_is_kept(self): + self.assertEqual( + configure_coverage_ci.parse_projects("hiprand,hiprand"), ["hiprand"] + ) + + def test_unknown_project_is_rejected(self): + with self.assertRaises(ValueError) as context: + configure_coverage_ci.parse_projects("hiprand,not-a-project") + self.assertIn("not-a-project", str(context.exception)) + + def test_all_alias_expands_to_every_project(self): + self.assertEqual( + configure_coverage_ci.parse_projects("all"), + sorted(configure_coverage_ci.COVERAGE_PROJECTS), + ) + + def test_rocm_libraries_all_expands_to_that_group(self): + self.assertEqual( + configure_coverage_ci.parse_projects("rocm_libraries_all"), + sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS), + ) + + def test_group_aliases_are_case_insensitive(self): + self.assertEqual( + configure_coverage_ci.parse_projects(" ALL "), + configure_coverage_ci.parse_projects("all"), + ) + self.assertEqual( + configure_coverage_ci.parse_projects("Rocm_Libraries_All"), + configure_coverage_ci.parse_projects("rocm_libraries_all"), + ) + + def test_alias_and_explicit_name_overlap_is_deduped(self): + self.assertEqual( + configure_coverage_ci.parse_projects("rocm_libraries_all,hiprand"), + configure_coverage_ci.parse_projects("rocm_libraries_all"), + ) + + def test_empty_group_alias_is_rejected(self): + # Update this test once a rocm-systems project is onboarded to coverage + # (the group becomes non-empty). + with self.assertRaises(ValueError): + configure_coverage_ci.parse_projects("rocm_systems_all") + + def test_unknown_alias_like_token_is_rejected(self): + with self.assertRaises(ValueError): + configure_coverage_ci.parse_projects("rocm_everything_all") + + +class SourceRepoPartitionTest(unittest.TestCase): + def test_groups_partition_all_projects(self): + libraries = set(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) + systems = set(configure_coverage_ci.ROCM_SYSTEMS_PROJECTS) + self.assertTrue(libraries.isdisjoint(systems)) + self.assertEqual( + libraries | systems, + set(configure_coverage_ci.COVERAGE_PROJECTS), + ) + + +class ParseAmdgpuFamiliesTest(unittest.TestCase): + def test_empty_falls_back_to_the_default_family(self): + self.assertEqual( + configure_coverage_ci.parse_amdgpu_families(""), + [configure_coverage_ci.DEFAULT_AMDGPU_FAMILIES], + ) + + def test_comma_separated_families_are_split(self): + self.assertEqual( + configure_coverage_ci.parse_amdgpu_families("gfx94X-dcgpu, gfx110X-all"), + ["gfx94X-dcgpu", "gfx110X-all"], + ) + + +class ParseConfigSourceTest(unittest.TestCase): + def test_repository_and_ref_are_split(self): + self.assertEqual( + configure_coverage_ci.parse_config_source("ROCm/rocm-libraries@develop"), + ("ROCm/rocm-libraries", "develop"), + ) + + def test_empty_falls_back_to_the_default_source(self): + self.assertEqual( + configure_coverage_ci.parse_config_source(""), + ("ROCm/rocm-libraries", "main"), + ) + + def test_missing_ref_is_rejected(self): + with self.assertRaises(ValueError): + configure_coverage_ci.parse_config_source("ROCm/rocm-libraries") + + +class BuildCoverageMatrixTest(unittest.TestCase): + def test_one_entry_per_project_and_family(self): + matrix = configure_coverage_ci.build_coverage_matrix( + ["hiprand"], + ["gfx94X-dcgpu", "gfx110X-all"], + "ROCm/rocm-libraries", + "main", + ) + self.assertEqual(len(matrix), 2) + self.assertEqual( + [entry["amdgpu_families"] for entry in matrix], + ["gfx94X-dcgpu", "gfx110X-all"], + ) + + def test_coverage_flag_uses_the_upper_case_project_name(self): + # therock_subproject.cmake only forwards the upper case spelling. + (entry,) = configure_coverage_ci.build_coverage_matrix( + ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" + ) + self.assertEqual(entry["coverage_flag"], "HIPRAND_ENABLE_COVERAGE") + + def test_entries_are_json_serializable(self): + matrix = configure_coverage_ci.build_coverage_matrix( + ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" + ) + self.assertEqual(json.loads(json.dumps(matrix)), matrix) + + def test_every_registered_project_declares_report_inputs(self): + for name, project in configure_coverage_ci.COVERAGE_PROJECTS.items(): + with self.subTest(project=name): + self.assertTrue(project.object_globs, "needs objects for llvm-cov") + self.assertTrue(project.fetch_artifact_args, "needs artifacts to fetch") + self.assertTrue(project.stage_project) + self.assertTrue(project.test_component) + + +class MainTest(unittest.TestCase): + def setUp(self): + self._orig_env = os.environ.copy() + + def tearDown(self): + os.environ.clear() + os.environ.update(self._orig_env) + + def test_writes_expected_outputs(self): + with tempfile.TemporaryDirectory() as tmp: + output_path = Path(tmp) / "github_output" + output_path.touch() + os.environ["GITHUB_OUTPUT"] = os.fspath(output_path) + os.environ["PROJECTS_TO_TEST"] = "hiprand" + os.environ["AMDGPU_FAMILIES"] = "gfx94X-dcgpu" + os.environ["COVERAGE_CONFIG_SOURCE"] = "ROCm/rocm-libraries@main" + + self.assertEqual(configure_coverage_ci.main([]), 0) + + written = output_path.read_text() + self.assertIn("coverage_matrix", written) + self.assertIn("dist_amdgpu_families", written) + self.assertIn("families_matrix_json", written) + self.assertIn("coverage_cmake_options", written) + self.assertIn("-DHIPRAND_ENABLE_COVERAGE=ON", written) + # compiler-runtime is always built so the profiling runtime exists. + self.assertIn("compiler-runtime,math-libs", written) + + +class EmitCmakeTest(unittest.TestCase): + def test_emits_group_lists_from_allowlist(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "therock_coverage_projects.cmake" + self.assertEqual(configure_coverage_ci.main(["--emit-cmake", str(out)]), 0) + text = out.read_text() + self.assertIn("set(THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS", text) + self.assertIn("hipRAND", text) + # rocm-systems group is empty today -> an empty set(). + self.assertRegex(text, r"set\(THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS\s*\)") + + def test_emit_cmake_needs_no_env(self): + # Must work without PROJECTS_TO_TEST / AMDGPU_FAMILIES / GITHUB_OUTPUT set. + import os + + saved = { + k: os.environ.pop(k, None) + for k in ( + "PROJECTS_TO_TEST", + "AMDGPU_FAMILIES", + "COVERAGE_CONFIG_SOURCE", + "GITHUB_OUTPUT", + ) + } + try: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "c.cmake" + self.assertEqual( + configure_coverage_ci.main(["--emit-cmake", str(out)]), 0 + ) + self.assertTrue(out.exists()) + finally: + for k, v in saved.items(): + if v is not None: + os.environ[k] = v + + +if __name__ == "__main__": + unittest.main() diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py new file mode 100644 index 00000000000..6f93fcd02b7 --- /dev/null +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -0,0 +1,213 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import os +import platform +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +# Add repo root to PYTHONPATH +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +import merge_coverage_report + + +def is_windows() -> bool: + return platform.system() == "Windows" + + +class TempDirTestBase(unittest.TestCase): + def setUp(self): + self._temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self._temp_dir.cleanup) + self.root = Path(self._temp_dir.name) + + def touch(self, relative_path: str) -> Path: + path = self.root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + return path + + +class FindLlvmToolTest(TempDirTestBase): + def test_prefers_the_rocm_distribution_copy(self): + bin_dir = self.root / "lib" / "llvm" / "bin" + bin_dir.mkdir(parents=True) + expected = bin_dir / f"llvm-cov{merge_coverage_report.EXECUTABLE_SUFFIX}" + expected.touch() + + with mock.patch("shutil.which", return_value="/usr/bin/llvm-cov"): + self.assertEqual( + merge_coverage_report.find_llvm_tool(bin_dir, "llvm-cov"), expected + ) + + def test_falls_back_to_path(self): + bin_dir = self.root / "empty" + bin_dir.mkdir() + + with mock.patch("shutil.which", return_value="/usr/bin/llvm-cov"): + self.assertEqual( + merge_coverage_report.find_llvm_tool(bin_dir, "llvm-cov"), + Path("/usr/bin/llvm-cov"), + ) + + def test_raises_when_the_tool_is_missing_everywhere(self): + bin_dir = self.root / "empty" + bin_dir.mkdir() + + with mock.patch("shutil.which", return_value=None): + with self.assertRaises(FileNotFoundError): + merge_coverage_report.find_llvm_tool(bin_dir, "llvm-cov") + + +class FindProfrawFilesTest(TempDirTestBase): + def test_searches_recursively_and_ignores_other_files(self): + self.touch("shard0/a-1234.profraw") + self.touch("shard1/nested/b-5678.profraw") + self.touch("shard1/test-output.xml") + + found = merge_coverage_report.find_profraw_files(self.root) + + self.assertEqual([f.name for f in found], ["a-1234.profraw", "b-5678.profraw"]) + + def test_empty_directory_yields_nothing(self): + self.assertEqual(merge_coverage_report.find_profraw_files(self.root), []) + + +class ResolveObjectsTest(TempDirTestBase): + @unittest.skipIf(is_windows(), "symlinks require elevated privileges on Windows") + def test_versioned_symlinks_collapse_to_one_object(self): + real = self.touch("lib/libhiprand.so.1.0") + (self.root / "lib" / "libhiprand.so").symlink_to(real) + (self.root / "lib" / "libhiprand.so.1").symlink_to(real) + + objects = merge_coverage_report.resolve_objects( + self.root, ["lib/libhiprand.so*"] + ) + + self.assertEqual(len(objects), 1) + self.assertEqual(objects[0].resolve(), real.resolve()) + + def test_multiple_globs_are_combined(self): + self.touch("lib/libhiprand.so") + self.touch("bin/hiprand_tool") + + objects = merge_coverage_report.resolve_objects( + self.root, ["lib/libhiprand.so*", "bin/hiprand_*"] + ) + + self.assertEqual( + sorted(o.name for o in objects), ["hiprand_tool", "libhiprand.so"] + ) + + def test_directories_are_not_treated_as_objects(self): + (self.root / "lib" / "libhiprand.so.d").mkdir(parents=True) + + self.assertEqual( + merge_coverage_report.resolve_objects(self.root, ["lib/libhiprand.so*"]), [] + ) + + def test_unmatched_glob_yields_nothing(self): + self.assertEqual( + merge_coverage_report.resolve_objects(self.root, ["lib/nothing*"]), [] + ) + + +class CommandConstructionTest(TempDirTestBase): + def test_merge_passes_every_profraw_file(self): + profraw_files = [self.touch("a.profraw"), self.touch("b.profraw")] + output = self.root / "out" / "coverage.profdata" + + with mock.patch("subprocess.run") as run: + merge_coverage_report.merge_profraw( + Path("/llvm/llvm-profdata"), profraw_files, output + ) + + command = run.call_args.args[0] + self.assertEqual(command[:3], ["/llvm/llvm-profdata", "merge", "-sparse"]) + self.assertEqual(command[-2:], [str(profraw_files[0]), str(profraw_files[1])]) + self.assertTrue(output.parent.is_dir()) + + def test_export_passes_the_first_object_positionally(self): + objects = [self.touch("lib/a.so"), self.touch("lib/b.so")] + output = self.root / "out" / "coverage.info" + + with mock.patch("subprocess.run") as run: + merge_coverage_report.export_lcov( + Path("/llvm/llvm-cov"), Path("/tmp/coverage.profdata"), objects, output + ) + + command = run.call_args.args[0] + self.assertEqual(command[:3], ["/llvm/llvm-cov", "export", str(objects[0])]) + self.assertIn("-object", command) + self.assertEqual( + command[-2:], ["-instr-profile=/tmp/coverage.profdata", "--format=lcov"] + ) + + +class MainTest(TempDirTestBase): + def _argv(self, *extra: str) -> list[str]: + return [ + "--profraw-dir", + os.fspath(self.root / "profraw"), + "--rocm-dir", + os.fspath(self.root / "rocm"), + "--object-globs", + "lib/libhiprand.so*", + *extra, + ] + + def test_missing_profiles_fail_by_default(self): + (self.root / "profraw").mkdir() + + self.assertEqual(merge_coverage_report.main(self._argv()), 1) + + def test_missing_profiles_are_tolerated_with_allow_empty(self): + (self.root / "profraw").mkdir() + + self.assertEqual(merge_coverage_report.main(self._argv("--allow-empty")), 0) + + def test_unmatched_object_globs_fail(self): + self.touch("profraw/a.profraw") + llvm_bin_dir = self.root / "rocm" / "lib" / "llvm" / "bin" + llvm_bin_dir.mkdir(parents=True) + for tool in ("llvm-profdata", "llvm-cov"): + (llvm_bin_dir / f"{tool}{merge_coverage_report.EXECUTABLE_SUFFIX}").touch() + + self.assertEqual(merge_coverage_report.main(self._argv()), 1) + + def test_happy_path_invokes_both_llvm_tools(self): + self.touch("profraw/shard0/a.profraw") + self.touch("rocm/lib/libhiprand.so") + llvm_bin_dir = self.root / "rocm" / "lib" / "llvm" / "bin" + llvm_bin_dir.mkdir(parents=True) + for tool in ("llvm-profdata", "llvm-cov"): + (llvm_bin_dir / f"{tool}{merge_coverage_report.EXECUTABLE_SUFFIX}").touch() + + with mock.patch("subprocess.run") as run: + exit_code = merge_coverage_report.main( + self._argv( + "--profdata-output", + os.fspath(self.root / "out" / "coverage.profdata"), + "--lcov-output", + os.fspath(self.root / "out" / "coverage.info"), + ) + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(run.call_count, 2) + invoked = [Path(call.args[0][0]).name for call in run.call_args_list] + self.assertEqual( + invoked, + [ + f"llvm-profdata{merge_coverage_report.EXECUTABLE_SUFFIX}", + f"llvm-cov{merge_coverage_report.EXECUTABLE_SUFFIX}", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/build_tools/tests/artifact_manager_tool_test.py b/build_tools/tests/artifact_manager_tool_test.py index 4fc925e17a0..3eeb4579cd9 100644 --- a/build_tools/tests/artifact_manager_tool_test.py +++ b/build_tools/tests/artifact_manager_tool_test.py @@ -720,6 +720,68 @@ def mock_extract(request: artifact_manager.ExtractRequest) -> Path: self.assertIn("test-artifact_lib_generic.tar.zst", fetched_keys) self.assertNotIn("second-artifact_lib_generic.tar.zst", fetched_keys) + def test_fetch_artifact_names_limits_to_named_artifacts(self) -> None: + """Test that --artifact-names narrows the fetch to an include list.""" + import artifact_manager + + self._create_staged_artifact("test-artifact", "lib", "generic") + self._create_staged_artifact("second-artifact", "lib", "generic") + + extract_calls: list[artifact_manager.ExtractRequest] = [] + + def mock_extract(request: artifact_manager.ExtractRequest) -> Path: + extract_calls.append(request) + return request.output_dir + + with mock.patch("artifact_manager.extract_artifact", mock_extract): + argv = [ + "fetch", + "--stage", + "all", + "--output-dir", + str(self.output_dir), + "--topology", + str(self.topology_path), + "--local-staging-dir", + str(self.staging_dir), + "--platform", + TEST_PLATFORM, + "--run-id", + "local", + "--artifact-names", + "second-artifact", + ] + + artifact_manager.main(argv) + + fetched_keys = [c.archive_path.name for c in extract_calls] + self.assertEqual(fetched_keys, ["second-artifact_lib_generic.tar.zst"]) + + def test_fetch_artifact_names_rejects_unknown_artifact(self) -> None: + """Test that --artifact-names validates artifact names.""" + import artifact_manager + + argv = [ + "fetch", + "--stage", + "all", + "--output-dir", + str(self.output_dir), + "--topology", + str(self.topology_path), + "--local-staging-dir", + str(self.staging_dir), + "--platform", + TEST_PLATFORM, + "--run-id", + "local", + "--artifact-names", + "unknown-artifact", + ] + + with self.assertRaises(ValueError): + artifact_manager.main(argv) + def test_fetch_exclude_artifacts_rejects_unknown_artifact(self) -> None: """Test that --exclude-artifacts validates artifact names.""" argv = [ diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 228e2d3d271..bdb5d3640f1 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -928,6 +928,15 @@ function(therock_cmake_subproject_activate target_name) list(APPEND _cmake_args "${${target_name}_CMAKE_ARGS}") + # Passthrough of -D_ENABLE_COVERAGE=ON to the subproject. CMake + # definitions are conventionally upper case, so only the upper case spelling + # is honored (i.e. HIPDNN_ENABLE_COVERAGE, not hipDNN_ENABLE_COVERAGE). + string(TOUPPER "${_logical_target_name}" _coverage_project_name) + set(_coverage_var_name "${_coverage_project_name}_ENABLE_COVERAGE") + if(DEFINED ${_coverage_var_name}) + list(APPEND _cmake_args "-D${_coverage_var_name}=${${_coverage_var_name}}") + endif() + # Derive the CMAKE_BUILD_TYPE from either {project}_BUILD_TYPE or the global # CMAKE_BUILD_TYPE. set(_cmake_build_type "${${target_name}_BUILD_TYPE}") diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index bf83f415594..a90ee394227 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -75,6 +75,14 @@ if(THEROCK_ENABLE_COMPILER) message(WARNING "LLVM parallel link jobs is unlimited. Memory usage may be much higher. Set -DLLVM_PARALLEL_LINK_JOBS to reduce memory use.") endif() + # Coverage instrumented builds need the ROCm profiling runtime + # (libclang_rt.profile_rocm) so that instrumented device code can write + # profraw files. The flag is only meaningful to compiler-rt, so forward it. + if(COMPILER_RT_BUILD_PROFILE_ROCM) + message(STATUS "COMPILER_RT_BUILD_PROFILE_ROCM enabled - building the ROCm profiling runtime") + list(APPEND _extra_llvm_cmake_args "-DCOMPILER_RT_BUILD_PROFILE_ROCM=ON") + endif() + # If the compiler is not pristine (i.e. has patches), then there will be a # ".amd-llvm.smrev" file present which must be used instead of the auto # computed git revision. If present, this will have a stable hash of diff --git a/docs/development/README.md b/docs/development/README.md index 2a576b0fff6..2cd18bfa7ca 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -10,6 +10,7 @@ - [Artifacts](artifacts.md) - [Build System](build_system.md) +- [Code Coverage](code_coverage.md) - [Dependencies](dependencies.md) - [Development Guide](development_guide.md) - [Installing Artifacts](installing_artifacts.md) diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index b5de8146d73..c3fc7e4bf3c 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -43,6 +43,15 @@ the job starts on a Windows runner, then runs artifact fetch, configure, build, and upload steps inside a WSL Ubuntu shell. See [WSL ROCDXG CI Stage](wsl_rocdxg.md) for details. +Code coverage runs on its own entry points, since it rebuilds the projects it +measures with instrumentation rather than reusing the regular build: + +- [`.github/workflows/multi_arch_ci_coverage.yml`](/.github/workflows/multi_arch_ci_coverage.yml) - PR coverage, opted into with the `ci:coverage` label +- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - nightly full-stack instrumented build + - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project build, test, and report for one GPU family + +See [Code Coverage](code_coverage.md) for details. + ## Build Phase TheRock builds ROCm components from source and produces **artifacts** - archive slices of key components. diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md new file mode 100644 index 00000000000..51337b2aa6f --- /dev/null +++ b/docs/development/code_coverage.md @@ -0,0 +1,145 @@ +# Code Coverage + +TheRock builds ROCm libraries with LLVM's source based code coverage +instrumentation and reports the results to +[Codecov](https://app.codecov.io/gh/ROCm/TheRock). Coverage is opt-in per +project: instrumentation slows a library down substantially, and the report is +only worth producing for projects whose test suites are thorough enough to give +a meaningful signal. + +## Enabling coverage for a local build + +Pass `-D_ENABLE_COVERAGE=ON` at the top level configure. The project +name must be upper case, matching the usual CMake convention for definitions: + +```bash +cmake -B build -GNinja . \ + -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ + -DHIPRAND_ENABLE_COVERAGE=ON \ + -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON +``` + +`therock_subproject.cmake` forwards the flag to the matching subproject, which +is responsible for translating it into compiler flags (usually +`-fprofile-instr-generate -fcoverage-mapping`). Passing the flag for a project +that does not implement it has no effect. + +`COMPILER_RT_BUILD_PROFILE_ROCM=ON` builds `libclang_rt.profile_rocm`, the +device side profiling runtime. Without it, instrumented device code cannot write +profiles, and you will only see host coverage. + +> [!NOTE] +> Coverage builds are incompatible with split kernel packaging. The CI +> workflows pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`; do the same if you +> build with a target family that enables kernel packing by default. + +### Enabling a whole group + +Three aggregate options turn on coverage for a whole component group instead of +naming each project. All default `OFF`: + +| Option | Instruments | +| ------------------------------------- | ------------------------------------------------------------------- | +| `THEROCK_COVERAGE_ROCM_LIBRARIES_ALL` | all coverage-enabled rocm-libraries projects (math-libs, ml-libs …) | +| `THEROCK_COVERAGE_ROCM_SYSTEMS_ALL` | all coverage-enabled rocm-systems projects | +| `THEROCK_COVERAGE_ALL` | both of the above | + +```bash +cmake -B build -GNinja . \ + -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ + -DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON \ + -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON +``` + +Each option expands to the individual `_ENABLE_COVERAGE` flags above, so +the same device-runtime and kernel-packing caveats apply. An explicit +`-D_ENABLE_COVERAGE=OFF` on the command line wins over the group flag. +Group membership is generated from the coverage registry (`COVERAGE_PROJECTS`), +so a group only ever contains onboarded projects; selecting a group with none +(currently rocm-systems) configures nothing and prints a warning. + +> [!NOTE] +> These options default `OFF` and have no effect on a normal, non-coverage build +> — no `_ENABLE_COVERAGE` flag is set. The membership list is +> regenerated from `COVERAGE_PROJECTS` on every configure (a small scripted step; +> it also means editing `configure_coverage_ci.py` triggers a reconfigure), but +> it only changes the build when one of the options is `ON`. + +## Producing a report locally + +Instrumented binaries write one `.profraw` file per process, named by the +`LLVM_PROFILE_FILE` environment variable. The `%p` (pid) and `%m` (binary +signature) substitutions keep concurrent processes from clobbering each other: + +```bash +export LLVM_PROFILE_FILE="$PWD/coverage-report/profraw/%p-%m.profraw" +ctest --test-dir build/math-libs/hiprand +``` + +`merge_coverage_report.py` then merges the profiles and exports lcov: + +```bash +python build_tools/github_actions/merge_coverage_report.py \ + --profraw-dir coverage-report/profraw \ + --rocm-dir build/dist/rocm \ + --object-globs "lib/libhiprand.so*" +``` + +The `llvm-profdata` and `llvm-cov` binaries must come from the same compiler +that built the instrumented objects, so the script prefers the copies under +`/lib/llvm/bin`. A version mismatch surfaces as an unhelpfully generic +"malformed instrumentation profile data" error. + +## Coverage CI + +Two workflows produce the reports that land in Codecov. Both take their project +list from `build_tools/github_actions/configure_coverage_ci.py`, which is the +registry of coverage-enabled projects and everything the pipeline needs to know +about each one: the CMake target, the build stage it lives in, its test +component, the object globs handed to `llvm-cov`, and its Codecov flag. + +| Workflow | Trigger | Build strategy | +| ------------------------------------ | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `multi_arch_ci_coverage.yml` | `ci:coverage` label on a PR, or manual dispatch | Builds the instrumented compiler-runtime once, then rebuilds only the target project for each matrix entry | +| `multi_arch_ci_coverage_nightly.yml` | Nightly schedule, or manual dispatch | Builds the whole instrumented stack once, then tests and reports on each project against that single build | + +Both delegate to `multi_arch_ci_coverage_linux.yml`, which owns the per-project +build, test, and aggregation sequence for one GPU family. Keeping the nesting at +three levels matters: GitHub refuses to run reusable workflows nested more than +four deep. + +The test jobs run through the same `test_component.yml` as regular CI, with +`coverage_enabled: true` adding an `LLVM_PROFILE_FILE` pointing into the +workspace and uploading the resulting profiles as an artifact. The aggregation +job downloads the profiles from every shard, merges them, and uploads the lcov +report. + +### Selecting projects to run + +Both workflows' `projects_to_test` input takes a comma-separated list of project +names, and also accepts three case-insensitive **group aliases** that expand to a +whole component group: `rocm_libraries_all`, `rocm_systems_all`, and `all`. They +may be mixed with explicit names, an empty input still means "every project", and +selecting a group with no onboarded projects fails the run with a clear error +rather than launching an empty matrix. + +The aliases are expanded to concrete project names inside +`configure_coverage_ci.py` before the job matrix is built, so nothing downstream +ever sees them: the coverage pipeline hands `fetch_test_configurations.py` +(shared with regular CI) an already-resolved per-project `test_component`, never +an alias. The `projects_to_test` input and these aliases exist only on the two +coverage workflows, so non-coverage test selection is unaffected. + +### Adding a project + +Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. Before doing +so, confirm that the project implements `_ENABLE_COVERAGE` in its own +CMake, and that a local instrumented build produces a non-empty report. A +project whose tests never load the instrumented library will build and test +cleanly but report zero coverage. + +Set the entry's `source_repo` (`ROCM_LIBRARIES` by default, or `ROCM_SYSTEMS`); +that is what routes the project into the correct group alias and +`THEROCK_COVERAGE_*_ALL` option. No CMake edit is needed — the group membership +lists are generated from `COVERAGE_PROJECTS` at configure time, so both the local +aggregate flags and the CI aliases pick up the new project automatically. diff --git a/docs/development/development_guide.md b/docs/development/development_guide.md index cf1e19b6671..23eaf2a650c 100644 --- a/docs/development/development_guide.md +++ b/docs/development/development_guide.md @@ -216,6 +216,25 @@ For example, this will add options to set tensilelite build parallel and keep bu -DhipBLASLt_CMAKE_ARGS="-DTENSILELITE_BUILD_PARALLEL_LEVEL=32;-DTENSILELITE_KEEP_BUILD_TMP=ON" ``` +- `{PROJECT}_ENABLE_COVERAGE` + +Builds a subproject with source based code coverage instrumentation, provided +that the subproject itself implements the flag. Unlike the flags above, the +project name must be upper case, matching the usual CMake convention for +definitions: `HIPDNN_ENABLE_COVERAGE`, not `hipDNN_ENABLE_COVERAGE`. + +```bash + # Instrument a single project + -DHIPRAND_ENABLE_COVERAGE=ON + + # Instrument several projects + -DHIPRAND_ENABLE_COVERAGE=ON -DROCBLAS_ENABLE_COVERAGE=ON +``` + +Instrumented device code additionally needs the ROCm profiling runtime, which is +built by passing `-DCOMPILER_RT_BUILD_PROFILE_ROCM=ON` to the top level +configure. See [Code coverage](code_coverage.md) for the full workflow. + ### Additional CMake developer ergonomic flags We add developer ergonomic flags as needed in order to support project-wide development activities. This currently includes: From 80dd9963f994209e78c62dd3d6bb4ed13947526e Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 1 Sep 2026 11:55:07 -0700 Subject: [PATCH 02/70] fix(ci): merge duplicate `if` keys in portable linux build stages --- .../multi_arch_build_portable_linux.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index f296098c05e..bfc72f4e19b 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -118,10 +118,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') && + !contains(inputs.skip_stages, 'compiler-runtime') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',compiler-runtime,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') && !contains(inputs.skip_stages, 'compiler-runtime') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -152,10 +152,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') && + !contains(inputs.skip_stages, 'runtime-tests') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',runtime-tests,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') && !contains(inputs.skip_stages, 'runtime-tests') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -186,10 +186,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && + !contains(inputs.skip_stages, 'wsl-rocdxg') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',wsl-rocdxg,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && !contains(inputs.skip_stages, 'wsl-rocdxg') }} uses: ./.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml secrets: inherit with: @@ -217,10 +217,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') && + !contains(inputs.skip_stages, 'math-libs') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',math-libs,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') && !contains(inputs.skip_stages, 'math-libs') }} strategy: # `fail-fast: false` lets all families complete even if one fails. # This is useful for CI (to see all failures at once), but for releases @@ -264,10 +264,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'comm-libs') && + !contains(inputs.skip_stages, 'comm-libs') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',comm-libs,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'comm-libs') && !contains(inputs.skip_stages, 'comm-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -300,10 +300,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'storage-libs') && + !contains(inputs.skip_stages, 'storage-libs') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',storage-libs,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'storage-libs') && !contains(inputs.skip_stages, 'storage-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -335,10 +335,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') && + !contains(inputs.skip_stages, 'debug-tools') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',debug-tools,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') && !contains(inputs.skip_stages, 'debug-tools') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -369,10 +369,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'dctools-core') && + !contains(inputs.skip_stages, 'dctools-core') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',dctools-core,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'dctools-core') && !contains(inputs.skip_stages, 'dctools-core') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -403,10 +403,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'profiler-apps') && + !contains(inputs.skip_stages, 'profiler-apps') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',profiler-apps,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'profiler-apps') && !contains(inputs.skip_stages, 'profiler-apps') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -437,10 +437,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'cv-libs') && + !contains(inputs.skip_stages, 'cv-libs') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',cv-libs,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'cv-libs') && !contains(inputs.skip_stages, 'cv-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -471,10 +471,10 @@ jobs: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'media-libs') && + !contains(inputs.skip_stages, 'media-libs') && (inputs.build_stages == '' || contains(format(',{0},', inputs.build_stages), ',media-libs,')) }} - if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'media-libs') && !contains(inputs.skip_stages, 'media-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: From 45d02409154408f5e9f20f13304b1b6f7dd62452 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 1 Sep 2026 16:42:49 -0700 Subject: [PATCH 03/70] fix(test): make merge_coverage_report command assertions path-separator agnostic --- .../tests/merge_coverage_report_test.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index 6f93fcd02b7..cc7a902154d 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -120,31 +120,30 @@ class CommandConstructionTest(TempDirTestBase): def test_merge_passes_every_profraw_file(self): profraw_files = [self.touch("a.profraw"), self.touch("b.profraw")] output = self.root / "out" / "coverage.profdata" + llvm_profdata = Path("/llvm/llvm-profdata") with mock.patch("subprocess.run") as run: - merge_coverage_report.merge_profraw( - Path("/llvm/llvm-profdata"), profraw_files, output - ) + merge_coverage_report.merge_profraw(llvm_profdata, profraw_files, output) command = run.call_args.args[0] - self.assertEqual(command[:3], ["/llvm/llvm-profdata", "merge", "-sparse"]) + self.assertEqual(command[:3], [str(llvm_profdata), "merge", "-sparse"]) self.assertEqual(command[-2:], [str(profraw_files[0]), str(profraw_files[1])]) self.assertTrue(output.parent.is_dir()) def test_export_passes_the_first_object_positionally(self): objects = [self.touch("lib/a.so"), self.touch("lib/b.so")] output = self.root / "out" / "coverage.info" + llvm_cov = Path("/llvm/llvm-cov") + profdata = Path("/tmp/coverage.profdata") with mock.patch("subprocess.run") as run: - merge_coverage_report.export_lcov( - Path("/llvm/llvm-cov"), Path("/tmp/coverage.profdata"), objects, output - ) + merge_coverage_report.export_lcov(llvm_cov, profdata, objects, output) command = run.call_args.args[0] - self.assertEqual(command[:3], ["/llvm/llvm-cov", "export", str(objects[0])]) + self.assertEqual(command[:3], [str(llvm_cov), "export", str(objects[0])]) self.assertIn("-object", command) self.assertEqual( - command[-2:], ["-instr-profile=/tmp/coverage.profdata", "--format=lcov"] + command[-2:], [f"-instr-profile={profdata}", "--format=lcov"] ) From fea36e0f25d49d516fe61902e155474f5fbe0c37 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Wed, 2 Sep 2026 06:07:23 -0500 Subject: [PATCH 04/70] refactor(ci): keep code coverage out of the shared build pipeline Review feedback asked for coverage to be an add-on that reuses the existing build process, rather than a feature the shared pipeline has to know about. multi_arch_build_portable_linux.yml is restored to its state on main, dropping the build_stages, stage_projects, and extra_cmake_options inputs along with the per-stage `if` rework that left the file with duplicate `if` keys and made it invalid YAML. The coverage workflows now call multi_arch_build_portable_linux_artifacts.yml once per stage they actually need, so no stage filtering is required anywhere. The nightly builds compiler-runtime and then math-libs as explicit jobs, which makes configure_coverage_ci.py's build_stages output unnecessary. The math-libs stage is deliberately not narrowed with stage_projects: its test jobs fetch artifacts by this run's id and so need the stage's full dependency set, and coverage_cmake_options already decides which projects get instrumented. COMPILER_RT_BUILD_PROFILE_ROCM is dropped from compiler/CMakeLists.txt, from the coverage workflows, and from the docs, since the ROCm profiling runtime is expected to come from the compiler build rather than from a TheRock level option. Co-authored-by: Cursor --- .../multi_arch_build_portable_linux.yml | 146 ++---------------- .github/workflows/multi_arch_ci_coverage.yml | 23 ++- .../multi_arch_ci_coverage_linux.yml | 30 ++-- .../multi_arch_ci_coverage_nightly.yml | 55 +++++-- .../github_actions/configure_coverage_ci.py | 7 +- .../tests/configure_coverage_ci_test.py | 5 +- compiler/CMakeLists.txt | 8 - docs/development/code_coverage.md | 26 ++-- docs/development/development_guide.md | 6 +- 9 files changed, 105 insertions(+), 201 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index c6e13444c01..10c6e586b35 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -46,24 +46,6 @@ on: type: string default: "" description: "Comma-separated artifact names to build (granular reuse). Empty = build all artifacts in stage." - build_stages: - type: string - default: "" - description: >- - Comma-separated allowlist of build stages to run. Empty runs every - stage. Variant pipelines such as coverage use this to build only the - stages they need. - stage_projects: - type: string - default: "" - description: >- - Space-separated project names to narrow per-arch stages down to - (e.g. "hiprand"). Empty builds every project in the stage. - extra_cmake_options: - type: string - default: "" - description: >- - Extra -D options appended to each stage's configure command. skip_stages: type: string default: "" @@ -119,14 +101,7 @@ jobs: # STAGE: compiler-runtime (generic) # ========================================================================== compiler-runtime: - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'compiler-runtime') && - !contains(inputs.skip_stages, 'compiler-runtime') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',compiler-runtime,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'compiler-runtime') && !contains(inputs.skip_stages, 'compiler-runtime') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -136,8 +111,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -153,14 +126,7 @@ jobs: # ========================================================================== emulation: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'emulation') && - !contains(inputs.skip_stages, 'emulation') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',emulation,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'emulation') && !contains(inputs.skip_stages, 'emulation') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -185,14 +151,7 @@ jobs: # ========================================================================== runtime-tests: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'runtime-tests') && - !contains(inputs.skip_stages, 'runtime-tests') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',runtime-tests,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'runtime-tests') && !contains(inputs.skip_stages, 'runtime-tests') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -202,8 +161,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -219,14 +176,7 @@ jobs: # ========================================================================== wsl-rocdxg: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && - !contains(inputs.skip_stages, 'wsl-rocdxg') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',wsl-rocdxg,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'wsl-rocdxg') && !contains(inputs.skip_stages, 'wsl-rocdxg') }} uses: ./.github/workflows/multi_arch_build_wsl_rocdxg_artifacts.yml secrets: inherit with: @@ -250,14 +200,7 @@ jobs: # ========================================================================== math-libs: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'math-libs') && - !contains(inputs.skip_stages, 'math-libs') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',math-libs,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'math-libs') && !contains(inputs.skip_stages, 'math-libs') }} strategy: # `fail-fast: false` lets all families complete even if one fails. # This is useful for CI (to see all failures at once), but for releases @@ -277,8 +220,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -298,14 +239,7 @@ jobs: # ========================================================================== comm-libs: needs: [compiler-runtime, emulation] - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'comm-libs') && - !contains(inputs.skip_stages, 'comm-libs') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',comm-libs,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'comm-libs') && !contains(inputs.skip_stages, 'comm-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -316,8 +250,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -334,14 +266,7 @@ jobs: # ========================================================================== storage-libs: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'storage-libs') && - !contains(inputs.skip_stages, 'storage-libs') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',storage-libs,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'storage-libs') && !contains(inputs.skip_stages, 'storage-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -352,8 +277,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -369,14 +292,7 @@ jobs: # ========================================================================== debug-tools: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'debug-tools') && - !contains(inputs.skip_stages, 'debug-tools') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',debug-tools,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'debug-tools') && !contains(inputs.skip_stages, 'debug-tools') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -386,8 +302,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -403,14 +317,7 @@ jobs: # ========================================================================== dctools-core: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'dctools-core') && - !contains(inputs.skip_stages, 'dctools-core') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',dctools-core,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'dctools-core') && !contains(inputs.skip_stages, 'dctools-core') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -420,8 +327,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -437,14 +342,7 @@ jobs: # ========================================================================== profiler-apps: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'profiler-apps') && - !contains(inputs.skip_stages, 'profiler-apps') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',profiler-apps,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'profiler-apps') && !contains(inputs.skip_stages, 'profiler-apps') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -454,8 +352,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -471,14 +367,7 @@ jobs: # ========================================================================== cv-libs: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'cv-libs') && - !contains(inputs.skip_stages, 'cv-libs') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',cv-libs,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'cv-libs') && !contains(inputs.skip_stages, 'cv-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -488,8 +377,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} @@ -505,14 +392,7 @@ jobs: # ========================================================================== media-libs: needs: compiler-runtime - if: >- - ${{ - !cancelled() && !failure() && - !contains(inputs.prebuilt_stages, 'media-libs') && - !contains(inputs.skip_stages, 'media-libs') && - (inputs.build_stages == '' || - contains(format(',{0},', inputs.build_stages), ',media-libs,')) - }} + if: ${{ !cancelled() && !failure() && !contains(inputs.prebuilt_stages, 'media-libs') && !contains(inputs.skip_stages, 'media-libs') }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -522,8 +402,6 @@ jobs: dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} rocm_package_version: ${{ inputs.rocm_package_version }} build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }} - stage_projects: ${{ inputs.stage_projects }} - extra_cmake_options: ${{ inputs.extra_cmake_options }} build_runs_on: ${{ inputs.build_runs_on }} release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} diff --git a/.github/workflows/multi_arch_ci_coverage.yml b/.github/workflows/multi_arch_ci_coverage.yml index 3f2224a47b6..73c3bafa8b7 100644 --- a/.github/workflows/multi_arch_ci_coverage.yml +++ b/.github/workflows/multi_arch_ci_coverage.yml @@ -63,7 +63,6 @@ jobs: outputs: coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} - families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} steps: - name: Checking out repository @@ -91,26 +90,22 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - # Instrumented device code needs the ROCm profiling runtime, which lives in - # compiler-rt. Building this stage once keeps the per-project jobs cheap and - # avoids several jobs racing to publish the same compiler artifacts. + # Each per-project stage below fetches its inbound artifacts by this run's id, + # so the run needs a compiler-runtime of its own. Building it once here keeps + # the per-project jobs cheap and avoids several of them racing to publish the + # same compiler artifacts. build_instrumented_compiler_runtime: name: Build Instrumented Compiler Runtime needs: setup_coverage_matrix if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} - uses: ./.github/workflows/multi_arch_build_portable_linux.yml + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: - artifact_group: multi-arch-coverage - matrix_per_family_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} + stage_name: compiler-runtime + stage_display_name: "Stage - Coverage Compiler Runtime" + timeout_minutes: 480 # 8 hours (compiler is big) dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - build_variant_label: coverage - build_variant_cmake_preset: "" - build_variant_suffix: coverage - build_stages: compiler-runtime - extra_cmake_options: >- - -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON - -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci permissions: diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 6d89808f1be..756cdf69566 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -7,10 +7,10 @@ # # build (instrumented) -> test (collect profraw) -> merge -> report # -# The compiler-runtime stage is expected to have been built already, by the -# caller, with the ROCm profiling runtime enabled. This workflow rebuilds only -# the stage that owns the target project, so a coverage run costs roughly one -# extra library build per project rather than a full instrumented stack. +# The compiler-runtime stage is expected to have been built already in this run +# by the caller. This workflow rebuilds only the stage that owns the target +# project, so a coverage run costs roughly one extra library build per project +# rather than a full instrumented stack. name: Multi-Arch CI Coverage - Linux @@ -33,7 +33,10 @@ on: build_stage: type: string default: math-libs - description: "Build stage that produces the instrumented project" + description: >- + Per-arch build stage that produces the instrumented project. Coverage + projects are library projects, so this is a per-arch stage (math-libs, + comm-libs) and is built for a single GPU family at a time. stage_project: type: string description: "Project name passed to configure_stage.py --projects" @@ -79,26 +82,25 @@ permissions: contents: read jobs: + # Only the stage that owns the target project is rebuilt. The compiler-runtime + # it links against was published earlier in this same run by the caller, so the + # stage's inbound artifact fetch picks it up by run id. build_coverage: name: Build Coverage (${{ inputs.project_name }}) if: ${{ inputs.build_enabled }} - uses: ./.github/workflows/multi_arch_build_portable_linux.yml + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: - artifact_group: multi-arch-coverage - matrix_per_family_json: >- - [{"amdgpu_family": "${{ inputs.amdgpu_families }}"}] + stage_name: ${{ inputs.build_stage }} + stage_display_name: "Stage - Coverage ${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" + timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal + amdgpu_family: ${{ inputs.amdgpu_families }} dist_amdgpu_families: ${{ inputs.amdgpu_families }} - build_variant_label: coverage - build_variant_cmake_preset: "" - build_variant_suffix: coverage - build_stages: ${{ inputs.build_stage }} stage_projects: ${{ inputs.stage_project }} # KPACK artifact splitting is disabled so the instrumented device code # stays in the library that llvm-cov is pointed at. extra_cmake_options: >- -D${{ inputs.coverage_flag }}=ON - -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF ${{ inputs.extra_cmake_options }} rocm_package_version: ${{ inputs.rocm_package_version }} diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 9861c97687b..43028b28502 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -9,6 +9,10 @@ # build per project, which is the right trade-off once several projects are # enabled. # +# The stack is assembled by naming the stages coverage needs and calling the +# shared per-stage build workflow for each, rather than running the whole +# multi_arch_build_portable_linux.yml pipeline and filtering stages out of it. +# # Phase 1 of RFC0014 runs every enabled component on the default architecture. # Change-based selection and multi-architecture expansion come later. @@ -53,7 +57,6 @@ jobs: dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} - build_stages: ${{ steps.matrix.outputs.build_stages }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} steps: - name: Checking out repository @@ -81,23 +84,48 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - build_instrumented_stack: - name: Build Full Instrumented Stack + # Built first so that every later stage in this run has a compiler-runtime to + # link against and to fetch as an inbound artifact. + build_instrumented_compiler_runtime: + name: Build Instrumented Compiler Runtime needs: setup_coverage_matrix if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} - uses: ./.github/workflows/multi_arch_build_portable_linux.yml + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml + secrets: inherit + with: + stage_name: compiler-runtime + stage_display_name: "Stage - Coverage Compiler Runtime" + timeout_minutes: 480 # 8 hours (compiler is big) + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + release_type: ci + permissions: + contents: read + id-token: write + + # The whole stage is built, not just the onboarded projects: the test jobs + # below run against this run's artifacts and need the complete dependency set. + # `coverage_cmake_options` decides which projects get instrumented; the rest + # build normally. + build_instrumented_math_libs: + name: Build Instrumented Math Libs + needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + family_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.families_matrix_json) }} + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: - artifact_group: multi-arch-coverage - matrix_per_family_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} + stage_name: math-libs + stage_display_name: "Stage - Coverage Math Libs (${{ matrix.family_info.amdgpu_family }})" + timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal + amdgpu_family: ${{ matrix.family_info.amdgpu_family }} dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - build_variant_label: coverage - build_variant_cmake_preset: "" - build_variant_suffix: coverage - build_stages: ${{ needs.setup_coverage_matrix.outputs.build_stages }} extra_cmake_options: >- ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} - -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci @@ -109,7 +137,7 @@ jobs: # stack built above (same run id, so the artifacts are already published). coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) - needs: [setup_coverage_matrix, build_instrumented_stack] + needs: [setup_coverage_matrix, build_instrumented_math_libs] # `!failure() && !cancelled()` alone would let this run when # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so # the matrix has to be checked explicitly. @@ -148,7 +176,8 @@ jobs: if: always() needs: - setup_coverage_matrix - - build_instrumented_stack + - build_instrumented_compiler_runtime + - build_instrumented_math_libs - coverage_report runs-on: ubuntu-24.04 steps: diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index eb1968aca79..e96854482d9 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -22,7 +22,6 @@ per-arch stages of multi_arch_build_portable_linux.yml. - coverage_cmake_options: every selected project's coverage flag, for the nightly full-stack instrumented build. - - build_stages: comma-separated stages the selected projects live in. """ import argparse @@ -82,6 +81,10 @@ class CoverageProject: # Projects that participate in coverage CI. Start small (RFC0014 phase 1) and # grow the list as component test suites become good enough to report on. +# +# The nightly workflow builds one instrumented job per stage, and currently only +# has a math-libs job. Onboarding a project from another stage also means adding +# that stage's build job to multi_arch_ci_coverage_nightly.yml. COVERAGE_PROJECTS: dict[str, CoverageProject] = { "hiprand": CoverageProject( cmake_target="hipRAND", @@ -266,7 +269,6 @@ def main(argv: Optional[list[str]] = None) -> int: f"-D{COVERAGE_PROJECTS[key].cmake_target.upper()}_ENABLE_COVERAGE=ON" for key in project_keys ] - build_stages = sorted({COVERAGE_PROJECTS[key].stage for key in project_keys}) outputs = { "coverage_matrix": json.dumps(matrix), "dist_amdgpu_families": ";".join(amdgpu_families), @@ -274,7 +276,6 @@ def main(argv: Optional[list[str]] = None) -> int: [{"amdgpu_family": family} for family in amdgpu_families] ), "coverage_cmake_options": " ".join(coverage_flags), - "build_stages": ",".join(["compiler-runtime"] + build_stages), } if args.print_matrix: diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 5a48621cb2a..f105859f380 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -177,8 +177,9 @@ def test_writes_expected_outputs(self): self.assertIn("families_matrix_json", written) self.assertIn("coverage_cmake_options", written) self.assertIn("-DHIPRAND_ENABLE_COVERAGE=ON", written) - # compiler-runtime is always built so the profiling runtime exists. - self.assertIn("compiler-runtime,math-libs", written) + # The workflow builds this stage, narrowed to this project. + self.assertIn('"build_stage": "math-libs"', written) + self.assertIn('"stage_project": "hiprand"', written) class EmitCmakeTest(unittest.TestCase): diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index a90ee394227..bf83f415594 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -75,14 +75,6 @@ if(THEROCK_ENABLE_COMPILER) message(WARNING "LLVM parallel link jobs is unlimited. Memory usage may be much higher. Set -DLLVM_PARALLEL_LINK_JOBS to reduce memory use.") endif() - # Coverage instrumented builds need the ROCm profiling runtime - # (libclang_rt.profile_rocm) so that instrumented device code can write - # profraw files. The flag is only meaningful to compiler-rt, so forward it. - if(COMPILER_RT_BUILD_PROFILE_ROCM) - message(STATUS "COMPILER_RT_BUILD_PROFILE_ROCM enabled - building the ROCm profiling runtime") - list(APPEND _extra_llvm_cmake_args "-DCOMPILER_RT_BUILD_PROFILE_ROCM=ON") - endif() - # If the compiler is not pristine (i.e. has patches), then there will be a # ".amd-llvm.smrev" file present which must be used instead of the auto # computed git revision. If present, this will have a stable hash of diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 51337b2aa6f..899a01e9732 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -15,8 +15,7 @@ name must be upper case, matching the usual CMake convention for definitions: ```bash cmake -B build -GNinja . \ -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ - -DHIPRAND_ENABLE_COVERAGE=ON \ - -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON + -DHIPRAND_ENABLE_COVERAGE=ON ``` `therock_subproject.cmake` forwards the flag to the matching subproject, which @@ -24,9 +23,11 @@ is responsible for translating it into compiler flags (usually `-fprofile-instr-generate -fcoverage-mapping`). Passing the flag for a project that does not implement it has no effect. -`COMPILER_RT_BUILD_PROFILE_ROCM=ON` builds `libclang_rt.profile_rocm`, the -device side profiling runtime. Without it, instrumented device code cannot write -profiles, and you will only see host coverage. +Device side profiling relies on the ROCm profiling runtime +(`libclang_rt.profile_rocm`) being present in the compiler build. TheRock does +not configure that from the top level; it comes from how amd-llvm's compiler-rt +is built. If a report shows host coverage only, check that the runtime is in the +compiler you built against. > [!NOTE] > Coverage builds are incompatible with split kernel packaging. The CI @@ -47,8 +48,7 @@ naming each project. All default `OFF`: ```bash cmake -B build -GNinja . \ -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ - -DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON \ - -DCOMPILER_RT_BUILD_PROFILE_ROCM=ON + -DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON ``` Each option expands to the individual `_ENABLE_COVERAGE` flags above, so @@ -104,9 +104,15 @@ component, the object globs handed to `llvm-cov`, and its Codecov flag. | `multi_arch_ci_coverage_nightly.yml` | Nightly schedule, or manual dispatch | Builds the whole instrumented stack once, then tests and reports on each project against that single build | Both delegate to `multi_arch_ci_coverage_linux.yml`, which owns the per-project -build, test, and aggregation sequence for one GPU family. Keeping the nesting at -three levels matters: GitHub refuses to run reusable workflows nested more than -four deep. +build, test, and aggregation sequence for one GPU family. + +Instrumented builds call `multi_arch_build_portable_linux_artifacts.yml` — the +same per-stage build workflow regular CI uses — one job per stage, rather than +going through the full `multi_arch_build_portable_linux.yml` pipeline. Coverage +only ever needs compiler-runtime plus the stage owning the project under test, so +naming those stages directly keeps the shared pipeline free of coverage-specific +stage filtering. `stage_projects` and `extra_cmake_options` on that workflow are +the only build-side hooks coverage adds. The test jobs run through the same `test_component.yml` as regular CI, with `coverage_enabled: true` adding an `LLVM_PROFILE_FILE` pointing into the diff --git a/docs/development/development_guide.md b/docs/development/development_guide.md index 23eaf2a650c..1053052aca2 100644 --- a/docs/development/development_guide.md +++ b/docs/development/development_guide.md @@ -231,9 +231,9 @@ definitions: `HIPDNN_ENABLE_COVERAGE`, not `hipDNN_ENABLE_COVERAGE`. -DHIPRAND_ENABLE_COVERAGE=ON -DROCBLAS_ENABLE_COVERAGE=ON ``` -Instrumented device code additionally needs the ROCm profiling runtime, which is -built by passing `-DCOMPILER_RT_BUILD_PROFILE_ROCM=ON` to the top level -configure. See [Code coverage](code_coverage.md) for the full workflow. +Instrumented device code additionally needs the ROCm profiling runtime, which +comes from the compiler build rather than from a top level TheRock option. See +[Code coverage](code_coverage.md) for the full workflow. ### Additional CMake developer ergonomic flags From 7c1920f27d0ee09e53c6368324981806214ddd1a Mon Sep 17 00:00:00 2001 From: jainprad Date: Thu, 3 Sep 2026 07:35:57 -0500 Subject: [PATCH 05/70] feat(ci): nightly code coverage via downstream trigger from the nightly Implements the nightly half of RFC0014: option C for scheduling and option B for artifacts, on the phase 1 scope of every onboarded project on the single default architecture (gfx942 / gfx94X-dcgpu). The regular nightly now dispatches multi_arch_ci_coverage_nightly.yml once its build stage finishes, passing its own run id as baseline_run_id. The cron on the coverage workflow is gone: it would have produced runs with no baseline. Coverage keeps its own run id, so a coverage failure leaves the nightly's status alone and the nightly's unstable test jobs do not gate it, while the baseline it tests against is still fresh from the same commit. The dispatch is gated on the COVERAGE_NIGHTLY_ENABLED repository variable and on the nightly channel. Instrumenting the whole stack in one build and then reporting per project means a project's dependencies would otherwise be instrumented too, which both writes stray profiles and moves the coverage denominator whenever a dependency changes. Test jobs therefore install the baseline run in full and overlay only the project under test from the coverage run. overlay_coverage_artifacts.py does the overlay; it works at the subproject stage directory level rather than the artifact level because TheRock's artifacts are grouped (rand carries both rocRAND and hipRAND), and fails rather than silently reporting coverage for a non-instrumented build. The baseline is read from its own release channel, since artifacts are bucketed per channel. configure_coverage_ci.py grows the artifact_names and artifact_relpaths the overlay needs, and now collapses a whole-group selection to THEROCK_COVERAGE_ROCM_LIBRARIES_ALL / _ROCM_SYSTEMS_ALL / THEROCK_COVERAGE_ALL, which is how the RFC describes the nightly full-stack build. quartz_tracking_id is threaded through the coverage workflows, which the new dispatch edge pulls into the release graph. --- .../multi_arch_ci_coverage_linux.yml | 41 ++++ .../multi_arch_ci_coverage_nightly.yml | 61 ++++- .../workflows/multi_arch_release_linux.yml | 45 +++- .github/workflows/test_component.yml | 55 ++++- .../github_actions/configure_coverage_ci.py | 59 ++++- .../overlay_coverage_artifacts.py | 223 +++++++++++++++++ .../tests/configure_coverage_ci_test.py | 56 ++++- .../tests/overlay_coverage_artifacts_test.py | 228 ++++++++++++++++++ docs/development/code_coverage.md | 62 ++++- 9 files changed, 809 insertions(+), 21 deletions(-) create mode 100644 build_tools/github_actions/overlay_coverage_artifacts.py create mode 100644 build_tools/github_actions/tests/overlay_coverage_artifacts_test.py diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 756cdf69566..56737379632 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -11,6 +11,12 @@ # by the caller. This workflow rebuilds only the stage that owns the target # project, so a coverage run costs roughly one extra library build per project # rather than a full instrumented stack. +# +# `baseline_run_id` switches the test half to the nightly hybrid model: the +# install tree comes from a regular nightly run and only the project under test +# is overlaid from this run, keeping instrumented dependencies out of the +# report. Without it the run's own artifacts are tested as built, which is what +# PR coverage wants since it only instruments the project it rebuilt. name: Multi-Arch CI Coverage - Linux @@ -50,6 +56,29 @@ on: type: string default: "" description: "Extra install_rocm_from_artifacts.py args for the report job" + baseline_run_id: + type: string + default: "" + description: >- + Regular nightly run to take non-instrumented dependencies from. Set + by the nightly caller, which instruments the whole stack in its own + run; the test jobs then install this run and overlay only the project + under test. Empty tests this run's artifacts as built. + baseline_release_type: + type: string + default: "nightly" + description: >- + Release channel baseline_run_id published to. Artifacts are bucketed + by channel, so the baseline is read from its own channel rather than + from this run's. + artifact_names: + type: string + default: "" + description: "Artifact names holding the instrumented project, for the hybrid fetch" + artifact_relpaths: + type: string + default: "" + description: "Subproject stage directories to overlay, for the hybrid fetch" codecov_flag: type: string default: "" @@ -77,6 +106,12 @@ on: description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." type: string default: "" + quartz_tracking_id: + description: >- + Leave empty. Internal Quartz tracking id, propagated from the caller + to link this run to its release lineage. + type: string + default: "" permissions: contents: read @@ -107,6 +142,7 @@ jobs: release_type: ${{ inputs.release_type }} repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: contents: read id-token: write @@ -184,6 +220,11 @@ jobs: build_variant: coverage coverage_enabled: true coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + coverage_baseline_run_id: ${{ inputs.baseline_run_id }} + coverage_baseline_release_type: ${{ inputs.baseline_release_type }} + coverage_artifact_names: ${{ inputs.artifact_names }} + coverage_artifact_relpaths: ${{ inputs.artifact_relpaths }} + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} aggregate_coverage: name: Coverage Report (${{ inputs.project_name }}) diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 43028b28502..7371b70cb68 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -13,24 +13,39 @@ # shared per-stage build workflow for each, rather than running the whole # multi_arch_build_portable_linux.yml pipeline and filtering stages out of it. # -# Phase 1 of RFC0014 runs every enabled component on the default architecture. -# Change-based selection and multi-architecture expansion come later. +# RFC0014 phase 1 runs every enabled component on one default architecture +# (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and +# multi-architecture expansion come later. +# +# Scheduling follows the RFC's option C: the regular nightly dispatches this +# workflow once its build finishes and hands over its run id as +# `baseline_run_id`, so there is no cron here. Coverage therefore gets its own +# run id -- a failure here leaves the nightly's status alone, and the nightly's +# unstable test jobs do not hold coverage up -- while still being pointed at a +# baseline that was just built from the same commit. Dispatching this workflow +# by hand with a `baseline_run_id` copied from a nightly run is the RFC's +# option B1 and works the same way. name: Multi-Arch CI Coverage Nightly on: - schedule: - # 3 AM UTC, after the regular nightly builds have settled. - - cron: "0 3 * * *" workflow_dispatch: inputs: + baseline_run_id: + type: string + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: + type: string + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" projects_to_test: type: string description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." default: "" amdgpu_families: type: string - description: "Comma-separated GPU families to collect coverage on." + description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." default: "gfx94X-dcgpu" coverage_config_source: type: string @@ -40,10 +55,23 @@ on: type: string description: "Test type to run against the instrumented build." default: "standard" + quartz_tracking_id: + type: string + description: >- + Leave empty. Internal Quartz tracking id, set automatically by the + nightly that dispatched this run to link it to its release lineage. + default: "" permissions: contents: read +# The two run ids are only connected through this input, so surface it in the +# run title rather than making people open the logs to find the baseline. +run-name: >- + Multi-Arch CI Coverage Nightly + | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} + | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} + concurrency: group: ${{ github.workflow }}-${{ github.run_id }} cancel-in-progress: false @@ -100,14 +128,17 @@ jobs: extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: contents: read id-token: write # The whole stage is built, not just the onboarded projects: the test jobs - # below run against this run's artifacts and need the complete dependency set. - # `coverage_cmake_options` decides which projects get instrumented; the rest - # build normally. + # below need the complete dependency set, and phase 1 instruments every + # onboarded project in one pass rather than per changed project. + # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the + # selection covers a whole group, and the individual project flags otherwise; + # everything not selected builds normally. build_instrumented_math_libs: name: Build Instrumented Math Libs needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] @@ -129,12 +160,15 @@ jobs: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: contents: read id-token: write - # One test + report job per project, all consuming the single instrumented - # stack built above (same run id, so the artifacts are already published). + # One test + report job per project. The instrumented stack built above is the + # only thing taken from this run: each job installs `baseline_run_id` and then + # overlays its own project, so a project's dependencies stay non-instrumented + # even though the build instrumented all of them at once. coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) needs: [setup_coverage_matrix, build_instrumented_math_libs] @@ -164,9 +198,14 @@ jobs: fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} codecov_flag: ${{ matrix.coverage.codecov_flag }} amdgpu_families: ${{ matrix.coverage.amdgpu_families }} + baseline_run_id: ${{ inputs.baseline_run_id }} + baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} + artifact_names: ${{ matrix.coverage.artifact_names }} + artifact_relpaths: ${{ matrix.coverage.artifact_relpaths }} rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: contents: read id-token: write diff --git a/.github/workflows/multi_arch_release_linux.yml b/.github/workflows/multi_arch_release_linux.yml index 160a41fb792..c797cc42f83 100644 --- a/.github/workflows/multi_arch_release_linux.yml +++ b/.github/workflows/multi_arch_release_linux.yml @@ -240,6 +240,49 @@ jobs: "quartz_tracking_id": "${{ inputs.quartz_tracking_id }}" } + # Nightly code coverage runs as its own workflow (RFC0014 option C) rather + # than as more jobs here: instrumented builds and tests are slow, and coverage + # failing should not colour the nightly's status. + # + # It needs a non-instrumented ROCm to test each instrumented project against, + # which is what this run just built, so the run id goes along as the baseline. + # Depending on build_artifacts alone is deliberate: the nightly's test jobs are + # dispatched separately and rarely all pass, and waiting on them would mean + # coverage rarely runs, while the artifacts coverage actually needs are ready + # as soon as the build is. + trigger_coverage_nightly: + needs: [build_artifacts] + name: Trigger Nightly Coverage + # Opt-in per repository while coverage is being rolled out. Only the nightly + # channel is a useful baseline: dev builds are dispatched ad hoc and would + # queue coverage runs against half-finished stacks. + if: >- + ${{ + vars.COVERAGE_NIGHTLY_ENABLED == 'true' && + startsWith(inputs.release_type, 'nightly') && + inputs.build_variant == 'release' && + contains(fromJSON(inputs.build_config).dist_amdgpu_families, 'gfx94X-dcgpu') + }} + runs-on: ubuntu-24.04 + permissions: + actions: write + steps: + - name: Trigger multi_arch_ci_coverage_nightly + uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 + with: + workflow: multi_arch_ci_coverage_nightly.yml + # Phase 1 collects coverage for every onboarded project on the single + # default architecture, so the project list is left at its default of + # "everything". test_type is deliberately not forwarded: instrumented + # tests already run several times longer than normal ones, so coverage + # keeps its own, narrower default instead of inheriting the nightly's. + inputs: | + { "baseline_run_id": "${{ github.run_id }}", + "baseline_release_type": "${{ inputs.release_type }}", + "amdgpu_families": "gfx94X-dcgpu", + "quartz_tracking_id": "${{ inputs.quartz_tracking_id }}" + } + # Install tests are dispatched asynchronously (sanity) so publish # is not blocked on GPU runner queues. See https://github.com/ROCm/TheRock/issues/5212 trigger_test_native_install: @@ -330,7 +373,7 @@ jobs: notify_quartz_completed: if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' }} - needs: [build_artifacts, build_tarballs, build_python_packages, build_native_deb_packages, build_native_rpm_packages, publish_to_release_buckets, trigger_test_artifacts_per_family, trigger_test_native_install, trigger_release_pytorch_wheels, trigger_release_jax_wheels] + needs: [build_artifacts, build_tarballs, build_python_packages, build_native_deb_packages, build_native_rpm_packages, publish_to_release_buckets, trigger_test_artifacts_per_family, trigger_coverage_nightly, trigger_test_native_install, trigger_release_pytorch_wheels, trigger_release_jax_wheels] name: "Quartz - completed - release Linux" runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index c231cf0f84d..f5acd942695 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -49,6 +49,35 @@ on: something project-specific to keep each project's profiles apart. type: string default: "coverage-profraw" + coverage_baseline_run_id: + description: >- + Regular nightly run holding the non-instrumented artifacts. When set, + the whole install tree comes from that run and only the project under + test is overlaid from this run's instrumented build, so its + dependencies stay out of the coverage report. Empty means the run's + own artifacts are already the ones to test. + type: string + default: "" + coverage_baseline_release_type: + description: >- + Release channel the baseline run published to. Artifacts are bucketed + by channel, so this has to match the run that produced them rather + than the coverage run reading them. + type: string + default: "nightly" + coverage_artifact_names: + description: >- + Comma-separated artifact names holding the instrumented project + (e.g. 'rand'). Required alongside coverage_baseline_run_id. + type: string + default: "" + coverage_artifact_relpaths: + description: >- + Comma-separated subproject stage directories to overlay from those + artifacts (e.g. 'math-libs/hipRAND/stage'). TheRock's artifacts are + grouped, so this is what keeps sibling projects non-instrumented. + type: string + default: "" default_container_image: type: string default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" @@ -98,6 +127,12 @@ jobs: QUARTZ_WORKFLOW_POSTFIX: "- test ROCm component" VENV_DIR: ${{ github.workspace }}/.venv ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + # Nightly coverage installs the baseline run and overlays the + # instrumented project afterwards; every other run installs its own + # artifacts directly. The channel travels with the run id because + # artifacts are bucketed by channel. + INSTALL_ARTIFACT_RUN_ID: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_run_id || (inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id) }}" + INSTALL_RELEASE_TYPE: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_release_type || inputs.release_type }}" OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -153,14 +188,32 @@ jobs: STEP_TIMEOUT_MINUTES: "15" # should match timeout-minutes specified above at this step uses: './.github/actions/setup_test_environment' with: - ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }} + ARTIFACT_RUN_ID: ${{ env.INSTALL_ARTIFACT_RUN_ID }} # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved ARTIFACT_GROUP: ${{ inputs.amdgpu_families }} AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }} VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} + RELEASE_TYPE: ${{ env.INSTALL_RELEASE_TYPE }} + + # The step above installed the baseline run, so every library here is + # non-instrumented. Replacing just this project's files leaves its + # dependencies out of the profiles the test run is about to write. + - name: Overlay instrumented coverage artifacts + if: ${{ inputs.coverage_baseline_run_id != '' }} + timeout-minutes: 15 + env: + GITHUB_TOKEN: ${{ github.token }} RELEASE_TYPE: ${{ inputs.release_type }} + run: | + python ./build_tools/github_actions/overlay_coverage_artifacts.py \ + --run-id="${{ github.run_id }}" \ + --amdgpu-families="${{ inputs.amdgpu_families }}" \ + --amdgpu-targets="${{ inputs.amdgpu_targets }}" \ + --artifact-names="${{ inputs.coverage_artifact_names }}" \ + --artifact-relpaths="${{ inputs.coverage_artifact_relpaths }}" \ + --install-dir="${{ env.OUTPUT_ARTIFACTS_DIR }}" # safe.directory must be set before Runner Health Status - name: Adjust git config diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index e96854482d9..0e4762551ca 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -20,8 +20,9 @@ - dist_amdgpu_families: semicolon-separated families, as CMake expects them. - families_matrix_json: JSON array of {amdgpu_family} objects for the per-arch stages of multi_arch_build_portable_linux.yml. - - coverage_cmake_options: every selected project's coverage flag, for the - nightly full-stack instrumented build. + - coverage_cmake_options: the coverage flags for the nightly full-stack + instrumented build, collapsed to a THEROCK_COVERAGE_*_ALL group option + whenever the selection covers a whole group. """ import argparse @@ -66,6 +67,13 @@ class CoverageProject: codecov_flag: Flag the report is filed under in Codecov. source_repo: The repo this project ships from (ROCM_LIBRARIES or ROCM_SYSTEMS); drives the group aliases. + artifact_names: BUILD_TOPOLOGY artifact(s) the instrumented project + ships in. These are grouped (`rand` holds both rocRAND and + hipRAND), which is why artifact_relpaths exists. + artifact_relpaths: Subproject stage directories inside those artifacts + that belong to this project alone. The nightly hybrid fetch copies + only these over the baseline install tree, so a run measuring + hipRAND does not also pick up an instrumented rocRAND. """ cmake_target: str @@ -77,6 +85,8 @@ class CoverageProject: fetch_artifact_args: str = "" codecov_flag: str = "" source_repo: str = ROCM_LIBRARIES + artifact_names: list[str] = field(default_factory=list) + artifact_relpaths: list[str] = field(default_factory=list) # Projects that participate in coverage CI. Start small (RFC0014 phase 1) and @@ -95,6 +105,8 @@ class CoverageProject: object_globs=["lib/libhiprand.so*"], fetch_artifact_args="--rand", codecov_flag="hipRAND", + artifact_names=["rand"], + artifact_relpaths=["math-libs/hipRAND/stage"], ), } @@ -202,11 +214,49 @@ def build_coverage_matrix( "fetch_artifact_args": project.fetch_artifact_args, "codecov_flag": project.codecov_flag or project_key, "amdgpu_families": family, + "artifact_names": ",".join(project.artifact_names), + "artifact_relpaths": ",".join(project.artifact_relpaths), } ) return matrix +def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: + """Picks the CMake coverage flags that instrument exactly this selection. + + A nightly run instruments every onboarded project, which is what the + THEROCK_COVERAGE_*_ALL group options say directly; a narrowed selection + falls back to naming each project. Both spellings reach the same + _ENABLE_COVERAGE flags, so this only affects how the configure + line reads. + """ + selected = set(project_keys) + covers_libraries = bool(ROCM_LIBRARIES_PROJECTS) and ROCM_LIBRARIES_PROJECTS <= ( + selected + ) + covers_systems = bool(ROCM_SYSTEMS_PROJECTS) and ROCM_SYSTEMS_PROJECTS <= selected + + options: list[str] = [] + grouped: set[str] = set() + if covers_libraries and covers_systems: + options.append("-DTHEROCK_COVERAGE_ALL=ON") + grouped |= ROCM_LIBRARIES_PROJECTS | ROCM_SYSTEMS_PROJECTS + else: + if covers_libraries: + options.append("-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON") + grouped |= ROCM_LIBRARIES_PROJECTS + if covers_systems: + options.append("-DTHEROCK_COVERAGE_ROCM_SYSTEMS_ALL=ON") + grouped |= ROCM_SYSTEMS_PROJECTS + + options.extend( + f"-D{COVERAGE_PROJECTS[key].cmake_target.upper()}_ENABLE_COVERAGE=ON" + for key in project_keys + if key not in grouped + ) + return options + + def emit_cmake(output_path: Path) -> None: """Writes the coverage group-membership lists as a CMake include. @@ -265,10 +315,7 @@ def main(argv: Optional[list[str]] = None) -> int: matrix = build_coverage_matrix( project_keys, amdgpu_families, config_repository, config_ref ) - coverage_flags = [ - f"-D{COVERAGE_PROJECTS[key].cmake_target.upper()}_ENABLE_COVERAGE=ON" - for key in project_keys - ] + coverage_flags = build_coverage_cmake_options(project_keys) outputs = { "coverage_matrix": json.dumps(matrix), "dist_amdgpu_families": ";".join(amdgpu_families), diff --git a/build_tools/github_actions/overlay_coverage_artifacts.py b/build_tools/github_actions/overlay_coverage_artifacts.py new file mode 100644 index 00000000000..557230e9123 --- /dev/null +++ b/build_tools/github_actions/overlay_coverage_artifacts.py @@ -0,0 +1,223 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Overlays one project's instrumented binaries onto a baseline install tree. + +Nightly coverage builds an instrumented stack in its own workflow run while the +regular nightly keeps publishing the normal one, so the two live under separate +run ids. A coverage test job wants exactly one instrumented project and +non-instrumented everything else: an instrumented dependency would emit its own +profiles and move the coverage denominator around whenever that dependency +changes. + +The test job gets there in two steps. It first installs the whole baseline run +the usual way, then calls this script to fetch the instrumented artifact from +the coverage run and copy just the project under test over the baseline copy. + +TheRock's artifacts are grouped -- `rand` carries both rocRAND and hipRAND -- +so a whole artifact is too coarse to overlay. Each artifact keeps its files +under the subproject stage directory they were built in +(`math-libs/hipRAND/stage/...`), and that is what --artifact-relpaths names, so +sibling projects in the same artifact stay non-instrumented. +""" + +import argparse +import logging +import os +import shutil +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) + +from artifact_manager import main as artifact_manager_main + +logging.basicConfig(level=logging.INFO, format="%(message)s") + +# Artifact components an instrumented project can contribute: `lib` the +# instrumented libraries, `test` the test binaries (instrumented in their own +# right for header-only projects), and `dev` the headers and CMake config that +# go with them. The rest only carry symbols and documentation. +EXCLUDED_COMPONENTS = ["run", "dbg", "doc"] + +# artifact_manager.py fetch explodes each archive into this subdirectory of its +# output directory, one directory per artifact. +FETCH_SUBDIR = "artifacts" + + +def split_list(raw: str) -> list[str]: + return [item.strip() for item in raw.split(",") if item.strip()] + + +def fetch_instrumented_artifacts( + *, + run_id: str, + artifact_names: list[str], + amdgpu_families: str, + amdgpu_targets: str, + output_dir: Path, + run_github_repo: str | None, +) -> Path: + """Downloads and explodes the named artifacts, returning their parent dir. + + Neither flattened nor bootstrapped: the overlay needs the per-subproject + directory layout that flattening would collapse. + """ + argv = [ + "fetch", + "--run-id", + run_id, + # These artifacts are outputs of the stage that built them rather than + # inbound to any stage, so the whole topology is the search space and + # --artifact-names does the narrowing. + "--stage", + "all", + "--artifact-names", + ",".join(artifact_names), + "--exclude-components", + ",".join(EXCLUDED_COMPONENTS), + "--amdgpu-families", + amdgpu_families, + "--output-dir", + os.fspath(output_dir), + ] + if amdgpu_targets: + argv.extend(["--amdgpu-targets", amdgpu_targets]) + if run_github_repo: + argv.extend(["--run-github-repo", run_github_repo]) + + logging.info("Fetching instrumented artifacts: %s", " ".join(argv)) + artifact_manager_main(argv) + return output_dir / FETCH_SUBDIR + + +def overlay_relpaths( + *, staging_dir: Path, relpaths: list[str], install_dir: Path +) -> int: + """Copies each artifact's subtree over the install tree. + + Returns the number of subtrees copied. Symlinks are preserved so the + libfoo.so -> libfoo.so.1 chain the loader follows still points at the + instrumented file. + """ + if not staging_dir.is_dir(): + return 0 + + copied = 0 + for artifact_dir in sorted(p for p in staging_dir.iterdir() if p.is_dir()): + for relpath in relpaths: + source = artifact_dir / relpath + if not source.is_dir(): + continue + logging.info("Overlaying %s onto %s", source, install_dir) + shutil.copytree( + source, + install_dir, + symlinks=True, + dirs_exist_ok=True, + ) + copied += 1 + return copied + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--run-id", + required=True, + help="Coverage workflow run that published the instrumented artifacts", + ) + parser.add_argument( + "--amdgpu-families", + required=True, + help="Semicolon-separated GPU families to fetch (e.g. 'gfx94X-dcgpu')", + ) + parser.add_argument( + "--amdgpu-targets", + default="", + help="Comma-separated individual GPU targets for split artifacts", + ) + parser.add_argument( + "--artifact-names", + required=True, + help="Comma-separated artifact names holding the instrumented project " + "(e.g. 'rand')", + ) + parser.add_argument( + "--artifact-relpaths", + required=True, + help="Comma-separated subproject stage directories to overlay " + "(e.g. 'math-libs/hipRAND/stage')", + ) + parser.add_argument( + "--install-dir", + type=Path, + required=True, + help="Baseline install tree to overlay the instrumented files onto", + ) + parser.add_argument( + "--staging-dir", + type=Path, + default=None, + help="Directory for the downloaded artifacts (default: a temporary directory)", + ) + parser.add_argument( + "--run-github-repo", + default=None, + help="GitHub repository for --run-id in 'owner/repo' format", + ) + args = parser.parse_args(argv) + + artifact_names = split_list(args.artifact_names) + relpaths = split_list(args.artifact_relpaths) + if not artifact_names or not relpaths: + logging.error( + "--artifact-names and --artifact-relpaths must both be non-empty." + ) + return 1 + + install_dir = args.install_dir.resolve() + if not install_dir.is_dir(): + logging.error( + "Install directory %s does not exist. The baseline artifacts must " + "be installed before the instrumented ones are overlaid.", + install_dir, + ) + return 1 + + with tempfile.TemporaryDirectory() as tmp: + staging_dir = args.staging_dir or Path(tmp) + staging_dir.mkdir(parents=True, exist_ok=True) + artifacts_dir = fetch_instrumented_artifacts( + run_id=args.run_id, + artifact_names=artifact_names, + amdgpu_families=args.amdgpu_families, + amdgpu_targets=args.amdgpu_targets, + output_dir=staging_dir, + run_github_repo=args.run_github_repo, + ) + copied = overlay_relpaths( + staging_dir=artifacts_dir, + relpaths=relpaths, + install_dir=install_dir, + ) + + # Silently testing the baseline build would report coverage against + # binaries that were never instrumented, so treat this as fatal. + if not copied: + logging.error( + "None of the artifact relpaths (%s) were found in the artifacts " + "fetched from run %s. The instrumented build did not produce the " + "expected layout.", + ", ".join(relpaths), + args.run_id, + ) + return 1 + + logging.info("Overlaid %d instrumented subtree(s) onto %s", copied, install_dir) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index f105859f380..c1d57d2ba9d 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -151,6 +151,58 @@ def test_every_registered_project_declares_report_inputs(self): self.assertTrue(project.stage_project) self.assertTrue(project.test_component) + def test_every_registered_project_declares_hybrid_fetch_inputs(self): + # Without these the nightly test job cannot tell which files belong to + # the project, and would test an entirely non-instrumented install. + for name, project in configure_coverage_ci.COVERAGE_PROJECTS.items(): + with self.subTest(project=name): + self.assertTrue(project.artifact_names, "needs artifacts to overlay") + self.assertTrue(project.artifact_relpaths, "needs relpaths to overlay") + + def test_hybrid_fetch_inputs_reach_the_matrix(self): + (entry,) = configure_coverage_ci.build_coverage_matrix( + ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" + ) + self.assertEqual(entry["artifact_names"], "rand") + self.assertEqual(entry["artifact_relpaths"], "math-libs/hipRAND/stage") + + +class BuildCoverageCmakeOptionsTest(unittest.TestCase): + def test_full_selection_collapses_to_the_group_option(self): + # What the nightly does: instrument every onboarded project at once. + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options( + sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) + ), + ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"], + ) + + def test_partial_selection_names_each_project(self): + # Only meaningful once a second rocm-libraries project is onboarded; + # until then the single project is also the whole group. + if len(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) < 2: + self.skipTest("needs at least two onboarded rocm-libraries projects") + first, *_ = sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options([first]), + [ + f"-D{configure_coverage_ci.COVERAGE_PROJECTS[first].cmake_target.upper()}" + "_ENABLE_COVERAGE=ON" + ], + ) + + def test_empty_group_does_not_produce_a_group_option(self): + # rocm-systems has no onboarded projects, so selecting only + # rocm-libraries must not claim to cover everything. + options = configure_coverage_ci.build_coverage_cmake_options( + sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) + ) + self.assertNotIn("-DTHEROCK_COVERAGE_ALL=ON", options) + self.assertNotIn("-DTHEROCK_COVERAGE_ROCM_SYSTEMS_ALL=ON", options) + + def test_no_selection_produces_no_options(self): + self.assertEqual(configure_coverage_ci.build_coverage_cmake_options([]), []) + class MainTest(unittest.TestCase): def setUp(self): @@ -176,7 +228,9 @@ def test_writes_expected_outputs(self): self.assertIn("dist_amdgpu_families", written) self.assertIn("families_matrix_json", written) self.assertIn("coverage_cmake_options", written) - self.assertIn("-DHIPRAND_ENABLE_COVERAGE=ON", written) + # hipRAND is currently the whole rocm-libraries group, so selecting + # it selects the group option. + self.assertIn("-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON", written) # The workflow builds this stage, narrowed to this project. self.assertIn('"build_stage": "math-libs"', written) self.assertIn('"stage_project": "hiprand"', written) diff --git a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py new file mode 100644 index 00000000000..47899cb9242 --- /dev/null +++ b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py @@ -0,0 +1,228 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import os +import platform +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +# Add repo root to PYTHONPATH +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +import overlay_coverage_artifacts + + +def is_windows() -> bool: + return platform.system() == "Windows" + + +class TempDirTestBase(unittest.TestCase): + def setUp(self): + self._temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self._temp_dir.cleanup) + self.root = Path(self._temp_dir.name) + + def write(self, relative_path: str, contents: str) -> Path: + path = self.root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents) + return path + + +class FetchInstrumentedArtifactsTest(TempDirTestBase): + def test_narrows_the_fetch_to_the_named_artifacts(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + artifacts_dir = overlay_coverage_artifacts.fetch_instrumented_artifacts( + run_id="99999", + artifact_names=["rand", "prim"], + amdgpu_families="gfx94X-dcgpu", + amdgpu_targets="", + output_dir=self.root / "staging", + run_github_repo=None, + ) + + (argv,) = fetch.call_args.args + self.assertEqual(argv[0], "fetch") + # These artifacts are stage outputs, not inbound to any stage. + self.assertEqual(argv[argv.index("--stage") + 1], "all") + self.assertEqual(argv[argv.index("--artifact-names") + 1], "rand,prim") + self.assertEqual(artifacts_dir, self.root / "staging" / "artifacts") + + def test_skips_components_that_cannot_be_instrumented(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + overlay_coverage_artifacts.fetch_instrumented_artifacts( + run_id="99999", + artifact_names=["rand"], + amdgpu_families="gfx94X-dcgpu", + amdgpu_targets="gfx942", + output_dir=self.root / "staging", + run_github_repo=None, + ) + + (argv,) = fetch.call_args.args + excluded = argv[argv.index("--exclude-components") + 1].split(",") + self.assertCountEqual(excluded, ["run", "dbg", "doc"]) + self.assertEqual(argv[argv.index("--amdgpu-targets") + 1], "gfx942") + + +class OverlayRelpathsTest(TempDirTestBase): + def _staged_artifact(self, artifact: str, relpath: str, files: dict[str, str]): + for name, contents in files.items(): + self.write(f"staging/{artifact}/{relpath}/{name}", contents) + + def test_copies_only_the_requested_relpath(self): + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/hipRAND/stage", + {"lib/libhiprand.so": "instrumented"}, + ) + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/rocRAND/stage", + {"lib/librocrand.so": "instrumented"}, + ) + self.write("install/lib/libhiprand.so", "baseline") + self.write("install/lib/librocrand.so", "baseline") + + copied = overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertEqual(copied, 1) + self.assertEqual( + (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" + ) + # A sibling project in the same grouped artifact stays as it was. + self.assertEqual( + (self.root / "install/lib/librocrand.so").read_text(), "baseline" + ) + + def test_merges_into_the_existing_tree(self): + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/hipRAND/stage", + {"lib/libhiprand.so": "instrumented"}, + ) + self.write("install/lib/libamdhip64.so", "baseline") + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertTrue((self.root / "install/lib/libamdhip64.so").exists()) + self.assertTrue((self.root / "install/lib/libhiprand.so").exists()) + + def test_counts_every_matching_artifact(self): + for component in ("lib", "test"): + self._staged_artifact( + f"rand_{component}_gfx942", + "math-libs/hipRAND/stage", + {f"{component}/file": component}, + ) + (self.root / "install").mkdir() + + self.assertEqual( + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ), + 2, + ) + + def test_reports_nothing_copied_when_the_relpath_is_absent(self): + self._staged_artifact( + "rand_lib_gfx942", "math-libs/rocRAND/stage", {"lib/librocrand.so": "x"} + ) + (self.root / "install").mkdir() + + self.assertEqual( + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ), + 0, + ) + + @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") + def test_preserves_symlinks(self): + # The loader follows libfoo.so -> libfoo.so.1; copying the link as a + # regular file would leave a stale non-instrumented copy behind it. + stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" + self.write(f"{stage}/libhiprand.so.1", "instrumented") + (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") + (self.root / "install").mkdir() + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertTrue((self.root / "install/lib/libhiprand.so").is_symlink()) + + +class MainTest(TempDirTestBase): + def _run(self, **overrides) -> int: + args = { + "--run-id": "99999", + "--amdgpu-families": "gfx94X-dcgpu", + "--artifact-names": "rand", + "--artifact-relpaths": "math-libs/hipRAND/stage", + "--install-dir": os.fspath(self.root / "install"), + "--staging-dir": os.fspath(self.root / "staging"), + } + args.update(overrides) + argv = [item for pair in args.items() for item in pair] + return overlay_coverage_artifacts.main(argv) + + def test_fails_when_the_install_tree_is_missing(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + self.assertEqual(self._run(), 1) + fetch.assert_not_called() + + def test_fails_when_nothing_was_overlaid(self): + # A silent pass here would report coverage for binaries that were never + # instrumented. + (self.root / "install").mkdir() + with mock.patch.object(overlay_coverage_artifacts, "artifact_manager_main"): + self.assertEqual(self._run(), 1) + + def test_fetches_and_overlays(self): + (self.root / "install").mkdir() + + def fake_fetch(argv): + self.write( + "staging/artifacts/rand_lib_gfx942/math-libs/hipRAND/stage" + "/lib/libhiprand.so", + "instrumented", + ) + + with mock.patch.object( + overlay_coverage_artifacts, + "artifact_manager_main", + side_effect=fake_fetch, + ): + self.assertEqual(self._run(), 0) + + self.assertEqual( + (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 899a01e9732..26e26e9525d 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -101,7 +101,7 @@ component, the object globs handed to `llvm-cov`, and its Codecov flag. | Workflow | Trigger | Build strategy | | ------------------------------------ | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | | `multi_arch_ci_coverage.yml` | `ci:coverage` label on a PR, or manual dispatch | Builds the instrumented compiler-runtime once, then rebuilds only the target project for each matrix entry | -| `multi_arch_ci_coverage_nightly.yml` | Nightly schedule, or manual dispatch | Builds the whole instrumented stack once, then tests and reports on each project against that single build | +| `multi_arch_ci_coverage_nightly.yml` | Dispatched by the regular nightly, or by hand | Builds the whole instrumented stack once, then tests and reports on each project against that single build | Both delegate to `multi_arch_ci_coverage_linux.yml`, which owns the per-project build, test, and aggregation sequence for one GPU family. @@ -120,6 +120,56 @@ workspace and uploading the resulting profiles as an artifact. The aggregation job downloads the profiles from every shard, merges them, and uploads the lcov report. +### Nightly coverage + +The regular nightly dispatches `multi_arch_ci_coverage_nightly.yml` once its +build stage finishes, handing over its own run id as `baseline_run_id`. Coverage +therefore has a run id of its own: a coverage failure does not colour the +nightly's status, and the nightly's test jobs — which rarely all pass — do not +hold coverage up. It waits on the build alone because that is the part producing +the artifacts coverage needs. + +The dispatch is gated on the `COVERAGE_NIGHTLY_ENABLED` repository variable +being `"true"`, so a repository or fork that does not want nightly coverage +simply never sets it. Dispatching the workflow by hand and pasting in a +`baseline_run_id` from a recent nightly does the same thing, which is the +supported way to reproduce or re-run a nightly report. + +Nightly coverage currently runs every onboarded project on one architecture +(gfx942, the `gfx94X-dcgpu` family). Running only what changed, and running more +architectures, come later. + +#### Why there are two run ids + +The nightly instruments the whole stack in one build, because building each +project separately would mean rebuilding its dependencies each time. Reporting, +though, has to stay per project: a coverage report for hipRAND should not shift +because rocRAND changed, and an instrumented rocRAND under an instrumented +hipRAND also writes its own profiles into the same run. + +Each test job therefore assembles a mostly non-instrumented install: + +1. `setup_test_environment` installs the **baseline** run in full — every + library non-instrumented, as the regular nightly built it. +1. `overlay_coverage_artifacts.py` fetches the artifact holding the project + under test from the **coverage** run and copies only that project's files + over the baseline ones. + +The baseline is read from the channel that published it (`baseline_release_type`, +normally `nightly`) rather than from the coverage run's own `ci` channel, since +artifacts are bucketed per channel. + +Step 2 is per project rather than per artifact because TheRock's artifacts are +grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit under +the subproject stage directory they were built in (`math-libs/hipRAND/stage`), +which is what `artifact_relpaths` in the coverage registry names. Nothing is +renamed along the way — the two runs write to different S3 directories, so the +instrumented `rand_lib_gfx942.tar.xz` and the regular one never collide. + +If the overlay finds none of the project's files, the job fails rather than +testing the baseline build and reporting coverage for binaries that were never +instrumented. + ### Selecting projects to run Both workflows' `projects_to_test` input takes a comma-separated list of project @@ -149,3 +199,13 @@ that is what routes the project into the correct group alias and `THEROCK_COVERAGE_*_ALL` option. No CMake edit is needed — the group membership lists are generated from `COVERAGE_PROJECTS` at configure time, so both the local aggregate flags and the CI aliases pick up the new project automatically. + +`artifact_names` and `artifact_relpaths` are what the nightly overlay uses, so +they need to name the artifact the project ships in and the subproject stage +directory holding its files. Getting `artifact_relpaths` wrong fails the nightly +test job outright rather than quietly reporting against a non-instrumented +build. + +A project whose stage is not yet built by `multi_arch_ci_coverage_nightly.yml` +also needs that stage's build job added there; today it builds compiler-runtime +and math-libs. From e805f9d28143b9e1fde3e74d37468194c3821f6b Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Thu, 3 Sep 2026 08:55:35 -0500 Subject: [PATCH 06/70] refactor(ci): drop PR coverage entry point, keep nightly only Coverage now has a single entry point, the nightly workflow dispatched downstream from the regular nightly. The label-triggered PR workflow and the per-project instrumented rebuild it needed are removed, which also takes the stage_projects hook back out of the shared per-stage build workflow. multi_arch_ci_coverage_linux.yml keeps only the test, merge, and report half; the instrumented stack is always built by its caller. The registry fields that existed solely to drive the per-project rebuild (stage_project, and the coverage_flag / build_stage matrix keys) go with it. Co-authored-by: Cursor --- ...ti_arch_build_portable_linux_artifacts.yml | 8 - .github/workflows/multi_arch_ci_coverage.yml | 167 ------------------ .../multi_arch_ci_coverage_linux.yml | 94 ++-------- .../multi_arch_ci_coverage_nightly.yml | 12 +- .../github_actions/configure_coverage_ci.py | 15 +- .../tests/configure_coverage_ci_test.py | 50 +++--- docs/development/ci_overview.md | 7 +- docs/development/code_coverage.md | 59 +++---- 8 files changed, 84 insertions(+), 328 deletions(-) delete mode 100644 .github/workflows/multi_arch_ci_coverage.yml diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 8d480ae9027..f247d70eabf 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -49,13 +49,6 @@ on: required: false default: "" description: "CMake preset name for this build variant (optional)" - stage_projects: - type: string - required: false - default: "" - description: >- - Space-separated project names to narrow the stage down to (e.g. - "hiprand"). Empty builds every project in the stage. extra_cmake_options: type: string required: false @@ -192,7 +185,6 @@ jobs: --artifacts="${{ inputs.rebuild_artifacts }}" \ --amdgpu-families="${{ inputs.amdgpu_family }}" \ --dist-amdgpu-families="${{ inputs.dist_amdgpu_families }}" \ - ${{ inputs.stage_projects && format('--projects {0}', inputs.stage_projects) || '' }} \ --manylinux \ --gha-output diff --git a/.github/workflows/multi_arch_ci_coverage.yml b/.github/workflows/multi_arch_ci_coverage.yml deleted file mode 100644 index 73c3bafa8b7..00000000000 --- a/.github/workflows/multi_arch_ci_coverage.yml +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-Arch CI Coverage -# -# Entry point for source based code coverage CI. Coverage builds are expensive -# and only meaningful for projects whose test suites report something useful, -# so this does not run on every PR: dispatch it manually, or add the -# `ci:coverage` label to a pull request. -# -# The instrumented compiler-runtime stage is built once and shared, then each -# selected project rebuilds only its own stage with instrumentation enabled. - -name: Multi-Arch CI Coverage - -on: - workflow_dispatch: - inputs: - projects_to_test: - type: string - description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." - default: "" - amdgpu_families: - type: string - description: "Comma-separated GPU families to collect coverage on." - default: "gfx94X-dcgpu" - coverage_config_source: - type: string - description: "Repository and ref holding the per-project coverage metadata." - default: "ROCm/rocm-libraries@main" - test_type: - type: string - description: "Test type to run against the instrumented build." - default: "quick" - pull_request: - types: - - labeled - - opened - - synchronize - -permissions: - contents: read - -concurrency: - # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency - # - # `github.workflow`: each workflow gets its own concurrency group - # `github.head_ref`: the branch name (only defined for pull_request events) - # `github.run_id` : fallback, unique for each run - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - setup_coverage_matrix: - name: Setup Coverage Matrix - # Coverage on a PR is opt-in: without the label this workflow does nothing. - if: >- - ${{ - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'ci:coverage') - }} - runs-on: ubuntu-24.04 - outputs: - coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} - dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} - rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} - steps: - - name: Checking out repository - id: checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: Generate coverage matrix - id: matrix - env: - PROJECTS_TO_TEST: ${{ inputs.projects_to_test || '' }} - AMDGPU_FAMILIES: ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} - COVERAGE_CONFIG_SOURCE: ${{ inputs.coverage_config_source || 'ROCm/rocm-libraries@main' }} - run: python ./build_tools/github_actions/configure_coverage_ci.py - - - name: Compute package version - id: rocm_package_version - run: | - python ./build_tools/compute_rocm_package_version.py \ - --release-type=ci \ - --override-git-sha=${{ steps.checkout.outputs.commit }} - - # Each per-project stage below fetches its inbound artifacts by this run's id, - # so the run needs a compiler-runtime of its own. Building it once here keeps - # the per-project jobs cheap and avoids several of them racing to publish the - # same compiler artifacts. - build_instrumented_compiler_runtime: - name: Build Instrumented Compiler Runtime - needs: setup_coverage_matrix - if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: compiler-runtime - stage_display_name: "Stage - Coverage Compiler Runtime" - timeout_minutes: 480 # 8 hours (compiler is big) - dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - release_type: ci - permissions: - contents: read - id-token: write - - coverage_linux: - name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) - needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] - # `!failure() && !cancelled()` alone would let this run when - # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so - # the matrix has to be checked explicitly. - if: >- - ${{ - !failure() && !cancelled() && - needs.setup_coverage_matrix.result == 'success' && - needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' - }} - strategy: - fail-fast: false - matrix: - coverage: ${{ fromJSON(needs.setup_coverage_matrix.outputs.coverage_matrix) }} - uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml - secrets: inherit - with: - project_name: ${{ matrix.coverage.project_name }} - coverage_flag: ${{ matrix.coverage.coverage_flag }} - build_stage: ${{ matrix.coverage.build_stage }} - stage_project: ${{ matrix.coverage.stage_project }} - test_component: ${{ matrix.coverage.test_component }} - object_globs: ${{ matrix.coverage.object_globs }} - fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} - codecov_flag: ${{ matrix.coverage.codecov_flag }} - amdgpu_families: ${{ matrix.coverage.amdgpu_families }} - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - test_type: ${{ inputs.test_type || 'quick' }} - release_type: ci - permissions: - contents: read - id-token: write - - ci_summary: - name: Coverage CI Summary - if: always() - needs: - - setup_coverage_matrix - - build_instrumented_compiler_runtime - - coverage_linux - runs-on: ubuntu-24.04 - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - - name: Evaluate workflow results - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python build_tools/github_actions/workflow_summary.py \ - --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 56737379632..cf1f206df78 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -3,20 +3,20 @@ # Multi-Arch CI Coverage - Linux # -# Runs the coverage pipeline for a single project on a single GPU family: +# Runs the test and report half of the coverage pipeline for a single project on +# a single GPU family: # -# build (instrumented) -> test (collect profraw) -> merge -> report +# test (collect profraw) -> merge -> report # -# The compiler-runtime stage is expected to have been built already in this run -# by the caller. This workflow rebuilds only the stage that owns the target -# project, so a coverage run costs roughly one extra library build per project -# rather than a full instrumented stack. +# The instrumented stack itself is built by the caller +# (multi_arch_ci_coverage_nightly.yml), which instruments every onboarded +# project in one pass. This workflow only consumes it. # -# `baseline_run_id` switches the test half to the nightly hybrid model: the -# install tree comes from a regular nightly run and only the project under test -# is overlaid from this run, keeping instrumented dependencies out of the -# report. Without it the run's own artifacts are tested as built, which is what -# PR coverage wants since it only instruments the project it rebuilt. +# `baseline_run_id` drives the hybrid install: the tree comes from a regular +# nightly run and only the project under test is overlaid from the instrumented +# run, keeping instrumented dependencies out of the report. Without it the +# instrumented run's own artifacts are tested as built, which measures the +# project's dependencies alongside it. name: Multi-Arch CI Coverage - Linux @@ -26,26 +26,6 @@ on: project_name: type: string description: "Coverage project key (e.g. hiprand)" - build_enabled: - type: boolean - default: true - description: >- - Build the instrumented project. Set to false when the caller already - built a full instrumented stack in this run (nightly coverage) and - only the test and report halves are needed. - coverage_flag: - type: string - description: "Per-project CMake coverage flag (e.g. HIPRAND_ENABLE_COVERAGE)" - build_stage: - type: string - default: math-libs - description: >- - Per-arch build stage that produces the instrumented project. Coverage - projects are library projects, so this is a per-arch stage (math-libs, - comm-libs) and is built for a single GPU family at a time. - stage_project: - type: string - description: "Project name passed to configure_stage.py --projects" test_component: type: string description: "Test matrix key for the project's tests" @@ -62,8 +42,8 @@ on: description: >- Regular nightly run to take non-instrumented dependencies from. Set by the nightly caller, which instruments the whole stack in its own - run; the test jobs then install this run and overlay only the project - under test. Empty tests this run's artifacts as built. + run; the test jobs then install that run and overlay only the project + under test. Empty tests the instrumented artifacts as built. baseline_release_type: type: string default: "nightly" @@ -85,16 +65,10 @@ on: description: "Codecov flag to file the report under" amdgpu_families: type: string - description: "GPU family to build and test coverage for" - extra_cmake_options: - type: string - default: "" - description: "Additional CMake options for the instrumented build" - rocm_package_version: - type: string + description: "GPU family to test coverage on" test_type: type: string - default: quick + default: standard release_type: type: string default: ci @@ -117,40 +91,8 @@ permissions: contents: read jobs: - # Only the stage that owns the target project is rebuilt. The compiler-runtime - # it links against was published earlier in this same run by the caller, so the - # stage's inbound artifact fetch picks it up by run id. - build_coverage: - name: Build Coverage (${{ inputs.project_name }}) - if: ${{ inputs.build_enabled }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: ${{ inputs.build_stage }} - stage_display_name: "Stage - Coverage ${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" - timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal - amdgpu_family: ${{ inputs.amdgpu_families }} - dist_amdgpu_families: ${{ inputs.amdgpu_families }} - stage_projects: ${{ inputs.stage_project }} - # KPACK artifact splitting is disabled so the instrumented device code - # stays in the library that llvm-cov is pointed at. - extra_cmake_options: >- - -D${{ inputs.coverage_flag }}=ON - -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF - ${{ inputs.extra_cmake_options }} - rocm_package_version: ${{ inputs.rocm_package_version }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - configure_test_matrix: name: Configure Coverage Tests (${{ inputs.project_name }}) - needs: build_coverage - if: ${{ !failure() && !cancelled() }} runs-on: ubuntu-24.04 outputs: components: ${{ steps.configure.outputs.components }} @@ -184,8 +126,8 @@ jobs: AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEST_TYPE: ${{ inputs.test_type }} TEST_LABELS: "[]" - # Only the instrumented project was rebuilt, so only its tests can - # contribute coverage. + # The report covers one project, so only that project's tests run + # here; the other onboarded projects get their own job. PROJECTS_TO_TEST: ${{ inputs.test_component }} CI_CONFIG_PATH: ci-config BUILD_VARIANT: coverage @@ -195,7 +137,7 @@ jobs: test_coverage: name: Test Coverage (${{ inputs.project_name }}) - needs: [build_coverage, configure_test_matrix] + needs: configure_test_matrix if: >- ${{ !failure() && !cancelled() && diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 7371b70cb68..868b48ebb88 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -4,10 +4,9 @@ # Multi-Arch CI Coverage - Nightly # # Builds one fully instrumented stack covering every coverage-enabled project, -# then runs each project's tests against it and reports separately. Compared to -# the per-PR coverage workflow this pays for a single large build instead of one -# build per project, which is the right trade-off once several projects are -# enabled. +# then runs each project's tests against it and reports separately. Paying for a +# single large build rather than one build per project is what keeps the cost +# flat as more projects are onboarded. # # The stack is assembled by naming the stages coverage needs and calling the # shared per-stage build workflow for each, rather than running the whole @@ -188,11 +187,7 @@ jobs: uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml secrets: inherit with: - build_enabled: false project_name: ${{ matrix.coverage.project_name }} - coverage_flag: ${{ matrix.coverage.coverage_flag }} - build_stage: ${{ matrix.coverage.build_stage }} - stage_project: ${{ matrix.coverage.stage_project }} test_component: ${{ matrix.coverage.test_component }} object_globs: ${{ matrix.coverage.object_globs }} fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} @@ -202,7 +197,6 @@ jobs: baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} artifact_names: ${{ matrix.coverage.artifact_names }} artifact_relpaths: ${{ matrix.coverage.artifact_relpaths }} - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 0e4762551ca..5140a70a161 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -1,7 +1,7 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Builds the job matrix for TheRock's code coverage CI workflows. +"""Builds the job matrix for TheRock's nightly code coverage CI workflow. Coverage is opt-in per project: instrumenting a library slows it down substantially and only pays off for projects whose test suites are good enough @@ -54,9 +54,9 @@ class CoverageProject: cmake_target: TheRock subproject name. Upper cased to form the `_ENABLE_COVERAGE` flag that therock_subproject.cmake forwards to the subproject. - stage: Build stage that produces the project. - stage_project: Name passed to configure_stage.py --projects, which - narrows the stage down so only this project is rebuilt. + stage: Build stage that produces the project. The nightly needs a build + job for this stage, so onboarding a project from a new stage means + adding one. test_component: Key in fetch_test_configurations.py's test matrix. coverage_config: Per-project coverage metadata file, relative to the root of the repository named by COVERAGE_CONFIG_SOURCE. @@ -78,7 +78,6 @@ class CoverageProject: cmake_target: str stage: str - stage_project: str test_component: str coverage_config: str object_globs: list[str] = field(default_factory=list) @@ -99,7 +98,6 @@ class CoverageProject: "hiprand": CoverageProject( cmake_target="hipRAND", stage="math-libs", - stage_project="hiprand", test_component="hiprand", coverage_config="projects/hiprand/test_categories_coverage.yaml", object_globs=["lib/libhiprand.so*"], @@ -200,12 +198,7 @@ def build_coverage_matrix( matrix.append( { "project_name": project_key, - # therock_subproject.cmake only honors the upper case - # spelling of the per-project coverage flag. - "coverage_flag": f"{project.cmake_target.upper()}_ENABLE_COVERAGE", "cmake_target": project.cmake_target, - "build_stage": project.stage, - "stage_project": project.stage_project, "test_component": project.test_component, "coverage_config": project.coverage_config, "coverage_config_repository": config_repository, diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index c1d57d2ba9d..9069b967031 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock # Add repo root to PYTHONPATH sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) @@ -130,13 +131,6 @@ def test_one_entry_per_project_and_family(self): ["gfx94X-dcgpu", "gfx110X-all"], ) - def test_coverage_flag_uses_the_upper_case_project_name(self): - # therock_subproject.cmake only forwards the upper case spelling. - (entry,) = configure_coverage_ci.build_coverage_matrix( - ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" - ) - self.assertEqual(entry["coverage_flag"], "HIPRAND_ENABLE_COVERAGE") - def test_entries_are_json_serializable(self): matrix = configure_coverage_ci.build_coverage_matrix( ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" @@ -148,7 +142,7 @@ def test_every_registered_project_declares_report_inputs(self): with self.subTest(project=name): self.assertTrue(project.object_globs, "needs objects for llvm-cov") self.assertTrue(project.fetch_artifact_args, "needs artifacts to fetch") - self.assertTrue(project.stage_project) + self.assertTrue(project.stage) self.assertTrue(project.test_component) def test_every_registered_project_declares_hybrid_fetch_inputs(self): @@ -177,19 +171,29 @@ def test_full_selection_collapses_to_the_group_option(self): ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"], ) - def test_partial_selection_names_each_project(self): - # Only meaningful once a second rocm-libraries project is onboarded; - # until then the single project is also the whole group. - if len(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) < 2: - self.skipTest("needs at least two onboarded rocm-libraries projects") - first, *_ = sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) - self.assertEqual( - configure_coverage_ci.build_coverage_cmake_options([first]), - [ - f"-D{configure_coverage_ci.COVERAGE_PROJECTS[first].cmake_target.upper()}" - "_ENABLE_COVERAGE=ON" - ], + def test_partial_selection_names_each_project_in_upper_case(self): + # therock_subproject.cmake only forwards the upper case spelling. Today + # hipRAND is the whole rocm-libraries group, so a second project has to + # be stubbed in to reach the per-project fallback at all. + registry = dict(configure_coverage_ci.COVERAGE_PROJECTS) + registry["rocblas"] = configure_coverage_ci.CoverageProject( + cmake_target="rocBLAS", + stage="math-libs", + test_component="rocblas", + coverage_config="projects/rocblas/test_categories_coverage.yaml", ) + with ( + mock.patch.object(configure_coverage_ci, "COVERAGE_PROJECTS", registry), + mock.patch.object( + configure_coverage_ci, + "ROCM_LIBRARIES_PROJECTS", + frozenset(registry), + ), + ): + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options(["rocblas"]), + ["-DROCBLAS_ENABLE_COVERAGE=ON"], + ) def test_empty_group_does_not_produce_a_group_option(self): # rocm-systems has no onboarded projects, so selecting only @@ -231,9 +235,9 @@ def test_writes_expected_outputs(self): # hipRAND is currently the whole rocm-libraries group, so selecting # it selects the group option. self.assertIn("-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON", written) - # The workflow builds this stage, narrowed to this project. - self.assertIn('"build_stage": "math-libs"', written) - self.assertIn('"stage_project": "hiprand"', written) + # The per-project test and overlay inputs the nightly reads back. + self.assertIn('"test_component": "hiprand"', written) + self.assertIn('"artifact_relpaths": "math-libs/hipRAND/stage"', written) class EmitCmakeTest(unittest.TestCase): diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index 2813a270a55..e3ed2836f46 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -50,12 +50,11 @@ the job starts on a Windows runner, then runs artifact fetch, configure, build, and upload steps inside a WSL Ubuntu shell. See [WSL ROCDXG CI Stage](wsl_rocdxg.md) for details. -Code coverage runs on its own entry points, since it rebuilds the projects it +Code coverage runs on its own entry point, since it rebuilds the projects it measures with instrumentation rather than reusing the regular build: -- [`.github/workflows/multi_arch_ci_coverage.yml`](/.github/workflows/multi_arch_ci_coverage.yml) - PR coverage, opted into with the `ci:coverage` label -- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - nightly full-stack instrumented build - - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project build, test, and report for one GPU family +- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - nightly full-stack instrumented build, dispatched by the regular nightly + - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project test and report for one GPU family See [Code Coverage](code_coverage.md) for details. diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 26e26e9525d..f496230e590 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -92,27 +92,25 @@ that built the instrumented objects, so the script prefers the copies under ## Coverage CI -Two workflows produce the reports that land in Codecov. Both take their project -list from `build_tools/github_actions/configure_coverage_ci.py`, which is the -registry of coverage-enabled projects and everything the pipeline needs to know -about each one: the CMake target, the build stage it lives in, its test -component, the object globs handed to `llvm-cov`, and its Codecov flag. - -| Workflow | Trigger | Build strategy | -| ------------------------------------ | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `multi_arch_ci_coverage.yml` | `ci:coverage` label on a PR, or manual dispatch | Builds the instrumented compiler-runtime once, then rebuilds only the target project for each matrix entry | -| `multi_arch_ci_coverage_nightly.yml` | Dispatched by the regular nightly, or by hand | Builds the whole instrumented stack once, then tests and reports on each project against that single build | - -Both delegate to `multi_arch_ci_coverage_linux.yml`, which owns the per-project -build, test, and aggregation sequence for one GPU family. - -Instrumented builds call `multi_arch_build_portable_linux_artifacts.yml` — the -same per-stage build workflow regular CI uses — one job per stage, rather than -going through the full `multi_arch_build_portable_linux.yml` pipeline. Coverage -only ever needs compiler-runtime plus the stage owning the project under test, so -naming those stages directly keeps the shared pipeline free of coverage-specific -stage filtering. `stage_projects` and `extra_cmake_options` on that workflow are -the only build-side hooks coverage adds. +Coverage reports are produced nightly by +`multi_arch_ci_coverage_nightly.yml`, which builds the whole instrumented stack +once and then tests and reports on each project against that single build. It +delegates to `multi_arch_ci_coverage_linux.yml` for the per-project test and +aggregation sequence on one GPU family. + +Both take their project list from +`build_tools/github_actions/configure_coverage_ci.py`, which is the registry of +coverage-enabled projects and everything the pipeline needs to know about each +one: the CMake target, the build stage it lives in, its test component, the +object globs handed to `llvm-cov`, and its Codecov flag. + +The instrumented build calls `multi_arch_build_portable_linux_artifacts.yml` — +the same per-stage build workflow regular CI uses — one job per stage, rather +than going through the full `multi_arch_build_portable_linux.yml` pipeline. +Coverage only ever needs compiler-runtime plus the stages owning the projects +under test, so naming those stages directly keeps the shared pipeline free of +coverage-specific stage filtering. `extra_cmake_options` on that workflow is the +only build-side hook coverage adds. The test jobs run through the same `test_component.yml` as regular CI, with `coverage_enabled: true` adding an `LLVM_PROFILE_FILE` pointing into the @@ -120,7 +118,7 @@ workspace and uploading the resulting profiles as an artifact. The aggregation job downloads the profiles from every shard, merges them, and uploads the lcov report. -### Nightly coverage +### Scheduling The regular nightly dispatches `multi_arch_ci_coverage_nightly.yml` once its build stage finishes, handing over its own run id as `baseline_run_id`. Coverage @@ -172,19 +170,20 @@ instrumented. ### Selecting projects to run -Both workflows' `projects_to_test` input takes a comma-separated list of project -names, and also accepts three case-insensitive **group aliases** that expand to a -whole component group: `rocm_libraries_all`, `rocm_systems_all`, and `all`. They -may be mixed with explicit names, an empty input still means "every project", and -selecting a group with no onboarded projects fails the run with a clear error -rather than launching an empty matrix. +A nightly run measures every onboarded project. Dispatching the workflow by hand +narrows that down: the `projects_to_test` input takes a comma-separated list of +project names, and also accepts three case-insensitive **group aliases** that +expand to a whole component group: `rocm_libraries_all`, `rocm_systems_all`, and +`all`. They may be mixed with explicit names, an empty input still means "every +project", and selecting a group with no onboarded projects fails the run with a +clear error rather than launching an empty matrix. The aliases are expanded to concrete project names inside `configure_coverage_ci.py` before the job matrix is built, so nothing downstream ever sees them: the coverage pipeline hands `fetch_test_configurations.py` (shared with regular CI) an already-resolved per-project `test_component`, never -an alias. The `projects_to_test` input and these aliases exist only on the two -coverage workflows, so non-coverage test selection is unaffected. +an alias. The `projects_to_test` input and these aliases exist only on the +coverage workflow, so non-coverage test selection is unaffected. ### Adding a project From 8a0b2dffd3e3c4d1d79948962241c252ceddf22f Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Thu, 3 Sep 2026 10:02:11 -0500 Subject: [PATCH 07/70] docs: add end-to-end nightly code coverage flow code_coverage.md covers the CMake options and onboarding a project, but not how a nightly report is actually produced. This adds the CI walkthrough: the dispatch and its gates, the instrumented build, how the baseline run id reaches the test jobs, the artifact overlay, and report generation. Linked from code_coverage.md, ci_overview.md, and the docs index. Co-authored-by: Cursor --- docs/development/README.md | 1 + docs/development/ci_overview.md | 4 +- docs/development/code_coverage.md | 5 + docs/development/code_coverage_flows.md | 213 ++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 docs/development/code_coverage_flows.md diff --git a/docs/development/README.md b/docs/development/README.md index 2cd18bfa7ca..633067b69ee 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -31,6 +31,7 @@ - [S3 Buckets](s3_buckets.md) - [GitHub Actions Debugging](github_actions_debugging.md) - [CI Behavior Manipulation](ci_behavior_manipulation.md) +- [Nightly Code Coverage Flow](code_coverage_flows.md) - [Stage Reuse](stage_reuse.md) - [CCache Troubleshooting](ccache_troubleshooting.md) - [WSL ROCDXG CI Stage](wsl_rocdxg.md) diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index e3ed2836f46..f943db7f0bd 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -56,7 +56,9 @@ measures with instrumentation rather than reusing the regular build: - [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - nightly full-stack instrumented build, dispatched by the regular nightly - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project test and report for one GPU family -See [Code Coverage](code_coverage.md) for details. +See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a +project, and [Nightly Code Coverage Flow](code_coverage_flows.md) for the +end-to-end CI walkthrough. ## Build Phase diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index f496230e590..3acbe3d2bff 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -92,6 +92,11 @@ that built the instrumented objects, so the script prefers the copies under ## Coverage CI +For a step-by-step walkthrough of a nightly run, from the dispatch through to +the Codecov upload, see +[Nightly Code Coverage Flow](code_coverage_flows.md). The rest of this section +is the summary. + Coverage reports are produced nightly by `multi_arch_ci_coverage_nightly.yml`, which builds the whole instrumented stack once and then tests and reports on each project against that single build. It diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md new file mode 100644 index 00000000000..6c223222dd3 --- /dev/null +++ b/docs/development/code_coverage_flows.md @@ -0,0 +1,213 @@ +# Nightly Code Coverage Flow + +An end-to-end walk through what happens between the regular nightly finishing +its build and a coverage report landing in Codecov. See +[Code Coverage](code_coverage.md) for enabling coverage on a local build, the +CMake options, and how to onboard a project. + +The short version: the regular nightly dispatches a separate coverage run and +hands over its own run id. The coverage run instruments the whole stack in one +build, then reconstructs per-project isolation at test time by installing the +regular nightly's non-instrumented artifacts and overlaying only the project +under test. + +```mermaid +graph TD + buildArtifacts[Nightly - build_artifacts] --> trigger[trigger_coverage_nightly] + trigger -.->|dispatch, baseline_run_id| matrix[Coverage - setup_coverage_matrix] + matrix --> compilerRuntime[Build instrumented compiler-runtime] + compilerRuntime --> mathLibs[Build instrumented math-libs] + mathLibs --> report[Per project: configure, test, report] + buildArtifacts -.->|non-instrumented artifacts| report + report --> codecov[Codecov] +``` + +Solid edges are job dependencies inside one workflow run. Dotted edges cross +run boundaries: the dispatch that starts the coverage run, and the baseline +artifacts its test jobs install. + +## Trigger point + +`multi_arch_release_linux.yml` (the regular nightly) owns the +`trigger_coverage_nightly` job. It depends on `build_artifacts` alone, not on +the nightly's test jobs: those are dispatched asynchronously and rarely all +pass, so waiting on them would mean coverage rarely runs, while the artifacts +coverage actually needs are ready as soon as the build is. + +Four conditions gate the dispatch: + +| Condition | Why | +| ----------------------------------------- | -------------------------------------------------------------------- | +| `vars.COVERAGE_NIGHTLY_ENABLED == 'true'` | Opt-in per repository while coverage is being rolled out | +| `release_type` starts with `nightly` | Dev builds are dispatched ad hoc and are not a useful baseline | +| `build_variant == 'release'` | Coverage measures the shipping configuration | +| Families include `gfx94X-dcgpu` | RFC0014 phase 1 collects coverage on the single default architecture | + +The job uses `benc-uk/workflow-dispatch` (pinned by SHA, with +`permissions: actions: write`) to start `multi_arch_ci_coverage_nightly.yml` +with four inputs: `baseline_run_id` set to `github.run_id`, +`baseline_release_type` set to the nightly's release channel, `amdgpu_families` +fixed to `gfx94X-dcgpu`, and `quartz_tracking_id`. + +`test_type` is deliberately not forwarded. Instrumented tests already run +several times longer than normal ones, so coverage keeps its own narrower +default rather than inheriting the nightly's. + +Dispatching `multi_arch_ci_coverage_nightly.yml` by hand and pasting in a +`baseline_run_id` from a recent nightly does exactly the same thing, and is the +supported way to reproduce or re-run a report. + +## The normal build + +Nothing about the regular nightly's build changes. `build_artifacts` produces +the ordinary non-instrumented artifacts and publishes them to the nightly +channel bucket under its own run id. Coverage treats that run as a read-only +input. + +## The instrumented build + +The dispatch creates a new workflow run with its own id, status, and logs. A +coverage failure therefore does not colour the nightly's status. + +`setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads +`PROJECTS_TO_TEST` (empty from a nightly dispatch, meaning every onboarded +project), `AMDGPU_FAMILIES`, and `COVERAGE_CONFIG_SOURCE`, and emits the job +matrix, the family list, and `coverage_cmake_options`. + +`build_instrumented_compiler_runtime` runs first. Every later stage in the run +fetches its inbound artifacts by run id, so the run needs a compiler-runtime of +its own to link against. + +`build_instrumented_math_libs` then builds the whole `math-libs` stage with +`coverage_cmake_options` appended to the configure line. With hipRAND the only +onboarded project, and hipRAND being the whole rocm-libraries group, that +resolves to `-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON`; `CMakeLists.txt` expands +the group option into the individual `_ENABLE_COVERAGE` flags and +`therock_subproject.cmake` forwards them into each subproject. + +Both stages call `multi_arch_build_portable_linux_artifacts.yml`, the same +per-stage build workflow regular CI uses, rather than going through the full +`multi_arch_build_portable_linux.yml` pipeline. Naming the two stages coverage +needs keeps the shared pipeline free of coverage-specific stage filtering. + +Both also pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`: split kernel +packaging would move instrumented device code out of the library `llvm-cov` is +later pointed at. + +The instrumented artifacts publish under `release_type: ci`, so they land in the +CI bucket while the baseline sits in the nightly bucket. That is also why the +instrumented and regular copies of an artifact can share a filename without +colliding. + +## Run ids + +Neither run can know the other's id in advance, since GitHub assigns the +coverage run's id only when the dispatch lands. The nightly already knows its +own, so the id travels downward: + +1. `trigger_coverage_nightly` sends `baseline_run_id: ${{ github.run_id }}`. +1. `multi_arch_ci_coverage_nightly.yml` receives it as a `workflow_dispatch` + input and forwards it, with `baseline_release_type`, to + `multi_arch_ci_coverage_linux.yml`. +1. That forwards both again to `test_component.yml` as + `coverage_baseline_run_id` and `coverage_baseline_release_type`. +1. `test_component.yml` derives `INSTALL_ARTIFACT_RUN_ID` and + `INSTALL_RELEASE_TYPE` from them, which is what `setup_test_environment` + installs from. + +The coverage run refers to its own artifacts as `github.run_id` throughout. + +The release channel travels with the run id rather than being assumed, because +artifacts are bucketed per channel: reading the baseline under the coverage +run's own `ci` channel would look in the wrong bucket entirely. Callers that +leave `coverage_baseline_run_id` empty get exactly the previous behaviour. + +Because the two runs are only connected through this one input, the coverage +workflow puts the baseline in its `run-name`, so it is visible in the Actions +list without opening the logs. `quartz_tracking_id` follows the same path to +keep the coverage run attached to its release lineage. + +## Test execution + +`coverage_report` fans out over the matrix, one call to +`multi_arch_ci_coverage_linux.yml` per project and GPU family. + +`configure_test_matrix` runs `fetch_test_configurations.py` — shared with +regular CI — narrowed to the one project this report covers, and outputs the +shard list. + +`test_coverage` then runs one `test_component.yml` job per shard with +`coverage_enabled: true`. Four things happen in order: + +1. **Install the baseline.** `setup_test_environment` runs against the baseline + run id and channel, filling the install tree with a complete, + non-instrumented ROCm. +1. **Overlay one project.** `overlay_coverage_artifacts.py` fetches the + instrumented artifact from the coverage run and copies only the project's + own subtree over the baseline copy. Details below. +1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to + a per-shard path using the `%p` (pid) and `%m` (binary signature) + substitutions, so a shard that forks or loads several instrumented libraries + does not overwrite its own profiles. +1. **Run the tests and upload.** The normal test script runs, then the profraw + files upload as a workflow artifact under `always()` — a failing shard still + exercised code. + +### Why the overlay exists + +A test job wants exactly one instrumented project and non-instrumented +everything else. An instrumented dependency would emit its own profiles and move +the coverage denominator around whenever that dependency changed. + +Building each project separately would give that for free, but would also mean +rebuilding every dependency once per project. The nightly instead instruments +the whole stack in one build and separates the projects afterwards. + +The overlay is per subproject directory rather than per artifact because +TheRock's artifacts are grouped: `rand` carries both rocRAND and hipRAND, so +overlaying a whole artifact would instrument the sibling too. Each project's +files sit under the subproject stage directory they were built in +(`math-libs/hipRAND/stage`), which is what `artifact_relpaths` in the coverage +registry names. Symlinks are preserved so the `libfoo.so -> libfoo.so.1` chain +the loader follows still resolves to the instrumented file. + +If none of the relpaths match, the script fails the job. Silently testing the +baseline would report coverage against binaries that were never instrumented. + +## Artifacts + +Three distinct kinds are in play: + +| Artifact | Where it lives | What it is for | +| ------------ | ----------------------------------- | ---------------------------------------------- | +| Baseline | nightly bucket, baseline run id | Every non-instrumented dependency | +| Instrumented | CI bucket, coverage run id | The project under test, and the LLVM tools | +| Profraw | GitHub Actions artifacts, per shard | Raw profiles, collected by the aggregation job | + +Profraw artifact names carry the project, GPU family, and shard index, so the +aggregation job can glob back exactly its own shards when several projects are +being measured in one run. + +## Report generation + +`aggregate_coverage` runs once per project, under `if: !cancelled()` so a +partial report still gets produced when some shards failed. + +It downloads every matching profraw artifact, then installs the **instrumented** +artifacts from the coverage run — not the baseline. `llvm-cov` needs the +binaries carrying the coverage mapping sections, and `llvm-profdata` and +`llvm-cov` themselves have to come from the same compiler that produced the +profiles. Both live under `lib/llvm/bin` of the installed distribution; a +version mismatch surfaces as an unhelpfully generic "malformed instrumentation +profile data" error. + +`merge_coverage_report.py` then collects the profiles recursively, expands the +project's `object_globs` and deduplicates them by real path (a versioned symlink +family resolves to one file), runs `llvm-profdata merge -sparse` into a single +profdata index, and exports lcov. Finding no profraw files at all is fatal: it +means either the tests never ran, or the instrumented libraries were not the +ones loaded at runtime. + +The lcov report uploads as a workflow artifact and goes to Codecov under the +project's flag. The Codecov step is skipped rather than failed when no +`CODECOV_TOKEN` is configured, so forks still get the lcov artifact. From 8e4797921b1cc6baa39f5ec0821acb1782217746 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 02:26:57 -0500 Subject: [PATCH 08/70] refactor(ci): defer the nightly downstream coverage trigger The RFC's option C, where the regular nightly dispatches coverage and hands over its run id, lands separately. This drops the trigger_coverage_nightly job, so multi_arch_release_linux.yml is untouched by this PR and coverage is started by hand with a baseline_run_id from a recent nightly (the RFC's option B1). Nothing downstream changes: baseline_run_id and baseline_release_type are already inputs, so adding the trigger later only means adding the dispatching job. Co-authored-by: Cursor --- .../multi_arch_ci_coverage_nightly.yml | 22 ++-- .../workflows/multi_arch_release_linux.yml | 45 +------- docs/development/ci_overview.md | 2 +- docs/development/code_coverage.md | 29 ++--- docs/development/code_coverage_flows.md | 109 ++++++++---------- 5 files changed, 78 insertions(+), 129 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 868b48ebb88..38ac468adb2 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -16,14 +16,16 @@ # (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and # multi-architecture expansion come later. # -# Scheduling follows the RFC's option C: the regular nightly dispatches this -# workflow once its build finishes and hands over its run id as -# `baseline_run_id`, so there is no cron here. Coverage therefore gets its own -# run id -- a failure here leaves the nightly's status alone, and the nightly's -# unstable test jobs do not hold coverage up -- while still being pointed at a -# baseline that was just built from the same commit. Dispatching this workflow -# by hand with a `baseline_run_id` copied from a nightly run is the RFC's -# option B1 and works the same way. +# Scheduling is the RFC's option B1 for now: dispatch this workflow by hand and +# pass the run id of a recent nightly as `baseline_run_id`. There is no cron +# here, and the regular nightly does not know about this workflow, so coverage +# has a run id of its own -- a failure here leaves the nightly's status alone, +# and the nightly's unstable test jobs do not hold coverage up. +# +# The RFC's option C, where the regular nightly dispatches this workflow once +# its build finishes and hands over its own run id automatically, is a later +# change. Everything below already takes `baseline_run_id` as an input, so that +# change only has to add the dispatching job. name: Multi-Arch CI Coverage Nightly @@ -57,8 +59,8 @@ on: quartz_tracking_id: type: string description: >- - Leave empty. Internal Quartz tracking id, set automatically by the - nightly that dispatched this run to link it to its release lineage. + Leave empty. Internal Quartz tracking id, used to link this run to a + release lineage once the nightly dispatches coverage automatically. default: "" permissions: diff --git a/.github/workflows/multi_arch_release_linux.yml b/.github/workflows/multi_arch_release_linux.yml index c797cc42f83..160a41fb792 100644 --- a/.github/workflows/multi_arch_release_linux.yml +++ b/.github/workflows/multi_arch_release_linux.yml @@ -240,49 +240,6 @@ jobs: "quartz_tracking_id": "${{ inputs.quartz_tracking_id }}" } - # Nightly code coverage runs as its own workflow (RFC0014 option C) rather - # than as more jobs here: instrumented builds and tests are slow, and coverage - # failing should not colour the nightly's status. - # - # It needs a non-instrumented ROCm to test each instrumented project against, - # which is what this run just built, so the run id goes along as the baseline. - # Depending on build_artifacts alone is deliberate: the nightly's test jobs are - # dispatched separately and rarely all pass, and waiting on them would mean - # coverage rarely runs, while the artifacts coverage actually needs are ready - # as soon as the build is. - trigger_coverage_nightly: - needs: [build_artifacts] - name: Trigger Nightly Coverage - # Opt-in per repository while coverage is being rolled out. Only the nightly - # channel is a useful baseline: dev builds are dispatched ad hoc and would - # queue coverage runs against half-finished stacks. - if: >- - ${{ - vars.COVERAGE_NIGHTLY_ENABLED == 'true' && - startsWith(inputs.release_type, 'nightly') && - inputs.build_variant == 'release' && - contains(fromJSON(inputs.build_config).dist_amdgpu_families, 'gfx94X-dcgpu') - }} - runs-on: ubuntu-24.04 - permissions: - actions: write - steps: - - name: Trigger multi_arch_ci_coverage_nightly - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: multi_arch_ci_coverage_nightly.yml - # Phase 1 collects coverage for every onboarded project on the single - # default architecture, so the project list is left at its default of - # "everything". test_type is deliberately not forwarded: instrumented - # tests already run several times longer than normal ones, so coverage - # keeps its own, narrower default instead of inheriting the nightly's. - inputs: | - { "baseline_run_id": "${{ github.run_id }}", - "baseline_release_type": "${{ inputs.release_type }}", - "amdgpu_families": "gfx94X-dcgpu", - "quartz_tracking_id": "${{ inputs.quartz_tracking_id }}" - } - # Install tests are dispatched asynchronously (sanity) so publish # is not blocked on GPU runner queues. See https://github.com/ROCm/TheRock/issues/5212 trigger_test_native_install: @@ -373,7 +330,7 @@ jobs: notify_quartz_completed: if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' }} - needs: [build_artifacts, build_tarballs, build_python_packages, build_native_deb_packages, build_native_rpm_packages, publish_to_release_buckets, trigger_test_artifacts_per_family, trigger_coverage_nightly, trigger_test_native_install, trigger_release_pytorch_wheels, trigger_release_jax_wheels] + needs: [build_artifacts, build_tarballs, build_python_packages, build_native_deb_packages, build_native_rpm_packages, publish_to_release_buckets, trigger_test_artifacts_per_family, trigger_test_native_install, trigger_release_pytorch_wheels, trigger_release_jax_wheels] name: "Quartz - completed - release Linux" runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index f943db7f0bd..993cc7a3118 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -53,7 +53,7 @@ and upload steps inside a WSL Ubuntu shell. See Code coverage runs on its own entry point, since it rebuilds the projects it measures with instrumentation rather than reusing the regular build: -- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - nightly full-stack instrumented build, dispatched by the regular nightly +- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - full-stack instrumented build, dispatched manually against a recent nightly - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project test and report for one GPU family See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 3acbe3d2bff..e7c670b9aaf 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -125,20 +125,21 @@ report. ### Scheduling -The regular nightly dispatches `multi_arch_ci_coverage_nightly.yml` once its -build stage finishes, handing over its own run id as `baseline_run_id`. Coverage -therefore has a run id of its own: a coverage failure does not colour the -nightly's status, and the nightly's test jobs — which rarely all pass — do not -hold coverage up. It waits on the build alone because that is the part producing -the artifacts coverage needs. - -The dispatch is gated on the `COVERAGE_NIGHTLY_ENABLED` repository variable -being `"true"`, so a repository or fork that does not want nightly coverage -simply never sets it. Dispatching the workflow by hand and pasting in a -`baseline_run_id` from a recent nightly does the same thing, which is the -supported way to reproduce or re-run a nightly report. - -Nightly coverage currently runs every onboarded project on one architecture +`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. Pick a recent +nightly run, pass its id as `baseline_run_id`, and leave +`baseline_release_type` at `nightly`. There is no cron trigger, and the regular +nightly does not know about this workflow. + +Coverage therefore has a run id of its own, which is what keeps a coverage +failure from colouring the nightly's status and keeps the nightly's test jobs — +which rarely all pass — from holding coverage up. + +Having the regular nightly dispatch this workflow automatically once its build +finishes, handing over its own run id, is a later change. Nothing else has to +move for it: `baseline_run_id` is already an input, so that change only adds the +dispatching job. + +Nightly coverage runs every onboarded project on one architecture (gfx942, the `gfx94X-dcgpu` family). Running only what changed, and running more architectures, come later. diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md index 6c223222dd3..107105d33f3 100644 --- a/docs/development/code_coverage_flows.md +++ b/docs/development/code_coverage_flows.md @@ -1,78 +1,66 @@ # Nightly Code Coverage Flow -An end-to-end walk through what happens between the regular nightly finishing -its build and a coverage report landing in Codecov. See -[Code Coverage](code_coverage.md) for enabling coverage on a local build, the -CMake options, and how to onboard a project. +An end-to-end walk through what happens between dispatching a coverage run and +a report landing in Codecov. See [Code Coverage](code_coverage.md) for enabling +coverage on a local build, the CMake options, and how to onboard a project. -The short version: the regular nightly dispatches a separate coverage run and -hands over its own run id. The coverage run instruments the whole stack in one -build, then reconstructs per-project isolation at test time by installing the -regular nightly's non-instrumented artifacts and overlaying only the project -under test. +The short version: a coverage run instruments the whole stack in one build, then +reconstructs per-project isolation at test time by installing a regular +nightly's non-instrumented artifacts and overlaying only the project under test. ```mermaid graph TD - buildArtifacts[Nightly - build_artifacts] --> trigger[trigger_coverage_nightly] - trigger -.->|dispatch, baseline_run_id| matrix[Coverage - setup_coverage_matrix] + dispatch[Manual dispatch with baseline_run_id] --> matrix[setup_coverage_matrix] matrix --> compilerRuntime[Build instrumented compiler-runtime] compilerRuntime --> mathLibs[Build instrumented math-libs] mathLibs --> report[Per project: configure, test, report] - buildArtifacts -.->|non-instrumented artifacts| report + nightly[Regular nightly run - baseline] -.->|non-instrumented artifacts| report report --> codecov[Codecov] ``` -Solid edges are job dependencies inside one workflow run. Dotted edges cross -run boundaries: the dispatch that starts the coverage run, and the baseline -artifacts its test jobs install. +Solid edges are job dependencies inside the coverage run. The dotted edge +crosses a run boundary: the baseline artifacts the test jobs install come from a +different workflow run entirely. ## Trigger point -`multi_arch_release_linux.yml` (the regular nightly) owns the -`trigger_coverage_nightly` job. It depends on `build_artifacts` alone, not on -the nightly's test jobs: those are dispatched asynchronously and rarely all -pass, so waiting on them would mean coverage rarely runs, while the artifacts -coverage actually needs are ready as soon as the build is. +`multi_arch_ci_coverage_nightly.yml` is started by hand, from the Actions tab or +the `gh` CLI. There is no cron trigger, and the regular nightly does not know +about this workflow. -Four conditions gate the dispatch: +Two inputs matter for a normal run. `baseline_run_id` is the run id of a recent +regular nightly, which supplies every non-instrumented dependency; +`baseline_release_type` is the channel that run published to, normally +`nightly`. The rest can be left at their defaults: every onboarded project, the +`gfx94X-dcgpu` family, and coverage's own `standard` test type, which is +narrower than the nightly's because instrumented tests already run several times +longer than normal ones. -| Condition | Why | -| ----------------------------------------- | -------------------------------------------------------------------- | -| `vars.COVERAGE_NIGHTLY_ENABLED == 'true'` | Opt-in per repository while coverage is being rolled out | -| `release_type` starts with `nightly` | Dev builds are dispatched ad hoc and are not a useful baseline | -| `build_variant == 'release'` | Coverage measures the shipping configuration | -| Families include `gfx94X-dcgpu` | RFC0014 phase 1 collects coverage on the single default architecture | +Leaving `baseline_run_id` empty is supported but means something different: the +run's own instrumented artifacts are tested as built, so a project's +instrumented dependencies are measured alongside it. -The job uses `benc-uk/workflow-dispatch` (pinned by SHA, with -`permissions: actions: write`) to start `multi_arch_ci_coverage_nightly.yml` -with four inputs: `baseline_run_id` set to `github.run_id`, -`baseline_release_type` set to the nightly's release channel, `amdgpu_families` -fixed to `gfx94X-dcgpu`, and `quartz_tracking_id`. - -`test_type` is deliberately not forwarded. Instrumented tests already run -several times longer than normal ones, so coverage keeps its own narrower -default rather than inheriting the nightly's. - -Dispatching `multi_arch_ci_coverage_nightly.yml` by hand and pasting in a -`baseline_run_id` from a recent nightly does exactly the same thing, and is the -supported way to reproduce or re-run a report. +Having the regular nightly dispatch this workflow automatically once its build +finishes, passing its own run id as `baseline_run_id`, is a later change. It +only needs to add a dispatching job — everything described below already takes +the baseline as an input. ## The normal build -Nothing about the regular nightly's build changes. `build_artifacts` produces -the ordinary non-instrumented artifacts and publishes them to the nightly -channel bucket under its own run id. Coverage treats that run as a read-only -input. +Nothing about the regular nightly changes. It produces the ordinary +non-instrumented artifacts and publishes them to the nightly channel bucket +under its own run id. Coverage treats that run as a read-only input, and the +nightly has no coverage-specific jobs in it. ## The instrumented build -The dispatch creates a new workflow run with its own id, status, and logs. A -coverage failure therefore does not colour the nightly's status. +The dispatch creates a workflow run with its own id, status, and logs, entirely +separate from the baseline nightly's. `setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads -`PROJECTS_TO_TEST` (empty from a nightly dispatch, meaning every onboarded -project), `AMDGPU_FAMILIES`, and `COVERAGE_CONFIG_SOURCE`, and emits the job -matrix, the family list, and `coverage_cmake_options`. +`PROJECTS_TO_TEST` (empty by default, meaning every onboarded project), +`AMDGPU_FAMILIES`, and `COVERAGE_CONFIG_SOURCE`, and emits the job matrix, the +family list, and `coverage_cmake_options`. `build_instrumented_compiler_runtime` runs first. Every later stage in the run fetches its inbound artifacts by run id, so the run needs a compiler-runtime of @@ -101,13 +89,13 @@ colliding. ## Run ids -Neither run can know the other's id in advance, since GitHub assigns the -coverage run's id only when the dispatch lands. The nightly already knows its -own, so the id travels downward: +Two runs are involved and each names the other's artifacts differently. The +coverage run refers to its own instrumented artifacts as `github.run_id` +throughout. The baseline is not discoverable, so it is threaded through as an +input: -1. `trigger_coverage_nightly` sends `baseline_run_id: ${{ github.run_id }}`. -1. `multi_arch_ci_coverage_nightly.yml` receives it as a `workflow_dispatch` - input and forwards it, with `baseline_release_type`, to +1. `multi_arch_ci_coverage_nightly.yml` takes `baseline_run_id` and + `baseline_release_type` as `workflow_dispatch` inputs and forwards them to `multi_arch_ci_coverage_linux.yml`. 1. That forwards both again to `test_component.yml` as `coverage_baseline_run_id` and `coverage_baseline_release_type`. @@ -115,17 +103,18 @@ own, so the id travels downward: `INSTALL_RELEASE_TYPE` from them, which is what `setup_test_environment` installs from. -The coverage run refers to its own artifacts as `github.run_id` throughout. +Because the two runs are connected only by that first input, and nothing in the +coverage run can look the baseline up, the workflow puts it in the `run-name` so +it is visible in the Actions list without opening the logs. The release channel travels with the run id rather than being assumed, because artifacts are bucketed per channel: reading the baseline under the coverage run's own `ci` channel would look in the wrong bucket entirely. Callers that leave `coverage_baseline_run_id` empty get exactly the previous behaviour. -Because the two runs are only connected through this one input, the coverage -workflow puts the baseline in its `run-name`, so it is visible in the Actions -list without opening the logs. `quartz_tracking_id` follows the same path to -keep the coverage run attached to its release lineage. +`quartz_tracking_id` follows the same path. It stays empty on a manual dispatch +and exists so a coverage run can be attached to a release lineage once the +nightly dispatches coverage automatically. ## Test execution From 4b448482bec42a0ccecad3db65fa263e7ad8eae0 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 02:56:37 -0500 Subject: [PATCH 09/70] DO NOT MERGE: run the coverage workflow from this branch The phase 1 work exists to show that an instrumented build and a coverage report work end to end, and it currently cannot show that. multi_arch_ci_coverage_nightly.yml is triggered by hand, but a workflow_dispatch is only offered once the file is on the default branch, so there is no way to run it before it merges and no way to see it in this PR's checks. Merging it unvalidated to make it validatable is the wrong order. A caller is not subject to that restriction. Paths under ./.github/workflows resolve against the caller's own commit, so a pull_request-triggered caller runs this branch's copy of the coverage workflow and reports as jobs of this PR. The coverage workflow therefore gains a workflow_call trigger, duplicating the dispatch inputs since Actions does not expand YAML anchors. The baseline is pinned rather than resolved, so that a change in the report means a change in this branch: Multi-Arch CI run 33990288042, whose Stage - Compiler Runtime and Stage - Math Libs (gfx94X-dcgpu) both succeeded. That run's own test jobs failed, which does not matter here, as only its build stages are consumed. It published to therock-ci-artifacts rather than the nightly bucket, so baseline_release_type is ci and not the default; the bucket for a run follows the release type it published under. A ROCm/TheRock run rather than a rockrel release nightly. This workflow only installs the baseline at test time and never configures against it, so the embedded artifact workspace paths that rule out a cross-repository baseline elsewhere do not apply, but keeping the baseline in this repository is one fewer variable while the instrumented build is still unproven. The caller also has to grant id-token and actions:read itself, because a called workflow cannot hold permissions its caller does not. The paths filter is deliberately narrow and concurrency cancels in progress: each run is a multi-hour instrumented build plus a full test suite, and it is not worth spending one on code that has already been superseded. This file is scaffolding and is not part of the design. Coverage on presubmit is precisely what phase 1 rejected, and leaving this in place would charge that cost to every PR touching coverage. Delete it, and the workflow_call trigger, once the run has gone green. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 81 +++++++++++++++++++ .../multi_arch_ci_coverage_nightly.yml | 39 +++++++++ 2 files changed, 120 insertions(+) create mode 100644 .github/workflows/coverage_nightly_pr_check.yml diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml new file mode 100644 index 00000000000..da36aeb2a95 --- /dev/null +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -0,0 +1,81 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# TEMPORARY. Delete this file, and the workflow_call trigger it relies on, +# before merging the coverage PR. +# +# multi_arch_ci_coverage_nightly.yml is triggered by hand, but a +# workflow_dispatch only appears once the file is on the default branch. There +# is therefore no way to run it from a branch and no way to see it in a PR's +# checks, which leaves the phase 1 work unable to demonstrate the one thing it +# exists to demonstrate. +# +# A caller has no such restriction: paths under ./.github/workflows resolve +# against the caller's own commit, so this runs the branch's copy of the +# coverage workflow and its jobs report as checks on this PR. +# +# This is scaffolding, not part of the design. Coverage on presubmit is exactly +# what the phase 1 scope rejects: one run is a multi-hour instrumented build +# plus a full test suite, which is why coverage is a separate, manually +# triggered workflow in the first place. Leaving this in place would charge +# that cost to every PR touching coverage. + +name: Coverage Nightly PR Check + +on: + pull_request: + # Narrow on purpose. Every run here is a multi-hour instrumented build, so + # only the files carrying coverage logic are worth spending one on. + paths: + - ".github/workflows/coverage_nightly_pr_check.yml" + - ".github/workflows/multi_arch_ci_coverage_nightly.yml" + - ".github/workflows/multi_arch_ci_coverage_linux.yml" + - ".github/workflows/test_component.yml" + - "build_tools/github_actions/configure_coverage_ci.py" + - "build_tools/github_actions/overlay_coverage_artifacts.py" + - "build_tools/github_actions/merge_coverage_report.py" + - "cmake/therock_subproject.cmake" + - "CMakeLists.txt" + +concurrency: + # A later push supersedes this run. Queueing multi-hour instrumented builds + # behind each other, to report on code that has already been replaced, only + # occupies build nodes the rest of CI needs. + group: coverage-nightly-pr-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + coverage: + name: Coverage + uses: ./.github/workflows/multi_arch_ci_coverage_nightly.yml + secrets: inherit + # A called workflow cannot hold more than its caller, so these are granted + # here even though the coverage workflow declares them itself: id-token for + # the AWS role the artifact fetches assume, actions:read for the baseline + # run lookup. + permissions: + contents: read + actions: read + id-token: write + with: + # Pinned rather than resolved, so that a change in the report means a + # change in this branch. + # + # ROCm/TheRock Multi-Arch CI run 33990288042 (commit ab8dfcd, 2026-09-05), + # whose Stage - Compiler Runtime and Stage - Math Libs (gfx94X-dcgpu) both + # succeeded. Its own test jobs failed, which does not matter here: only + # the build stages are consumed. + # + # A ROCm/TheRock run rather than a rockrel release nightly. The two are + # interchangeable for this workflow, which only installs the baseline at + # test time and never configures against it, but staying in-repository + # keeps the embedded artifact workspace paths (/__w/TheRock/TheRock) + # matching this run's, which is one fewer variable while the instrumented + # build is still unproven. + baseline_run_id: "33990288042" + # That run published to therock-ci-artifacts, not the nightly bucket, so + # the baseline has to be read from the ci channel rather than the default. + baseline_release_type: "ci" + amdgpu_families: "gfx94X-dcgpu" + projects_to_test: "hiprand" + test_type: "standard" diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 38ac468adb2..cbcec6ddf5e 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -63,6 +63,45 @@ on: release lineage once the nightly dispatches coverage automatically. default: "" + # TEMPORARY, paired with coverage_nightly_pr_check.yml. A workflow_dispatch + # is only offered once the file is on the default branch, so this workflow + # cannot be run from a branch and cannot be shown in a PR's checks. A caller + # is not subject to that, so the check workflow runs this branch's copy. + # Actions does not expand YAML anchors, so the inputs are repeated verbatim. + # Remove this trigger with the check workflow once the run has gone green. + workflow_call: + inputs: + baseline_run_id: + type: string + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: + type: string + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" + projects_to_test: + type: string + description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." + default: "" + amdgpu_families: + type: string + description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." + default: "gfx94X-dcgpu" + coverage_config_source: + type: string + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" + test_type: + type: string + description: "Test type to run against the instrumented build." + default: "standard" + quartz_tracking_id: + type: string + description: >- + Leave empty. Internal Quartz tracking id, used to link this run to a + release lineage once the nightly dispatches coverage automatically. + default: "" + permissions: contents: read From 0585dc3fca64279caec9e4b4591e8f86f59e32c0 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 05:27:44 -0500 Subject: [PATCH 10/70] refactor(ci): make the coverage run self-contained The nightly coverage run built an instrumented stack and then, at test time, threw most of it away: each job installed a regular nightly's artifacts in full and overlaid only the project under test, so the project's dependencies stayed non-instrumented. That is unnecessary. A coverage run builds every stage its selection needs and instruments only the projects the selection names, so everything those projects depend on is already built normally in the same run. Its own artifacts are the stack to test against. The isolation the overlay was protecting comes from somewhere else anyway. merge_coverage_report.py hands llvm-cov an explicit -object per binary, and llvm-cov only reports functions found in those objects' coverage mappings, so counters written by a sibling project are never looked up. A selection covering several projects still yields one report per project. Removing it also removes the failure mode it introduced. Run 34098007786 died in the overlay step, where shutil.copytree could not replace an existing libhiprand.so -> .so.1 -> .so.1.1 chain that the baseline install had already put in place. Drops the script and its tests, the baseline_run_id and baseline_release_type inputs threaded through four workflows, and the --artifact-names flag added to artifact_manager.py for the overlay's sake alone. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 29 +-- .../multi_arch_ci_coverage_linux.yml | 41 +--- .../multi_arch_ci_coverage_nightly.yml | 113 +++++---- .github/workflows/test_component.yml | 55 +---- build_tools/artifact_manager.py | 34 +-- .../overlay_coverage_artifacts.py | 223 ----------------- .../tests/overlay_coverage_artifacts_test.py | 228 ------------------ .../tests/artifact_manager_tool_test.py | 62 ----- 8 files changed, 90 insertions(+), 695 deletions(-) delete mode 100644 build_tools/github_actions/overlay_coverage_artifacts.py delete mode 100644 build_tools/github_actions/tests/overlay_coverage_artifacts_test.py diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index da36aeb2a95..f9b0e45ffcf 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -32,7 +32,6 @@ on: - ".github/workflows/multi_arch_ci_coverage_linux.yml" - ".github/workflows/test_component.yml" - "build_tools/github_actions/configure_coverage_ci.py" - - "build_tools/github_actions/overlay_coverage_artifacts.py" - "build_tools/github_actions/merge_coverage_report.py" - "cmake/therock_subproject.cmake" - "CMakeLists.txt" @@ -51,31 +50,17 @@ jobs: secrets: inherit # A called workflow cannot hold more than its caller, so these are granted # here even though the coverage workflow declares them itself: id-token for - # the AWS role the artifact fetches assume, actions:read for the baseline - # run lookup. + # the AWS role the artifact fetches assume, actions:read for the profraw + # artifact downloads. permissions: contents: read actions: read id-token: write with: - # Pinned rather than resolved, so that a change in the report means a - # change in this branch. - # - # ROCm/TheRock Multi-Arch CI run 33990288042 (commit ab8dfcd, 2026-09-05), - # whose Stage - Compiler Runtime and Stage - Math Libs (gfx94X-dcgpu) both - # succeeded. Its own test jobs failed, which does not matter here: only - # the build stages are consumed. - # - # A ROCm/TheRock run rather than a rockrel release nightly. The two are - # interchangeable for this workflow, which only installs the baseline at - # test time and never configures against it, but staying in-repository - # keeps the embedded artifact workspace paths (/__w/TheRock/TheRock) - # matching this run's, which is one fewer variable while the instrumented - # build is still unproven. - baseline_run_id: "33990288042" - # That run published to therock-ci-artifacts, not the nightly bucket, so - # the baseline has to be read from the ci channel rather than the default. - baseline_release_type: "ci" - amdgpu_families: "gfx94X-dcgpu" + # One project, so the build instruments hipRAND and nothing else and the + # run's own artifacts are already an isolated stack. Naming a wider + # selection here would work, but each addition is another instrumented + # library in a build that is already the expensive part of the run. projects_to_test: "hiprand" + amdgpu_families: "gfx94X-dcgpu" test_type: "standard" diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index cf1f206df78..5236ed03b0b 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -9,14 +9,14 @@ # test (collect profraw) -> merge -> report # # The instrumented stack itself is built by the caller -# (multi_arch_ci_coverage_nightly.yml), which instruments every onboarded -# project in one pass. This workflow only consumes it. +# (multi_arch_ci_coverage_nightly.yml). This workflow only consumes it, and it +# consumes it from that one run: the coverage run builds every stage it needs, +# instrumenting only the projects the selection named, so its own artifacts are +# already the stack to test against. # -# `baseline_run_id` drives the hybrid install: the tree comes from a regular -# nightly run and only the project under test is overlaid from the instrumented -# run, keeping instrumented dependencies out of the report. Without it the -# instrumented run's own artifacts are tested as built, which measures the -# project's dependencies alongside it. +# The report is scoped by `object_globs` rather than by what is instrumented. +# llvm-cov only reports functions found in the objects it is given, so a +# selection covering several projects still yields one report per project. name: Multi-Arch CI Coverage - Linux @@ -36,29 +36,6 @@ on: type: string default: "" description: "Extra install_rocm_from_artifacts.py args for the report job" - baseline_run_id: - type: string - default: "" - description: >- - Regular nightly run to take non-instrumented dependencies from. Set - by the nightly caller, which instruments the whole stack in its own - run; the test jobs then install that run and overlay only the project - under test. Empty tests the instrumented artifacts as built. - baseline_release_type: - type: string - default: "nightly" - description: >- - Release channel baseline_run_id published to. Artifacts are bucketed - by channel, so the baseline is read from its own channel rather than - from this run's. - artifact_names: - type: string - default: "" - description: "Artifact names holding the instrumented project, for the hybrid fetch" - artifact_relpaths: - type: string - default: "" - description: "Subproject stage directories to overlay, for the hybrid fetch" codecov_flag: type: string default: "" @@ -162,10 +139,6 @@ jobs: build_variant: coverage coverage_enabled: true coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} - coverage_baseline_run_id: ${{ inputs.baseline_run_id }} - coverage_baseline_release_type: ${{ inputs.baseline_release_type }} - coverage_artifact_names: ${{ inputs.artifact_names }} - coverage_artifact_relpaths: ${{ inputs.artifact_relpaths }} quartz_tracking_id: ${{ inputs.quartz_tracking_id }} aggregate_coverage: diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index cbcec6ddf5e..9ae7859cde3 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -16,30 +16,22 @@ # (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and # multi-architecture expansion come later. # -# Scheduling is the RFC's option B1 for now: dispatch this workflow by hand and -# pass the run id of a recent nightly as `baseline_run_id`. There is no cron +# The run is self-contained. It builds every stage it needs and instruments only +# the projects named by `projects_to_test`, so everything those projects depend +# on is built normally in the same run and its artifacts are already the stack +# to test against. There is no second run id to coordinate and no baseline to +# mix in. +# +# Scheduling is manual for now: dispatch this workflow by hand. There is no cron # here, and the regular nightly does not know about this workflow, so coverage # has a run id of its own -- a failure here leaves the nightly's status alone, # and the nightly's unstable test jobs do not hold coverage up. -# -# The RFC's option C, where the regular nightly dispatches this workflow once -# its build finishes and hands over its own run id automatically, is a later -# change. Everything below already takes `baseline_run_id` as an input, so that -# change only has to add the dispatching job. name: Multi-Arch CI Coverage Nightly on: workflow_dispatch: inputs: - baseline_run_id: - type: string - description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." - default: "" - baseline_release_type: - type: string - description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." - default: "nightly" projects_to_test: type: string description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." @@ -71,14 +63,6 @@ on: # Remove this trigger with the check workflow once the run has gone green. workflow_call: inputs: - baseline_run_id: - type: string - description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." - default: "" - baseline_release_type: - type: string - description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." - default: "nightly" projects_to_test: type: string description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." @@ -105,11 +89,11 @@ on: permissions: contents: read -# The two run ids are only connected through this input, so surface it in the -# run title rather than making people open the logs to find the baseline. +# Which projects a run measured is the thing people look for afterwards, so put +# it in the title rather than making them open the setup job's log. run-name: >- Multi-Arch CI Coverage Nightly - | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} + | ${{ inputs.projects_to_test || 'all onboarded projects' }} | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} concurrency: @@ -125,6 +109,8 @@ jobs: dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} + generic_stages_json: ${{ steps.matrix.outputs.generic_stages_json }} + needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} steps: - name: Checking out repository @@ -152,8 +138,10 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - # Built first so that every later stage in this run has a compiler-runtime to - # link against and to fetch as an inbound artifact. + # Built first and unconditionally: every other stage takes its inbound + # artifacts from this one, so no selection can skip it. It also receives the + # coverage flags, because rocm-systems projects such as rocprofiler-sdk and + # amdsmi are built here rather than in a stage of their own. build_instrumented_compiler_runtime: name: Build Instrumented Compiler Runtime needs: setup_coverage_matrix @@ -165,7 +153,41 @@ jobs: stage_display_name: "Stage - Coverage Compiler Runtime" timeout_minutes: 480 # 8 hours (compiler is big) dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + extra_cmake_options: >- + ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + permissions: + contents: read + id-token: write + + # The build-once stages the selection needs, other than compiler-runtime. + # Empty for a selection confined to math-libs, which is why the matrix is + # guarded: fromJSON('[]') produces a job with no combinations. + build_instrumented_generic_stages: + name: Build Instrumented Stages + needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + if: >- + ${{ + !failure() && !cancelled() && + needs.setup_coverage_matrix.outputs.generic_stages_json != '[]' + }} + strategy: + fail-fast: false + matrix: + stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.generic_stages_json) }} + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml + secrets: inherit + with: + stage_name: ${{ matrix.stage_info.stage_name }} + stage_display_name: ${{ matrix.stage_info.stage_display_name }} + timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + extra_cmake_options: >- + ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} @@ -174,15 +196,18 @@ jobs: id-token: write # The whole stage is built, not just the onboarded projects: the test jobs - # below need the complete dependency set, and phase 1 instruments every - # onboarded project in one pass rather than per changed project. - # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the - # selection covers a whole group, and the individual project flags otherwise; - # everything not selected builds normally. + # below need the complete dependency set. `coverage_cmake_options` is the + # THEROCK_COVERAGE_*_ALL group option when the selection covers a whole group, + # and the individual project flags otherwise; everything not selected builds + # normally. build_instrumented_math_libs: name: Build Instrumented Math Libs needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] - if: ${{ !failure() && !cancelled() }} + if: >- + ${{ + !failure() && !cancelled() && + needs.setup_coverage_matrix.outputs.needs_math_libs == 'true' + }} strategy: fail-fast: false matrix: @@ -205,13 +230,16 @@ jobs: contents: read id-token: write - # One test + report job per project. The instrumented stack built above is the - # only thing taken from this run: each job installs `baseline_run_id` and then - # overlays its own project, so a project's dependencies stay non-instrumented - # even though the build instrumented all of them at once. + # One test + report job per project, all against the stack built above. A + # selection naming several projects instruments all of them in that one build, + # and each job still reports on its own: llvm-cov is given only that project's + # objects, so counters from the others are never looked up. coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) - needs: [setup_coverage_matrix, build_instrumented_math_libs] + needs: + - setup_coverage_matrix + - build_instrumented_generic_stages + - build_instrumented_math_libs # `!failure() && !cancelled()` alone would let this run when # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so # the matrix has to be checked explicitly. @@ -234,10 +262,6 @@ jobs: fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} codecov_flag: ${{ matrix.coverage.codecov_flag }} amdgpu_families: ${{ matrix.coverage.amdgpu_families }} - baseline_run_id: ${{ inputs.baseline_run_id }} - baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} - artifact_names: ${{ matrix.coverage.artifact_names }} - artifact_relpaths: ${{ matrix.coverage.artifact_relpaths }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} @@ -251,6 +275,7 @@ jobs: needs: - setup_coverage_matrix - build_instrumented_compiler_runtime + - build_instrumented_generic_stages - build_instrumented_math_libs - coverage_report runs-on: ubuntu-24.04 diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index f5acd942695..c231cf0f84d 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -49,35 +49,6 @@ on: something project-specific to keep each project's profiles apart. type: string default: "coverage-profraw" - coverage_baseline_run_id: - description: >- - Regular nightly run holding the non-instrumented artifacts. When set, - the whole install tree comes from that run and only the project under - test is overlaid from this run's instrumented build, so its - dependencies stay out of the coverage report. Empty means the run's - own artifacts are already the ones to test. - type: string - default: "" - coverage_baseline_release_type: - description: >- - Release channel the baseline run published to. Artifacts are bucketed - by channel, so this has to match the run that produced them rather - than the coverage run reading them. - type: string - default: "nightly" - coverage_artifact_names: - description: >- - Comma-separated artifact names holding the instrumented project - (e.g. 'rand'). Required alongside coverage_baseline_run_id. - type: string - default: "" - coverage_artifact_relpaths: - description: >- - Comma-separated subproject stage directories to overlay from those - artifacts (e.g. 'math-libs/hipRAND/stage'). TheRock's artifacts are - grouped, so this is what keeps sibling projects non-instrumented. - type: string - default: "" default_container_image: type: string default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" @@ -127,12 +98,6 @@ jobs: QUARTZ_WORKFLOW_POSTFIX: "- test ROCm component" VENV_DIR: ${{ github.workspace }}/.venv ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - # Nightly coverage installs the baseline run and overlays the - # instrumented project afterwards; every other run installs its own - # artifacts directly. The channel travels with the run id because - # artifacts are bucketed by channel. - INSTALL_ARTIFACT_RUN_ID: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_run_id || (inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id) }}" - INSTALL_RELEASE_TYPE: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_release_type || inputs.release_type }}" OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -188,32 +153,14 @@ jobs: STEP_TIMEOUT_MINUTES: "15" # should match timeout-minutes specified above at this step uses: './.github/actions/setup_test_environment' with: - ARTIFACT_RUN_ID: ${{ env.INSTALL_ARTIFACT_RUN_ID }} + ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }} # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved ARTIFACT_GROUP: ${{ inputs.amdgpu_families }} AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }} VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} - RELEASE_TYPE: ${{ env.INSTALL_RELEASE_TYPE }} - - # The step above installed the baseline run, so every library here is - # non-instrumented. Replacing just this project's files leaves its - # dependencies out of the profiles the test run is about to write. - - name: Overlay instrumented coverage artifacts - if: ${{ inputs.coverage_baseline_run_id != '' }} - timeout-minutes: 15 - env: - GITHUB_TOKEN: ${{ github.token }} RELEASE_TYPE: ${{ inputs.release_type }} - run: | - python ./build_tools/github_actions/overlay_coverage_artifacts.py \ - --run-id="${{ github.run_id }}" \ - --amdgpu-families="${{ inputs.amdgpu_families }}" \ - --amdgpu-targets="${{ inputs.amdgpu_targets }}" \ - --artifact-names="${{ inputs.coverage_artifact_names }}" \ - --artifact-relpaths="${{ inputs.coverage_artifact_relpaths }}" \ - --install-dir="${{ env.OUTPUT_ARTIFACTS_DIR }}" # safe.directory must be set before Runner Health Status - name: Adjust git config diff --git a/build_tools/artifact_manager.py b/build_tools/artifact_manager.py index 121482653a7..ddb5d23f0c1 100644 --- a/build_tools/artifact_manager.py +++ b/build_tools/artifact_manager.py @@ -236,12 +236,11 @@ def parse_excluded_components(raw_components: str) -> set[str]: return components -def parse_artifact_names( +def parse_excluded_artifacts( raw_artifacts: str, valid_artifacts: set[str], - option_name: str, ) -> set[str]: - """Parse and validate a comma- or semicolon-separated artifact name list.""" + """Parse and validate artifact names passed to --exclude-artifacts.""" artifacts = { artifact.strip() for artifact in raw_artifacts.replace(";", ",").split(",") @@ -250,7 +249,7 @@ def parse_artifact_names( invalid_artifacts = artifacts - valid_artifacts if invalid_artifacts: raise ValueError( - f"Invalid artifact name(s) for {option_name}: " + "Invalid artifact name(s): " f"{', '.join(sorted(invalid_artifacts))}. " f"Valid artifacts are: {', '.join(sorted(valid_artifacts))}" ) @@ -470,23 +469,9 @@ def do_fetch(args: argparse.Namespace): ) target_families = parse_target_families(args) - valid_artifacts = set(topology.artifacts.keys()) - - # An include list narrows the stage's inbound set; hybrid fetches use it to - # pull one instrumented artifact from a coverage run and everything else - # from a regular run. - included_artifacts = parse_artifact_names( - args.artifact_names, valid_artifacts, "--artifact-names" - ) - if included_artifacts: - log(f"Limiting to artifacts: {', '.join(sorted(included_artifacts))}") - inbound &= included_artifacts - if not inbound: - log("No requested artifacts are inbound for this stage") - return - - excluded_artifacts = parse_artifact_names( - args.exclude_artifacts, valid_artifacts, "--exclude-artifacts" + excluded_artifacts = parse_excluded_artifacts( + args.exclude_artifacts, + set(topology.artifacts.keys()), ) if excluded_artifacts: log(f"Excluding artifacts: {', '.join(sorted(excluded_artifacts))}") @@ -1356,13 +1341,6 @@ def main(argv: Optional[List[str]] = None): help="Comma- or semicolon-separated artifact components to exclude " f"when fetching. Valid components: {', '.join(ARTIFACT_COMPONENTS)}", ) - fetch_parser.add_argument( - "--artifact-names", - type=str, - default="", - help="Comma- or semicolon-separated artifact names to fetch. Empty " - "(the default) fetches every artifact inbound to the stage.", - ) fetch_parser.add_argument( "--exclude-artifacts", type=str, diff --git a/build_tools/github_actions/overlay_coverage_artifacts.py b/build_tools/github_actions/overlay_coverage_artifacts.py deleted file mode 100644 index 557230e9123..00000000000 --- a/build_tools/github_actions/overlay_coverage_artifacts.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Overlays one project's instrumented binaries onto a baseline install tree. - -Nightly coverage builds an instrumented stack in its own workflow run while the -regular nightly keeps publishing the normal one, so the two live under separate -run ids. A coverage test job wants exactly one instrumented project and -non-instrumented everything else: an instrumented dependency would emit its own -profiles and move the coverage denominator around whenever that dependency -changes. - -The test job gets there in two steps. It first installs the whole baseline run -the usual way, then calls this script to fetch the instrumented artifact from -the coverage run and copy just the project under test over the baseline copy. - -TheRock's artifacts are grouped -- `rand` carries both rocRAND and hipRAND -- -so a whole artifact is too coarse to overlay. Each artifact keeps its files -under the subproject stage directory they were built in -(`math-libs/hipRAND/stage/...`), and that is what --artifact-relpaths names, so -sibling projects in the same artifact stay non-instrumented. -""" - -import argparse -import logging -import os -import shutil -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) - -from artifact_manager import main as artifact_manager_main - -logging.basicConfig(level=logging.INFO, format="%(message)s") - -# Artifact components an instrumented project can contribute: `lib` the -# instrumented libraries, `test` the test binaries (instrumented in their own -# right for header-only projects), and `dev` the headers and CMake config that -# go with them. The rest only carry symbols and documentation. -EXCLUDED_COMPONENTS = ["run", "dbg", "doc"] - -# artifact_manager.py fetch explodes each archive into this subdirectory of its -# output directory, one directory per artifact. -FETCH_SUBDIR = "artifacts" - - -def split_list(raw: str) -> list[str]: - return [item.strip() for item in raw.split(",") if item.strip()] - - -def fetch_instrumented_artifacts( - *, - run_id: str, - artifact_names: list[str], - amdgpu_families: str, - amdgpu_targets: str, - output_dir: Path, - run_github_repo: str | None, -) -> Path: - """Downloads and explodes the named artifacts, returning their parent dir. - - Neither flattened nor bootstrapped: the overlay needs the per-subproject - directory layout that flattening would collapse. - """ - argv = [ - "fetch", - "--run-id", - run_id, - # These artifacts are outputs of the stage that built them rather than - # inbound to any stage, so the whole topology is the search space and - # --artifact-names does the narrowing. - "--stage", - "all", - "--artifact-names", - ",".join(artifact_names), - "--exclude-components", - ",".join(EXCLUDED_COMPONENTS), - "--amdgpu-families", - amdgpu_families, - "--output-dir", - os.fspath(output_dir), - ] - if amdgpu_targets: - argv.extend(["--amdgpu-targets", amdgpu_targets]) - if run_github_repo: - argv.extend(["--run-github-repo", run_github_repo]) - - logging.info("Fetching instrumented artifacts: %s", " ".join(argv)) - artifact_manager_main(argv) - return output_dir / FETCH_SUBDIR - - -def overlay_relpaths( - *, staging_dir: Path, relpaths: list[str], install_dir: Path -) -> int: - """Copies each artifact's subtree over the install tree. - - Returns the number of subtrees copied. Symlinks are preserved so the - libfoo.so -> libfoo.so.1 chain the loader follows still points at the - instrumented file. - """ - if not staging_dir.is_dir(): - return 0 - - copied = 0 - for artifact_dir in sorted(p for p in staging_dir.iterdir() if p.is_dir()): - for relpath in relpaths: - source = artifact_dir / relpath - if not source.is_dir(): - continue - logging.info("Overlaying %s onto %s", source, install_dir) - shutil.copytree( - source, - install_dir, - symlinks=True, - dirs_exist_ok=True, - ) - copied += 1 - return copied - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--run-id", - required=True, - help="Coverage workflow run that published the instrumented artifacts", - ) - parser.add_argument( - "--amdgpu-families", - required=True, - help="Semicolon-separated GPU families to fetch (e.g. 'gfx94X-dcgpu')", - ) - parser.add_argument( - "--amdgpu-targets", - default="", - help="Comma-separated individual GPU targets for split artifacts", - ) - parser.add_argument( - "--artifact-names", - required=True, - help="Comma-separated artifact names holding the instrumented project " - "(e.g. 'rand')", - ) - parser.add_argument( - "--artifact-relpaths", - required=True, - help="Comma-separated subproject stage directories to overlay " - "(e.g. 'math-libs/hipRAND/stage')", - ) - parser.add_argument( - "--install-dir", - type=Path, - required=True, - help="Baseline install tree to overlay the instrumented files onto", - ) - parser.add_argument( - "--staging-dir", - type=Path, - default=None, - help="Directory for the downloaded artifacts (default: a temporary directory)", - ) - parser.add_argument( - "--run-github-repo", - default=None, - help="GitHub repository for --run-id in 'owner/repo' format", - ) - args = parser.parse_args(argv) - - artifact_names = split_list(args.artifact_names) - relpaths = split_list(args.artifact_relpaths) - if not artifact_names or not relpaths: - logging.error( - "--artifact-names and --artifact-relpaths must both be non-empty." - ) - return 1 - - install_dir = args.install_dir.resolve() - if not install_dir.is_dir(): - logging.error( - "Install directory %s does not exist. The baseline artifacts must " - "be installed before the instrumented ones are overlaid.", - install_dir, - ) - return 1 - - with tempfile.TemporaryDirectory() as tmp: - staging_dir = args.staging_dir or Path(tmp) - staging_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir = fetch_instrumented_artifacts( - run_id=args.run_id, - artifact_names=artifact_names, - amdgpu_families=args.amdgpu_families, - amdgpu_targets=args.amdgpu_targets, - output_dir=staging_dir, - run_github_repo=args.run_github_repo, - ) - copied = overlay_relpaths( - staging_dir=artifacts_dir, - relpaths=relpaths, - install_dir=install_dir, - ) - - # Silently testing the baseline build would report coverage against - # binaries that were never instrumented, so treat this as fatal. - if not copied: - logging.error( - "None of the artifact relpaths (%s) were found in the artifacts " - "fetched from run %s. The instrumented build did not produce the " - "expected layout.", - ", ".join(relpaths), - args.run_id, - ) - return 1 - - logging.info("Overlaid %d instrumented subtree(s) onto %s", copied, install_dir) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py deleted file mode 100644 index 47899cb9242..00000000000 --- a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -import os -import platform -import sys -import tempfile -import unittest -from pathlib import Path -from unittest import mock - -# Add repo root to PYTHONPATH -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) - -import overlay_coverage_artifacts - - -def is_windows() -> bool: - return platform.system() == "Windows" - - -class TempDirTestBase(unittest.TestCase): - def setUp(self): - self._temp_dir = tempfile.TemporaryDirectory() - self.addCleanup(self._temp_dir.cleanup) - self.root = Path(self._temp_dir.name) - - def write(self, relative_path: str, contents: str) -> Path: - path = self.root / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(contents) - return path - - -class FetchInstrumentedArtifactsTest(TempDirTestBase): - def test_narrows_the_fetch_to_the_named_artifacts(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - artifacts_dir = overlay_coverage_artifacts.fetch_instrumented_artifacts( - run_id="99999", - artifact_names=["rand", "prim"], - amdgpu_families="gfx94X-dcgpu", - amdgpu_targets="", - output_dir=self.root / "staging", - run_github_repo=None, - ) - - (argv,) = fetch.call_args.args - self.assertEqual(argv[0], "fetch") - # These artifacts are stage outputs, not inbound to any stage. - self.assertEqual(argv[argv.index("--stage") + 1], "all") - self.assertEqual(argv[argv.index("--artifact-names") + 1], "rand,prim") - self.assertEqual(artifacts_dir, self.root / "staging" / "artifacts") - - def test_skips_components_that_cannot_be_instrumented(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - overlay_coverage_artifacts.fetch_instrumented_artifacts( - run_id="99999", - artifact_names=["rand"], - amdgpu_families="gfx94X-dcgpu", - amdgpu_targets="gfx942", - output_dir=self.root / "staging", - run_github_repo=None, - ) - - (argv,) = fetch.call_args.args - excluded = argv[argv.index("--exclude-components") + 1].split(",") - self.assertCountEqual(excluded, ["run", "dbg", "doc"]) - self.assertEqual(argv[argv.index("--amdgpu-targets") + 1], "gfx942") - - -class OverlayRelpathsTest(TempDirTestBase): - def _staged_artifact(self, artifact: str, relpath: str, files: dict[str, str]): - for name, contents in files.items(): - self.write(f"staging/{artifact}/{relpath}/{name}", contents) - - def test_copies_only_the_requested_relpath(self): - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/hipRAND/stage", - {"lib/libhiprand.so": "instrumented"}, - ) - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/rocRAND/stage", - {"lib/librocrand.so": "instrumented"}, - ) - self.write("install/lib/libhiprand.so", "baseline") - self.write("install/lib/librocrand.so", "baseline") - - copied = overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertEqual(copied, 1) - self.assertEqual( - (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" - ) - # A sibling project in the same grouped artifact stays as it was. - self.assertEqual( - (self.root / "install/lib/librocrand.so").read_text(), "baseline" - ) - - def test_merges_into_the_existing_tree(self): - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/hipRAND/stage", - {"lib/libhiprand.so": "instrumented"}, - ) - self.write("install/lib/libamdhip64.so", "baseline") - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertTrue((self.root / "install/lib/libamdhip64.so").exists()) - self.assertTrue((self.root / "install/lib/libhiprand.so").exists()) - - def test_counts_every_matching_artifact(self): - for component in ("lib", "test"): - self._staged_artifact( - f"rand_{component}_gfx942", - "math-libs/hipRAND/stage", - {f"{component}/file": component}, - ) - (self.root / "install").mkdir() - - self.assertEqual( - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ), - 2, - ) - - def test_reports_nothing_copied_when_the_relpath_is_absent(self): - self._staged_artifact( - "rand_lib_gfx942", "math-libs/rocRAND/stage", {"lib/librocrand.so": "x"} - ) - (self.root / "install").mkdir() - - self.assertEqual( - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ), - 0, - ) - - @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") - def test_preserves_symlinks(self): - # The loader follows libfoo.so -> libfoo.so.1; copying the link as a - # regular file would leave a stale non-instrumented copy behind it. - stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" - self.write(f"{stage}/libhiprand.so.1", "instrumented") - (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") - (self.root / "install").mkdir() - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertTrue((self.root / "install/lib/libhiprand.so").is_symlink()) - - -class MainTest(TempDirTestBase): - def _run(self, **overrides) -> int: - args = { - "--run-id": "99999", - "--amdgpu-families": "gfx94X-dcgpu", - "--artifact-names": "rand", - "--artifact-relpaths": "math-libs/hipRAND/stage", - "--install-dir": os.fspath(self.root / "install"), - "--staging-dir": os.fspath(self.root / "staging"), - } - args.update(overrides) - argv = [item for pair in args.items() for item in pair] - return overlay_coverage_artifacts.main(argv) - - def test_fails_when_the_install_tree_is_missing(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - self.assertEqual(self._run(), 1) - fetch.assert_not_called() - - def test_fails_when_nothing_was_overlaid(self): - # A silent pass here would report coverage for binaries that were never - # instrumented. - (self.root / "install").mkdir() - with mock.patch.object(overlay_coverage_artifacts, "artifact_manager_main"): - self.assertEqual(self._run(), 1) - - def test_fetches_and_overlays(self): - (self.root / "install").mkdir() - - def fake_fetch(argv): - self.write( - "staging/artifacts/rand_lib_gfx942/math-libs/hipRAND/stage" - "/lib/libhiprand.so", - "instrumented", - ) - - with mock.patch.object( - overlay_coverage_artifacts, - "artifact_manager_main", - side_effect=fake_fetch, - ): - self.assertEqual(self._run(), 0) - - self.assertEqual( - (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/tests/artifact_manager_tool_test.py b/build_tools/tests/artifact_manager_tool_test.py index 3eeb4579cd9..4fc925e17a0 100644 --- a/build_tools/tests/artifact_manager_tool_test.py +++ b/build_tools/tests/artifact_manager_tool_test.py @@ -720,68 +720,6 @@ def mock_extract(request: artifact_manager.ExtractRequest) -> Path: self.assertIn("test-artifact_lib_generic.tar.zst", fetched_keys) self.assertNotIn("second-artifact_lib_generic.tar.zst", fetched_keys) - def test_fetch_artifact_names_limits_to_named_artifacts(self) -> None: - """Test that --artifact-names narrows the fetch to an include list.""" - import artifact_manager - - self._create_staged_artifact("test-artifact", "lib", "generic") - self._create_staged_artifact("second-artifact", "lib", "generic") - - extract_calls: list[artifact_manager.ExtractRequest] = [] - - def mock_extract(request: artifact_manager.ExtractRequest) -> Path: - extract_calls.append(request) - return request.output_dir - - with mock.patch("artifact_manager.extract_artifact", mock_extract): - argv = [ - "fetch", - "--stage", - "all", - "--output-dir", - str(self.output_dir), - "--topology", - str(self.topology_path), - "--local-staging-dir", - str(self.staging_dir), - "--platform", - TEST_PLATFORM, - "--run-id", - "local", - "--artifact-names", - "second-artifact", - ] - - artifact_manager.main(argv) - - fetched_keys = [c.archive_path.name for c in extract_calls] - self.assertEqual(fetched_keys, ["second-artifact_lib_generic.tar.zst"]) - - def test_fetch_artifact_names_rejects_unknown_artifact(self) -> None: - """Test that --artifact-names validates artifact names.""" - import artifact_manager - - argv = [ - "fetch", - "--stage", - "all", - "--output-dir", - str(self.output_dir), - "--topology", - str(self.topology_path), - "--local-staging-dir", - str(self.staging_dir), - "--platform", - TEST_PLATFORM, - "--run-id", - "local", - "--artifact-names", - "unknown-artifact", - ] - - with self.assertRaises(ValueError): - artifact_manager.main(argv) - def test_fetch_exclude_artifacts_rejects_unknown_artifact(self) -> None: """Test that --exclude-artifacts validates artifact names.""" argv = [ From 1fa0945c80f0aceb1b3a64354b0f7c408b73c7e6 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 05:28:02 -0500 Subject: [PATCH 11/70] fix(ci): forward each project's own coverage option THEROCK_COVERAGE_* set _ENABLE_COVERAGE and therock_subproject.cmake passed that name straight through to the subproject. Almost no project implements it. Of the projects registered here only hipBLASLt, hipSPARSE and hipDNN use that spelling; the rest use BUILD_CODE_COVERAGE, CODE_COVERAGE or ENABLE_CODE_COVERAGE. hipRAND, the only project the registry had until now, gates its instrumentation on BUILD_CODE_COVERAGE. So -DHIPRAND_ENABLE_COVERAGE=ON was forwarded, ignored, and the build produced uninstrumented binaries. The tests would have written no profraw files and the report job would have failed on finding none. Run 34098007786 never got that far. Registry entries now name the option their project actually implements, and therock_subproject.cmake translates TheRock's flag into it. The translation is per subproject on purpose: BUILD_CODE_COVERAGE and CODE_COVERAGE are generic names, so defining either globally would instrument everything in the build that understands it, which is what per-project coverage exists to avoid. Also expands the registry to the 27 projects across rocm-libraries and rocm-systems, so a run can measure a chosen subset, and makes the nightly build only the stages that subset needs. Eight of those cannot be measured by this pipeline. hipSPARSE, rocALUTION and rocprofiler-sdk instrument with gcov, which writes .gcda files rather than the .profraw llvm-profdata merges; MIOpen, aqlprofile, amdsmi, rocprofiler-compute and rocprofiler-systems have no build-time coverage option at all. They stay registered so the gap is recorded, but carry an unsupported_reason instead of an option name, stay out of the group aliases, and are rejected with that reason when named rather than failing hours later with an empty report. Option names verified against the pinned rocm-libraries and rocm-systems commits. Co-authored-by: Cursor --- CMakeLists.txt | 10 +- .../github_actions/configure_coverage_ci.py | 501 ++++++++++++++++-- .../tests/configure_coverage_ci_test.py | 206 +++++-- cmake/therock_subproject.cmake | 24 +- docs/development/code_coverage.md | 94 ++-- docs/development/code_coverage_flows.md | 207 +++----- 6 files changed, 767 insertions(+), 275 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ca4655d235..20072d4f49b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,10 +215,12 @@ option(THEROCK_USE_SAFE_DEPENDENCY_PROVIDER "Enable the safe dependency provider ################################################################################ # Coverage Options # Enable code-coverage instrumentation for a whole component group by setting the -# per-project _ENABLE_COVERAGE flags that therock_subproject.cmake -# forwards to each subproject. The per-group project lists are generated from the -# coverage registry (COVERAGE_PROJECTS in -# build_tools/github_actions/configure_coverage_ci.py) at configure time. +# per-project _ENABLE_COVERAGE flags. therock_subproject.cmake turns +# each one into whichever option that subproject actually implements, using the +# THEROCK_COVERAGE_OPTION_ map generated below. Both the group lists +# and that map come from the coverage registry (COVERAGE_PROJECTS in +# build_tools/github_actions/configure_coverage_ci.py) at configure time, and +# cover only the projects that emit LLVM profile data. ################################################################################ option(THEROCK_COVERAGE_ROCM_LIBRARIES_ALL diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 5140a70a161..5f93a8ddd75 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -45,6 +45,26 @@ ROCM_LIBRARIES = "rocm-libraries" ROCM_SYSTEMS = "rocm-systems" +# Build stages that can carry an instrumented project. compiler-runtime is +# always built because every other stage takes its inbound artifacts from it; +# the rest are built only when the selection needs them. +STAGE_COMPILER_RUNTIME = "compiler-runtime" +STAGE_MATH_LIBS = "math-libs" +STAGE_COMM_LIBS = "comm-libs" +STAGE_PROFILER_APPS = "profiler-apps" + +# Only math-libs is built per GPU family; the others are built once. +PER_ARCH_STAGES = frozenset({STAGE_MATH_LIBS}) + +# Display names for the per-stage build jobs, so the Actions UI reads the same +# way as the regular nightly's. +STAGE_DISPLAY_NAMES: dict[str, str] = { + STAGE_COMPILER_RUNTIME: "Coverage Compiler Runtime", + STAGE_MATH_LIBS: "Coverage Math Libs", + STAGE_COMM_LIBS: "Coverage Comm Libs", + STAGE_PROFILER_APPS: "Coverage Profiler Apps", +} + @dataclass(frozen=True) class CoverageProject: @@ -52,8 +72,19 @@ class CoverageProject: Attributes: cmake_target: TheRock subproject name. Upper cased to form the - `_ENABLE_COVERAGE` flag that therock_subproject.cmake - forwards to the subproject. + `_ENABLE_COVERAGE` flag, which is TheRock's own knob and + is not what the subproject sees; see coverage_option. + coverage_option: CMake option the upstream project implements to turn + its instrumentation on. These names are not standardised -- most + projects use `BUILD_CODE_COVERAGE` or `CODE_COVERAGE`, a few use + `_ENABLE_COVERAGE`, RCCL uses `ENABLE_CODE_COVERAGE` -- + so therock_subproject.cmake translates TheRock's flag into this + name when configuring the subproject. Being per-subproject is what + keeps the generic names from instrumenting the whole build. + unsupported_reason: Set instead of coverage_option when the project + cannot be measured by this pipeline. Such a project stays in the + registry so the gap is recorded, but is left out of the group + aliases and rejected if named explicitly. stage: Build stage that produces the project. The nightly needs a build job for this stage, so onboarding a project from a new stage means adding one. @@ -67,44 +98,345 @@ class CoverageProject: codecov_flag: Flag the report is filed under in Codecov. source_repo: The repo this project ships from (ROCM_LIBRARIES or ROCM_SYSTEMS); drives the group aliases. - artifact_names: BUILD_TOPOLOGY artifact(s) the instrumented project - ships in. These are grouped (`rand` holds both rocRAND and - hipRAND), which is why artifact_relpaths exists. - artifact_relpaths: Subproject stage directories inside those artifacts - that belong to this project alone. The nightly hybrid fetch copies - only these over the baseline install tree, so a run measuring - hipRAND does not also pick up an instrumented rocRAND. + extra_cmake_targets: Additional subprojects to instrument alongside + cmake_target. Header-only projects report against their own test + binaries, and some of those are built by a sibling subproject + (rocPRIM's live in rocPRIM_tests), which therefore has to be + instrumented too or the binaries carry no coverage mapping. """ cmake_target: str stage: str test_component: str coverage_config: str + coverage_option: str = "" + unsupported_reason: str = "" object_globs: list[str] = field(default_factory=list) fetch_artifact_args: str = "" codecov_flag: str = "" source_repo: str = ROCM_LIBRARIES - artifact_names: list[str] = field(default_factory=list) - artifact_relpaths: list[str] = field(default_factory=list) + extra_cmake_targets: list[str] = field(default_factory=list) -# Projects that participate in coverage CI. Start small (RFC0014 phase 1) and -# grow the list as component test suites become good enough to report on. +# Projects that participate in coverage CI. +# +# A run never instruments this whole list unless it is asked to. `PROJECTS_TO_TEST` +# selects a subset by name, or by one of the group aliases below, and only the +# stages that subset needs get built. Selecting everything is possible and +# expensive: instrumented builds are substantially slower than normal ones. +# +# A selection covering several projects instruments all of them in one build and +# still reports on each separately, because `object_globs` is what scopes a +# report: llvm-cov only reports functions found in the objects it is handed, so +# counters emitted by a sibling project are never looked up. # -# The nightly workflow builds one instrumented job per stage, and currently only -# has a math-libs job. Onboarding a project from another stage also means adding -# that stage's build job to multi_arch_ci_coverage_nightly.yml. +# Header-only projects (rocPRIM, hipCUB, rocThrust, rocWMMA) ship no library to +# report against, so their object_globs name their installed test binaries +# instead, and their fetch args include --tests to bring those binaries into the +# report job. COVERAGE_PROJECTS: dict[str, CoverageProject] = { + # + # rocm-libraries -- math-libs stage + # + "rocrand": CoverageProject( + cmake_target="rocRAND", + coverage_option="CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocrand", + coverage_config="projects/rocrand/test_categories_coverage.yaml", + object_globs=["lib/librocrand.so*"], + fetch_artifact_args="--rand", + codecov_flag="rocRAND", + ), "hiprand": CoverageProject( cmake_target="hipRAND", - stage="math-libs", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, test_component="hiprand", coverage_config="projects/hiprand/test_categories_coverage.yaml", object_globs=["lib/libhiprand.so*"], fetch_artifact_args="--rand", codecov_flag="hipRAND", - artifact_names=["rand"], - artifact_relpaths=["math-libs/hipRAND/stage"], + ), + "rocfft": CoverageProject( + cmake_target="rocFFT", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocfft", + coverage_config="projects/rocfft/test_categories_coverage.yaml", + object_globs=["lib/librocfft.so*"], + fetch_artifact_args="--fft", + codecov_flag="rocFFT", + ), + "hipfft": CoverageProject( + cmake_target="hipFFT", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipfft", + coverage_config="projects/hipfft/test_categories_coverage.yaml", + object_globs=["lib/libhipfft.so*"], + fetch_artifact_args="--fft", + codecov_flag="hipFFT", + ), + "rocblas": CoverageProject( + cmake_target="rocBLAS", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocblas", + coverage_config="projects/rocblas/test_categories_coverage.yaml", + object_globs=["lib/librocblas.so*"], + fetch_artifact_args="--blas", + codecov_flag="rocBLAS", + ), + "hipblas": CoverageProject( + cmake_target="hipBLAS", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipblas", + coverage_config="projects/hipblas/test_categories_coverage.yaml", + object_globs=["lib/libhipblas.so*"], + fetch_artifact_args="--blas", + codecov_flag="hipBLAS", + ), + "hipblaslt": CoverageProject( + cmake_target="hipBLASLt", + coverage_option="HIPBLASLT_ENABLE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipblaslt", + coverage_config="projects/hipblaslt/test_categories_coverage.yaml", + object_globs=["lib/libhipblaslt.so*"], + fetch_artifact_args="--blas", + codecov_flag="hipBLASLt", + ), + "rocsparse": CoverageProject( + cmake_target="rocSPARSE", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocsparse", + coverage_config="projects/rocsparse/test_categories_coverage.yaml", + object_globs=["lib/librocsparse.so*"], + fetch_artifact_args="--sparse", + codecov_flag="rocSPARSE", + ), + "hipsparse": CoverageProject( + cmake_target="hipSPARSE", + unsupported_reason=( + "HIPSPARSE_ENABLE_COVERAGE selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" + ), + stage=STAGE_MATH_LIBS, + test_component="hipsparse", + coverage_config="projects/hipsparse/test_categories_coverage.yaml", + object_globs=["lib/libhipsparse.so*"], + fetch_artifact_args="--sparse", + codecov_flag="hipSPARSE", + ), + "hipsparselt": CoverageProject( + cmake_target="hipSPARSELt", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipsparselt", + coverage_config="projects/hipsparselt/test_categories_coverage.yaml", + object_globs=["lib/libhipsparselt.so*"], + fetch_artifact_args="--sparse", + codecov_flag="hipSPARSELt", + ), + "rocsolver": CoverageProject( + cmake_target="rocSOLVER", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocsolver", + coverage_config="projects/rocsolver/test_categories_coverage.yaml", + object_globs=["lib/librocsolver.so*"], + fetch_artifact_args="--solver", + codecov_flag="rocSOLVER", + ), + "hipsolver": CoverageProject( + cmake_target="hipSOLVER", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipsolver", + coverage_config="projects/hipsolver/test_categories_coverage.yaml", + object_globs=["lib/libhipsolver.so*"], + fetch_artifact_args="--solver", + codecov_flag="hipSOLVER", + ), + "rocalution": CoverageProject( + cmake_target="rocALUTION", + unsupported_reason=( + "BUILD_CODE_COVERAGE selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" + ), + stage=STAGE_MATH_LIBS, + test_component="rocalution", + coverage_config="projects/rocalution/test_categories_coverage.yaml", + object_globs=["lib/librocalution.so*"], + fetch_artifact_args="--rocalution", + codecov_flag="rocALUTION", + ), + "hiptensor": CoverageProject( + cmake_target="hipTensor", + coverage_option="CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hiptensor", + coverage_config="projects/hiptensor/test_categories_coverage.yaml", + object_globs=["lib/libhiptensor.so*"], + fetch_artifact_args="--hiptensor", + codecov_flag="hipTensor", + ), + "miopen": CoverageProject( + cmake_target="MIOpen", + unsupported_reason="no coverage option in its CMake", + stage=STAGE_MATH_LIBS, + test_component="miopen", + coverage_config="projects/miopen/test_categories_coverage.yaml", + object_globs=["lib/libMIOpen.so*"], + fetch_artifact_args="--miopen", + codecov_flag="MIOpen", + ), + "hipdnn": CoverageProject( + cmake_target="hipDNN", + coverage_option="HIPDNN_ENABLE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipdnn", + coverage_config="projects/hipdnn/test_categories_coverage.yaml", + object_globs=["lib/libhipdnn.so*"], + fetch_artifact_args="--hipdnn", + codecov_flag="hipDNN", + ), + # + # rocm-libraries -- math-libs stage, header-only + # + "rocprim": CoverageProject( + cmake_target="rocPRIM", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocprim", + coverage_config="projects/rocprim/test_categories_coverage.yaml", + object_globs=["bin/test_*"], + fetch_artifact_args="--prim --tests", + codecov_flag="rocPRIM", + # rocPRIM's tests are a sibling subproject, so both stage dirs are + # overlaid and both are instrumented. + extra_cmake_targets=["rocPRIM_tests"], + ), + "hipcub": CoverageProject( + cmake_target="hipCUB", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="hipcub", + coverage_config="projects/hipcub/test_categories_coverage.yaml", + object_globs=["bin/test_*"], + fetch_artifact_args="--prim --tests", + codecov_flag="hipCUB", + ), + "rocthrust": CoverageProject( + cmake_target="rocThrust", + coverage_option="CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocthrust", + coverage_config="projects/rocthrust/test_categories_coverage.yaml", + object_globs=["bin/test_*"], + fetch_artifact_args="--prim --tests", + codecov_flag="rocThrust", + ), + "rocwmma": CoverageProject( + cmake_target="rocWMMA", + coverage_option="CODE_COVERAGE", + stage=STAGE_MATH_LIBS, + test_component="rocwmma", + coverage_config="projects/rocwmma/test_categories_coverage.yaml", + object_globs=["bin/*_test*"], + fetch_artifact_args="--rocwmma --tests", + codecov_flag="rocWMMA", + ), + # + # rocm-systems -- comm-libs stage + # + "rccl": CoverageProject( + cmake_target="rccl", + coverage_option="ENABLE_CODE_COVERAGE", + stage=STAGE_COMM_LIBS, + test_component="rccl", + coverage_config="projects/rccl/test_categories_coverage.yaml", + object_globs=["lib/librccl.so*"], + fetch_artifact_args="--rccl", + codecov_flag="rccl", + source_repo=ROCM_SYSTEMS, + ), + "rocshmem": CoverageProject( + cmake_target="rocshmem", + coverage_option="BUILD_CODE_COVERAGE", + stage=STAGE_COMM_LIBS, + test_component="rocshmem", + coverage_config="projects/rocshmem/test_categories_coverage.yaml", + object_globs=["lib/librocshmem.so*"], + fetch_artifact_args="--rocshmem", + codecov_flag="rocSHMEM", + source_repo=ROCM_SYSTEMS, + ), + # + # rocm-systems -- compiler-runtime stage + # + "rocprofiler-sdk": CoverageProject( + cmake_target="rocprofiler-sdk", + unsupported_reason=( + "ROCPROFILER_BUILD_CODECOV selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" + ), + stage=STAGE_COMPILER_RUNTIME, + test_component="rocprofiler-sdk", + coverage_config="projects/rocprofiler-sdk/test_categories_coverage.yaml", + object_globs=["lib/librocprofiler-sdk.so*"], + fetch_artifact_args="--rocprofiler-sdk", + codecov_flag="rocprofiler-sdk", + source_repo=ROCM_SYSTEMS, + ), + "aqlprofile": CoverageProject( + cmake_target="aqlprofile", + unsupported_reason="no coverage option in its CMake", + stage=STAGE_COMPILER_RUNTIME, + test_component="aqlprofile", + coverage_config="projects/aqlprofile/test_categories_coverage.yaml", + object_globs=["lib/libhsa-amd-aqlprofile*.so*"], + fetch_artifact_args="--aqlprofile", + codecov_flag="aqlprofile", + source_repo=ROCM_SYSTEMS, + ), + "amdsmi": CoverageProject( + cmake_target="amdsmi", + unsupported_reason="no coverage option in its CMake", + stage=STAGE_COMPILER_RUNTIME, + test_component="amdsmi", + coverage_config="projects/amdsmi/test_categories_coverage.yaml", + object_globs=["lib/libamd_smi.so*"], + fetch_artifact_args="--base-only", + codecov_flag="amdsmi", + source_repo=ROCM_SYSTEMS, + ), + "rocprofiler-compute": CoverageProject( + cmake_target="rocprofiler-compute", + unsupported_reason="a Python tool with no native instrumentation option", + stage=STAGE_COMPILER_RUNTIME, + test_component="rocprofiler-compute", + coverage_config="projects/rocprofiler-compute/test_categories_coverage.yaml", + object_globs=["lib/librocprofiler-compute*.so*"], + fetch_artifact_args="--rocprofiler-compute", + codecov_flag="rocprofiler-compute", + source_repo=ROCM_SYSTEMS, + ), + # + # rocm-systems -- profiler-apps stage + # + "rocprofiler-systems": CoverageProject( + cmake_target="rocprofiler-systems", + unsupported_reason=( + "no build-time coverage option; its coverage feature instruments other programs at runtime" + ), + stage=STAGE_PROFILER_APPS, + test_component="rocprofiler-systems", + coverage_config="projects/rocprofiler-systems/test_categories_coverage.yaml", + object_globs=["lib/librocprofiler-systems*.so*"], + fetch_artifact_args="--rocprofiler-systems", + codecov_flag="rocprofiler-systems", + source_repo=ROCM_SYSTEMS, ), } @@ -113,12 +445,31 @@ class CoverageProject: assert ( _proj.source_repo in _VALID_SOURCE_REPOS ), f"{_key}: source_repo must be one of {sorted(_VALID_SOURCE_REPOS)}" - + # A project in a stage the workflow cannot build would be selectable and + # then silently tested against uninstrumented binaries. + assert ( + _proj.stage in STAGE_DISPLAY_NAMES + ), f"{_key}: stage must be one of {sorted(STAGE_DISPLAY_NAMES)}" + # An entry with neither would be selectable and instrument nothing; one + # with both leaves it ambiguous whether the project can be measured. + assert bool(_proj.coverage_option) != bool( + _proj.unsupported_reason + ), f"{_key}: set exactly one of coverage_option and unsupported_reason" + +SUPPORTED_PROJECTS: frozenset[str] = frozenset( + k for k, v in COVERAGE_PROJECTS.items() if v.coverage_option +) +# Group membership is limited to what can actually be measured, so a group +# alias never schedules a job that is guaranteed to fail for want of profiles. ROCM_LIBRARIES_PROJECTS: frozenset[str] = frozenset( - k for k, v in COVERAGE_PROJECTS.items() if v.source_repo == ROCM_LIBRARIES + k + for k, v in COVERAGE_PROJECTS.items() + if v.source_repo == ROCM_LIBRARIES and v.coverage_option ) ROCM_SYSTEMS_PROJECTS: frozenset[str] = frozenset( - k for k, v in COVERAGE_PROJECTS.items() if v.source_repo == ROCM_SYSTEMS + k + for k, v in COVERAGE_PROJECTS.items() + if v.source_repo == ROCM_SYSTEMS and v.coverage_option ) # Group-alias tokens accepted by PROJECTS_TO_TEST (and the projects_to_test CI @@ -126,7 +477,7 @@ class CoverageProject: _GROUP_ALIASES: dict[str, frozenset[str]] = { "rocm_libraries_all": ROCM_LIBRARIES_PROJECTS, "rocm_systems_all": ROCM_SYSTEMS_PROJECTS, - "all": frozenset(COVERAGE_PROJECTS), + "all": SUPPORTED_PROJECTS, } @@ -139,10 +490,11 @@ def parse_projects(raw_projects: str) -> list[str]: """ requested = [p.strip().lower() for p in raw_projects.split(",") if p.strip()] if not requested: - return sorted(COVERAGE_PROJECTS) + return sorted(SUPPORTED_PROJECTS) expanded: list[str] = [] unknown: list[str] = [] + unsupported: list[str] = [] for token in requested: if token in _GROUP_ALIASES: group = _GROUP_ALIASES[token] @@ -152,17 +504,32 @@ def parse_projects(raw_projects: str) -> list[str]: "projects are onboarded to coverage yet." ) expanded.extend(sorted(group)) - elif token in COVERAGE_PROJECTS: + elif token in SUPPORTED_PROJECTS: expanded.append(token) + elif token in COVERAGE_PROJECTS: + unsupported.append(token) else: unknown.append(token) if unknown: raise ValueError( f"Unknown coverage project(s): {', '.join(sorted(unknown))}. " - f"Coverage-enabled projects are: {', '.join(sorted(COVERAGE_PROJECTS))}. " + f"Coverage-enabled projects are: {', '.join(sorted(SUPPORTED_PROJECTS))}. " f"Group aliases: {', '.join(sorted(_GROUP_ALIASES))}." ) + + # Failing here beats scheduling a build and several hours of tests that + # cannot produce a report. + if unsupported: + detail = "; ".join( + f"{name} ({COVERAGE_PROJECTS[name].unsupported_reason})" + for name in sorted(unsupported) + ) + raise ValueError( + f"Coverage is not supported for: {detail}. These projects are " + "registered so the gap is tracked, but cannot be measured until " + "they emit LLVM profraw data." + ) # Preserve first-seen order, drop duplicates (explicit + alias overlap). return list(dict.fromkeys(expanded)) @@ -207,8 +574,6 @@ def build_coverage_matrix( "fetch_artifact_args": project.fetch_artifact_args, "codecov_flag": project.codecov_flag or project_key, "amdgpu_families": family, - "artifact_names": ",".join(project.artifact_names), - "artifact_relpaths": ",".join(project.artifact_relpaths), } ) return matrix @@ -242,32 +607,72 @@ def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: options.append("-DTHEROCK_COVERAGE_ROCM_SYSTEMS_ALL=ON") grouped |= ROCM_SYSTEMS_PROJECTS - options.extend( - f"-D{COVERAGE_PROJECTS[key].cmake_target.upper()}_ENABLE_COVERAGE=ON" - for key in project_keys - if key not in grouped - ) + # The group options already cover extra_cmake_targets, because emit_cmake + # writes them into the group lists CMakeLists.txt expands. + for key in project_keys: + if key in grouped: + continue + project = COVERAGE_PROJECTS[key] + for target in [project.cmake_target, *project.extra_cmake_targets]: + options.append(f"-D{target.upper()}_ENABLE_COVERAGE=ON") return options +def build_stage_plan(project_keys: list[str]) -> tuple[list[dict], bool]: + """Works out which build stages the selection needs. + + Returns the generic (built-once) stages other than compiler-runtime, and + whether the per-architecture math-libs stage is needed. compiler-runtime is + excluded because it is built unconditionally: every other stage takes its + inbound artifacts from it, so there is no selection that does not need it. + """ + stages = {COVERAGE_PROJECTS[key].stage for key in project_keys} + needs_math_libs = STAGE_MATH_LIBS in stages + + generic = sorted(stages - PER_ARCH_STAGES - {STAGE_COMPILER_RUNTIME}) + return ( + [ + { + "stage_name": stage, + "stage_display_name": f"Stage - {STAGE_DISPLAY_NAMES[stage]}", + } + for stage in generic + ], + needs_math_libs, + ) + + def emit_cmake(output_path: Path) -> None: - """Writes the coverage group-membership lists as a CMake include. + """Writes the coverage group lists and option-name map as a CMake include. - Groups COVERAGE_PROJECTS by source_repo and emits one set() per group, using - each project's cmake_target (upper cased by CMake to form the - _ENABLE_COVERAGE flag). CMakeLists.txt reads this so it does not - hardcode the project names. + Two things come out of here. The group lists drive the + THEROCK_COVERAGE_*_ALL options, and are limited to projects that can be + measured. The option map tells therock_subproject.cmake which flag each + subproject actually implements, since the names are not standardised + upstream. CMakeLists.txt reads both so it does not hardcode either. """ def targets(repo: str) -> list[str]: - return sorted( - ( - p.cmake_target - for p in COVERAGE_PROJECTS.values() - if p.source_repo == repo - ), - key=str.lower, - ) + names: set[str] = set() + for p in COVERAGE_PROJECTS.values(): + if p.source_repo != repo or not p.coverage_option: + continue + # extra_cmake_targets are part of the group: a group flag that + # instrumented rocPRIM but not rocPRIM_tests would leave rocPRIM's + # only reportable binaries without a coverage mapping. + names.update([p.cmake_target, *p.extra_cmake_targets]) + return sorted(names, key=str.lower) + + # A sibling subproject is built from the same upstream source as its parent, + # so it takes the same option. + option_lines = [] + for p in sorted(COVERAGE_PROJECTS.values(), key=lambda v: v.cmake_target.lower()): + if not p.coverage_option: + continue + for target in sorted([p.cmake_target, *p.extra_cmake_targets], key=str.lower): + option_lines.append( + f"set(THEROCK_COVERAGE_OPTION_{target.upper()} {p.coverage_option})\n" + ) output_path.write_text( "# Generated by configure_coverage_ci.py --emit-cmake; do not edit.\n" @@ -275,6 +680,7 @@ def targets(repo: str) -> list[str]: "# build_tools/github_actions/configure_coverage_ci.py instead.\n" f"set(THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS {' '.join(targets(ROCM_LIBRARIES))})\n" f"set(THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS {' '.join(targets(ROCM_SYSTEMS))})\n" + + "".join(option_lines) ) @@ -309,6 +715,7 @@ def main(argv: Optional[list[str]] = None) -> int: project_keys, amdgpu_families, config_repository, config_ref ) coverage_flags = build_coverage_cmake_options(project_keys) + generic_stages, needs_math_libs = build_stage_plan(project_keys) outputs = { "coverage_matrix": json.dumps(matrix), "dist_amdgpu_families": ";".join(amdgpu_families), @@ -316,6 +723,8 @@ def main(argv: Optional[list[str]] = None) -> int: [{"amdgpu_family": family} for family in amdgpu_families] ), "coverage_cmake_options": " ".join(coverage_flags), + "generic_stages_json": json.dumps(generic_stages), + "needs_math_libs": "true" if needs_math_libs else "false", } if args.print_matrix: diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 9069b967031..02dddf00004 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -16,10 +16,12 @@ class ParseProjectsTest(unittest.TestCase): - def test_empty_selects_every_coverage_project(self): + def test_empty_selects_every_measurable_project(self): + # Registered-but-unmeasurable projects are deliberately left out: an + # empty selection should not schedule a job that cannot report. self.assertEqual( configure_coverage_ci.parse_projects(""), - sorted(configure_coverage_ci.COVERAGE_PROJECTS), + sorted(configure_coverage_ci.SUPPORTED_PROJECTS), ) def test_whitespace_and_case_are_normalized(self): @@ -35,10 +37,10 @@ def test_unknown_project_is_rejected(self): configure_coverage_ci.parse_projects("hiprand,not-a-project") self.assertIn("not-a-project", str(context.exception)) - def test_all_alias_expands_to_every_project(self): + def test_all_alias_expands_to_every_measurable_project(self): self.assertEqual( configure_coverage_ci.parse_projects("all"), - sorted(configure_coverage_ci.COVERAGE_PROJECTS), + sorted(configure_coverage_ci.SUPPORTED_PROJECTS), ) def test_rocm_libraries_all_expands_to_that_group(self): @@ -64,10 +66,15 @@ def test_alias_and_explicit_name_overlap_is_deduped(self): ) def test_empty_group_alias_is_rejected(self): - # Update this test once a rocm-systems project is onboarded to coverage - # (the group becomes non-empty). - with self.assertRaises(ValueError): - configure_coverage_ci.parse_projects("rocm_systems_all") + # Both groups are populated, so the empty case needs an empty group to + # exist at all. Selecting nothing is a caller mistake worth reporting + # rather than silently producing no jobs. + with mock.patch.dict( + configure_coverage_ci._GROUP_ALIASES, + {"rocm_systems_all": frozenset()}, + ): + with self.assertRaises(ValueError): + configure_coverage_ci.parse_projects("rocm_systems_all") def test_unknown_alias_like_token_is_rejected(self): with self.assertRaises(ValueError): @@ -75,13 +82,13 @@ def test_unknown_alias_like_token_is_rejected(self): class SourceRepoPartitionTest(unittest.TestCase): - def test_groups_partition_all_projects(self): + def test_groups_partition_the_measurable_projects(self): libraries = set(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) systems = set(configure_coverage_ci.ROCM_SYSTEMS_PROJECTS) self.assertTrue(libraries.isdisjoint(systems)) self.assertEqual( libraries | systems, - set(configure_coverage_ci.COVERAGE_PROJECTS), + set(configure_coverage_ci.SUPPORTED_PROJECTS), ) @@ -145,20 +152,38 @@ def test_every_registered_project_declares_report_inputs(self): self.assertTrue(project.stage) self.assertTrue(project.test_component) - def test_every_registered_project_declares_hybrid_fetch_inputs(self): - # Without these the nightly test job cannot tell which files belong to - # the project, and would test an entirely non-instrumented install. - for name, project in configure_coverage_ci.COVERAGE_PROJECTS.items(): + def test_every_measurable_project_names_an_upstream_option(self): + # TheRock's _ENABLE_COVERAGE is only a knob; instrumentation + # happens when the subproject's own option is set. An entry without one + # builds and tests cleanly and then reports nothing. + for name in sorted(configure_coverage_ci.SUPPORTED_PROJECTS): + with self.subTest(project=name): + self.assertTrue( + configure_coverage_ci.COVERAGE_PROJECTS[name].coverage_option + ) + + def test_unsupported_project_is_rejected_with_the_reason(self): + with self.assertRaises(ValueError) as context: + configure_coverage_ci.parse_projects("miopen") + self.assertIn("not supported", str(context.exception)) + self.assertIn("no coverage option", str(context.exception)) + + def test_unsupported_projects_stay_out_of_the_group_aliases(self): + # They remain registered so the gap is tracked, but must not be + # scheduled by a group selection. + for name in ("miopen", "hipsparse", "rocalution", "rocprofiler-sdk"): with self.subTest(project=name): - self.assertTrue(project.artifact_names, "needs artifacts to overlay") - self.assertTrue(project.artifact_relpaths, "needs relpaths to overlay") + self.assertIn(name, configure_coverage_ci.COVERAGE_PROJECTS) + self.assertNotIn(name, configure_coverage_ci.parse_projects("all")) - def test_hybrid_fetch_inputs_reach_the_matrix(self): + def test_object_globs_reach_the_matrix(self): + # object_globs is what scopes a report to one project, so it has to + # survive into the matrix in the spelling llvm-cov is handed. (entry,) = configure_coverage_ci.build_coverage_matrix( ["hiprand"], ["gfx94X-dcgpu"], "ROCm/rocm-libraries", "main" ) - self.assertEqual(entry["artifact_names"], "rand") - self.assertEqual(entry["artifact_relpaths"], "math-libs/hipRAND/stage") + self.assertEqual(entry["object_globs"], "lib/libhiprand.so*") + self.assertEqual(entry["fetch_artifact_args"], "--rand") class BuildCoverageCmakeOptionsTest(unittest.TestCase): @@ -171,43 +196,80 @@ def test_full_selection_collapses_to_the_group_option(self): ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"], ) + def test_both_groups_collapse_to_the_combined_option(self): + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options( + sorted(configure_coverage_ci.SUPPORTED_PROJECTS) + ), + ["-DTHEROCK_COVERAGE_ALL=ON"], + ) + def test_partial_selection_names_each_project_in_upper_case(self): - # therock_subproject.cmake only forwards the upper case spelling. Today - # hipRAND is the whole rocm-libraries group, so a second project has to - # be stubbed in to reach the per-project fallback at all. - registry = dict(configure_coverage_ci.COVERAGE_PROJECTS) - registry["rocblas"] = configure_coverage_ci.CoverageProject( - cmake_target="rocBLAS", - stage="math-libs", - test_component="rocblas", - coverage_config="projects/rocblas/test_categories_coverage.yaml", + # therock_subproject.cmake only forwards the upper case spelling. + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options(["rocblas"]), + ["-DROCBLAS_ENABLE_COVERAGE=ON"], ) - with ( - mock.patch.object(configure_coverage_ci, "COVERAGE_PROJECTS", registry), - mock.patch.object( - configure_coverage_ci, - "ROCM_LIBRARIES_PROJECTS", - frozenset(registry), - ), - ): - self.assertEqual( - configure_coverage_ci.build_coverage_cmake_options(["rocblas"]), - ["-DROCBLAS_ENABLE_COVERAGE=ON"], - ) - def test_empty_group_does_not_produce_a_group_option(self): - # rocm-systems has no onboarded projects, so selecting only - # rocm-libraries must not claim to cover everything. + def test_partial_selection_also_names_extra_targets(self): + # rocPRIM reports against binaries built by rocPRIM_tests, so naming + # rocPRIM alone would instrument nothing the report reads. + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options(["rocprim"]), + ["-DROCPRIM_ENABLE_COVERAGE=ON", "-DROCPRIM_TESTS_ENABLE_COVERAGE=ON"], + ) + + def test_one_group_does_not_claim_to_cover_the_other(self): options = configure_coverage_ci.build_coverage_cmake_options( sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) ) - self.assertNotIn("-DTHEROCK_COVERAGE_ALL=ON", options) - self.assertNotIn("-DTHEROCK_COVERAGE_ROCM_SYSTEMS_ALL=ON", options) + self.assertEqual(options, ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"]) def test_no_selection_produces_no_options(self): self.assertEqual(configure_coverage_ci.build_coverage_cmake_options([]), []) +class BuildStagePlanTest(unittest.TestCase): + def test_math_libs_only_selection_needs_no_generic_stage(self): + generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["hiprand"]) + self.assertEqual(generic, []) + self.assertTrue(needs_math_libs) + + def test_compiler_runtime_project_adds_no_stage_of_its_own(self): + # compiler-runtime is built unconditionally, so a project living there + # must not also appear as a generic stage to build. + generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["amdsmi"]) + self.assertEqual(generic, []) + self.assertFalse(needs_math_libs) + + def test_generic_stage_project_is_named_for_the_build_job(self): + generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["rccl"]) + self.assertEqual( + generic, + [ + { + "stage_name": "comm-libs", + "stage_display_name": "Stage - Coverage Comm Libs", + } + ], + ) + self.assertFalse(needs_math_libs) + + def test_stages_are_deduplicated_across_projects(self): + generic, _ = configure_coverage_ci.build_stage_plan(["rccl", "rocshmem"]) + self.assertEqual([entry["stage_name"] for entry in generic], ["comm-libs"]) + + def test_selecting_everything_names_every_non_default_stage(self): + generic, needs_math_libs = configure_coverage_ci.build_stage_plan( + sorted(configure_coverage_ci.COVERAGE_PROJECTS) + ) + self.assertEqual( + [entry["stage_name"] for entry in generic], + ["comm-libs", "profiler-apps"], + ) + self.assertTrue(needs_math_libs) + + class MainTest(unittest.TestCase): def setUp(self): self._orig_env = os.environ.copy() @@ -232,12 +294,14 @@ def test_writes_expected_outputs(self): self.assertIn("dist_amdgpu_families", written) self.assertIn("families_matrix_json", written) self.assertIn("coverage_cmake_options", written) - # hipRAND is currently the whole rocm-libraries group, so selecting - # it selects the group option. - self.assertIn("-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON", written) + # One project out of a populated group names that project alone. + self.assertIn("-DHIPRAND_ENABLE_COVERAGE=ON", written) # The per-project test and overlay inputs the nightly reads back. self.assertIn('"test_component": "hiprand"', written) - self.assertIn('"artifact_relpaths": "math-libs/hipRAND/stage"', written) + self.assertIn('"object_globs": "lib/libhiprand.so*"', written) + # hipRAND is a math-libs project, so no other stage is built. + self.assertIn("needs_math_libs=true", written) + self.assertIn("generic_stages_json=[]", written) class EmitCmakeTest(unittest.TestCase): @@ -248,8 +312,48 @@ def test_emits_group_lists_from_allowlist(self): text = out.read_text() self.assertIn("set(THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS", text) self.assertIn("hipRAND", text) - # rocm-systems group is empty today -> an empty set(). - self.assertRegex(text, r"set\(THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS\s*\)") + self.assertIn("set(THEROCK_COVERAGE_ROCM_SYSTEMS_PROJECTS", text) + self.assertIn("rccl", text) + # A group flag has to reach the sibling subproject that builds a + # header-only project's tests, or the group instruments rocPRIM and + # leaves its only reportable binaries without a coverage mapping. + self.assertIn("rocPRIM_tests", text) + # Group membership follows source_repo, not the stage. + libraries_line = next( + line + for line in text.splitlines() + if line.startswith("set(THEROCK_COVERAGE_ROCM_LIBRARIES_PROJECTS") + ) + self.assertNotIn("rccl", libraries_line) + # Unmeasurable projects are excluded, or a group build would set a + # flag their CMake does not implement. + self.assertNotIn("MIOpen", text) + self.assertNotIn("hipSPARSE ", text) + + def test_emits_the_upstream_option_name_per_subproject(self): + # The generic names are why this is a per-subproject map: setting + # BUILD_CODE_COVERAGE globally would instrument everything that happens + # to understand it. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "therock_coverage_projects.cmake" + self.assertEqual(configure_coverage_ci.main(["--emit-cmake", str(out)]), 0) + text = out.read_text() + self.assertIn( + "set(THEROCK_COVERAGE_OPTION_HIPRAND BUILD_CODE_COVERAGE)", text + ) + self.assertIn("set(THEROCK_COVERAGE_OPTION_ROCRAND CODE_COVERAGE)", text) + self.assertIn( + "set(THEROCK_COVERAGE_OPTION_RCCL ENABLE_CODE_COVERAGE)", text + ) + self.assertIn( + "set(THEROCK_COVERAGE_OPTION_HIPBLASLT HIPBLASLT_ENABLE_COVERAGE)", text + ) + # A sibling subproject builds from the same source, so it takes the + # same option as its parent. + self.assertIn( + "set(THEROCK_COVERAGE_OPTION_ROCPRIM_TESTS BUILD_CODE_COVERAGE)", text + ) + self.assertNotIn("THEROCK_COVERAGE_OPTION_MIOPEN", text) def test_emit_cmake_needs_no_env(self): # Must work without PROJECTS_TO_TEST / AMDGPU_FAMILIES / GITHUB_OUTPUT set. diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index bdb5d3640f1..159977416ad 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -928,13 +928,29 @@ function(therock_cmake_subproject_activate target_name) list(APPEND _cmake_args "${${target_name}_CMAKE_ARGS}") - # Passthrough of -D_ENABLE_COVERAGE=ON to the subproject. CMake - # definitions are conventionally upper case, so only the upper case spelling - # is honored (i.e. HIPDNN_ENABLE_COVERAGE, not hipDNN_ENABLE_COVERAGE). + # Translate TheRock's -D_ENABLE_COVERAGE into whichever option the + # subproject actually implements. CMake definitions are conventionally upper + # case, so only the upper case spelling of TheRock's flag is honored (i.e. + # HIPDNN_ENABLE_COVERAGE, not hipDNN_ENABLE_COVERAGE). + # + # The names upstream are not standardised: most projects take + # BUILD_CODE_COVERAGE or CODE_COVERAGE, a few take _ENABLE_COVERAGE, + # RCCL takes ENABLE_CODE_COVERAGE. The generic ones are the reason this is + # done per subproject rather than by setting a global: defining + # BUILD_CODE_COVERAGE for the whole build would instrument every project that + # happens to understand it, which is exactly what per-project coverage is + # trying to avoid. string(TOUPPER "${_logical_target_name}" _coverage_project_name) set(_coverage_var_name "${_coverage_project_name}_ENABLE_COVERAGE") if(DEFINED ${_coverage_var_name}) - list(APPEND _cmake_args "-D${_coverage_var_name}=${${_coverage_var_name}}") + set(_coverage_option "${THEROCK_COVERAGE_OPTION_${_coverage_project_name}}") + if(NOT _coverage_option) + message(FATAL_ERROR + "${_logical_target_name}: ${_coverage_var_name} was set but the project " + "has no coverage option registered. Add one to COVERAGE_PROJECTS in " + "build_tools/github_actions/configure_coverage_ci.py.") + endif() + list(APPEND _cmake_args "-D${_coverage_option}=${${_coverage_var_name}}") endif() # Derive the CMAKE_BUILD_TYPE from either {project}_BUILD_TYPE or the global diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index e7c670b9aaf..d9a2deef595 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -125,54 +125,35 @@ report. ### Scheduling -`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. Pick a recent -nightly run, pass its id as `baseline_run_id`, and leave -`baseline_release_type` at `nightly`. There is no cron trigger, and the regular -nightly does not know about this workflow. +`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. There is no cron +trigger, and the regular nightly does not know about this workflow. Coverage therefore has a run id of its own, which is what keeps a coverage failure from colouring the nightly's status and keeps the nightly's test jobs — which rarely all pass — from holding coverage up. -Having the regular nightly dispatch this workflow automatically once its build -finishes, handing over its own run id, is a later change. Nothing else has to -move for it: `baseline_run_id` is already an input, so that change only adds the -dispatching job. +Nightly coverage runs the projects named by `projects_to_test` on one +architecture (gfx942, the `gfx94X-dcgpu` family). Running only what changed, and +running more architectures, come later. -Nightly coverage runs every onboarded project on one architecture -(gfx942, the `gfx94X-dcgpu` family). Running only what changed, and running more -architectures, come later. +#### The run is self-contained -#### Why there are two run ids +A coverage run builds every stage its selection needs, starting from +compiler-runtime, and instruments only the projects that were selected. +Everything those projects depend on is built normally in the same run, so the +run's own artifacts are already the stack to test against. The test jobs install +them the same way regular CI installs its own, and there is no second run id to +supply. -The nightly instruments the whole stack in one build, because building each -project separately would mean rebuilding its dependencies each time. Reporting, -though, has to stay per project: a coverage report for hipRAND should not shift -because rocRAND changed, and an instrumented rocRAND under an instrumented -hipRAND also writes its own profiles into the same run. +What keeps a report scoped to one project is `object_globs`, not what happens to +be instrumented. `llvm-cov export` is handed only that project's objects and +reports only functions found in their coverage mappings, so a selection covering +several projects still yields one report per project. -Each test job therefore assembles a mostly non-instrumented install: - -1. `setup_test_environment` installs the **baseline** run in full — every - library non-instrumented, as the regular nightly built it. -1. `overlay_coverage_artifacts.py` fetches the artifact holding the project - under test from the **coverage** run and copies only that project's files - over the baseline ones. - -The baseline is read from the channel that published it (`baseline_release_type`, -normally `nightly`) rather than from the coverage run's own `ci` channel, since -artifacts are bucketed per channel. - -Step 2 is per project rather than per artifact because TheRock's artifacts are -grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit under -the subproject stage directory they were built in (`math-libs/hipRAND/stage`), -which is what `artifact_relpaths` in the coverage registry names. Nothing is -renamed along the way — the two runs write to different S3 directories, so the -instrumented `rand_lib_gfx942.tar.xz` and the regular one never collide. - -If the overlay finds none of the project's files, the job fails rather than -testing the baseline build and reporting coverage for binaries that were never -instrumented. +Selecting more projects therefore costs build time rather than accuracy: each +one adds an instrumented library, and instrumented libraries are slower to build +and slower to run under test. Naming the projects you actually want is the cheap +path. ### Selecting projects to run @@ -193,11 +174,26 @@ coverage workflow, so non-coverage test selection is unaffected. ### Adding a project -Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. Before doing -so, confirm that the project implements `_ENABLE_COVERAGE` in its own -CMake, and that a local instrumented build produces a non-empty report. A -project whose tests never load the instrumented library will build and test -cleanly but report zero coverage. +Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. + +Its `coverage_option` has to name the CMake option the project actually +implements, and that option has to select LLVM source-based instrumentation +(`-fprofile-instr-generate -fcoverage-mapping`). Neither can be assumed. The +names are not standardised — most projects use `BUILD_CODE_COVERAGE`, several +use `CODE_COVERAGE`, a few use `_ENABLE_COVERAGE`, RCCL uses +`ENABLE_CODE_COVERAGE` — and some projects instrument with gcov +(`-fprofile-arcs`) instead, which writes `.gcda` files this pipeline cannot +read. `therock_subproject.cmake` translates TheRock's `_ENABLE_COVERAGE` +into whichever name you register, per subproject, so a generic name does not +leak into the rest of the build. + +A project that cannot be measured yet gets an `unsupported_reason` instead. It +stays in the registry so the gap is recorded, stays out of the group aliases, +and is rejected with that reason if someone names it. + +Confirm a local instrumented build produces a non-empty report before relying on +a new entry. A project whose tests never load the instrumented library builds +and tests cleanly and then reports nothing. Set the entry's `source_repo` (`ROCM_LIBRARIES` by default, or `ROCM_SYSTEMS`); that is what routes the project into the correct group alias and @@ -205,11 +201,11 @@ that is what routes the project into the correct group alias and lists are generated from `COVERAGE_PROJECTS` at configure time, so both the local aggregate flags and the CI aliases pick up the new project automatically. -`artifact_names` and `artifact_relpaths` are what the nightly overlay uses, so -they need to name the artifact the project ships in and the subproject stage -directory holding its files. Getting `artifact_relpaths` wrong fails the nightly -test job outright rather than quietly reporting against a non-instrumented -build. +`object_globs` names the binaries the report is generated from, relative to the +install tree. For most projects that is the shared library +(`lib/librocblas.so*`); header-only projects have none, so they name their +installed test binaries instead. Getting it wrong fails the report job, which +refuses to run when no glob matches rather than publishing zero coverage. A project whose stage is not yet built by `multi_arch_ci_coverage_nightly.yml` also needs that stage's build job added there; today it builds compiler-runtime diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md index 107105d33f3..904ff8e2cd9 100644 --- a/docs/development/code_coverage_flows.md +++ b/docs/development/code_coverage_flows.md @@ -4,23 +4,23 @@ An end-to-end walk through what happens between dispatching a coverage run and a report landing in Codecov. See [Code Coverage](code_coverage.md) for enabling coverage on a local build, the CMake options, and how to onboard a project. -The short version: a coverage run instruments the whole stack in one build, then -reconstructs per-project isolation at test time by installing a regular -nightly's non-instrumented artifacts and overlaying only the project under test. +The short version: a coverage run builds every stage its selection needs, +instruments only the selected projects, and reports on each of them separately +from that one run. ```mermaid graph TD - dispatch[Manual dispatch with baseline_run_id] --> matrix[setup_coverage_matrix] + dispatch[Manual dispatch with projects_to_test] --> matrix[setup_coverage_matrix] matrix --> compilerRuntime[Build instrumented compiler-runtime] + compilerRuntime --> generic[Build other needed stages] compilerRuntime --> mathLibs[Build instrumented math-libs] - mathLibs --> report[Per project: configure, test, report] - nightly[Regular nightly run - baseline] -.->|non-instrumented artifacts| report + generic --> report[Per project: configure, test, report] + mathLibs --> report report --> codecov[Codecov] ``` -Solid edges are job dependencies inside the coverage run. The dotted edge -crosses a run boundary: the baseline artifacts the test jobs install come from a -different workflow run entirely. +Every edge is a job dependency inside the one coverage run. Nothing crosses a +run boundary. ## Trigger point @@ -28,93 +28,69 @@ different workflow run entirely. the `gh` CLI. There is no cron trigger, and the regular nightly does not know about this workflow. -Two inputs matter for a normal run. `baseline_run_id` is the run id of a recent -regular nightly, which supplies every non-instrumented dependency; -`baseline_release_type` is the channel that run published to, normally -`nightly`. The rest can be left at their defaults: every onboarded project, the -`gfx94X-dcgpu` family, and coverage's own `standard` test type, which is -narrower than the nightly's because instrumented tests already run several times -longer than normal ones. +The input that matters is `projects_to_test`: a comma-separated list of project +names, or one of the group aliases. The rest can be left at their defaults — the +`gfx94X-dcgpu` family and coverage's own `standard` test type, which is narrower +than the nightly's because instrumented tests already run several times longer +than normal ones. -Leaving `baseline_run_id` empty is supported but means something different: the -run's own instrumented artifacts are tested as built, so a project's -instrumented dependencies are measured alongside it. +Leaving `projects_to_test` empty selects every onboarded project. That works, +but it instruments all of them in one build, which is the expensive way to run +this. Name what you want measured. -Having the regular nightly dispatch this workflow automatically once its build -finishes, passing its own run id as `baseline_run_id`, is a later change. It -only needs to add a dispatching job — everything described below already takes -the baseline as an input. +## The regular nightly -## The normal build - -Nothing about the regular nightly changes. It produces the ordinary -non-instrumented artifacts and publishes them to the nightly channel bucket -under its own run id. Coverage treats that run as a read-only input, and the -nightly has no coverage-specific jobs in it. +Unaffected, and not consulted. It has no coverage-specific jobs, and a coverage +run neither reads its artifacts nor depends on it having succeeded. ## The instrumented build -The dispatch creates a workflow run with its own id, status, and logs, entirely -separate from the baseline nightly's. +The dispatch creates a workflow run with its own id, status, and logs. `setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads -`PROJECTS_TO_TEST` (empty by default, meaning every onboarded project), -`AMDGPU_FAMILIES`, and `COVERAGE_CONFIG_SOURCE`, and emits the job matrix, the -family list, and `coverage_cmake_options`. - -`build_instrumented_compiler_runtime` runs first. Every later stage in the run -fetches its inbound artifacts by run id, so the run needs a compiler-runtime of -its own to link against. - -`build_instrumented_math_libs` then builds the whole `math-libs` stage with -`coverage_cmake_options` appended to the configure line. With hipRAND the only -onboarded project, and hipRAND being the whole rocm-libraries group, that -resolves to `-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON`; `CMakeLists.txt` expands -the group option into the individual `_ENABLE_COVERAGE` flags and -`therock_subproject.cmake` forwards them into each subproject. - -Both stages call `multi_arch_build_portable_linux_artifacts.yml`, the same -per-stage build workflow regular CI uses, rather than going through the full -`multi_arch_build_portable_linux.yml` pipeline. Naming the two stages coverage +`PROJECTS_TO_TEST`, `AMDGPU_FAMILIES` and `COVERAGE_CONFIG_SOURCE` and emits the +per-project job matrix, the family list, `coverage_cmake_options`, and the build +plan: `generic_stages_json` and `needs_math_libs`. + +`build_instrumented_compiler_runtime` runs first and unconditionally. Every other +stage takes its inbound artifacts from it — all 19 of `math-libs`' inbound +artifacts are produced there — so no selection can skip it. It also receives the +coverage flags, since several rocm-systems projects (`rocprofiler-sdk`, +`aqlprofile`, `amdsmi`, `rocprofiler-compute`) are built in that stage. + +`build_instrumented_generic_stages` then fans out over `generic_stages_json`, +the build-once stages the selection needs beyond compiler-runtime — `comm-libs` +for rccl and rocshmem, `profiler-apps` for rocprofiler-systems. It is skipped +entirely for a selection confined to math-libs. + +`build_instrumented_math_libs` fans out over the GPU families, and is skipped +when `needs_math_libs` is false. + +Every stage gets `coverage_cmake_options` appended to its configure line. +`configure_coverage_ci.py` collapses a selection covering a whole group into +that group's option, so a full run reads `-DTHEROCK_COVERAGE_ALL=ON` while a +narrow one reads `-DHIPRAND_ENABLE_COVERAGE=ON`. `CMakeLists.txt` expands the +group options into individual `_ENABLE_COVERAGE` flags. Everything not +selected builds normally. + +`_ENABLE_COVERAGE` is TheRock's own knob and is not what the subproject +sees. Upstream option names are not standardised — hipRAND takes +`BUILD_CODE_COVERAGE`, rocRAND takes `CODE_COVERAGE`, RCCL takes +`ENABLE_CODE_COVERAGE`, hipBLASLt takes `HIPBLASLT_ENABLE_COVERAGE` — so +`therock_subproject.cmake` translates TheRock's flag into the name registered +for that project. Doing it per subproject is what keeps the generic spellings +from instrumenting everything in the build that happens to understand them. + +All the build jobs call `multi_arch_build_portable_linux_artifacts.yml`, the +same per-stage build workflow regular CI uses, rather than going through the +full `multi_arch_build_portable_linux.yml` pipeline. Naming the stages coverage needs keeps the shared pipeline free of coverage-specific stage filtering. -Both also pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`: split kernel +They also pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`: split kernel packaging would move instrumented device code out of the library `llvm-cov` is later pointed at. -The instrumented artifacts publish under `release_type: ci`, so they land in the -CI bucket while the baseline sits in the nightly bucket. That is also why the -instrumented and regular copies of an artifact can share a filename without -colliding. - -## Run ids - -Two runs are involved and each names the other's artifacts differently. The -coverage run refers to its own instrumented artifacts as `github.run_id` -throughout. The baseline is not discoverable, so it is threaded through as an -input: - -1. `multi_arch_ci_coverage_nightly.yml` takes `baseline_run_id` and - `baseline_release_type` as `workflow_dispatch` inputs and forwards them to - `multi_arch_ci_coverage_linux.yml`. -1. That forwards both again to `test_component.yml` as - `coverage_baseline_run_id` and `coverage_baseline_release_type`. -1. `test_component.yml` derives `INSTALL_ARTIFACT_RUN_ID` and - `INSTALL_RELEASE_TYPE` from them, which is what `setup_test_environment` - installs from. - -Because the two runs are connected only by that first input, and nothing in the -coverage run can look the baseline up, the workflow puts it in the `run-name` so -it is visible in the Actions list without opening the logs. - -The release channel travels with the run id rather than being assumed, because -artifacts are bucketed per channel: reading the baseline under the coverage -run's own `ci` channel would look in the wrong bucket entirely. Callers that -leave `coverage_baseline_run_id` empty get exactly the previous behaviour. - -`quartz_tracking_id` follows the same path. It stays empty on a manual dispatch -and exists so a coverage run can be attached to a release lineage once the -nightly dispatches coverage automatically. +The artifacts publish under `release_type: ci`, under this run's id. ## Test execution @@ -126,14 +102,11 @@ regular CI — narrowed to the one project this report covers, and outputs the shard list. `test_coverage` then runs one `test_component.yml` job per shard with -`coverage_enabled: true`. Four things happen in order: - -1. **Install the baseline.** `setup_test_environment` runs against the baseline - run id and channel, filling the install tree with a complete, - non-instrumented ROCm. -1. **Overlay one project.** `overlay_coverage_artifacts.py` fetches the - instrumented artifact from the coverage run and copies only the project's - own subtree over the baseline copy. Details below. +`coverage_enabled: true`. Three things happen in order: + +1. **Install this run's artifacts.** `setup_test_environment` runs against + `github.run_id` and the `ci` channel, exactly as it does for regular CI. No + coverage-specific artifact handling is involved. 1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to a per-shard path using the `%p` (pid) and `%m` (binary signature) substitutions, so a shard that forks or loads several instrumented libraries @@ -142,35 +115,26 @@ shard list. files upload as a workflow artifact under `always()` — a failing shard still exercised code. -### Why the overlay exists - -A test job wants exactly one instrumented project and non-instrumented -everything else. An instrumented dependency would emit its own profiles and move -the coverage denominator around whenever that dependency changed. +### What scopes a report to one project -Building each project separately would give that for free, but would also mean -rebuilding every dependency once per project. The nightly instead instruments -the whole stack in one build and separates the projects afterwards. +`object_globs`, not what happens to be instrumented. -The overlay is per subproject directory rather than per artifact because -TheRock's artifacts are grouped: `rand` carries both rocRAND and hipRAND, so -overlaying a whole artifact would instrument the sibling too. Each project's -files sit under the subproject stage directory they were built in -(`math-libs/hipRAND/stage`), which is what `artifact_relpaths` in the coverage -registry names. Symlinks are preserved so the `libfoo.so -> libfoo.so.1` chain -the loader follows still resolves to the instrumented file. +`llvm-cov export` is handed an explicit `-object` for each of the project's +binaries and reports only the functions found in their coverage mappings. +Counters that a sibling project wrote into the same merged profdata are never +looked up. A selection naming several projects therefore still produces one +report per project, from a single build and a single run. -If none of the relpaths match, the script fails the job. Silently testing the -baseline would report coverage against binaries that were never instrumented. +The cost of selecting more projects is build and test time — each instrumented +library is slower to compile and slower to exercise — rather than accuracy. ## Artifacts -Three distinct kinds are in play: +Two kinds are in play: | Artifact | Where it lives | What it is for | | ------------ | ----------------------------------- | ---------------------------------------------- | -| Baseline | nightly bucket, baseline run id | Every non-instrumented dependency | -| Instrumented | CI bucket, coverage run id | The project under test, and the LLVM tools | +| Instrumented | CI bucket, this run's id | The stack under test, and the LLVM tools | | Profraw | GitHub Actions artifacts, per shard | Raw profiles, collected by the aggregation job | Profraw artifact names carry the project, GPU family, and shard index, so the @@ -182,20 +146,21 @@ being measured in one run. `aggregate_coverage` runs once per project, under `if: !cancelled()` so a partial report still gets produced when some shards failed. -It downloads every matching profraw artifact, then installs the **instrumented** -artifacts from the coverage run — not the baseline. `llvm-cov` needs the -binaries carrying the coverage mapping sections, and `llvm-profdata` and -`llvm-cov` themselves have to come from the same compiler that produced the -profiles. Both live under `lib/llvm/bin` of the installed distribution; a -version mismatch surfaces as an unhelpfully generic "malformed instrumentation -profile data" error. +It downloads every matching profraw artifact, then installs the run's artifacts +again. `llvm-cov` needs the binaries carrying the coverage mapping sections, and +`llvm-profdata` and `llvm-cov` themselves have to come from the same compiler +that produced the profiles. Both live under `lib/llvm/bin` of the installed +distribution; a version mismatch surfaces as an unhelpfully generic "malformed +instrumentation profile data" error. `merge_coverage_report.py` then collects the profiles recursively, expands the project's `object_globs` and deduplicates them by real path (a versioned symlink family resolves to one file), runs `llvm-profdata merge -sparse` into a single -profdata index, and exports lcov. Finding no profraw files at all is fatal: it -means either the tests never ran, or the instrumented libraries were not the -ones loaded at runtime. +profdata index, and exports lcov. Two conditions are fatal rather than reported +as zero coverage: finding no profraw files at all, which means the tests never +ran or the instrumented libraries were not the ones loaded at runtime; and +matching no objects, which means `object_globs` does not describe what the +project actually installs. The lcov report uploads as a workflow artifact and goes to Codecov under the project's flag. The Codecov step is skipped rather than failed when no From fbadbae3851045870a30e2b89201c754d9f3e2db Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 06:57:04 -0500 Subject: [PATCH 12/70] feat(ci): run nightly coverage against a real nightly baseline Phase 1 instruments the whole stack in one build but has to report per project, so a test job installs a non-instrumented baseline run and overlays only the project under test. That path was removed in 0585dc3f on the argument that object_globs already scope the report. They scope the export, but not what gets measured: inline and template code from headers is emitted linkonce_odr into every binary that uses it, so an instrumented sibling writes counters that llvm-profdata merges into this project's and no -object choice separates again. It is worst for the header-only projects (rocPRIM, hipCUB, rocThrust, rocWMMA), where nearly all the measured code lives in headers. This reverts that commit and restores baseline_run_id as the RFC specifies. The overlay crashed in run 34098007786 because shutil.copytree recreates symlinks with os.symlink(), which raises once the baseline install has already placed the libfoo.so -> libfoo.so.1 chain -- dirs_exist_ok forgives directory collisions, not symlink ones. Replaced with a merge_tree() that unlinks what it is about to replace, covered by two regression tests. The PR check now pins ROCm/rockrel 34068476531, a real Multi-Arch Release (nightly), and selects every project rather than one. Unlike PR #7944, which copies baseline stages into its own run and builds on top of them, nothing here configures CMake against the baseline, so the embedded /__w/rockrel/rockrel paths that rule out release nightlies there do not matter: the baseline is only installed on the test runner, where prebuilt binaries run and no package config is read. Selecting everything exposed two gaps a single-project run could not: - comm-libs takes hipify and rocjitsu from emulation, which the coverage workflow never built, so selecting rccl or rocshmem would have failed on a missing inbound artifact. build_stage_plan now derives the required stages transitively from the build topology and orders them into waves, and the workflow builds one wave per job. - The artifacts bucket lookup resolves the run through the GitHub API against this repository, which 404s for a rockrel id. Plumbed a RUN_GITHUB_REPO passthrough down to install_rocm_from_artifacts.py. Also added a test asserting every registry entry names the stage the topology says builds its artifacts. The registry duplicates that fact with nothing keeping the two in step, and a project moving upstream would otherwise surface as a stage never built and an overlay that finds nothing, hours into a run. Co-authored-by: Cursor --- .../actions/setup_test_environment/action.yml | 9 + .../workflows/coverage_nightly_pr_check.yml | 42 ++- .../multi_arch_ci_coverage_linux.yml | 48 +++- .../multi_arch_ci_coverage_nightly.yml | 140 ++++++--- .github/workflows/test_component.yml | 63 +++- build_tools/artifact_manager.py | 34 ++- .../github_actions/configure_coverage_ci.py | 176 ++++++++++-- .../overlay_coverage_artifacts.py | 252 ++++++++++++++++ .../tests/configure_coverage_ci_test.py | 72 +++-- .../tests/merge_coverage_report_test.py | 4 +- .../tests/overlay_coverage_artifacts_test.py | 270 ++++++++++++++++++ .../tests/artifact_manager_tool_test.py | 62 ++++ docs/development/code_coverage.md | 67 +++-- docs/development/code_coverage_flows.md | 30 +- 14 files changed, 1137 insertions(+), 132 deletions(-) create mode 100644 build_tools/github_actions/overlay_coverage_artifacts.py create mode 100644 build_tools/github_actions/tests/overlay_coverage_artifacts_test.py diff --git a/.github/actions/setup_test_environment/action.yml b/.github/actions/setup_test_environment/action.yml index b28e5b29bac..5e694640428 100644 --- a/.github/actions/setup_test_environment/action.yml +++ b/.github/actions/setup_test_environment/action.yml @@ -21,6 +21,13 @@ inputs: ARTIFACT_RUN_ID: description: (string) Run ID to download artifacts from (`env.ARTIFACT_RUN_ID` or a value like `19716407405`) required: true + RUN_GITHUB_REPO: + description: >- + (string) Repository owning ARTIFACT_RUN_ID, in 'owner/repo' form. Only + needed when the run belongs to another repository, such as a rockrel + release nightly; the bucket lookup queries the run through the API and + fails if it looks in the wrong place. Defaults to this repository. + default: "" FETCH_ARTIFACT_ARGS: description: (string) Extra arguments to install_rocm_from_artifacts.py (e.g. `--blas --tests`) RELEASE_TYPE: @@ -68,6 +75,7 @@ runs: ARTIFACT_GROUP: ${{ inputs.ARTIFACT_GROUP }} AMDGPU_TARGETS: ${{ inputs.AMDGPU_TARGETS }} ARTIFACT_RUN_ID: ${{ inputs.ARTIFACT_RUN_ID }} + RUN_GITHUB_REPO: ${{ inputs.RUN_GITHUB_REPO }} FETCH_ARTIFACT_ARGS: ${{ inputs.FETCH_ARTIFACT_ARGS }} RELEASE_TYPE: ${{ inputs.RELEASE_TYPE }} GITHUB_TOKEN: ${{ github.token }} @@ -76,6 +84,7 @@ runs: run: | python ./build_tools/install_rocm_from_artifacts.py \ --run-id=${ARTIFACT_RUN_ID} \ + ${RUN_GITHUB_REPO:+--run-github-repo=${RUN_GITHUB_REPO}} \ --artifact-group=${ARTIFACT_GROUP} \ --amdgpu-targets=${AMDGPU_TARGETS} \ --output-dir=${OUTPUT_ARTIFACTS_DIR} \ diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index f9b0e45ffcf..2aa34ccd671 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -32,6 +32,7 @@ on: - ".github/workflows/multi_arch_ci_coverage_linux.yml" - ".github/workflows/test_component.yml" - "build_tools/github_actions/configure_coverage_ci.py" + - "build_tools/github_actions/overlay_coverage_artifacts.py" - "build_tools/github_actions/merge_coverage_report.py" - "cmake/therock_subproject.cmake" - "CMakeLists.txt" @@ -50,17 +51,44 @@ jobs: secrets: inherit # A called workflow cannot hold more than its caller, so these are granted # here even though the coverage workflow declares them itself: id-token for - # the AWS role the artifact fetches assume, actions:read for the profraw - # artifact downloads. + # the AWS role the artifact fetches assume, actions:read for the baseline + # run lookup. permissions: contents: read actions: read id-token: write with: - # One project, so the build instruments hipRAND and nothing else and the - # run's own artifacts are already an isolated stack. Naming a wider - # selection here would work, but each addition is another instrumented - # library in a build that is already the expensive part of the run. - projects_to_test: "hiprand" + # Pinned rather than resolved, so that a change in the report means a + # change in this branch. + # + # ROCm/rockrel run 34068476531, "Multi-Arch Release (nightly)" on main, + # 2026-09-07, succeeded. A real nightly rather than a CI run, because + # that is what a dispatched coverage run will be given in practice and + # the phase 1 PoC is worth exercising against the real thing. + # + # Being a rockrel run is safe here. Its artifacts embed the workspace + # they were built in (/__w/rockrel/rockrel), which breaks any consumer + # that configures CMake against them, but this workflow never does: it + # builds the instrumented stack from source in its own workspace and + # only installs the baseline on the test runner, where prebuilt binaries + # are executed and no package config is read. + # + # Verified before pinning that this run publishes gfx942 artifacts for + # every selected project -- blas, fft, hipdnn, hiptensor, prim, rand, + # rccl, rocshmem, rocwmma, solver and sparse, plus the _test archives the + # header-only projects report against. + baseline_run_id: "34068476531" + # The bucket lookup queries the run through the GitHub API, so it has to + # be told where the run lives; ROCm/TheRock returns 404 for a rockrel id. + baseline_repository: "ROCm/rockrel" + # That run published to therock-nightly-artifacts, which is the default, + # but naming it keeps the pin and its channel in one place. + baseline_release_type: "nightly" amdgpu_families: "gfx94X-dcgpu" + # The whole registry, which is what phase 1 specifies: instrument the + # stack once and report every onboarded project. Expensive on purpose -- + # this is the case a single-project run cannot demonstrate, since the + # overlay only has to hold sibling instrumentation out of the install + # when there are instrumented siblings to hold out. + projects_to_test: "all" test_type: "standard" diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 5236ed03b0b..e4ab8b505d9 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -9,14 +9,14 @@ # test (collect profraw) -> merge -> report # # The instrumented stack itself is built by the caller -# (multi_arch_ci_coverage_nightly.yml). This workflow only consumes it, and it -# consumes it from that one run: the coverage run builds every stage it needs, -# instrumenting only the projects the selection named, so its own artifacts are -# already the stack to test against. +# (multi_arch_ci_coverage_nightly.yml), which instruments every onboarded +# project in one pass. This workflow only consumes it. # -# The report is scoped by `object_globs` rather than by what is instrumented. -# llvm-cov only reports functions found in the objects it is given, so a -# selection covering several projects still yields one report per project. +# `baseline_run_id` drives the hybrid install: the tree comes from a regular +# nightly run and only the project under test is overlaid from the instrumented +# run, keeping instrumented dependencies out of the report. Without it the +# instrumented run's own artifacts are tested as built, which measures the +# project's dependencies alongside it. name: Multi-Arch CI Coverage - Linux @@ -36,6 +36,35 @@ on: type: string default: "" description: "Extra install_rocm_from_artifacts.py args for the report job" + baseline_run_id: + type: string + default: "" + description: >- + Regular nightly run to take non-instrumented dependencies from. Set + by the nightly caller, which instruments the whole stack in its own + run; the test jobs then install that run and overlay only the project + under test. Empty tests the instrumented artifacts as built. + baseline_release_type: + type: string + default: "nightly" + description: >- + Release channel baseline_run_id published to. Artifacts are bucketed + by channel, so the baseline is read from its own channel rather than + from this run's. + baseline_repository: + type: string + default: "" + description: >- + Repository owning baseline_run_id, in 'owner/repo' form. Needed when + the baseline is a release nightly, which ROCm/rockrel publishes. + artifact_names: + type: string + default: "" + description: "Artifact names holding the instrumented project, for the hybrid fetch" + artifact_relpaths: + type: string + default: "" + description: "Subproject stage directories to overlay, for the hybrid fetch" codecov_flag: type: string default: "" @@ -139,6 +168,11 @@ jobs: build_variant: coverage coverage_enabled: true coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + coverage_baseline_run_id: ${{ inputs.baseline_run_id }} + coverage_baseline_release_type: ${{ inputs.baseline_release_type }} + coverage_baseline_repository: ${{ inputs.baseline_repository }} + coverage_artifact_names: ${{ inputs.artifact_names }} + coverage_artifact_relpaths: ${{ inputs.artifact_relpaths }} quartz_tracking_id: ${{ inputs.quartz_tracking_id }} aggregate_coverage: diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 9ae7859cde3..ae06c7418a8 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -16,22 +16,34 @@ # (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and # multi-architecture expansion come later. # -# The run is self-contained. It builds every stage it needs and instruments only -# the projects named by `projects_to_test`, so everything those projects depend -# on is built normally in the same run and its artifacts are already the stack -# to test against. There is no second run id to coordinate and no baseline to -# mix in. -# -# Scheduling is manual for now: dispatch this workflow by hand. There is no cron +# Scheduling is the RFC's option B1 for now: dispatch this workflow by hand and +# pass the run id of a recent nightly as `baseline_run_id`. There is no cron # here, and the regular nightly does not know about this workflow, so coverage # has a run id of its own -- a failure here leaves the nightly's status alone, # and the nightly's unstable test jobs do not hold coverage up. +# +# The RFC's option C, where the regular nightly dispatches this workflow once +# its build finishes and hands over its own run id automatically, is a later +# change. Everything below already takes `baseline_run_id` as an input, so that +# change only has to add the dispatching job. name: Multi-Arch CI Coverage Nightly on: workflow_dispatch: inputs: + baseline_run_id: + type: string + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: + type: string + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" + baseline_repository: + type: string + description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." + default: "" projects_to_test: type: string description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." @@ -63,6 +75,18 @@ on: # Remove this trigger with the check workflow once the run has gone green. workflow_call: inputs: + baseline_run_id: + type: string + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: + type: string + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" + baseline_repository: + type: string + description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." + default: "" projects_to_test: type: string description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." @@ -89,11 +113,11 @@ on: permissions: contents: read -# Which projects a run measured is the thing people look for afterwards, so put -# it in the title rather than making them open the setup job's log. +# The two run ids are only connected through this input, so surface it in the +# run title rather than making people open the logs to find the baseline. run-name: >- Multi-Arch CI Coverage Nightly - | ${{ inputs.projects_to_test || 'all onboarded projects' }} + | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} concurrency: @@ -109,7 +133,8 @@ jobs: dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} - generic_stages_json: ${{ steps.matrix.outputs.generic_stages_json }} + stage_wave1_json: ${{ steps.matrix.outputs.stage_wave1_json }} + stage_wave2_json: ${{ steps.matrix.outputs.stage_wave2_json }} needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} steps: @@ -138,10 +163,8 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - # Built first and unconditionally: every other stage takes its inbound - # artifacts from this one, so no selection can skip it. It also receives the - # coverage flags, because rocm-systems projects such as rocprofiler-sdk and - # amdsmi are built here rather than in a stage of their own. + # Built first so that every later stage in this run has a compiler-runtime to + # link against and to fetch as an inbound artifact. build_instrumented_compiler_runtime: name: Build Instrumented Compiler Runtime needs: setup_coverage_matrix @@ -153,6 +176,41 @@ jobs: stage_display_name: "Stage - Coverage Compiler Runtime" timeout_minutes: 480 # 8 hours (compiler is big) dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} + release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + permissions: + contents: read + id-token: write + + # The built-once stages the selection reaches into, in dependency order. + # Selecting rccl or rocshmem asks for comm-libs, which takes hipify and + # rocjitsu from emulation, so emulation builds first and comm-libs second. + # configure_coverage_ci.py derives both waves from the build topology and + # fails if a selection ever needs a third. + # + # A selection confined to math-libs leaves both waves empty, which has to be + # guarded: fromJSON('[]') produces a job with no combinations. + build_instrumented_stages_wave1: + name: Build Instrumented Stages (wave 1) + needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + if: >- + ${{ + !failure() && !cancelled() && + needs.setup_coverage_matrix.outputs.stage_wave1_json != '[]' + }} + strategy: + fail-fast: false + matrix: + stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.stage_wave1_json) }} + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml + secrets: inherit + with: + stage_name: ${{ matrix.stage_info.stage_name }} + stage_display_name: ${{ matrix.stage_info.stage_display_name }} + timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} extra_cmake_options: >- ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF @@ -163,21 +221,23 @@ jobs: contents: read id-token: write - # The build-once stages the selection needs, other than compiler-runtime. - # Empty for a selection confined to math-libs, which is why the matrix is - # guarded: fromJSON('[]') produces a job with no combinations. - build_instrumented_generic_stages: - name: Build Instrumented Stages - needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + build_instrumented_stages_wave2: + name: Build Instrumented Stages (wave 2) + needs: + [ + setup_coverage_matrix, + build_instrumented_compiler_runtime, + build_instrumented_stages_wave1, + ] if: >- ${{ !failure() && !cancelled() && - needs.setup_coverage_matrix.outputs.generic_stages_json != '[]' + needs.setup_coverage_matrix.outputs.stage_wave2_json != '[]' }} strategy: fail-fast: false matrix: - stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.generic_stages_json) }} + stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.stage_wave2_json) }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -196,10 +256,11 @@ jobs: id-token: write # The whole stage is built, not just the onboarded projects: the test jobs - # below need the complete dependency set. `coverage_cmake_options` is the - # THEROCK_COVERAGE_*_ALL group option when the selection covers a whole group, - # and the individual project flags otherwise; everything not selected builds - # normally. + # below need the complete dependency set, and phase 1 instruments every + # onboarded project in one pass rather than per changed project. + # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the + # selection covers a whole group, and the individual project flags otherwise; + # everything not selected builds normally. build_instrumented_math_libs: name: Build Instrumented Math Libs needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] @@ -230,16 +291,19 @@ jobs: contents: read id-token: write - # One test + report job per project, all against the stack built above. A - # selection naming several projects instruments all of them in that one build, - # and each job still reports on its own: llvm-cov is given only that project's - # objects, so counters from the others are never looked up. + # One test + report job per project. The instrumented stack built above is the + # only thing taken from this run: each job installs `baseline_run_id` and then + # overlays its own project, so a project's dependencies stay non-instrumented + # even though the build instrumented all of them at once. coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) needs: - - setup_coverage_matrix - - build_instrumented_generic_stages - - build_instrumented_math_libs + [ + setup_coverage_matrix, + build_instrumented_stages_wave1, + build_instrumented_stages_wave2, + build_instrumented_math_libs, + ] # `!failure() && !cancelled()` alone would let this run when # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so # the matrix has to be checked explicitly. @@ -262,6 +326,11 @@ jobs: fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} codecov_flag: ${{ matrix.coverage.codecov_flag }} amdgpu_families: ${{ matrix.coverage.amdgpu_families }} + baseline_run_id: ${{ inputs.baseline_run_id }} + baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} + baseline_repository: ${{ inputs.baseline_repository }} + artifact_names: ${{ matrix.coverage.artifact_names }} + artifact_relpaths: ${{ matrix.coverage.artifact_relpaths }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} @@ -275,7 +344,8 @@ jobs: needs: - setup_coverage_matrix - build_instrumented_compiler_runtime - - build_instrumented_generic_stages + - build_instrumented_stages_wave1 + - build_instrumented_stages_wave2 - build_instrumented_math_libs - coverage_report runs-on: ubuntu-24.04 diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index c231cf0f84d..8b12370f003 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -49,6 +49,42 @@ on: something project-specific to keep each project's profiles apart. type: string default: "coverage-profraw" + coverage_baseline_run_id: + description: >- + Regular nightly run holding the non-instrumented artifacts. When set, + the whole install tree comes from that run and only the project under + test is overlaid from this run's instrumented build, so its + dependencies stay out of the coverage report. Empty means the run's + own artifacts are already the ones to test. + type: string + default: "" + coverage_baseline_release_type: + description: >- + Release channel the baseline run published to. Artifacts are bucketed + by channel, so this has to match the run that produced them rather + than the coverage run reading them. + type: string + default: "nightly" + coverage_baseline_repository: + description: >- + Repository owning coverage_baseline_run_id, in 'owner/repo' form. + Needed when the baseline is a release nightly, which is published by + ROCm/rockrel rather than this repository. Empty uses this repository. + type: string + default: "" + coverage_artifact_names: + description: >- + Comma-separated artifact names holding the instrumented project + (e.g. 'rand'). Required alongside coverage_baseline_run_id. + type: string + default: "" + coverage_artifact_relpaths: + description: >- + Comma-separated subproject stage directories to overlay from those + artifacts (e.g. 'math-libs/hipRAND/stage'). TheRock's artifacts are + grouped, so this is what keeps sibling projects non-instrumented. + type: string + default: "" default_container_image: type: string default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" @@ -98,6 +134,12 @@ jobs: QUARTZ_WORKFLOW_POSTFIX: "- test ROCm component" VENV_DIR: ${{ github.workspace }}/.venv ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + # Nightly coverage installs the baseline run and overlays the + # instrumented project afterwards; every other run installs its own + # artifacts directly. The channel travels with the run id because + # artifacts are bucketed by channel. + INSTALL_ARTIFACT_RUN_ID: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_run_id || (inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id) }}" + INSTALL_RELEASE_TYPE: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_release_type || inputs.release_type }}" OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -153,14 +195,33 @@ jobs: STEP_TIMEOUT_MINUTES: "15" # should match timeout-minutes specified above at this step uses: './.github/actions/setup_test_environment' with: - ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }} + ARTIFACT_RUN_ID: ${{ env.INSTALL_ARTIFACT_RUN_ID }} + RUN_GITHUB_REPO: ${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_repository || '' }} # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved ARTIFACT_GROUP: ${{ inputs.amdgpu_families }} AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }} VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} + RELEASE_TYPE: ${{ env.INSTALL_RELEASE_TYPE }} + + # The step above installed the baseline run, so every library here is + # non-instrumented. Replacing just this project's files leaves its + # dependencies out of the profiles the test run is about to write. + - name: Overlay instrumented coverage artifacts + if: ${{ inputs.coverage_baseline_run_id != '' }} + timeout-minutes: 15 + env: + GITHUB_TOKEN: ${{ github.token }} RELEASE_TYPE: ${{ inputs.release_type }} + run: | + python ./build_tools/github_actions/overlay_coverage_artifacts.py \ + --run-id="${{ github.run_id }}" \ + --amdgpu-families="${{ inputs.amdgpu_families }}" \ + --amdgpu-targets="${{ inputs.amdgpu_targets }}" \ + --artifact-names="${{ inputs.coverage_artifact_names }}" \ + --artifact-relpaths="${{ inputs.coverage_artifact_relpaths }}" \ + --install-dir="${{ env.OUTPUT_ARTIFACTS_DIR }}" # safe.directory must be set before Runner Health Status - name: Adjust git config diff --git a/build_tools/artifact_manager.py b/build_tools/artifact_manager.py index ddb5d23f0c1..121482653a7 100644 --- a/build_tools/artifact_manager.py +++ b/build_tools/artifact_manager.py @@ -236,11 +236,12 @@ def parse_excluded_components(raw_components: str) -> set[str]: return components -def parse_excluded_artifacts( +def parse_artifact_names( raw_artifacts: str, valid_artifacts: set[str], + option_name: str, ) -> set[str]: - """Parse and validate artifact names passed to --exclude-artifacts.""" + """Parse and validate a comma- or semicolon-separated artifact name list.""" artifacts = { artifact.strip() for artifact in raw_artifacts.replace(";", ",").split(",") @@ -249,7 +250,7 @@ def parse_excluded_artifacts( invalid_artifacts = artifacts - valid_artifacts if invalid_artifacts: raise ValueError( - "Invalid artifact name(s): " + f"Invalid artifact name(s) for {option_name}: " f"{', '.join(sorted(invalid_artifacts))}. " f"Valid artifacts are: {', '.join(sorted(valid_artifacts))}" ) @@ -469,9 +470,23 @@ def do_fetch(args: argparse.Namespace): ) target_families = parse_target_families(args) - excluded_artifacts = parse_excluded_artifacts( - args.exclude_artifacts, - set(topology.artifacts.keys()), + valid_artifacts = set(topology.artifacts.keys()) + + # An include list narrows the stage's inbound set; hybrid fetches use it to + # pull one instrumented artifact from a coverage run and everything else + # from a regular run. + included_artifacts = parse_artifact_names( + args.artifact_names, valid_artifacts, "--artifact-names" + ) + if included_artifacts: + log(f"Limiting to artifacts: {', '.join(sorted(included_artifacts))}") + inbound &= included_artifacts + if not inbound: + log("No requested artifacts are inbound for this stage") + return + + excluded_artifacts = parse_artifact_names( + args.exclude_artifacts, valid_artifacts, "--exclude-artifacts" ) if excluded_artifacts: log(f"Excluding artifacts: {', '.join(sorted(excluded_artifacts))}") @@ -1341,6 +1356,13 @@ def main(argv: Optional[List[str]] = None): help="Comma- or semicolon-separated artifact components to exclude " f"when fetching. Valid components: {', '.join(ARTIFACT_COMPONENTS)}", ) + fetch_parser.add_argument( + "--artifact-names", + type=str, + default="", + help="Comma- or semicolon-separated artifact names to fetch. Empty " + "(the default) fetches every artifact inbound to the stage.", + ) fetch_parser.add_argument( "--exclude-artifacts", type=str, diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 5f93a8ddd75..3a1c306c7e9 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -34,8 +34,10 @@ from typing import Optional sys.path.insert(0, os.fspath(Path(__file__).resolve().parent)) +sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) from github_actions_api import gha_set_output +from _therock_utils.build_topology import get_topology DEFAULT_AMDGPU_FAMILIES = "gfx94X-dcgpu" DEFAULT_COVERAGE_CONFIG_SOURCE = "ROCm/rocm-libraries@main" @@ -53,16 +55,16 @@ STAGE_COMM_LIBS = "comm-libs" STAGE_PROFILER_APPS = "profiler-apps" -# Only math-libs is built per GPU family; the others are built once. -PER_ARCH_STAGES = frozenset({STAGE_MATH_LIBS}) - # Display names for the per-stage build jobs, so the Actions UI reads the same -# way as the regular nightly's. +# way as the regular nightly's. Stages a project can be registered against are +# validated against these keys; emulation is here because build_stage_plan can +# pull it in as a dependency of comm-libs, not because a project lives there. STAGE_DISPLAY_NAMES: dict[str, str] = { STAGE_COMPILER_RUNTIME: "Coverage Compiler Runtime", STAGE_MATH_LIBS: "Coverage Math Libs", STAGE_COMM_LIBS: "Coverage Comm Libs", STAGE_PROFILER_APPS: "Coverage Profiler Apps", + "emulation": "Coverage Emulation", } @@ -85,6 +87,13 @@ class CoverageProject: cannot be measured by this pipeline. Such a project stays in the registry so the gap is recorded, but is left out of the group aliases and rejected if named explicitly. + artifact_names: BUILD_TOPOLOGY artifact(s) the instrumented project + ships in. These are grouped (`rand` holds both rocRAND and + hipRAND), which is why artifact_relpaths exists. + artifact_relpaths: Subproject stage directories inside those artifacts + that belong to this project alone. The nightly hybrid fetch copies + only these over the baseline install tree, so a run measuring + hipRAND does not also pick up an instrumented rocRAND. stage: Build stage that produces the project. The nightly needs a build job for this stage, so onboarding a project from a new stage means adding one. @@ -111,6 +120,8 @@ class CoverageProject: coverage_config: str coverage_option: str = "" unsupported_reason: str = "" + artifact_names: list[str] = field(default_factory=list) + artifact_relpaths: list[str] = field(default_factory=list) object_globs: list[str] = field(default_factory=list) fetch_artifact_args: str = "" codecov_flag: str = "" @@ -140,6 +151,8 @@ class CoverageProject: # "rocrand": CoverageProject( cmake_target="rocRAND", + artifact_names=["rand"], + artifact_relpaths=["math-libs/rocRAND/stage"], coverage_option="CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocrand", @@ -150,6 +163,8 @@ class CoverageProject: ), "hiprand": CoverageProject( cmake_target="hipRAND", + artifact_names=["rand"], + artifact_relpaths=["math-libs/hipRAND/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hiprand", @@ -160,6 +175,8 @@ class CoverageProject: ), "rocfft": CoverageProject( cmake_target="rocFFT", + artifact_names=["fft"], + artifact_relpaths=["math-libs/rocFFT/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocfft", @@ -170,6 +187,8 @@ class CoverageProject: ), "hipfft": CoverageProject( cmake_target="hipFFT", + artifact_names=["fft"], + artifact_relpaths=["math-libs/hipFFT/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipfft", @@ -180,6 +199,8 @@ class CoverageProject: ), "rocblas": CoverageProject( cmake_target="rocBLAS", + artifact_names=["blas"], + artifact_relpaths=["math-libs/BLAS/rocBLAS/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocblas", @@ -190,6 +211,8 @@ class CoverageProject: ), "hipblas": CoverageProject( cmake_target="hipBLAS", + artifact_names=["blas"], + artifact_relpaths=["math-libs/BLAS/hipBLAS/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipblas", @@ -200,6 +223,8 @@ class CoverageProject: ), "hipblaslt": CoverageProject( cmake_target="hipBLASLt", + artifact_names=["blas"], + artifact_relpaths=["math-libs/BLAS/hipBLASLt/stage"], coverage_option="HIPBLASLT_ENABLE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipblaslt", @@ -210,6 +235,8 @@ class CoverageProject: ), "rocsparse": CoverageProject( cmake_target="rocSPARSE", + artifact_names=["sparse"], + artifact_relpaths=["math-libs/BLAS/rocSPARSE/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocsparse", @@ -220,6 +247,8 @@ class CoverageProject: ), "hipsparse": CoverageProject( cmake_target="hipSPARSE", + artifact_names=["sparse"], + artifact_relpaths=["math-libs/BLAS/hipSPARSE/stage"], unsupported_reason=( "HIPSPARSE_ENABLE_COVERAGE selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" ), @@ -232,6 +261,8 @@ class CoverageProject: ), "hipsparselt": CoverageProject( cmake_target="hipSPARSELt", + artifact_names=["sparse"], + artifact_relpaths=["math-libs/BLAS/hipSPARSELt/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipsparselt", @@ -242,6 +273,8 @@ class CoverageProject: ), "rocsolver": CoverageProject( cmake_target="rocSOLVER", + artifact_names=["solver"], + artifact_relpaths=["math-libs/BLAS/rocSOLVER/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocsolver", @@ -252,6 +285,8 @@ class CoverageProject: ), "hipsolver": CoverageProject( cmake_target="hipSOLVER", + artifact_names=["solver"], + artifact_relpaths=["math-libs/BLAS/hipSOLVER/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipsolver", @@ -262,6 +297,8 @@ class CoverageProject: ), "rocalution": CoverageProject( cmake_target="rocALUTION", + artifact_names=["rocalution"], + artifact_relpaths=["math-libs/rocALUTION/stage"], unsupported_reason=( "BUILD_CODE_COVERAGE selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" ), @@ -274,6 +311,8 @@ class CoverageProject: ), "hiptensor": CoverageProject( cmake_target="hipTensor", + artifact_names=["hiptensor"], + artifact_relpaths=["math-libs/hipTensor/stage"], coverage_option="CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hiptensor", @@ -284,6 +323,8 @@ class CoverageProject: ), "miopen": CoverageProject( cmake_target="MIOpen", + artifact_names=["miopen"], + artifact_relpaths=["ml-libs/MIOpen/stage"], unsupported_reason="no coverage option in its CMake", stage=STAGE_MATH_LIBS, test_component="miopen", @@ -294,6 +335,8 @@ class CoverageProject: ), "hipdnn": CoverageProject( cmake_target="hipDNN", + artifact_names=["hipdnn"], + artifact_relpaths=["ml-libs/hipDNN/stage"], coverage_option="HIPDNN_ENABLE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipdnn", @@ -307,6 +350,8 @@ class CoverageProject: # "rocprim": CoverageProject( cmake_target="rocPRIM", + artifact_names=["prim"], + artifact_relpaths=["math-libs/rocPRIM/stage", "math-libs/rocPRIM_tests/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocprim", @@ -320,6 +365,8 @@ class CoverageProject: ), "hipcub": CoverageProject( cmake_target="hipCUB", + artifact_names=["prim"], + artifact_relpaths=["math-libs/hipCUB/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hipcub", @@ -330,6 +377,8 @@ class CoverageProject: ), "rocthrust": CoverageProject( cmake_target="rocThrust", + artifact_names=["prim"], + artifact_relpaths=["math-libs/rocThrust/stage"], coverage_option="CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocthrust", @@ -340,6 +389,8 @@ class CoverageProject: ), "rocwmma": CoverageProject( cmake_target="rocWMMA", + artifact_names=["rocwmma"], + artifact_relpaths=["math-libs/rocWMMA/stage"], coverage_option="CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocwmma", @@ -353,6 +404,8 @@ class CoverageProject: # "rccl": CoverageProject( cmake_target="rccl", + artifact_names=["rccl"], + artifact_relpaths=["comm-libs/rccl/stage"], coverage_option="ENABLE_CODE_COVERAGE", stage=STAGE_COMM_LIBS, test_component="rccl", @@ -364,6 +417,8 @@ class CoverageProject: ), "rocshmem": CoverageProject( cmake_target="rocshmem", + artifact_names=["rocshmem"], + artifact_relpaths=["comm-libs/rocshmem/stage"], coverage_option="BUILD_CODE_COVERAGE", stage=STAGE_COMM_LIBS, test_component="rocshmem", @@ -378,6 +433,8 @@ class CoverageProject: # "rocprofiler-sdk": CoverageProject( cmake_target="rocprofiler-sdk", + artifact_names=["rocprofiler-sdk"], + artifact_relpaths=["profiler/rocprofiler-sdk/stage"], unsupported_reason=( "ROCPROFILER_BUILD_CODECOV selects gcov instrumentation, which writes .gcda files rather than the .profraw this pipeline merges" ), @@ -391,6 +448,8 @@ class CoverageProject: ), "aqlprofile": CoverageProject( cmake_target="aqlprofile", + artifact_names=["aqlprofile"], + artifact_relpaths=["profiler/aqlprofile/stage"], unsupported_reason="no coverage option in its CMake", stage=STAGE_COMPILER_RUNTIME, test_component="aqlprofile", @@ -402,6 +461,8 @@ class CoverageProject: ), "amdsmi": CoverageProject( cmake_target="amdsmi", + artifact_names=["core-amdsmi"], + artifact_relpaths=["core/amdsmi/stage"], unsupported_reason="no coverage option in its CMake", stage=STAGE_COMPILER_RUNTIME, test_component="amdsmi", @@ -413,6 +474,8 @@ class CoverageProject: ), "rocprofiler-compute": CoverageProject( cmake_target="rocprofiler-compute", + artifact_names=["rocprofiler-compute"], + artifact_relpaths=["profiler/rocprofiler-compute/stage"], unsupported_reason="a Python tool with no native instrumentation option", stage=STAGE_COMPILER_RUNTIME, test_component="rocprofiler-compute", @@ -427,6 +490,8 @@ class CoverageProject: # "rocprofiler-systems": CoverageProject( cmake_target="rocprofiler-systems", + artifact_names=["rocprofiler-systems"], + artifact_relpaths=["profiler/rocprofiler-systems/stage"], unsupported_reason=( "no build-time coverage option; its coverage feature instruments other programs at runtime" ), @@ -455,6 +520,11 @@ class CoverageProject: assert bool(_proj.coverage_option) != bool( _proj.unsupported_reason ), f"{_key}: set exactly one of coverage_option and unsupported_reason" + # Without these the nightly test job cannot tell which files belong to the + # project, and would measure an entirely non-instrumented install. + if _proj.coverage_option: + assert _proj.artifact_names, f"{_key}: needs artifacts to overlay" + assert _proj.artifact_relpaths, f"{_key}: needs relpaths to overlay" SUPPORTED_PROJECTS: frozenset[str] = frozenset( k for k, v in COVERAGE_PROJECTS.items() if v.coverage_option @@ -574,6 +644,8 @@ def build_coverage_matrix( "fetch_artifact_args": project.fetch_artifact_args, "codecov_flag": project.codecov_flag or project_key, "amdgpu_families": family, + "artifact_names": ",".join(project.artifact_names), + "artifact_relpaths": ",".join(project.artifact_relpaths), } ) return matrix @@ -618,28 +690,70 @@ def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: return options -def build_stage_plan(project_keys: list[str]) -> tuple[list[dict], bool]: - """Works out which build stages the selection needs. +def _stage_display_name(stage: str) -> str: + return f"Stage - {STAGE_DISPLAY_NAMES.get(stage, f'Coverage {stage}')}" + + +def build_stage_plan(project_keys: list[str]) -> tuple[list[list[dict]], bool]: + """Works out which build stages the selection needs, and in what order. - Returns the generic (built-once) stages other than compiler-runtime, and - whether the per-architecture math-libs stage is needed. compiler-runtime is - excluded because it is built unconditionally: every other stage takes its - inbound artifacts from it, so there is no selection that does not need it. + Returns the generic (built-once) stages grouped into waves that can each + build in parallel, plus whether the per-architecture math-libs stage is + needed. compiler-runtime is excluded because it is built unconditionally: + every other stage takes its inbound artifacts from it, so there is no + selection that does not need it. + + A project's own stage is not the whole answer, which is why the topology is + consulted rather than just the registry. Selecting rccl asks for comm-libs, + but comm-libs takes hipify and rocjitsu from emulation, so emulation has to + be built too -- and built first, hence the waves. """ - stages = {COVERAGE_PROJECTS[key].stage for key in project_keys} - needs_math_libs = STAGE_MATH_LIBS in stages - - generic = sorted(stages - PER_ARCH_STAGES - {STAGE_COMPILER_RUNTIME}) - return ( - [ - { - "stage_name": stage, - "stage_display_name": f"Stage - {STAGE_DISPLAY_NAMES[stage]}", - } - for stage in generic - ], - needs_math_libs, - ) + topology = get_topology() + + def producers(stage: str) -> set[str]: + return { + topology.get_stage_for_artifact(artifact) + for artifact in topology.get_inbound_artifacts(stage) + } + + selected = {COVERAGE_PROJECTS[key].stage for key in project_keys} + needs_math_libs = STAGE_MATH_LIBS in selected + + required: set[str] = set() + queue = list(selected) + while queue: + stage = queue.pop() + if stage in required: + continue + required.add(stage) + queue.extend(producers(stage) - required) + + # math-libs has its own per-architecture job, and compiler-runtime is + # always built, so neither belongs in the generic waves. Both are treated + # as already satisfied when ordering what is left. + built = {STAGE_COMPILER_RUNTIME, STAGE_MATH_LIBS} + remaining = required - built + + waves: list[list[dict]] = [] + while remaining: + wave = sorted(s for s in remaining if not (producers(s) - built - {s})) + if not wave: + # Dropping the stages silently would leave the build to fail much + # later, on a missing inbound artifact. + raise ValueError( + f"cannot order coverage build stages {sorted(remaining)}: " + "their inbound artifacts form a cycle" + ) + waves.append( + [ + {"stage_name": s, "stage_display_name": _stage_display_name(s)} + for s in wave + ] + ) + built.update(wave) + remaining -= set(wave) + + return waves, needs_math_libs def emit_cmake(output_path: Path) -> None: @@ -715,7 +829,16 @@ def main(argv: Optional[list[str]] = None) -> int: project_keys, amdgpu_families, config_repository, config_ref ) coverage_flags = build_coverage_cmake_options(project_keys) - generic_stages, needs_math_libs = build_stage_plan(project_keys) + stage_waves, needs_math_libs = build_stage_plan(project_keys) + # The workflow has one job per wave, and Actions cannot generate jobs, so a + # third wave has to be a build error here rather than a stage the workflow + # quietly skips. + if len(stage_waves) > 2: + raise ValueError( + f"build_stage_plan produced {len(stage_waves)} stage waves but " + "multi_arch_ci_coverage_nightly.yml only has jobs for two; add " + "another build_instrumented_generic_stages_* job to match" + ) outputs = { "coverage_matrix": json.dumps(matrix), "dist_amdgpu_families": ";".join(amdgpu_families), @@ -723,7 +846,8 @@ def main(argv: Optional[list[str]] = None) -> int: [{"amdgpu_family": family} for family in amdgpu_families] ), "coverage_cmake_options": " ".join(coverage_flags), - "generic_stages_json": json.dumps(generic_stages), + "stage_wave1_json": json.dumps(stage_waves[0] if stage_waves else []), + "stage_wave2_json": json.dumps(stage_waves[1] if len(stage_waves) > 1 else []), "needs_math_libs": "true" if needs_math_libs else "false", } diff --git a/build_tools/github_actions/overlay_coverage_artifacts.py b/build_tools/github_actions/overlay_coverage_artifacts.py new file mode 100644 index 00000000000..0c67cf87963 --- /dev/null +++ b/build_tools/github_actions/overlay_coverage_artifacts.py @@ -0,0 +1,252 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Overlays one project's instrumented binaries onto a baseline install tree. + +Nightly coverage builds an instrumented stack in its own workflow run while the +regular nightly keeps publishing the normal one, so the two live under separate +run ids. A coverage test job wants exactly one instrumented project and +non-instrumented everything else: an instrumented dependency would emit its own +profiles and move the coverage denominator around whenever that dependency +changes. + +The test job gets there in two steps. It first installs the whole baseline run +the usual way, then calls this script to fetch the instrumented artifact from +the coverage run and copy just the project under test over the baseline copy. + +TheRock's artifacts are grouped -- `rand` carries both rocRAND and hipRAND -- +so a whole artifact is too coarse to overlay. Each artifact keeps its files +under the subproject stage directory they were built in +(`math-libs/hipRAND/stage/...`), and that is what --artifact-relpaths names, so +sibling projects in the same artifact stay non-instrumented. +""" + +import argparse +import logging +import os +import shutil +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) + +from artifact_manager import main as artifact_manager_main + +logging.basicConfig(level=logging.INFO, format="%(message)s") + +# Artifact components an instrumented project can contribute: `lib` the +# instrumented libraries, `test` the test binaries (instrumented in their own +# right for header-only projects), and `dev` the headers and CMake config that +# go with them. The rest only carry symbols and documentation. +EXCLUDED_COMPONENTS = ["run", "dbg", "doc"] + +# artifact_manager.py fetch explodes each archive into this subdirectory of its +# output directory, one directory per artifact. +FETCH_SUBDIR = "artifacts" + + +def split_list(raw: str) -> list[str]: + return [item.strip() for item in raw.split(",") if item.strip()] + + +def fetch_instrumented_artifacts( + *, + run_id: str, + artifact_names: list[str], + amdgpu_families: str, + amdgpu_targets: str, + output_dir: Path, + run_github_repo: str | None, +) -> Path: + """Downloads and explodes the named artifacts, returning their parent dir. + + Neither flattened nor bootstrapped: the overlay needs the per-subproject + directory layout that flattening would collapse. + """ + argv = [ + "fetch", + "--run-id", + run_id, + # These artifacts are outputs of the stage that built them rather than + # inbound to any stage, so the whole topology is the search space and + # --artifact-names does the narrowing. + "--stage", + "all", + "--artifact-names", + ",".join(artifact_names), + "--exclude-components", + ",".join(EXCLUDED_COMPONENTS), + "--amdgpu-families", + amdgpu_families, + "--output-dir", + os.fspath(output_dir), + ] + if amdgpu_targets: + argv.extend(["--amdgpu-targets", amdgpu_targets]) + if run_github_repo: + argv.extend(["--run-github-repo", run_github_repo]) + + logging.info("Fetching instrumented artifacts: %s", " ".join(argv)) + artifact_manager_main(argv) + return output_dir / FETCH_SUBDIR + + +def merge_tree(source: Path, dest: Path) -> None: + """Copies the `source` tree over `dest`, replacing entries already there. + + shutil.copytree(dirs_exist_ok=True) does not work for this. It recreates + symlinks with os.symlink(), which raises FileExistsError as soon as the + baseline install has already put the libfoo.so -> libfoo.so.1 chain in + place -- which it always has, since the overlay runs after a full install. + dirs_exist_ok only forgives collisions on directories. + + Directories are merged; everything else is replaced outright, so the + instrumented file wins over the baseline one. + """ + dest.mkdir(parents=True, exist_ok=True) + for entry in source.iterdir(): + target = dest / entry.name + if entry.is_symlink(): + # exists() is False for a dangling symlink, hence both checks. + if target.is_symlink() or target.exists(): + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(target) + else: + target.unlink() + os.symlink(os.readlink(entry), target) + elif entry.is_dir(): + merge_tree(entry, target) + else: + # Unlink first: copy2 follows a symlink at the destination and + # would write through it, overwriting the baseline file it names + # while leaving the link itself behind. + if target.is_symlink(): + target.unlink() + shutil.copy2(entry, target) + + +def overlay_relpaths( + *, staging_dir: Path, relpaths: list[str], install_dir: Path +) -> int: + """Copies each artifact's subtree over the install tree. + + Returns the number of subtrees copied. Symlinks are preserved so the + libfoo.so -> libfoo.so.1 chain the loader follows still points at the + instrumented file. + """ + if not staging_dir.is_dir(): + return 0 + + copied = 0 + for artifact_dir in sorted(p for p in staging_dir.iterdir() if p.is_dir()): + for relpath in relpaths: + source = artifact_dir / relpath + if not source.is_dir(): + continue + logging.info("Overlaying %s onto %s", source, install_dir) + merge_tree(source, install_dir) + copied += 1 + return copied + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--run-id", + required=True, + help="Coverage workflow run that published the instrumented artifacts", + ) + parser.add_argument( + "--amdgpu-families", + required=True, + help="Semicolon-separated GPU families to fetch (e.g. 'gfx94X-dcgpu')", + ) + parser.add_argument( + "--amdgpu-targets", + default="", + help="Comma-separated individual GPU targets for split artifacts", + ) + parser.add_argument( + "--artifact-names", + required=True, + help="Comma-separated artifact names holding the instrumented project " + "(e.g. 'rand')", + ) + parser.add_argument( + "--artifact-relpaths", + required=True, + help="Comma-separated subproject stage directories to overlay " + "(e.g. 'math-libs/hipRAND/stage')", + ) + parser.add_argument( + "--install-dir", + type=Path, + required=True, + help="Baseline install tree to overlay the instrumented files onto", + ) + parser.add_argument( + "--staging-dir", + type=Path, + default=None, + help="Directory for the downloaded artifacts (default: a temporary directory)", + ) + parser.add_argument( + "--run-github-repo", + default=None, + help="GitHub repository for --run-id in 'owner/repo' format", + ) + args = parser.parse_args(argv) + + artifact_names = split_list(args.artifact_names) + relpaths = split_list(args.artifact_relpaths) + if not artifact_names or not relpaths: + logging.error( + "--artifact-names and --artifact-relpaths must both be non-empty." + ) + return 1 + + install_dir = args.install_dir.resolve() + if not install_dir.is_dir(): + logging.error( + "Install directory %s does not exist. The baseline artifacts must " + "be installed before the instrumented ones are overlaid.", + install_dir, + ) + return 1 + + with tempfile.TemporaryDirectory() as tmp: + staging_dir = args.staging_dir or Path(tmp) + staging_dir.mkdir(parents=True, exist_ok=True) + artifacts_dir = fetch_instrumented_artifacts( + run_id=args.run_id, + artifact_names=artifact_names, + amdgpu_families=args.amdgpu_families, + amdgpu_targets=args.amdgpu_targets, + output_dir=staging_dir, + run_github_repo=args.run_github_repo, + ) + copied = overlay_relpaths( + staging_dir=artifacts_dir, + relpaths=relpaths, + install_dir=install_dir, + ) + + # Silently testing the baseline build would report coverage against + # binaries that were never instrumented, so treat this as fatal. + if not copied: + logging.error( + "None of the artifact relpaths (%s) were found in the artifacts " + "fetched from run %s. The instrumented build did not produce the " + "expected layout.", + ", ".join(relpaths), + args.run_id, + ) + return 1 + + logging.info("Overlaid %d instrumented subtree(s) onto %s", copied, install_dir) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 02dddf00004..868c9bd2a1a 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -11,8 +11,10 @@ # Add repo root to PYTHONPATH sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +sys.path.insert(0, os.fspath(Path(__file__).parents[2])) import configure_coverage_ci +from _therock_utils.build_topology import get_topology class ParseProjectsTest(unittest.TestCase): @@ -230,22 +232,34 @@ def test_no_selection_produces_no_options(self): class BuildStagePlanTest(unittest.TestCase): + @staticmethod + def _names(waves): + return [[entry["stage_name"] for entry in wave] for wave in waves] + def test_math_libs_only_selection_needs_no_generic_stage(self): - generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["hiprand"]) - self.assertEqual(generic, []) + waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["hiprand"]) + self.assertEqual(waves, []) self.assertTrue(needs_math_libs) def test_compiler_runtime_project_adds_no_stage_of_its_own(self): # compiler-runtime is built unconditionally, so a project living there # must not also appear as a generic stage to build. - generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["amdsmi"]) - self.assertEqual(generic, []) + waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["amdsmi"]) + self.assertEqual(waves, []) + self.assertFalse(needs_math_libs) + + def test_generic_stage_project_pulls_in_what_that_stage_consumes(self): + # comm-libs takes hipify and rocjitsu from emulation, so selecting rccl + # has to build emulation as well -- and before comm-libs, or the build + # fails on a missing inbound artifact. + waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["rccl"]) + self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) self.assertFalse(needs_math_libs) - def test_generic_stage_project_is_named_for_the_build_job(self): - generic, needs_math_libs = configure_coverage_ci.build_stage_plan(["rccl"]) + def test_generic_stage_is_named_for_the_build_job(self): + waves, _ = configure_coverage_ci.build_stage_plan(["rccl"]) self.assertEqual( - generic, + waves[-1], [ { "stage_name": "comm-libs", @@ -253,22 +267,45 @@ def test_generic_stage_project_is_named_for_the_build_job(self): } ], ) - self.assertFalse(needs_math_libs) def test_stages_are_deduplicated_across_projects(self): - generic, _ = configure_coverage_ci.build_stage_plan(["rccl", "rocshmem"]) - self.assertEqual([entry["stage_name"] for entry in generic], ["comm-libs"]) + waves, _ = configure_coverage_ci.build_stage_plan(["rccl", "rocshmem"]) + self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) def test_selecting_everything_names_every_non_default_stage(self): - generic, needs_math_libs = configure_coverage_ci.build_stage_plan( - sorted(configure_coverage_ci.COVERAGE_PROJECTS) - ) - self.assertEqual( - [entry["stage_name"] for entry in generic], - ["comm-libs", "profiler-apps"], + waves, needs_math_libs = configure_coverage_ci.build_stage_plan( + sorted(configure_coverage_ci.SUPPORTED_PROJECTS) ) + self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) self.assertTrue(needs_math_libs) + def test_no_selection_needs_more_waves_than_the_workflow_has_jobs(self): + # multi_arch_ci_coverage_nightly.yml has two wave jobs. main() raises + # past that, but the registry should not get there in the first place. + waves, _ = configure_coverage_ci.build_stage_plan( + sorted(configure_coverage_ci.COVERAGE_PROJECTS) + ) + self.assertLessEqual(len(waves), 2) + + +class RegistryMatchesBuildTopologyTest(unittest.TestCase): + """The registry duplicates facts the build topology already knows. + + Nothing keeps the two in step at runtime, so a project moving between + stages upstream would otherwise show up as a stage that is never built and + an overlay that finds nothing, hours into a run. + """ + + def test_every_project_names_the_stage_that_builds_its_artifacts(self): + topology = get_topology() + for key, project in sorted(configure_coverage_ci.COVERAGE_PROJECTS.items()): + for artifact in project.artifact_names: + with self.subTest(project=key, artifact=artifact): + self.assertEqual( + topology.get_stage_for_artifact(artifact), + project.stage, + ) + class MainTest(unittest.TestCase): def setUp(self): @@ -301,7 +338,8 @@ def test_writes_expected_outputs(self): self.assertIn('"object_globs": "lib/libhiprand.so*"', written) # hipRAND is a math-libs project, so no other stage is built. self.assertIn("needs_math_libs=true", written) - self.assertIn("generic_stages_json=[]", written) + self.assertIn("stage_wave1_json=[]", written) + self.assertIn("stage_wave2_json=[]", written) class EmitCmakeTest(unittest.TestCase): diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index cc7a902154d..cb54ae5c270 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -142,9 +142,7 @@ def test_export_passes_the_first_object_positionally(self): command = run.call_args.args[0] self.assertEqual(command[:3], [str(llvm_cov), "export", str(objects[0])]) self.assertIn("-object", command) - self.assertEqual( - command[-2:], [f"-instr-profile={profdata}", "--format=lcov"] - ) + self.assertEqual(command[-2:], [f"-instr-profile={profdata}", "--format=lcov"]) class MainTest(TempDirTestBase): diff --git a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py new file mode 100644 index 00000000000..2d94a59c5f5 --- /dev/null +++ b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py @@ -0,0 +1,270 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import os +import platform +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +# Add repo root to PYTHONPATH +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +import overlay_coverage_artifacts + + +def is_windows() -> bool: + return platform.system() == "Windows" + + +class TempDirTestBase(unittest.TestCase): + def setUp(self): + self._temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self._temp_dir.cleanup) + self.root = Path(self._temp_dir.name) + + def write(self, relative_path: str, contents: str) -> Path: + path = self.root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents) + return path + + +class FetchInstrumentedArtifactsTest(TempDirTestBase): + def test_narrows_the_fetch_to_the_named_artifacts(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + artifacts_dir = overlay_coverage_artifacts.fetch_instrumented_artifacts( + run_id="99999", + artifact_names=["rand", "prim"], + amdgpu_families="gfx94X-dcgpu", + amdgpu_targets="", + output_dir=self.root / "staging", + run_github_repo=None, + ) + + (argv,) = fetch.call_args.args + self.assertEqual(argv[0], "fetch") + # These artifacts are stage outputs, not inbound to any stage. + self.assertEqual(argv[argv.index("--stage") + 1], "all") + self.assertEqual(argv[argv.index("--artifact-names") + 1], "rand,prim") + self.assertEqual(artifacts_dir, self.root / "staging" / "artifacts") + + def test_skips_components_that_cannot_be_instrumented(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + overlay_coverage_artifacts.fetch_instrumented_artifacts( + run_id="99999", + artifact_names=["rand"], + amdgpu_families="gfx94X-dcgpu", + amdgpu_targets="gfx942", + output_dir=self.root / "staging", + run_github_repo=None, + ) + + (argv,) = fetch.call_args.args + excluded = argv[argv.index("--exclude-components") + 1].split(",") + self.assertCountEqual(excluded, ["run", "dbg", "doc"]) + self.assertEqual(argv[argv.index("--amdgpu-targets") + 1], "gfx942") + + +class OverlayRelpathsTest(TempDirTestBase): + def _staged_artifact(self, artifact: str, relpath: str, files: dict[str, str]): + for name, contents in files.items(): + self.write(f"staging/{artifact}/{relpath}/{name}", contents) + + def test_copies_only_the_requested_relpath(self): + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/hipRAND/stage", + {"lib/libhiprand.so": "instrumented"}, + ) + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/rocRAND/stage", + {"lib/librocrand.so": "instrumented"}, + ) + self.write("install/lib/libhiprand.so", "baseline") + self.write("install/lib/librocrand.so", "baseline") + + copied = overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertEqual(copied, 1) + self.assertEqual( + (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" + ) + # A sibling project in the same grouped artifact stays as it was. + self.assertEqual( + (self.root / "install/lib/librocrand.so").read_text(), "baseline" + ) + + def test_merges_into_the_existing_tree(self): + self._staged_artifact( + "rand_lib_gfx942", + "math-libs/hipRAND/stage", + {"lib/libhiprand.so": "instrumented"}, + ) + self.write("install/lib/libamdhip64.so", "baseline") + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertTrue((self.root / "install/lib/libamdhip64.so").exists()) + self.assertTrue((self.root / "install/lib/libhiprand.so").exists()) + + def test_counts_every_matching_artifact(self): + for component in ("lib", "test"): + self._staged_artifact( + f"rand_{component}_gfx942", + "math-libs/hipRAND/stage", + {f"{component}/file": component}, + ) + (self.root / "install").mkdir() + + self.assertEqual( + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ), + 2, + ) + + def test_reports_nothing_copied_when_the_relpath_is_absent(self): + self._staged_artifact( + "rand_lib_gfx942", "math-libs/rocRAND/stage", {"lib/librocrand.so": "x"} + ) + (self.root / "install").mkdir() + + self.assertEqual( + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ), + 0, + ) + + @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") + def test_preserves_symlinks(self): + # The loader follows libfoo.so -> libfoo.so.1; copying the link as a + # regular file would leave a stale non-instrumented copy behind it. + stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" + self.write(f"{stage}/libhiprand.so.1", "instrumented") + (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") + (self.root / "install").mkdir() + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + self.assertTrue((self.root / "install/lib/libhiprand.so").is_symlink()) + + @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") + def test_replaces_a_symlink_the_baseline_already_installed(self): + # The overlay always runs after a full baseline install, so the whole + # symlink chain is already on disk. shutil.copytree used to fail here + # with "File exists: 'libhiprand.so.1' -> '.../libhiprand.so'" -- + # dirs_exist_ok forgives directory collisions, not symlink ones. + stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" + self.write(f"{stage}/libhiprand.so.1", "instrumented") + (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") + self.write("install/lib/libhiprand.so.1", "baseline") + (self.root / "install/lib/libhiprand.so").symlink_to("libhiprand.so.1") + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + link = self.root / "install/lib/libhiprand.so" + self.assertTrue(link.is_symlink()) + self.assertEqual(link.read_text(), "instrumented") + + @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") + def test_replaces_a_baseline_regular_file_with_a_symlink(self): + # Some baselines ship the resolved file where the instrumented artifact + # ships a link. Leaving the file in place would keep the loader on + # non-instrumented code. + stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" + self.write(f"{stage}/libhiprand.so.1", "instrumented") + (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") + self.write("install/lib/libhiprand.so", "baseline") + + overlay_coverage_artifacts.overlay_relpaths( + staging_dir=self.root / "staging", + relpaths=["math-libs/hipRAND/stage"], + install_dir=self.root / "install", + ) + + link = self.root / "install/lib/libhiprand.so" + self.assertTrue(link.is_symlink()) + self.assertEqual(link.read_text(), "instrumented") + + +class MainTest(TempDirTestBase): + def _run(self, **overrides) -> int: + args = { + "--run-id": "99999", + "--amdgpu-families": "gfx94X-dcgpu", + "--artifact-names": "rand", + "--artifact-relpaths": "math-libs/hipRAND/stage", + "--install-dir": os.fspath(self.root / "install"), + "--staging-dir": os.fspath(self.root / "staging"), + } + args.update(overrides) + argv = [item for pair in args.items() for item in pair] + return overlay_coverage_artifacts.main(argv) + + def test_fails_when_the_install_tree_is_missing(self): + with mock.patch.object( + overlay_coverage_artifacts, "artifact_manager_main" + ) as fetch: + self.assertEqual(self._run(), 1) + fetch.assert_not_called() + + def test_fails_when_nothing_was_overlaid(self): + # A silent pass here would report coverage for binaries that were never + # instrumented. + (self.root / "install").mkdir() + with mock.patch.object(overlay_coverage_artifacts, "artifact_manager_main"): + self.assertEqual(self._run(), 1) + + def test_fetches_and_overlays(self): + (self.root / "install").mkdir() + + def fake_fetch(argv): + self.write( + "staging/artifacts/rand_lib_gfx942/math-libs/hipRAND/stage" + "/lib/libhiprand.so", + "instrumented", + ) + + with mock.patch.object( + overlay_coverage_artifacts, + "artifact_manager_main", + side_effect=fake_fetch, + ): + self.assertEqual(self._run(), 0) + + self.assertEqual( + (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/build_tools/tests/artifact_manager_tool_test.py b/build_tools/tests/artifact_manager_tool_test.py index 4fc925e17a0..3eeb4579cd9 100644 --- a/build_tools/tests/artifact_manager_tool_test.py +++ b/build_tools/tests/artifact_manager_tool_test.py @@ -720,6 +720,68 @@ def mock_extract(request: artifact_manager.ExtractRequest) -> Path: self.assertIn("test-artifact_lib_generic.tar.zst", fetched_keys) self.assertNotIn("second-artifact_lib_generic.tar.zst", fetched_keys) + def test_fetch_artifact_names_limits_to_named_artifacts(self) -> None: + """Test that --artifact-names narrows the fetch to an include list.""" + import artifact_manager + + self._create_staged_artifact("test-artifact", "lib", "generic") + self._create_staged_artifact("second-artifact", "lib", "generic") + + extract_calls: list[artifact_manager.ExtractRequest] = [] + + def mock_extract(request: artifact_manager.ExtractRequest) -> Path: + extract_calls.append(request) + return request.output_dir + + with mock.patch("artifact_manager.extract_artifact", mock_extract): + argv = [ + "fetch", + "--stage", + "all", + "--output-dir", + str(self.output_dir), + "--topology", + str(self.topology_path), + "--local-staging-dir", + str(self.staging_dir), + "--platform", + TEST_PLATFORM, + "--run-id", + "local", + "--artifact-names", + "second-artifact", + ] + + artifact_manager.main(argv) + + fetched_keys = [c.archive_path.name for c in extract_calls] + self.assertEqual(fetched_keys, ["second-artifact_lib_generic.tar.zst"]) + + def test_fetch_artifact_names_rejects_unknown_artifact(self) -> None: + """Test that --artifact-names validates artifact names.""" + import artifact_manager + + argv = [ + "fetch", + "--stage", + "all", + "--output-dir", + str(self.output_dir), + "--topology", + str(self.topology_path), + "--local-staging-dir", + str(self.staging_dir), + "--platform", + TEST_PLATFORM, + "--run-id", + "local", + "--artifact-names", + "unknown-artifact", + ] + + with self.assertRaises(ValueError): + artifact_manager.main(argv) + def test_fetch_exclude_artifacts_rejects_unknown_artifact(self) -> None: """Test that --exclude-artifacts validates artifact names.""" argv = [ diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index d9a2deef595..9081506d434 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -125,35 +125,62 @@ report. ### Scheduling -`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. There is no cron -trigger, and the regular nightly does not know about this workflow. +`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. Pick a recent +nightly run, pass its id as `baseline_run_id`, and leave `baseline_release_type` +at `nightly`. There is no cron trigger, and the regular nightly does not know +about this workflow. Coverage therefore has a run id of its own, which is what keeps a coverage failure from colouring the nightly's status and keeps the nightly's test jobs — which rarely all pass — from holding coverage up. +Having the regular nightly dispatch this workflow automatically once its build +finishes, handing over its own run id, is a later change. Nothing else has to +move for it: `baseline_run_id` is already an input, so that change only adds the +dispatching job. + Nightly coverage runs the projects named by `projects_to_test` on one architecture (gfx942, the `gfx94X-dcgpu` family). Running only what changed, and running more architectures, come later. -#### The run is self-contained - -A coverage run builds every stage its selection needs, starting from -compiler-runtime, and instruments only the projects that were selected. -Everything those projects depend on is built normally in the same run, so the -run's own artifacts are already the stack to test against. The test jobs install -them the same way regular CI installs its own, and there is no second run id to -supply. - -What keeps a report scoped to one project is `object_globs`, not what happens to -be instrumented. `llvm-cov export` is handed only that project's objects and -reports only functions found in their coverage mappings, so a selection covering -several projects still yields one report per project. - -Selecting more projects therefore costs build time rather than accuracy: each -one adds an instrumented library, and instrumented libraries are slower to build -and slower to run under test. Naming the projects you actually want is the cheap -path. +#### Why there are two run ids + +The nightly instruments the whole stack in one build, because building each +project separately would mean rebuilding its dependencies each time. Reporting, +though, has to stay per project: a coverage report for hipRAND should not shift +because rocRAND changed, and an instrumented rocRAND under an instrumented +hipRAND also writes its own profiles into the same run. + +Each test job therefore assembles a mostly non-instrumented install: + +1. `setup_test_environment` installs the **baseline** run in full — every + library non-instrumented, as the regular nightly built it. +1. `overlay_coverage_artifacts.py` fetches the artifact holding the project + under test from the **coverage** run and copies only that project's files + over the baseline ones. + +The baseline is read from the channel that published it +(`baseline_release_type`, normally `nightly`) rather than from the coverage +run's own `ci` channel, since artifacts are bucketed per channel. + +Step 2 is per project rather than per artifact because TheRock's artifacts are +grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit under +the subproject stage directory they were built in (`math-libs/hipRAND/stage`), +which is what `artifact_relpaths` in the coverage registry names. Nothing is +renamed along the way — the two runs write to different S3 directories, so the +instrumented `rand_lib_gfx942.tar.xz` and the regular one never collide. + +If the overlay finds none of the project's files, the job fails rather than +testing the baseline build and reporting coverage for binaries that were never +instrumented. + +`object_globs` then scopes the report itself: `llvm-cov export` is handed only +that project's objects, so a selection covering several projects still yields +one report per project. The overlay and the globs solve different halves of the +same problem. The globs keep a sibling's whole functions out of the report; the +overlay keeps a sibling's instrumented inline and header code from merging into +this project's counters in the first place, which globs cannot undo because +`linkonce_odr` definitions land in every binary that uses them. ### Selecting projects to run diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md index 904ff8e2cd9..f83772fe276 100644 --- a/docs/development/code_coverage_flows.md +++ b/docs/development/code_coverage_flows.md @@ -104,9 +104,12 @@ shard list. `test_coverage` then runs one `test_component.yml` job per shard with `coverage_enabled: true`. Three things happen in order: -1. **Install this run's artifacts.** `setup_test_environment` runs against - `github.run_id` and the `ci` channel, exactly as it does for regular CI. No - coverage-specific artifact handling is involved. +1. **Install the baseline, then overlay.** `setup_test_environment` runs against + `baseline_run_id` and `baseline_release_type`, installing a fully + non-instrumented stack. `overlay_coverage_artifacts.py` then fetches this + run's artifact for the project under test and copies only that project's + stage directories over it, so exactly one library in the install is + instrumented. 1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to a per-shard path using the `%p` (pid) and `%m` (binary signature) substitutions, so a shard that forks or loads several instrumented libraries @@ -117,16 +120,23 @@ shard list. ### What scopes a report to one project -`object_globs`, not what happens to be instrumented. +Two mechanisms, covering different leaks. -`llvm-cov export` is handed an explicit `-object` for each of the project's -binaries and reports only the functions found in their coverage mappings. -Counters that a sibling project wrote into the same merged profdata are never -looked up. A selection naming several projects therefore still produces one -report per project, from a single build and a single run. +`object_globs` scopes the export. `llvm-cov export` is handed an explicit +`-object` for each of the project's binaries and reports only the functions +found in their coverage mappings, so a sibling's whole functions never appear. + +The overlay scopes what is instrumented at all. Globs alone would not be enough, +because inline and template code defined in headers is emitted `linkonce_odr` +into every binary that uses it. If a sibling library were also instrumented, its +copies of those functions would write counters that `llvm-profdata` merges into +this project's, and no `-object` choice could separate them again. Installing a +non-instrumented baseline and overlaying one project keeps that from arising — +which matters most for the header-only projects (rocPRIM, hipCUB, rocThrust, +rocWMMA), where nearly all the measured code lives in headers. The cost of selecting more projects is build and test time — each instrumented -library is slower to compile and slower to exercise — rather than accuracy. +library is slower to compile and slower to exercise. ## Artifacts From 387ac8a9a521865e66271c5527c31f9d850ae259 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 08:20:09 -0500 Subject: [PATCH 13/70] ci: narrow the coverage PR check to hipRAND Run 34119302776 selected every onboarded project and found the stack is not ready to be instrumented all at once: - libclang_rt.profile_rocm.a is linked without -ldl or -lpthread, so an instrumented executable cannot resolve dlsym, dladdr or pthread_once. Shared libraries tolerate the undefined symbols, which is why this stayed hidden until projects that build their own binaries were selected: rocRAND's tools, hipCUB's and rocPRIM's tests. - rccl's CMake refuses coverage outside a Debug build, and TheRock builds Release. - rocshmem does not survive coverage together with HIP relocatable device code: "unknown special variable with appending linkage: llvm.global_ctors". Narrow to hipRAND, which built cleanly instrumented in 34098007786, so the baseline install and the per-project overlay can be proven on their own. The rocm-systems projects wait on the second and third problem, the rest of math-libs on the first; the selection widens once they are fixed. Nothing about the workflow changes -- it still resolves the whole registry when asked, and the emulation/comm-libs wave jobs stay in place for when rccl and rocshmem come back. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 2aa34ccd671..042098133a0 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -85,10 +85,23 @@ jobs: # but naming it keeps the pin and its channel in one place. baseline_release_type: "nightly" amdgpu_families: "gfx94X-dcgpu" - # The whole registry, which is what phase 1 specifies: instrument the - # stack once and report every onboarded project. Expensive on purpose -- - # this is the case a single-project run cannot demonstrate, since the - # overlay only has to hold sibling instrumentation out of the install - # when there are instrumented siblings to hold out. - projects_to_test: "all" + # hipRAND alone while the baseline and overlay path is being proven. The + # phase 1 scope is the whole registry, and the workflow supports it, but + # run 34119302776 showed the stack is not ready to be instrumented all at + # once: + # + # - libclang_rt.profile_rocm.a is linked without -ldl or -lpthread, so + # any instrumented executable fails to resolve dlsym, dladdr and + # pthread_once. Shared libraries tolerate the undefined symbols, so + # this only bites projects that build their own binaries: rocRAND's + # tools, and hipCUB's and rocPRIM's tests. + # - rccl's CMake refuses coverage outside a Debug build, and TheRock + # builds Release. + # - rocshmem does not survive coverage together with HIP relocatable + # device code ("unknown special variable with appending linkage"). + # + # The rocm-systems projects (rccl, rocshmem) are out until the second and + # third are addressed; the rest of math-libs waits on the first. hipRAND + # is unaffected -- it built cleanly instrumented in run 34098007786. + projects_to_test: "hiprand" test_type: "standard" From a04976869b50c6d81c5ed167a2d550ff8a6330c5 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 09:51:14 -0500 Subject: [PATCH 14/70] fix(cmake): link the profile runtime's dependencies for coverage builds -fprofile-instr-generate links libclang_rt.profile_rocm.a, whose InstrProfilingPlatformROCm.cpp calls dlsym, dladdr and pthread_once. The driver puts neither -ldl nor -lpthread on the link line, and the manylinux base still keeps those symbols in libdl and libpthread rather than in libc, so an instrumented shared library ends up with undefined references. The library itself links, since shared objects tolerate that, and the failure surfaces one step later when an executable links against it: ld.lld: error: undefined reference: dlsym >>> referenced by library/libhiprand.so.1.1 >>> (disallowed by --no-allow-shlib-undefined) This blocked every instrumented project, not just the ones building their own executables, and is why run 34126836356 failed on hipRAND alone. It stayed hidden until 1fa0945c made the coverage flag actually reach the subproject; before that hipRAND was building uninstrumented. Add the two libraries to the subproject's linker flags alongside the coverage option. The flags go through the project init file, which appends to CMAKE__LINKER_FLAGS and so lands ahead of the objects that need the symbols; push-state/--no-as-needed keeps the DT_NEEDED entries from being dropped on that account, and pop-state keeps the relaxation from applying to anything else on the line. Verified the mechanism outside ROCm, reproducing the shape of the failure: a static archive with an unresolved reference linked into a shared library, an executable linking against it under --no-allow-shlib-undefined, and the same init-file flags fixing it and recording the expected DT_NEEDED. Co-authored-by: Cursor --- cmake/therock_subproject.cmake | 34 +++++++++++++++++++++++++++++-- docs/development/code_coverage.md | 9 ++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 159977416ad..54adf17f8c6 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -913,6 +913,35 @@ function(therock_cmake_subproject_activate target_name) _therock_get_flag_init_contents(_flag_init_contents "${target_name}") string(APPEND _init_contents "${_flag_init_contents}") + # Give the LLVM profile runtime the libraries it depends on. + # + # -fprofile-instr-generate links libclang_rt.profile_rocm.a, whose + # InstrProfilingPlatformROCm.cpp calls dlsym, dladdr and pthread_once. The + # driver does not put -ldl or -lpthread on the link line, and on the + # manylinux base those symbols are still in libdl and libpthread rather than + # in libc, so an instrumented shared library is left with undefined + # references. The library itself links -- shared objects tolerate that -- but + # every executable linking against it then fails + # --no-allow-shlib-undefined, which is how this shows up: hipRAND's tests + # cannot link against an instrumented libhiprand.so. + # + # push-state/pop-state keeps --no-as-needed scoped to these two, so the + # DT_NEEDED entries are recorded no matter where the flags land relative to + # the objects that need them, without forcing a DT_NEEDED for every other + # library on the line. + string(TOUPPER "${_logical_target_name}" _coverage_project_name) + set(_coverage_var_name "${_coverage_project_name}_ENABLE_COVERAGE") + if(NOT MSVC AND ${_coverage_var_name}) + set(_coverage_link_flags + " -Wl,--push-state,--no-as-needed -ldl -lpthread -Wl,--pop-state") + string(APPEND _init_contents + "string(APPEND CMAKE_EXE_LINKER_FLAGS \"${_coverage_link_flags}\")\n") + string(APPEND _init_contents + "string(APPEND CMAKE_SHARED_LINKER_FLAGS \"${_coverage_link_flags}\")\n") + string(APPEND _init_contents + "string(APPEND CMAKE_MODULE_LINKER_FLAGS \"${_coverage_link_flags}\")\n") + endif() + file(CONFIGURE OUTPUT "${_cmake_project_init_file}" CONTENT "${_init_contents}" @ONLY ESCAPE_QUOTES) list(APPEND _fprint_files "${_cmake_project_init_file}") list(APPEND _fprint_files "${ROCM_BUILD_FLAGS_STATE_FILE}") @@ -940,8 +969,9 @@ function(therock_cmake_subproject_activate target_name) # BUILD_CODE_COVERAGE for the whole build would instrument every project that # happens to understand it, which is exactly what per-project coverage is # trying to avoid. - string(TOUPPER "${_logical_target_name}" _coverage_project_name) - set(_coverage_var_name "${_coverage_project_name}_ENABLE_COVERAGE") + # + # _coverage_project_name and _coverage_var_name are set further up, where the + # profile runtime's link libraries are added to the init file. if(DEFINED ${_coverage_var_name}) set(_coverage_option "${THEROCK_COVERAGE_OPTION_${_coverage_project_name}}") if(NOT _coverage_option) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 9081506d434..96f61ecb2f2 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -29,6 +29,15 @@ not configure that from the top level; it comes from how amd-llvm's compiler-rt is built. If a report shows host coverage only, check that the runtime is in the compiler you built against. +That runtime calls `dlsym`, `dladdr` and `pthread_once`, and the driver does not +put `-ldl` or `-lpthread` on the link line for it. On the manylinux base those +symbols are still in `libdl` and `libpthread` rather than in `libc`, so an +instrumented shared library would be left with undefined references, and any +executable linking against it would fail `--no-allow-shlib-undefined`. +`therock_subproject.cmake` adds the two libraries whenever it enables coverage +for a subproject, so this is handled; it is worth knowing about if you +instrument a project outside the normal flow. + > [!NOTE] > Coverage builds are incompatible with split kernel packaging. The CI > workflows pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`; do the same if you From 1e6f606eee30664a7adcb55f5013459bd0a7a5a9 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 13:45:43 -0500 Subject: [PATCH 15/70] ci(coverage): build only the stages phase 1 can use The staged build jobs existed for one dependency edge: comm-libs takes hipify and rocjitsu from emulation, so selecting rccl or rocshmem had to build emulation first. Those two projects are the only consumers, and both are blocked upstream -- rccl's CMake refuses coverage outside a Debug build, and rocshmem fails on coverage plus HIP relocatable device code. The other 17 measurable projects are all in math-libs, so the wave jobs were carrying no traffic. Drop them, along with the topology walk and wave ordering in configure_coverage_ci.py. build_stage_plan() becomes resolve_build_stages(), which checks the selection against BUILDABLE_STAGES and reports whether math-libs is needed. Naming an unbuildable project is now an error at matrix time. Without it nothing would be built for that project and the failure would not surface until the test job found no instrumented files to overlay, hours into the run. An empty selection meant every measurable project, which would now reject itself, so it narrows to the buildable set. That applies to the default only; an explicit request is refused rather than reduced. The ordering constraint is recorded under "Adding a stage" in docs/development/code_coverage.md for whoever restores comm-libs. Co-authored-by: Cursor --- .../multi_arch_ci_coverage_nightly.yml | 85 +---------- .../github_actions/configure_coverage_ci.py | 140 +++++++----------- .../tests/configure_coverage_ci_test.py | 108 +++++++------- docs/development/code_coverage.md | 37 +++-- docs/development/code_coverage_flows.md | 33 +++-- 5 files changed, 154 insertions(+), 249 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index ae06c7418a8..9e016a616e6 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -46,7 +46,7 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." + description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group alias: rocm_libraries_all. Empty = every project this workflow can build. rccl and rocshmem need the comm-libs build job, which phase 1 does not have, so naming them (or rocm_systems_all / all) is rejected." default: "" amdgpu_families: type: string @@ -89,7 +89,7 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group aliases: rocm_libraries_all, rocm_systems_all, all (combinable with explicit names). Empty = all coverage-enabled projects." + description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group alias: rocm_libraries_all. Empty = every project this workflow can build. rccl and rocshmem need the comm-libs build job, which phase 1 does not have, so naming them (or rocm_systems_all / all) is rejected." default: "" amdgpu_families: type: string @@ -133,8 +133,6 @@ jobs: dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} - stage_wave1_json: ${{ steps.matrix.outputs.stage_wave1_json }} - stage_wave2_json: ${{ steps.matrix.outputs.stage_wave2_json }} needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} steps: @@ -184,77 +182,6 @@ jobs: contents: read id-token: write - # The built-once stages the selection reaches into, in dependency order. - # Selecting rccl or rocshmem asks for comm-libs, which takes hipify and - # rocjitsu from emulation, so emulation builds first and comm-libs second. - # configure_coverage_ci.py derives both waves from the build topology and - # fails if a selection ever needs a third. - # - # A selection confined to math-libs leaves both waves empty, which has to be - # guarded: fromJSON('[]') produces a job with no combinations. - build_instrumented_stages_wave1: - name: Build Instrumented Stages (wave 1) - needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] - if: >- - ${{ - !failure() && !cancelled() && - needs.setup_coverage_matrix.outputs.stage_wave1_json != '[]' - }} - strategy: - fail-fast: false - matrix: - stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.stage_wave1_json) }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: ${{ matrix.stage_info.stage_name }} - stage_display_name: ${{ matrix.stage_info.stage_display_name }} - timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal - dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: >- - ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} - -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - release_type: ci - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - - build_instrumented_stages_wave2: - name: Build Instrumented Stages (wave 2) - needs: - [ - setup_coverage_matrix, - build_instrumented_compiler_runtime, - build_instrumented_stages_wave1, - ] - if: >- - ${{ - !failure() && !cancelled() && - needs.setup_coverage_matrix.outputs.stage_wave2_json != '[]' - }} - strategy: - fail-fast: false - matrix: - stage_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.stage_wave2_json) }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: ${{ matrix.stage_info.stage_name }} - stage_display_name: ${{ matrix.stage_info.stage_display_name }} - timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal - dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: >- - ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} - -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - release_type: ci - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - # The whole stage is built, not just the onboarded projects: the test jobs # below need the complete dependency set, and phase 1 instruments every # onboarded project in one pass rather than per changed project. @@ -297,11 +224,13 @@ jobs: # even though the build instrumented all of them at once. coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) + # compiler-runtime is named explicitly rather than relied on through + # math-libs: a selection that needs no math-libs build still tests against + # binaries that stage produced. needs: [ setup_coverage_matrix, - build_instrumented_stages_wave1, - build_instrumented_stages_wave2, + build_instrumented_compiler_runtime, build_instrumented_math_libs, ] # `!failure() && !cancelled()` alone would let this run when @@ -344,8 +273,6 @@ jobs: needs: - setup_coverage_matrix - build_instrumented_compiler_runtime - - build_instrumented_stages_wave1 - - build_instrumented_stages_wave2 - build_instrumented_math_libs - coverage_report runs-on: ubuntu-24.04 diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 3a1c306c7e9..7d8a999e311 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -37,7 +37,6 @@ sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) from github_actions_api import gha_set_output -from _therock_utils.build_topology import get_topology DEFAULT_AMDGPU_FAMILIES = "gfx94X-dcgpu" DEFAULT_COVERAGE_CONFIG_SOURCE = "ROCm/rocm-libraries@main" @@ -48,24 +47,28 @@ ROCM_SYSTEMS = "rocm-systems" # Build stages that can carry an instrumented project. compiler-runtime is -# always built because every other stage takes its inbound artifacts from it; -# the rest are built only when the selection needs them. +# always built because every other stage takes its inbound artifacts from it. STAGE_COMPILER_RUNTIME = "compiler-runtime" STAGE_MATH_LIBS = "math-libs" STAGE_COMM_LIBS = "comm-libs" STAGE_PROFILER_APPS = "profiler-apps" -# Display names for the per-stage build jobs, so the Actions UI reads the same -# way as the regular nightly's. Stages a project can be registered against are -# validated against these keys; emulation is here because build_stage_plan can -# pull it in as a dependency of comm-libs, not because a project lives there. -STAGE_DISPLAY_NAMES: dict[str, str] = { - STAGE_COMPILER_RUNTIME: "Coverage Compiler Runtime", - STAGE_MATH_LIBS: "Coverage Math Libs", - STAGE_COMM_LIBS: "Coverage Comm Libs", - STAGE_PROFILER_APPS: "Coverage Profiler Apps", - "emulation": "Coverage Emulation", -} +# Stages a project may be registered against, validated below. +KNOWN_STAGES = frozenset( + { + STAGE_COMPILER_RUNTIME, + STAGE_MATH_LIBS, + STAGE_COMM_LIBS, + STAGE_PROFILER_APPS, + } +) + +# Phase 1 has build jobs for these two only. The jobs that built any other +# stage were removed while their sole measurable consumers -- rccl and rocshmem +# in comm-libs -- are blocked upstream, so a selection reaching further is +# rejected by resolve_build_stages() rather than failing hours later on a +# missing inbound artifact. +BUILDABLE_STAGES = frozenset({STAGE_COMPILER_RUNTIME, STAGE_MATH_LIBS}) @dataclass(frozen=True) @@ -513,8 +516,8 @@ class CoverageProject: # A project in a stage the workflow cannot build would be selectable and # then silently tested against uninstrumented binaries. assert ( - _proj.stage in STAGE_DISPLAY_NAMES - ), f"{_key}: stage must be one of {sorted(STAGE_DISPLAY_NAMES)}" + _proj.stage in KNOWN_STAGES + ), f"{_key}: stage must be one of {sorted(KNOWN_STAGES)}" # An entry with neither would be selectable and instrument nothing; one # with both leaves it ambiguous whether the project can be measured. assert bool(_proj.coverage_option) != bool( @@ -529,6 +532,12 @@ class CoverageProject: SUPPORTED_PROJECTS: frozenset[str] = frozenset( k for k, v in COVERAGE_PROJECTS.items() if v.coverage_option ) +# What an empty selection defaults to: everything measurable that this workflow +# also has a build job for. Naming a project outside BUILDABLE_STAGES stays an +# error, so the narrowing applies only to the default, never to an explicit ask. +DEFAULT_PROJECTS: frozenset[str] = frozenset( + k for k in SUPPORTED_PROJECTS if COVERAGE_PROJECTS[k].stage in BUILDABLE_STAGES +) # Group membership is limited to what can actually be measured, so a group # alias never schedules a job that is guaranteed to fail for want of profiles. ROCM_LIBRARIES_PROJECTS: frozenset[str] = frozenset( @@ -557,10 +566,15 @@ def parse_projects(raw_projects: str) -> list[str]: Recognizes the group aliases 'rocm_libraries_all', 'rocm_systems_all', and 'all', which expand to a whole component group and may be mixed with explicit project names. Matching is case-insensitive. + + An empty selection means every project this workflow can build, which is + narrower than 'all' while comm-libs has no build job. Asking for those + projects by name or via a group alias still fails in + resolve_build_stages(), so nothing is dropped from an explicit request. """ requested = [p.strip().lower() for p in raw_projects.split(",") if p.strip()] if not requested: - return sorted(SUPPORTED_PROJECTS) + return sorted(DEFAULT_PROJECTS) expanded: list[str] = [] unknown: list[str] = [] @@ -690,70 +704,27 @@ def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: return options -def _stage_display_name(stage: str) -> str: - return f"Stage - {STAGE_DISPLAY_NAMES.get(stage, f'Coverage {stage}')}" - - -def build_stage_plan(project_keys: list[str]) -> tuple[list[list[dict]], bool]: - """Works out which build stages the selection needs, and in what order. +def resolve_build_stages(project_keys: list[str]) -> set[str]: + """Returns the stages the selection needs, rejecting what phase 1 cannot build. - Returns the generic (built-once) stages grouped into waves that can each - build in parallel, plus whether the per-architecture math-libs stage is - needed. compiler-runtime is excluded because it is built unconditionally: - every other stage takes its inbound artifacts from it, so there is no - selection that does not need it. - - A project's own stage is not the whole answer, which is why the topology is - consulted rather than just the registry. Selecting rccl asks for comm-libs, - but comm-libs takes hipify and rocjitsu from emulation, so emulation has to - be built too -- and built first, hence the waves. + Only compiler-runtime and math-libs have build jobs. A project from any + other stage would have nothing built for it, and that would not surface + until the test job found no instrumented files to overlay, hours in, so it + is an error here instead. """ - topology = get_topology() - - def producers(stage: str) -> set[str]: - return { - topology.get_stage_for_artifact(artifact) - for artifact in topology.get_inbound_artifacts(stage) - } - - selected = {COVERAGE_PROJECTS[key].stage for key in project_keys} - needs_math_libs = STAGE_MATH_LIBS in selected - - required: set[str] = set() - queue = list(selected) - while queue: - stage = queue.pop() - if stage in required: - continue - required.add(stage) - queue.extend(producers(stage) - required) - - # math-libs has its own per-architecture job, and compiler-runtime is - # always built, so neither belongs in the generic waves. Both are treated - # as already satisfied when ordering what is left. - built = {STAGE_COMPILER_RUNTIME, STAGE_MATH_LIBS} - remaining = required - built - - waves: list[list[dict]] = [] - while remaining: - wave = sorted(s for s in remaining if not (producers(s) - built - {s})) - if not wave: - # Dropping the stages silently would leave the build to fail much - # later, on a missing inbound artifact. - raise ValueError( - f"cannot order coverage build stages {sorted(remaining)}: " - "their inbound artifacts form a cycle" - ) - waves.append( - [ - {"stage_name": s, "stage_display_name": _stage_display_name(s)} - for s in wave - ] + stages = {COVERAGE_PROJECTS[key].stage for key in project_keys} + unbuildable = sorted(stages - BUILDABLE_STAGES) + if unbuildable: + blocked = sorted( + key for key in project_keys if COVERAGE_PROJECTS[key].stage in unbuildable ) - built.update(wave) - remaining -= set(wave) - - return waves, needs_math_libs + raise ValueError( + f"cannot build coverage for {', '.join(blocked)}: this workflow has " + f"build jobs for {', '.join(sorted(BUILDABLE_STAGES))} only, not " + f"{', '.join(unbuildable)}. Restore the staged build jobs in " + "multi_arch_ci_coverage_nightly.yml to select these projects." + ) + return stages def emit_cmake(output_path: Path) -> None: @@ -829,16 +800,7 @@ def main(argv: Optional[list[str]] = None) -> int: project_keys, amdgpu_families, config_repository, config_ref ) coverage_flags = build_coverage_cmake_options(project_keys) - stage_waves, needs_math_libs = build_stage_plan(project_keys) - # The workflow has one job per wave, and Actions cannot generate jobs, so a - # third wave has to be a build error here rather than a stage the workflow - # quietly skips. - if len(stage_waves) > 2: - raise ValueError( - f"build_stage_plan produced {len(stage_waves)} stage waves but " - "multi_arch_ci_coverage_nightly.yml only has jobs for two; add " - "another build_instrumented_generic_stages_* job to match" - ) + needs_math_libs = STAGE_MATH_LIBS in resolve_build_stages(project_keys) outputs = { "coverage_matrix": json.dumps(matrix), "dist_amdgpu_families": ";".join(amdgpu_families), @@ -846,8 +808,6 @@ def main(argv: Optional[list[str]] = None) -> int: [{"amdgpu_family": family} for family in amdgpu_families] ), "coverage_cmake_options": " ".join(coverage_flags), - "stage_wave1_json": json.dumps(stage_waves[0] if stage_waves else []), - "stage_wave2_json": json.dumps(stage_waves[1] if len(stage_waves) > 1 else []), "needs_math_libs": "true" if needs_math_libs else "false", } diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 868c9bd2a1a..244be12dedc 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -18,12 +18,27 @@ class ParseProjectsTest(unittest.TestCase): - def test_empty_selects_every_measurable_project(self): + def test_empty_selects_every_buildable_project(self): # Registered-but-unmeasurable projects are deliberately left out: an - # empty selection should not schedule a job that cannot report. + # empty selection should not schedule a job that cannot report. Nor + # should it pick projects whose stage this workflow cannot build. self.assertEqual( configure_coverage_ci.parse_projects(""), - sorted(configure_coverage_ci.SUPPORTED_PROJECTS), + sorted(configure_coverage_ci.DEFAULT_PROJECTS), + ) + + def test_default_selection_is_one_the_workflow_can_actually_build(self): + configure_coverage_ci.resolve_build_stages( + configure_coverage_ci.parse_projects("") + ) + + def test_default_is_narrower_than_all_while_comm_libs_has_no_build_job(self): + self.assertEqual( + sorted( + configure_coverage_ci.SUPPORTED_PROJECTS + - configure_coverage_ci.DEFAULT_PROJECTS + ), + ["rccl", "rocshmem"], ) def test_whitespace_and_case_are_normalized(self): @@ -231,61 +246,48 @@ def test_no_selection_produces_no_options(self): self.assertEqual(configure_coverage_ci.build_coverage_cmake_options([]), []) -class BuildStagePlanTest(unittest.TestCase): - @staticmethod - def _names(waves): - return [[entry["stage_name"] for entry in wave] for wave in waves] - - def test_math_libs_only_selection_needs_no_generic_stage(self): - waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["hiprand"]) - self.assertEqual(waves, []) - self.assertTrue(needs_math_libs) +class ResolveBuildStagesTest(unittest.TestCase): + def test_math_libs_project_needs_the_math_libs_stage(self): + self.assertEqual( + configure_coverage_ci.resolve_build_stages(["hiprand"]), + {"math-libs"}, + ) - def test_compiler_runtime_project_adds_no_stage_of_its_own(self): + def test_compiler_runtime_project_needs_no_math_libs_build(self): # compiler-runtime is built unconditionally, so a project living there - # must not also appear as a generic stage to build. - waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["amdsmi"]) - self.assertEqual(waves, []) - self.assertFalse(needs_math_libs) - - def test_generic_stage_project_pulls_in_what_that_stage_consumes(self): - # comm-libs takes hipify and rocjitsu from emulation, so selecting rccl - # has to build emulation as well -- and before comm-libs, or the build - # fails on a missing inbound artifact. - waves, needs_math_libs = configure_coverage_ci.build_stage_plan(["rccl"]) - self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) - self.assertFalse(needs_math_libs) - - def test_generic_stage_is_named_for_the_build_job(self): - waves, _ = configure_coverage_ci.build_stage_plan(["rccl"]) + # must not drag in the per-architecture math-libs job. self.assertEqual( - waves[-1], - [ - { - "stage_name": "comm-libs", - "stage_display_name": "Stage - Coverage Comm Libs", - } - ], + configure_coverage_ci.resolve_build_stages(["amdsmi"]), + {"compiler-runtime"}, ) - def test_stages_are_deduplicated_across_projects(self): - waves, _ = configure_coverage_ci.build_stage_plan(["rccl", "rocshmem"]) - self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) - - def test_selecting_everything_names_every_non_default_stage(self): - waves, needs_math_libs = configure_coverage_ci.build_stage_plan( - sorted(configure_coverage_ci.SUPPORTED_PROJECTS) + def test_whole_rocm_libraries_group_stays_within_math_libs(self): + keys = configure_coverage_ci.parse_projects("rocm_libraries_all") + self.assertEqual( + configure_coverage_ci.resolve_build_stages(keys), {"math-libs"} ) - self.assertEqual(self._names(waves), [["emulation"], ["comm-libs"]]) - self.assertTrue(needs_math_libs) - - def test_no_selection_needs_more_waves_than_the_workflow_has_jobs(self): - # multi_arch_ci_coverage_nightly.yml has two wave jobs. main() raises - # past that, but the registry should not get there in the first place. - waves, _ = configure_coverage_ci.build_stage_plan( - sorted(configure_coverage_ci.COVERAGE_PROJECTS) + + def test_comm_libs_project_is_rejected_with_a_actionable_message(self): + # comm-libs has no build job, so selecting rccl would leave the test + # job with nothing to overlay. It has to fail here instead. + with self.assertRaises(ValueError) as caught: + configure_coverage_ci.resolve_build_stages(["rccl"]) + message = str(caught.exception) + self.assertIn("rccl", message) + self.assertIn("comm-libs", message) + self.assertIn("multi_arch_ci_coverage_nightly.yml", message) + + def test_rejection_names_every_unbuildable_project_not_just_the_first(self): + with self.assertRaises(ValueError) as caught: + configure_coverage_ci.resolve_build_stages(["hiprand", "rccl", "rocshmem"]) + message = str(caught.exception) + self.assertIn("rccl", message) + self.assertIn("rocshmem", message) + + def test_every_buildable_stage_is_one_a_project_can_register_against(self): + self.assertTrue( + configure_coverage_ci.BUILDABLE_STAGES <= configure_coverage_ci.KNOWN_STAGES ) - self.assertLessEqual(len(waves), 2) class RegistryMatchesBuildTopologyTest(unittest.TestCase): @@ -336,10 +338,8 @@ def test_writes_expected_outputs(self): # The per-project test and overlay inputs the nightly reads back. self.assertIn('"test_component": "hiprand"', written) self.assertIn('"object_globs": "lib/libhiprand.so*"', written) - # hipRAND is a math-libs project, so no other stage is built. + # hipRAND is a math-libs project, so that stage has to be built. self.assertIn("needs_math_libs=true", written) - self.assertIn("stage_wave1_json=[]", written) - self.assertIn("stage_wave2_json=[]", written) class EmitCmakeTest(unittest.TestCase): diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 96f61ecb2f2..4a98b6941fc 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -193,13 +193,20 @@ this project's counters in the first place, which globs cannot undo because ### Selecting projects to run -A nightly run measures every onboarded project. Dispatching the workflow by hand -narrows that down: the `projects_to_test` input takes a comma-separated list of -project names, and also accepts three case-insensitive **group aliases** that -expand to a whole component group: `rocm_libraries_all`, `rocm_systems_all`, and -`all`. They may be mixed with explicit names, an empty input still means "every -project", and selecting a group with no onboarded projects fails the run with a -clear error rather than launching an empty matrix. +A nightly run measures every onboarded project the workflow can build. +Dispatching by hand narrows that down: the `projects_to_test` input takes a +comma-separated list of project names, and also accepts three case-insensitive +**group aliases** that expand to a whole component group: `rocm_libraries_all`, +`rocm_systems_all`, and `all`. They may be mixed with explicit names, and +selecting a group with no onboarded projects fails the run with a clear error +rather than launching an empty matrix. + +An empty input means every project with a build job, which today is +`rocm_libraries_all`. `rccl` and `rocshmem` are onboarded but live in +`comm-libs`, which this workflow does not build, so they are excluded from the +default — and naming them, or `rocm_systems_all` or `all`, is rejected outright. +The narrowing applies only to the default; an explicit request is never silently +reduced. See [Adding a stage](#adding-a-stage). The aliases are expanded to concrete project names inside `configure_coverage_ci.py` before the job matrix is built, so nothing downstream @@ -243,6 +250,16 @@ install tree. For most projects that is the shared library installed test binaries instead. Getting it wrong fails the report job, which refuses to run when no glob matches rather than publishing zero coverage. -A project whose stage is not yet built by `multi_arch_ci_coverage_nightly.yml` -also needs that stage's build job added there; today it builds compiler-runtime -and math-libs. +### Adding a stage + +`multi_arch_ci_coverage_nightly.yml` builds compiler-runtime and math-libs, and +`configure_coverage_ci.py` rejects any selection that reaches beyond them +(`BUILDABLE_STAGES`). That covers 17 of the 19 measurable projects. + +Onboarding a project from another stage needs a build job for that stage added +to the workflow, and the stage added to `BUILDABLE_STAGES`. Order matters: a +stage consumes inbound artifacts from other stages, so anything it depends on +has to be built first. `comm-libs`, the stage `rccl` and `rocshmem` need, takes +`hipify` and `rocjitsu` from `emulation` — so it needs two jobs, emulation then +comm-libs, not one. Consult `build_topology` rather than assuming; a missing +inbound artifact does not surface until well into the build. diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md index f83772fe276..f2d3df63b25 100644 --- a/docs/development/code_coverage_flows.md +++ b/docs/development/code_coverage_flows.md @@ -4,7 +4,7 @@ An end-to-end walk through what happens between dispatching a coverage run and a report landing in Codecov. See [Code Coverage](code_coverage.md) for enabling coverage on a local build, the CMake options, and how to onboard a project. -The short version: a coverage run builds every stage its selection needs, +The short version: a coverage run builds the stages its selection needs, instruments only the selected projects, and reports on each of them separately from that one run. @@ -12,10 +12,8 @@ from that one run. graph TD dispatch[Manual dispatch with projects_to_test] --> matrix[setup_coverage_matrix] matrix --> compilerRuntime[Build instrumented compiler-runtime] - compilerRuntime --> generic[Build other needed stages] compilerRuntime --> mathLibs[Build instrumented math-libs] - generic --> report[Per project: configure, test, report] - mathLibs --> report + mathLibs --> report[Per project: configure, test, report] report --> codecov[Codecov] ``` @@ -34,9 +32,14 @@ names, or one of the group aliases. The rest can be left at their defaults — t than the nightly's because instrumented tests already run several times longer than normal ones. -Leaving `projects_to_test` empty selects every onboarded project. That works, -but it instruments all of them in one build, which is the expensive way to run -this. Name what you want measured. +Leaving `projects_to_test` empty selects every onboarded project this workflow +can build. That works, but it instruments all of them in one build, which is the +expensive way to run this. Name what you want measured. + +`rccl` and `rocshmem` are the exception. They live in `comm-libs`, which has no +build job here, so naming them — or `rocm_systems_all` or `all` — is rejected by +`setup_coverage_matrix` rather than failing hours later with nothing to overlay. +See [Adding a stage](code_coverage.md#adding-a-stage). ## The regular nightly @@ -49,8 +52,9 @@ The dispatch creates a workflow run with its own id, status, and logs. `setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads `PROJECTS_TO_TEST`, `AMDGPU_FAMILIES` and `COVERAGE_CONFIG_SOURCE` and emits the -per-project job matrix, the family list, `coverage_cmake_options`, and the build -plan: `generic_stages_json` and `needs_math_libs`. +per-project job matrix, the family list, `coverage_cmake_options`, and +`needs_math_libs`. It also rejects any selection naming a project whose stage +this workflow has no build job for. `build_instrumented_compiler_runtime` runs first and unconditionally. Every other stage takes its inbound artifacts from it — all 19 of `math-libs`' inbound @@ -58,15 +62,12 @@ artifacts are produced there — so no selection can skip it. It also receives t coverage flags, since several rocm-systems projects (`rocprofiler-sdk`, `aqlprofile`, `amdsmi`, `rocprofiler-compute`) are built in that stage. -`build_instrumented_generic_stages` then fans out over `generic_stages_json`, -the build-once stages the selection needs beyond compiler-runtime — `comm-libs` -for rccl and rocshmem, `profiler-apps` for rocprofiler-systems. It is skipped -entirely for a selection confined to math-libs. - `build_instrumented_math_libs` fans out over the GPU families, and is skipped -when `needs_math_libs` is false. +when `needs_math_libs` is false. It is the only other stage built: 17 of the 19 +measurable projects live there, and the remaining two are blocked upstream, so +phase 1 carries no job for any further stage. -Every stage gets `coverage_cmake_options` appended to its configure line. +Both stages get `coverage_cmake_options` appended to their configure line. `configure_coverage_ci.py` collapses a selection covering a whole group into that group's option, so a full run reads `-DTHEROCK_COVERAGE_ALL=ON` while a narrow one reads `-DHIPRAND_ENABLE_COVERAGE=ON`. `CMakeLists.txt` expands the From 515f8d682b35e6be203b528225512bcdad94c152 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 15:28:02 -0500 Subject: [PATCH 16/70] ci(coverage): widen the PR check to the compiled rocm-libraries Run 34135220913 proved the path end to end on hipRAND once the profile runtime's own dependencies were added to the link line: instrumented build, baseline install, overlay, tests, and a 77% line report. The narrowing to a single project has served its purpose. Select the 13 rocm-libraries projects that build a shared library. The four header-only ones are left out: they have no library of their own to instrument, so they report against instrumented test binaries, and their inline code is the linkonce_odr case the overlay cannot fully isolate. That is worth doing, but worth doing on its own. rccl and rocshmem are not selectable at all now that comm-libs has no build job, and both are blocked upstream regardless. hipBLASLt is included deliberately despite failing to link hipblaslt-test with "unable to find library -lrocroller" in run 34119302776. Whether that is coverage-related was never established, and this is how we find out. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 042098133a0..a296b86c0a2 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -73,10 +73,11 @@ jobs: # only installs the baseline on the test runner, where prebuilt binaries # are executed and no package config is read. # - # Verified before pinning that this run publishes gfx942 artifacts for - # every selected project -- blas, fft, hipdnn, hiptensor, prim, rand, - # rccl, rocshmem, rocwmma, solver and sparse, plus the _test archives the - # header-only projects report against. + # Verified before pinning that this run publishes artifacts covering + # every selected project: blas, fft, hipdnn, hiptensor, rand, solver and + # sparse. Not all are per-target -- hipdnn is generic throughout and + # solver's tests are generic -- so checking only for _gfx942 understates + # what is there. baseline_run_id: "34068476531" # The bucket lookup queries the run through the GitHub API, so it has to # be told where the run lives; ROCm/TheRock returns 404 for a rockrel id. @@ -85,23 +86,30 @@ jobs: # but naming it keeps the pin and its channel in one place. baseline_release_type: "nightly" amdgpu_families: "gfx94X-dcgpu" - # hipRAND alone while the baseline and overlay path is being proven. The - # phase 1 scope is the whole registry, and the workflow supports it, but - # run 34119302776 showed the stack is not ready to be instrumented all at - # once: + # Every rocm-libraries project that builds a shared library. Run + # 34135220913 proved the path end to end on hipRAND -- instrumented + # build, baseline install, overlay, tests and a 77% report -- once + # libclang_rt.profile_rocm.a's own dependencies (-ldl, -lpthread) were + # added to the link line, which had been failing every instrumented + # binary before that. # - # - libclang_rt.profile_rocm.a is linked without -ldl or -lpthread, so - # any instrumented executable fails to resolve dlsym, dladdr and - # pthread_once. Shared libraries tolerate the undefined symbols, so - # this only bites projects that build their own binaries: rocRAND's - # tools, and hipCUB's and rocPRIM's tests. - # - rccl's CMake refuses coverage outside a Debug build, and TheRock - # builds Release. - # - rocshmem does not survive coverage together with HIP relocatable - # device code ("unknown special variable with appending linkage"). + # Named explicitly rather than using rocm_libraries_all because the four + # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA) are left + # out. They have no library of their own to instrument, so they report + # against instrumented test binaries instead, and their inline code is + # the linkonce_odr case the overlay cannot fully isolate. Worth doing, + # but worth doing separately. # - # The rocm-systems projects (rccl, rocshmem) are out until the second and - # third are addressed; the rest of math-libs waits on the first. hipRAND - # is unaffected -- it built cleanly instrumented in run 34098007786. - projects_to_test: "hiprand" + # rccl and rocshmem are absent because they are not selectable: they + # live in comm-libs, which has no build job here. Both are also blocked + # upstream -- rccl's CMake refuses coverage outside a Debug build, and + # rocshmem fails on coverage plus HIP relocatable device code. + # + # hipBLASLt is included deliberately even though it failed to link + # hipblaslt-test with "unable to find library -lrocroller" in run + # 34119302776. Whether that is coverage-related was never established, + # and this run is how we find out. + projects_to_test: >- + hipblas,hipblaslt,hipdnn,hipfft,hiprand,hipsolver,hipsparselt, + hiptensor,rocblas,rocfft,rocrand,rocsolver,rocsparse test_type: "standard" From 4d4f8f7a89c2a0a62347b1bd0b220bef3158f340 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 7 Sep 2026 16:47:48 -0500 Subject: [PATCH 17/70] coverage: take hipBLASLt out of the automatic selections Run 34159520920 answered the question the last commit set out to ask: the hipBLASLt link failure is caused by coverage, and the cause is upstream. The coverage-only branch of hipblaslt's clients/CMakeLists.txt links hipblaslt-test against a bare `rocroller` while the lines around it use imported targets like roc::tensilelite-host. rocRoller exports under roc::, and in TheRock it is a separate subproject consumed through its package config, so `rocroller` is not a target in hipBLASLt's build and survives to the linker as -lrocroller with no -L to resolve it. hipBLASLt's standalone build has rocRoller in-tree and so never notices. Nothing in TheRock can fix this: rocm-libraries is a pinned submodule and patches/ only covers third-party. Confirmed against the non-coverage build of the same commit, which succeeds and never links rocroller into hipblaslt-test at all, because the whole block is behind HIPBLASLT_ENABLE_COVERAGE. Rather than only dropping it from the PR check, which would leave the nightly's own default selection broken, record it as a blocked_reason. That is a new registry field for a project that is measurable but whose instrumented build fails, as distinct from unsupported_reason, which means it cannot be measured at all and is not expected to change. A blocked project leaves the default selection and every group alias, including the group lists --emit-cmake writes, so a group build cannot set a flag the Python side just declined to. It stays selectable by name, with the reason printed as a warning: rerunning it is the only way to find out the block has been lifted. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 9 +-- .../github_actions/configure_coverage_ci.py | 61 ++++++++++++++-- .../tests/configure_coverage_ci_test.py | 73 +++++++++++++++++-- docs/development/code_coverage.md | 24 ++++++ 4 files changed, 151 insertions(+), 16 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index a296b86c0a2..ea4a91cf75e 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -105,11 +105,10 @@ jobs: # upstream -- rccl's CMake refuses coverage outside a Debug build, and # rocshmem fails on coverage plus HIP relocatable device code. # - # hipBLASLt is included deliberately even though it failed to link - # hipblaslt-test with "unable to find library -lrocroller" in run - # 34119302776. Whether that is coverage-related was never established, - # and this run is how we find out. + # hipBLASLt is absent for the same reason it is absent from the default + # selection: its instrumented build does not link. See its + # blocked_reason in configure_coverage_ci.py. projects_to_test: >- - hipblas,hipblaslt,hipdnn,hipfft,hiprand,hipsolver,hipsparselt, + hipblas,hipdnn,hipfft,hiprand,hipsolver,hipsparselt, hiptensor,rocblas,rocfft,rocrand,rocsolver,rocsparse test_type: "standard" diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 7d8a999e311..8c0ea849a46 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -90,6 +90,12 @@ class CoverageProject: cannot be measured by this pipeline. Such a project stays in the registry so the gap is recorded, but is left out of the group aliases and rejected if named explicitly. + blocked_reason: Set alongside coverage_option when the project is + measurable in principle but its instrumented build is currently + broken. Unlike unsupported_reason this is expected to be + temporary, so the project only drops out of the default selection + and the group aliases; naming it still works, which is how the + block gets retested once the fix lands. artifact_names: BUILD_TOPOLOGY artifact(s) the instrumented project ships in. These are grouped (`rand` holds both rocRAND and hipRAND), which is why artifact_relpaths exists. @@ -123,6 +129,7 @@ class CoverageProject: coverage_config: str coverage_option: str = "" unsupported_reason: str = "" + blocked_reason: str = "" artifact_names: list[str] = field(default_factory=list) artifact_relpaths: list[str] = field(default_factory=list) object_globs: list[str] = field(default_factory=list) @@ -229,6 +236,17 @@ class CoverageProject: artifact_names=["blas"], artifact_relpaths=["math-libs/BLAS/hipBLASLt/stage"], coverage_option="HIPBLASLT_ENABLE_COVERAGE", + # The coverage-only branch of clients/CMakeLists.txt links hipblaslt-test + # against a bare `rocroller` while its neighbours use the imported + # targets. rocRoller exports as roc::rocroller, and under TheRock it is a + # separate subproject found via its package config, so the bare name is + # not a target here and reaches the linker as -lrocroller with no -L to + # resolve it. hipBLASLt's own build gets away with it by having rocRoller + # in-tree. Fix belongs upstream in ROCm/rocm-libraries. + blocked_reason=( + "hipblaslt-test fails to link with 'unable to find library " + "-lrocroller'; its coverage build needs roc::rocroller upstream" + ), stage=STAGE_MATH_LIBS, test_component="hipblaslt", coverage_config="projects/hipblaslt/test_categories_coverage.yaml", @@ -532,23 +550,31 @@ class CoverageProject: SUPPORTED_PROJECTS: frozenset[str] = frozenset( k for k, v in COVERAGE_PROJECTS.items() if v.coverage_option ) +# Measurable, but the instrumented build is broken today. Kept selectable by +# name so the block can be retested, and kept out of everything that picks +# projects on the caller's behalf. +BLOCKED_PROJECTS: frozenset[str] = frozenset( + k for k, v in COVERAGE_PROJECTS.items() if v.blocked_reason +) # What an empty selection defaults to: everything measurable that this workflow # also has a build job for. Naming a project outside BUILDABLE_STAGES stays an # error, so the narrowing applies only to the default, never to an explicit ask. DEFAULT_PROJECTS: frozenset[str] = frozenset( - k for k in SUPPORTED_PROJECTS if COVERAGE_PROJECTS[k].stage in BUILDABLE_STAGES + k + for k in SUPPORTED_PROJECTS - BLOCKED_PROJECTS + if COVERAGE_PROJECTS[k].stage in BUILDABLE_STAGES ) # Group membership is limited to what can actually be measured, so a group # alias never schedules a job that is guaranteed to fail for want of profiles. ROCM_LIBRARIES_PROJECTS: frozenset[str] = frozenset( k for k, v in COVERAGE_PROJECTS.items() - if v.source_repo == ROCM_LIBRARIES and v.coverage_option + if v.source_repo == ROCM_LIBRARIES and v.coverage_option and not v.blocked_reason ) ROCM_SYSTEMS_PROJECTS: frozenset[str] = frozenset( k for k, v in COVERAGE_PROJECTS.items() - if v.source_repo == ROCM_SYSTEMS and v.coverage_option + if v.source_repo == ROCM_SYSTEMS and v.coverage_option and not v.blocked_reason ) # Group-alias tokens accepted by PROJECTS_TO_TEST (and the projects_to_test CI @@ -556,7 +582,7 @@ class CoverageProject: _GROUP_ALIASES: dict[str, frozenset[str]] = { "rocm_libraries_all": ROCM_LIBRARIES_PROJECTS, "rocm_systems_all": ROCM_SYSTEMS_PROJECTS, - "all": SUPPORTED_PROJECTS, + "all": SUPPORTED_PROJECTS - BLOCKED_PROJECTS, } @@ -571,6 +597,11 @@ def parse_projects(raw_projects: str) -> list[str]: narrower than 'all' while comm-libs has no build job. Asking for those projects by name or via a group alias still fails in resolve_build_stages(), so nothing is dropped from an explicit request. + + Projects with a known-broken instrumented build are likewise left out of + the default and the aliases, but naming one is honoured with a warning + rather than an error: retesting a block is the only way to notice it has + been fixed. """ requested = [p.strip().lower() for p in raw_projects.split(",") if p.strip()] if not requested: @@ -615,7 +646,18 @@ def parse_projects(raw_projects: str) -> list[str]: "they emit LLVM profraw data." ) # Preserve first-seen order, drop duplicates (explicit + alias overlap). - return list(dict.fromkeys(expanded)) + selection = list(dict.fromkeys(expanded)) + + # Not fatal: the caller went out of their way to name these, and the only + # way to find out a block has been lifted is to run into it again. + for name in selection: + blocked_reason = COVERAGE_PROJECTS[name].blocked_reason + if blocked_reason: + print( + f"warning: {name} is expected to fail: {blocked_reason}", + file=sys.stderr, + ) + return selection def parse_amdgpu_families(raw_families: str) -> list[str]: @@ -732,7 +774,9 @@ def emit_cmake(output_path: Path) -> None: Two things come out of here. The group lists drive the THEROCK_COVERAGE_*_ALL options, and are limited to projects that can be - measured. The option map tells therock_subproject.cmake which flag each + measured and are not currently blocked. The option map is not: a blocked + project stays in it so that naming it explicitly still instruments it. + The option map tells therock_subproject.cmake which flag each subproject actually implements, since the names are not standardised upstream. CMakeLists.txt reads both so it does not hardcode either. """ @@ -742,6 +786,11 @@ def targets(repo: str) -> list[str]: for p in COVERAGE_PROJECTS.values(): if p.source_repo != repo or not p.coverage_option: continue + # Kept in step with the Python group sets: a group flag that + # instrumented a project whose instrumented build is known to fail + # would take the whole stage down with it. + if p.blocked_reason: + continue # extra_cmake_targets are part of the group: a group flag that # instrumented rocPRIM but not rocPRIM_tests would leave rocPRIM's # only reportable binaries without a coverage mapping. diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 244be12dedc..cb106777148 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -1,6 +1,8 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT +import contextlib +import io import json import os import sys @@ -32,15 +34,44 @@ def test_default_selection_is_one_the_workflow_can_actually_build(self): configure_coverage_ci.parse_projects("") ) - def test_default_is_narrower_than_all_while_comm_libs_has_no_build_job(self): + def test_default_leaves_out_what_cannot_be_built_or_is_known_broken(self): self.assertEqual( sorted( configure_coverage_ci.SUPPORTED_PROJECTS - configure_coverage_ci.DEFAULT_PROJECTS ), - ["rccl", "rocshmem"], + # rccl and rocshmem for want of a comm-libs build job, hipblaslt + # because its instrumented build does not link. + ["hipblaslt", "rccl", "rocshmem"], ) + def test_blocked_project_is_left_out_of_every_alias(self): + # An alias is the caller delegating the choice, so it must not hand + # back a project whose build is known to fail. + for alias in ("all", "rocm_libraries_all", "rocm_systems_all"): + with self.subTest(alias=alias): + self.assertFalse( + configure_coverage_ci.BLOCKED_PROJECTS + & configure_coverage_ci._GROUP_ALIASES[alias] + ) + + def test_blocked_project_is_still_honoured_when_named(self): + # Retesting is the only way to notice the block has been lifted. + for name in sorted(configure_coverage_ci.BLOCKED_PROJECTS): + with self.subTest(project=name): + self.assertEqual(configure_coverage_ci.parse_projects(name), [name]) + + def test_naming_a_blocked_project_warns_with_the_reason(self): + for name in sorted(configure_coverage_ci.BLOCKED_PROJECTS): + with self.subTest(project=name): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + configure_coverage_ci.parse_projects(name) + self.assertIn( + configure_coverage_ci.COVERAGE_PROJECTS[name].blocked_reason, + stderr.getvalue(), + ) + def test_whitespace_and_case_are_normalized(self): self.assertEqual(configure_coverage_ci.parse_projects(" HipRand "), ["hiprand"]) @@ -57,7 +88,10 @@ def test_unknown_project_is_rejected(self): def test_all_alias_expands_to_every_measurable_project(self): self.assertEqual( configure_coverage_ci.parse_projects("all"), - sorted(configure_coverage_ci.SUPPORTED_PROJECTS), + sorted( + configure_coverage_ci.SUPPORTED_PROJECTS + - configure_coverage_ci.BLOCKED_PROJECTS + ), ) def test_rocm_libraries_all_expands_to_that_group(self): @@ -105,7 +139,10 @@ def test_groups_partition_the_measurable_projects(self): self.assertTrue(libraries.isdisjoint(systems)) self.assertEqual( libraries | systems, - set(configure_coverage_ci.SUPPORTED_PROJECTS), + set( + configure_coverage_ci.SUPPORTED_PROJECTS + - configure_coverage_ci.BLOCKED_PROJECTS + ), ) @@ -216,11 +253,22 @@ def test_full_selection_collapses_to_the_group_option(self): def test_both_groups_collapse_to_the_combined_option(self): self.assertEqual( configure_coverage_ci.build_coverage_cmake_options( - sorted(configure_coverage_ci.SUPPORTED_PROJECTS) + sorted( + configure_coverage_ci.SUPPORTED_PROJECTS + - configure_coverage_ci.BLOCKED_PROJECTS + ) ), ["-DTHEROCK_COVERAGE_ALL=ON"], ) + def test_blocked_project_is_named_rather_than_folded_into_a_group(self): + # It is outside the group lists, so the group flag would not reach it + # and the explicit ask would silently build uninstrumented. + self.assertEqual( + configure_coverage_ci.build_coverage_cmake_options(["hipblaslt"]), + ["-DHIPBLASLT_ENABLE_COVERAGE=ON"], + ) + def test_partial_selection_names_each_project_in_upper_case(self): # therock_subproject.cmake only forwards the upper case spelling. self.assertEqual( @@ -367,6 +415,21 @@ def test_emits_group_lists_from_allowlist(self): # flag their CMake does not implement. self.assertNotIn("MIOpen", text) self.assertNotIn("hipSPARSE ", text) + # The CMake group lists and the Python group sets decide the same + # thing in two places, so a blocked project has to leave both or a + # group build sets the flag the Python side just declined to. + self.assertNotIn("hipBLASLt", libraries_line) + + def test_blocked_project_keeps_its_option_mapping(self): + # Excluded from the group lists but still nameable, which needs the + # translation to its upstream option name to survive. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "therock_coverage_projects.cmake" + self.assertEqual(configure_coverage_ci.main(["--emit-cmake", str(out)]), 0) + self.assertIn( + "set(THEROCK_COVERAGE_OPTION_HIPBLASLT HIPBLASLT_ENABLE_COVERAGE)", + out.read_text(), + ) def test_emits_the_upstream_option_name_per_subproject(self): # The generic names are why this is a per-subproject map: setting diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 4a98b6941fc..532ec8cc300 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -208,6 +208,12 @@ default — and naming them, or `rocm_systems_all` or `all`, is rejected outrigh The narrowing applies only to the default; an explicit request is never silently reduced. See [Adding a stage](#adding-a-stage). +`hipblaslt` is excluded from the default and the aliases too, for a different +reason: it is measurable, but its instrumented build currently fails to link. +Naming it is honoured with a warning rather than rejected, since rerunning it +is how the block gets noticed as fixed. See +[Blocked projects](#blocked-projects). + The aliases are expanded to concrete project names inside `configure_coverage_ci.py` before the job matrix is built, so nothing downstream ever sees them: the coverage pipeline hands `fetch_test_configurations.py` @@ -234,6 +240,24 @@ A project that cannot be measured yet gets an `unsupported_reason` instead. It stays in the registry so the gap is recorded, stays out of the group aliases, and is rejected with that reason if someone names it. +### Blocked projects + +A project that is measurable but whose instrumented build is broken gets a +`blocked_reason` alongside its `coverage_option`. Unlike `unsupported_reason` +this is expected to be temporary, so the project is only dropped from the +default selection and the group aliases — including the group lists +`--emit-cmake` writes, so a group build does not set a flag the Python side +just declined to set. Naming it still works and prints the reason as a warning. + +`hipblaslt` is the current example. Its coverage-only branch of +`clients/CMakeLists.txt` links `hipblaslt-test` against a bare `rocroller` +while the surrounding lines use imported targets. rocRoller exports as +`roc::rocroller`, and under TheRock it is a separate subproject found through +its package config, so the bare name is not a target here and reaches the +linker as `-lrocroller` with no `-L` to resolve it. hipBLASLt's own build +avoids this by having rocRoller in-tree. The fix belongs upstream in +`ROCm/rocm-libraries`; once it lands, drop the `blocked_reason`. + Confirm a local instrumented build produces a non-empty report before relying on a new entry. A project whose tests never load the instrumented library builds and tests cleanly and then reports nothing. From 2ece46af23603e6fe19456ae9e27aabbca0d69ae Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 8 Sep 2026 02:03:00 -0500 Subject: [PATCH 18/70] coverage: measure host code only, and fix two wrong registry entries Run 34164420766 built all twelve projects and reported six of them. The six failures were three separate problems. Keep the instrumentation off the GPU. The upstream coverage options pass -fprofile-instr-generate and -fcoverage-mapping unqualified and the HIP driver forwards both to the device compilation, so kernels were instrumented as well. rocSPARSE showed what that costs: all 22683 tests pass, then the HIP runtime aborts with "Cannot create GlobalVar Obj for symbol: __profd_..." on a rocPRIM trampoline kernel, because libclang_rt.profile_rocm looks the device profile symbols up with hipModuleGetGlobal at exit. rocBLAS and rocSOLVER are the same story without the diagnostic: 101618 and 20514 tests pass, then a segfault where the profile is written. All four also wrote a device profile named gfx942:sramecc+:xnack-.0., which upload-artifact rejects for the colons. -Xarch_device confines the negation to the device compilation. Checked against the ROCm clang in this tree: with it, the device image loses __profd_, __profc_ and the per-TU __llvm_profile_sections_ symbol the runtime enumerates, while the host keeps -fprofile-instrument=clang and its coverage mapping. Order is what makes this work, so it goes on the compile rule after rather than in CMAKE__FLAGS, which the project's own COMPILE_OPTIONS would otherwise override. The rule is not set yet in CMAKE_PROJECT_TOP_LEVEL_INCLUDES, so it needs its own file passed as CMAKE_PROJECT_INCLUDE. hipDNN's object glob named a library that does not exist. Its shared library is libhipdnn_backend.so, as TheRock's own packaging metadata already recorded. Profiles were being collected the whole time and thrown away for want of anything to attribute them to. hipTensor's option is HIPTENSOR_CODE_COVERAGE, not the bare CODE_COVERAGE its neighbour rocALUTION uses. Passing the wrong name is silent, which is exactly how this got missed: the library built uninstrumented and the tests passed having written no profile at all. Finally, give coverage runs four times the test budget. Instrumented builds are -O0 -g and count every branch, so rocRAND tests that normally take milliseconds took 25 to 60 seconds each and the shard hit the 15 minute wall a long way from finishing. Co-authored-by: Cursor --- .github/workflows/test_component.yml | 15 ++++- .../github_actions/configure_coverage_ci.py | 9 ++- cmake/therock_subproject.cmake | 54 +++++++++++++++++ docs/development/code_coverage.md | 59 +++++++++++++++++-- docs/development/code_coverage_flows.md | 5 +- 5 files changed, 131 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index 8b12370f003..23637ef4f9a 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -140,6 +140,17 @@ jobs: # artifacts are bucketed by channel. INSTALL_ARTIFACT_RUN_ID: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_run_id || (inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id) }}" INSTALL_RELEASE_TYPE: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_release_type || inputs.release_type }}" + # Instrumented libraries are built -O0 -g and counted on every branch, so + # the same suite takes far longer than the per-component budget assumes: + # rocRAND tests that normally run in milliseconds took 25 to 60 seconds + # each, and the shard hit its 15 minute wall a long way from finishing. + # + # A floor rather than a multiplier, because the budgets are not written to + # a common scale -- rocBLAS asks for 288 minutes, past the job's own cap, + # while rocRAND asks for 15 -- so multiplying stretches the ones that were + # already generous and still leaves the tight ones tight. Only the floor + # moves, and only for coverage. + TEST_TIMEOUT_MINUTES: "${{ (inputs.coverage_enabled && fromJSON(inputs.component).timeout_minutes < 60) && 60 || fromJSON(inputs.component).timeout_minutes }}" OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -280,7 +291,7 @@ jobs: - name: Test id: test - timeout-minutes: ${{ fromJSON(inputs.component).timeout_minutes }} + timeout-minutes: ${{ env.TEST_TIMEOUT_MINUTES }} env: SHARD_INDEX: ${{ matrix.shard }} TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }} @@ -297,7 +308,7 @@ jobs: RETRY_THIS_STEP: "true" RETRY_COUNT: "1" RETRY_DELAY: "7" - STEP_TIMEOUT_MINUTES: ${{ fromJSON(inputs.component).timeout_minutes }} + STEP_TIMEOUT_MINUTES: ${{ env.TEST_TIMEOUT_MINUTES }} STEP_NAME: >- Test ${{ fromJSON(inputs.component).job_name }} (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }}) diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 8c0ea849a46..0d889c728c3 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -334,7 +334,10 @@ class CoverageProject: cmake_target="hipTensor", artifact_names=["hiptensor"], artifact_relpaths=["math-libs/hipTensor/stage"], - coverage_option="CODE_COVERAGE", + # Prefixed, unlike rocALUTION's bare CODE_COVERAGE just above. Getting + # this wrong is silent: the option is simply never read, the library + # builds uninstrumented, and the tests pass having written no profile. + coverage_option="HIPTENSOR_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="hiptensor", coverage_config="projects/hiptensor/test_categories_coverage.yaml", @@ -362,7 +365,9 @@ class CoverageProject: stage=STAGE_MATH_LIBS, test_component="hipdnn", coverage_config="projects/hipdnn/test_categories_coverage.yaml", - object_globs=["lib/libhipdnn.so*"], + # Not libhipdnn.so: hipDNN's shared library is the backend, which is + # also the name TheRock's own packaging metadata records for it. + object_globs=["lib/libhipdnn_backend.so*"], fetch_artifact_args="--hipdnn", codecov_flag="hipDNN", ), diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 54adf17f8c6..37b1ba3a552 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -528,6 +528,7 @@ function(therock_cmake_subproject_declare target_name) THEROCK_PREFIX_DIR "${_prefix_dir}" THEROCK_CMAKE_SOURCE_DIR "${_cmake_source_dir}" THEROCK_CMAKE_PROJECT_INIT_FILE "${ARG_BINARY_DIR}/${ARG_BUILD_DIR}_init.cmake" + THEROCK_CMAKE_PROJECT_COVERAGE_FILE "${ARG_BINARY_DIR}/${ARG_BUILD_DIR}_coverage.cmake" THEROCK_CMAKE_PROJECT_TOOLCHAIN_FILE "${ARG_BINARY_DIR}/${ARG_BUILD_DIR}_toolchain.cmake" THEROCK_CMAKE_ARGS "${ARG_CMAKE_ARGS}" THEROCK_CMAKE_INCLUDES "${ARG_CMAKE_INCLUDES}" @@ -648,6 +649,7 @@ function(therock_cmake_subproject_activate target_name) get_target_property(_cmake_args "${target_name}" THEROCK_CMAKE_ARGS) get_target_property(_cmake_includes "${target_name}" THEROCK_CMAKE_INCLUDES) get_target_property(_cmake_project_init_file "${target_name}" THEROCK_CMAKE_PROJECT_INIT_FILE) + get_target_property(_cmake_project_coverage_file "${target_name}" THEROCK_CMAKE_PROJECT_COVERAGE_FILE) get_target_property(_cmake_project_toolchain_file "${target_name}" THEROCK_CMAKE_PROJECT_TOOLCHAIN_FILE) get_target_property(_cmake_source_dir "${target_name}" THEROCK_CMAKE_SOURCE_DIR) get_target_property(_exclude_from_all "${target_name}" THEROCK_EXCLUDE_FROM_ALL) @@ -942,6 +944,56 @@ function(therock_cmake_subproject_activate target_name) "string(APPEND CMAKE_MODULE_LINKER_FLAGS \"${_coverage_link_flags}\")\n") endif() + # Keep the instrumentation off the GPU. + # + # The upstream coverage options pass -fprofile-instr-generate and + # -fcoverage-mapping unqualified, and the HIP driver forwards both to the + # device compilation, so kernels are instrumented too. That is not what this + # pipeline measures, and it does not survive contact with the runtime: the + # device image gains a per-TU __llvm_profile_sections_ symbol that + # libclang_rt.profile_rocm.a looks up with hipModuleGetGlobal at exit, and a + # __profd_ for every kernel. Symbols the runtime cannot resolve abort the + # process after the tests have already passed (rocSPARSE, on a rocPRIM + # trampoline kernel), and where the readback does succeed it writes a second + # profile named after the device -- gfx942:sramecc+:xnack-.0. -- + # which cannot be uploaded as an artifact because of the colons. + # + # -Xarch_device confines the negation to the device compilation; the host + # keeps -fprofile-instrument=clang and its coverage mapping. Order decides + # this, since the last of a -f/-fno- pair wins and the project's own flags + # come from COMPILE_OPTIONS, which land after CMAKE__FLAGS. Appending + # to the compile rule after is the one position that is reliably + # last. The rule does not exist yet in CMAKE_PROJECT_TOP_LEVEL_INCLUDES, + # which runs before any language is enabled, hence the separate file below. + # + # This reaches the languages a project() call enables, which is where the + # HIP sources in these projects are compiled -- as CXX, through the amd-hip + # toolchain. A language brought up later by a bare enable_language() is not + # covered, and would need the same rewrite from wherever that happens. + set(_coverage_project_include_arg) + set(_coverage_configure_depends) + if(NOT MSVC AND ${_coverage_var_name}) + set(_coverage_host_only_contents + "# Generated by therock_subproject.cmake; see the coverage comment there.\n") + # Runs on every project() call, so the rewrite has to be idempotent. + # -Wno-unused-command-line-argument because -Xarch_device is unused for + # the plain C++ sources that make up most of these projects, and several + # of them build with -Werror. + foreach(_coverage_lang IN ITEMS C CXX HIP) + string(APPEND _coverage_host_only_contents + "if(CMAKE_${_coverage_lang}_COMPILE_OBJECT AND NOT CMAKE_${_coverage_lang}_COMPILE_OBJECT MATCHES \"fno-profile-instr-generate\")\n" + " string(REPLACE \"\" \" -Xarch_device -fno-profile-instr-generate -Xarch_device -fno-coverage-mapping -Wno-unused-command-line-argument\"\n" + " CMAKE_${_coverage_lang}_COMPILE_OBJECT \"\${CMAKE_${_coverage_lang}_COMPILE_OBJECT}\")\n" + "endif()\n") + endforeach() + file(CONFIGURE OUTPUT "${_cmake_project_coverage_file}" + CONTENT "${_coverage_host_only_contents}" @ONLY ESCAPE_QUOTES) + list(APPEND _fprint_files "${_cmake_project_coverage_file}") + set(_coverage_project_include_arg + "-DCMAKE_PROJECT_INCLUDE=${_cmake_project_coverage_file}") + set(_coverage_configure_depends "${_cmake_project_coverage_file}") + endif() + file(CONFIGURE OUTPUT "${_cmake_project_init_file}" CONTENT "${_init_contents}" @ONLY ESCAPE_QUOTES) list(APPEND _fprint_files "${_cmake_project_init_file}") list(APPEND _fprint_files "${ROCM_BUILD_FLAGS_STATE_FILE}") @@ -1094,6 +1146,7 @@ function(therock_cmake_subproject_activate target_name) "-DTHEROCK_STAGE_INSTALL_ROOT=${_stage_dir}" "-DCMAKE_TOOLCHAIN_FILE=${_cmake_project_toolchain_file}" "-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${_cmake_project_init_file}" + ${_coverage_project_include_arg} ${_cmake_args} # CMake doesn't always generate a compile_commands.json so touch one to keep # the build graph sane. @@ -1111,6 +1164,7 @@ function(therock_cmake_subproject_activate target_name) "${_cmake_source_dir}/CMakeLists.txt" "${_cmake_project_toolchain_file}" "${_cmake_project_init_file}" + ${_coverage_configure_depends} "${_global_post_include}" "${ROCM_BUILD_FLAGS_STATE_FILE}" ${_extra_depends} diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 532ec8cc300..4002e6ae2fd 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -23,11 +23,25 @@ is responsible for translating it into compiler flags (usually `-fprofile-instr-generate -fcoverage-mapping`). Passing the flag for a project that does not implement it has no effect. -Device side profiling relies on the ROCm profiling runtime -(`libclang_rt.profile_rocm`) being present in the compiler build. TheRock does -not configure that from the top level; it comes from how amd-llvm's compiler-rt -is built. If a report shows host coverage only, check that the runtime is in the -compiler you built against. +This pipeline measures host code only, and deliberately keeps the +instrumentation off the GPU. The upstream coverage options pass +`-fprofile-instr-generate` and `-fcoverage-mapping` unqualified and the HIP +driver forwards both to the device compilation, so kernels would be +instrumented too. That does not survive contact with the runtime: the device +image gains a per-translation-unit `__llvm_profile_sections_` symbol that +`libclang_rt.profile_rocm` looks up with `hipModuleGetGlobal` at exit, plus a +`__profd_` for every kernel. A symbol the runtime cannot resolve aborts the +process after the tests have already passed, and where the readback succeeds it +writes a second profile named after the device +(`gfx942:sramecc+:xnack-.0.`) that cannot be uploaded as a CI +artifact because of the colons. + +`therock_subproject.cmake` therefore appends `-Xarch_device +-fno-profile-instr-generate -Xarch_device -fno-coverage-mapping` to the compile +rule of any subproject it enables coverage for. It has to be the compile rule +rather than `CMAKE__FLAGS`: the last of a `-f`/`-fno-` pair wins, and the +project's own flags arrive later, as `COMPILE_OPTIONS`. Measuring GPU code +would mean undoing this and solving the two problems above. That runtime calls `dlsym`, `dladdr` and `pthread_once`, and the driver does not put `-ldl` or `-lpthread` on the link line for it. On the manylinux base those @@ -74,6 +88,32 @@ so a group only ever contains onboarded projects; selecting a group with none > it also means editing `configure_coverage_ci.py` triggers a reconfigure), but > it only changes the build when one of the options is `ON`. +### Host code only + +What gets measured is host code. The upstream coverage options pass +`-fprofile-instr-generate` and `-fcoverage-mapping` unqualified and the HIP +driver forwards both to the device compilation, so kernels would be instrumented +too. `therock_subproject.cmake` cancels that with `-Xarch_device`, which is +scoped to the device compilation and leaves the host instrumentation alone. + +This is not just a matter of scope. An instrumented device image carries a +per-translation-unit `__llvm_profile_sections_` symbol that +`libclang_rt.profile_rocm.a` looks up through `hipModuleGetGlobal` when the +process exits, plus a `__profd_` for every kernel. A symbol the runtime cannot +resolve aborts the process *after* the tests have passed — rocSPARSE hit this on +an inlined rocPRIM kernel — and where the readback does succeed the runtime +writes an extra profile named after the device +(`gfx942:sramecc+:xnack-.0.`), which cannot be uploaded as a CI +artifact because of the colons. + +The cancellation has to be the last coverage flag on the compile line, since the +later of a `-f`/`-fno-` pair wins and the project's own flags arrive through +`COMPILE_OPTIONS`, after `CMAKE__FLAGS`. It is therefore appended to +`CMAKE__COMPILE_OBJECT` from a generated file passed as +`CMAKE_PROJECT_INCLUDE`, rather than to the flags variables; the compile rule +does not exist yet in `CMAKE_PROJECT_TOP_LEVEL_INCLUDES`, which runs before any +language is enabled. + ## Producing a report locally Instrumented binaries write one `.profraw` file per process, named by the @@ -132,6 +172,15 @@ workspace and uploading the resulting profiles as an artifact. The aggregation job downloads the profiles from every shard, merges them, and uploads the lcov report. +`coverage_enabled` also puts a floor under the per-component test timeout. +Instrumented libraries are built `-O0 -g` and count every branch, so a suite +takes much longer than its usual budget assumes: rocRAND tests that normally +finish in milliseconds took 25 to 60 seconds each and ran out of their 15 +minutes. It is a floor rather than a multiplier because the budgets are not on a +common scale — rocBLAS asks for 288 minutes, past the job's own cap, while +rocRAND asks for 15 — so scaling would stretch the generous ones and still leave +the tight ones tight. + ### Scheduling `multi_arch_ci_coverage_nightly.yml` is dispatched by hand. Pick a recent diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md index f2d3df63b25..60376b323d9 100644 --- a/docs/development/code_coverage_flows.md +++ b/docs/development/code_coverage_flows.md @@ -88,8 +88,9 @@ full `multi_arch_build_portable_linux.yml` pipeline. Naming the stages coverage needs keeps the shared pipeline free of coverage-specific stage filtering. They also pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`: split kernel -packaging would move instrumented device code out of the library `llvm-cov` is -later pointed at. +packaging rearranges the code objects embedded in the library `llvm-cov` is +later pointed at. Coverage is host-only, so this may no longer be load bearing, +but it has not been retested since and the flag is cheap to keep. The artifacts publish under `release_type: ci`, under this run's id. From f1cd9294fc362b45b3a5f8e0d0022087dc4fc79b Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 8 Sep 2026 02:57:08 -0500 Subject: [PATCH 19/70] coverage: fix the lint failures on the host-only change timeout-minutes takes a number and the env context hands back strings, so the step needs fromJSON around it. actionlint catches this; the value is still computed once in the job env rather than twice at the two use sites. mdformat cannot wrap inside a code span, so the flag list was left as one overlong line. It reads better as a block anyway. Also records in the PR check what run 34164420766 established, since the project list there is otherwise hard to account for. Co-authored-by: Cursor --- .github/workflows/coverage_nightly_pr_check.yml | 5 ++++- .github/workflows/test_component.yml | 3 ++- docs/development/code_coverage.md | 17 +++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index ea4a91cf75e..41e0a4920f6 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -91,7 +91,10 @@ jobs: # build, baseline install, overlay, tests and a 77% report -- once # libclang_rt.profile_rocm.a's own dependencies (-ldl, -lpthread) were # added to the link line, which had been failing every instrumented - # binary before that. + # binary before that. Run 34164420766 then took the instrumented build + # of all twelve green and reported six of them; the six that failed are + # what the host-only scoping, the timeout floor and the two registry + # corrections in this branch are for. # # Named explicitly rather than using rocm_libraries_all because the four # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA) are left diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index 23637ef4f9a..5db36b06665 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -291,7 +291,8 @@ jobs: - name: Test id: test - timeout-minutes: ${{ env.TEST_TIMEOUT_MINUTES }} + # fromJSON because env values are strings and this wants a number. + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT_MINUTES) }} env: SHARD_INDEX: ${{ matrix.shard }} TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }} diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 4002e6ae2fd..1a5581c563b 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -36,12 +36,17 @@ writes a second profile named after the device (`gfx942:sramecc+:xnack-.0.`) that cannot be uploaded as a CI artifact because of the colons. -`therock_subproject.cmake` therefore appends `-Xarch_device --fno-profile-instr-generate -Xarch_device -fno-coverage-mapping` to the compile -rule of any subproject it enables coverage for. It has to be the compile rule -rather than `CMAKE__FLAGS`: the last of a `-f`/`-fno-` pair wins, and the -project's own flags arrive later, as `COMPILE_OPTIONS`. Measuring GPU code -would mean undoing this and solving the two problems above. +`therock_subproject.cmake` therefore appends the negation, scoped to the device +compilation, for any subproject it enables coverage for: + +``` +-Xarch_device -fno-profile-instr-generate -Xarch_device -fno-coverage-mapping +``` + +It has to go on the compile rule rather than in `CMAKE__FLAGS`: the last +of a `-f`/`-fno-` pair wins, and the project's own flags arrive later, as +`COMPILE_OPTIONS`. Measuring GPU code would mean undoing this and solving the +two problems above. That runtime calls `dlsym`, `dladdr` and `pthread_once`, and the driver does not put `-ldl` or `-lpthread` on the link line for it. On the manylinux base those From eec2a1acbb9af8cde99253985ac8e79a9be4c370 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 8 Sep 2026 04:32:51 -0500 Subject: [PATCH 20/70] coverage: block hipTensor, whose instrumented build exceeds the code model Correcting hipTensor's option name to HIPTENSOR_CODE_COVERAGE instrumented it for the first time, and it does not link: libhiptensor.so grows past the 2GB the small code model allows and ld.lld fails on out-of-range R_X86_64_PC32 relocations from .rodata into .text. Upstream enables coverage as a blanket add_compile_options over a library built on composable_kernel, so the coverage mapping and name data scale with an already very large amount of instantiated template code. The option name stays correct so that retesting the block is one field away. Taking it out of the PR check list matters as much as the blocked_reason, since a named project is still honoured. A stage is built as a unit, so this one link failure failed math-libs in run 34201973090 and skipped the report for all twelve. Co-authored-by: Cursor --- .../workflows/coverage_nightly_pr_check.yml | 16 ++++++---- .../github_actions/configure_coverage_ci.py | 14 +++++++++ .../tests/configure_coverage_ci_test.py | 4 +-- docs/development/code_coverage.md | 29 ++++++++++++++----- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 41e0a4920f6..b3f0a47c5b1 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -74,7 +74,7 @@ jobs: # are executed and no package config is read. # # Verified before pinning that this run publishes artifacts covering - # every selected project: blas, fft, hipdnn, hiptensor, rand, solver and + # every selected project: blas, fft, hipdnn, rand, solver and # sparse. Not all are per-target -- hipdnn is generic throughout and # solver's tests are generic -- so checking only for _gfx942 understates # what is there. @@ -108,10 +108,14 @@ jobs: # upstream -- rccl's CMake refuses coverage outside a Debug build, and # rocshmem fails on coverage plus HIP relocatable device code. # - # hipBLASLt is absent for the same reason it is absent from the default - # selection: its instrumented build does not link. See its - # blocked_reason in configure_coverage_ci.py. + # hipBLASLt and hipTensor are absent for the same reason they are absent + # from the default selection: neither instrumented build links. See their + # blocked_reason in configure_coverage_ci.py. hipTensor was still in this + # list for run 34201973090, which is how its failure was found: the two + # registry corrections meant it was instrumented for the first time, and + # a stage is built as a unit, so its link error failed math-libs and + # skipped the report for all twelve. projects_to_test: >- - hipblas,hipdnn,hipfft,hiprand,hipsolver,hipsparselt, - hiptensor,rocblas,rocfft,rocrand,rocsolver,rocsparse + hipblas,hipdnn,hipfft,hiprand,hipsolver, + hipsparselt,rocblas,rocfft,rocrand,rocsolver,rocsparse test_type: "standard" diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 0d889c728c3..25b7930aa3a 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -338,6 +338,20 @@ class CoverageProject: # this wrong is silent: the option is simply never read, the library # builds uninstrumented, and the tests pass having written no profile. coverage_option="HIPTENSOR_CODE_COVERAGE", + # Instrumenting hipTensor pushes libhiptensor.so past the 2GB that the + # small code model allows, and the link dies on out-of-range + # R_X86_64_PC32 relocations from .rodata into .text. Upstream applies + # coverage as a blanket add_compile_options(-fprofile-instr-generate + # -fcoverage-mapping), and hipTensor is built on composable_kernel, so + # the coverage mapping and name data scale with an already enormous + # amount of instantiated template code. Building it with a larger code + # model is the likely fix and belongs upstream. Blocked meanwhile + # because the failure takes the whole math-libs stage with it, and + # every other project in the stage along with it. + blocked_reason=( + "libhiptensor.so exceeds the small code model when instrumented; " + "the link fails on out-of-range R_X86_64_PC32 relocations" + ), stage=STAGE_MATH_LIBS, test_component="hiptensor", coverage_config="projects/hiptensor/test_categories_coverage.yaml", diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index cb106777148..61b08005698 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -41,8 +41,8 @@ def test_default_leaves_out_what_cannot_be_built_or_is_known_broken(self): - configure_coverage_ci.DEFAULT_PROJECTS ), # rccl and rocshmem for want of a comm-libs build job, hipblaslt - # because its instrumented build does not link. - ["hipblaslt", "rccl", "rocshmem"], + # and hiptensor because their instrumented builds do not link. + ["hipblaslt", "hiptensor", "rccl", "rocshmem"], ) def test_blocked_project_is_left_out_of_every_alias(self): diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 1a5581c563b..78f7372c6dc 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -262,10 +262,10 @@ default — and naming them, or `rocm_systems_all` or `all`, is rejected outrigh The narrowing applies only to the default; an explicit request is never silently reduced. See [Adding a stage](#adding-a-stage). -`hipblaslt` is excluded from the default and the aliases too, for a different -reason: it is measurable, but its instrumented build currently fails to link. -Naming it is honoured with a warning rather than rejected, since rerunning it -is how the block gets noticed as fixed. See +`hipblaslt` and `hiptensor` are excluded from the default and the aliases too, +for a different reason: both are measurable, but their instrumented builds +currently fail to link. Naming one is honoured with a warning rather than +rejected, since rerunning it is how the block gets noticed as fixed. See [Blocked projects](#blocked-projects). The aliases are expanded to concrete project names inside @@ -303,14 +303,29 @@ default selection and the group aliases — including the group lists `--emit-cmake` writes, so a group build does not set a flag the Python side just declined to set. Naming it still works and prints the reason as a warning. -`hipblaslt` is the current example. Its coverage-only branch of +There are two examples today, and both fixes belong upstream in +`ROCm/rocm-libraries`; once either lands, drop that project's +`blocked_reason`. + +`hipblaslt` fails at link. Its coverage-only branch of `clients/CMakeLists.txt` links `hipblaslt-test` against a bare `rocroller` while the surrounding lines use imported targets. rocRoller exports as `roc::rocroller`, and under TheRock it is a separate subproject found through its package config, so the bare name is not a target here and reaches the linker as `-lrocroller` with no `-L` to resolve it. hipBLASLt's own build -avoids this by having rocRoller in-tree. The fix belongs upstream in -`ROCm/rocm-libraries`; once it lands, drop the `blocked_reason`. +avoids this by having rocRoller in-tree. + +`hiptensor` outgrows the small code model. Upstream enables coverage as a +blanket `add_compile_options(-fprofile-instr-generate -fcoverage-mapping)`, and +hipTensor is built on composable_kernel, so the coverage mapping and name data +scale with an already very large amount of instantiated template code. The +result pushes `libhiptensor.so` past 2GB and the link fails on out-of-range +`R_X86_64_PC32` relocations from `.rodata` into `.text`. Building it with a +larger code model is the likely fix. + +Blocking matters more than it looks here, because a stage is built as a unit: a +single project whose instrumented build fails takes the whole stage with it, and +every other project's report along with it. Confirm a local instrumented build produces a non-empty report before relying on a new entry. A project whose tests never load the instrumented library builds From 255d8a1f91611199e31dbf4aadc8bf34c01e038c Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 8 Sep 2026 09:27:18 -0500 Subject: [PATCH 21/70] docs: merge the coverage flow walkthrough into code_coverage.md The split put the same explanations in two places. Both documents described the option-name translation, the host-only scoping, the baseline-and-overlay arrangement and the object_globs export scoping, and code_coverage.md had grown two separate accounts of host-only instrumentation against each other. Nothing kept them in step. code_coverage.md now runs local build, local report, then CI in one order, with each mechanism explained where it first appears and referred back to afterwards. The CI half keeps everything the flow document carried, including the job graph. Readers who arrived via the Infrastructure index or ci_overview.md now land on the Coverage CI section rather than a separate file. Co-authored-by: Cursor --- docs/development/README.md | 2 +- docs/development/ci_overview.md | 4 +- docs/development/code_coverage.md | 446 ++++++++++++++---------- docs/development/code_coverage_flows.md | 179 ---------- 4 files changed, 274 insertions(+), 357 deletions(-) delete mode 100644 docs/development/code_coverage_flows.md diff --git a/docs/development/README.md b/docs/development/README.md index 633067b69ee..4e75f94ab38 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -31,7 +31,7 @@ - [S3 Buckets](s3_buckets.md) - [GitHub Actions Debugging](github_actions_debugging.md) - [CI Behavior Manipulation](ci_behavior_manipulation.md) -- [Nightly Code Coverage Flow](code_coverage_flows.md) +- [Coverage CI](code_coverage.md#coverage-ci) - [Stage Reuse](stage_reuse.md) - [CCache Troubleshooting](ccache_troubleshooting.md) - [WSL ROCDXG CI Stage](wsl_rocdxg.md) diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index 993cc7a3118..db12ce663d0 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -57,8 +57,8 @@ measures with instrumentation rather than reusing the regular build: - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project test and report for one GPU family See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a -project, and [Nightly Code Coverage Flow](code_coverage_flows.md) for the -end-to-end CI walkthrough. +project, and [Coverage CI](code_coverage.md#coverage-ci) for the end-to-end +walkthrough of a run. ## Build Phase diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 78f7372c6dc..0747554afe7 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -18,49 +18,26 @@ cmake -B build -GNinja . \ -DHIPRAND_ENABLE_COVERAGE=ON ``` -`therock_subproject.cmake` forwards the flag to the matching subproject, which -is responsible for translating it into compiler flags (usually -`-fprofile-instr-generate -fcoverage-mapping`). Passing the flag for a project -that does not implement it has no effect. - -This pipeline measures host code only, and deliberately keeps the -instrumentation off the GPU. The upstream coverage options pass -`-fprofile-instr-generate` and `-fcoverage-mapping` unqualified and the HIP -driver forwards both to the device compilation, so kernels would be -instrumented too. That does not survive contact with the runtime: the device -image gains a per-translation-unit `__llvm_profile_sections_` symbol that -`libclang_rt.profile_rocm` looks up with `hipModuleGetGlobal` at exit, plus a -`__profd_` for every kernel. A symbol the runtime cannot resolve aborts the -process after the tests have already passed, and where the readback succeeds it -writes a second profile named after the device -(`gfx942:sramecc+:xnack-.0.`) that cannot be uploaded as a CI -artifact because of the colons. - -`therock_subproject.cmake` therefore appends the negation, scoped to the device -compilation, for any subproject it enables coverage for: - -``` --Xarch_device -fno-profile-instr-generate -Xarch_device -fno-coverage-mapping -``` - -It has to go on the compile rule rather than in `CMAKE__FLAGS`: the last -of a `-f`/`-fno-` pair wins, and the project's own flags arrive later, as -`COMPILE_OPTIONS`. Measuring GPU code would mean undoing this and solving the -two problems above. - -That runtime calls `dlsym`, `dladdr` and `pthread_once`, and the driver does not -put `-ldl` or `-lpthread` on the link line for it. On the manylinux base those -symbols are still in `libdl` and `libpthread` rather than in `libc`, so an -instrumented shared library would be left with undefined references, and any -executable linking against it would fail `--no-allow-shlib-undefined`. -`therock_subproject.cmake` adds the two libraries whenever it enables coverage -for a subproject, so this is handled; it is worth knowing about if you -instrument a project outside the normal flow. +`_ENABLE_COVERAGE` is TheRock's own knob, and it is not what the +subproject sees. Upstream option names are not standardised — hipRAND takes +`BUILD_CODE_COVERAGE`, rocRAND takes `CODE_COVERAGE`, RCCL takes +`ENABLE_CODE_COVERAGE`, hipBLASLt takes `HIPBLASLT_ENABLE_COVERAGE` — so +`therock_subproject.cmake` translates TheRock's flag into whichever name is +registered for that project in `COVERAGE_PROJECTS`. Translating per subproject +is what keeps the generic spellings from instrumenting everything in the build +that happens to understand them. + +Passing the flag for a project with no registered option is a configure error +rather than a silent no-op, since the failure it replaces — an uninstrumented +build that tests cleanly and reports nothing — surfaces hours later. > [!NOTE] > Coverage builds are incompatible with split kernel packaging. The CI > workflows pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`; do the same if you -> build with a target family that enables kernel packing by default. +> build with a target family that enables kernel packing by default. Split +> packaging rearranges the code objects embedded in the library `llvm-cov` is +> later pointed at. Coverage is host-only now, so this may no longer be load +> bearing, but it has not been retested and the flag is cheap to keep. ### Enabling a whole group @@ -79,45 +56,64 @@ cmake -B build -GNinja . \ -DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON ``` -Each option expands to the individual `_ENABLE_COVERAGE` flags above, so -the same device-runtime and kernel-packing caveats apply. An explicit -`-D_ENABLE_COVERAGE=OFF` on the command line wins over the group flag. -Group membership is generated from the coverage registry (`COVERAGE_PROJECTS`), -so a group only ever contains onboarded projects; selecting a group with none -(currently rocm-systems) configures nothing and prints a warning. +Each option expands to the individual `_ENABLE_COVERAGE` flags above, +so the same caveats apply. An explicit `-D_ENABLE_COVERAGE=OFF` on the +command line wins over the group flag. Group membership is generated from the +coverage registry (`COVERAGE_PROJECTS`), so a group only ever contains +onboarded projects. > [!NOTE] -> These options default `OFF` and have no effect on a normal, non-coverage build -> — no `_ENABLE_COVERAGE` flag is set. The membership list is -> regenerated from `COVERAGE_PROJECTS` on every configure (a small scripted step; -> it also means editing `configure_coverage_ci.py` triggers a reconfigure), but -> it only changes the build when one of the options is `ON`. +> These options default `OFF` and have no effect on a normal, non-coverage +> build — no `_ENABLE_COVERAGE` flag is set. The membership list is +> regenerated from `COVERAGE_PROJECTS` on every configure (a small scripted +> step; it also means editing `configure_coverage_ci.py` triggers a +> reconfigure), but it only changes the build when one of the options is `ON`. ### Host code only -What gets measured is host code. The upstream coverage options pass -`-fprofile-instr-generate` and `-fcoverage-mapping` unqualified and the HIP -driver forwards both to the device compilation, so kernels would be instrumented -too. `therock_subproject.cmake` cancels that with `-Xarch_device`, which is -scoped to the device compilation and leaves the host instrumentation alone. +What gets measured is host code, and the instrumentation is deliberately kept +off the GPU. The upstream coverage options pass `-fprofile-instr-generate` and +`-fcoverage-mapping` unqualified, and the HIP driver forwards both to the +device compilation, so kernels would be instrumented too. -This is not just a matter of scope. An instrumented device image carries a -per-translation-unit `__llvm_profile_sections_` symbol that +That does not survive contact with the runtime. An instrumented device image +carries a per-translation-unit `__llvm_profile_sections_` symbol that `libclang_rt.profile_rocm.a` looks up through `hipModuleGetGlobal` when the process exits, plus a `__profd_` for every kernel. A symbol the runtime cannot -resolve aborts the process *after* the tests have passed — rocSPARSE hit this on -an inlined rocPRIM kernel — and where the readback does succeed the runtime -writes an extra profile named after the device +resolve aborts the process *after* the tests have already passed — rocSPARSE +hit this on an inlined rocPRIM kernel — and where the readback does succeed, +the runtime writes an extra profile named after the device (`gfx942:sramecc+:xnack-.0.`), which cannot be uploaded as a CI artifact because of the colons. -The cancellation has to be the last coverage flag on the compile line, since the -later of a `-f`/`-fno-` pair wins and the project's own flags arrive through -`COMPILE_OPTIONS`, after `CMAKE__FLAGS`. It is therefore appended to -`CMAKE__COMPILE_OBJECT` from a generated file passed as -`CMAKE_PROJECT_INCLUDE`, rather than to the flags variables; the compile rule +`therock_subproject.cmake` therefore appends the negation, scoped to the device +compilation, for any subproject it enables coverage for: + +``` +-Xarch_device -fno-profile-instr-generate -Xarch_device -fno-coverage-mapping +``` + +Position is what makes this work. The later of a `-f`/`-fno-` pair wins, and +the project's own flags arrive through `COMPILE_OPTIONS`, after +`CMAKE__FLAGS` — so the negation is appended to +`CMAKE__COMPILE_OBJECT` rather than to the flags variables. That rule does not exist yet in `CMAKE_PROJECT_TOP_LEVEL_INCLUDES`, which runs before any -language is enabled. +language is enabled, so it is applied from a generated file passed as +`CMAKE_PROJECT_INCLUDE`. Measuring GPU code would mean undoing this and solving +the two problems above. + +### The profile runtime's link dependencies + +`libclang_rt.profile_rocm.a` calls `dlsym`, `dladdr` and `pthread_once`, and +the driver does not put `-ldl` or `-lpthread` on the link line for it. On the +manylinux base those symbols are still in `libdl` and `libpthread` rather than +in `libc`, so an instrumented shared library would be left with undefined +references, and any executable linking against it would fail +`--no-allow-shlib-undefined`. + +`therock_subproject.cmake` adds the two libraries whenever it enables coverage +for a subproject, so this is handled. It is worth knowing about if you +instrument a project outside the normal flow. ## Producing a report locally @@ -141,70 +137,125 @@ python build_tools/github_actions/merge_coverage_report.py \ The `llvm-profdata` and `llvm-cov` binaries must come from the same compiler that built the instrumented objects, so the script prefers the copies under -`/lib/llvm/bin`. A version mismatch surfaces as an unhelpfully generic -"malformed instrumentation profile data" error. +`/lib/llvm/bin`. A version mismatch surfaces as an unhelpfully +generic "malformed instrumentation profile data" error. ## Coverage CI -For a step-by-step walkthrough of a nightly run, from the dispatch through to -the Codecov upload, see -[Nightly Code Coverage Flow](code_coverage_flows.md). The rest of this section -is the summary. +A coverage run builds the stages its selection needs, instruments only the +selected projects, and reports on each of them separately from that one build. -Coverage reports are produced nightly by -`multi_arch_ci_coverage_nightly.yml`, which builds the whole instrumented stack -once and then tests and reports on each project against that single build. It -delegates to `multi_arch_ci_coverage_linux.yml` for the per-project test and -aggregation sequence on one GPU family. +```mermaid +graph TD + dispatch[Manual dispatch with projects_to_test] --> matrix[setup_coverage_matrix] + matrix --> compilerRuntime[Build instrumented compiler-runtime] + compilerRuntime --> mathLibs[Build instrumented math-libs] + mathLibs --> report[Per project: configure, test, report] + report --> codecov[Codecov] +``` -Both take their project list from -`build_tools/github_actions/configure_coverage_ci.py`, which is the registry of +Every edge is a job dependency inside the one coverage run. +`multi_arch_ci_coverage_nightly.yml` is the top level; it delegates to +`multi_arch_ci_coverage_linux.yml` for the per-project test and aggregation +sequence on one GPU family. Both take their project list from +`build_tools/github_actions/configure_coverage_ci.py`, the registry of coverage-enabled projects and everything the pipeline needs to know about each one: the CMake target, the build stage it lives in, its test component, the object globs handed to `llvm-cov`, and its Codecov flag. -The instrumented build calls `multi_arch_build_portable_linux_artifacts.yml` — -the same per-stage build workflow regular CI uses — one job per stage, rather -than going through the full `multi_arch_build_portable_linux.yml` pipeline. -Coverage only ever needs compiler-runtime plus the stages owning the projects -under test, so naming those stages directly keeps the shared pipeline free of -coverage-specific stage filtering. `extra_cmake_options` on that workflow is the -only build-side hook coverage adds. +### Dispatching a run -The test jobs run through the same `test_component.yml` as regular CI, with -`coverage_enabled: true` adding an `LLVM_PROFILE_FILE` pointing into the -workspace and uploading the resulting profiles as an artifact. The aggregation -job downloads the profiles from every shard, merges them, and uploads the lcov -report. +`multi_arch_ci_coverage_nightly.yml` is started by hand, from the Actions tab +or the `gh` CLI. There is no cron trigger, and the regular nightly does not +know about this workflow — it has no coverage-specific jobs, and a coverage run +neither reads its artifacts nor depends on it having succeeded. -`coverage_enabled` also puts a floor under the per-component test timeout. -Instrumented libraries are built `-O0 -g` and count every branch, so a suite -takes much longer than its usual budget assumes: rocRAND tests that normally -finish in milliseconds took 25 to 60 seconds each and ran out of their 15 -minutes. It is a floor rather than a multiplier because the budgets are not on a -common scale — rocBLAS asks for 288 minutes, past the job's own cap, while -rocRAND asks for 15 — so scaling would stretch the generous ones and still leave -the tight ones tight. +Pick a recent nightly run, pass its id as `baseline_run_id`, and leave +`baseline_release_type` at `nightly`. The other input that matters is +`projects_to_test`. The rest can be left at their defaults — the +`gfx94X-dcgpu` family and coverage's own `standard` test type, which is +narrower than the nightly's because instrumented tests already run several +times longer than normal ones. -### Scheduling - -`multi_arch_ci_coverage_nightly.yml` is dispatched by hand. Pick a recent -nightly run, pass its id as `baseline_run_id`, and leave `baseline_release_type` -at `nightly`. There is no cron trigger, and the regular nightly does not know -about this workflow. - -Coverage therefore has a run id of its own, which is what keeps a coverage -failure from colouring the nightly's status and keeps the nightly's test jobs — -which rarely all pass — from holding coverage up. +Coverage having a run id of its own is what keeps a coverage failure from +colouring the nightly's status, and keeps the nightly's test jobs — which +rarely all pass — from holding coverage up. Having the regular nightly dispatch this workflow automatically once its build finishes, handing over its own run id, is a later change. Nothing else has to -move for it: `baseline_run_id` is already an input, so that change only adds the -dispatching job. +move for it: `baseline_run_id` is already an input, so that change only adds +the dispatching job. Running only what changed, and running more architectures +than gfx942, come later too. + +### The instrumented build + +`setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads +`PROJECTS_TO_TEST`, `AMDGPU_FAMILIES` and `COVERAGE_CONFIG_SOURCE` and emits +the per-project job matrix, the family list, `coverage_cmake_options` and +`needs_math_libs`. It also rejects any selection naming a project whose stage +this workflow has no build job for, so a bad request stops in seconds rather +than hours. + +`build_instrumented_compiler_runtime` runs first and unconditionally. Every +other stage takes its inbound artifacts from it — all 19 of `math-libs`' +inbound artifacts are produced there — so no selection can skip it. It also +receives the coverage flags, since several rocm-systems projects +(`rocprofiler-sdk`, `aqlprofile`, `amdsmi`, `rocprofiler-compute`) are built in +that stage. + +`build_instrumented_math_libs` fans out over the GPU families, and is skipped +when `needs_math_libs` is false. It is the only other stage built: 17 of the 19 +measurable projects live there, and the remaining two are blocked upstream, so +phase 1 carries no job for any further stage. + +Both stages get `coverage_cmake_options` appended to their configure line. +`configure_coverage_ci.py` collapses a selection covering a whole group into +that group's option, so a full run reads `-DTHEROCK_COVERAGE_ALL=ON` while a +narrow one reads `-DHIPRAND_ENABLE_COVERAGE=ON`. `CMakeLists.txt` expands the +group options into individual `_ENABLE_COVERAGE` flags, and +`therock_subproject.cmake` translates each of those into the project's own +option name. Everything not selected builds normally. + +All the build jobs call `multi_arch_build_portable_linux_artifacts.yml`, the +same per-stage build workflow regular CI uses, rather than going through the +full `multi_arch_build_portable_linux.yml` pipeline. Naming the stages coverage +needs keeps the shared pipeline free of coverage-specific stage filtering, and +`extra_cmake_options` on that workflow is the only build-side hook coverage +adds. + +The artifacts publish under `release_type: ci`, under this run's id. + +### Test execution + +`coverage_report` fans out over the matrix, one call to +`multi_arch_ci_coverage_linux.yml` per project and GPU family. +`configure_test_matrix` runs `fetch_test_configurations.py` — shared with +regular CI — narrowed to the one project this report covers, and outputs the +shard list. `test_coverage` then runs one `test_component.yml` job per shard +with `coverage_enabled: true`. Three things happen in order: + +1. **Install the baseline, then overlay.** `setup_test_environment` runs + against `baseline_run_id` and `baseline_release_type`, installing a fully + non-instrumented stack. `overlay_coverage_artifacts.py` then fetches this + run's artifact for the project under test and copies only that project's + stage directories over it, so exactly one library in the install is + instrumented. +1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to + a per-shard path using the `%p` and `%m` substitutions, so a shard that + forks or loads several instrumented libraries does not overwrite its own + profiles. +1. **Run the tests and upload.** The normal test script runs, then the profraw + files upload as a workflow artifact under `always()` — a failing shard still + exercised code. -Nightly coverage runs the projects named by `projects_to_test` on one -architecture (gfx942, the `gfx94X-dcgpu` family). Running only what changed, and -running more architectures, come later. +`coverage_enabled` also puts a floor under the per-component test timeout. +Instrumented libraries are built `-O0 -g` and count every branch, so a suite +takes much longer than its usual budget assumes: rocRAND tests that normally +finish in milliseconds took 25 to 60 seconds each and ran out of their 15 +minutes. It is a floor rather than a multiplier because the budgets are not on +a common scale — rocBLAS asks for 288 minutes, past the job's own cap, while +rocRAND asks for 15 — so scaling would stretch the generous ones and still +leave the tight ones tight. #### Why there are two run ids @@ -212,40 +263,83 @@ The nightly instruments the whole stack in one build, because building each project separately would mean rebuilding its dependencies each time. Reporting, though, has to stay per project: a coverage report for hipRAND should not shift because rocRAND changed, and an instrumented rocRAND under an instrumented -hipRAND also writes its own profiles into the same run. - -Each test job therefore assembles a mostly non-instrumented install: - -1. `setup_test_environment` installs the **baseline** run in full — every - library non-instrumented, as the regular nightly built it. -1. `overlay_coverage_artifacts.py` fetches the artifact holding the project - under test from the **coverage** run and copies only that project's files - over the baseline ones. +hipRAND also writes its own profiles into the same run. Hence the baseline +install and the overlay described above. The baseline is read from the channel that published it (`baseline_release_type`, normally `nightly`) rather than from the coverage run's own `ci` channel, since artifacts are bucketed per channel. -Step 2 is per project rather than per artifact because TheRock's artifacts are -grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit under -the subproject stage directory they were built in (`math-libs/hipRAND/stage`), -which is what `artifact_relpaths` in the coverage registry names. Nothing is -renamed along the way — the two runs write to different S3 directories, so the -instrumented `rand_lib_gfx942.tar.xz` and the regular one never collide. +The overlay is per project rather than per artifact because TheRock's artifacts +are grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit +under the subproject stage directory they were built in +(`math-libs/hipRAND/stage`), which is what `artifact_relpaths` in the registry +names. Nothing is renamed along the way — the two runs write to different S3 +directories, so the instrumented `rand_lib_gfx942.tar.xz` and the regular one +never collide. If the overlay finds none of the project's files, the job fails rather than testing the baseline build and reporting coverage for binaries that were never instrumented. -`object_globs` then scopes the report itself: `llvm-cov export` is handed only -that project's objects, so a selection covering several projects still yields -one report per project. The overlay and the globs solve different halves of the -same problem. The globs keep a sibling's whole functions out of the report; the -overlay keeps a sibling's instrumented inline and header code from merging into -this project's counters in the first place, which globs cannot undo because -`linkonce_odr` definitions land in every binary that uses them. +#### What scopes a report to one project + +Two mechanisms, covering different leaks. -### Selecting projects to run +`object_globs` scopes the export. `llvm-cov export` is handed an explicit +`-object` for each of the project's binaries and reports only the functions +found in their coverage mappings, so a sibling's whole functions never appear. + +The overlay scopes what is instrumented at all. Globs alone would not be +enough, because inline and template code defined in headers is emitted +`linkonce_odr` into every binary that uses it. If a sibling library were also +instrumented, its copies of those functions would write counters that +`llvm-profdata` merges into this project's, and no `-object` choice could +separate them again. Installing a non-instrumented baseline and overlaying one +project keeps that from arising — which matters most for the header-only +projects (rocPRIM, hipCUB, rocThrust, rocWMMA), where nearly all the measured +code lives in headers. + +The cost of selecting more projects is build and test time; each instrumented +library is slower to compile and slower to exercise. + +### Artifacts + +Two kinds are in play: + +| Artifact | Where it lives | What it is for | +| ------------ | ----------------------------------- | ---------------------------------------------- | +| Instrumented | CI bucket, this run's id | The stack under test, and the LLVM tools | +| Profraw | GitHub Actions artifacts, per shard | Raw profiles, collected by the aggregation job | + +Profraw artifact names carry the project, GPU family and shard index, so the +aggregation job can glob back exactly its own shards when several projects are +being measured in one run. + +### Report generation + +`aggregate_coverage` runs once per project, under `if: !cancelled()` so a +partial report still gets produced when some shards failed. + +It downloads every matching profraw artifact, then installs the run's artifacts +again — `llvm-cov` needs the binaries carrying the coverage mapping sections, +and the LLVM tools have to come from the same compiler that produced the +profiles. + +`merge_coverage_report.py` then collects the profiles recursively, expands the +project's `object_globs` and deduplicates them by real path (a versioned +symlink family resolves to one file), runs `llvm-profdata merge -sparse` into a +single profdata index, and exports lcov. Two conditions are fatal rather than +reported as zero coverage: finding no profraw files at all, which means the +tests never ran or the instrumented libraries were not the ones loaded at +runtime; and matching no objects, which means `object_globs` does not describe +what the project actually installs. + +The lcov report uploads as a workflow artifact and goes to Codecov under the +project's flag. The Codecov step is skipped rather than failed when no +`CODECOV_TOKEN` is configured, so forks still get the lcov artifact. + +## Selecting projects to run A nightly run measures every onboarded project the workflow can build. Dispatching by hand narrows that down: the `projects_to_test` input takes a @@ -256,11 +350,15 @@ selecting a group with no onboarded projects fails the run with a clear error rather than launching an empty matrix. An empty input means every project with a build job, which today is -`rocm_libraries_all`. `rccl` and `rocshmem` are onboarded but live in -`comm-libs`, which this workflow does not build, so they are excluded from the -default — and naming them, or `rocm_systems_all` or `all`, is rejected outright. -The narrowing applies only to the default; an explicit request is never silently -reduced. See [Adding a stage](#adding-a-stage). +`rocm_libraries_all`. That works, but it instruments all of them in one build, +which is the expensive way to run this; name what you want measured. + +`rccl` and `rocshmem` are onboarded but live in `comm-libs`, which this +workflow does not build, so they are excluded from the default — and naming +them, or `rocm_systems_all` or `all`, is rejected outright by +`setup_coverage_matrix` rather than failing hours later with nothing to +overlay. The narrowing applies only to the default; an explicit request is +never silently reduced. See [Adding a stage](#adding-a-stage). `hipblaslt` and `hiptensor` are excluded from the default and the aliases too, for a different reason: both are measurable, but their instrumented builds @@ -269,13 +367,14 @@ rejected, since rerunning it is how the block gets noticed as fixed. See [Blocked projects](#blocked-projects). The aliases are expanded to concrete project names inside -`configure_coverage_ci.py` before the job matrix is built, so nothing downstream -ever sees them: the coverage pipeline hands `fetch_test_configurations.py` -(shared with regular CI) an already-resolved per-project `test_component`, never -an alias. The `projects_to_test` input and these aliases exist only on the -coverage workflow, so non-coverage test selection is unaffected. +`configure_coverage_ci.py` before the job matrix is built, so nothing +downstream ever sees them: the coverage pipeline hands +`fetch_test_configurations.py` (shared with regular CI) an already-resolved +per-project `test_component`, never an alias. The `projects_to_test` input and +these aliases exist only on the coverage workflow, so non-coverage test +selection is unaffected. -### Adding a project +## Adding a project Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. @@ -286,15 +385,29 @@ names are not standardised — most projects use `BUILD_CODE_COVERAGE`, several use `CODE_COVERAGE`, a few use `_ENABLE_COVERAGE`, RCCL uses `ENABLE_CODE_COVERAGE` — and some projects instrument with gcov (`-fprofile-arcs`) instead, which writes `.gcda` files this pipeline cannot -read. `therock_subproject.cmake` translates TheRock's `_ENABLE_COVERAGE` -into whichever name you register, per subproject, so a generic name does not -leak into the rest of the build. +read. A project that cannot be measured yet gets an `unsupported_reason` instead. It stays in the registry so the gap is recorded, stays out of the group aliases, and is rejected with that reason if someone names it. -### Blocked projects +Confirm a local instrumented build produces a non-empty report before relying +on a new entry. A project whose tests never load the instrumented library +builds and tests cleanly and then reports nothing. + +Set the entry's `source_repo` (`ROCM_LIBRARIES` by default, or `ROCM_SYSTEMS`); +that is what routes the project into the correct group alias and +`THEROCK_COVERAGE_*_ALL` option. No CMake edit is needed — the group membership +lists are generated from `COVERAGE_PROJECTS` at configure time, so both the +local aggregate flags and the CI aliases pick up the new project automatically. + +`object_globs` names the binaries the report is generated from, relative to the +install tree. For most projects that is the shared library +(`lib/librocblas.so*`); header-only projects have none, so they name their +installed test binaries instead. Getting it wrong fails the report job, which +refuses to run when no glob matches rather than publishing zero coverage. + +## Blocked projects A project that is measurable but whose instrumented build is broken gets a `blocked_reason` alongside its `coverage_option`. Unlike `unsupported_reason` @@ -303,9 +416,12 @@ default selection and the group aliases — including the group lists `--emit-cmake` writes, so a group build does not set a flag the Python side just declined to set. Naming it still works and prints the reason as a warning. +Blocking matters more than it looks, because a stage is built as a unit: a +single project whose instrumented build fails takes the whole stage with it, +and every other project's report along with it. + There are two examples today, and both fixes belong upstream in -`ROCm/rocm-libraries`; once either lands, drop that project's -`blocked_reason`. +`ROCm/rocm-libraries`; once either lands, drop that project's `blocked_reason`. `hipblaslt` fails at link. Its coverage-only branch of `clients/CMakeLists.txt` links `hipblaslt-test` against a bare `rocroller` @@ -323,27 +439,7 @@ result pushes `libhiptensor.so` past 2GB and the link fails on out-of-range `R_X86_64_PC32` relocations from `.rodata` into `.text`. Building it with a larger code model is the likely fix. -Blocking matters more than it looks here, because a stage is built as a unit: a -single project whose instrumented build fails takes the whole stage with it, and -every other project's report along with it. - -Confirm a local instrumented build produces a non-empty report before relying on -a new entry. A project whose tests never load the instrumented library builds -and tests cleanly and then reports nothing. - -Set the entry's `source_repo` (`ROCM_LIBRARIES` by default, or `ROCM_SYSTEMS`); -that is what routes the project into the correct group alias and -`THEROCK_COVERAGE_*_ALL` option. No CMake edit is needed — the group membership -lists are generated from `COVERAGE_PROJECTS` at configure time, so both the local -aggregate flags and the CI aliases pick up the new project automatically. - -`object_globs` names the binaries the report is generated from, relative to the -install tree. For most projects that is the shared library -(`lib/librocblas.so*`); header-only projects have none, so they name their -installed test binaries instead. Getting it wrong fails the report job, which -refuses to run when no glob matches rather than publishing zero coverage. - -### Adding a stage +## Adding a stage `multi_arch_ci_coverage_nightly.yml` builds compiler-runtime and math-libs, and `configure_coverage_ci.py` rejects any selection that reaches beyond them diff --git a/docs/development/code_coverage_flows.md b/docs/development/code_coverage_flows.md deleted file mode 100644 index 60376b323d9..00000000000 --- a/docs/development/code_coverage_flows.md +++ /dev/null @@ -1,179 +0,0 @@ -# Nightly Code Coverage Flow - -An end-to-end walk through what happens between dispatching a coverage run and -a report landing in Codecov. See [Code Coverage](code_coverage.md) for enabling -coverage on a local build, the CMake options, and how to onboard a project. - -The short version: a coverage run builds the stages its selection needs, -instruments only the selected projects, and reports on each of them separately -from that one run. - -```mermaid -graph TD - dispatch[Manual dispatch with projects_to_test] --> matrix[setup_coverage_matrix] - matrix --> compilerRuntime[Build instrumented compiler-runtime] - compilerRuntime --> mathLibs[Build instrumented math-libs] - mathLibs --> report[Per project: configure, test, report] - report --> codecov[Codecov] -``` - -Every edge is a job dependency inside the one coverage run. Nothing crosses a -run boundary. - -## Trigger point - -`multi_arch_ci_coverage_nightly.yml` is started by hand, from the Actions tab or -the `gh` CLI. There is no cron trigger, and the regular nightly does not know -about this workflow. - -The input that matters is `projects_to_test`: a comma-separated list of project -names, or one of the group aliases. The rest can be left at their defaults — the -`gfx94X-dcgpu` family and coverage's own `standard` test type, which is narrower -than the nightly's because instrumented tests already run several times longer -than normal ones. - -Leaving `projects_to_test` empty selects every onboarded project this workflow -can build. That works, but it instruments all of them in one build, which is the -expensive way to run this. Name what you want measured. - -`rccl` and `rocshmem` are the exception. They live in `comm-libs`, which has no -build job here, so naming them — or `rocm_systems_all` or `all` — is rejected by -`setup_coverage_matrix` rather than failing hours later with nothing to overlay. -See [Adding a stage](code_coverage.md#adding-a-stage). - -## The regular nightly - -Unaffected, and not consulted. It has no coverage-specific jobs, and a coverage -run neither reads its artifacts nor depends on it having succeeded. - -## The instrumented build - -The dispatch creates a workflow run with its own id, status, and logs. - -`setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads -`PROJECTS_TO_TEST`, `AMDGPU_FAMILIES` and `COVERAGE_CONFIG_SOURCE` and emits the -per-project job matrix, the family list, `coverage_cmake_options`, and -`needs_math_libs`. It also rejects any selection naming a project whose stage -this workflow has no build job for. - -`build_instrumented_compiler_runtime` runs first and unconditionally. Every other -stage takes its inbound artifacts from it — all 19 of `math-libs`' inbound -artifacts are produced there — so no selection can skip it. It also receives the -coverage flags, since several rocm-systems projects (`rocprofiler-sdk`, -`aqlprofile`, `amdsmi`, `rocprofiler-compute`) are built in that stage. - -`build_instrumented_math_libs` fans out over the GPU families, and is skipped -when `needs_math_libs` is false. It is the only other stage built: 17 of the 19 -measurable projects live there, and the remaining two are blocked upstream, so -phase 1 carries no job for any further stage. - -Both stages get `coverage_cmake_options` appended to their configure line. -`configure_coverage_ci.py` collapses a selection covering a whole group into -that group's option, so a full run reads `-DTHEROCK_COVERAGE_ALL=ON` while a -narrow one reads `-DHIPRAND_ENABLE_COVERAGE=ON`. `CMakeLists.txt` expands the -group options into individual `_ENABLE_COVERAGE` flags. Everything not -selected builds normally. - -`_ENABLE_COVERAGE` is TheRock's own knob and is not what the subproject -sees. Upstream option names are not standardised — hipRAND takes -`BUILD_CODE_COVERAGE`, rocRAND takes `CODE_COVERAGE`, RCCL takes -`ENABLE_CODE_COVERAGE`, hipBLASLt takes `HIPBLASLT_ENABLE_COVERAGE` — so -`therock_subproject.cmake` translates TheRock's flag into the name registered -for that project. Doing it per subproject is what keeps the generic spellings -from instrumenting everything in the build that happens to understand them. - -All the build jobs call `multi_arch_build_portable_linux_artifacts.yml`, the -same per-stage build workflow regular CI uses, rather than going through the -full `multi_arch_build_portable_linux.yml` pipeline. Naming the stages coverage -needs keeps the shared pipeline free of coverage-specific stage filtering. - -They also pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`: split kernel -packaging rearranges the code objects embedded in the library `llvm-cov` is -later pointed at. Coverage is host-only, so this may no longer be load bearing, -but it has not been retested since and the flag is cheap to keep. - -The artifacts publish under `release_type: ci`, under this run's id. - -## Test execution - -`coverage_report` fans out over the matrix, one call to -`multi_arch_ci_coverage_linux.yml` per project and GPU family. - -`configure_test_matrix` runs `fetch_test_configurations.py` — shared with -regular CI — narrowed to the one project this report covers, and outputs the -shard list. - -`test_coverage` then runs one `test_component.yml` job per shard with -`coverage_enabled: true`. Three things happen in order: - -1. **Install the baseline, then overlay.** `setup_test_environment` runs against - `baseline_run_id` and `baseline_release_type`, installing a fully - non-instrumented stack. `overlay_coverage_artifacts.py` then fetches this - run's artifact for the project under test and copies only that project's - stage directories over it, so exactly one library in the install is - instrumented. -1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to - a per-shard path using the `%p` (pid) and `%m` (binary signature) - substitutions, so a shard that forks or loads several instrumented libraries - does not overwrite its own profiles. -1. **Run the tests and upload.** The normal test script runs, then the profraw - files upload as a workflow artifact under `always()` — a failing shard still - exercised code. - -### What scopes a report to one project - -Two mechanisms, covering different leaks. - -`object_globs` scopes the export. `llvm-cov export` is handed an explicit -`-object` for each of the project's binaries and reports only the functions -found in their coverage mappings, so a sibling's whole functions never appear. - -The overlay scopes what is instrumented at all. Globs alone would not be enough, -because inline and template code defined in headers is emitted `linkonce_odr` -into every binary that uses it. If a sibling library were also instrumented, its -copies of those functions would write counters that `llvm-profdata` merges into -this project's, and no `-object` choice could separate them again. Installing a -non-instrumented baseline and overlaying one project keeps that from arising — -which matters most for the header-only projects (rocPRIM, hipCUB, rocThrust, -rocWMMA), where nearly all the measured code lives in headers. - -The cost of selecting more projects is build and test time — each instrumented -library is slower to compile and slower to exercise. - -## Artifacts - -Two kinds are in play: - -| Artifact | Where it lives | What it is for | -| ------------ | ----------------------------------- | ---------------------------------------------- | -| Instrumented | CI bucket, this run's id | The stack under test, and the LLVM tools | -| Profraw | GitHub Actions artifacts, per shard | Raw profiles, collected by the aggregation job | - -Profraw artifact names carry the project, GPU family, and shard index, so the -aggregation job can glob back exactly its own shards when several projects are -being measured in one run. - -## Report generation - -`aggregate_coverage` runs once per project, under `if: !cancelled()` so a -partial report still gets produced when some shards failed. - -It downloads every matching profraw artifact, then installs the run's artifacts -again. `llvm-cov` needs the binaries carrying the coverage mapping sections, and -`llvm-profdata` and `llvm-cov` themselves have to come from the same compiler -that produced the profiles. Both live under `lib/llvm/bin` of the installed -distribution; a version mismatch surfaces as an unhelpfully generic "malformed -instrumentation profile data" error. - -`merge_coverage_report.py` then collects the profiles recursively, expands the -project's `object_globs` and deduplicates them by real path (a versioned symlink -family resolves to one file), runs `llvm-profdata merge -sparse` into a single -profdata index, and exports lcov. Two conditions are fatal rather than reported -as zero coverage: finding no profraw files at all, which means the tests never -ran or the instrumented libraries were not the ones loaded at runtime; and -matching no objects, which means `object_globs` does not describe what the -project actually installs. - -The lcov report uploads as a workflow artifact and goes to Codecov under the -project's flag. The Codecov step is skipped rather than failed when no -`CODECOV_TOKEN` is configured, so forks still get the lcov artifact. From ba01c077864308e2b4e25d780132fe985729fcd1 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 8 Sep 2026 09:57:42 -0500 Subject: [PATCH 22/70] docs: add the plain-language coverage walkthrough code_coverage.md answers "what are the options and how do I onboard a project", which is the wrong shape for someone meeting this pipeline for the first time. This is the narrative version: why two builds are involved, what the overlay is for, and what each of the eight files in the critical path contributes, followed by an appendix covering every file in the change. The two link to each other rather than repeating; the reference keeps the mechanisms and this keeps the story. Co-authored-by: Cursor --- docs/development/README.md | 1 + docs/development/code_coverage.md | 4 + .../nightly_coverage_implementation.md | 359 ++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 docs/development/nightly_coverage_implementation.md diff --git a/docs/development/README.md b/docs/development/README.md index 4e75f94ab38..5c60adbb6d7 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -11,6 +11,7 @@ - [Artifacts](artifacts.md) - [Build System](build_system.md) - [Code Coverage](code_coverage.md) +- [Nightly Coverage Implementation](nightly_coverage_implementation.md) - a plain-language walkthrough of how a coverage report is produced - [Dependencies](dependencies.md) - [Development Guide](development_guide.md) - [Installing Artifacts](installing_artifacts.md) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 0747554afe7..54f64c6601f 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -7,6 +7,10 @@ project: instrumentation slows a library down substantially, and the report is only worth producing for projects whose test suites are thorough enough to give a meaningful signal. +This page is the reference. For a plain-language walkthrough of a single run, +start with +[Nightly Coverage Implementation](nightly_coverage_implementation.md). + ## Enabling coverage for a local build Pass `-D_ENABLE_COVERAGE=ON` at the top level configure. The project diff --git a/docs/development/nightly_coverage_implementation.md b/docs/development/nightly_coverage_implementation.md new file mode 100644 index 00000000000..2af86734a51 --- /dev/null +++ b/docs/development/nightly_coverage_implementation.md @@ -0,0 +1,359 @@ +# Code Coverage in TheRock — A Simple Walkthrough + +How a coverage report gets produced, start to finish, in plain terms. This is +the narrative introduction; [Code Coverage](code_coverage.md) is the reference, +with the CMake options, the CI walkthrough and how to onboard a project. + +______________________________________________________________________ + +## What is this trying to do? + +Answer one question: **when we run hipRAND's tests, how much of hipRAND's code +actually runs?** + +The green run answered it — 77% of lines, 82% of functions. Getting to that +number is what everything below is about. + +______________________________________________________________________ + +## First, three words you need + +**Instrumented** — a library compiled with extra bookkeeping. Every time a line +runs, it bumps a counter. Instrumented code is slower and bigger, which is why +we don't ship it. You get it by compiling with `-fprofile-instr-generate`. + +**profraw** — the counter file an instrumented program dumps when it exits. +Raw, per-process, not human-readable. + +**lcov** — the final, readable report: which lines ran and which didn't. + +The whole pipeline is just: build instrumented → run tests → collect profraw → +convert to lcov. + +______________________________________________________________________ + +## The one idea that explains the design + +You might expect: build everything instrumented, run the tests, done. + +The problem is that hipRAND doesn't run alone. Its tests also load rocRAND, +hipBLAS, the HIP runtime and more. If *those* were instrumented too, they'd all +write counters, and hipRAND's report would include code that isn't hipRAND's. +Worse, hipRAND's percentage would move whenever some unrelated library changed. + +So we want **exactly one instrumented library, and everything else normal.** + +Building "everything else normal" from scratch would take hours. But a normal +build of the whole stack already exists — the **regular nightly** runs every +night. So we reuse it: + +> Take last night's normal build, and swap in **just** our instrumented hipRAND. + +That is the core trick. It's why two different builds are involved. + +______________________________________________________________________ + +## The two builds + +| | Where it comes from | What we take | +| ------------ | --------------------------------------------------------------- | ----------------------------- | +| **Baseline** | Last night's regular nightly (`ROCm/rockrel` run `34068476531`) | The whole ROCm stack, normal | +| **Coverage** | Our own run, right now | Only the instrumented hipRAND | + +______________________________________________________________________ + +## The flow + +```mermaid +flowchart TD + A["1. Someone starts a coverage run
(picks: which project?)"] + B["2. Work out the plan
'test hipRAND, instrument hipRAND'"] + C["3. Build the compiler + runtime"] + D["4. Build math-libs
hipRAND instrumented, rest normal"] + E["5. On a GPU machine:
install last night's normal build"] + F["6. Swap in our instrumented hipRAND"] + G["7. Run hipRAND's tests
counters written to .profraw"] + H["8. Merge profraw, convert to lcov"] + I["Report: 77% of lines covered"] + + A --> B --> C --> D --> E --> F --> G --> H --> I + + style D fill:#ffe6cc + style F fill:#ffe6cc + style I fill:#d5e8d4 +``` + +The two orange boxes are the parts unique to coverage. Everything else is +TheRock's ordinary build and test machinery. + +______________________________________________________________________ + +## Step by step + +### Step 1 — Start the run + +Today it starts automatically when a pull request touches a coverage file. That +trigger (`coverage_nightly_pr_check.yml`) is temporary scaffolding for +validation and gets deleted before merge. After that, you start it by hand from +the Actions tab. + +You choose one thing that matters: **which projects to measure**, through the +`projects_to_test` input. It takes a comma-separated list of project names, or +one of the group aliases (`rocm_libraries_all`, `rocm_systems_all`, `all`). +Leaving it empty measures every onboarded project the workflow can build. The +walkthrough below follows a single-project run, `hiprand`, because it is the +easiest to read; nothing changes for a larger selection except how many test +and report jobs fan out at the end. + +### Step 2 — Work out the plan + +`setup_coverage_matrix`*, about 6 seconds* + +A script (`configure_coverage_ci.py`) looks up hipRAND in its registry and +figures out everything downstream needs: + +- Turn coverage on with `-DHIPRAND_ENABLE_COVERAGE=ON` +- hipRAND lives in the `math-libs` build stage, so build that +- Its files live in `math-libs/hipRAND/stage` +- Report on `lib/libhiprand.so*` +- Run the test suite called `hiprand` + +If you asked for something that can't be built, it stops **here** with a clear +message rather than failing two hours later. + +### Step 3 — Build the compiler + +*about 7 minutes* + +Coverage needs the compiler's profiling runtime, so this is built first. Nothing +project-specific happens yet. + +### Step 4 — Build the libraries, with hipRAND instrumented + +*about 59 minutes — the expensive step* + +The whole `math-libs` stage gets built, but only hipRAND is instrumented. The +flag flows through three hands to get there: + +``` +configure_coverage_ci.py says -DHIPRAND_ENABLE_COVERAGE=ON + ↓ +therock_subproject.cmake translates it to -DBUILD_CODE_COVERAGE=ON + ↓ +hipRAND's own CMakeLists adds -fprofile-instr-generate +``` + +**Why the translation?** Because projects don't agree on a name. hipRAND calls +its option `BUILD_CODE_COVERAGE`, RCCL calls it `ENABLE_CODE_COVERAGE`, others +use something else. TheRock uses one consistent name and translates per project. +Skipping this step was a real bug — hipRAND built without instrumentation and +quietly reported nothing. + +**Why build all of math-libs?** Because a stage, not a project, is the unit CI +knows how to build. The coverage workflow hands the ordinary build workflow +`stage_name: math-libs`, and everything downstream keys off that one input: +`fetch_sources.py --stage`, the configure step's `--stage`, and finally +`cmake --build --target stage-math-libs`. "Build only hipRAND" isn't a smaller +version of that job — it's a job that doesn't exist. There's a floor anyway, +since hipRAND sits on rocRAND, which sits on rocPRIM and the HIP runtime. + +This is the design working, not a cost being tolerated. The whole point is to +pay for one instrumented stack per night and test every project against it, so +the build is the same size whether you measure one project or twelve. The +numbers say so: hipRAND alone took 59 minutes, and all twelve took 68. + +> Careful with the two verbs: the whole stage is **built**, but only the +> selected projects are **instrumented**. Everything else compiles normally. +> Keeping each project's numbers to itself happens later, in steps 6 and 8 — +> not by restricting what gets built. + +### Step 5 — Install last night's normal build + +*on a real GPU machine* + +The test machine downloads and installs the entire baseline stack — normal, +uninstrumented, nothing special. + +At this moment there is **no** instrumented code on the machine at all. + +### Step 6 — Swap in the instrumented hipRAND + +`overlay_coverage_artifacts.py` + +Now copy our instrumented hipRAND files over the baseline's copies: + +``` +Overlaying .../rand_dev.../math-libs/hipRAND/stage onto .../build +Overlaying .../rand_lib.../math-libs/hipRAND/stage onto .../build +Overlaying .../rand_test.../math-libs/hipRAND/stage onto .../build +Overlaid 3 instrumented subtree(s) +``` + +Result: a complete, working ROCm install where **exactly one library** counts +what it does. + +> Note it copies `math-libs/hipRAND/stage`, not the whole `rand` package. That +> package holds **both** rocRAND and hipRAND, and we want rocRAND left alone. + +### Step 7 — Run the tests + +hipRAND's normal test suite runs, unchanged. One environment variable tells the +instrumented library where to dump its counters: + +``` +LLVM_PROFILE_FILE=.../hiprand-shard1-%p-%m.profraw +``` + +`%p` is the process id and `%m` identifies the binary, so nothing overwrites +anything else. This run produced **8** profraw files. + +### Step 8 — Build the report + +`merge_coverage_report.py`*, about 30 seconds* + +Three things happen: + +1. `llvm-profdata` merges the 8 profraw files into one index +1. `llvm-cov` reads that index plus `lib/libhiprand.so*` and writes lcov +1. The lcov file is uploaded (and sent to Codecov if a token is configured) + +Pointing `llvm-cov` at only hipRAND's library is the second safeguard: even if +something else were instrumented, it wouldn't reach the report. + +______________________________________________________________________ + +## The result + +From run +[34135220913](https://github.com/ROCm/TheRock/actions/runs/34135220913): + +``` +Merging 8 profraw file(s) into coverage.profdata +Exporting lcov for 1 object(s) to coverage.info +``` + +| | | +| ----------------- | --------------------- | +| Lines covered | 155 / 201 — **77.1%** | +| Functions covered | 28 / 34 — **82.4%** | +| File measured | `hiprand.cpp` | +| Total time | about 70 minutes | + +______________________________________________________________________ + +## Which files do what + +| File | Its one job | +| ------------------------------------ | -------------------------------------------- | +| `coverage_nightly_pr_check.yml` | Starts the run (temporary) | +| `multi_arch_ci_coverage_nightly.yml` | Runs the builds in order | +| `multi_arch_ci_coverage_linux.yml` | Per project: test, then report | +| `test_component.yml` | Installs, swaps, runs tests | +| `configure_coverage_ci.py` | The registry — knows every project's details | +| `therock_subproject.cmake` | Translates the coverage flag; fixes linking | +| `overlay_coverage_artifacts.py` | Does the swap in step 6 | +| `merge_coverage_report.py` | Makes the report in step 8 | + +______________________________________________________________________ + +## Appendix — every file in the PR + +Twenty-one files: eleven new, ten edits to things that already existed. The +table above covers the eight you need for the flow; this is the complete list, +grouped by the part of the pipeline each one belongs to. + +| # | Part | What lives here | +| --- | ------------------------------ | --------------------------------------------------- | +| 1 | Entry points and orchestration | Starting a run and sequencing the jobs | +| 2 | Planning | Deciding what to build, instrument, test and report | +| 3 | Build and instrumentation | Turning the coverage flag into compiler flags | +| 4 | Artifacts and test execution | Assembling the install tree and running tests | +| 5 | Reporting | profraw to lcov | +| 6 | Unit tests | The fast checks that run on every PR | +| 7 | Documentation | — | + +A useful thing to notice while reading: every one of the ten edits is additive +and defaults to off or empty, so regular CI behaves exactly as before. Coverage +rides on the existing build and test machinery rather than forking it. The one +exception worth knowing about is `CMakeLists.txt`, which runs the registry +script on *every* configure, coverage or not, to learn the project lists — a +few milliseconds, and the reason CMake and CI cannot drift apart. + +### 1. Entry points and orchestration + +All three are new files. + +| File | What it does | Called by | Calls | +| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `coverage_nightly_pr_check.yml` **(temporary — deleted before merge)** | Makes the coverage workflow runnable from a branch and visible as a PR check, which `workflow_dispatch` alone cannot do | GitHub, on a `pull_request` touching any of nine coverage paths — deliberately narrow, since every run is a multi-hour build | `multi_arch_ci_coverage_nightly.yml` | +| `multi_arch_ci_coverage_nightly.yml` | The real entry point and top-level orchestrator. Five jobs: `setup_coverage_matrix`, `build_instrumented_compiler_runtime`, `build_instrumented_math_libs`, `coverage_report` (project × GPU family), `ci_summary` | A human, from the Actions tab. Today also the PR check above, via a `workflow_call` trigger removed alongside it | `configure_coverage_ci.py`; `multi_arch_build_portable_linux_artifacts.yml` twice (compiler-runtime, math-libs); `multi_arch_ci_coverage_linux.yml` per project | +| `multi_arch_ci_coverage_linux.yml` | Everything for one project on one GPU family, in three jobs: `configure_test_matrix`, `test_coverage`, `aggregate_coverage` | `multi_arch_ci_coverage_nightly.yml`, once per matrix entry | `configure_coverage_ci.py`, `test_component.yml`, `artifact_manager.py`, `merge_coverage_report.py` | + +### 2. Planning + +One new file, and the reason the rest of the pipeline stays consistent. + +| File | What it does | Called by | Calls | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| `build_tools/github_actions/configure_coverage_ci.py` | The registry and single source of truth. Per project it records the build stage, the upstream option name, the artifacts and stage directories to overlay, the object globs to report on, and the test suite to run. Resolves a selection into a job matrix and rejects anything unbuildable up front, so a bad request stops in six seconds rather than two hours | Three places, which is what makes it the source of truth: the nightly's matrix job, the per-project workflow's configure job, and `CMakeLists.txt` via `--emit-cmake` | Nothing — pure data and logic | + +### 3. Build and instrumentation + +All three already existed. The flags they add are inert unless coverage is on, +with the one caveat from above: `CMakeLists.txt` reads the registry on every +configure regardless. + +| File | What it does | Called by | Calls | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `multi_arch_build_portable_linux_artifacts.yml` | Builds one stage in a container and uploads its artifacts. Coverage adds an `extra_cmake_options` input, appended **after** the stage-derived arguments, since the last `-D` wins. This is how the coverage flags reach the build without coverage needing a build workflow of its own | Regular multi-arch CI, and now the coverage nightly | The configure step, then `cmake --build --target stage-` | +| `CMakeLists.txt` | Declares the three group options (`THEROCK_COVERAGE_ALL`, `..._ROCM_LIBRARIES_ALL`, `..._ROCM_SYSTEMS_ALL`) and expands whichever is set into individual `_ENABLE_COVERAGE` flags. An explicit `-DFOO_ENABLE_COVERAGE=OFF` still wins | Every CMake configure, coverage or not | `configure_coverage_ci.py --emit-cmake`, then includes the generated `therock_coverage_projects.cmake` | +| `cmake/therock_subproject.cmake` | Where the generic flag becomes a real one. Translates `_ENABLE_COVERAGE` into whatever the subproject implements; adds `-ldl` and `-lpthread`, which the profile runtime needs and the driver does not supply; and scopes instrumentation to host code with `-Xarch_device`, injected into the compile rule so it lands after the project's own flags | Every subproject declaration in the build | Generates a small CMake file handed to the subproject as `CMAKE_PROJECT_INCLUDE` | + +### 4. Artifacts and test execution + +This is the part that assembles a working install in which exactly one library +is instrumented. Only `overlay_coverage_artifacts.py` is new; the other three +already existed and are extended. + +| File | What it does | Called by | Calls | +| -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `test_component.yml` | The shared test workflow behind every component test in the repo. Coverage adds seven inputs and four steps: install from the baseline run, overlay the instrumented project, point `LLVM_PROFILE_FILE` at a per-shard path, and upload the profraw files. Also raises the test timeout to a 60-minute floor, since instrumented `-O0` builds run far slower | Every component-test caller in the repo, and now `multi_arch_ci_coverage_linux.yml` | `setup_test_environment`, then `overlay_coverage_artifacts.py` (only when a baseline run is set), then the project's own test suite | +| `.github/actions/setup_test_environment/action.yml` | Composite action that downloads and installs a ROCm build on the test machine. Coverage adds `RUN_GITHUB_REPO`, because the baseline is published by `ROCm/rockrel` and the bucket lookup fails if it queries the wrong repository | `test_component.yml` only | `install_rocm_from_artifacts.py` | +| `build_tools/github_actions/overlay_coverage_artifacts.py` **(new)** | Step 6, the swap. Copies the instrumented project's stage directories over the baseline install. Most of its care goes into filesystem edge cases: merging into an existing tree, preserving symlinks, and either kind of symlink/regular-file collision | `test_component.yml` | `artifact_manager.py`, imported as a module rather than shelled out to, fetching with `--artifact-names` and `--stage all` | +| `build_tools/artifact_manager.py` | Pre-existing artifact transport: pushes a stage's outputs to S3 and fetches them back. Coverage adds `--artifact-names`, an allowlist, because the existing `--exclude-artifacts` denylist is the wrong shape for fetching one project's files | Six pre-existing callers (`fetch_artifacts.py`, `build_tarballs.py`, `post_stage_upload.py`, `baseline_runs.py`, `stage_reuse_decision.py`, `expand_amdgpu_families.py`) plus `overlay_coverage_artifacts.py` | The S3 artifact backend | + +### 5. Reporting + +| File | What it does | Called by | Calls | +| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | --------------------------------------------- | +| `build_tools/github_actions/merge_coverage_report.py` **(new)** | Step 8. Finds the profraw files, resolves the object globs (collapsing versioned symlinks like `libfoo.so`, `.so.1`, `.so.1.2` into one object), merges and exports lcov. Fails loudly when a glob matches nothing, which is what caught the hipDNN naming bug instead of quietly emitting an empty report | `multi_arch_ci_coverage_linux.yml`, in `aggregate_coverage` | `llvm-profdata merge`, then `llvm-cov export` | + +### 6. Unit tests + +All four run in the **Unit Tests** workflow, whose "Test build_tools" step runs +`pytest` from `build_tools/` on Ubuntu and Windows, on every pull request. No +GPU, no network, no build — the whole set finishes in about two seconds. That +speed is the point: the alternative way to catch these mistakes is a two-hour +instrumented run on a GPU runner, which is how the hipDNN and hipTensor bugs +were actually found. + +| File | Covers | Notable cases | +| ---------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `configure_coverage_ci_test.py` **(new)** | The registry | Alias expansion, blocked projects, and whole-registry invariants: every project declares report inputs and names an upstream option | +| `overlay_coverage_artifacts_test.py` **(new)** | The swap | Symlink preservation, both file/symlink crossing cases, and failing loudly when nothing was overlaid | +| `merge_coverage_report_test.py` **(new)** | The report | Tool lookup, versioned-symlink collapsing, and the failure policy for empty profiles and unmatched globs | +| `artifact_manager_tool_test.py` | `--artifact-names` | Two tests beside the existing `--exclude-artifacts` ones, keeping allowlist and denylist symmetric | + +Note the last one edits a large pre-existing file rather than adding a new one. + +### 7. Documentation + +| File | What it covers | +| --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs/development/code_coverage.md` **(new)** | The reference, in one file: the CMake options, the end-to-end CI walkthrough, why instrumentation is host-only, why the test timeout has a floor, and how to onboard a project or a stage | +| `docs/development/nightly_coverage_implementation.md` **(new)** | This document — the narrative introduction to the same pipeline | +| `docs/development/ci_overview.md` | Places coverage among the CI entry points, noting it is separate because it rebuilds what it measures | +| `docs/development/development_guide.md` | Documents `{PROJECT}_ENABLE_COVERAGE` next to the other build flags, including the upper-case rule: `HIPDNN_ENABLE_COVERAGE`, not `hipDNN_ENABLE_COVERAGE` | +| `docs/development/README.md` | Index entries for the new documents | + +______________________________________________________________________ From cb091283715e03a9d4b617fa9530e8c4be641c8a Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Wed, 9 Sep 2026 04:10:34 -0500 Subject: [PATCH 23/70] coverage: publish an HTML report and a text summary beside the lcov The report job produced coverage.info and nothing else. That is the format Codecov consumes, not one anybody reads: answering "what is the number" meant opening Codecov, and there was no line-by-line view attached to the run at all. merge_coverage_report.py now renders the one profdata index three ways. llvm-cov report gives coverage_summary.txt, the per-file table with the totals on the last line, and llvm-cov show gives html/index.html, source annotated line by line and C++ names demangled. All three go into the existing coverage-report-- artifact, and the totals are also written to the job's summary page so the number is visible without downloading anything. Only the HTML opens source files; lcov and the table are produced from the coverage mappings alone. The report job checks out TheRock, not the projects it measures, so it now runs fetch_sources.py for the stage that built the project, which the registry already records and which is threaded through the matrix as build_stage. Paths are recorded as the build container saw them, so --path-equivalence maps that prefix onto the workspace. That fetch is continue-on-error and runs after the artifacts are already down. It is the only step here that can fail for reasons of its own -- a large shallow checkout on a runner that has just downloaded an instrumented stack -- and losing it should cost the annotated view rather than the whole report. Files generated into the build tree, version headers and the like, exist in no checkout and show as uncovered in the HTML; the text table still counts them. Co-authored-by: Cursor --- .../multi_arch_ci_coverage_linux.yml | 67 +++++++- .../multi_arch_ci_coverage_nightly.yml | 1 + .../github_actions/configure_coverage_ci.py | 3 + .../github_actions/merge_coverage_report.py | 147 +++++++++++++++++- .../tests/merge_coverage_report_test.py | 85 ++++++++++ docs/development/code_coverage.md | 57 +++++-- 6 files changed, 342 insertions(+), 18 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index e4ab8b505d9..a37c05082eb 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -36,6 +36,13 @@ on: type: string default: "" description: "Extra install_rocm_from_artifacts.py args for the report job" + build_stage: + type: string + default: "" + description: >- + Build stage owning the project. The report job fetches this stage's + sources so the HTML rendering has something to annotate; the lcov and + text outputs do not need them. Empty skips the fetch and the HTML. baseline_run_id: type: string default: "" @@ -230,21 +237,75 @@ jobs: --output-dir=${OUTPUT_ARTIFACTS_DIR} \ ${FETCH_ARTIFACT_ARGS} - - name: Merge profraw and generate lcov report + # Only the HTML rendering reads source, so this runs after the artifacts + # are already down: a failure here, including running the runner out of + # disk, costs the annotated view and leaves lcov and the text table. + - name: Fetch sources for the annotated report + if: ${{ inputs.build_stage != '' }} + id: fetch_sources + timeout-minutes: 30 + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + STAGE_NAME: ${{ inputs.build_stage }} + run: | + gh auth setup-git --hostname github.com + ./build_tools/fetch_sources.py --stage "${STAGE_NAME}" --jobs 12 --depth 1 + + - name: Merge profraw and generate reports + env: + # Sources are recorded at the path the build container used, which is + # not where they sit here, so llvm-cov is told how to get from one to + # the other. Generated files under build/ are in neither and are + # reported as uncovered; the text summary still counts them. + BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} run: | + html_args=() + if [[ "${HTML_ENABLED}" == "true" ]]; then + html_args=( + --html-output="${REPORT_DIR}/html" + --project-title="${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" + ) + else + echo "::warning::Sources unavailable; skipping the annotated HTML report." + fi + python ./build_tools/github_actions/merge_coverage_report.py \ --profraw-dir="${PROFRAW_DIR}" \ --rocm-dir="${OUTPUT_ARTIFACTS_DIR}" \ --object-globs="${{ inputs.object_globs }}" \ --profdata-output="${REPORT_DIR}/coverage.profdata" \ - --lcov-output="${REPORT_DIR}/coverage.info" + --lcov-output="${REPORT_DIR}/coverage.info" \ + --summary-output="${REPORT_DIR}/coverage_summary.txt" \ + --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ + "${html_args[@]}" + + # Puts the totals on the run's summary page, so the number is readable + # without downloading anything. + - name: Publish the summary to the job page + if: ${{ always() }} + run: | + summary="${REPORT_DIR}/coverage_summary.txt" + [[ -f "${summary}" ]] || exit 0 + { + echo "## Coverage: ${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" + echo '```' + tail -n 3 "${summary}" + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" - name: Upload coverage report if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-report-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} - path: ${{ env.REPORT_DIR }}/coverage.info + # The whole directory: coverage.info for Codecov, coverage_summary.txt + # for the numbers, and html/index.html for the annotated report. + path: | + ${{ env.REPORT_DIR }}/coverage.info + ${{ env.REPORT_DIR }}/coverage_summary.txt + ${{ env.REPORT_DIR }}/html if-no-files-found: warn # Skipped rather than failed when the repository has no Codecov token, diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 9e016a616e6..7dc7859e9c6 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -253,6 +253,7 @@ jobs: test_component: ${{ matrix.coverage.test_component }} object_globs: ${{ matrix.coverage.object_globs }} fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} + build_stage: ${{ matrix.coverage.build_stage }} codecov_flag: ${{ matrix.coverage.codecov_flag }} amdgpu_families: ${{ matrix.coverage.amdgpu_families }} baseline_run_id: ${{ inputs.baseline_run_id }} diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 25b7930aa3a..5903c67f3da 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -717,6 +717,9 @@ def build_coverage_matrix( "coverage_config_ref": config_ref, "object_globs": ",".join(project.object_globs), "fetch_artifact_args": project.fetch_artifact_args, + # The report job fetches this stage's sources so the HTML + # rendering has something to annotate. + "build_stage": project.stage, "codecov_flag": project.codecov_flag or project_key, "amdgpu_families": family, "artifact_names": ",".join(project.artifact_names), diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py index e5f4b6eea2a..ec3b742d100 100644 --- a/build_tools/github_actions/merge_coverage_report.py +++ b/build_tools/github_actions/merge_coverage_report.py @@ -8,6 +8,12 @@ merges them into a single `.profdata` index and exports lcov for the instrumented objects. +Three renderings come out of the same index. lcov is what Codecov consumes, a +text table is what a reader wants when the question is just "what is the +number", and the HTML tree is the annotated line-by-line view. The last one is +the only one that reads the source files: the others report from the coverage +mappings alone. + The `llvm-profdata` and `llvm-cov` binaries must come from the same compiler that built the instrumented objects; a version mismatch produces an unhelpfully generic "malformed instrumentation profile data" error. Both live @@ -87,14 +93,33 @@ def merge_profraw(llvm_profdata: Path, profraw_files: list[Path], output: Path) subprocess.run(command, check=True) -def export_lcov( - llvm_cov: Path, profdata: Path, objects: list[Path], output: Path -) -> None: +def build_cov_command( + llvm_cov: Path, + subcommand: str, + profdata: Path, + objects: list[Path], + path_equivalence: str | None, +) -> list[str]: + """Assembles the part of an llvm-cov command line the three outputs share.""" # llvm-cov takes the first object positionally and the rest via -object. - command = [str(llvm_cov), "export", str(objects[0])] + command = [str(llvm_cov), subcommand, str(objects[0])] for obj in objects[1:]: command.extend(["-object", str(obj)]) - command.extend([f"-instr-profile={profdata}", "--format=lcov"]) + command.append(f"-instr-profile={profdata}") + if path_equivalence: + command.append(f"-path-equivalence={path_equivalence}") + return command + + +def export_lcov( + llvm_cov: Path, + profdata: Path, + objects: list[Path], + output: Path, + path_equivalence: str | None = None, +) -> None: + command = build_cov_command(llvm_cov, "export", profdata, objects, path_equivalence) + command.append("--format=lcov") output.parent.mkdir(parents=True, exist_ok=True) logging.info("Exporting lcov for %d object(s) to %s", len(objects), output) @@ -102,6 +127,58 @@ def export_lcov( subprocess.run(command, check=True, stdout=lcov_file) +def write_summary( + llvm_cov: Path, + profdata: Path, + objects: list[Path], + output: Path, + path_equivalence: str | None = None, +) -> None: + """Writes the per-file coverage table, and echoes the totals to the log. + + This reads nothing but the coverage mappings, so it is the one rendering + that is complete even when the sources are not on the machine. + """ + command = build_cov_command(llvm_cov, "report", profdata, objects, path_equivalence) + + output.parent.mkdir(parents=True, exist_ok=True) + logging.info("Writing coverage summary to %s", output) + result = subprocess.run(command, check=True, capture_output=True, text=True) + output.write_text(result.stdout) + + for line in result.stdout.splitlines(): + if line.startswith("TOTAL"): + logging.info("%s", line) + + +def write_html( + llvm_cov: Path, + profdata: Path, + objects: list[Path], + output_dir: Path, + path_equivalence: str | None = None, + project_title: str | None = None, + demangler: Path | None = None, +) -> None: + """Renders the browsable report into output_dir, entry point index.html. + + Unlike the other two this opens the source files, so anything not on the + machine is reported by llvm-cov as uncovered rather than annotated. Files + generated into the build tree are never in a source checkout, so a few + misses here are expected and not worth failing over. + """ + command = build_cov_command(llvm_cov, "show", profdata, objects, path_equivalence) + command.extend(["--format=html", f"-output-dir={output_dir}"]) + if project_title: + command.append(f"--project-title={project_title}") + if demangler: + command.append(f"-Xdemangler={demangler}") + + output_dir.mkdir(parents=True, exist_ok=True) + logging.info("Writing HTML report to %s", output_dir / "index.html") + subprocess.run(command, check=True) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -142,6 +219,34 @@ def main(argv: list[str] | None = None) -> int: default=Path("coverage-report/coverage.info"), help="Path for the generated lcov report", ) + parser.add_argument( + "--summary-output", + type=Path, + default=None, + help="Path for the plain text per-file coverage table. Needs no " + "sources, so it is complete wherever it runs", + ) + parser.add_argument( + "--html-output", + type=Path, + default=None, + help="Directory for the browsable HTML report, entry point " + "index.html. Renders source, so the sources must be present and " + "--path-equivalence set if they moved since the build", + ) + parser.add_argument( + "--path-equivalence", + type=str, + default=None, + help="',' remapping for source paths recorded at build " + "time, e.g. '/__w/TheRock/TheRock,${GITHUB_WORKSPACE}'", + ) + parser.add_argument( + "--project-title", + type=str, + default=None, + help="Heading for the HTML report's index page", + ) parser.add_argument( "--allow-empty", action="store_true", @@ -177,8 +282,38 @@ def main(argv: list[str] | None = None) -> int: return 1 merge_profraw(llvm_profdata, profraw_files, args.profdata_output) - export_lcov(llvm_cov, args.profdata_output, objects, args.lcov_output) + export_lcov( + llvm_cov, + args.profdata_output, + objects, + args.lcov_output, + args.path_equivalence, + ) logging.info("Wrote coverage report to %s", args.lcov_output) + + if args.summary_output: + write_summary( + llvm_cov, + args.profdata_output, + objects, + args.summary_output, + args.path_equivalence, + ) + + if args.html_output: + # Demangling is what makes the C++ report readable; skip it rather + # than fail if this distribution has no llvm-cxxfilt. + demangler = llvm_bin_dir / f"llvm-cxxfilt{EXECUTABLE_SUFFIX}" + write_html( + llvm_cov, + args.profdata_output, + objects, + args.html_output, + args.path_equivalence, + args.project_title, + demangler if demangler.is_file() else None, + ) + return 0 diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index cb54ae5c270..5d71435cb66 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -144,6 +144,64 @@ def test_export_passes_the_first_object_positionally(self): self.assertIn("-object", command) self.assertEqual(command[-2:], [f"-instr-profile={profdata}", "--format=lcov"]) + def test_summary_reports_and_captures_the_table(self): + objects = [self.touch("lib/a.so")] + output = self.root / "out" / "coverage_summary.txt" + table = "Filename Cover\nTOTAL 77.78%\n" + + with mock.patch("subprocess.run", return_value=mock.Mock(stdout=table)) as run: + merge_coverage_report.write_summary( + Path("/llvm/llvm-cov"), Path("/tmp/c.profdata"), objects, output + ) + + self.assertEqual(run.call_args.args[0][1], "report") + self.assertEqual(output.read_text(), table) + + def test_html_renders_into_a_directory(self): + objects = [self.touch("lib/a.so")] + output_dir = self.root / "out" / "html" + + with mock.patch("subprocess.run") as run: + merge_coverage_report.write_html( + Path("/llvm/llvm-cov"), + Path("/tmp/c.profdata"), + objects, + output_dir, + project_title="hiprand", + demangler=Path("/llvm/llvm-cxxfilt"), + ) + + command = run.call_args.args[0] + self.assertEqual(command[1], "show") + self.assertIn("--format=html", command) + self.assertIn(f"-output-dir={output_dir}", command) + self.assertIn("--project-title=hiprand", command) + self.assertIn("-Xdemangler=/llvm/llvm-cxxfilt", command) + + def test_path_equivalence_reaches_every_rendering(self): + objects = [self.touch("lib/a.so")] + remap = "/__w/TheRock/TheRock,/home/runner/work" + + for build in ( + lambda: merge_coverage_report.export_lcov( + Path("/llvm/llvm-cov"), + Path("/tmp/c.profdata"), + objects, + self.root / "out" / "coverage.info", + remap, + ), + lambda: merge_coverage_report.write_html( + Path("/llvm/llvm-cov"), + Path("/tmp/c.profdata"), + objects, + self.root / "out" / "html", + remap, + ), + ): + with mock.patch("subprocess.run") as run: + build() + self.assertIn(f"-path-equivalence={remap}", run.call_args.args[0]) + class MainTest(TempDirTestBase): def _argv(self, *extra: str) -> list[str]: @@ -205,6 +263,33 @@ def test_happy_path_invokes_both_llvm_tools(self): ], ) + def test_summary_and_html_are_opt_in(self): + self.touch("profraw/shard0/a.profraw") + self.touch("rocm/lib/libhiprand.so") + llvm_bin_dir = self.root / "rocm" / "lib" / "llvm" / "bin" + llvm_bin_dir.mkdir(parents=True) + for tool in ("llvm-profdata", "llvm-cov"): + (llvm_bin_dir / f"{tool}{merge_coverage_report.EXECUTABLE_SUFFIX}").touch() + + with mock.patch("subprocess.run", return_value=mock.Mock(stdout="")) as run: + exit_code = merge_coverage_report.main( + self._argv( + "--profdata-output", + os.fspath(self.root / "out" / "coverage.profdata"), + "--lcov-output", + os.fspath(self.root / "out" / "coverage.info"), + "--summary-output", + os.fspath(self.root / "out" / "coverage_summary.txt"), + "--html-output", + os.fspath(self.root / "out" / "html"), + ) + ) + + self.assertEqual(exit_code, 0) + # merge, then export, report and show off the one index. + subcommands = [call.args[0][1] for call in run.call_args_list] + self.assertEqual(subcommands, ["merge", "export", "report", "show"]) + if __name__ == "__main__": unittest.main() diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 54f64c6601f..8113b752e64 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -144,6 +144,25 @@ that built the instrumented objects, so the script prefers the copies under `/lib/llvm/bin`. A version mismatch surfaces as an unhelpfully generic "malformed instrumentation profile data" error. +`--summary-output` adds the per-file table as plain text and `--html-output` +the browsable report, entry point `index.html`: + +```bash +python build_tools/github_actions/merge_coverage_report.py \ + --profraw-dir coverage-report/profraw \ + --rocm-dir build/dist/rocm \ + --object-globs "lib/libhiprand.so*" \ + --summary-output coverage-report/coverage_summary.txt \ + --html-output coverage-report/html +``` + +Only the HTML reads the source files; lcov and the text table are produced +from the coverage mappings alone and are complete wherever they run. Source +paths are recorded as they were at build time, so if the tree has moved +since, `--path-equivalence ,` tells `llvm-cov` how to get from one +to the other. Files generated into the build tree are in no source checkout +and are reported as uncovered in the HTML; the text table still counts them. + ## Coverage CI A coverage run builds the stages its selection needs, instruments only the @@ -333,15 +352,35 @@ profiles. `merge_coverage_report.py` then collects the profiles recursively, expands the project's `object_globs` and deduplicates them by real path (a versioned symlink family resolves to one file), runs `llvm-profdata merge -sparse` into a -single profdata index, and exports lcov. Two conditions are fatal rather than -reported as zero coverage: finding no profraw files at all, which means the -tests never ran or the instrumented libraries were not the ones loaded at -runtime; and matching no objects, which means `object_globs` does not describe -what the project actually installs. - -The lcov report uploads as a workflow artifact and goes to Codecov under the -project's flag. The Codecov step is skipped rather than failed when no -`CODECOV_TOKEN` is configured, so forks still get the lcov artifact. +single profdata index, and renders it three ways. Two conditions are fatal +rather than reported as zero coverage: finding no profraw files at all, which +means the tests never ran or the instrumented libraries were not the ones +loaded at runtime; and matching no objects, which means `object_globs` does not +describe what the project actually installs. + +The `coverage-report--` artifact holds all three: + +| File | What it is | +| ---------------------- | ----------------------------------------------------- | +| `coverage.info` | lcov, the format Codecov consumes | +| `coverage_summary.txt` | The per-file table, with the totals on the final line | +| `html/index.html` | The browsable report, source annotated line by line | + +The totals are also written to the job's summary page, so the number is +readable without downloading anything. + +Only the HTML rendering opens source files, and the report job checks out +TheRock rather than the projects it measures, so it first runs +`fetch_sources.py` for the stage that built the project. Source paths are +recorded as the build container saw them, and `--path-equivalence` maps that +prefix onto the workspace. That step is `continue-on-error`: losing it costs +the annotated view and nothing else, since lcov and the text table come from +the coverage mappings. Files generated into the build tree — version headers +and the like — exist in no checkout and show as uncovered in the HTML while +the text table still counts them. + +The Codecov step is skipped rather than failed when no `CODECOV_TOKEN` is +configured, so forks still get the artifact. ## Selecting projects to run From 476a37805ec0c9f806eccd2a317775743d3eaebb Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Thu, 10 Sep 2026 05:26:47 -0500 Subject: [PATCH 24/70] docs: correct the coverage entry point description in the CI overview The baseline nightly is optional, and passing one is precisely the case where the coverage run does reuse the regular build, so describing the run as not reusing it was backwards. The report job also emits a text summary and an HTML report now, not just the lcov file. Also list the coverage docs in Further Reading, which had been the only body reference missing from those lists. Co-authored-by: Cursor --- docs/development/ci_overview.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index db12ce663d0..d33df349104 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -50,11 +50,11 @@ the job starts on a Windows runner, then runs artifact fetch, configure, build, and upload steps inside a WSL Ubuntu shell. See [WSL ROCDXG CI Stage](wsl_rocdxg.md) for details. -Code coverage runs on its own entry point, since it rebuilds the projects it -measures with instrumentation rather than reusing the regular build: +Code coverage has its own entry point, because the projects it measures have to +be rebuilt with instrumentation before their tests can be run: -- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - full-stack instrumented build, dispatched manually against a recent nightly - - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - per-project test and report for one GPU family +- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - builds one instrumented stack covering every coverage-enabled project. Dispatched by hand: pass the run id of a recent nightly as `baseline_run_id` to take the non-instrumented dependencies from it, or leave it empty to build the whole stack within the run + - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - runs one project's tests on one GPU family, then turns the collected profiles into an lcov file, a text summary and a browsable HTML report See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a project, and [Coverage CI](code_coverage.md#coverage-ci) for the end-to-end @@ -168,6 +168,7 @@ See [workflow_outputs.md](workflow_outputs.md) for the S3 layout structure and [ - [artifacts.md](artifacts.md) - Artifact organization and packaging - [build_system.md](build_system.md) - CMake build architecture +- [code_coverage.md](code_coverage.md) - Instrumented builds and coverage reports - [dependencies.md](dependencies.md) - Dependency management - [installing_artifacts.md](installing_artifacts.md) - Installing ROCm from artifacts @@ -184,6 +185,7 @@ See [workflow_outputs.md](workflow_outputs.md) for the S3 layout structure and [ - [workflow_outputs.md](workflow_outputs.md) - CI output directory structure - [github_actions_debugging.md](github_actions_debugging.md) - Debugging GitHub Actions - [ci_behavior_manipulation.md](ci_behavior_manipulation.md) - Controlling CI behavior with labels and inputs +- [code_coverage.md#coverage-ci](code_coverage.md#coverage-ci) - How a coverage run is dispatched, built, tested and reported - [stage_reuse.md](stage_reuse.md) - Reusing unaffected build stages from a baseline run - [manifest_diff.md](manifest_diff.md) - Manifest diff report (submodule SHA changes between two commits) From 5d0b916b172dd762f6d3d0855b74ab99ff59145e Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 15 Sep 2026 03:21:43 -0500 Subject: [PATCH 25/70] ci(coverage): stop passing the deprecated KPACK_SPLIT_ARTIFACTS flag THEROCK_FLAG_KPACK_SPLIT_ARTIFACTS is being deprecated, so the coverage build jobs no longer opt out of split packaging. The compiler-runtime job carried nothing else in extra_cmake_options, so that input goes away there entirely. Drops the note telling local builds to pass the same flag. It already recorded that host-only instrumentation likely made the opt-out unnecessary; pointing readers at a deprecated option is worse than leaving the caveat unstated. Co-authored-by: Cursor --- .github/workflows/multi_arch_ci_coverage_nightly.yml | 5 +---- docs/development/code_coverage.md | 8 -------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage_nightly.yml index 7dc7859e9c6..dab9c62849d 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage_nightly.yml @@ -174,7 +174,6 @@ jobs: stage_display_name: "Stage - Coverage Compiler Runtime" timeout_minutes: 480 # 8 hours (compiler is big) dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} @@ -208,9 +207,7 @@ jobs: timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal amdgpu_family: ${{ matrix.family_info.amdgpu_family }} dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: >- - ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} - -DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF + extra_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 8113b752e64..1336698cf7a 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -35,14 +35,6 @@ Passing the flag for a project with no registered option is a configure error rather than a silent no-op, since the failure it replaces — an uninstrumented build that tests cleanly and reports nothing — surfaces hours later. -> [!NOTE] -> Coverage builds are incompatible with split kernel packaging. The CI -> workflows pass `-DTHEROCK_FLAG_KPACK_SPLIT_ARTIFACTS=OFF`; do the same if you -> build with a target family that enables kernel packing by default. Split -> packaging rearranges the code objects embedded in the library `llvm-cov` is -> later pointed at. Coverage is host-only now, so this may no longer be load -> bearing, but it has not been retested and the flag is cheap to keep. - ### Enabling a whole group Three aggregate options turn on coverage for a whole component group instead of From 47d647b5eead635e4255f1ef3da0d390abaef178 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Sun, 20 Sep 2026 20:01:57 -0700 Subject: [PATCH 26/70] refactor(ci): drive code coverage via test_code_coverage_component and 3-layer workflows Restructure coverage CI into three layers so a Windows layer can be added without duplicating the shared coverage-matrix logic: - multi_arch_ci_coverage.yml (renamed from _nightly): thin parent that computes the matrix once and dispatches the per-platform layer. - multi_arch_ci_coverage_linux.yml (new): per-platform build + fan-out; builds the instrumented stack once, then fans out per project. - multi_arch_ci_coverage_report.yml (renamed from _linux): shared per-project test + aggregate/report. The per-project test job now runs test_code_coverage_component.yml + install_rocm_code_coverage_build.py (install the generic tree from the baseline run, overlay only the project under test from the coverage run) instead of test_component.yml's inline overlay_coverage_artifacts.py path. Split the installer's --run-github-repo (generic baseline fetch) from a new --code-coverage-run-github-repo (instrumented replacement): the baseline (ROCm/rockrel or ROCm/rocm-libraries) and the coverage run (ROCm/TheRock) live in different repos and the artifact bucket lookup is keyed on run-id + repo. coverage_nightly_pr_check.yml is retained (repointed at the renamed parent) so the change stays testable on PRs; the nightly dispatch path cannot fire on a PR trigger. Adds docs/development/coverage_nightly_trigger_snippet.md, a reference for the rocm-libraries nightly to dispatch the orchestrator via a GitHub App token (RFC option C). Co-Authored-By: Claude --- .../workflows/coverage_nightly_pr_check.yml | 11 +- ...nightly.yml => multi_arch_ci_coverage.yml} | 142 ++----- .../multi_arch_ci_coverage_linux.yml | 376 ++++++------------ .../multi_arch_ci_coverage_report.yml | 305 ++++++++++++++ .../test_code_coverage_component.yml | 28 +- .../github_actions/configure_coverage_ci.py | 2 +- .../tests/configure_coverage_ci_test.py | 2 +- .../install_rocm_code_coverage_build.py | 38 +- .../install_rocm_code_coverage_build_test.py | 105 +++++ docs/development/ci_overview.md | 5 +- docs/development/code_coverage.md | 40 +- .../coverage_nightly_trigger_snippet.md | 104 +++++ .../nightly_coverage_implementation.md | 36 +- 13 files changed, 770 insertions(+), 424 deletions(-) rename .github/workflows/{multi_arch_ci_coverage_nightly.yml => multi_arch_ci_coverage.yml} (57%) create mode 100644 .github/workflows/multi_arch_ci_coverage_report.yml create mode 100644 build_tools/tests/install_rocm_code_coverage_build_test.py create mode 100644 docs/development/coverage_nightly_trigger_snippet.md diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index b3f0a47c5b1..64cdfc24daa 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -4,7 +4,7 @@ # TEMPORARY. Delete this file, and the workflow_call trigger it relies on, # before merging the coverage PR. # -# multi_arch_ci_coverage_nightly.yml is triggered by hand, but a +# multi_arch_ci_coverage.yml is triggered by hand, but a # workflow_dispatch only appears once the file is on the default branch. There # is therefore no way to run it from a branch and no way to see it in a PR's # checks, which leaves the phase 1 work unable to demonstrate the one thing it @@ -28,12 +28,13 @@ on: # only the files carrying coverage logic are worth spending one on. paths: - ".github/workflows/coverage_nightly_pr_check.yml" - - ".github/workflows/multi_arch_ci_coverage_nightly.yml" + - ".github/workflows/multi_arch_ci_coverage.yml" - ".github/workflows/multi_arch_ci_coverage_linux.yml" - - ".github/workflows/test_component.yml" + - ".github/workflows/multi_arch_ci_coverage_report.yml" + - ".github/workflows/test_code_coverage_component.yml" - "build_tools/github_actions/configure_coverage_ci.py" - - "build_tools/github_actions/overlay_coverage_artifacts.py" - "build_tools/github_actions/merge_coverage_report.py" + - "build_tools/install_rocm_code_coverage_build.py" - "cmake/therock_subproject.cmake" - "CMakeLists.txt" @@ -47,7 +48,7 @@ concurrency: jobs: coverage: name: Coverage - uses: ./.github/workflows/multi_arch_ci_coverage_nightly.yml + uses: ./.github/workflows/multi_arch_ci_coverage.yml secrets: inherit # A called workflow cannot hold more than its caller, so these are granted # here even though the coverage workflow declares them itself: id-token for diff --git a/.github/workflows/multi_arch_ci_coverage_nightly.yml b/.github/workflows/multi_arch_ci_coverage.yml similarity index 57% rename from .github/workflows/multi_arch_ci_coverage_nightly.yml rename to .github/workflows/multi_arch_ci_coverage.yml index dab9c62849d..ee00098ec58 100644 --- a/.github/workflows/multi_arch_ci_coverage_nightly.yml +++ b/.github/workflows/multi_arch_ci_coverage.yml @@ -1,33 +1,25 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -# Multi-Arch CI Coverage - Nightly +# Multi-Arch CI Coverage (thin parent / orchestrator) # -# Builds one fully instrumented stack covering every coverage-enabled project, -# then runs each project's tests against it and reports separately. Paying for a -# single large build rather than one build per project is what keeps the cost -# flat as more projects are onboarded. -# -# The stack is assembled by naming the stages coverage needs and calling the -# shared per-stage build workflow for each, rather than running the whole -# multi_arch_build_portable_linux.yml pipeline and filtering stages out of it. +# Computes the shared coverage matrix once (setup_coverage_matrix) and hands it +# to a per-platform build+fanout layer (multi_arch_ci_coverage_linux.yml today; +# a Windows sibling later). The instrumented build and the per-project +# test+report live in those layers, not here, so adding a platform duplicates +# none of the matrix logic. # # RFC0014 phase 1 runs every enabled component on one default architecture # (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and # multi-architecture expansion come later. # -# Scheduling is the RFC's option B1 for now: dispatch this workflow by hand and -# pass the run id of a recent nightly as `baseline_run_id`. There is no cron -# here, and the regular nightly does not know about this workflow, so coverage -# has a run id of its own -- a failure here leaves the nightly's status alone, -# and the nightly's unstable test jobs do not hold coverage up. -# -# The RFC's option C, where the regular nightly dispatches this workflow once -# its build finishes and hands over its own run id automatically, is a later -# change. Everything below already takes `baseline_run_id` as an input, so that -# change only has to add the dispatching job. +# Coverage runs as its own run, separate from the regular nightly: the nightly +# dispatches this workflow once its build finishes and hands over its own run id +# as `baseline_run_id` (RFC option C). Coverage having a run id of its own means +# a failure here leaves the nightly's status alone, and the nightly's unstable +# test jobs do not hold coverage up. -name: Multi-Arch CI Coverage Nightly +name: Multi-Arch CI Coverage on: workflow_dispatch: @@ -116,7 +108,7 @@ permissions: # The two run ids are only connected through this input, so surface it in the # run title rather than making people open the logs to find the baseline. run-name: >- - Multi-Arch CI Coverage Nightly + Multi-Arch CI Coverage | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} @@ -161,103 +153,29 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - # Built first so that every later stage in this run has a compiler-runtime to - # link against and to fetch as an inbound artifact. - build_instrumented_compiler_runtime: - name: Build Instrumented Compiler Runtime + # The build + fanout half is per-platform. The shared matrix computed above is + # handed to the Linux layer, which builds the instrumented stack once and then + # fans out to the per-project report workflow. A Windows layer would be a + # sibling job here, reusing setup_coverage_matrix and the report workflow + # unchanged. + coverage_linux: + name: Coverage - Linux needs: setup_coverage_matrix + # `fromJSON('')` is a hard error downstream, and an empty selection has + # nothing to build, so the matrix is checked explicitly before fanning out. if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml - secrets: inherit - with: - stage_name: compiler-runtime - stage_display_name: "Stage - Coverage Compiler Runtime" - timeout_minutes: 480 # 8 hours (compiler is big) - dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - release_type: ci - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - - # The whole stage is built, not just the onboarded projects: the test jobs - # below need the complete dependency set, and phase 1 instruments every - # onboarded project in one pass rather than per changed project. - # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the - # selection covers a whole group, and the individual project flags otherwise; - # everything not selected builds normally. - build_instrumented_math_libs: - name: Build Instrumented Math Libs - needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] - if: >- - ${{ - !failure() && !cancelled() && - needs.setup_coverage_matrix.outputs.needs_math_libs == 'true' - }} - strategy: - fail-fast: false - matrix: - family_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.families_matrix_json) }} - uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml + uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml secrets: inherit with: - stage_name: math-libs - stage_display_name: "Stage - Coverage Math Libs (${{ matrix.family_info.amdgpu_family }})" - timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal - amdgpu_family: ${{ matrix.family_info.amdgpu_family }} + coverage_matrix: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix }} dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - extra_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + families_matrix_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} + coverage_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + needs_math_libs: ${{ needs.setup_coverage_matrix.outputs.needs_math_libs }} rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - release_type: ci - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - - # One test + report job per project. The instrumented stack built above is the - # only thing taken from this run: each job installs `baseline_run_id` and then - # overlays its own project, so a project's dependencies stay non-instrumented - # even though the build instrumented all of them at once. - coverage_report: - name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) - # compiler-runtime is named explicitly rather than relied on through - # math-libs: a selection that needs no math-libs build still tests against - # binaries that stage produced. - needs: - [ - setup_coverage_matrix, - build_instrumented_compiler_runtime, - build_instrumented_math_libs, - ] - # `!failure() && !cancelled()` alone would let this run when - # setup_coverage_matrix was skipped, and fromJSON('') is a hard error, so - # the matrix has to be checked explicitly. - if: >- - ${{ - !failure() && !cancelled() && - needs.setup_coverage_matrix.result == 'success' && - needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' - }} - strategy: - fail-fast: false - matrix: - coverage: ${{ fromJSON(needs.setup_coverage_matrix.outputs.coverage_matrix) }} - uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml - secrets: inherit - with: - project_name: ${{ matrix.coverage.project_name }} - test_component: ${{ matrix.coverage.test_component }} - object_globs: ${{ matrix.coverage.object_globs }} - fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} - build_stage: ${{ matrix.coverage.build_stage }} - codecov_flag: ${{ matrix.coverage.codecov_flag }} - amdgpu_families: ${{ matrix.coverage.amdgpu_families }} baseline_run_id: ${{ inputs.baseline_run_id }} baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} baseline_repository: ${{ inputs.baseline_repository }} - artifact_names: ${{ matrix.coverage.artifact_names }} - artifact_relpaths: ${{ matrix.coverage.artifact_relpaths }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} @@ -266,13 +184,11 @@ jobs: id-token: write ci_summary: - name: Nightly Coverage Summary + name: Coverage Summary if: always() needs: - setup_coverage_matrix - - build_instrumented_compiler_runtime - - build_instrumented_math_libs - - coverage_report + - coverage_linux runs-on: ubuntu-24.04 steps: - name: Checking out repository diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index a37c05082eb..c614bb24709 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -3,319 +3,169 @@ # Multi-Arch CI Coverage - Linux # -# Runs the test and report half of the coverage pipeline for a single project on -# a single GPU family: +# Per-platform build + fanout layer. Builds one fully instrumented Linux stack +# covering every coverage-enabled project, then fans out to +# multi_arch_ci_coverage_report.yml once per project to test against that stack +# and report separately. Paying for a single large build rather than one build +# per project is what keeps the cost flat as more projects are onboarded. # -# test (collect profraw) -> merge -> report +# The stack is assembled by naming the stages coverage needs and calling the +# shared per-stage build workflow for each, rather than running the whole +# multi_arch_build_portable_linux.yml pipeline and filtering stages out of it. # -# The instrumented stack itself is built by the caller -# (multi_arch_ci_coverage_nightly.yml), which instruments every onboarded -# project in one pass. This workflow only consumes it. -# -# `baseline_run_id` drives the hybrid install: the tree comes from a regular -# nightly run and only the project under test is overlaid from the instrumented -# run, keeping instrumented dependencies out of the report. Without it the -# instrumented run's own artifacts are tested as built, which measures the -# project's dependencies alongside it. +# The shared coverage matrix and build configuration are computed once by the +# parent (multi_arch_ci_coverage.yml) and handed down as inputs, so that adding +# multi_arch_ci_coverage_windows.yml later duplicates none of that logic -- it +# only mirrors the build + fanout below with Windows runners. name: Multi-Arch CI Coverage - Linux on: workflow_call: inputs: - project_name: + coverage_matrix: type: string - description: "Coverage project key (e.g. hiprand)" - test_component: + description: "JSON array of per-project coverage matrix entries produced by setup_coverage_matrix." + dist_amdgpu_families: type: string - description: "Test matrix key for the project's tests" - object_globs: + description: "GPU families the instrumented stack is distributed for." + families_matrix_json: type: string - description: "Comma-separated globs matching the instrumented binaries" - fetch_artifact_args: + description: "JSON array of per-family build matrix entries for the math-libs stage." + coverage_cmake_options: type: string - default: "" - description: "Extra install_rocm_from_artifacts.py args for the report job" - build_stage: + description: "Extra CMake options enabling instrumentation for the selected projects." + needs_math_libs: type: string - default: "" - description: >- - Build stage owning the project. The report job fetches this stage's - sources so the HTML rendering has something to annotate; the lcov and - text outputs do not need them. Empty skips the fetch and the HTML. + description: "'true' when the selection requires the math-libs stage to be built." + rocm_package_version: + type: string + description: "Computed ROCm package version stamped onto the instrumented build." baseline_run_id: type: string default: "" - description: >- - Regular nightly run to take non-instrumented dependencies from. Set - by the nightly caller, which instruments the whole stack in its own - run; the test jobs then install that run and overlay only the project - under test. Empty tests the instrumented artifacts as built. + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." baseline_release_type: type: string default: "nightly" - description: >- - Release channel baseline_run_id published to. Artifacts are bucketed - by channel, so the baseline is read from its own channel rather than - from this run's. + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." baseline_repository: type: string default: "" - description: >- - Repository owning baseline_run_id, in 'owner/repo' form. Needed when - the baseline is a release nightly, which ROCm/rockrel publishes. - artifact_names: - type: string - default: "" - description: "Artifact names holding the instrumented project, for the hybrid fetch" - artifact_relpaths: - type: string - default: "" - description: "Subproject stage directories to overlay, for the hybrid fetch" - codecov_flag: - type: string - default: "" - description: "Codecov flag to file the report under" - amdgpu_families: - type: string - description: "GPU family to test coverage on" + description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." test_type: type: string - default: standard + default: "standard" + description: "Test type to run against the instrumented build." release_type: type: string - default: ci - repository: - description: "Repository to checkout. Defaults to github.repository." - type: string - default: "" - ref: - description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." + default: "ci" + description: "Release channel this coverage run's own instrumented artifacts publish to." + quartz_tracking_id: type: string default: "" - quartz_tracking_id: description: >- Leave empty. Internal Quartz tracking id, propagated from the caller - to link this run to its release lineage. - type: string - default: "" + to link this run to a release lineage once the nightly dispatches + coverage automatically. permissions: contents: read jobs: - configure_test_matrix: - name: Configure Coverage Tests (${{ inputs.project_name }}) - runs-on: ubuntu-24.04 - outputs: - components: ${{ steps.configure.outputs.components }} - platform: ${{ steps.configure.outputs.platform }} - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Checkout CI config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - with: - repository: ROCm/therock-ci-config - ref: main - path: ci-config - continue-on-error: true - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: Configuring coverage test options - id: configure - env: - ARTIFACT_GROUP: multi-arch-coverage - AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} - TEST_TYPE: ${{ inputs.test_type }} - TEST_LABELS: "[]" - # The report covers one project, so only that project's tests run - # here; the other onboarded projects get their own job. - PROJECTS_TO_TEST: ${{ inputs.test_component }} - CI_CONFIG_PATH: ci-config - BUILD_VARIANT: coverage - run: | - python ./build_tools/github_actions/fetch_test_configurations.py \ - --platform=linux + # Built first so that every later stage in this run has a compiler-runtime to + # link against and to fetch as an inbound artifact. + build_instrumented_compiler_runtime: + name: Build Instrumented Compiler Runtime + if: ${{ inputs.coverage_matrix != '[]' }} + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml + secrets: inherit + with: + stage_name: compiler-runtime + stage_display_name: "Stage - Coverage Compiler Runtime" + timeout_minutes: 480 # 8 hours (compiler is big) + dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} + rocm_package_version: ${{ inputs.rocm_package_version }} + release_type: ci + quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + permissions: + contents: read + id-token: write - test_coverage: - name: Test Coverage (${{ inputs.project_name }}) - needs: configure_test_matrix + # The whole stage is built, not just the onboarded projects: the test jobs + # below need the complete dependency set, and phase 1 instruments every + # onboarded project in one pass rather than per changed project. + # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the + # selection covers a whole group, and the individual project flags otherwise; + # everything not selected builds normally. + build_instrumented_math_libs: + name: Build Instrumented Math Libs + needs: build_instrumented_compiler_runtime if: >- ${{ !failure() && !cancelled() && - needs.configure_test_matrix.outputs.components != '' && - needs.configure_test_matrix.outputs.components != '[]' + inputs.needs_math_libs == 'true' }} strategy: fail-fast: false matrix: - components: ${{ fromJSON(needs.configure_test_matrix.outputs.components || '[]') }} - uses: ./.github/workflows/test_component.yml + family_info: ${{ fromJSON(inputs.families_matrix_json) }} + uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: - artifact_group: multi-arch-coverage - amdgpu_families: ${{ inputs.amdgpu_families }} - test_runs_on: ${{ matrix.components.test_runner }} - platform: ${{ needs.configure_test_matrix.outputs.platform }} - component: ${{ toJSON(matrix.components) }} - release_type: ${{ inputs.release_type }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - build_variant: coverage - coverage_enabled: true - coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} - coverage_baseline_run_id: ${{ inputs.baseline_run_id }} - coverage_baseline_release_type: ${{ inputs.baseline_release_type }} - coverage_baseline_repository: ${{ inputs.baseline_repository }} - coverage_artifact_names: ${{ inputs.artifact_names }} - coverage_artifact_relpaths: ${{ inputs.artifact_relpaths }} + stage_name: math-libs + stage_display_name: "Stage - Coverage Math Libs (${{ matrix.family_info.amdgpu_family }})" + timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal + amdgpu_family: ${{ matrix.family_info.amdgpu_family }} + dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} + extra_cmake_options: ${{ inputs.coverage_cmake_options }} + rocm_package_version: ${{ inputs.rocm_package_version }} + release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + permissions: + contents: read + id-token: write - aggregate_coverage: - name: Coverage Report (${{ inputs.project_name }}) - needs: [configure_test_matrix, test_coverage] - # Runs even when some shards failed: a partial report is more useful than - # none, and a failing test still exercises code. + # One test + report job per project. The instrumented stack built above is the + # only thing taken from this run: each job installs `baseline_run_id` and then + # replaces its own project, so a project's dependencies stay non-instrumented + # even though the build instrumented all of them at once. + coverage_report: + name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) + # compiler-runtime is named explicitly rather than relied on through + # math-libs: a selection that needs no math-libs build still tests against + # binaries that stage produced. + needs: + [ + build_instrumented_compiler_runtime, + build_instrumented_math_libs, + ] + # `!failure() && !cancelled()` alone would let this run when the parent's + # matrix was empty, and fromJSON('') is a hard error, so the matrix has to + # be checked explicitly. if: >- ${{ - !cancelled() && - needs.configure_test_matrix.result == 'success' && - needs.configure_test_matrix.outputs.components != '[]' + !failure() && !cancelled() && + inputs.coverage_matrix != '[]' }} - runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + coverage: ${{ fromJSON(inputs.coverage_matrix) }} + uses: ./.github/workflows/multi_arch_ci_coverage_report.yml + secrets: inherit + with: + project_name: ${{ matrix.coverage.project_name }} + test_component: ${{ matrix.coverage.test_component }} + object_globs: ${{ matrix.coverage.object_globs }} + fetch_artifact_args: ${{ matrix.coverage.fetch_artifact_args }} + build_stage: ${{ matrix.coverage.build_stage }} + codecov_flag: ${{ matrix.coverage.codecov_flag }} + amdgpu_families: ${{ matrix.coverage.amdgpu_families }} + baseline_run_id: ${{ inputs.baseline_run_id }} + baseline_repository: ${{ inputs.baseline_repository }} + test_type: ${{ inputs.test_type || 'standard' }} + release_type: ${{ inputs.release_type || 'ci' }} permissions: contents: read id-token: write - env: - OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build - PROFRAW_DIR: ${{ github.workspace }}/coverage-profraw - REPORT_DIR: ${{ github.workspace }}/coverage-report - HAS_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN != '' }} - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - with: - repository: ${{ inputs.repository || github.repository }} - ref: ${{ inputs.ref }} - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: Download profraw files from all shards - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - pattern: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }}-* - path: ${{ env.PROFRAW_DIR }} - continue-on-error: true - - - name: Install python deps - run: pip install -r requirements.txt - - - name: Download instrumented artifacts - env: - GITHUB_TOKEN: ${{ github.token }} - RELEASE_TYPE: ${{ inputs.release_type }} - FETCH_ARTIFACT_ARGS: ${{ inputs.fetch_artifact_args }} - run: | - python ./build_tools/install_rocm_from_artifacts.py \ - --run-id=${{ github.run_id }} \ - --artifact-group=${{ inputs.amdgpu_families }} \ - --output-dir=${OUTPUT_ARTIFACTS_DIR} \ - ${FETCH_ARTIFACT_ARGS} - - # Only the HTML rendering reads source, so this runs after the artifacts - # are already down: a failure here, including running the runner out of - # disk, costs the annotated view and leaves lcov and the text table. - - name: Fetch sources for the annotated report - if: ${{ inputs.build_stage != '' }} - id: fetch_sources - timeout-minutes: 30 - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - STAGE_NAME: ${{ inputs.build_stage }} - run: | - gh auth setup-git --hostname github.com - ./build_tools/fetch_sources.py --stage "${STAGE_NAME}" --jobs 12 --depth 1 - - - name: Merge profraw and generate reports - env: - # Sources are recorded at the path the build container used, which is - # not where they sit here, so llvm-cov is told how to get from one to - # the other. Generated files under build/ are in neither and are - # reported as uncovered; the text summary still counts them. - BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} - run: | - html_args=() - if [[ "${HTML_ENABLED}" == "true" ]]; then - html_args=( - --html-output="${REPORT_DIR}/html" - --project-title="${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" - ) - else - echo "::warning::Sources unavailable; skipping the annotated HTML report." - fi - - python ./build_tools/github_actions/merge_coverage_report.py \ - --profraw-dir="${PROFRAW_DIR}" \ - --rocm-dir="${OUTPUT_ARTIFACTS_DIR}" \ - --object-globs="${{ inputs.object_globs }}" \ - --profdata-output="${REPORT_DIR}/coverage.profdata" \ - --lcov-output="${REPORT_DIR}/coverage.info" \ - --summary-output="${REPORT_DIR}/coverage_summary.txt" \ - --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ - "${html_args[@]}" - - # Puts the totals on the run's summary page, so the number is readable - # without downloading anything. - - name: Publish the summary to the job page - if: ${{ always() }} - run: | - summary="${REPORT_DIR}/coverage_summary.txt" - [[ -f "${summary}" ]] || exit 0 - { - echo "## Coverage: ${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" - echo '```' - tail -n 3 "${summary}" - echo '```' - } >> "${GITHUB_STEP_SUMMARY}" - - - name: Upload coverage report - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: coverage-report-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} - # The whole directory: coverage.info for Codecov, coverage_summary.txt - # for the numbers, and html/index.html for the annotated report. - path: | - ${{ env.REPORT_DIR }}/coverage.info - ${{ env.REPORT_DIR }}/coverage_summary.txt - ${{ env.REPORT_DIR }}/html - if-no-files-found: warn - - # Skipped rather than failed when the repository has no Codecov token, - # so forks and first-time setups still get the lcov artifact above. - - name: Upload report to Codecov - if: ${{ env.HAS_CODECOV_TOKEN == 'true' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ env.REPORT_DIR }}/coverage.info - flags: ${{ inputs.codecov_flag || inputs.project_name }} - name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} - fail_ci_if_error: false diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml new file mode 100644 index 00000000000..082fa62ae72 --- /dev/null +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -0,0 +1,305 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Multi-Arch CI Coverage - Report +# +# Runs the test and report half of the coverage pipeline for a single project on +# a single GPU family: +# +# test (collect profraw) -> merge -> report +# +# The instrumented stack itself is built by the caller +# (multi_arch_ci_coverage_linux.yml), which instruments every onboarded +# project in one pass. This workflow only consumes it. +# +# `baseline_run_id` drives the hybrid install: the tree comes from a regular +# nightly run and only the project under test is replaced from the instrumented +# run, keeping instrumented dependencies out of the report. Without it the +# instrumented run's own artifacts are tested as built, which measures the +# project's dependencies alongside it. + +name: Multi-Arch CI Coverage - Report + +on: + workflow_call: + inputs: + project_name: + type: string + description: "Coverage project key (e.g. hiprand)" + test_component: + type: string + description: "Test matrix key for the project's tests" + object_globs: + type: string + description: "Comma-separated globs matching the instrumented binaries" + fetch_artifact_args: + type: string + default: "" + description: "Extra install_rocm_from_artifacts.py args for the report job" + build_stage: + type: string + default: "" + description: >- + Build stage owning the project. The report job fetches this stage's + sources so the HTML rendering has something to annotate; the lcov and + text outputs do not need them. Empty skips the fetch and the HTML. + baseline_run_id: + type: string + default: "" + description: >- + Regular nightly run to take non-instrumented dependencies from. Set + by the nightly caller, which instruments the whole stack in its own + run; the test jobs then install that run and overlay only the project + under test. Empty tests the instrumented artifacts as built. + baseline_repository: + type: string + default: "" + description: >- + Repository owning baseline_run_id, in 'owner/repo' form. Needed when + the baseline is a release nightly, which ROCm/rockrel publishes. + codecov_flag: + type: string + default: "" + description: "Codecov flag to file the report under" + amdgpu_families: + type: string + description: "GPU family to test coverage on" + amdgpu_targets: + type: string + default: "" + description: "Specific GPU targets to test, passed through to the installer. Empty derives them from amdgpu_families." + test_type: + type: string + default: standard + release_type: + type: string + default: ci + repository: + description: "Repository to checkout. Defaults to github.repository." + type: string + default: "" + ref: + description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref." + type: string + default: "" + +permissions: + contents: read + +jobs: + configure_test_matrix: + name: Configure Coverage Tests (${{ inputs.project_name }}) + runs-on: ubuntu-24.04 + outputs: + components: ${{ steps.configure.outputs.components }} + platform: ${{ steps.configure.outputs.platform }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} + + - name: Checkout CI config + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ROCm/therock-ci-config + ref: main + path: ci-config + continue-on-error: true + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Configuring coverage test options + id: configure + env: + ARTIFACT_GROUP: multi-arch-coverage + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} + TEST_TYPE: ${{ inputs.test_type }} + TEST_LABELS: "[]" + # The report covers one project, so only that project's tests run + # here; the other onboarded projects get their own job. + PROJECTS_TO_TEST: ${{ inputs.test_component }} + CI_CONFIG_PATH: ci-config + BUILD_VARIANT: coverage + run: | + python ./build_tools/github_actions/fetch_test_configurations.py \ + --platform=linux + + test_coverage: + name: Test Coverage (${{ inputs.project_name }}) + needs: configure_test_matrix + if: >- + ${{ + !failure() && !cancelled() && + needs.configure_test_matrix.outputs.components != '' && + needs.configure_test_matrix.outputs.components != '[]' + }} + strategy: + fail-fast: false + matrix: + components: ${{ fromJSON(needs.configure_test_matrix.outputs.components || '[]') }} + uses: ./.github/workflows/test_code_coverage_component.yml + secrets: inherit + with: + # The installer takes the generic tree from baseline_run_id and replaces + # only this project's libs from the instrumented run (this workflow's own + # run id), deriving the replacement artifact name and folder from the + # component job_name -- so artifact_names/relpaths are no longer needed. + generic_run_id: ${{ inputs.baseline_run_id }} + baseline_repository: ${{ inputs.baseline_repository }} + artifact_group: multi-arch-coverage + amdgpu_families: ${{ inputs.amdgpu_families }} + amdgpu_targets: ${{ inputs.amdgpu_targets }} + test_runs_on: ${{ matrix.components.test_runner }} + platform: ${{ needs.configure_test_matrix.outputs.platform }} + component: ${{ toJSON(matrix.components) }} + release_type: ${{ inputs.release_type }} + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + build_variant: coverage + coverage_enabled: true + coverage_artifact_prefix: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + + aggregate_coverage: + name: Coverage Report (${{ inputs.project_name }}) + needs: [configure_test_matrix, test_coverage] + # Runs even when some shards failed: a partial report is more useful than + # none, and a failing test still exercises code. + if: >- + ${{ + !cancelled() && + needs.configure_test_matrix.result == 'success' && + needs.configure_test_matrix.outputs.components != '[]' + }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build + PROFRAW_DIR: ${{ github.workspace }}/coverage-profraw + REPORT_DIR: ${{ github.workspace }}/coverage-report + HAS_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN != '' }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Download profraw files from all shards + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: coverage-profraw-${{ inputs.project_name }}-${{ inputs.amdgpu_families }}-* + path: ${{ env.PROFRAW_DIR }} + continue-on-error: true + + - name: Install python deps + run: pip install -r requirements.txt + + - name: Download instrumented artifacts + env: + GITHUB_TOKEN: ${{ github.token }} + RELEASE_TYPE: ${{ inputs.release_type }} + FETCH_ARTIFACT_ARGS: ${{ inputs.fetch_artifact_args }} + run: | + python ./build_tools/install_rocm_from_artifacts.py \ + --run-id=${{ github.run_id }} \ + --artifact-group=${{ inputs.amdgpu_families }} \ + --output-dir=${OUTPUT_ARTIFACTS_DIR} \ + ${FETCH_ARTIFACT_ARGS} + + # Only the HTML rendering reads source, so this runs after the artifacts + # are already down: a failure here, including running the runner out of + # disk, costs the annotated view and leaves lcov and the text table. + - name: Fetch sources for the annotated report + if: ${{ inputs.build_stage != '' }} + id: fetch_sources + timeout-minutes: 30 + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + STAGE_NAME: ${{ inputs.build_stage }} + run: | + gh auth setup-git --hostname github.com + ./build_tools/fetch_sources.py --stage "${STAGE_NAME}" --jobs 12 --depth 1 + + - name: Merge profraw and generate reports + env: + # Sources are recorded at the path the build container used, which is + # not where they sit here, so llvm-cov is told how to get from one to + # the other. Generated files under build/ are in neither and are + # reported as uncovered; the text summary still counts them. + BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} + run: | + html_args=() + if [[ "${HTML_ENABLED}" == "true" ]]; then + html_args=( + --html-output="${REPORT_DIR}/html" + --project-title="${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" + ) + else + echo "::warning::Sources unavailable; skipping the annotated HTML report." + fi + + python ./build_tools/github_actions/merge_coverage_report.py \ + --profraw-dir="${PROFRAW_DIR}" \ + --rocm-dir="${OUTPUT_ARTIFACTS_DIR}" \ + --object-globs="${{ inputs.object_globs }}" \ + --profdata-output="${REPORT_DIR}/coverage.profdata" \ + --lcov-output="${REPORT_DIR}/coverage.info" \ + --summary-output="${REPORT_DIR}/coverage_summary.txt" \ + --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ + "${html_args[@]}" + + # Puts the totals on the run's summary page, so the number is readable + # without downloading anything. + - name: Publish the summary to the job page + if: ${{ always() }} + run: | + summary="${REPORT_DIR}/coverage_summary.txt" + [[ -f "${summary}" ]] || exit 0 + { + echo "## Coverage: ${{ inputs.project_name }} (${{ inputs.amdgpu_families }})" + echo '```' + tail -n 3 "${summary}" + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload coverage report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-report-${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + # The whole directory: coverage.info for Codecov, coverage_summary.txt + # for the numbers, and html/index.html for the annotated report. + path: | + ${{ env.REPORT_DIR }}/coverage.info + ${{ env.REPORT_DIR }}/coverage_summary.txt + ${{ env.REPORT_DIR }}/html + if-no-files-found: warn + + # Skipped rather than failed when the repository has no Codecov token, + # so forks and first-time setups still get the lcov artifact above. + - name: Upload report to Codecov + if: ${{ env.HAS_CODECOV_TOKEN == 'true' }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ env.REPORT_DIR }}/coverage.info + flags: ${{ inputs.codecov_flag || inputs.project_name }} + name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + fail_ci_if_error: false diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 3cbd9c26a5d..4050907c373 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -9,6 +9,14 @@ on: generic_run_id: type: string default: "" + baseline_repository: + description: >- + Repository owning generic_run_id (owner/repo), passed to the installer + as --run-github-repo for the generic fetch. Empty lets the installer + default to the current repository. The instrumented replacement always + resolves against this coverage run's own repository. + type: string + default: "" artifact_group: type: string amdgpu_families: @@ -136,16 +144,34 @@ jobs: env: RELEASE_TYPE: ${{ inputs.release_type }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} + BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} run: | pip3 install -r requirements-test.txt + # The generic (baseline) tree comes from generic_run_id. When it is + # empty (self-contained dispatch), fall back to this run's own + # instrumented artifacts so the install still resolves -- the + # subsequent --replace then re-extracts the same instrumented libs, + # leaving the whole stack instrumented, which is the documented + # self-contained behavior. + generic_run_id="${{ inputs.generic_run_id }}" + if [[ -z "${generic_run_id}" ]]; then + generic_run_id="${{ github.run_id }}" + fi + # Only pass --run-github-repo when a baseline repository was given, so + # an empty value lets the installer default to the current repository. + run_github_repo_arg=() + if [[ -n "${BASELINE_REPOSITORY}" ]]; then + run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") + fi python3 build_tools/install_rocm_code_coverage_build.py \ --code-coverage-run-id=${{ env.CODE_COVERAGE_RUN_ID }} \ - --run-id=${{ inputs.generic_run_id }} \ + --run-id="${generic_run_id}" \ --artifact-group=${{ inputs.amdgpu_families }} \ --amdgpu-targets=${{ inputs.amdgpu_targets }} \ --tests \ --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \ --replace-${{ fromJSON(inputs.component).job_name }} \ + "${run_github_repo_arg[@]}" \ ${FETCH_ARTIFACT_ARGS} # safe.directory must be set before Runner Health Status diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 5903c67f3da..1c41df48310 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -786,7 +786,7 @@ def resolve_build_stages(project_keys: list[str]) -> set[str]: f"cannot build coverage for {', '.join(blocked)}: this workflow has " f"build jobs for {', '.join(sorted(BUILDABLE_STAGES))} only, not " f"{', '.join(unbuildable)}. Restore the staged build jobs in " - "multi_arch_ci_coverage_nightly.yml to select these projects." + "multi_arch_ci_coverage_linux.yml to select these projects." ) return stages diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 61b08005698..478cd3d990b 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -323,7 +323,7 @@ def test_comm_libs_project_is_rejected_with_a_actionable_message(self): message = str(caught.exception) self.assertIn("rccl", message) self.assertIn("comm-libs", message) - self.assertIn("multi_arch_ci_coverage_nightly.yml", message) + self.assertIn("multi_arch_ci_coverage_linux.yml", message) def test_rejection_names_every_unbuildable_project_not_just_the_first(self): with self.assertRaises(ValueError) as caught: diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index 2a34bec63b3..0013d43f627 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -97,15 +97,25 @@ def _target_families(family, amdgpu_targets): return families -def download_replacement_artifacts(code_coverage_run_id, artifact_names, opts): +def download_replacement_artifacts( + code_coverage_run_id, artifact_names, opts, code_coverage_run_github_repo +): """Download the instrumented replacement artifacts from the code-coverage run. Uses the code coverage run ID as the run-id for the S3 backend, then fetches every component tar matching the requested artifact names and target family. + + The instrumented replacement lives in a DIFFERENT repo than the generic + baseline: the generic build comes from --run-id in --run-github-repo (e.g. + ROCm/rocm-libraries), while the instrumented coverage run comes from + --code-coverage-run-id in --code-coverage-run-github-repo (the coverage + run's own repo, e.g. ROCm/TheRock). The backend's S3 bucket lookup 404s if + the run id is queried against the wrong owning repo, so we must pass the + code-coverage repo here rather than reusing opts.run_github_repo. """ backend = create_backend_from_env( run_id=code_coverage_run_id, - github_repository=opts.run_github_repo, + github_repository=code_coverage_run_github_repo, platform=platform.system().lower(), ) log(f"Fetching replacement artifacts from {backend.base_uri}") @@ -242,6 +252,23 @@ def main(argv): type=str, help="run id of the build from which instrumental components needs to be replaced", ) + # The instrumented replacement fetch is keyed on a DIFFERENT repo than the + # generic install: the generic baseline uses --run-github-repo (e.g. + # ROCm/rocm-libraries), while the coverage run lives in its own repo (e.g. + # ROCm/TheRock). Defaults to $GITHUB_REPOSITORY so CI resolves to the + # coverage run's own repo automatically. Consumed by parse_known_args so it + # does NOT leak into extra_args / the generic install. + parser.add_argument( + "--code-coverage-run-github-repo", + type=str, + default=os.environ.get("GITHUB_REPOSITORY"), + help=( + "GitHub repository (owner/name) that owns --code-coverage-run-id, " + "used to resolve the instrumented artifact backend. Defaults to " + "$GITHUB_REPOSITORY (the coverage run's own repo). This is separate " + "from --run-github-repo, which owns the generic --run-id build." + ), + ) artifacts_group = parser.add_argument_group("replace_comps") for comp in COMPONENT_MAP.keys(): artifacts_group.add_argument( @@ -272,9 +299,12 @@ def main(argv): "--code-coverage-run-id is required when using --replace-* options" ) - # download selected component artifacts + # download selected component artifacts (instrumented, from the coverage repo) dest_dir = download_replacement_artifacts( - args.code_coverage_run_id, artifacts.keys(), opts + args.code_coverage_run_id, + artifacts.keys(), + opts, + args.code_coverage_run_github_repo, ) # replace selected library folder paths in selected component artifacts diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py new file mode 100644 index 00000000000..6ed166bcf91 --- /dev/null +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for install_rocm_code_coverage_build.py. + +Focuses on the generic-vs-instrumented repo split: the generic install is keyed +on --run-github-repo while the instrumented replacement fetch must be keyed on +--code-coverage-run-github-repo (defaulting to $GITHUB_REPOSITORY). +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +import install_rocm_code_coverage_build as mod + + +class _FakeBackend: + """Minimal ArtifactBackend stand-in sufficient for the replacement fetch.""" + + def __init__(self): + self.base_uri = "s3://fake-bucket/fake" + + def list_artifacts(self, name_filter=None): + # One instrumented blas tar for the requested family. + return ["blas_lib_generic.tar.xz"] + + +class TestCodeCoverageRepoSplit(unittest.TestCase): + def _run_main(self, extra_argv, env=None): + """Run main() with all network boundaries mocked, capturing kwargs. + + Returns the github_repository kwarg passed to create_backend_from_env + for the instrumented replacement fetch. + """ + captured = {} + + def fake_create_backend(**kwargs): + captured["github_repository"] = kwargs.get("github_repository") + captured["run_id"] = kwargs.get("run_id") + return _FakeBackend() + + def fake_find_available(artifact_names, target_families, available): + # Pretend the single listed artifact matches. + return list(available) + + with tempfile.TemporaryDirectory() as tmp: + argv = [ + "--code-coverage-run-id", + "222", + "--replace-rocblas", + "--artifact-group", + "gfx94X-dcgpu", + "--output-dir", + tmp, + "--run-id", + "111", + "--run-github-repo", + "ROCm/rocm-libraries", + ] + extra_argv + + patched_env = env or {} + with ( + mock.patch.object(mod, "install_from_artifacts_main"), + mock.patch.object( + mod, "create_backend_from_env", side_effect=fake_create_backend + ), + mock.patch.object( + mod, "find_available_artifacts", side_effect=fake_find_available + ), + mock.patch.object( + mod, "download_artifact", return_value=Path(tmp) / "x" + ), + # No archives to extract -> replace step is a no-op. + mock.patch.object(mod, "replace_instrumented_libraries"), + mock.patch.dict(os.environ, patched_env, clear=False), + ): + mod.main(argv) + return captured + + def test_replacement_uses_code_coverage_repo_not_generic_repo(self): + captured = self._run_main( + ["--code-coverage-run-github-repo", "ROCm/TheRock"] + ) + self.assertEqual(captured["run_id"], "222") + self.assertEqual(captured["github_repository"], "ROCm/TheRock") + self.assertNotEqual(captured["github_repository"], "ROCm/rocm-libraries") + + def test_replacement_repo_defaults_to_github_repository_env(self): + # main() must be re-imported so the argparse default picks up the env, + # since the default is evaluated at parse time inside main(). + captured = self._run_main( + [], env={"GITHUB_REPOSITORY": "ROCm/TheRock"} + ) + self.assertEqual(captured["github_repository"], "ROCm/TheRock") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index d33df349104..d01fdda0138 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -53,8 +53,9 @@ and upload steps inside a WSL Ubuntu shell. See Code coverage has its own entry point, because the projects it measures have to be rebuilt with instrumentation before their tests can be run: -- [`.github/workflows/multi_arch_ci_coverage_nightly.yml`](/.github/workflows/multi_arch_ci_coverage_nightly.yml) - builds one instrumented stack covering every coverage-enabled project. Dispatched by hand: pass the run id of a recent nightly as `baseline_run_id` to take the non-instrumented dependencies from it, or leave it empty to build the whole stack within the run - - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - runs one project's tests on one GPU family, then turns the collected profiles into an lcov file, a text summary and a browsable HTML report +- [`.github/workflows/multi_arch_ci_coverage.yml`](/.github/workflows/multi_arch_ci_coverage.yml) - thin top-level orchestrator (`setup_coverage_matrix`, `coverage_linux`, `ci_summary`). Dispatched by hand or by the nightly: pass the run id of a recent nightly as `baseline_run_id` to take the non-instrumented dependencies from it, or leave it empty to build the whole stack within the run + - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - the per-platform build + fan-out layer: builds the instrumented compiler-runtime and math-libs, then fans out over projects (a future `multi_arch_ci_coverage_windows.yml` will mirror it) + - [`.github/workflows/multi_arch_ci_coverage_report.yml`](/.github/workflows/multi_arch_ci_coverage_report.yml) - runs one project's tests on one GPU family, then turns the collected profiles into an lcov file, a text summary and a browsable HTML report See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a project, and [Coverage CI](code_coverage.md#coverage-ci) for the end-to-end diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 1336698cf7a..1a88c915a49 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -170,9 +170,11 @@ graph TD ``` Every edge is a job dependency inside the one coverage run. -`multi_arch_ci_coverage_nightly.yml` is the top level; it delegates to -`multi_arch_ci_coverage_linux.yml` for the per-project test and aggregation -sequence on one GPU family. Both take their project list from +`multi_arch_ci_coverage.yml` is the thin top-level orchestrator; it delegates to +`multi_arch_ci_coverage_linux.yml`, the per-platform build + fan-out layer that +builds the instrumented stacks and calls `multi_arch_ci_coverage_report.yml` for +the per-project test and aggregation sequence on one GPU family. All take their +project list from `build_tools/github_actions/configure_coverage_ci.py`, the registry of coverage-enabled projects and everything the pipeline needs to know about each one: the CMake target, the build stage it lives in, its test component, the @@ -180,10 +182,12 @@ object globs handed to `llvm-cov`, and its Codecov flag. ### Dispatching a run -`multi_arch_ci_coverage_nightly.yml` is started by hand, from the Actions tab -or the `gh` CLI. There is no cron trigger, and the regular nightly does not -know about this workflow — it has no coverage-specific jobs, and a coverage run -neither reads its artifacts nor depends on it having succeeded. +`multi_arch_ci_coverage.yml` is started by hand, from the Actions tab or the +`gh` CLI, and can now also be dispatched by the consumer nightly (RFC option C): +`ROCm/rocm-libraries`'s `therock-multi-arch-ci-nightly.yml` triggers it as a +separate run via a GitHub App token, passing its own run id as `baseline_run_id`. +A coverage run has no cron trigger of its own, and it neither reads the nightly's +artifacts nor depends on it having succeeded — it only takes the baseline run id. Pick a recent nightly run, pass its id as `baseline_run_id`, and leave `baseline_release_type` at `nightly`. The other input that matters is @@ -243,18 +247,18 @@ The artifacts publish under `release_type: ci`, under this run's id. ### Test execution `coverage_report` fans out over the matrix, one call to -`multi_arch_ci_coverage_linux.yml` per project and GPU family. +`multi_arch_ci_coverage_report.yml` per project and GPU family. `configure_test_matrix` runs `fetch_test_configurations.py` — shared with regular CI — narrowed to the one project this report covers, and outputs the -shard list. `test_coverage` then runs one `test_component.yml` job per shard -with `coverage_enabled: true`. Three things happen in order: - -1. **Install the baseline, then overlay.** `setup_test_environment` runs - against `baseline_run_id` and `baseline_release_type`, installing a fully - non-instrumented stack. `overlay_coverage_artifacts.py` then fetches this - run's artifact for the project under test and copies only that project's - stage directories over it, so exactly one library in the install is - instrumented. +shard list. `test_coverage` then runs one `test_code_coverage_component.yml` job +per shard. Three things happen in order: + +1. **Install the baseline, then swap in the instrumented project.** + `install_rocm_code_coverage_build.py` installs the generic non-instrumented + tree from `--run-id` (the baseline nightly run) and replaces only the + project-under-test's libraries from `--code-coverage-run-id` (this coverage + run's own id), deriving the artifact name and folder from the component + `job_name`, so exactly one library in the install is instrumented. 1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to a per-shard path using the `%p` and `%m` substitutions, so a shard that forks or loads several instrumented libraries does not overwrite its own @@ -476,7 +480,7 @@ larger code model is the likely fix. ## Adding a stage -`multi_arch_ci_coverage_nightly.yml` builds compiler-runtime and math-libs, and +`multi_arch_ci_coverage_linux.yml` builds compiler-runtime and math-libs, and `configure_coverage_ci.py` rejects any selection that reaches beyond them (`BUILDABLE_STAGES`). That covers 17 of the 19 measurable projects. diff --git a/docs/development/coverage_nightly_trigger_snippet.md b/docs/development/coverage_nightly_trigger_snippet.md new file mode 100644 index 00000000000..8ac4cd51259 --- /dev/null +++ b/docs/development/coverage_nightly_trigger_snippet.md @@ -0,0 +1,104 @@ +# Reference: `trigger_coverage` job for the rocm-libraries nightly + +This snippet belongs in **`ROCm/rocm-libraries`** +`.github/workflows/therock-multi-arch-ci-nightly.yml`, not in this repo. It is +kept here only as the canonical reference for the code-coverage dispatch +contract. TheRock cannot edit the external repo; copy this into the nightly when +wiring the real driver. + +The nightly's regular build is **non-instrumented**, so it can only ever be the +baseline that coverage is measured against, never the thing coverage runs on. +The coverage orchestrator (`ROCm/TheRock` +`.github/workflows/multi_arch_ci_coverage.yml`) therefore has to run as a +**separate** GitHub Actions run with its own `run_id`; a nested `uses:` call +would share the nightly's `run_id` and collide the generic and instrumented +trees the installer needs to keep apart. The nightly hands over its own +`run_id` as `baseline_run_id`. + +Cross-repo dispatch is authenticated with a **GitHub App token** +(`actions/create-github-app-token`), modelled on `ROCm/rocm-systems` +`.github/workflows/rocm_ci_caller.yml`. No classic PAT is introduced. + +> **One-time org setup (not code):** a GitHub App reachable from rocm-libraries +> must be installed on **ROCm/TheRock** with **`actions: write`**, so the token +> minted below is allowed to call `createWorkflowDispatch` on TheRock. + +## `workflow_dispatch` input (add to the nightly's `on:` block) + +```yaml +on: + workflow_dispatch: + inputs: + # ... existing inputs ... + enable_coverage: + description: >- + Dispatch the ROCm/TheRock code-coverage orchestrator after the build + and test jobs succeed. Nightly-only: a coverage run is a multi-hour + instrumented build, unfit for presubmit. + type: boolean + default: false +``` + +## `trigger_coverage` job + +```yaml +jobs: + # ... existing setup, linux_build_and_test, etc. ... + + trigger_coverage: + name: Trigger TheRock Coverage + needs: [setup, linux_build_and_test] + # Gate on the switch, on there being a build to baseline against, and on the + # test job having succeeded -- coverage measures a run that actually built. + if: >- + ${{ + inputs.enable_coverage && + needs.setup.outputs.enable_build_jobs == 'true' && + needs.linux_build_and_test.result == 'success' + }} + runs-on: ubuntu-24.04 + steps: + - name: Generate GitHub App token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.THEROCK_COVERAGE_APP_ID }} + private-key: ${{ secrets.THEROCK_COVERAGE_APP_PRIVATE_KEY }} + owner: "ROCm" + repositories: "TheRock" + + - name: Dispatch TheRock coverage orchestrator + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v8.0.0 + with: + github-token: ${{ steps.generate-token.outputs.token }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'ROCm', + repo: 'TheRock', + workflow_id: 'multi_arch_ci_coverage.yml', + // Pin to a known-good TheRock ref; bump when the coverage + // workflows change in a way this nightly depends on. + ref: 'main', + inputs: { + // The nightly's own run supplies the non-instrumented baseline. + baseline_run_id: String(context.runId), + baseline_repository: 'ROCm/rocm-libraries', + // Match the channel this nightly publishes to. + baseline_release_type: 'nightly', + // Records the exact rocm-libraries commit that drove coverage. + coverage_config_source: '${{ github.repository }}@${{ github.sha }}', + projects_to_test: 'hipblas,hipdnn,hipfft,hiprand,hipsolver,hipsparselt,rocblas,rocfft,rocrand,rocsolver,rocsparse', + amdgpu_families: 'gfx94X-dcgpu', + }, + }); +``` + +Notes: + +- `baseline_run_id` is stringified because `createWorkflowDispatch` inputs must + be strings; `context.runId` is a number. +- `ref` pins which TheRock copy of the coverage workflows runs. A + `workflow_dispatch` always executes the workflow file from that ref. +- `baseline_release_type` and `projects_to_test` should track whatever the + nightly actually built and published; the values above match the phase 1 + default selection. diff --git a/docs/development/nightly_coverage_implementation.md b/docs/development/nightly_coverage_implementation.md index 2af86734a51..4a6805bb5eb 100644 --- a/docs/development/nightly_coverage_implementation.md +++ b/docs/development/nightly_coverage_implementation.md @@ -178,7 +178,7 @@ At this moment there is **no** instrumented code on the machine at all. ### Step 6 — Swap in the instrumented hipRAND -`overlay_coverage_artifacts.py` +`install_rocm_code_coverage_build.py` Now copy our instrumented hipRAND files over the baseline's copies: @@ -246,12 +246,13 @@ ______________________________________________________________________ | File | Its one job | | ------------------------------------ | -------------------------------------------- | | `coverage_nightly_pr_check.yml` | Starts the run (temporary) | -| `multi_arch_ci_coverage_nightly.yml` | Runs the builds in order | -| `multi_arch_ci_coverage_linux.yml` | Per project: test, then report | -| `test_component.yml` | Installs, swaps, runs tests | +| `multi_arch_ci_coverage.yml` | Thin orchestrator: setup, linux, summary | +| `multi_arch_ci_coverage_linux.yml` | Builds instrumented stacks, then fans out | +| `multi_arch_ci_coverage_report.yml` | Per project: test, then report | +| `test_code_coverage_component.yml` | Installs, swaps, runs tests | | `configure_coverage_ci.py` | The registry — knows every project's details | | `therock_subproject.cmake` | Translates the coverage flag; fixes linking | -| `overlay_coverage_artifacts.py` | Does the swap in step 6 | +| `install_rocm_code_coverage_build.py` | Does the baseline install + project swap | | `merge_coverage_report.py` | Makes the report in step 8 | ______________________________________________________________________ @@ -281,13 +282,14 @@ few milliseconds, and the reason CMake and CI cannot drift apart. ### 1. Entry points and orchestration -All three are new files. +All four are new files. | File | What it does | Called by | Calls | | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `coverage_nightly_pr_check.yml` **(temporary — deleted before merge)** | Makes the coverage workflow runnable from a branch and visible as a PR check, which `workflow_dispatch` alone cannot do | GitHub, on a `pull_request` touching any of nine coverage paths — deliberately narrow, since every run is a multi-hour build | `multi_arch_ci_coverage_nightly.yml` | -| `multi_arch_ci_coverage_nightly.yml` | The real entry point and top-level orchestrator. Five jobs: `setup_coverage_matrix`, `build_instrumented_compiler_runtime`, `build_instrumented_math_libs`, `coverage_report` (project × GPU family), `ci_summary` | A human, from the Actions tab. Today also the PR check above, via a `workflow_call` trigger removed alongside it | `configure_coverage_ci.py`; `multi_arch_build_portable_linux_artifacts.yml` twice (compiler-runtime, math-libs); `multi_arch_ci_coverage_linux.yml` per project | -| `multi_arch_ci_coverage_linux.yml` | Everything for one project on one GPU family, in three jobs: `configure_test_matrix`, `test_coverage`, `aggregate_coverage` | `multi_arch_ci_coverage_nightly.yml`, once per matrix entry | `configure_coverage_ci.py`, `test_component.yml`, `artifact_manager.py`, `merge_coverage_report.py` | +| `coverage_nightly_pr_check.yml` **(temporary — deleted before merge)** | Makes the coverage workflow runnable from a branch and visible as a PR check, which `workflow_dispatch` alone cannot do | GitHub, on a `pull_request` touching any of nine coverage paths — deliberately narrow, since every run is a multi-hour build | `multi_arch_ci_coverage.yml` | +| `multi_arch_ci_coverage.yml` | The real entry point and thin top-level orchestrator. Three jobs: `setup_coverage_matrix`, `coverage_linux` (dispatches the Linux layer), `ci_summary`. Keeps `workflow_dispatch` plus a temporary `workflow_call` | A human, from the Actions tab, or the consumer nightly via App-token dispatch (option C). Today also the PR check above, via the `workflow_call` trigger removed alongside it | `configure_coverage_ci.py`; `multi_arch_ci_coverage_linux.yml` | +| `multi_arch_ci_coverage_linux.yml` | The per-platform build + fan-out layer. Three jobs: `build_instrumented_compiler_runtime`, `build_instrumented_math_libs`, `coverage_report` (project × GPU family). A future `multi_arch_ci_coverage_windows.yml` mirrors it | `multi_arch_ci_coverage.yml` | `multi_arch_build_portable_linux_artifacts.yml` twice (compiler-runtime, math-libs); `multi_arch_ci_coverage_report.yml` per project | +| `multi_arch_ci_coverage_report.yml` | Everything for one project on one GPU family, in three jobs: `configure_test_matrix`, `test_coverage`, `aggregate_coverage` | `multi_arch_ci_coverage_linux.yml`, once per matrix entry | `configure_coverage_ci.py`, `test_code_coverage_component.yml`, `artifact_manager.py`, `merge_coverage_report.py` | ### 2. Planning @@ -312,21 +314,23 @@ configure regardless. ### 4. Artifacts and test execution This is the part that assembles a working install in which exactly one library -is instrumented. Only `overlay_coverage_artifacts.py` is new; the other three -already existed and are extended. +is instrumented. `test_code_coverage_component.yml` and +`install_rocm_code_coverage_build.py` are the new, coverage-only path; +`test_component.yml` is the shared component-test workflow that coverage no +longer uses. | File | What it does | Called by | Calls | | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `test_component.yml` | The shared test workflow behind every component test in the repo. Coverage adds seven inputs and four steps: install from the baseline run, overlay the instrumented project, point `LLVM_PROFILE_FILE` at a per-shard path, and upload the profraw files. Also raises the test timeout to a 60-minute floor, since instrumented `-O0` builds run far slower | Every component-test caller in the repo, and now `multi_arch_ci_coverage_linux.yml` | `setup_test_environment`, then `overlay_coverage_artifacts.py` (only when a baseline run is set), then the project's own test suite | -| `.github/actions/setup_test_environment/action.yml` | Composite action that downloads and installs a ROCm build on the test machine. Coverage adds `RUN_GITHUB_REPO`, because the baseline is published by `ROCm/rockrel` and the bucket lookup fails if it queries the wrong repository | `test_component.yml` only | `install_rocm_from_artifacts.py` | -| `build_tools/github_actions/overlay_coverage_artifacts.py` **(new)** | Step 6, the swap. Copies the instrumented project's stage directories over the baseline install. Most of its care goes into filesystem edge cases: merging into an existing tree, preserving symlinks, and either kind of symlink/regular-file collision | `test_component.yml` | `artifact_manager.py`, imported as a module rather than shelled out to, fetching with `--artifact-names` and `--stage all` | -| `build_tools/artifact_manager.py` | Pre-existing artifact transport: pushes a stage's outputs to S3 and fetches them back. Coverage adds `--artifact-names`, an allowlist, because the existing `--exclude-artifacts` denylist is the wrong shape for fetching one project's files | Six pre-existing callers (`fetch_artifacts.py`, `build_tarballs.py`, `post_stage_upload.py`, `baseline_runs.py`, `stage_reuse_decision.py`, `expand_amdgpu_families.py`) plus `overlay_coverage_artifacts.py` | The S3 artifact backend | +| `test_code_coverage_component.yml` **(new)** | The coverage-only test workflow. Installs from the baseline run and swaps in the instrumented project, points `LLVM_PROFILE_FILE` at a per-shard path, and uploads the profraw files. Also raises the test timeout to a 60-minute floor, since instrumented `-O0` builds run far slower. `test_component.yml`, the shared component-test workflow, was left unedited and is no longer on the coverage path | `multi_arch_ci_coverage_report.yml` | `setup_test_environment`, then `install_rocm_code_coverage_build.py`, then the project's own test suite | +| `.github/actions/setup_test_environment/action.yml` | Composite action that downloads and installs a ROCm build on the test machine. Coverage adds `RUN_GITHUB_REPO`, because the baseline is published by `ROCm/rockrel` and the bucket lookup fails if it queries the wrong repository | `test_component.yml` and `test_code_coverage_component.yml` | `install_rocm_from_artifacts.py` | +| `build_tools/install_rocm_code_coverage_build.py` **(new)** | The swap. Installs the generic tree from `--run-id` (the baseline nightly run) and replaces only the project-under-test's libraries from `--code-coverage-run-id` (this coverage run's own id), deriving the artifact name and folder from the component `job_name` | `test_code_coverage_component.yml` | `artifact_manager.py`, imported as a module rather than shelled out to, fetching with `--artifact-names` and `--stage all` | +| `build_tools/artifact_manager.py` | Pre-existing artifact transport: pushes a stage's outputs to S3 and fetches them back. Coverage adds `--artifact-names`, an allowlist, because the existing `--exclude-artifacts` denylist is the wrong shape for fetching one project's files | Six pre-existing callers (`fetch_artifacts.py`, `build_tarballs.py`, `post_stage_upload.py`, `baseline_runs.py`, `stage_reuse_decision.py`, `expand_amdgpu_families.py`) plus `install_rocm_code_coverage_build.py` | The S3 artifact backend | ### 5. Reporting | File | What it does | Called by | Calls | | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | --------------------------------------------- | -| `build_tools/github_actions/merge_coverage_report.py` **(new)** | Step 8. Finds the profraw files, resolves the object globs (collapsing versioned symlinks like `libfoo.so`, `.so.1`, `.so.1.2` into one object), merges and exports lcov. Fails loudly when a glob matches nothing, which is what caught the hipDNN naming bug instead of quietly emitting an empty report | `multi_arch_ci_coverage_linux.yml`, in `aggregate_coverage` | `llvm-profdata merge`, then `llvm-cov export` | +| `build_tools/github_actions/merge_coverage_report.py` **(new)** | Step 8. Finds the profraw files, resolves the object globs (collapsing versioned symlinks like `libfoo.so`, `.so.1`, `.so.1.2` into one object), merges and exports lcov. Fails loudly when a glob matches nothing, which is what caught the hipDNN naming bug instead of quietly emitting an empty report | `multi_arch_ci_coverage_report.yml`, in `aggregate_coverage` | `llvm-profdata merge`, then `llvm-cov export` | ### 6. Unit tests From 11b1273511b1b10eeddd5b98c00ab18ee1906ad3 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Sun, 20 Sep 2026 20:07:07 -0700 Subject: [PATCH 27/70] style(ci): apply black to code coverage installer test Collapse two short helper calls onto single lines to satisfy black 25.11.0 (the repo-pinned version), matching what pre-commit would produce. --- .../tests/install_rocm_code_coverage_build_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py index 6ed166bcf91..ca3dd3ff512 100644 --- a/build_tools/tests/install_rocm_code_coverage_build_test.py +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -85,9 +85,7 @@ def fake_find_available(artifact_names, target_families, available): return captured def test_replacement_uses_code_coverage_repo_not_generic_repo(self): - captured = self._run_main( - ["--code-coverage-run-github-repo", "ROCm/TheRock"] - ) + captured = self._run_main(["--code-coverage-run-github-repo", "ROCm/TheRock"]) self.assertEqual(captured["run_id"], "222") self.assertEqual(captured["github_repository"], "ROCm/TheRock") self.assertNotEqual(captured["github_repository"], "ROCm/rocm-libraries") @@ -95,9 +93,7 @@ def test_replacement_uses_code_coverage_repo_not_generic_repo(self): def test_replacement_repo_defaults_to_github_repository_env(self): # main() must be re-imported so the argparse default picks up the env, # since the default is evaluated at parse time inside main(). - captured = self._run_main( - [], env={"GITHUB_REPOSITORY": "ROCm/TheRock"} - ) + captured = self._run_main([], env={"GITHUB_REPOSITORY": "ROCm/TheRock"}) self.assertEqual(captured["github_repository"], "ROCm/TheRock") From ef3dc04821654fe950126e6499ffd46c8c3f9a55 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Sun, 20 Sep 2026 20:38:39 -0700 Subject: [PATCH 28/70] docs(ci): tighten verbose coverage comments to plain style Rewrite essay-style comments and docstrings in the coverage workflows and the code-coverage installer into concise, plain English. Comments/docstrings/input descriptions only -- no code, keys, defaults, or behavior change. --- .../workflows/coverage_nightly_pr_check.yml | 118 ++++++++---------- .github/workflows/multi_arch_ci_coverage.yml | 57 ++++----- .../multi_arch_ci_coverage_linux.yml | 57 ++++----- .../multi_arch_ci_coverage_report.yml | 63 +++++----- .../test_code_coverage_component.yml | 71 +++++------ .../install_rocm_code_coverage_build.py | 57 ++++----- 6 files changed, 181 insertions(+), 242 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 64cdfc24daa..38953bc693a 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -1,31 +1,27 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -# TEMPORARY. Delete this file, and the workflow_call trigger it relies on, -# before merging the coverage PR. +# TEMPORARY. Delete this file and the workflow_call trigger it relies on before +# merging the coverage PR. # -# multi_arch_ci_coverage.yml is triggered by hand, but a -# workflow_dispatch only appears once the file is on the default branch. There -# is therefore no way to run it from a branch and no way to see it in a PR's -# checks, which leaves the phase 1 work unable to demonstrate the one thing it -# exists to demonstrate. +# multi_arch_ci_coverage.yml runs by hand, but workflow_dispatch only appears +# once the file is on the default branch -- so it can't run from a branch or +# show in a PR's checks, leaving phase 1 unable to demonstrate itself. # # A caller has no such restriction: paths under ./.github/workflows resolve -# against the caller's own commit, so this runs the branch's copy of the -# coverage workflow and its jobs report as checks on this PR. +# against the caller's commit, so this runs the branch's copy and its jobs +# report as checks on this PR. # -# This is scaffolding, not part of the design. Coverage on presubmit is exactly -# what the phase 1 scope rejects: one run is a multi-hour instrumented build -# plus a full test suite, which is why coverage is a separate, manually -# triggered workflow in the first place. Leaving this in place would charge -# that cost to every PR touching coverage. +# Scaffolding, not design. Presubmit coverage is what phase 1 rejects: each run +# is a multi-hour instrumented build plus full test suite, which is why coverage +# is a separate manual workflow. Leaving this in charges that cost to every PR. name: Coverage Nightly PR Check on: pull_request: - # Narrow on purpose. Every run here is a multi-hour instrumented build, so - # only the files carrying coverage logic are worth spending one on. + # Narrow on purpose: each run is a multi-hour build, so only files carrying + # coverage logic are worth one. paths: - ".github/workflows/coverage_nightly_pr_check.yml" - ".github/workflows/multi_arch_ci_coverage.yml" @@ -39,9 +35,8 @@ on: - "CMakeLists.txt" concurrency: - # A later push supersedes this run. Queueing multi-hour instrumented builds - # behind each other, to report on code that has already been replaced, only - # occupies build nodes the rest of CI needs. + # A later push supersedes this run: queueing multi-hour builds to report on + # already-replaced code just occupies nodes the rest of CI needs. group: coverage-nightly-pr-check-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -50,72 +45,59 @@ jobs: name: Coverage uses: ./.github/workflows/multi_arch_ci_coverage.yml secrets: inherit - # A called workflow cannot hold more than its caller, so these are granted - # here even though the coverage workflow declares them itself: id-token for - # the AWS role the artifact fetches assume, actions:read for the baseline - # run lookup. + # A called workflow can't hold more than its caller, so grant these here too: + # id-token for the AWS role the artifact fetches assume, actions:read for the + # baseline run lookup. permissions: contents: read actions: read id-token: write with: - # Pinned rather than resolved, so that a change in the report means a - # change in this branch. + # Pinned, not resolved, so a change in the report means a change in this branch. # # ROCm/rockrel run 34068476531, "Multi-Arch Release (nightly)" on main, - # 2026-09-07, succeeded. A real nightly rather than a CI run, because - # that is what a dispatched coverage run will be given in practice and - # the phase 1 PoC is worth exercising against the real thing. + # 2026-09-07, succeeded. A real nightly, not a CI run, since that's what a + # dispatched coverage run gets in practice. # - # Being a rockrel run is safe here. Its artifacts embed the workspace - # they were built in (/__w/rockrel/rockrel), which breaks any consumer - # that configures CMake against them, but this workflow never does: it - # builds the instrumented stack from source in its own workspace and - # only installs the baseline on the test runner, where prebuilt binaries - # are executed and no package config is read. + # A rockrel run is safe here. Its artifacts embed their build workspace + # (/__w/rockrel/rockrel), which breaks consumers that configure CMake + # against them -- but this workflow never does: it builds the instrumented + # stack from source and only installs the baseline on the test runner, + # where prebuilt binaries run and no package config is read. # - # Verified before pinning that this run publishes artifacts covering - # every selected project: blas, fft, hipdnn, rand, solver and - # sparse. Not all are per-target -- hipdnn is generic throughout and - # solver's tests are generic -- so checking only for _gfx942 understates - # what is there. + # Verified before pinning that this run publishes artifacts for every + # selected project: blas, fft, hipdnn, rand, solver, sparse. Not all are + # per-target (hipdnn and solver's tests are generic), so checking only for + # _gfx942 understates what's there. baseline_run_id: "34068476531" - # The bucket lookup queries the run through the GitHub API, so it has to - # be told where the run lives; ROCm/TheRock returns 404 for a rockrel id. + # The bucket lookup queries the run via the GitHub API, so it needs the + # owning repo; ROCm/TheRock 404s for a rockrel id. baseline_repository: "ROCm/rockrel" - # That run published to therock-nightly-artifacts, which is the default, - # but naming it keeps the pin and its channel in one place. + # That run published to therock-nightly-artifacts (the default); naming it + # keeps the pin and channel together. baseline_release_type: "nightly" amdgpu_families: "gfx94X-dcgpu" # Every rocm-libraries project that builds a shared library. Run - # 34135220913 proved the path end to end on hipRAND -- instrumented - # build, baseline install, overlay, tests and a 77% report -- once - # libclang_rt.profile_rocm.a's own dependencies (-ldl, -lpthread) were - # added to the link line, which had been failing every instrumented - # binary before that. Run 34164420766 then took the instrumented build - # of all twelve green and reported six of them; the six that failed are - # what the host-only scoping, the timeout floor and the two registry - # corrections in this branch are for. + # 34135220913 proved the path end to end on hipRAND (build, install, + # overlay, tests, 77% report) once libclang_rt.profile_rocm.a's own deps + # (-ldl, -lpthread) were added to the link line. Run 34164420766 then built + # all twelve green and reported six; the six failures are what the host-only + # scoping, timeout floor and two registry corrections in this branch fix. # - # Named explicitly rather than using rocm_libraries_all because the four - # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA) are left - # out. They have no library of their own to instrument, so they report - # against instrumented test binaries instead, and their inline code is - # the linkonce_odr case the overlay cannot fully isolate. Worth doing, - # but worth doing separately. + # Named explicitly, not rocm_libraries_all, to leave out the four + # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA): they have no + # library to instrument, report against instrumented test binaries, and hit + # the linkonce_odr case the overlay can't isolate. Worth doing separately. # - # rccl and rocshmem are absent because they are not selectable: they - # live in comm-libs, which has no build job here. Both are also blocked - # upstream -- rccl's CMake refuses coverage outside a Debug build, and - # rocshmem fails on coverage plus HIP relocatable device code. + # rccl and rocshmem aren't selectable: they live in comm-libs, which has no + # build job here. Both are also blocked upstream -- rccl's CMake refuses + # coverage outside Debug, and rocshmem fails on coverage + HIP RDC. # - # hipBLASLt and hipTensor are absent for the same reason they are absent - # from the default selection: neither instrumented build links. See their - # blocked_reason in configure_coverage_ci.py. hipTensor was still in this - # list for run 34201973090, which is how its failure was found: the two - # registry corrections meant it was instrumented for the first time, and - # a stage is built as a unit, so its link error failed math-libs and - # skipped the report for all twelve. + # hipBLASLt and hipTensor are absent because neither instrumented build + # links (see blocked_reason in configure_coverage_ci.py). hipTensor was + # still listed for run 34201973090, which found its failure: the registry + # corrections instrumented it for the first time, and a stage builds as a + # unit, so its link error failed math-libs and skipped all twelve reports. projects_to_test: >- hipblas,hipdnn,hipfft,hiprand,hipsolver, hipsparselt,rocblas,rocfft,rocrand,rocsolver,rocsparse diff --git a/.github/workflows/multi_arch_ci_coverage.yml b/.github/workflows/multi_arch_ci_coverage.yml index ee00098ec58..d299f68e514 100644 --- a/.github/workflows/multi_arch_ci_coverage.yml +++ b/.github/workflows/multi_arch_ci_coverage.yml @@ -5,19 +5,16 @@ # # Computes the shared coverage matrix once (setup_coverage_matrix) and hands it # to a per-platform build+fanout layer (multi_arch_ci_coverage_linux.yml today; -# a Windows sibling later). The instrumented build and the per-project -# test+report live in those layers, not here, so adding a platform duplicates -# none of the matrix logic. +# Windows later). The build and per-project test+report live in those layers, +# so adding a platform duplicates no matrix logic. # -# RFC0014 phase 1 runs every enabled component on one default architecture -# (gfx942, i.e. the gfx94X-dcgpu family). Change-based selection and -# multi-architecture expansion come later. +# RFC0014 phase 1: every enabled component on one architecture (gfx94X-dcgpu). +# Change-based selection and multi-arch come later. # -# Coverage runs as its own run, separate from the regular nightly: the nightly -# dispatches this workflow once its build finishes and hands over its own run id -# as `baseline_run_id` (RFC option C). Coverage having a run id of its own means -# a failure here leaves the nightly's status alone, and the nightly's unstable -# test jobs do not hold coverage up. +# Coverage runs separately from the nightly, which dispatches it and passes its +# own run id as `baseline_run_id` (RFC option C). A separate run id means a +# failure here doesn't touch the nightly's status, nor its unstable tests hold +# coverage up. name: Multi-Arch CI Coverage @@ -38,7 +35,7 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group alias: rocm_libraries_all. Empty = every project this workflow can build. rccl and rocshmem need the comm-libs build job, which phase 1 does not have, so naming them (or rocm_systems_all / all) is rejected." + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." default: "" amdgpu_families: type: string @@ -55,16 +52,15 @@ on: quartz_tracking_id: type: string description: >- - Leave empty. Internal Quartz tracking id, used to link this run to a - release lineage once the nightly dispatches coverage automatically. + Leave empty. Internal Quartz tracking id linking this run to a release + lineage once the nightly dispatches coverage automatically. default: "" - # TEMPORARY, paired with coverage_nightly_pr_check.yml. A workflow_dispatch - # is only offered once the file is on the default branch, so this workflow - # cannot be run from a branch and cannot be shown in a PR's checks. A caller - # is not subject to that, so the check workflow runs this branch's copy. - # Actions does not expand YAML anchors, so the inputs are repeated verbatim. - # Remove this trigger with the check workflow once the run has gone green. + # TEMPORARY, paired with coverage_nightly_pr_check.yml. workflow_dispatch only + # appears once the file is on the default branch, so this can't run from a + # branch or show in PR checks; a caller can, so the check workflow runs this + # branch's copy. Actions doesn't expand YAML anchors, so inputs are repeated. + # Remove this trigger with the check workflow once the run is green. workflow_call: inputs: baseline_run_id: @@ -81,7 +77,7 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand); case-insensitive. Group alias: rocm_libraries_all. Empty = every project this workflow can build. rccl and rocshmem need the comm-libs build job, which phase 1 does not have, so naming them (or rocm_systems_all / all) is rejected." + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." default: "" amdgpu_families: type: string @@ -98,15 +94,14 @@ on: quartz_tracking_id: type: string description: >- - Leave empty. Internal Quartz tracking id, used to link this run to a - release lineage once the nightly dispatches coverage automatically. + Leave empty. Internal Quartz tracking id linking this run to a release + lineage once the nightly dispatches coverage automatically. default: "" permissions: contents: read -# The two run ids are only connected through this input, so surface it in the -# run title rather than making people open the logs to find the baseline. +# Surface the baseline in the run title so people don't dig through logs for it. run-name: >- Multi-Arch CI Coverage | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} @@ -153,16 +148,14 @@ jobs: --release-type=ci \ --override-git-sha=${{ steps.checkout.outputs.commit }} - # The build + fanout half is per-platform. The shared matrix computed above is - # handed to the Linux layer, which builds the instrumented stack once and then - # fans out to the per-project report workflow. A Windows layer would be a - # sibling job here, reusing setup_coverage_matrix and the report workflow - # unchanged. + # Build + fanout is per-platform. The Linux layer builds the instrumented + # stack once, then fans out to the per-project report workflow. A Windows + # layer would be a sibling job reusing the same matrix and report workflow. coverage_linux: name: Coverage - Linux needs: setup_coverage_matrix - # `fromJSON('')` is a hard error downstream, and an empty selection has - # nothing to build, so the matrix is checked explicitly before fanning out. + # Empty selection has nothing to build, and fromJSON('') hard-errors, so + # check the matrix explicitly. if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml secrets: inherit diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index c614bb24709..5c1b3973c9e 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -3,20 +3,18 @@ # Multi-Arch CI Coverage - Linux # -# Per-platform build + fanout layer. Builds one fully instrumented Linux stack -# covering every coverage-enabled project, then fans out to -# multi_arch_ci_coverage_report.yml once per project to test against that stack -# and report separately. Paying for a single large build rather than one build -# per project is what keeps the cost flat as more projects are onboarded. +# Per-platform build + fanout layer. Builds one instrumented Linux stack for +# every coverage-enabled project, then fans out to +# multi_arch_ci_coverage_report.yml once per project. One large build instead +# of one per project keeps cost flat as projects are onboarded. # -# The stack is assembled by naming the stages coverage needs and calling the -# shared per-stage build workflow for each, rather than running the whole -# multi_arch_build_portable_linux.yml pipeline and filtering stages out of it. +# The stack is assembled by calling the shared per-stage build workflow for each +# needed stage, rather than running multi_arch_build_portable_linux.yml and +# filtering stages out. # -# The shared coverage matrix and build configuration are computed once by the -# parent (multi_arch_ci_coverage.yml) and handed down as inputs, so that adding -# multi_arch_ci_coverage_windows.yml later duplicates none of that logic -- it -# only mirrors the build + fanout below with Windows runners. +# The matrix and build config are computed once by the parent +# (multi_arch_ci_coverage.yml) and passed in, so a future Windows layer +# duplicates none of it -- it only mirrors the build + fanout with Windows runners. name: Multi-Arch CI Coverage - Linux @@ -65,16 +63,15 @@ on: type: string default: "" description: >- - Leave empty. Internal Quartz tracking id, propagated from the caller - to link this run to a release lineage once the nightly dispatches - coverage automatically. + Leave empty. Internal Quartz tracking id, propagated from the caller to + link this run to a release lineage once the nightly dispatches coverage. permissions: contents: read jobs: - # Built first so that every later stage in this run has a compiler-runtime to - # link against and to fetch as an inbound artifact. + # Built first so later stages have a compiler-runtime to link against and + # fetch as an inbound artifact. build_instrumented_compiler_runtime: name: Build Instrumented Compiler Runtime if: ${{ inputs.coverage_matrix != '[]' }} @@ -92,12 +89,10 @@ jobs: contents: read id-token: write - # The whole stage is built, not just the onboarded projects: the test jobs - # below need the complete dependency set, and phase 1 instruments every - # onboarded project in one pass rather than per changed project. - # `coverage_cmake_options` is the THEROCK_COVERAGE_*_ALL group option when the - # selection covers a whole group, and the individual project flags otherwise; - # everything not selected builds normally. + # Build the whole stage, not just onboarded projects: test jobs need the full + # dependency set, and phase 1 instruments every project in one pass. + # coverage_cmake_options is the THEROCK_COVERAGE_*_ALL group option for a whole + # group, else individual project flags; unselected projects build normally. build_instrumented_math_libs: name: Build Instrumented Math Libs needs: build_instrumented_compiler_runtime @@ -126,23 +121,19 @@ jobs: contents: read id-token: write - # One test + report job per project. The instrumented stack built above is the - # only thing taken from this run: each job installs `baseline_run_id` and then - # replaces its own project, so a project's dependencies stay non-instrumented - # even though the build instrumented all of them at once. + # One test + report job per project. Each installs baseline_run_id and replaces + # only its own project from this run's instrumented stack, so dependencies stay + # non-instrumented even though the build instrumented them all at once. coverage_report: name: Coverage::${{ matrix.coverage.project_name }} (${{ matrix.coverage.amdgpu_families }}) - # compiler-runtime is named explicitly rather than relied on through - # math-libs: a selection that needs no math-libs build still tests against - # binaries that stage produced. + # Name compiler-runtime explicitly, not via math-libs: a selection with no + # math-libs build still tests against binaries that stage produced. needs: [ build_instrumented_compiler_runtime, build_instrumented_math_libs, ] - # `!failure() && !cancelled()` alone would let this run when the parent's - # matrix was empty, and fromJSON('') is a hard error, so the matrix has to - # be checked explicitly. + # Also guard the empty matrix: fromJSON('') would hard-error. if: >- ${{ !failure() && !cancelled() && diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 082fa62ae72..2404cb26f35 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -8,15 +8,13 @@ # # test (collect profraw) -> merge -> report # -# The instrumented stack itself is built by the caller -# (multi_arch_ci_coverage_linux.yml), which instruments every onboarded -# project in one pass. This workflow only consumes it. +# The instrumented stack is built by the caller +# (multi_arch_ci_coverage_linux.yml); this workflow only consumes it. # -# `baseline_run_id` drives the hybrid install: the tree comes from a regular -# nightly run and only the project under test is replaced from the instrumented -# run, keeping instrumented dependencies out of the report. Without it the -# instrumented run's own artifacts are tested as built, which measures the -# project's dependencies alongside it. +# baseline_run_id drives the hybrid install: the tree comes from a regular +# nightly and only the project under test is replaced from the instrumented run, +# keeping instrumented dependencies out of the report. Without it, the +# instrumented artifacts are tested as built, measuring dependencies alongside it. name: Multi-Arch CI Coverage - Report @@ -40,17 +38,16 @@ on: type: string default: "" description: >- - Build stage owning the project. The report job fetches this stage's - sources so the HTML rendering has something to annotate; the lcov and - text outputs do not need them. Empty skips the fetch and the HTML. + Build stage owning the project. Its sources are fetched so the HTML + report can annotate them; lcov and text outputs don't need them. + Empty skips the fetch and the HTML. baseline_run_id: type: string default: "" description: >- - Regular nightly run to take non-instrumented dependencies from. Set - by the nightly caller, which instruments the whole stack in its own - run; the test jobs then install that run and overlay only the project - under test. Empty tests the instrumented artifacts as built. + Regular nightly run supplying non-instrumented dependencies. Test jobs + install it and overlay only the project under test. Empty tests the + instrumented artifacts as built. baseline_repository: type: string default: "" @@ -122,8 +119,8 @@ jobs: AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEST_TYPE: ${{ inputs.test_type }} TEST_LABELS: "[]" - # The report covers one project, so only that project's tests run - # here; the other onboarded projects get their own job. + # One project per report, so only its tests run here; other projects + # get their own job. PROJECTS_TO_TEST: ${{ inputs.test_component }} CI_CONFIG_PATH: ci-config BUILD_VARIANT: coverage @@ -147,10 +144,9 @@ jobs: uses: ./.github/workflows/test_code_coverage_component.yml secrets: inherit with: - # The installer takes the generic tree from baseline_run_id and replaces - # only this project's libs from the instrumented run (this workflow's own - # run id), deriving the replacement artifact name and folder from the - # component job_name -- so artifact_names/relpaths are no longer needed. + # Installer takes the generic tree from baseline_run_id and replaces only + # this project's libs from this run's instrumented artifacts, deriving the + # artifact name and folder from the component job_name. generic_run_id: ${{ inputs.baseline_run_id }} baseline_repository: ${{ inputs.baseline_repository }} artifact_group: multi-arch-coverage @@ -169,8 +165,8 @@ jobs: aggregate_coverage: name: Coverage Report (${{ inputs.project_name }}) needs: [configure_test_matrix, test_coverage] - # Runs even when some shards failed: a partial report is more useful than - # none, and a failing test still exercises code. + # Runs even if some shards failed: a partial report beats none, and a + # failing test still exercises code. if: >- ${{ !cancelled() && @@ -221,9 +217,9 @@ jobs: --output-dir=${OUTPUT_ARTIFACTS_DIR} \ ${FETCH_ARTIFACT_ARGS} - # Only the HTML rendering reads source, so this runs after the artifacts - # are already down: a failure here, including running the runner out of - # disk, costs the annotated view and leaves lcov and the text table. + # Only HTML rendering reads source, so fetch after the artifacts are down: + # a failure here (e.g. out of disk) costs only the annotated view, leaving + # lcov and the text table. - name: Fetch sources for the annotated report if: ${{ inputs.build_stage != '' }} id: fetch_sources @@ -238,10 +234,10 @@ jobs: - name: Merge profraw and generate reports env: - # Sources are recorded at the path the build container used, which is - # not where they sit here, so llvm-cov is told how to get from one to - # the other. Generated files under build/ are in neither and are - # reported as uncovered; the text summary still counts them. + # Sources are recorded at the build container's path, not where they + # sit here, so tell llvm-cov how to map between them. Generated files + # under build/ are in neither and report as uncovered; the text + # summary still counts them. BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} run: | @@ -265,8 +261,7 @@ jobs: --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ "${html_args[@]}" - # Puts the totals on the run's summary page, so the number is readable - # without downloading anything. + # Put totals on the run summary page, readable without downloading. - name: Publish the summary to the job page if: ${{ always() }} run: | @@ -292,8 +287,8 @@ jobs: ${{ env.REPORT_DIR }}/html if-no-files-found: warn - # Skipped rather than failed when the repository has no Codecov token, - # so forks and first-time setups still get the lcov artifact above. + # Skip (not fail) when there's no Codecov token, so forks and first-time + # setups still get the lcov artifact above. - name: Upload report to Codecov if: ${{ env.HAS_CODECOV_TOKEN == 'true' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 4050907c373..1d6277bb710 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -11,10 +11,10 @@ on: default: "" baseline_repository: description: >- - Repository owning generic_run_id (owner/repo), passed to the installer - as --run-github-repo for the generic fetch. Empty lets the installer - default to the current repository. The instrumented replacement always - resolves against this coverage run's own repository. + Repository owning generic_run_id (owner/repo), passed as + --run-github-repo. Empty defaults to the current repository. The + instrumented replacement always resolves against this coverage run's + own repository. type: string default: "" artifact_group: @@ -45,16 +45,14 @@ on: default: "release" coverage_enabled: description: >- - Collect llvm profraw files from the test run and upload them as a - workflow artifact. Only meaningful against a coverage instrumented - build. + Collect llvm profraw files from the test run and upload them. + Only meaningful against a coverage-instrumented build. type: boolean default: false coverage_artifact_prefix: description: >- - Prefix for the uploaded profraw artifact name. A coverage run tests - several projects in one workflow run, so the caller sets this to - something project-specific to keep each project profiles apart. + Prefix for the uploaded profraw artifact name. Set per-project so one + coverage run's projects keep their profiles apart. type: string default: "coverage-profraw" default_container_image: @@ -78,22 +76,20 @@ jobs: outputs: summary: ${{ steps.summary.outputs.summary }} container: - # If the component has specified an alternate image, honor that option. - # Otherwise use the default image. + # Honor the component's alternate image if set, else the default. image: >- ${{ inputs.platform == 'linux' && (fromJSON(inputs.component).container_image || inputs.default_container_image) || null }} - # Container options are configured in build_tools/github_actions/fetch_test_configurations.py - # and include base options (ipc, user, ulimit, seccomp), GPU options (devices, groups), - # and any job-specific options + # Container options come from fetch_test_configurations.py: base (ipc, + # user, ulimit, seccomp), GPU (devices, groups), and job-specific options. options: ${{ fromJSON(inputs.component).container_options }} strategy: fail-fast: false matrix: - # The shard array is based on "total_shards" from "fetch_test_configurations.py" - # The test executable will shard based on the array. (ex: [1, 2, 3, 4] = four test shards) + # Shard array from fetch_test_configurations.py's total_shards. + # The test executable shards on it (e.g. [1,2,3,4] = four shards). shard: ${{ fromJSON(inputs.component).shard_arr }} defaults: run: @@ -106,9 +102,8 @@ jobs: AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} ARTIFACT_GROUP: ${{ inputs.artifact_group }} BUILD_VARIANT: ${{ inputs.build_variant }} - # Benchmark results database API endpoints for performance metrics collection - # NOTE: These secrets are only required for benchmark results submission in nightly CI runs. - # For PR/push workflows, secret retrieval will fail gracefully and benchmarks will skip API submission. + # Benchmark results DB endpoints. Only needed for nightly benchmark + # submission; PR/push runs fail retrieval gracefully and skip submission. BENCHMARK_DB_URL: ${{ secrets.BENCHMARK_DB_URL }} BENCHMARK_DB_FALLBACK_URL: ${{ secrets.BENCHMARK_DB_FALLBACK_URL }} steps: @@ -147,18 +142,15 @@ jobs: BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} run: | pip3 install -r requirements-test.txt - # The generic (baseline) tree comes from generic_run_id. When it is - # empty (self-contained dispatch), fall back to this run's own - # instrumented artifacts so the install still resolves -- the - # subsequent --replace then re-extracts the same instrumented libs, - # leaving the whole stack instrumented, which is the documented - # self-contained behavior. + # Baseline tree comes from generic_run_id. When empty (self-contained + # dispatch), fall back to this run's own instrumented artifacts; + # --replace then re-extracts them, leaving the whole stack instrumented. generic_run_id="${{ inputs.generic_run_id }}" if [[ -z "${generic_run_id}" ]]; then generic_run_id="${{ github.run_id }}" fi - # Only pass --run-github-repo when a baseline repository was given, so - # an empty value lets the installer default to the current repository. + # Pass --run-github-repo only when set, so empty defaults to the + # current repository. run_github_repo_arg=() if [[ -n "${BASELINE_REPOSITORY}" ]]; then run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") @@ -205,8 +197,8 @@ jobs: echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV # Instrumented binaries write one profraw per process. %p (pid) and %m - # (binary signature) keep the files distinct so a shard that forks or - # loads several instrumented libraries does not overwrite itself. + # (binary signature) keep files distinct so a forking/multi-lib shard + # doesn't overwrite itself. - name: Prepare coverage profile directory if: ${{ inputs.coverage_enabled }} env: @@ -216,8 +208,8 @@ jobs: mkdir -p "${GITHUB_WORKSPACE}/coverage-report/profraw" echo "LLVM_PROFILE_FILE=${GITHUB_WORKSPACE}/coverage-report/profraw/${TEST_COMPONENT}-shard${SHARD_INDEX}-%p-%m.profraw" >> "${GITHUB_ENV}" - # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription. - # Must be done at runtime since container env vars aren't in GitHub's env context. + # Limit threads from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid + # oversubscription. Done at runtime; container env isn't in GitHub's context. - name: Set thread limits run: | if [[ -n "${KUBE_CPU_REQUEST:-}" ]]; then @@ -238,12 +230,11 @@ jobs: TEST_TYPE: ${{ fromJSON(inputs.component).test_type }} TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }} ROCM_KPACK_DEBUG: "1" - # Windows hip-tests (PAL=1, ROCR=0) set this via matrix; other components - # leave it empty. #3587 - # An empty GPU_ENABLE_PAL is misread as ROCr on Windows (atoi("")==0), - # whose Windows memory path OOMs some tests (e.g. rocsparse bsr*0 graph_test). - # Default unset components to the platform backend, mirroring the runtime fix - # in ROCm/rocm-systems#8050 (Windows -> PAL, Linux -> ROCr). + # Windows hip-tests (PAL=1, ROCR=0) set this via matrix; others leave + # it empty. #3587 An empty value is misread as ROCr on Windows + # (atoi("")==0), whose memory path OOMs some tests (e.g. rocsparse + # bsr*0 graph_test). Default unset components to the platform backend, + # per ROCm/rocm-systems#8050 (Windows -> PAL, Linux -> ROCr). GPU_ENABLE_PAL: ${{ fromJSON(inputs.component).gpu_enable_pal || (inputs.platform == 'windows' && '1' || '0') }} RETRY_THIS_STEP: "true" RETRY_COUNT: "1" @@ -257,8 +248,8 @@ jobs: run: | ${{ fromJSON(inputs.component).test_script }} - # Uploaded even when the test step failed: partial coverage from a failing - # shard is still useful, and the aggregation job tolerates missing shards. + # Upload even on test failure: partial coverage is useful and the + # aggregation job tolerates missing shards. - name: Upload coverage profraw files if: ${{ always() && inputs.coverage_enabled }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index 0013d43f627..9dd555aec36 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -35,9 +35,8 @@ from _therock_utils.artifacts import ArtifactName from _therock_utils.cmake_amdgpu_targets import amdgpu_family_map, expand_families -# Maps each --replace- flag (argparse dest) to the TheRock artifact name -# that ships the instrumented library. rocBLAS is packaged in the 'blas' -# artifact and rocSOLVER in the 'solver' artifact (see BUILD_TOPOLOGY.toml). +# Maps each --replace- flag to its TheRock artifact and library folder. +# e.g. rocBLAS ships in the 'blas' artifact (see BUILD_TOPOLOGY.toml). COMPONENT_MAP = { # component: (artifact_name, library_folder), "rocfft": ("fft", "rocFFT"), @@ -67,10 +66,9 @@ def _read_passthrough_options(passthrough_argv): - """Read (without consuming) options shared with install_rocm_from_artifacts. + """Read options shared with install_rocm_from_artifacts without consuming them. - These are parsed non-destructively so the same argv can still be forwarded - to install_rocm_from_artifacts.py unchanged. + Parsed non-destructively so the same argv still forwards unchanged. """ reader = argparse.ArgumentParser(add_help=False) # --amdgpu-family and --artifact-group share a dest, mirroring install_rocm_from_artifacts. @@ -86,8 +84,8 @@ def _read_passthrough_options(passthrough_argv): def _target_families(family, amdgpu_targets): """Build the family match set: generic + family + expanded gfx targets. - blas/solver are target-specific artifacts named per family (mono-arch) or - per gfx target (kpack-split), so both spellings must be matched. + Target-specific artifacts (blas/solver) are named per family (mono-arch) or + per gfx target (kpack-split), so match both spellings. """ families = ["generic"] if family: @@ -100,18 +98,12 @@ def _target_families(family, amdgpu_targets): def download_replacement_artifacts( code_coverage_run_id, artifact_names, opts, code_coverage_run_github_repo ): - """Download the instrumented replacement artifacts from the code-coverage run. - - Uses the code coverage run ID as the run-id for the S3 backend, then fetches - every component tar matching the requested artifact names and target family. - - The instrumented replacement lives in a DIFFERENT repo than the generic - baseline: the generic build comes from --run-id in --run-github-repo (e.g. - ROCm/rocm-libraries), while the instrumented coverage run comes from - --code-coverage-run-id in --code-coverage-run-github-repo (the coverage - run's own repo, e.g. ROCm/TheRock). The backend's S3 bucket lookup 404s if - the run id is queried against the wrong owning repo, so we must pass the - code-coverage repo here rather than reusing opts.run_github_repo. + """Download instrumented replacement artifacts from the code-coverage run. + + Fetches every component tar matching the requested artifact names and family. + The coverage run lives in its own repo (--code-coverage-run-github-repo), + separate from the generic baseline's --run-github-repo. Pass it explicitly: + the S3 bucket lookup 404s if the run id is queried against the wrong repo. """ backend = create_backend_from_env( run_id=code_coverage_run_id, @@ -152,9 +144,8 @@ def download_replacement_artifacts( def _replace_scoped_member(tf, member, dest_path, output_dir, relpaths): """Write a single archive member into the flattened install tree. - Mirrors the file/symlink/dir/hardlink handling used when TheRock flattens - an artifact archive, so replaced files keep their exec bits and link - structure. Any existing file/symlink at dest_path is removed first. + Mirrors TheRock's file/symlink/dir/hardlink flattening so replaced files + keep their exec bits and link structure. Removes any existing file first. """ if dest_path.is_symlink() or (dest_path.exists() and not dest_path.is_dir()): os.unlink(dest_path) @@ -173,8 +164,8 @@ def _replace_scoped_member(tf, member, dest_path, output_dir, relpaths): elif member.issym(): dest_path.symlink_to(member.linkname) elif member.islnk(): - # Hardlink target is archive-relative; strip its manifest prefix so it - # resolves to the already-written file in the flattened output tree. + # Hardlink target is archive-relative; strip the manifest prefix to + # resolve it against the already-written flattened file. for prefix in relpaths: prefix_slash = prefix + "/" if member.linkname.startswith(prefix_slash): @@ -192,10 +183,9 @@ def _replace_scoped_member(tf, member, dest_path, output_dir, relpaths): def replace_instrumented_libraries(artifacts, dest_dir, output_dir): """Extract instrumented libs from downloaded archives into the install tree. - For every replacement archive under dest_dir, read its artifact_manifest.txt - to learn the relpath prefixes, then flatten (strip prefix) each member into - output_dir -- but only members whose scoped path matches the artifact's - library folder (rocBLAS/rocSOLVER), so unrelated files are left in place. + Reads each archive's artifact_manifest.txt for relpath prefixes, then flattens + members into output_dir -- but only those matching the artifact's library + folder (e.g. rocBLAS), leaving unrelated files in place. """ archives = sorted( p for p in dest_dir.iterdir() if p.name.endswith((".tar.zst", ".tar.xz")) @@ -252,12 +242,9 @@ def main(argv): type=str, help="run id of the build from which instrumental components needs to be replaced", ) - # The instrumented replacement fetch is keyed on a DIFFERENT repo than the - # generic install: the generic baseline uses --run-github-repo (e.g. - # ROCm/rocm-libraries), while the coverage run lives in its own repo (e.g. - # ROCm/TheRock). Defaults to $GITHUB_REPOSITORY so CI resolves to the - # coverage run's own repo automatically. Consumed by parse_known_args so it - # does NOT leak into extra_args / the generic install. + # Repo owning --code-coverage-run-id (the instrumented run), separate from + # --run-github-repo (the generic baseline). Defaults to $GITHUB_REPOSITORY. + # parse_known_args consumes it so it does not leak into the generic install. parser.add_argument( "--code-coverage-run-github-repo", type=str, From 4f177cd61f14384e315d12fb1433dc80bba54123 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Mon, 21 Sep 2026 11:59:12 -0700 Subject: [PATCH 29/70] refactor(ci): remove L1 thin parent; make _linux.yml the direct dispatch target Per-platform coverage dispatch mirrors the nightly's own pattern: the nightly dispatches multi_arch_ci_linux.yml directly, so trigger_coverage_linux should dispatch multi_arch_ci_coverage_linux.yml directly with no shared orchestrator in between. - Delete multi_arch_ci_coverage.yml (L1 thin parent): its setup_coverage_matrix and ci_summary jobs move into _linux.yml; its workflow_dispatch / workflow_call triggers move there too. - Rewrite multi_arch_ci_coverage_linux.yml as the self-contained Linux entry point: add setup_coverage_matrix, workflow_dispatch + workflow_call triggers, run-name, concurrency; switch build jobs from inputs.* to needs.setup_coverage_matrix.outputs.*; add ci_summary. - Update coverage_nightly_pr_check.yml: point uses: at _linux.yml directly; remove multi_arch_ci_coverage.yml from paths filter. - Update coverage_nightly_trigger_snippet.md: split single enable_coverage into enable_coverage_linux + enable_coverage_windows; dispatch targets are now multi_arch_ci_coverage_linux.yml and multi_arch_ci_coverage_windows.yml. --- .../workflows/coverage_nightly_pr_check.yml | 9 +- .github/workflows/multi_arch_ci_coverage.yml | 195 ------------------ .../multi_arch_ci_coverage_linux.yml | 190 ++++++++++++----- .../coverage_nightly_trigger_snippet.md | 97 +++++++-- 4 files changed, 224 insertions(+), 267 deletions(-) delete mode 100644 .github/workflows/multi_arch_ci_coverage.yml diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 38953bc693a..59b5e60c2d0 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -4,9 +4,9 @@ # TEMPORARY. Delete this file and the workflow_call trigger it relies on before # merging the coverage PR. # -# multi_arch_ci_coverage.yml runs by hand, but workflow_dispatch only appears -# once the file is on the default branch -- so it can't run from a branch or -# show in a PR's checks, leaving phase 1 unable to demonstrate itself. +# multi_arch_ci_coverage_linux.yml runs by hand, but workflow_dispatch only +# appears once the file is on the default branch -- so it can't run from a +# branch or show in a PR's checks, leaving phase 1 unable to demonstrate itself. # # A caller has no such restriction: paths under ./.github/workflows resolve # against the caller's commit, so this runs the branch's copy and its jobs @@ -24,7 +24,6 @@ on: # coverage logic are worth one. paths: - ".github/workflows/coverage_nightly_pr_check.yml" - - ".github/workflows/multi_arch_ci_coverage.yml" - ".github/workflows/multi_arch_ci_coverage_linux.yml" - ".github/workflows/multi_arch_ci_coverage_report.yml" - ".github/workflows/test_code_coverage_component.yml" @@ -43,7 +42,7 @@ concurrency: jobs: coverage: name: Coverage - uses: ./.github/workflows/multi_arch_ci_coverage.yml + uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml secrets: inherit # A called workflow can't hold more than its caller, so grant these here too: # id-token for the AWS role the artifact fetches assume, actions:read for the diff --git a/.github/workflows/multi_arch_ci_coverage.yml b/.github/workflows/multi_arch_ci_coverage.yml deleted file mode 100644 index d299f68e514..00000000000 --- a/.github/workflows/multi_arch_ci_coverage.yml +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Multi-Arch CI Coverage (thin parent / orchestrator) -# -# Computes the shared coverage matrix once (setup_coverage_matrix) and hands it -# to a per-platform build+fanout layer (multi_arch_ci_coverage_linux.yml today; -# Windows later). The build and per-project test+report live in those layers, -# so adding a platform duplicates no matrix logic. -# -# RFC0014 phase 1: every enabled component on one architecture (gfx94X-dcgpu). -# Change-based selection and multi-arch come later. -# -# Coverage runs separately from the nightly, which dispatches it and passes its -# own run id as `baseline_run_id` (RFC option C). A separate run id means a -# failure here doesn't touch the nightly's status, nor its unstable tests hold -# coverage up. - -name: Multi-Arch CI Coverage - -on: - workflow_dispatch: - inputs: - baseline_run_id: - type: string - description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." - default: "" - baseline_release_type: - type: string - description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." - default: "nightly" - baseline_repository: - type: string - description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." - default: "" - projects_to_test: - type: string - description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." - default: "" - amdgpu_families: - type: string - description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." - default: "gfx94X-dcgpu" - coverage_config_source: - type: string - description: "Repository and ref holding the per-project coverage metadata." - default: "ROCm/rocm-libraries@main" - test_type: - type: string - description: "Test type to run against the instrumented build." - default: "standard" - quartz_tracking_id: - type: string - description: >- - Leave empty. Internal Quartz tracking id linking this run to a release - lineage once the nightly dispatches coverage automatically. - default: "" - - # TEMPORARY, paired with coverage_nightly_pr_check.yml. workflow_dispatch only - # appears once the file is on the default branch, so this can't run from a - # branch or show in PR checks; a caller can, so the check workflow runs this - # branch's copy. Actions doesn't expand YAML anchors, so inputs are repeated. - # Remove this trigger with the check workflow once the run is green. - workflow_call: - inputs: - baseline_run_id: - type: string - description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." - default: "" - baseline_release_type: - type: string - description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." - default: "nightly" - baseline_repository: - type: string - description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." - default: "" - projects_to_test: - type: string - description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." - default: "" - amdgpu_families: - type: string - description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." - default: "gfx94X-dcgpu" - coverage_config_source: - type: string - description: "Repository and ref holding the per-project coverage metadata." - default: "ROCm/rocm-libraries@main" - test_type: - type: string - description: "Test type to run against the instrumented build." - default: "standard" - quartz_tracking_id: - type: string - description: >- - Leave empty. Internal Quartz tracking id linking this run to a release - lineage once the nightly dispatches coverage automatically. - default: "" - -permissions: - contents: read - -# Surface the baseline in the run title so people don't dig through logs for it. -run-name: >- - Multi-Arch CI Coverage - | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} - | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} - -concurrency: - group: ${{ github.workflow }}-${{ github.run_id }} - cancel-in-progress: false - -jobs: - setup_coverage_matrix: - name: Setup Coverage Matrix - runs-on: ubuntu-24.04 - outputs: - coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} - dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} - families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} - coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} - needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} - rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} - steps: - - name: Checking out repository - id: checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - - - name: Setting up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.12 - - - name: Generate coverage matrix - id: matrix - env: - PROJECTS_TO_TEST: ${{ inputs.projects_to_test || '' }} - AMDGPU_FAMILIES: ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} - COVERAGE_CONFIG_SOURCE: ${{ inputs.coverage_config_source || 'ROCm/rocm-libraries@main' }} - run: python ./build_tools/github_actions/configure_coverage_ci.py - - - name: Compute package version - id: rocm_package_version - run: | - python ./build_tools/compute_rocm_package_version.py \ - --release-type=ci \ - --override-git-sha=${{ steps.checkout.outputs.commit }} - - # Build + fanout is per-platform. The Linux layer builds the instrumented - # stack once, then fans out to the per-project report workflow. A Windows - # layer would be a sibling job reusing the same matrix and report workflow. - coverage_linux: - name: Coverage - Linux - needs: setup_coverage_matrix - # Empty selection has nothing to build, and fromJSON('') hard-errors, so - # check the matrix explicitly. - if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} - uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml - secrets: inherit - with: - coverage_matrix: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix }} - dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} - families_matrix_json: ${{ needs.setup_coverage_matrix.outputs.families_matrix_json }} - coverage_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} - needs_math_libs: ${{ needs.setup_coverage_matrix.outputs.needs_math_libs }} - rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} - baseline_run_id: ${{ inputs.baseline_run_id }} - baseline_release_type: ${{ inputs.baseline_release_type || 'nightly' }} - baseline_repository: ${{ inputs.baseline_repository }} - test_type: ${{ inputs.test_type || 'standard' }} - release_type: ci - quartz_tracking_id: ${{ inputs.quartz_tracking_id }} - permissions: - contents: read - id-token: write - - ci_summary: - name: Coverage Summary - if: always() - needs: - - setup_coverage_matrix - - coverage_linux - runs-on: ubuntu-24.04 - steps: - - name: Checking out repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 15 - - name: Evaluate workflow results - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python build_tools/github_actions/workflow_summary.py \ - --needs-json '${{ toJSON(needs) }}' diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 5c1b3973c9e..3211b1c2151 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -3,86 +3,163 @@ # Multi-Arch CI Coverage - Linux # -# Per-platform build + fanout layer. Builds one instrumented Linux stack for -# every coverage-enabled project, then fans out to -# multi_arch_ci_coverage_report.yml once per project. One large build instead -# of one per project keeps cost flat as projects are onboarded. +# Self-contained Linux coverage entry point. Computes the coverage matrix, +# builds one instrumented Linux stack for every coverage-enabled project, then +# fans out to multi_arch_ci_coverage_report.yml once per project. # -# The stack is assembled by calling the shared per-stage build workflow for each -# needed stage, rather than running multi_arch_build_portable_linux.yml and -# filtering stages out. +# Mirrors the nightly pattern: the nightly dispatches multi_arch_ci_linux.yml +# directly; trigger_coverage_linux in the nightly dispatches this workflow +# directly. A future Windows mirror copies this file, swapping the build +# reusable and runners; multi_arch_ci_coverage_report.yml is reused unchanged. # -# The matrix and build config are computed once by the parent -# (multi_arch_ci_coverage.yml) and passed in, so a future Windows layer -# duplicates none of it -- it only mirrors the build + fanout with Windows runners. +# RFC0014 phase 1: every enabled component on one architecture (gfx94X-dcgpu). +# Change-based selection and multi-arch come later. name: Multi-Arch CI Coverage - Linux on: - workflow_call: + workflow_dispatch: inputs: - coverage_matrix: + baseline_run_id: type: string - description: "JSON array of per-project coverage matrix entries produced by setup_coverage_matrix." - dist_amdgpu_families: + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: type: string - description: "GPU families the instrumented stack is distributed for." - families_matrix_json: + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" + baseline_repository: type: string - description: "JSON array of per-family build matrix entries for the math-libs stage." - coverage_cmake_options: + description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." + default: "" + projects_to_test: type: string - description: "Extra CMake options enabling instrumentation for the selected projects." - needs_math_libs: + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." + default: "" + amdgpu_families: type: string - description: "'true' when the selection requires the math-libs stage to be built." - rocm_package_version: + description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." + default: "gfx94X-dcgpu" + coverage_config_source: type: string - description: "Computed ROCm package version stamped onto the instrumented build." - baseline_run_id: + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" + test_type: + type: string + description: "Test type to run against the instrumented build." + default: "standard" + quartz_tracking_id: type: string + description: >- + Leave empty. Internal Quartz tracking id linking this run to a release + lineage once the nightly dispatches coverage automatically. default: "" + + # TEMPORARY, paired with coverage_nightly_pr_check.yml. workflow_dispatch only + # appears once the file is on the default branch, so it can't run from a + # branch or show in PR checks; a caller can, so the check workflow runs this + # branch's copy. Actions doesn't expand YAML anchors, so inputs are repeated. + # Remove this trigger with the check workflow once the run is green. + workflow_call: + inputs: + baseline_run_id: + type: string description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" baseline_release_type: type: string - default: "nightly" description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" baseline_repository: type: string - default: "" description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." + default: "" + projects_to_test: + type: string + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." + default: "" + amdgpu_families: + type: string + description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." + default: "gfx94X-dcgpu" + coverage_config_source: + type: string + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" test_type: type: string - default: "standard" description: "Test type to run against the instrumented build." - release_type: - type: string - default: "ci" - description: "Release channel this coverage run's own instrumented artifacts publish to." + default: "standard" quartz_tracking_id: type: string - default: "" description: >- - Leave empty. Internal Quartz tracking id, propagated from the caller to - link this run to a release lineage once the nightly dispatches coverage. + Leave empty. Internal Quartz tracking id linking this run to a release + lineage once the nightly dispatches coverage automatically. + default: "" permissions: contents: read +run-name: >- + Multi-Arch CI Coverage - Linux + | baseline: ${{ inputs.baseline_run_id || 'none (self-contained)' }} + | ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.run_id }} + cancel-in-progress: false + jobs: + setup_coverage_matrix: + name: Setup Coverage Matrix + runs-on: ubuntu-24.04 + outputs: + coverage_matrix: ${{ steps.matrix.outputs.coverage_matrix }} + dist_amdgpu_families: ${{ steps.matrix.outputs.dist_amdgpu_families }} + families_matrix_json: ${{ steps.matrix.outputs.families_matrix_json }} + coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} + needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} + rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} + steps: + - name: Checking out repository + id: checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + + - name: Generate coverage matrix + id: matrix + env: + PROJECTS_TO_TEST: ${{ inputs.projects_to_test || '' }} + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families || 'gfx94X-dcgpu' }} + COVERAGE_CONFIG_SOURCE: ${{ inputs.coverage_config_source || 'ROCm/rocm-libraries@main' }} + run: python ./build_tools/github_actions/configure_coverage_ci.py + + - name: Compute package version + id: rocm_package_version + run: | + python ./build_tools/compute_rocm_package_version.py \ + --release-type=ci \ + --override-git-sha=${{ steps.checkout.outputs.commit }} + # Built first so later stages have a compiler-runtime to link against and # fetch as an inbound artifact. build_instrumented_compiler_runtime: name: Build Instrumented Compiler Runtime - if: ${{ inputs.coverage_matrix != '[]' }} + needs: setup_coverage_matrix + if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: stage_name: compiler-runtime stage_display_name: "Stage - Coverage Compiler Runtime" timeout_minutes: 480 # 8 hours (compiler is big) - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - rocm_package_version: ${{ inputs.rocm_package_version }} + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: @@ -95,16 +172,16 @@ jobs: # group, else individual project flags; unselected projects build normally. build_instrumented_math_libs: name: Build Instrumented Math Libs - needs: build_instrumented_compiler_runtime + needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] if: >- ${{ !failure() && !cancelled() && - inputs.needs_math_libs == 'true' + needs.setup_coverage_matrix.outputs.needs_math_libs == 'true' }} strategy: fail-fast: false matrix: - family_info: ${{ fromJSON(inputs.families_matrix_json) }} + family_info: ${{ fromJSON(needs.setup_coverage_matrix.outputs.families_matrix_json) }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml secrets: inherit with: @@ -112,9 +189,9 @@ jobs: stage_display_name: "Stage - Coverage Math Libs (${{ matrix.family_info.amdgpu_family }})" timeout_minutes: 480 # 8 hours; instrumented builds are slower than normal amdgpu_family: ${{ matrix.family_info.amdgpu_family }} - dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }} - extra_cmake_options: ${{ inputs.coverage_cmake_options }} - rocm_package_version: ${{ inputs.rocm_package_version }} + dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + extra_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} + rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} permissions: @@ -130,6 +207,7 @@ jobs: # math-libs build still tests against binaries that stage produced. needs: [ + setup_coverage_matrix, build_instrumented_compiler_runtime, build_instrumented_math_libs, ] @@ -137,12 +215,12 @@ jobs: if: >- ${{ !failure() && !cancelled() && - inputs.coverage_matrix != '[]' + needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} strategy: fail-fast: false matrix: - coverage: ${{ fromJSON(inputs.coverage_matrix) }} + coverage: ${{ fromJSON(needs.setup_coverage_matrix.outputs.coverage_matrix) }} uses: ./.github/workflows/multi_arch_ci_coverage_report.yml secrets: inherit with: @@ -156,7 +234,27 @@ jobs: baseline_run_id: ${{ inputs.baseline_run_id }} baseline_repository: ${{ inputs.baseline_repository }} test_type: ${{ inputs.test_type || 'standard' }} - release_type: ${{ inputs.release_type || 'ci' }} + release_type: ci permissions: contents: read id-token: write + + ci_summary: + name: Coverage Summary + if: always() + needs: + - setup_coverage_matrix + - build_instrumented_compiler_runtime + - build_instrumented_math_libs + - coverage_report + runs-on: ubuntu-24.04 + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + - name: Evaluate workflow results + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python build_tools/github_actions/workflow_summary.py \ + --needs-json '${{ toJSON(needs) }}' diff --git a/docs/development/coverage_nightly_trigger_snippet.md b/docs/development/coverage_nightly_trigger_snippet.md index 8ac4cd51259..d54d22632ca 100644 --- a/docs/development/coverage_nightly_trigger_snippet.md +++ b/docs/development/coverage_nightly_trigger_snippet.md @@ -1,4 +1,4 @@ -# Reference: `trigger_coverage` job for the rocm-libraries nightly +# Reference: `trigger_coverage_linux` / `trigger_coverage_windows` jobs for the rocm-libraries nightly This snippet belongs in **`ROCm/rocm-libraries`** `.github/workflows/therock-multi-arch-ci-nightly.yml`, not in this repo. It is @@ -8,51 +8,61 @@ wiring the real driver. The nightly's regular build is **non-instrumented**, so it can only ever be the baseline that coverage is measured against, never the thing coverage runs on. -The coverage orchestrator (`ROCm/TheRock` -`.github/workflows/multi_arch_ci_coverage.yml`) therefore has to run as a +Each platform's coverage orchestrator (`multi_arch_ci_coverage_linux.yml` / +`multi_arch_ci_coverage_windows.yml` in `ROCm/TheRock`) therefore runs as a **separate** GitHub Actions run with its own `run_id`; a nested `uses:` call would share the nightly's `run_id` and collide the generic and instrumented -trees the installer needs to keep apart. The nightly hands over its own -`run_id` as `baseline_run_id`. +trees the installer needs to keep apart. + +Linux and Windows build at different speeds. Dispatching per-platform as soon as +each build finishes mirrors the nightly's own pattern (which dispatches +`multi_arch_ci_linux.yml` and `multi_arch_ci_windows.yml` independently) and +avoids forcing the faster platform to wait for the slower one. Cross-repo dispatch is authenticated with a **GitHub App token** (`actions/create-github-app-token`), modelled on `ROCm/rocm-systems` `.github/workflows/rocm_ci_caller.yml`. No classic PAT is introduced. > **One-time org setup (not code):** a GitHub App reachable from rocm-libraries -> must be installed on **ROCm/TheRock** with **`actions: write`**, so the token -> minted below is allowed to call `createWorkflowDispatch` on TheRock. +> must be installed on **ROCm/TheRock** with **`actions: write`**, so the tokens +> minted below are allowed to call `createWorkflowDispatch` on TheRock. -## `workflow_dispatch` input (add to the nightly's `on:` block) +## `workflow_dispatch` inputs (add to the nightly's `on:` block) ```yaml on: workflow_dispatch: inputs: # ... existing inputs ... - enable_coverage: + enable_coverage_linux: description: >- - Dispatch the ROCm/TheRock code-coverage orchestrator after the build - and test jobs succeed. Nightly-only: a coverage run is a multi-hour - instrumented build, unfit for presubmit. + Dispatch the ROCm/TheRock Linux code-coverage orchestrator after the + Linux build and test jobs succeed. Nightly-only: a coverage run is a + multi-hour instrumented build, unfit for presubmit. + type: boolean + default: false + enable_coverage_windows: + description: >- + Dispatch the ROCm/TheRock Windows code-coverage orchestrator after the + Windows build and test jobs succeed. type: boolean default: false ``` -## `trigger_coverage` job +## `trigger_coverage_linux` job ```yaml jobs: # ... existing setup, linux_build_and_test, etc. ... - trigger_coverage: - name: Trigger TheRock Coverage + trigger_coverage_linux: + name: Trigger TheRock Linux Coverage needs: [setup, linux_build_and_test] # Gate on the switch, on there being a build to baseline against, and on the - # test job having succeeded -- coverage measures a run that actually built. + # Linux test job having succeeded. if: >- ${{ - inputs.enable_coverage && + inputs.enable_coverage_linux && needs.setup.outputs.enable_build_jobs == 'true' && needs.linux_build_and_test.result == 'success' }} @@ -67,7 +77,7 @@ jobs: owner: "ROCm" repositories: "TheRock" - - name: Dispatch TheRock coverage orchestrator + - name: Dispatch TheRock Linux coverage orchestrator uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v8.0.0 with: github-token: ${{ steps.generate-token.outputs.token }} @@ -75,7 +85,7 @@ jobs: await github.rest.actions.createWorkflowDispatch({ owner: 'ROCm', repo: 'TheRock', - workflow_id: 'multi_arch_ci_coverage.yml', + workflow_id: 'multi_arch_ci_coverage_linux.yml', // Pin to a known-good TheRock ref; bump when the coverage // workflows change in a way this nightly depends on. ref: 'main', @@ -93,12 +103,57 @@ jobs: }); ``` +## `trigger_coverage_windows` job (add when Windows coverage is ready) + +```yaml + trigger_coverage_windows: + name: Trigger TheRock Windows Coverage + needs: [setup, windows_build_and_test] + if: >- + ${{ + inputs.enable_coverage_windows && + needs.setup.outputs.enable_build_jobs == 'true' && + needs.windows_build_and_test.result == 'success' + }} + runs-on: ubuntu-24.04 + steps: + - name: Generate GitHub App token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.THEROCK_COVERAGE_APP_ID }} + private-key: ${{ secrets.THEROCK_COVERAGE_APP_PRIVATE_KEY }} + owner: "ROCm" + repositories: "TheRock" + + - name: Dispatch TheRock Windows coverage orchestrator + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v8.0.0 + with: + github-token: ${{ steps.generate-token.outputs.token }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'ROCm', + repo: 'TheRock', + workflow_id: 'multi_arch_ci_coverage_windows.yml', + ref: 'main', + inputs: { + baseline_run_id: String(context.runId), + baseline_repository: 'ROCm/rocm-libraries', + baseline_release_type: 'nightly', + coverage_config_source: '${{ github.repository }}@${{ github.sha }}', + projects_to_test: '...', + amdgpu_families: '...', + }, + }); +``` + Notes: - `baseline_run_id` is stringified because `createWorkflowDispatch` inputs must be strings; `context.runId` is a number. +- Both jobs pass the **same** `baseline_run_id` (the nightly is one run) but + dispatch to different workflow files and different `amdgpu_families`. - `ref` pins which TheRock copy of the coverage workflows runs. A `workflow_dispatch` always executes the workflow file from that ref. - `baseline_release_type` and `projects_to_test` should track whatever the - nightly actually built and published; the values above match the phase 1 - default selection. + nightly actually built and published. From acefa4912f9a94823e91b50dbd24638be25eb78b Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Mon, 21 Sep 2026 15:48:01 -0700 Subject: [PATCH 30/70] docs(coverage): trim docs and drop nightly walkthrough --- docs/development/README.md | 1 - docs/development/code_coverage.md | 580 ++++++------------ .../coverage_nightly_trigger_snippet.md | 49 +- .../nightly_coverage_implementation.md | 363 ----------- 4 files changed, 202 insertions(+), 791 deletions(-) delete mode 100644 docs/development/nightly_coverage_implementation.md diff --git a/docs/development/README.md b/docs/development/README.md index 5c60adbb6d7..4e75f94ab38 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -11,7 +11,6 @@ - [Artifacts](artifacts.md) - [Build System](build_system.md) - [Code Coverage](code_coverage.md) -- [Nightly Coverage Implementation](nightly_coverage_implementation.md) - a plain-language walkthrough of how a coverage report is produced - [Dependencies](dependencies.md) - [Development Guide](development_guide.md) - [Installing Artifacts](installing_artifacts.md) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 1a88c915a49..767fc435a09 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -1,145 +1,93 @@ # Code Coverage -TheRock builds ROCm libraries with LLVM's source based code coverage -instrumentation and reports the results to +TheRock builds ROCm libraries with LLVM source-based coverage and reports to [Codecov](https://app.codecov.io/gh/ROCm/TheRock). Coverage is opt-in per -project: instrumentation slows a library down substantially, and the report is -only worth producing for projects whose test suites are thorough enough to give -a meaningful signal. +project — instrumentation slows a library substantially, and a report is only +worth producing when the test suite gives a meaningful signal. -This page is the reference. For a plain-language walkthrough of a single run, -start with -[Nightly Coverage Implementation](nightly_coverage_implementation.md). +## How it works -## Enabling coverage for a local build +- **Instrumented** — compiled with `-fprofile-instr-generate`; every line bumps + a counter. Slower and bigger; never shipped. +- **profraw** — the counter file an instrumented process writes on exit. +- **lcov** — the final report: which lines ran and which did not. -Pass `-D_ENABLE_COVERAGE=ON` at the top level configure. The project -name must be upper case, matching the usual CMake convention for definitions: +Pipeline: build instrumented → run tests → collect profraw → convert to lcov. -```bash -cmake -B build -GNinja . \ - -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ - -DHIPRAND_ENABLE_COVERAGE=ON -``` +A project's tests load its dependencies too. If those were instrumented, their +counters would pollute the report. So exactly one library is instrumented; the +rest come from the regular nightly build (the **baseline**): -`_ENABLE_COVERAGE` is TheRock's own knob, and it is not what the -subproject sees. Upstream option names are not standardised — hipRAND takes -`BUILD_CODE_COVERAGE`, rocRAND takes `CODE_COVERAGE`, RCCL takes -`ENABLE_CODE_COVERAGE`, hipBLASLt takes `HIPBLASLT_ENABLE_COVERAGE` — so -`therock_subproject.cmake` translates TheRock's flag into whichever name is -registered for that project in `COVERAGE_PROJECTS`. Translating per subproject -is what keeps the generic spellings from instrumenting everything in the build -that happens to understand them. +| | Source | What it provides | +|---|---|---| +| **Baseline** | Regular nightly run | Whole ROCm stack, non-instrumented | +| **Coverage** | This run | Only the one instrumented project | -Passing the flag for a project with no registered option is a configure error -rather than a silent no-op, since the failure it replaces — an uninstrumented -build that tests cleanly and reports nothing — surfaces hours later. +The test runner installs the baseline, then replaces only the selected +project's libraries with instrumented ones. See [Coverage CI](#coverage-ci). -### Enabling a whole group - -Three aggregate options turn on coverage for a whole component group instead of -naming each project. All default `OFF`: - -| Option | Instruments | -| ------------------------------------- | ------------------------------------------------------------------- | -| `THEROCK_COVERAGE_ROCM_LIBRARIES_ALL` | all coverage-enabled rocm-libraries projects (math-libs, ml-libs …) | -| `THEROCK_COVERAGE_ROCM_SYSTEMS_ALL` | all coverage-enabled rocm-systems projects | -| `THEROCK_COVERAGE_ALL` | both of the above | +## Enabling coverage for a local build ```bash cmake -B build -GNinja . \ -DTHEROCK_AMDGPU_FAMILIES=gfx94X-dcgpu \ - -DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON + -DHIPRAND_ENABLE_COVERAGE=ON ``` -Each option expands to the individual `_ENABLE_COVERAGE` flags above, -so the same caveats apply. An explicit `-D_ENABLE_COVERAGE=OFF` on the -command line wins over the group flag. Group membership is generated from the -coverage registry (`COVERAGE_PROJECTS`), so a group only ever contains -onboarded projects. +`_ENABLE_COVERAGE` is TheRock's own flag; upstream option names are +not standardised (hipRAND: `BUILD_CODE_COVERAGE`, rocRAND: `CODE_COVERAGE`, +RCCL: `ENABLE_CODE_COVERAGE`), so `therock_subproject.cmake` translates it to +whichever name `COVERAGE_PROJECTS` registers for that project. Passing the flag +for an unregistered project is a configure error, not a silent no-op. -> [!NOTE] -> These options default `OFF` and have no effect on a normal, non-coverage -> build — no `_ENABLE_COVERAGE` flag is set. The membership list is -> regenerated from `COVERAGE_PROJECTS` on every configure (a small scripted -> step; it also means editing `configure_coverage_ci.py` triggers a -> reconfigure), but it only changes the build when one of the options is `ON`. +### Enabling a whole group -### Host code only +| Option | Instruments | +| --- | --- | +| `THEROCK_COVERAGE_ROCM_LIBRARIES_ALL` | all coverage-enabled rocm-libraries projects | +| `THEROCK_COVERAGE_ROCM_SYSTEMS_ALL` | all coverage-enabled rocm-systems projects | +| `THEROCK_COVERAGE_ALL` | both of the above | -What gets measured is host code, and the instrumentation is deliberately kept -off the GPU. The upstream coverage options pass `-fprofile-instr-generate` and -`-fcoverage-mapping` unqualified, and the HIP driver forwards both to the -device compilation, so kernels would be instrumented too. +Each expands to individual `_ENABLE_COVERAGE` flags. An explicit +`-D_ENABLE_COVERAGE=OFF` wins over a group flag. Membership is +generated from `COVERAGE_PROJECTS`; these options default `OFF` and have no +effect on a normal build. -That does not survive contact with the runtime. An instrumented device image -carries a per-translation-unit `__llvm_profile_sections_` symbol that -`libclang_rt.profile_rocm.a` looks up through `hipModuleGetGlobal` when the -process exits, plus a `__profd_` for every kernel. A symbol the runtime cannot -resolve aborts the process *after* the tests have already passed — rocSPARSE -hit this on an inlined rocPRIM kernel — and where the readback does succeed, -the runtime writes an extra profile named after the device -(`gfx942:sramecc+:xnack-.0.`), which cannot be uploaded as a CI -artifact because of the colons. +### Host code only -`therock_subproject.cmake` therefore appends the negation, scoped to the device -compilation, for any subproject it enables coverage for: +Upstream coverage flags are unqualified and the HIP driver forwards them to +device compilation. Two failures result: the runtime aborts looking up missing +profiling symbols in the device image, and when readback succeeds it writes a +profile named after the device target (colons included, breaking CI artifact +upload). `therock_subproject.cmake` appends the negation scoped to device +compilation: ``` -Xarch_device -fno-profile-instr-generate -Xarch_device -fno-coverage-mapping ``` -Position is what makes this work. The later of a `-f`/`-fno-` pair wins, and -the project's own flags arrive through `COMPILE_OPTIONS`, after -`CMAKE__FLAGS` — so the negation is appended to -`CMAKE__COMPILE_OBJECT` rather than to the flags variables. That rule -does not exist yet in `CMAKE_PROJECT_TOP_LEVEL_INCLUDES`, which runs before any -language is enabled, so it is applied from a generated file passed as -`CMAKE_PROJECT_INCLUDE`. Measuring GPU code would mean undoing this and solving -the two problems above. +The later of a `-f`/`-fno-` pair wins; the project's own flags arrive after +`CMAKE__FLAGS`, so the negation is appended to +`CMAKE__COMPILE_OBJECT` via a generated file passed as +`CMAKE_PROJECT_INCLUDE`. ### The profile runtime's link dependencies -`libclang_rt.profile_rocm.a` calls `dlsym`, `dladdr` and `pthread_once`, and -the driver does not put `-ldl` or `-lpthread` on the link line for it. On the -manylinux base those symbols are still in `libdl` and `libpthread` rather than -in `libc`, so an instrumented shared library would be left with undefined -references, and any executable linking against it would fail -`--no-allow-shlib-undefined`. - -`therock_subproject.cmake` adds the two libraries whenever it enables coverage -for a subproject, so this is handled. It is worth knowing about if you -instrument a project outside the normal flow. +`libclang_rt.profile_rocm.a` needs `-ldl` and `-lpthread`, which the driver +does not supply. On the manylinux base these are not in libc, so an instrumented +shared library gets undefined references and any executable linking it fails +`--no-allow-shlib-undefined`. `therock_subproject.cmake` adds both libraries +automatically. ## Producing a report locally -Instrumented binaries write one `.profraw` file per process, named by the -`LLVM_PROFILE_FILE` environment variable. The `%p` (pid) and `%m` (binary -signature) substitutions keep concurrent processes from clobbering each other: +Set `LLVM_PROFILE_FILE` with `%p`/`%m` substitutions to keep per-process files +distinct, run the tests, then call `merge_coverage_report.py`: ```bash export LLVM_PROFILE_FILE="$PWD/coverage-report/profraw/%p-%m.profraw" ctest --test-dir build/math-libs/hiprand -``` - -`merge_coverage_report.py` then merges the profiles and exports lcov: -```bash -python build_tools/github_actions/merge_coverage_report.py \ - --profraw-dir coverage-report/profraw \ - --rocm-dir build/dist/rocm \ - --object-globs "lib/libhiprand.so*" -``` - -The `llvm-profdata` and `llvm-cov` binaries must come from the same compiler -that built the instrumented objects, so the script prefers the copies under -`/lib/llvm/bin`. A version mismatch surfaces as an unhelpfully -generic "malformed instrumentation profile data" error. - -`--summary-output` adds the per-file table as plain text and `--html-output` -the browsable report, entry point `index.html`: - -```bash python build_tools/github_actions/merge_coverage_report.py \ --profraw-dir coverage-report/profraw \ --rocm-dir build/dist/rocm \ @@ -148,346 +96,186 @@ python build_tools/github_actions/merge_coverage_report.py \ --html-output coverage-report/html ``` -Only the HTML reads the source files; lcov and the text table are produced -from the coverage mappings alone and are complete wherever they run. Source -paths are recorded as they were at build time, so if the tree has moved -since, `--path-equivalence ,` tells `llvm-cov` how to get from one -to the other. Files generated into the build tree are in no source checkout -and are reported as uncovered in the HTML; the text table still counts them. +`llvm-profdata` and `llvm-cov` must match the compiler that built the objects; +the script prefers the copies under `/lib/llvm/bin`. Only the HTML +reads source files; lcov and the text table work from coverage mappings alone. +Use `--path-equivalence ,` if the source tree has moved since the +build. ## Coverage CI -A coverage run builds the stages its selection needs, instruments only the -selected projects, and reports on each of them separately from that one build. - ```mermaid graph TD - dispatch[Manual dispatch with projects_to_test] --> matrix[setup_coverage_matrix] + dispatch[Dispatch with projects_to_test] --> matrix[setup_coverage_matrix] matrix --> compilerRuntime[Build instrumented compiler-runtime] compilerRuntime --> mathLibs[Build instrumented math-libs] mathLibs --> report[Per project: configure, test, report] report --> codecov[Codecov] ``` -Every edge is a job dependency inside the one coverage run. -`multi_arch_ci_coverage.yml` is the thin top-level orchestrator; it delegates to -`multi_arch_ci_coverage_linux.yml`, the per-platform build + fan-out layer that -builds the instrumented stacks and calls `multi_arch_ci_coverage_report.yml` for -the per-project test and aggregation sequence on one GPU family. All take their -project list from -`build_tools/github_actions/configure_coverage_ci.py`, the registry of -coverage-enabled projects and everything the pipeline needs to know about each -one: the CMake target, the build stage it lives in, its test component, the -object globs handed to `llvm-cov`, and its Codecov flag. +`multi_arch_ci_coverage_linux.yml` is the entry point: it computes the matrix, +builds the instrumented stack, then fans out to +`multi_arch_ci_coverage_report.yml` per project per GPU family. +`configure_coverage_ci.py` is the registry — CMake target, build stage, test +component, object globs, and Codecov flag for every onboarded project. ### Dispatching a run -`multi_arch_ci_coverage.yml` is started by hand, from the Actions tab or the -`gh` CLI, and can now also be dispatched by the consumer nightly (RFC option C): -`ROCm/rocm-libraries`'s `therock-multi-arch-ci-nightly.yml` triggers it as a -separate run via a GitHub App token, passing its own run id as `baseline_run_id`. -A coverage run has no cron trigger of its own, and it neither reads the nightly's -artifacts nor depends on it having succeeded — it only takes the baseline run id. - -Pick a recent nightly run, pass its id as `baseline_run_id`, and leave -`baseline_release_type` at `nightly`. The other input that matters is -`projects_to_test`. The rest can be left at their defaults — the -`gfx94X-dcgpu` family and coverage's own `standard` test type, which is -narrower than the nightly's because instrumented tests already run several -times longer than normal ones. - -Coverage having a run id of its own is what keeps a coverage failure from -colouring the nightly's status, and keeps the nightly's test jobs — which -rarely all pass — from holding coverage up. - -Having the regular nightly dispatch this workflow automatically once its build -finishes, handing over its own run id, is a later change. Nothing else has to -move for it: `baseline_run_id` is already an input, so that change only adds -the dispatching job. Running only what changed, and running more architectures -than gfx942, come later too. +Start from the Actions tab or `gh` CLI, or let the rocm-libraries nightly +dispatch it via a GitHub App token. There is no cron trigger. + +Pass a recent nightly run id as `baseline_run_id` (keep `baseline_release_type` +at `nightly`) and set `projects_to_test`; other inputs use sensible defaults. +Running coverage separately keeps failures off the nightly's status. Automatic +nightly dispatch is a follow-up; `baseline_run_id` is already the right input. ### The instrumented build -`setup_coverage_matrix` runs `configure_coverage_ci.py`, which reads -`PROJECTS_TO_TEST`, `AMDGPU_FAMILIES` and `COVERAGE_CONFIG_SOURCE` and emits -the per-project job matrix, the family list, `coverage_cmake_options` and -`needs_math_libs`. It also rejects any selection naming a project whose stage -this workflow has no build job for, so a bad request stops in seconds rather -than hours. - -`build_instrumented_compiler_runtime` runs first and unconditionally. Every -other stage takes its inbound artifacts from it — all 19 of `math-libs`' -inbound artifacts are produced there — so no selection can skip it. It also -receives the coverage flags, since several rocm-systems projects -(`rocprofiler-sdk`, `aqlprofile`, `amdsmi`, `rocprofiler-compute`) are built in -that stage. - -`build_instrumented_math_libs` fans out over the GPU families, and is skipped -when `needs_math_libs` is false. It is the only other stage built: 17 of the 19 -measurable projects live there, and the remaining two are blocked upstream, so -phase 1 carries no job for any further stage. - -Both stages get `coverage_cmake_options` appended to their configure line. -`configure_coverage_ci.py` collapses a selection covering a whole group into -that group's option, so a full run reads `-DTHEROCK_COVERAGE_ALL=ON` while a -narrow one reads `-DHIPRAND_ENABLE_COVERAGE=ON`. `CMakeLists.txt` expands the -group options into individual `_ENABLE_COVERAGE` flags, and -`therock_subproject.cmake` translates each of those into the project's own -option name. Everything not selected builds normally. - -All the build jobs call `multi_arch_build_portable_linux_artifacts.yml`, the -same per-stage build workflow regular CI uses, rather than going through the -full `multi_arch_build_portable_linux.yml` pipeline. Naming the stages coverage -needs keeps the shared pipeline free of coverage-specific stage filtering, and -`extra_cmake_options` on that workflow is the only build-side hook coverage -adds. - -The artifacts publish under `release_type: ci`, under this run's id. +`setup_coverage_matrix` runs `configure_coverage_ci.py`, emitting the matrix, +`coverage_cmake_options`, and `needs_math_libs`. Selections naming a project +whose stage has no build job fail immediately. + +`build_instrumented_compiler_runtime` always runs first — every other stage +pulls its inbound artifacts from it. `build_instrumented_math_libs` fans out +over GPU families and is skipped when `needs_math_libs` is false; 17 of the 19 +measurable projects live there. + +`coverage_cmake_options` is appended to each stage's configure line. A +full-group selection collapses to `-DTHEROCK_COVERAGE_ALL=ON`; a narrow one +names each project. `CMakeLists.txt` expands group options to +`_ENABLE_COVERAGE` flags; `therock_subproject.cmake` translates each +to the project's upstream name. Everything not selected builds normally. + +Build jobs use `multi_arch_build_portable_linux_artifacts.yml` directly; +`extra_cmake_options` is the only build-side hook coverage adds. Artifacts +publish under `release_type: ci`. ### Test execution -`coverage_report` fans out over the matrix, one call to -`multi_arch_ci_coverage_report.yml` per project and GPU family. -`configure_test_matrix` runs `fetch_test_configurations.py` — shared with -regular CI — narrowed to the one project this report covers, and outputs the -shard list. `test_coverage` then runs one `test_code_coverage_component.yml` job -per shard. Three things happen in order: - -1. **Install the baseline, then swap in the instrumented project.** - `install_rocm_code_coverage_build.py` installs the generic non-instrumented - tree from `--run-id` (the baseline nightly run) and replaces only the - project-under-test's libraries from `--code-coverage-run-id` (this coverage - run's own id), deriving the artifact name and folder from the component - `job_name`, so exactly one library in the install is instrumented. -1. **Point the runtime at a profile directory.** `LLVM_PROFILE_FILE` is set to - a per-shard path using the `%p` and `%m` substitutions, so a shard that - forks or loads several instrumented libraries does not overwrite its own - profiles. -1. **Run the tests and upload.** The normal test script runs, then the profraw - files upload as a workflow artifact under `always()` — a failing shard still +Per shard, `test_code_coverage_component.yml` does three things: + +1. **Install baseline + swap project.** `install_rocm_code_coverage_build.py` + installs from `--run-id` (the nightly) and replaces only the + project-under-test from `--code-coverage-run-id`. +1. **Set `LLVM_PROFILE_FILE`** to a per-shard path (`%p`/`%m`) so concurrent + processes don't overwrite each other. +1. **Run tests and upload profraw** under `always()` — a failing shard still exercised code. -`coverage_enabled` also puts a floor under the per-component test timeout. -Instrumented libraries are built `-O0 -g` and count every branch, so a suite -takes much longer than its usual budget assumes: rocRAND tests that normally -finish in milliseconds took 25 to 60 seconds each and ran out of their 15 -minutes. It is a floor rather than a multiplier because the budgets are not on -a common scale — rocBLAS asks for 288 minutes, past the job's own cap, while -rocRAND asks for 15 — so scaling would stretch the generous ones and still -leave the tight ones tight. +`coverage_enabled` raises the test timeout to a 60-minute floor. Instrumented +libraries are `-O0 -g`; rocRAND tests that finish in milliseconds normally took +25–60 seconds each. A floor rather than a multiplier avoids over-extending +budgets that are already generous (rocBLAS: 288 min). #### Why there are two run ids -The nightly instruments the whole stack in one build, because building each -project separately would mean rebuilding its dependencies each time. Reporting, -though, has to stay per project: a coverage report for hipRAND should not shift -because rocRAND changed, and an instrumented rocRAND under an instrumented -hipRAND also writes its own profiles into the same run. Hence the baseline -install and the overlay described above. - -The baseline is read from the channel that published it -(`baseline_release_type`, normally `nightly`) rather than from the coverage -run's own `ci` channel, since artifacts are bucketed per channel. - -The overlay is per project rather than per artifact because TheRock's artifacts -are grouped: `rand` carries both rocRAND and hipRAND. Each project's files sit -under the subproject stage directory they were built in -(`math-libs/hipRAND/stage`), which is what `artifact_relpaths` in the registry -names. Nothing is renamed along the way — the two runs write to different S3 -directories, so the instrumented `rand_lib_gfx942.tar.xz` and the regular one -never collide. - -If the overlay finds none of the project's files, the job fails rather than -testing the baseline build and reporting coverage for binaries that were never -instrumented. +The whole stack is built instrumented once to avoid rebuilding per-project +dependencies. But reporting must be per project: an instrumented rocRAND writes +its own profiles alongside hipRAND's, shifting hipRAND's numbers on unrelated +changes. The baseline-install-then-overlay isolates exactly one project. -#### What scopes a report to one project +The baseline is read from `baseline_release_type` (normally `nightly`), not the +coverage run's `ci` channel — artifacts are bucketed per channel. The overlay +targets `math-libs/hipRAND/stage` rather than the whole `rand` artifact (which +also contains rocRAND). The two runs write to separate S3 paths; no files +collide. An empty overlay fails the job rather than reporting coverage for +uninstrumented binaries. -Two mechanisms, covering different leaks. +#### What scopes a report to one project -`object_globs` scopes the export. `llvm-cov export` is handed an explicit -`-object` for each of the project's binaries and reports only the functions -found in their coverage mappings, so a sibling's whole functions never appear. +`object_globs` limits what `llvm-cov` reports on. The overlay limits what is +instrumented: inline and template code is `linkonce_odr` and lands in every +binary that uses it — an instrumented sibling writes counters that +`llvm-profdata` merges in, unreachable by any `-object` filter. This matters +most for header-only projects (rocPRIM, hipCUB, rocThrust, rocWMMA). -The overlay scopes what is instrumented at all. Globs alone would not be -enough, because inline and template code defined in headers is emitted -`linkonce_odr` into every binary that uses it. If a sibling library were also -instrumented, its copies of those functions would write counters that -`llvm-profdata` merges into this project's, and no `-object` choice could -separate them again. Installing a non-instrumented baseline and overlaying one -project keeps that from arising — which matters most for the header-only -projects (rocPRIM, hipCUB, rocThrust, rocWMMA), where nearly all the measured -code lives in headers. +### Artifacts -The cost of selecting more projects is build and test time; each instrumented -library is slower to compile and slower to exercise. +| Artifact | Where | Purpose | +| --- | --- | --- | +| Instrumented | CI bucket, this run | Stack under test + LLVM tools | +| Profraw | GitHub Actions artifacts, per shard | Raw profiles for aggregation | -### Artifacts +Profraw names include the project, GPU family, and shard index. -Two kinds are in play: +### Report generation -| Artifact | Where it lives | What it is for | -| ------------ | ----------------------------------- | ---------------------------------------------- | -| Instrumented | CI bucket, this run's id | The stack under test, and the LLVM tools | -| Profraw | GitHub Actions artifacts, per shard | Raw profiles, collected by the aggregation job | +`aggregate_coverage` runs under `if: !cancelled()` so partial reports survive +shard failures. It downloads profraw artifacts, reinstalls the run's artifacts +(LLVM tools must match the compiler that produced the profiles), then runs +`merge_coverage_report.py`. Two conditions are fatal: no profraw files (tests +didn't run or the wrong library was loaded) and no matching objects (`object_globs` +doesn't match the project's install layout). -Profraw artifact names carry the project, GPU family and shard index, so the -aggregation job can glob back exactly its own shards when several projects are -being measured in one run. +The `coverage-report--` artifact: -### Report generation +| File | What it is | +| --- | --- | +| `coverage.info` | lcov for Codecov | +| `coverage_summary.txt` | per-file table with totals | +| `html/index.html` | browsable annotated report | -`aggregate_coverage` runs once per project, under `if: !cancelled()` so a -partial report still gets produced when some shards failed. - -It downloads every matching profraw artifact, then installs the run's artifacts -again — `llvm-cov` needs the binaries carrying the coverage mapping sections, -and the LLVM tools have to come from the same compiler that produced the -profiles. - -`merge_coverage_report.py` then collects the profiles recursively, expands the -project's `object_globs` and deduplicates them by real path (a versioned -symlink family resolves to one file), runs `llvm-profdata merge -sparse` into a -single profdata index, and renders it three ways. Two conditions are fatal -rather than reported as zero coverage: finding no profraw files at all, which -means the tests never ran or the instrumented libraries were not the ones -loaded at runtime; and matching no objects, which means `object_globs` does not -describe what the project actually installs. - -The `coverage-report--` artifact holds all three: - -| File | What it is | -| ---------------------- | ----------------------------------------------------- | -| `coverage.info` | lcov, the format Codecov consumes | -| `coverage_summary.txt` | The per-file table, with the totals on the final line | -| `html/index.html` | The browsable report, source annotated line by line | - -The totals are also written to the job's summary page, so the number is -readable without downloading anything. - -Only the HTML rendering opens source files, and the report job checks out -TheRock rather than the projects it measures, so it first runs -`fetch_sources.py` for the stage that built the project. Source paths are -recorded as the build container saw them, and `--path-equivalence` maps that -prefix onto the workspace. That step is `continue-on-error`: losing it costs -the annotated view and nothing else, since lcov and the text table come from -the coverage mappings. Files generated into the build tree — version headers -and the like — exist in no checkout and show as uncovered in the HTML while -the text table still counts them. - -The Codecov step is skipped rather than failed when no `CODECOV_TOKEN` is -configured, so forks still get the artifact. +Totals are also written to the job summary page. The HTML step runs +`fetch_sources.py` and remaps paths with `--path-equivalence`; it is +`continue-on-error` (losing it costs only the annotated view). Codecov is +skipped, not failed, when `CODECOV_TOKEN` is absent. ## Selecting projects to run -A nightly run measures every onboarded project the workflow can build. -Dispatching by hand narrows that down: the `projects_to_test` input takes a -comma-separated list of project names, and also accepts three case-insensitive -**group aliases** that expand to a whole component group: `rocm_libraries_all`, -`rocm_systems_all`, and `all`. They may be mixed with explicit names, and -selecting a group with no onboarded projects fails the run with a clear error -rather than launching an empty matrix. - -An empty input means every project with a build job, which today is -`rocm_libraries_all`. That works, but it instruments all of them in one build, -which is the expensive way to run this; name what you want measured. - -`rccl` and `rocshmem` are onboarded but live in `comm-libs`, which this -workflow does not build, so they are excluded from the default — and naming -them, or `rocm_systems_all` or `all`, is rejected outright by -`setup_coverage_matrix` rather than failing hours later with nothing to -overlay. The narrowing applies only to the default; an explicit request is -never silently reduced. See [Adding a stage](#adding-a-stage). - -`hipblaslt` and `hiptensor` are excluded from the default and the aliases too, -for a different reason: both are measurable, but their instrumented builds -currently fail to link. Naming one is honoured with a warning rather than -rejected, since rerunning it is how the block gets noticed as fixed. See +`projects_to_test` takes comma-separated project names or group aliases +(`rocm_libraries_all`, `rocm_systems_all`, `all`). Empty selects every project +with a build job (currently `rocm_libraries_all`). + +`rccl` and `rocshmem` live in `comm-libs`, which has no build job; they are +excluded from the default and rejected (not just dropped) when named or reached +via an alias. See [Adding a stage](#adding-a-stage). + +`hipblaslt` and `hiptensor` are excluded from the default and aliases because +their instrumented builds fail. Naming one still works and prints the reason — +retesting is how the block gets noticed as fixed. See [Blocked projects](#blocked-projects). -The aliases are expanded to concrete project names inside -`configure_coverage_ci.py` before the job matrix is built, so nothing -downstream ever sees them: the coverage pipeline hands -`fetch_test_configurations.py` (shared with regular CI) an already-resolved -per-project `test_component`, never an alias. The `projects_to_test` input and -these aliases exist only on the coverage workflow, so non-coverage test -selection is unaffected. +Aliases resolve to concrete names before the matrix is built; nothing downstream +sees them. ## Adding a project Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. -Its `coverage_option` has to name the CMake option the project actually -implements, and that option has to select LLVM source-based instrumentation -(`-fprofile-instr-generate -fcoverage-mapping`). Neither can be assumed. The -names are not standardised — most projects use `BUILD_CODE_COVERAGE`, several -use `CODE_COVERAGE`, a few use `_ENABLE_COVERAGE`, RCCL uses -`ENABLE_CODE_COVERAGE` — and some projects instrument with gcov -(`-fprofile-arcs`) instead, which writes `.gcda` files this pipeline cannot -read. - -A project that cannot be measured yet gets an `unsupported_reason` instead. It -stays in the registry so the gap is recorded, stays out of the group aliases, -and is rejected with that reason if someone names it. - -Confirm a local instrumented build produces a non-empty report before relying -on a new entry. A project whose tests never load the instrumented library -builds and tests cleanly and then reports nothing. - -Set the entry's `source_repo` (`ROCM_LIBRARIES` by default, or `ROCM_SYSTEMS`); -that is what routes the project into the correct group alias and -`THEROCK_COVERAGE_*_ALL` option. No CMake edit is needed — the group membership -lists are generated from `COVERAGE_PROJECTS` at configure time, so both the -local aggregate flags and the CI aliases pick up the new project automatically. - -`object_globs` names the binaries the report is generated from, relative to the -install tree. For most projects that is the shared library -(`lib/librocblas.so*`); header-only projects have none, so they name their -installed test binaries instead. Getting it wrong fails the report job, which -refuses to run when no glob matches rather than publishing zero coverage. +`coverage_option` must name the CMake option the project implements, and it must +select LLVM instrumentation (not gcov, which writes `.gcda` files this pipeline +can't read). Names vary: most use `BUILD_CODE_COVERAGE` or `CODE_COVERAGE`; +RCCL uses `ENABLE_CODE_COVERAGE`. + +Verify a local instrumented build produces a non-empty report before committing +the entry. Set `source_repo` (`ROCM_LIBRARIES` or `ROCM_SYSTEMS`) to route the +project into the correct group alias; membership lists are generated at configure +time, so no CMake edit is needed. `object_globs` must match what the project +actually installs — a miss fails the report job rather than publishing zero. ## Blocked projects -A project that is measurable but whose instrumented build is broken gets a -`blocked_reason` alongside its `coverage_option`. Unlike `unsupported_reason` -this is expected to be temporary, so the project is only dropped from the -default selection and the group aliases — including the group lists -`--emit-cmake` writes, so a group build does not set a flag the Python side -just declined to set. Naming it still works and prints the reason as a warning. - -Blocking matters more than it looks, because a stage is built as a unit: a -single project whose instrumented build fails takes the whole stage with it, -and every other project's report along with it. - -There are two examples today, and both fixes belong upstream in -`ROCm/rocm-libraries`; once either lands, drop that project's `blocked_reason`. - -`hipblaslt` fails at link. Its coverage-only branch of -`clients/CMakeLists.txt` links `hipblaslt-test` against a bare `rocroller` -while the surrounding lines use imported targets. rocRoller exports as -`roc::rocroller`, and under TheRock it is a separate subproject found through -its package config, so the bare name is not a target here and reaches the -linker as `-lrocroller` with no `-L` to resolve it. hipBLASLt's own build -avoids this by having rocRoller in-tree. - -`hiptensor` outgrows the small code model. Upstream enables coverage as a -blanket `add_compile_options(-fprofile-instr-generate -fcoverage-mapping)`, and -hipTensor is built on composable_kernel, so the coverage mapping and name data -scale with an already very large amount of instantiated template code. The -result pushes `libhiptensor.so` past 2GB and the link fails on out-of-range -`R_X86_64_PC32` relocations from `.rodata` into `.text`. Building it with a -larger code model is the likely fix. +A `blocked_reason` alongside `coverage_option` means the instrumented build is +broken. The project is dropped from the default and group aliases (including +`--emit-cmake` output) but can still be named explicitly, printing the reason. +A stage is built as a unit, so one blocked project takes the whole stage down. +Drop `blocked_reason` once the upstream fix lands. + +`hipblaslt` — its coverage branch links against bare `rocroller` while the rest +of its CMakeLists uses `roc::rocroller`. Under TheRock rocRoller is a separate +subproject; the bare name reaches the linker as `-lrocroller` with no `-L`. + +`hiptensor` — upstream applies coverage as a blanket `add_compile_options` on a +composable_kernel-based library. The coverage mapping data exceeds 2GB, failing +the link with out-of-range `R_X86_64_PC32` relocations. A larger code model is +the likely fix. ## Adding a stage -`multi_arch_ci_coverage_linux.yml` builds compiler-runtime and math-libs, and -`configure_coverage_ci.py` rejects any selection that reaches beyond them -(`BUILDABLE_STAGES`). That covers 17 of the 19 measurable projects. - -Onboarding a project from another stage needs a build job for that stage added -to the workflow, and the stage added to `BUILDABLE_STAGES`. Order matters: a -stage consumes inbound artifacts from other stages, so anything it depends on -has to be built first. `comm-libs`, the stage `rccl` and `rocshmem` need, takes -`hipify` and `rocjitsu` from `emulation` — so it needs two jobs, emulation then -comm-libs, not one. Consult `build_topology` rather than assuming; a missing -inbound artifact does not surface until well into the build. +`configure_coverage_ci.py` rejects selections reaching beyond `compiler-runtime` +and `math-libs` (`BUILDABLE_STAGES`). To add a stage: add a build job in +`multi_arch_ci_coverage_linux.yml` and add the stage name to `BUILDABLE_STAGES`. +Order matters — a stage needs its inbound artifacts built first. `comm-libs` +(needed by `rccl`/`rocshmem`) depends on `emulation`; that's two jobs, not one. +Consult `build_topology`; a missing inbound artifact won't surface until deep in +the build. diff --git a/docs/development/coverage_nightly_trigger_snippet.md b/docs/development/coverage_nightly_trigger_snippet.md index d54d22632ca..08b46c0d07c 100644 --- a/docs/development/coverage_nightly_trigger_snippet.md +++ b/docs/development/coverage_nightly_trigger_snippet.md @@ -1,31 +1,19 @@ # Reference: `trigger_coverage_linux` / `trigger_coverage_windows` jobs for the rocm-libraries nightly -This snippet belongs in **`ROCm/rocm-libraries`** -`.github/workflows/therock-multi-arch-ci-nightly.yml`, not in this repo. It is -kept here only as the canonical reference for the code-coverage dispatch -contract. TheRock cannot edit the external repo; copy this into the nightly when -wiring the real driver. +Snippet for **`ROCm/rocm-libraries`** +`.github/workflows/therock-multi-arch-ci-nightly.yml`. Copy here when wiring +the real driver; TheRock cannot edit that repo. -The nightly's regular build is **non-instrumented**, so it can only ever be the -baseline that coverage is measured against, never the thing coverage runs on. -Each platform's coverage orchestrator (`multi_arch_ci_coverage_linux.yml` / -`multi_arch_ci_coverage_windows.yml` in `ROCm/TheRock`) therefore runs as a -**separate** GitHub Actions run with its own `run_id`; a nested `uses:` call -would share the nightly's `run_id` and collide the generic and instrumented -trees the installer needs to keep apart. +The nightly build is non-instrumented and can only be a baseline. Coverage must +run as a **separate** GitHub Actions run (its own `run_id`) so the generic and +instrumented artifact sets stay distinct. Dispatching per-platform as each build +finishes avoids cross-platform waits and matches the nightly's own pattern. -Linux and Windows build at different speeds. Dispatching per-platform as soon as -each build finishes mirrors the nightly's own pattern (which dispatches -`multi_arch_ci_linux.yml` and `multi_arch_ci_windows.yml` independently) and -avoids forcing the faster platform to wait for the slower one. +Auth uses a **GitHub App token** (`actions/create-github-app-token`), modelled +on `ROCm/rocm-systems/.github/workflows/rocm_ci_caller.yml`. No classic PAT. -Cross-repo dispatch is authenticated with a **GitHub App token** -(`actions/create-github-app-token`), modelled on `ROCm/rocm-systems` -`.github/workflows/rocm_ci_caller.yml`. No classic PAT is introduced. - -> **One-time org setup (not code):** a GitHub App reachable from rocm-libraries -> must be installed on **ROCm/TheRock** with **`actions: write`**, so the tokens -> minted below are allowed to call `createWorkflowDispatch` on TheRock. +> **One-time org setup:** a GitHub App reachable from rocm-libraries must be +> installed on **ROCm/TheRock** with **`actions: write`**. ## `workflow_dispatch` inputs (add to the nightly's `on:` block) @@ -149,11 +137,10 @@ jobs: Notes: -- `baseline_run_id` is stringified because `createWorkflowDispatch` inputs must - be strings; `context.runId` is a number. -- Both jobs pass the **same** `baseline_run_id` (the nightly is one run) but - dispatch to different workflow files and different `amdgpu_families`. -- `ref` pins which TheRock copy of the coverage workflows runs. A - `workflow_dispatch` always executes the workflow file from that ref. -- `baseline_release_type` and `projects_to_test` should track whatever the - nightly actually built and published. +- `baseline_run_id` must be a string (`String(context.runId)`). +- Both jobs share the same `baseline_run_id` but dispatch to different workflow + files and `amdgpu_families`. +- `ref` pins the TheRock workflow copy; update it when the coverage workflows + change in a breaking way. +- `baseline_release_type` and `projects_to_test` should match what the nightly + built. diff --git a/docs/development/nightly_coverage_implementation.md b/docs/development/nightly_coverage_implementation.md deleted file mode 100644 index 4a6805bb5eb..00000000000 --- a/docs/development/nightly_coverage_implementation.md +++ /dev/null @@ -1,363 +0,0 @@ -# Code Coverage in TheRock — A Simple Walkthrough - -How a coverage report gets produced, start to finish, in plain terms. This is -the narrative introduction; [Code Coverage](code_coverage.md) is the reference, -with the CMake options, the CI walkthrough and how to onboard a project. - -______________________________________________________________________ - -## What is this trying to do? - -Answer one question: **when we run hipRAND's tests, how much of hipRAND's code -actually runs?** - -The green run answered it — 77% of lines, 82% of functions. Getting to that -number is what everything below is about. - -______________________________________________________________________ - -## First, three words you need - -**Instrumented** — a library compiled with extra bookkeeping. Every time a line -runs, it bumps a counter. Instrumented code is slower and bigger, which is why -we don't ship it. You get it by compiling with `-fprofile-instr-generate`. - -**profraw** — the counter file an instrumented program dumps when it exits. -Raw, per-process, not human-readable. - -**lcov** — the final, readable report: which lines ran and which didn't. - -The whole pipeline is just: build instrumented → run tests → collect profraw → -convert to lcov. - -______________________________________________________________________ - -## The one idea that explains the design - -You might expect: build everything instrumented, run the tests, done. - -The problem is that hipRAND doesn't run alone. Its tests also load rocRAND, -hipBLAS, the HIP runtime and more. If *those* were instrumented too, they'd all -write counters, and hipRAND's report would include code that isn't hipRAND's. -Worse, hipRAND's percentage would move whenever some unrelated library changed. - -So we want **exactly one instrumented library, and everything else normal.** - -Building "everything else normal" from scratch would take hours. But a normal -build of the whole stack already exists — the **regular nightly** runs every -night. So we reuse it: - -> Take last night's normal build, and swap in **just** our instrumented hipRAND. - -That is the core trick. It's why two different builds are involved. - -______________________________________________________________________ - -## The two builds - -| | Where it comes from | What we take | -| ------------ | --------------------------------------------------------------- | ----------------------------- | -| **Baseline** | Last night's regular nightly (`ROCm/rockrel` run `34068476531`) | The whole ROCm stack, normal | -| **Coverage** | Our own run, right now | Only the instrumented hipRAND | - -______________________________________________________________________ - -## The flow - -```mermaid -flowchart TD - A["1. Someone starts a coverage run
(picks: which project?)"] - B["2. Work out the plan
'test hipRAND, instrument hipRAND'"] - C["3. Build the compiler + runtime"] - D["4. Build math-libs
hipRAND instrumented, rest normal"] - E["5. On a GPU machine:
install last night's normal build"] - F["6. Swap in our instrumented hipRAND"] - G["7. Run hipRAND's tests
counters written to .profraw"] - H["8. Merge profraw, convert to lcov"] - I["Report: 77% of lines covered"] - - A --> B --> C --> D --> E --> F --> G --> H --> I - - style D fill:#ffe6cc - style F fill:#ffe6cc - style I fill:#d5e8d4 -``` - -The two orange boxes are the parts unique to coverage. Everything else is -TheRock's ordinary build and test machinery. - -______________________________________________________________________ - -## Step by step - -### Step 1 — Start the run - -Today it starts automatically when a pull request touches a coverage file. That -trigger (`coverage_nightly_pr_check.yml`) is temporary scaffolding for -validation and gets deleted before merge. After that, you start it by hand from -the Actions tab. - -You choose one thing that matters: **which projects to measure**, through the -`projects_to_test` input. It takes a comma-separated list of project names, or -one of the group aliases (`rocm_libraries_all`, `rocm_systems_all`, `all`). -Leaving it empty measures every onboarded project the workflow can build. The -walkthrough below follows a single-project run, `hiprand`, because it is the -easiest to read; nothing changes for a larger selection except how many test -and report jobs fan out at the end. - -### Step 2 — Work out the plan - -`setup_coverage_matrix`*, about 6 seconds* - -A script (`configure_coverage_ci.py`) looks up hipRAND in its registry and -figures out everything downstream needs: - -- Turn coverage on with `-DHIPRAND_ENABLE_COVERAGE=ON` -- hipRAND lives in the `math-libs` build stage, so build that -- Its files live in `math-libs/hipRAND/stage` -- Report on `lib/libhiprand.so*` -- Run the test suite called `hiprand` - -If you asked for something that can't be built, it stops **here** with a clear -message rather than failing two hours later. - -### Step 3 — Build the compiler - -*about 7 minutes* - -Coverage needs the compiler's profiling runtime, so this is built first. Nothing -project-specific happens yet. - -### Step 4 — Build the libraries, with hipRAND instrumented - -*about 59 minutes — the expensive step* - -The whole `math-libs` stage gets built, but only hipRAND is instrumented. The -flag flows through three hands to get there: - -``` -configure_coverage_ci.py says -DHIPRAND_ENABLE_COVERAGE=ON - ↓ -therock_subproject.cmake translates it to -DBUILD_CODE_COVERAGE=ON - ↓ -hipRAND's own CMakeLists adds -fprofile-instr-generate -``` - -**Why the translation?** Because projects don't agree on a name. hipRAND calls -its option `BUILD_CODE_COVERAGE`, RCCL calls it `ENABLE_CODE_COVERAGE`, others -use something else. TheRock uses one consistent name and translates per project. -Skipping this step was a real bug — hipRAND built without instrumentation and -quietly reported nothing. - -**Why build all of math-libs?** Because a stage, not a project, is the unit CI -knows how to build. The coverage workflow hands the ordinary build workflow -`stage_name: math-libs`, and everything downstream keys off that one input: -`fetch_sources.py --stage`, the configure step's `--stage`, and finally -`cmake --build --target stage-math-libs`. "Build only hipRAND" isn't a smaller -version of that job — it's a job that doesn't exist. There's a floor anyway, -since hipRAND sits on rocRAND, which sits on rocPRIM and the HIP runtime. - -This is the design working, not a cost being tolerated. The whole point is to -pay for one instrumented stack per night and test every project against it, so -the build is the same size whether you measure one project or twelve. The -numbers say so: hipRAND alone took 59 minutes, and all twelve took 68. - -> Careful with the two verbs: the whole stage is **built**, but only the -> selected projects are **instrumented**. Everything else compiles normally. -> Keeping each project's numbers to itself happens later, in steps 6 and 8 — -> not by restricting what gets built. - -### Step 5 — Install last night's normal build - -*on a real GPU machine* - -The test machine downloads and installs the entire baseline stack — normal, -uninstrumented, nothing special. - -At this moment there is **no** instrumented code on the machine at all. - -### Step 6 — Swap in the instrumented hipRAND - -`install_rocm_code_coverage_build.py` - -Now copy our instrumented hipRAND files over the baseline's copies: - -``` -Overlaying .../rand_dev.../math-libs/hipRAND/stage onto .../build -Overlaying .../rand_lib.../math-libs/hipRAND/stage onto .../build -Overlaying .../rand_test.../math-libs/hipRAND/stage onto .../build -Overlaid 3 instrumented subtree(s) -``` - -Result: a complete, working ROCm install where **exactly one library** counts -what it does. - -> Note it copies `math-libs/hipRAND/stage`, not the whole `rand` package. That -> package holds **both** rocRAND and hipRAND, and we want rocRAND left alone. - -### Step 7 — Run the tests - -hipRAND's normal test suite runs, unchanged. One environment variable tells the -instrumented library where to dump its counters: - -``` -LLVM_PROFILE_FILE=.../hiprand-shard1-%p-%m.profraw -``` - -`%p` is the process id and `%m` identifies the binary, so nothing overwrites -anything else. This run produced **8** profraw files. - -### Step 8 — Build the report - -`merge_coverage_report.py`*, about 30 seconds* - -Three things happen: - -1. `llvm-profdata` merges the 8 profraw files into one index -1. `llvm-cov` reads that index plus `lib/libhiprand.so*` and writes lcov -1. The lcov file is uploaded (and sent to Codecov if a token is configured) - -Pointing `llvm-cov` at only hipRAND's library is the second safeguard: even if -something else were instrumented, it wouldn't reach the report. - -______________________________________________________________________ - -## The result - -From run -[34135220913](https://github.com/ROCm/TheRock/actions/runs/34135220913): - -``` -Merging 8 profraw file(s) into coverage.profdata -Exporting lcov for 1 object(s) to coverage.info -``` - -| | | -| ----------------- | --------------------- | -| Lines covered | 155 / 201 — **77.1%** | -| Functions covered | 28 / 34 — **82.4%** | -| File measured | `hiprand.cpp` | -| Total time | about 70 minutes | - -______________________________________________________________________ - -## Which files do what - -| File | Its one job | -| ------------------------------------ | -------------------------------------------- | -| `coverage_nightly_pr_check.yml` | Starts the run (temporary) | -| `multi_arch_ci_coverage.yml` | Thin orchestrator: setup, linux, summary | -| `multi_arch_ci_coverage_linux.yml` | Builds instrumented stacks, then fans out | -| `multi_arch_ci_coverage_report.yml` | Per project: test, then report | -| `test_code_coverage_component.yml` | Installs, swaps, runs tests | -| `configure_coverage_ci.py` | The registry — knows every project's details | -| `therock_subproject.cmake` | Translates the coverage flag; fixes linking | -| `install_rocm_code_coverage_build.py` | Does the baseline install + project swap | -| `merge_coverage_report.py` | Makes the report in step 8 | - -______________________________________________________________________ - -## Appendix — every file in the PR - -Twenty-one files: eleven new, ten edits to things that already existed. The -table above covers the eight you need for the flow; this is the complete list, -grouped by the part of the pipeline each one belongs to. - -| # | Part | What lives here | -| --- | ------------------------------ | --------------------------------------------------- | -| 1 | Entry points and orchestration | Starting a run and sequencing the jobs | -| 2 | Planning | Deciding what to build, instrument, test and report | -| 3 | Build and instrumentation | Turning the coverage flag into compiler flags | -| 4 | Artifacts and test execution | Assembling the install tree and running tests | -| 5 | Reporting | profraw to lcov | -| 6 | Unit tests | The fast checks that run on every PR | -| 7 | Documentation | — | - -A useful thing to notice while reading: every one of the ten edits is additive -and defaults to off or empty, so regular CI behaves exactly as before. Coverage -rides on the existing build and test machinery rather than forking it. The one -exception worth knowing about is `CMakeLists.txt`, which runs the registry -script on *every* configure, coverage or not, to learn the project lists — a -few milliseconds, and the reason CMake and CI cannot drift apart. - -### 1. Entry points and orchestration - -All four are new files. - -| File | What it does | Called by | Calls | -| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `coverage_nightly_pr_check.yml` **(temporary — deleted before merge)** | Makes the coverage workflow runnable from a branch and visible as a PR check, which `workflow_dispatch` alone cannot do | GitHub, on a `pull_request` touching any of nine coverage paths — deliberately narrow, since every run is a multi-hour build | `multi_arch_ci_coverage.yml` | -| `multi_arch_ci_coverage.yml` | The real entry point and thin top-level orchestrator. Three jobs: `setup_coverage_matrix`, `coverage_linux` (dispatches the Linux layer), `ci_summary`. Keeps `workflow_dispatch` plus a temporary `workflow_call` | A human, from the Actions tab, or the consumer nightly via App-token dispatch (option C). Today also the PR check above, via the `workflow_call` trigger removed alongside it | `configure_coverage_ci.py`; `multi_arch_ci_coverage_linux.yml` | -| `multi_arch_ci_coverage_linux.yml` | The per-platform build + fan-out layer. Three jobs: `build_instrumented_compiler_runtime`, `build_instrumented_math_libs`, `coverage_report` (project × GPU family). A future `multi_arch_ci_coverage_windows.yml` mirrors it | `multi_arch_ci_coverage.yml` | `multi_arch_build_portable_linux_artifacts.yml` twice (compiler-runtime, math-libs); `multi_arch_ci_coverage_report.yml` per project | -| `multi_arch_ci_coverage_report.yml` | Everything for one project on one GPU family, in three jobs: `configure_test_matrix`, `test_coverage`, `aggregate_coverage` | `multi_arch_ci_coverage_linux.yml`, once per matrix entry | `configure_coverage_ci.py`, `test_code_coverage_component.yml`, `artifact_manager.py`, `merge_coverage_report.py` | - -### 2. Planning - -One new file, and the reason the rest of the pipeline stays consistent. - -| File | What it does | Called by | Calls | -| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | -| `build_tools/github_actions/configure_coverage_ci.py` | The registry and single source of truth. Per project it records the build stage, the upstream option name, the artifacts and stage directories to overlay, the object globs to report on, and the test suite to run. Resolves a selection into a job matrix and rejects anything unbuildable up front, so a bad request stops in six seconds rather than two hours | Three places, which is what makes it the source of truth: the nightly's matrix job, the per-project workflow's configure job, and `CMakeLists.txt` via `--emit-cmake` | Nothing — pure data and logic | - -### 3. Build and instrumentation - -All three already existed. The flags they add are inert unless coverage is on, -with the one caveat from above: `CMakeLists.txt` reads the registry on every -configure regardless. - -| File | What it does | Called by | Calls | -| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `multi_arch_build_portable_linux_artifacts.yml` | Builds one stage in a container and uploads its artifacts. Coverage adds an `extra_cmake_options` input, appended **after** the stage-derived arguments, since the last `-D` wins. This is how the coverage flags reach the build without coverage needing a build workflow of its own | Regular multi-arch CI, and now the coverage nightly | The configure step, then `cmake --build --target stage-` | -| `CMakeLists.txt` | Declares the three group options (`THEROCK_COVERAGE_ALL`, `..._ROCM_LIBRARIES_ALL`, `..._ROCM_SYSTEMS_ALL`) and expands whichever is set into individual `_ENABLE_COVERAGE` flags. An explicit `-DFOO_ENABLE_COVERAGE=OFF` still wins | Every CMake configure, coverage or not | `configure_coverage_ci.py --emit-cmake`, then includes the generated `therock_coverage_projects.cmake` | -| `cmake/therock_subproject.cmake` | Where the generic flag becomes a real one. Translates `_ENABLE_COVERAGE` into whatever the subproject implements; adds `-ldl` and `-lpthread`, which the profile runtime needs and the driver does not supply; and scopes instrumentation to host code with `-Xarch_device`, injected into the compile rule so it lands after the project's own flags | Every subproject declaration in the build | Generates a small CMake file handed to the subproject as `CMAKE_PROJECT_INCLUDE` | - -### 4. Artifacts and test execution - -This is the part that assembles a working install in which exactly one library -is instrumented. `test_code_coverage_component.yml` and -`install_rocm_code_coverage_build.py` are the new, coverage-only path; -`test_component.yml` is the shared component-test workflow that coverage no -longer uses. - -| File | What it does | Called by | Calls | -| -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `test_code_coverage_component.yml` **(new)** | The coverage-only test workflow. Installs from the baseline run and swaps in the instrumented project, points `LLVM_PROFILE_FILE` at a per-shard path, and uploads the profraw files. Also raises the test timeout to a 60-minute floor, since instrumented `-O0` builds run far slower. `test_component.yml`, the shared component-test workflow, was left unedited and is no longer on the coverage path | `multi_arch_ci_coverage_report.yml` | `setup_test_environment`, then `install_rocm_code_coverage_build.py`, then the project's own test suite | -| `.github/actions/setup_test_environment/action.yml` | Composite action that downloads and installs a ROCm build on the test machine. Coverage adds `RUN_GITHUB_REPO`, because the baseline is published by `ROCm/rockrel` and the bucket lookup fails if it queries the wrong repository | `test_component.yml` and `test_code_coverage_component.yml` | `install_rocm_from_artifacts.py` | -| `build_tools/install_rocm_code_coverage_build.py` **(new)** | The swap. Installs the generic tree from `--run-id` (the baseline nightly run) and replaces only the project-under-test's libraries from `--code-coverage-run-id` (this coverage run's own id), deriving the artifact name and folder from the component `job_name` | `test_code_coverage_component.yml` | `artifact_manager.py`, imported as a module rather than shelled out to, fetching with `--artifact-names` and `--stage all` | -| `build_tools/artifact_manager.py` | Pre-existing artifact transport: pushes a stage's outputs to S3 and fetches them back. Coverage adds `--artifact-names`, an allowlist, because the existing `--exclude-artifacts` denylist is the wrong shape for fetching one project's files | Six pre-existing callers (`fetch_artifacts.py`, `build_tarballs.py`, `post_stage_upload.py`, `baseline_runs.py`, `stage_reuse_decision.py`, `expand_amdgpu_families.py`) plus `install_rocm_code_coverage_build.py` | The S3 artifact backend | - -### 5. Reporting - -| File | What it does | Called by | Calls | -| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | --------------------------------------------- | -| `build_tools/github_actions/merge_coverage_report.py` **(new)** | Step 8. Finds the profraw files, resolves the object globs (collapsing versioned symlinks like `libfoo.so`, `.so.1`, `.so.1.2` into one object), merges and exports lcov. Fails loudly when a glob matches nothing, which is what caught the hipDNN naming bug instead of quietly emitting an empty report | `multi_arch_ci_coverage_report.yml`, in `aggregate_coverage` | `llvm-profdata merge`, then `llvm-cov export` | - -### 6. Unit tests - -All four run in the **Unit Tests** workflow, whose "Test build_tools" step runs -`pytest` from `build_tools/` on Ubuntu and Windows, on every pull request. No -GPU, no network, no build — the whole set finishes in about two seconds. That -speed is the point: the alternative way to catch these mistakes is a two-hour -instrumented run on a GPU runner, which is how the hipDNN and hipTensor bugs -were actually found. - -| File | Covers | Notable cases | -| ---------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | -| `configure_coverage_ci_test.py` **(new)** | The registry | Alias expansion, blocked projects, and whole-registry invariants: every project declares report inputs and names an upstream option | -| `overlay_coverage_artifacts_test.py` **(new)** | The swap | Symlink preservation, both file/symlink crossing cases, and failing loudly when nothing was overlaid | -| `merge_coverage_report_test.py` **(new)** | The report | Tool lookup, versioned-symlink collapsing, and the failure policy for empty profiles and unmatched globs | -| `artifact_manager_tool_test.py` | `--artifact-names` | Two tests beside the existing `--exclude-artifacts` ones, keeping allowlist and denylist symmetric | - -Note the last one edits a large pre-existing file rather than adding a new one. - -### 7. Documentation - -| File | What it covers | -| --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docs/development/code_coverage.md` **(new)** | The reference, in one file: the CMake options, the end-to-end CI walkthrough, why instrumentation is host-only, why the test timeout has a floor, and how to onboard a project or a stage | -| `docs/development/nightly_coverage_implementation.md` **(new)** | This document — the narrative introduction to the same pipeline | -| `docs/development/ci_overview.md` | Places coverage among the CI entry points, noting it is separate because it rebuilds what it measures | -| `docs/development/development_guide.md` | Documents `{PROJECT}_ENABLE_COVERAGE` next to the other build flags, including the upper-case rule: `HIPDNN_ENABLE_COVERAGE`, not `hipDNN_ENABLE_COVERAGE` | -| `docs/development/README.md` | Index entries for the new documents | - -______________________________________________________________________ From cba8728153cd64321a36e45a35e4c7cde879ce60 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Mon, 21 Sep 2026 15:55:45 -0700 Subject: [PATCH 31/70] style(ci): simplify comments in coverage workflows and scripts --- .../workflows/coverage_nightly_pr_check.yml | 23 +++++-------- .../multi_arch_ci_coverage_linux.yml | 32 ++++++++----------- .../github_actions/configure_coverage_ci.py | 10 +++--- 3 files changed, 25 insertions(+), 40 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 59b5e60c2d0..56a7cd55e98 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -1,20 +1,14 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -# TEMPORARY. Delete this file and the workflow_call trigger it relies on before -# merging the coverage PR. +# TEMPORARY. Delete this file and the workflow_call trigger before merging. # -# multi_arch_ci_coverage_linux.yml runs by hand, but workflow_dispatch only -# appears once the file is on the default branch -- so it can't run from a -# branch or show in a PR's checks, leaving phase 1 unable to demonstrate itself. +# workflow_dispatch only appears once the file is on the default branch, so it +# can't trigger from a branch. A caller can: paths resolve against the caller's +# commit, so this runs the branch copy as a PR check. # -# A caller has no such restriction: paths under ./.github/workflows resolve -# against the caller's commit, so this runs the branch's copy and its jobs -# report as checks on this PR. -# -# Scaffolding, not design. Presubmit coverage is what phase 1 rejects: each run -# is a multi-hour instrumented build plus full test suite, which is why coverage -# is a separate manual workflow. Leaving this in charges that cost to every PR. +# Presubmit coverage is too expensive for every PR (multi-hour instrumented +# build + full test suite); coverage is a separate manual workflow by design. name: Coverage Nightly PR Check @@ -52,7 +46,7 @@ jobs: actions: read id-token: write with: - # Pinned, not resolved, so a change in the report means a change in this branch. + # Pinned baseline: a report change must reflect a code change, not a baseline drift. # # ROCm/rockrel run 34068476531, "Multi-Arch Release (nightly)" on main, # 2026-09-07, succeeded. A real nightly, not a CI run, since that's what a @@ -80,8 +74,7 @@ jobs: # 34135220913 proved the path end to end on hipRAND (build, install, # overlay, tests, 77% report) once libclang_rt.profile_rocm.a's own deps # (-ldl, -lpthread) were added to the link line. Run 34164420766 then built - # all twelve green and reported six; the six failures are what the host-only - # scoping, timeout floor and two registry corrections in this branch fix. + # all twelve green and reported six. # # Named explicitly, not rocm_libraries_all, to leave out the four # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA): they have no diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 3211b1c2151..e099e0ae2c3 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -12,8 +12,7 @@ # directly. A future Windows mirror copies this file, swapping the build # reusable and runners; multi_arch_ci_coverage_report.yml is reused unchanged. # -# RFC0014 phase 1: every enabled component on one architecture (gfx94X-dcgpu). -# Change-based selection and multi-arch come later. +# Currently targets gfx94X-dcgpu; change-based selection and multi-arch come later. name: Multi-Arch CI Coverage - Linux @@ -34,11 +33,11 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem require a comm-libs build job (not present), so rocm_systems_all and all are rejected." default: "" amdgpu_families: type: string - description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." + description: "Comma-separated GPU families to collect coverage on." default: "gfx94X-dcgpu" coverage_config_source: type: string @@ -50,16 +49,13 @@ on: default: "standard" quartz_tracking_id: type: string - description: >- - Leave empty. Internal Quartz tracking id linking this run to a release - lineage once the nightly dispatches coverage automatically. + description: "Leave empty. Reserved for release tracking." default: "" - # TEMPORARY, paired with coverage_nightly_pr_check.yml. workflow_dispatch only - # appears once the file is on the default branch, so it can't run from a - # branch or show in PR checks; a caller can, so the check workflow runs this - # branch's copy. Actions doesn't expand YAML anchors, so inputs are repeated. - # Remove this trigger with the check workflow once the run is green. + # TEMPORARY, paired with coverage_nightly_pr_check.yml. + # workflow_dispatch requires the file to be on the default branch; a caller + # has no such restriction. Actions doesn't expand YAML anchors, so inputs are + # repeated. Remove this trigger with the check workflow once the run is green. workflow_call: inputs: baseline_run_id: @@ -76,11 +72,11 @@ on: default: "" projects_to_test: type: string - description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem need the comm-libs build job (absent in phase 1), so they (and rocm_systems_all / all) are rejected." + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem require a comm-libs build job (not present), so rocm_systems_all and all are rejected." default: "" amdgpu_families: type: string - description: "Comma-separated GPU families to collect coverage on. Phase 1 uses the single default architecture." + description: "Comma-separated GPU families to collect coverage on." default: "gfx94X-dcgpu" coverage_config_source: type: string @@ -92,9 +88,7 @@ on: default: "standard" quartz_tracking_id: type: string - description: >- - Leave empty. Internal Quartz tracking id linking this run to a release - lineage once the nightly dispatches coverage automatically. + description: "Leave empty. Reserved for release tracking." default: "" permissions: @@ -166,8 +160,8 @@ jobs: contents: read id-token: write - # Build the whole stage, not just onboarded projects: test jobs need the full - # dependency set, and phase 1 instruments every project in one pass. + # Build the whole stage: test jobs need the full dependency set, and + # instrumenting as a unit keeps the per-project build cost flat. # coverage_cmake_options is the THEROCK_COVERAGE_*_ALL group option for a whole # group, else individual project flags; unselected projects build normally. build_instrumented_math_libs: diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 1c41df48310..640e5d2628a 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -63,11 +63,9 @@ } ) -# Phase 1 has build jobs for these two only. The jobs that built any other -# stage were removed while their sole measurable consumers -- rccl and rocshmem -# in comm-libs -- are blocked upstream, so a selection reaching further is -# rejected by resolve_build_stages() rather than failing hours later on a -# missing inbound artifact. +# Only compiler-runtime and math-libs have build jobs. rccl and rocshmem +# (comm-libs) are blocked upstream, so selecting them is rejected here rather +# than failing hours later on a missing inbound artifact. BUILDABLE_STAGES = frozenset({STAGE_COMPILER_RUNTIME, STAGE_MATH_LIBS}) @@ -769,7 +767,7 @@ def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: def resolve_build_stages(project_keys: list[str]) -> set[str]: - """Returns the stages the selection needs, rejecting what phase 1 cannot build. + """Returns the stages the selection needs, rejecting stages with no build job. Only compiler-runtime and math-libs have build jobs. A project from any other stage would have nothing built for it, and that would not surface From 7e333da0998ca66e894c69859b9037aedd416395 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Mon, 21 Sep 2026 16:10:17 -0700 Subject: [PATCH 32/70] fix(test): use Path for demangler assertion; apply mdformat to code_coverage.md --- .../tests/merge_coverage_report_test.py | 2 +- docs/development/code_coverage.md | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index 5d71435cb66..1272aff92b4 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -176,7 +176,7 @@ def test_html_renders_into_a_directory(self): self.assertIn("--format=html", command) self.assertIn(f"-output-dir={output_dir}", command) self.assertIn("--project-title=hiprand", command) - self.assertIn("-Xdemangler=/llvm/llvm-cxxfilt", command) + self.assertIn(f"-Xdemangler={Path('/llvm/llvm-cxxfilt')}", command) def test_path_equivalence_reaches_every_rendering(self): objects = [self.touch("lib/a.so")] diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 767fc435a09..087672bdb2a 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -18,10 +18,10 @@ A project's tests load its dependencies too. If those were instrumented, their counters would pollute the report. So exactly one library is instrumented; the rest come from the regular nightly build (the **baseline**): -| | Source | What it provides | -|---|---|---| +| | Source | What it provides | +| ------------ | ------------------- | ---------------------------------- | | **Baseline** | Regular nightly run | Whole ROCm stack, non-instrumented | -| **Coverage** | This run | Only the one instrumented project | +| **Coverage** | This run | Only the one instrumented project | The test runner installs the baseline, then replaces only the selected project's libraries with instrumented ones. See [Coverage CI](#coverage-ci). @@ -42,11 +42,11 @@ for an unregistered project is a configure error, not a silent no-op. ### Enabling a whole group -| Option | Instruments | -| --- | --- | +| Option | Instruments | +| ------------------------------------- | -------------------------------------------- | | `THEROCK_COVERAGE_ROCM_LIBRARIES_ALL` | all coverage-enabled rocm-libraries projects | -| `THEROCK_COVERAGE_ROCM_SYSTEMS_ALL` | all coverage-enabled rocm-systems projects | -| `THEROCK_COVERAGE_ALL` | both of the above | +| `THEROCK_COVERAGE_ROCM_SYSTEMS_ALL` | all coverage-enabled rocm-systems projects | +| `THEROCK_COVERAGE_ALL` | both of the above | Each expands to individual `_ENABLE_COVERAGE` flags. An explicit `-D_ENABLE_COVERAGE=OFF` wins over a group flag. Membership is @@ -191,10 +191,10 @@ most for header-only projects (rocPRIM, hipCUB, rocThrust, rocWMMA). ### Artifacts -| Artifact | Where | Purpose | -| --- | --- | --- | -| Instrumented | CI bucket, this run | Stack under test + LLVM tools | -| Profraw | GitHub Actions artifacts, per shard | Raw profiles for aggregation | +| Artifact | Where | Purpose | +| ------------ | ----------------------------------- | ----------------------------- | +| Instrumented | CI bucket, this run | Stack under test + LLVM tools | +| Profraw | GitHub Actions artifacts, per shard | Raw profiles for aggregation | Profraw names include the project, GPU family, and shard index. @@ -209,11 +209,11 @@ doesn't match the project's install layout). The `coverage-report--` artifact: -| File | What it is | -| --- | --- | -| `coverage.info` | lcov for Codecov | +| File | What it is | +| ---------------------- | -------------------------- | +| `coverage.info` | lcov for Codecov | | `coverage_summary.txt` | per-file table with totals | -| `html/index.html` | browsable annotated report | +| `html/index.html` | browsable annotated report | Totals are also written to the job summary page. The HTML step runs `fetch_sources.py` and remaps paths with `--path-equivalence`; it is From c0c48d219b5545399388ca557e27777ff03af15f Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Mon, 21 Sep 2026 20:33:50 -0700 Subject: [PATCH 33/70] fix(ci): add setup-python before coverage install step --- .github/workflows/test_code_coverage_component.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 1d6277bb710..0d14934b987 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -134,6 +134,11 @@ jobs: repository: ${{ inputs.repository || github.repository }} ref: ${{ inputs.ref }} + - name: Setting up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + - name: Install TheRock Code Coverage Build timeout-minutes: 15 env: @@ -141,7 +146,7 @@ jobs: FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} run: | - pip3 install -r requirements-test.txt + pip install -r requirements-test.txt # Baseline tree comes from generic_run_id. When empty (self-contained # dispatch), fall back to this run's own instrumented artifacts; # --replace then re-extracts them, leaving the whole stack instrumented. From 9e4f0c6f1c201a9e1625c5f6313cfb850d25dc9a Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 22 Sep 2026 19:00:49 +0530 Subject: [PATCH 34/70] feat(ci): add baseline_release_type input for coverage baseline ## Motivation The multi-arch coverage report needs to compare against a baseline that may be published under a different release type than the current run (e.g. a release nightly from ROCm/rockrel). Reusing `release_type` for both the current and baseline runs forced them to share the same type, preventing comparison against a differently-typed baseline. ## Technical Details - Add a new `baseline_release_type` workflow input (string, defaults to `nightly`). - Pass `inputs.baseline_release_type` instead of `inputs.release_type` as the `release_type` for the baseline coverage job, decoupling the baseline's release type from the current run's. --- .github/workflows/multi_arch_ci_coverage_report.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 2404cb26f35..f9be1aa12b3 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -54,6 +54,9 @@ on: description: >- Repository owning baseline_run_id, in 'owner/repo' form. Needed when the baseline is a release nightly, which ROCm/rockrel publishes. + baseline_release_type: + type: string + default: nightly codecov_flag: type: string default: "" @@ -155,7 +158,7 @@ jobs: test_runs_on: ${{ matrix.components.test_runner }} platform: ${{ needs.configure_test_matrix.outputs.platform }} component: ${{ toJSON(matrix.components) }} - release_type: ${{ inputs.release_type }} + release_type: ${{ inputs.baseline_release_type }} repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} build_variant: coverage From af064e4337b6bd8abed714c689af376070270df3 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 22 Sep 2026 21:50:12 +0530 Subject: [PATCH 35/70] Dropping baseline_repository parameter --- .github/workflows/coverage_nightly_pr_check.yml | 4 ++-- .github/workflows/test_code_coverage_component.yml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 56a7cd55e98..5c93afaf984 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -62,10 +62,10 @@ jobs: # selected project: blas, fft, hipdnn, rand, solver, sparse. Not all are # per-target (hipdnn and solver's tests are generic), so checking only for # _gfx942 understates what's there. - baseline_run_id: "34068476531" + baseline_run_id: "35620066949" # The bucket lookup queries the run via the GitHub API, so it needs the # owning repo; ROCm/TheRock 404s for a rockrel id. - baseline_repository: "ROCm/rockrel" + baseline_repository: "ROCm/TheRock" # That run published to therock-nightly-artifacts (the default); naming it # keeps the pin and channel together. baseline_release_type: "nightly" diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 0d14934b987..7dee12bf2b0 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -144,7 +144,7 @@ jobs: env: RELEASE_TYPE: ${{ inputs.release_type }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} - BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} +# BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} run: | pip install -r requirements-test.txt # Baseline tree comes from generic_run_id. When empty (self-contained @@ -157,9 +157,9 @@ jobs: # Pass --run-github-repo only when set, so empty defaults to the # current repository. run_github_repo_arg=() - if [[ -n "${BASELINE_REPOSITORY}" ]]; then - run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") - fi +# if [[ -n "${BASELINE_REPOSITORY}" ]]; then +# run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") +# fi python3 build_tools/install_rocm_code_coverage_build.py \ --code-coverage-run-id=${{ env.CODE_COVERAGE_RUN_ID }} \ --run-id="${generic_run_id}" \ @@ -168,7 +168,7 @@ jobs: --tests \ --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \ --replace-${{ fromJSON(inputs.component).job_name }} \ - "${run_github_repo_arg[@]}" \ +# "${run_github_repo_arg[@]}" \ ${FETCH_ARTIFACT_ARGS} # safe.directory must be set before Runner Health Status From 96766ae4671a69d8e2214fa26c6863385925e008 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 22 Sep 2026 21:57:29 +0530 Subject: [PATCH 36/70] Dropping baseline_repository parameter --- .github/workflows/test_code_coverage_component.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 7dee12bf2b0..3ed17e9c99b 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -168,8 +168,8 @@ jobs: --tests \ --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \ --replace-${{ fromJSON(inputs.component).job_name }} \ -# "${run_github_repo_arg[@]}" \ ${FETCH_ARTIFACT_ARGS} +# "${run_github_repo_arg[@]}" \ # safe.directory must be set before Runner Health Status - name: Adjust git config From 81154af2936997964773f102c6e3aef04f90e26d Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 22 Sep 2026 22:16:20 +0530 Subject: [PATCH 37/70] Dropping baseline_repository parameter --- .github/workflows/test_code_coverage_component.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 3ed17e9c99b..e47c28367b9 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -157,9 +157,9 @@ jobs: # Pass --run-github-repo only when set, so empty defaults to the # current repository. run_github_repo_arg=() -# if [[ -n "${BASELINE_REPOSITORY}" ]]; then -# run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") -# fi + #if [[ -n "${BASELINE_REPOSITORY}" ]]; then + # run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") + #fi python3 build_tools/install_rocm_code_coverage_build.py \ --code-coverage-run-id=${{ env.CODE_COVERAGE_RUN_ID }} \ --run-id="${generic_run_id}" \ @@ -169,7 +169,7 @@ jobs: --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \ --replace-${{ fromJSON(inputs.component).job_name }} \ ${FETCH_ARTIFACT_ARGS} -# "${run_github_repo_arg[@]}" \ + #"${run_github_repo_arg[@]}" \ # safe.directory must be set before Runner Health Status - name: Adjust git config From 08f3f9133562cfd29b48e510c6ffaae530ec7c4e Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 22 Sep 2026 11:58:08 -0700 Subject: [PATCH 38/70] fix(ci): use ci release type for instrumented artifact lookup RELEASE_TYPE=nightly in the test env caused the replacement artifact lookup to target s3://therock-nightly-artifacts/ instead of s3://therock-ci-artifacts/ where the instrumented build uploaded. Add --code-coverage-release-type (default 'ci') to install_rocm_code_coverage_build.py and thread it through create_backend_from_env so Phase 2 uses the correct bucket. --- .../_therock_utils/artifact_backend.py | 10 +++++++++- .../install_rocm_code_coverage_build.py | 20 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/build_tools/_therock_utils/artifact_backend.py b/build_tools/_therock_utils/artifact_backend.py index 357839ba537..182c2851bbb 100644 --- a/build_tools/_therock_utils/artifact_backend.py +++ b/build_tools/_therock_utils/artifact_backend.py @@ -334,6 +334,7 @@ def create_backend_from_env( run_id: Optional[str] = None, github_repository: Optional[str] = None, platform: Optional[str] = None, + release_type: Optional[str] = None, ) -> ArtifactBackend: """Create the appropriate backend based on environment variables. @@ -344,6 +345,10 @@ def create_backend_from_env( For S3 backend (when THEROCK_LOCAL_STAGING_DIR is not set): - Uses WorkflowOutputRoot.from_workflow_run() for bucket selection + + Args: + release_type: Release type override (e.g. "ci", "nightly"). If None, + falls back to the RELEASE_TYPE environment variable. """ import platform as platform_module @@ -363,6 +368,9 @@ def create_backend_from_env( ) output_root = WorkflowOutputRoot.from_workflow_run( - run_id=run_id, platform=platform_name, github_repository=github_repository + run_id=run_id, + platform=platform_name, + github_repository=github_repository, + release_type=release_type, ) return S3Backend(output_root=output_root) diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index 9dd555aec36..11eeb7e6c4a 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -96,7 +96,11 @@ def _target_families(family, amdgpu_targets): def download_replacement_artifacts( - code_coverage_run_id, artifact_names, opts, code_coverage_run_github_repo + code_coverage_run_id, + artifact_names, + opts, + code_coverage_run_github_repo, + code_coverage_release_type, ): """Download instrumented replacement artifacts from the code-coverage run. @@ -109,6 +113,7 @@ def download_replacement_artifacts( run_id=code_coverage_run_id, github_repository=code_coverage_run_github_repo, platform=platform.system().lower(), + release_type=code_coverage_release_type, ) log(f"Fetching replacement artifacts from {backend.base_uri}") @@ -256,6 +261,18 @@ def main(argv): "from --run-github-repo, which owns the generic --run-id build." ), ) + parser.add_argument( + "--code-coverage-release-type", + type=str, + default="ci", + help=( + "Release type of the instrumented build identified by " + "--code-coverage-run-id (e.g. 'ci', 'nightly'). Controls which S3 " + "bucket is used for the replacement artifact lookup. Defaults to " + "'ci' because instrumented builds always run as CI jobs. This is " + "separate from the baseline's RELEASE_TYPE env var." + ), + ) artifacts_group = parser.add_argument_group("replace_comps") for comp in COMPONENT_MAP.keys(): artifacts_group.add_argument( @@ -292,6 +309,7 @@ def main(argv): artifacts.keys(), opts, args.code_coverage_run_github_repo, + args.code_coverage_release_type, ) # replace selected library folder paths in selected component artifacts From ba77777334a856587add463e44d8efb3d7c8ac8f Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 22 Sep 2026 16:08:32 -0700 Subject: [PATCH 39/70] fix(ci): restore cross-repo baseline plumbing for coverage install Pass --run-github-repo from baseline_repository, forward baseline_release_type through the linux workflow, and repin the PR check to rockrel nightly 35670146294 so the generic baseline fetch and bucket agree. --- .github/workflows/coverage_nightly_pr_check.yml | 10 +++++----- .github/workflows/multi_arch_ci_coverage_linux.yml | 1 + .github/workflows/test_code_coverage_component.yml | 14 ++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index 5c93afaf984..eb1616af044 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -48,8 +48,8 @@ jobs: with: # Pinned baseline: a report change must reflect a code change, not a baseline drift. # - # ROCm/rockrel run 34068476531, "Multi-Arch Release (nightly)" on main, - # 2026-09-07, succeeded. A real nightly, not a CI run, since that's what a + # ROCm/rockrel run 35670146294, "Multi-Arch Release (nightly)" on main, + # 2026-09-22, succeeded. A real nightly, not a CI run, since that's what a # dispatched coverage run gets in practice. # # A rockrel run is safe here. Its artifacts embed their build workspace @@ -58,14 +58,14 @@ jobs: # stack from source and only installs the baseline on the test runner, # where prebuilt binaries run and no package config is read. # - # Verified before pinning that this run publishes artifacts for every + # Verify before pinning that this run publishes artifacts for every # selected project: blas, fft, hipdnn, rand, solver, sparse. Not all are # per-target (hipdnn and solver's tests are generic), so checking only for # _gfx942 understates what's there. - baseline_run_id: "35620066949" + baseline_run_id: "35670146294" # The bucket lookup queries the run via the GitHub API, so it needs the # owning repo; ROCm/TheRock 404s for a rockrel id. - baseline_repository: "ROCm/TheRock" + baseline_repository: "ROCm/rockrel" # That run published to therock-nightly-artifacts (the default); naming it # keeps the pin and channel together. baseline_release_type: "nightly" diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index e099e0ae2c3..1de155e1aac 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -226,6 +226,7 @@ jobs: codecov_flag: ${{ matrix.coverage.codecov_flag }} amdgpu_families: ${{ matrix.coverage.amdgpu_families }} baseline_run_id: ${{ inputs.baseline_run_id }} + baseline_release_type: ${{ inputs.baseline_release_type }} baseline_repository: ${{ inputs.baseline_repository }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index e47c28367b9..b28924ae678 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -144,7 +144,7 @@ jobs: env: RELEASE_TYPE: ${{ inputs.release_type }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} -# BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} + BASELINE_REPOSITORY: ${{ inputs.baseline_repository }} run: | pip install -r requirements-test.txt # Baseline tree comes from generic_run_id. When empty (self-contained @@ -155,21 +155,23 @@ jobs: generic_run_id="${{ github.run_id }}" fi # Pass --run-github-repo only when set, so empty defaults to the - # current repository. + # current repository. The generic baseline lives in a different repo + # than this coverage run (rockrel for the PR check, rocm-libraries for + # the nightly); the S3 bucket lookup 404s if that repo is not named. run_github_repo_arg=() - #if [[ -n "${BASELINE_REPOSITORY}" ]]; then - # run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") - #fi + if [[ -n "${BASELINE_REPOSITORY}" ]]; then + run_github_repo_arg=(--run-github-repo="${BASELINE_REPOSITORY}") + fi python3 build_tools/install_rocm_code_coverage_build.py \ --code-coverage-run-id=${{ env.CODE_COVERAGE_RUN_ID }} \ --run-id="${generic_run_id}" \ + "${run_github_repo_arg[@]}" \ --artifact-group=${{ inputs.amdgpu_families }} \ --amdgpu-targets=${{ inputs.amdgpu_targets }} \ --tests \ --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \ --replace-${{ fromJSON(inputs.component).job_name }} \ ${FETCH_ARTIFACT_ARGS} - #"${run_github_repo_arg[@]}" \ # safe.directory must be set before Runner Health Status - name: Adjust git config From 82a0f5b462f10a09e1bbc3be005302a384c74e82 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 22 Sep 2026 16:09:32 -0700 Subject: [PATCH 40/70] test(ci): trim PR-check coverage to hipblas,hipdnn --- .github/workflows/coverage_nightly_pr_check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml index eb1616af044..7afe1691779 100644 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ b/.github/workflows/coverage_nightly_pr_check.yml @@ -91,6 +91,5 @@ jobs: # corrections instrumented it for the first time, and a stage builds as a # unit, so its link error failed math-libs and skipped all twelve reports. projects_to_test: >- - hipblas,hipdnn,hipfft,hiprand,hipsolver, - hipsparselt,rocblas,rocfft,rocrand,rocsolver,rocsparse + hipblas,hipdnn test_type: "standard" From 3f6170fe3062c3d619d113eb5687fa3d2c7c8152 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Tue, 22 Sep 2026 20:21:16 -0700 Subject: [PATCH 41/70] fix(ci): drop empty additional_requirements_files default --- build_tools/github_actions/fetch_test_configurations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 7d538545606..e7411da4135 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -149,10 +149,9 @@ def _family_matches( ) -# Common settings applied to all jobs -_common_settings = { - "additional_requirements_files": [], -} +# Common settings applied to all jobs. Kept empty so components omit keys they +# do not set (an empty additional_requirements_files trips the `!= ''` guard). +_common_settings = {} # Common settings for rocgdb jobs _rocgdb_common = { From 9a3f764b81f13a72b55951e3d409efd63dedf653 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Wed, 23 Sep 2026 11:15:29 -0700 Subject: [PATCH 42/70] fix(ci): pin TheRock repo/ref for coverage checkouts on cross-repo callers --- .github/workflows/multi_arch_ci_coverage_linux.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 1de155e1aac..9c52b429a77 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -119,6 +119,10 @@ jobs: id: checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 timeout-minutes: 15 + with: + # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. + repository: ROCm/TheRock + ref: ${{ github.job_workflow_sha || github.sha }} - name: Setting up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -230,6 +234,9 @@ jobs: baseline_repository: ${{ inputs.baseline_repository }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci + # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. + repository: ROCm/TheRock + ref: ${{ github.job_workflow_sha || github.sha }} permissions: contents: read id-token: write @@ -247,6 +254,10 @@ jobs: - name: Checking out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 timeout-minutes: 15 + with: + # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. + repository: ROCm/TheRock + ref: ${{ github.job_workflow_sha || github.sha }} - name: Evaluate workflow results env: GITHUB_TOKEN: ${{ github.token }} From cdc1f1568218a2662d4bdd1ddfbdfd4993c3fffc Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Wed, 23 Sep 2026 13:23:23 -0700 Subject: [PATCH 43/70] fix(ci): take repository/ref as inputs for coverage checkouts --- .../multi_arch_ci_coverage_linux.yml | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 9c52b429a77..338f83b3e13 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -51,6 +51,14 @@ on: type: string description: "Leave empty. Reserved for release tracking." default: "" + repository: + type: string + description: "Repository holding this workflow's scripts. Empty defaults to github.repository; cross-repo callers pass ROCm/TheRock." + default: "" + ref: + type: string + description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." + default: "" # TEMPORARY, paired with coverage_nightly_pr_check.yml. # workflow_dispatch requires the file to be on the default branch; a caller @@ -90,6 +98,14 @@ on: type: string description: "Leave empty. Reserved for release tracking." default: "" + repository: + type: string + description: "Repository holding this workflow's scripts. Empty defaults to github.repository; cross-repo callers pass ROCm/TheRock." + default: "" + ref: + type: string + description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." + default: "" permissions: contents: read @@ -120,9 +136,9 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 timeout-minutes: 15 with: - # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. - repository: ROCm/TheRock - ref: ${{ github.job_workflow_sha || github.sha }} + # Cross-repo callers pass ROCm/TheRock + ref; same-repo defaults to github.repository. + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} - name: Setting up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -234,9 +250,9 @@ jobs: baseline_repository: ${{ inputs.baseline_repository }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci - # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. - repository: ROCm/TheRock - ref: ${{ github.job_workflow_sha || github.sha }} + # Forwarded to L2's checkouts and the component jobs. + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} permissions: contents: read id-token: write @@ -255,9 +271,9 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 timeout-minutes: 15 with: - # Called cross-repo (rocm-libraries), so pin TheRock + this workflow's commit. - repository: ROCm/TheRock - ref: ${{ github.job_workflow_sha || github.sha }} + # Cross-repo callers pass ROCm/TheRock + ref; same-repo defaults to github.repository. + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} - name: Evaluate workflow results env: GITHUB_TOKEN: ${{ github.token }} From 864a16e48a6c383c36af8c1ebf93ef224a51cc20 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Wed, 23 Sep 2026 13:52:49 -0700 Subject: [PATCH 44/70] fix(ci): pass repository/ref to instrumented build calls --- .github/workflows/multi_arch_ci_coverage_linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 338f83b3e13..71eb04fc4e9 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -176,6 +176,9 @@ jobs: rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + # TheRock scripts for the instrumented build; cross-repo callers pass ROCm/TheRock + ref. + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} permissions: contents: read id-token: write @@ -208,6 +211,9 @@ jobs: rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci quartz_tracking_id: ${{ inputs.quartz_tracking_id }} + # TheRock scripts for the instrumented build; cross-repo callers pass ROCm/TheRock + ref. + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} permissions: contents: read id-token: write From 5085c48714ee10ab7687a5c2a9d7ae26c72a30e6 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Fri, 25 Sep 2026 10:13:49 -0700 Subject: [PATCH 45/70] refactor(ci): drop unreachable coverage overlay path --- .github/workflows/test_component.yml | 125 +------- .../overlay_coverage_artifacts.py | 252 ---------------- .../tests/overlay_coverage_artifacts_test.py | 270 ------------------ 3 files changed, 3 insertions(+), 644 deletions(-) delete mode 100644 build_tools/github_actions/overlay_coverage_artifacts.py delete mode 100644 build_tools/github_actions/tests/overlay_coverage_artifacts_test.py diff --git a/.github/workflows/test_component.yml b/.github/workflows/test_component.yml index 72210d33208..493a52345d8 100644 --- a/.github/workflows/test_component.yml +++ b/.github/workflows/test_component.yml @@ -35,56 +35,6 @@ on: description: "Build variant (e.g., release, asan). Used to skip tests incompatible with certain builds." type: string default: "release" - coverage_enabled: - description: >- - Collect llvm profraw files from the test run and upload them as a - workflow artifact. Only meaningful against a coverage instrumented - build. - type: boolean - default: false - coverage_artifact_prefix: - description: >- - Prefix for the uploaded profraw artifact name. A coverage run tests - several projects in one workflow run, so the caller sets this to - something project-specific to keep each project's profiles apart. - type: string - default: "coverage-profraw" - coverage_baseline_run_id: - description: >- - Regular nightly run holding the non-instrumented artifacts. When set, - the whole install tree comes from that run and only the project under - test is overlaid from this run's instrumented build, so its - dependencies stay out of the coverage report. Empty means the run's - own artifacts are already the ones to test. - type: string - default: "" - coverage_baseline_release_type: - description: >- - Release channel the baseline run published to. Artifacts are bucketed - by channel, so this has to match the run that produced them rather - than the coverage run reading them. - type: string - default: "nightly" - coverage_baseline_repository: - description: >- - Repository owning coverage_baseline_run_id, in 'owner/repo' form. - Needed when the baseline is a release nightly, which is published by - ROCm/rockrel rather than this repository. Empty uses this repository. - type: string - default: "" - coverage_artifact_names: - description: >- - Comma-separated artifact names holding the instrumented project - (e.g. 'rand'). Required alongside coverage_baseline_run_id. - type: string - default: "" - coverage_artifact_relpaths: - description: >- - Comma-separated subproject stage directories to overlay from those - artifacts (e.g. 'math-libs/hipRAND/stage'). TheRock's artifacts are - grouped, so this is what keeps sibling projects non-instrumented. - type: string - default: "" default_container_image: type: string default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98" @@ -134,23 +84,6 @@ jobs: QUARTZ_WORKFLOW_POSTFIX: "- test ROCm component" VENV_DIR: ${{ github.workspace }}/.venv ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" - # Nightly coverage installs the baseline run and overlays the - # instrumented project afterwards; every other run installs its own - # artifacts directly. The channel travels with the run id because - # artifacts are bucketed by channel. - INSTALL_ARTIFACT_RUN_ID: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_run_id || (inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id) }}" - INSTALL_RELEASE_TYPE: "${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_release_type || inputs.release_type }}" - # Instrumented libraries are built -O0 -g and counted on every branch, so - # the same suite takes far longer than the per-component budget assumes: - # rocRAND tests that normally run in milliseconds took 25 to 60 seconds - # each, and the shard hit its 15 minute wall a long way from finishing. - # - # A floor rather than a multiplier, because the budgets are not written to - # a common scale -- rocBLAS asks for 288 minutes, past the job's own cap, - # while rocRAND asks for 15 -- so multiplying stretches the ones that were - # already generous and still leaves the tight ones tight. Only the floor - # moves, and only for coverage. - TEST_TIMEOUT_MINUTES: "${{ (inputs.coverage_enabled && fromJSON(inputs.component).timeout_minutes < 60) && 60 || fromJSON(inputs.component).timeout_minutes }}" OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -208,33 +141,14 @@ jobs: STEP_TIMEOUT_MINUTES: ${{ inputs.build_variant == 'asan' && '60' || '15' }} uses: './.github/actions/setup_test_environment' with: - ARTIFACT_RUN_ID: ${{ env.INSTALL_ARTIFACT_RUN_ID }} - RUN_GITHUB_REPO: ${{ inputs.coverage_baseline_run_id != '' && inputs.coverage_baseline_repository || '' }} + ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }} # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved ARTIFACT_GROUP: ${{ inputs.amdgpu_families }} AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }} VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }} - RELEASE_TYPE: ${{ env.INSTALL_RELEASE_TYPE }} - - # The step above installed the baseline run, so every library here is - # non-instrumented. Replacing just this project's files leaves its - # dependencies out of the profiles the test run is about to write. - - name: Overlay instrumented coverage artifacts - if: ${{ inputs.coverage_baseline_run_id != '' }} - timeout-minutes: 15 - env: - GITHUB_TOKEN: ${{ github.token }} RELEASE_TYPE: ${{ inputs.release_type }} - run: | - python ./build_tools/github_actions/overlay_coverage_artifacts.py \ - --run-id="${{ github.run_id }}" \ - --amdgpu-families="${{ inputs.amdgpu_families }}" \ - --amdgpu-targets="${{ inputs.amdgpu_targets }}" \ - --artifact-names="${{ inputs.coverage_artifact_names }}" \ - --artifact-relpaths="${{ inputs.coverage_artifact_relpaths }}" \ - --install-dir="${{ env.OUTPUT_ARTIFACTS_DIR }}" # safe.directory must be set before Runner Health Status - name: Adjust git config @@ -278,26 +192,6 @@ jobs: --requirements-files "${REQUIREMENTS_FILES}" - # TODO(#3755): Clean this up after final sanitizer environment variable flags are determined - - name: Enable sanitizer-specific test flags - if: ${{ contains(inputs.artifact_group, 'asan') }} - run: | - echo "ASAN_OPTIONS=detect_odr_violation=0:quarantine_size_mb=600" >> $GITHUB_ENV - echo "HSA_XNACK=1" >> $GITHUB_ENV - echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV - - # Instrumented binaries write one profraw per process. %p (pid) and %m - # (binary signature) keep the files distinct so a shard that forks or - # loads several instrumented libraries does not overwrite itself. - - name: Prepare coverage profile directory - if: ${{ inputs.coverage_enabled }} - env: - TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }} - SHARD_INDEX: ${{ matrix.shard }} - run: | - mkdir -p "${GITHUB_WORKSPACE}/coverage-report/profraw" - echo "LLVM_PROFILE_FILE=${GITHUB_WORKSPACE}/coverage-report/profraw/${TEST_COMPONENT}-shard${SHARD_INDEX}-%p-%m.profraw" >> "${GITHUB_ENV}" - # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription. # Must be done at runtime since container env vars aren't in GitHub's env context. - name: Set thread limits @@ -313,8 +207,7 @@ jobs: - name: Test id: test - # fromJSON because env values are strings and this wants a number. - timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT_MINUTES) }} + timeout-minutes: ${{ fromJSON(inputs.component).timeout_minutes }} env: SHARD_INDEX: ${{ matrix.shard }} TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }} @@ -331,7 +224,7 @@ jobs: RETRY_THIS_STEP: "true" RETRY_COUNT: "1" RETRY_DELAY: "7" - STEP_TIMEOUT_MINUTES: ${{ env.TEST_TIMEOUT_MINUTES }} + STEP_TIMEOUT_MINUTES: ${{ fromJSON(inputs.component).timeout_minutes }} STEP_NAME: >- Test ${{ fromJSON(inputs.component).job_name }} (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }}) @@ -343,18 +236,6 @@ jobs: run: | # zizmor: ignore[template-injection] ${{ fromJSON(inputs.component).test_script }} - # Uploaded even when the test step failed: partial coverage from a failing - # shard is still useful, and the aggregation job tolerates missing shards. - - name: Upload coverage profraw files - if: ${{ always() && inputs.coverage_enabled }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: >- - ${{ inputs.coverage_artifact_prefix }}-${{ fromJSON(inputs.component).job_name }}-shard${{ matrix.shard }} - path: ${{ github.workspace }}/coverage-report/profraw/*.profraw - if-no-files-found: warn - retention-days: 3 - - name: Print test reproduction command if: ${{ failure() && steps.test.outcome == 'failure' }} env: diff --git a/build_tools/github_actions/overlay_coverage_artifacts.py b/build_tools/github_actions/overlay_coverage_artifacts.py deleted file mode 100644 index 0c67cf87963..00000000000 --- a/build_tools/github_actions/overlay_coverage_artifacts.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Overlays one project's instrumented binaries onto a baseline install tree. - -Nightly coverage builds an instrumented stack in its own workflow run while the -regular nightly keeps publishing the normal one, so the two live under separate -run ids. A coverage test job wants exactly one instrumented project and -non-instrumented everything else: an instrumented dependency would emit its own -profiles and move the coverage denominator around whenever that dependency -changes. - -The test job gets there in two steps. It first installs the whole baseline run -the usual way, then calls this script to fetch the instrumented artifact from -the coverage run and copy just the project under test over the baseline copy. - -TheRock's artifacts are grouped -- `rand` carries both rocRAND and hipRAND -- -so a whole artifact is too coarse to overlay. Each artifact keeps its files -under the subproject stage directory they were built in -(`math-libs/hipRAND/stage/...`), and that is what --artifact-relpaths names, so -sibling projects in the same artifact stay non-instrumented. -""" - -import argparse -import logging -import os -import shutil -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, os.fspath(Path(__file__).resolve().parents[1])) - -from artifact_manager import main as artifact_manager_main - -logging.basicConfig(level=logging.INFO, format="%(message)s") - -# Artifact components an instrumented project can contribute: `lib` the -# instrumented libraries, `test` the test binaries (instrumented in their own -# right for header-only projects), and `dev` the headers and CMake config that -# go with them. The rest only carry symbols and documentation. -EXCLUDED_COMPONENTS = ["run", "dbg", "doc"] - -# artifact_manager.py fetch explodes each archive into this subdirectory of its -# output directory, one directory per artifact. -FETCH_SUBDIR = "artifacts" - - -def split_list(raw: str) -> list[str]: - return [item.strip() for item in raw.split(",") if item.strip()] - - -def fetch_instrumented_artifacts( - *, - run_id: str, - artifact_names: list[str], - amdgpu_families: str, - amdgpu_targets: str, - output_dir: Path, - run_github_repo: str | None, -) -> Path: - """Downloads and explodes the named artifacts, returning their parent dir. - - Neither flattened nor bootstrapped: the overlay needs the per-subproject - directory layout that flattening would collapse. - """ - argv = [ - "fetch", - "--run-id", - run_id, - # These artifacts are outputs of the stage that built them rather than - # inbound to any stage, so the whole topology is the search space and - # --artifact-names does the narrowing. - "--stage", - "all", - "--artifact-names", - ",".join(artifact_names), - "--exclude-components", - ",".join(EXCLUDED_COMPONENTS), - "--amdgpu-families", - amdgpu_families, - "--output-dir", - os.fspath(output_dir), - ] - if amdgpu_targets: - argv.extend(["--amdgpu-targets", amdgpu_targets]) - if run_github_repo: - argv.extend(["--run-github-repo", run_github_repo]) - - logging.info("Fetching instrumented artifacts: %s", " ".join(argv)) - artifact_manager_main(argv) - return output_dir / FETCH_SUBDIR - - -def merge_tree(source: Path, dest: Path) -> None: - """Copies the `source` tree over `dest`, replacing entries already there. - - shutil.copytree(dirs_exist_ok=True) does not work for this. It recreates - symlinks with os.symlink(), which raises FileExistsError as soon as the - baseline install has already put the libfoo.so -> libfoo.so.1 chain in - place -- which it always has, since the overlay runs after a full install. - dirs_exist_ok only forgives collisions on directories. - - Directories are merged; everything else is replaced outright, so the - instrumented file wins over the baseline one. - """ - dest.mkdir(parents=True, exist_ok=True) - for entry in source.iterdir(): - target = dest / entry.name - if entry.is_symlink(): - # exists() is False for a dangling symlink, hence both checks. - if target.is_symlink() or target.exists(): - if target.is_dir() and not target.is_symlink(): - shutil.rmtree(target) - else: - target.unlink() - os.symlink(os.readlink(entry), target) - elif entry.is_dir(): - merge_tree(entry, target) - else: - # Unlink first: copy2 follows a symlink at the destination and - # would write through it, overwriting the baseline file it names - # while leaving the link itself behind. - if target.is_symlink(): - target.unlink() - shutil.copy2(entry, target) - - -def overlay_relpaths( - *, staging_dir: Path, relpaths: list[str], install_dir: Path -) -> int: - """Copies each artifact's subtree over the install tree. - - Returns the number of subtrees copied. Symlinks are preserved so the - libfoo.so -> libfoo.so.1 chain the loader follows still points at the - instrumented file. - """ - if not staging_dir.is_dir(): - return 0 - - copied = 0 - for artifact_dir in sorted(p for p in staging_dir.iterdir() if p.is_dir()): - for relpath in relpaths: - source = artifact_dir / relpath - if not source.is_dir(): - continue - logging.info("Overlaying %s onto %s", source, install_dir) - merge_tree(source, install_dir) - copied += 1 - return copied - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--run-id", - required=True, - help="Coverage workflow run that published the instrumented artifacts", - ) - parser.add_argument( - "--amdgpu-families", - required=True, - help="Semicolon-separated GPU families to fetch (e.g. 'gfx94X-dcgpu')", - ) - parser.add_argument( - "--amdgpu-targets", - default="", - help="Comma-separated individual GPU targets for split artifacts", - ) - parser.add_argument( - "--artifact-names", - required=True, - help="Comma-separated artifact names holding the instrumented project " - "(e.g. 'rand')", - ) - parser.add_argument( - "--artifact-relpaths", - required=True, - help="Comma-separated subproject stage directories to overlay " - "(e.g. 'math-libs/hipRAND/stage')", - ) - parser.add_argument( - "--install-dir", - type=Path, - required=True, - help="Baseline install tree to overlay the instrumented files onto", - ) - parser.add_argument( - "--staging-dir", - type=Path, - default=None, - help="Directory for the downloaded artifacts (default: a temporary directory)", - ) - parser.add_argument( - "--run-github-repo", - default=None, - help="GitHub repository for --run-id in 'owner/repo' format", - ) - args = parser.parse_args(argv) - - artifact_names = split_list(args.artifact_names) - relpaths = split_list(args.artifact_relpaths) - if not artifact_names or not relpaths: - logging.error( - "--artifact-names and --artifact-relpaths must both be non-empty." - ) - return 1 - - install_dir = args.install_dir.resolve() - if not install_dir.is_dir(): - logging.error( - "Install directory %s does not exist. The baseline artifacts must " - "be installed before the instrumented ones are overlaid.", - install_dir, - ) - return 1 - - with tempfile.TemporaryDirectory() as tmp: - staging_dir = args.staging_dir or Path(tmp) - staging_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir = fetch_instrumented_artifacts( - run_id=args.run_id, - artifact_names=artifact_names, - amdgpu_families=args.amdgpu_families, - amdgpu_targets=args.amdgpu_targets, - output_dir=staging_dir, - run_github_repo=args.run_github_repo, - ) - copied = overlay_relpaths( - staging_dir=artifacts_dir, - relpaths=relpaths, - install_dir=install_dir, - ) - - # Silently testing the baseline build would report coverage against - # binaries that were never instrumented, so treat this as fatal. - if not copied: - logging.error( - "None of the artifact relpaths (%s) were found in the artifacts " - "fetched from run %s. The instrumented build did not produce the " - "expected layout.", - ", ".join(relpaths), - args.run_id, - ) - return 1 - - logging.info("Overlaid %d instrumented subtree(s) onto %s", copied, install_dir) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py b/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py deleted file mode 100644 index 2d94a59c5f5..00000000000 --- a/build_tools/github_actions/tests/overlay_coverage_artifacts_test.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -import os -import platform -import sys -import tempfile -import unittest -from pathlib import Path -from unittest import mock - -# Add repo root to PYTHONPATH -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) - -import overlay_coverage_artifacts - - -def is_windows() -> bool: - return platform.system() == "Windows" - - -class TempDirTestBase(unittest.TestCase): - def setUp(self): - self._temp_dir = tempfile.TemporaryDirectory() - self.addCleanup(self._temp_dir.cleanup) - self.root = Path(self._temp_dir.name) - - def write(self, relative_path: str, contents: str) -> Path: - path = self.root / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(contents) - return path - - -class FetchInstrumentedArtifactsTest(TempDirTestBase): - def test_narrows_the_fetch_to_the_named_artifacts(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - artifacts_dir = overlay_coverage_artifacts.fetch_instrumented_artifacts( - run_id="99999", - artifact_names=["rand", "prim"], - amdgpu_families="gfx94X-dcgpu", - amdgpu_targets="", - output_dir=self.root / "staging", - run_github_repo=None, - ) - - (argv,) = fetch.call_args.args - self.assertEqual(argv[0], "fetch") - # These artifacts are stage outputs, not inbound to any stage. - self.assertEqual(argv[argv.index("--stage") + 1], "all") - self.assertEqual(argv[argv.index("--artifact-names") + 1], "rand,prim") - self.assertEqual(artifacts_dir, self.root / "staging" / "artifacts") - - def test_skips_components_that_cannot_be_instrumented(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - overlay_coverage_artifacts.fetch_instrumented_artifacts( - run_id="99999", - artifact_names=["rand"], - amdgpu_families="gfx94X-dcgpu", - amdgpu_targets="gfx942", - output_dir=self.root / "staging", - run_github_repo=None, - ) - - (argv,) = fetch.call_args.args - excluded = argv[argv.index("--exclude-components") + 1].split(",") - self.assertCountEqual(excluded, ["run", "dbg", "doc"]) - self.assertEqual(argv[argv.index("--amdgpu-targets") + 1], "gfx942") - - -class OverlayRelpathsTest(TempDirTestBase): - def _staged_artifact(self, artifact: str, relpath: str, files: dict[str, str]): - for name, contents in files.items(): - self.write(f"staging/{artifact}/{relpath}/{name}", contents) - - def test_copies_only_the_requested_relpath(self): - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/hipRAND/stage", - {"lib/libhiprand.so": "instrumented"}, - ) - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/rocRAND/stage", - {"lib/librocrand.so": "instrumented"}, - ) - self.write("install/lib/libhiprand.so", "baseline") - self.write("install/lib/librocrand.so", "baseline") - - copied = overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertEqual(copied, 1) - self.assertEqual( - (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" - ) - # A sibling project in the same grouped artifact stays as it was. - self.assertEqual( - (self.root / "install/lib/librocrand.so").read_text(), "baseline" - ) - - def test_merges_into_the_existing_tree(self): - self._staged_artifact( - "rand_lib_gfx942", - "math-libs/hipRAND/stage", - {"lib/libhiprand.so": "instrumented"}, - ) - self.write("install/lib/libamdhip64.so", "baseline") - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertTrue((self.root / "install/lib/libamdhip64.so").exists()) - self.assertTrue((self.root / "install/lib/libhiprand.so").exists()) - - def test_counts_every_matching_artifact(self): - for component in ("lib", "test"): - self._staged_artifact( - f"rand_{component}_gfx942", - "math-libs/hipRAND/stage", - {f"{component}/file": component}, - ) - (self.root / "install").mkdir() - - self.assertEqual( - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ), - 2, - ) - - def test_reports_nothing_copied_when_the_relpath_is_absent(self): - self._staged_artifact( - "rand_lib_gfx942", "math-libs/rocRAND/stage", {"lib/librocrand.so": "x"} - ) - (self.root / "install").mkdir() - - self.assertEqual( - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ), - 0, - ) - - @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") - def test_preserves_symlinks(self): - # The loader follows libfoo.so -> libfoo.so.1; copying the link as a - # regular file would leave a stale non-instrumented copy behind it. - stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" - self.write(f"{stage}/libhiprand.so.1", "instrumented") - (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") - (self.root / "install").mkdir() - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - self.assertTrue((self.root / "install/lib/libhiprand.so").is_symlink()) - - @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") - def test_replaces_a_symlink_the_baseline_already_installed(self): - # The overlay always runs after a full baseline install, so the whole - # symlink chain is already on disk. shutil.copytree used to fail here - # with "File exists: 'libhiprand.so.1' -> '.../libhiprand.so'" -- - # dirs_exist_ok forgives directory collisions, not symlink ones. - stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" - self.write(f"{stage}/libhiprand.so.1", "instrumented") - (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") - self.write("install/lib/libhiprand.so.1", "baseline") - (self.root / "install/lib/libhiprand.so").symlink_to("libhiprand.so.1") - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - link = self.root / "install/lib/libhiprand.so" - self.assertTrue(link.is_symlink()) - self.assertEqual(link.read_text(), "instrumented") - - @unittest.skipIf(is_windows(), "symlinks require elevation on Windows") - def test_replaces_a_baseline_regular_file_with_a_symlink(self): - # Some baselines ship the resolved file where the instrumented artifact - # ships a link. Leaving the file in place would keep the loader on - # non-instrumented code. - stage = "staging/rand_lib_gfx942/math-libs/hipRAND/stage/lib" - self.write(f"{stage}/libhiprand.so.1", "instrumented") - (self.root / stage / "libhiprand.so").symlink_to("libhiprand.so.1") - self.write("install/lib/libhiprand.so", "baseline") - - overlay_coverage_artifacts.overlay_relpaths( - staging_dir=self.root / "staging", - relpaths=["math-libs/hipRAND/stage"], - install_dir=self.root / "install", - ) - - link = self.root / "install/lib/libhiprand.so" - self.assertTrue(link.is_symlink()) - self.assertEqual(link.read_text(), "instrumented") - - -class MainTest(TempDirTestBase): - def _run(self, **overrides) -> int: - args = { - "--run-id": "99999", - "--amdgpu-families": "gfx94X-dcgpu", - "--artifact-names": "rand", - "--artifact-relpaths": "math-libs/hipRAND/stage", - "--install-dir": os.fspath(self.root / "install"), - "--staging-dir": os.fspath(self.root / "staging"), - } - args.update(overrides) - argv = [item for pair in args.items() for item in pair] - return overlay_coverage_artifacts.main(argv) - - def test_fails_when_the_install_tree_is_missing(self): - with mock.patch.object( - overlay_coverage_artifacts, "artifact_manager_main" - ) as fetch: - self.assertEqual(self._run(), 1) - fetch.assert_not_called() - - def test_fails_when_nothing_was_overlaid(self): - # A silent pass here would report coverage for binaries that were never - # instrumented. - (self.root / "install").mkdir() - with mock.patch.object(overlay_coverage_artifacts, "artifact_manager_main"): - self.assertEqual(self._run(), 1) - - def test_fetches_and_overlays(self): - (self.root / "install").mkdir() - - def fake_fetch(argv): - self.write( - "staging/artifacts/rand_lib_gfx942/math-libs/hipRAND/stage" - "/lib/libhiprand.so", - "instrumented", - ) - - with mock.patch.object( - overlay_coverage_artifacts, - "artifact_manager_main", - side_effect=fake_fetch, - ): - self.assertEqual(self._run(), 0) - - self.assertEqual( - (self.root / "install/lib/libhiprand.so").read_text(), "instrumented" - ) - - -if __name__ == "__main__": - unittest.main() From 7be1cf8585e0bc14503bdc88afaecfbf70e347a6 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Fri, 25 Sep 2026 10:17:22 -0700 Subject: [PATCH 46/70] fix(ci): restore additional_requirements_files default, harden coverage guard --- .github/workflows/test_code_coverage_component.yml | 4 +++- build_tools/github_actions/fetch_test_configurations.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index a29da393c0f..6e9936a7e83 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -192,7 +192,9 @@ jobs: python ./build_tools/print_driver_gpu_info.py - name: Setup Additional Requirements - if: ${{ fromJSON(inputs.component).additional_requirements_files != '' }} + # First-element check skips cleanly when the list is empty ([]) or the + # key is absent, without depending on how an empty list coerces to ''. + if: ${{ fromJSON(inputs.component).additional_requirements_files[0] != '' }} run: | python ./build_tools/install_additional_requirements.py \ --requirements-files ${{ join(fromJSON(inputs.component).additional_requirements_files, ',') }} diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index e7411da4135..7d538545606 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -149,9 +149,10 @@ def _family_matches( ) -# Common settings applied to all jobs. Kept empty so components omit keys they -# do not set (an empty additional_requirements_files trips the `!= ''` guard). -_common_settings = {} +# Common settings applied to all jobs +_common_settings = { + "additional_requirements_files": [], +} # Common settings for rocgdb jobs _rocgdb_common = { From df04afe64a4700f86318b7c68b17e39439469427 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Fri, 25 Sep 2026 10:19:27 -0700 Subject: [PATCH 47/70] chore(ci): remove temporary coverage PR-check scaffolding --- .../workflows/coverage_nightly_pr_check.yml | 95 ------------------- .../multi_arch_ci_coverage_linux.yml | 47 --------- 2 files changed, 142 deletions(-) delete mode 100644 .github/workflows/coverage_nightly_pr_check.yml diff --git a/.github/workflows/coverage_nightly_pr_check.yml b/.github/workflows/coverage_nightly_pr_check.yml deleted file mode 100644 index 7afe1691779..00000000000 --- a/.github/workflows/coverage_nightly_pr_check.yml +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# TEMPORARY. Delete this file and the workflow_call trigger before merging. -# -# workflow_dispatch only appears once the file is on the default branch, so it -# can't trigger from a branch. A caller can: paths resolve against the caller's -# commit, so this runs the branch copy as a PR check. -# -# Presubmit coverage is too expensive for every PR (multi-hour instrumented -# build + full test suite); coverage is a separate manual workflow by design. - -name: Coverage Nightly PR Check - -on: - pull_request: - # Narrow on purpose: each run is a multi-hour build, so only files carrying - # coverage logic are worth one. - paths: - - ".github/workflows/coverage_nightly_pr_check.yml" - - ".github/workflows/multi_arch_ci_coverage_linux.yml" - - ".github/workflows/multi_arch_ci_coverage_report.yml" - - ".github/workflows/test_code_coverage_component.yml" - - "build_tools/github_actions/configure_coverage_ci.py" - - "build_tools/github_actions/merge_coverage_report.py" - - "build_tools/install_rocm_code_coverage_build.py" - - "cmake/therock_subproject.cmake" - - "CMakeLists.txt" - -concurrency: - # A later push supersedes this run: queueing multi-hour builds to report on - # already-replaced code just occupies nodes the rest of CI needs. - group: coverage-nightly-pr-check-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - coverage: - name: Coverage - uses: ./.github/workflows/multi_arch_ci_coverage_linux.yml - secrets: inherit - # A called workflow can't hold more than its caller, so grant these here too: - # id-token for the AWS role the artifact fetches assume, actions:read for the - # baseline run lookup. - permissions: - contents: read - actions: read - id-token: write - with: - # Pinned baseline: a report change must reflect a code change, not a baseline drift. - # - # ROCm/rockrel run 35670146294, "Multi-Arch Release (nightly)" on main, - # 2026-09-22, succeeded. A real nightly, not a CI run, since that's what a - # dispatched coverage run gets in practice. - # - # A rockrel run is safe here. Its artifacts embed their build workspace - # (/__w/rockrel/rockrel), which breaks consumers that configure CMake - # against them -- but this workflow never does: it builds the instrumented - # stack from source and only installs the baseline on the test runner, - # where prebuilt binaries run and no package config is read. - # - # Verify before pinning that this run publishes artifacts for every - # selected project: blas, fft, hipdnn, rand, solver, sparse. Not all are - # per-target (hipdnn and solver's tests are generic), so checking only for - # _gfx942 understates what's there. - baseline_run_id: "35670146294" - # The bucket lookup queries the run via the GitHub API, so it needs the - # owning repo; ROCm/TheRock 404s for a rockrel id. - baseline_repository: "ROCm/rockrel" - # That run published to therock-nightly-artifacts (the default); naming it - # keeps the pin and channel together. - baseline_release_type: "nightly" - amdgpu_families: "gfx94X-dcgpu" - # Every rocm-libraries project that builds a shared library. Run - # 34135220913 proved the path end to end on hipRAND (build, install, - # overlay, tests, 77% report) once libclang_rt.profile_rocm.a's own deps - # (-ldl, -lpthread) were added to the link line. Run 34164420766 then built - # all twelve green and reported six. - # - # Named explicitly, not rocm_libraries_all, to leave out the four - # header-only projects (hipCUB, rocPRIM, rocThrust, rocWMMA): they have no - # library to instrument, report against instrumented test binaries, and hit - # the linkonce_odr case the overlay can't isolate. Worth doing separately. - # - # rccl and rocshmem aren't selectable: they live in comm-libs, which has no - # build job here. Both are also blocked upstream -- rccl's CMake refuses - # coverage outside Debug, and rocshmem fails on coverage + HIP RDC. - # - # hipBLASLt and hipTensor are absent because neither instrumented build - # links (see blocked_reason in configure_coverage_ci.py). hipTensor was - # still listed for run 34201973090, which found its failure: the registry - # corrections instrumented it for the first time, and a stage builds as a - # unit, so its link error failed math-libs and skipped all twelve reports. - projects_to_test: >- - hipblas,hipdnn - test_type: "standard" diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 71eb04fc4e9..3646990856c 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -60,53 +60,6 @@ on: description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." default: "" - # TEMPORARY, paired with coverage_nightly_pr_check.yml. - # workflow_dispatch requires the file to be on the default branch; a caller - # has no such restriction. Actions doesn't expand YAML anchors, so inputs are - # repeated. Remove this trigger with the check workflow once the run is green. - workflow_call: - inputs: - baseline_run_id: - type: string - description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." - default: "" - baseline_release_type: - type: string - description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." - default: "nightly" - baseline_repository: - type: string - description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." - default: "" - projects_to_test: - type: string - description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem require a comm-libs build job (not present), so rocm_systems_all and all are rejected." - default: "" - amdgpu_families: - type: string - description: "Comma-separated GPU families to collect coverage on." - default: "gfx94X-dcgpu" - coverage_config_source: - type: string - description: "Repository and ref holding the per-project coverage metadata." - default: "ROCm/rocm-libraries@main" - test_type: - type: string - description: "Test type to run against the instrumented build." - default: "standard" - quartz_tracking_id: - type: string - description: "Leave empty. Reserved for release tracking." - default: "" - repository: - type: string - description: "Repository holding this workflow's scripts. Empty defaults to github.repository; cross-repo callers pass ROCm/TheRock." - default: "" - ref: - type: string - description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." - default: "" - permissions: contents: read From 30d41caf2691ce0b570c046de3dd6d714406f0fa Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Fri, 25 Sep 2026 10:35:10 -0700 Subject: [PATCH 48/70] fix(ci): restore workflow_call trigger needed by cross-repo caller --- .../multi_arch_ci_coverage_linux.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 3646990856c..c3f9cc1303b 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -60,6 +60,50 @@ on: description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." default: "" + # Required so cross-repo callers (rocm-libraries nightly) can invoke via `uses:`. + workflow_call: + inputs: + baseline_run_id: + type: string + description: "Regular nightly run id supplying the non-instrumented dependencies. Empty tests this run's own instrumented stack end to end." + default: "" + baseline_release_type: + type: string + description: "Release channel baseline_run_id published to; artifacts are bucketed by channel." + default: "nightly" + baseline_repository: + type: string + description: "Repository owning baseline_run_id (owner/repo). Release nightlies come from ROCm/rockrel; empty means this repository." + default: "" + projects_to_test: + type: string + description: "Comma-separated coverage projects (e.g. hiprand), case-insensitive. Alias: rocm_libraries_all. Empty = every buildable project. rccl/rocshmem require a comm-libs build job (not present), so rocm_systems_all and all are rejected." + default: "" + amdgpu_families: + type: string + description: "Comma-separated GPU families to collect coverage on." + default: "gfx94X-dcgpu" + coverage_config_source: + type: string + description: "Repository and ref holding the per-project coverage metadata." + default: "ROCm/rocm-libraries@main" + test_type: + type: string + description: "Test type to run against the instrumented build." + default: "standard" + quartz_tracking_id: + type: string + description: "Leave empty. Reserved for release tracking." + default: "" + repository: + type: string + description: "Repository holding this workflow's scripts. Empty defaults to github.repository; cross-repo callers pass ROCm/TheRock." + default: "" + ref: + type: string + description: "Ref/SHA of this workflow's scripts. Cross-repo callers pass the ref they pinned in uses:." + default: "" + permissions: contents: read From e1100108fce139099e5355d8d18878654fe22cf0 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Fri, 25 Sep 2026 12:53:37 -0700 Subject: [PATCH 49/70] docs(ci): drop hand-copied coverage trigger snippet; fix stale dispatch wording --- .../multi_arch_ci_coverage_linux.yml | 7 +- docs/development/code_coverage.md | 2 +- .../coverage_nightly_trigger_snippet.md | 146 ------------------ 3 files changed, 4 insertions(+), 151 deletions(-) delete mode 100644 docs/development/coverage_nightly_trigger_snippet.md diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index c3f9cc1303b..e1afdce9c9e 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -7,10 +7,9 @@ # builds one instrumented Linux stack for every coverage-enabled project, then # fans out to multi_arch_ci_coverage_report.yml once per project. # -# Mirrors the nightly pattern: the nightly dispatches multi_arch_ci_linux.yml -# directly; trigger_coverage_linux in the nightly dispatches this workflow -# directly. A future Windows mirror copies this file, swapping the build -# reusable and runners; multi_arch_ci_coverage_report.yml is reused unchanged. +# The rocm-libraries nightly invokes this via a cross-repo reusable `uses:` call. +# A future Windows mirror copies this file, swapping the build reusable and +# runners; multi_arch_ci_coverage_report.yml is reused unchanged. # # Currently targets gfx94X-dcgpu; change-based selection and multi-arch come later. diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 087672bdb2a..a810b159fc8 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -122,7 +122,7 @@ component, object globs, and Codecov flag for every onboarded project. ### Dispatching a run Start from the Actions tab or `gh` CLI, or let the rocm-libraries nightly -dispatch it via a GitHub App token. There is no cron trigger. +call it via a cross-repo reusable `uses:` workflow call. There is no cron trigger. Pass a recent nightly run id as `baseline_run_id` (keep `baseline_release_type` at `nightly`) and set `projects_to_test`; other inputs use sensible defaults. diff --git a/docs/development/coverage_nightly_trigger_snippet.md b/docs/development/coverage_nightly_trigger_snippet.md deleted file mode 100644 index 08b46c0d07c..00000000000 --- a/docs/development/coverage_nightly_trigger_snippet.md +++ /dev/null @@ -1,146 +0,0 @@ -# Reference: `trigger_coverage_linux` / `trigger_coverage_windows` jobs for the rocm-libraries nightly - -Snippet for **`ROCm/rocm-libraries`** -`.github/workflows/therock-multi-arch-ci-nightly.yml`. Copy here when wiring -the real driver; TheRock cannot edit that repo. - -The nightly build is non-instrumented and can only be a baseline. Coverage must -run as a **separate** GitHub Actions run (its own `run_id`) so the generic and -instrumented artifact sets stay distinct. Dispatching per-platform as each build -finishes avoids cross-platform waits and matches the nightly's own pattern. - -Auth uses a **GitHub App token** (`actions/create-github-app-token`), modelled -on `ROCm/rocm-systems/.github/workflows/rocm_ci_caller.yml`. No classic PAT. - -> **One-time org setup:** a GitHub App reachable from rocm-libraries must be -> installed on **ROCm/TheRock** with **`actions: write`**. - -## `workflow_dispatch` inputs (add to the nightly's `on:` block) - -```yaml -on: - workflow_dispatch: - inputs: - # ... existing inputs ... - enable_coverage_linux: - description: >- - Dispatch the ROCm/TheRock Linux code-coverage orchestrator after the - Linux build and test jobs succeed. Nightly-only: a coverage run is a - multi-hour instrumented build, unfit for presubmit. - type: boolean - default: false - enable_coverage_windows: - description: >- - Dispatch the ROCm/TheRock Windows code-coverage orchestrator after the - Windows build and test jobs succeed. - type: boolean - default: false -``` - -## `trigger_coverage_linux` job - -```yaml -jobs: - # ... existing setup, linux_build_and_test, etc. ... - - trigger_coverage_linux: - name: Trigger TheRock Linux Coverage - needs: [setup, linux_build_and_test] - # Gate on the switch, on there being a build to baseline against, and on the - # Linux test job having succeeded. - if: >- - ${{ - inputs.enable_coverage_linux && - needs.setup.outputs.enable_build_jobs == 'true' && - needs.linux_build_and_test.result == 'success' - }} - runs-on: ubuntu-24.04 - steps: - - name: Generate GitHub App token - id: generate-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.THEROCK_COVERAGE_APP_ID }} - private-key: ${{ secrets.THEROCK_COVERAGE_APP_PRIVATE_KEY }} - owner: "ROCm" - repositories: "TheRock" - - - name: Dispatch TheRock Linux coverage orchestrator - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v8.0.0 - with: - github-token: ${{ steps.generate-token.outputs.token }} - script: | - await github.rest.actions.createWorkflowDispatch({ - owner: 'ROCm', - repo: 'TheRock', - workflow_id: 'multi_arch_ci_coverage_linux.yml', - // Pin to a known-good TheRock ref; bump when the coverage - // workflows change in a way this nightly depends on. - ref: 'main', - inputs: { - // The nightly's own run supplies the non-instrumented baseline. - baseline_run_id: String(context.runId), - baseline_repository: 'ROCm/rocm-libraries', - // Match the channel this nightly publishes to. - baseline_release_type: 'nightly', - // Records the exact rocm-libraries commit that drove coverage. - coverage_config_source: '${{ github.repository }}@${{ github.sha }}', - projects_to_test: 'hipblas,hipdnn,hipfft,hiprand,hipsolver,hipsparselt,rocblas,rocfft,rocrand,rocsolver,rocsparse', - amdgpu_families: 'gfx94X-dcgpu', - }, - }); -``` - -## `trigger_coverage_windows` job (add when Windows coverage is ready) - -```yaml - trigger_coverage_windows: - name: Trigger TheRock Windows Coverage - needs: [setup, windows_build_and_test] - if: >- - ${{ - inputs.enable_coverage_windows && - needs.setup.outputs.enable_build_jobs == 'true' && - needs.windows_build_and_test.result == 'success' - }} - runs-on: ubuntu-24.04 - steps: - - name: Generate GitHub App token - id: generate-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.THEROCK_COVERAGE_APP_ID }} - private-key: ${{ secrets.THEROCK_COVERAGE_APP_PRIVATE_KEY }} - owner: "ROCm" - repositories: "TheRock" - - - name: Dispatch TheRock Windows coverage orchestrator - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v8.0.0 - with: - github-token: ${{ steps.generate-token.outputs.token }} - script: | - await github.rest.actions.createWorkflowDispatch({ - owner: 'ROCm', - repo: 'TheRock', - workflow_id: 'multi_arch_ci_coverage_windows.yml', - ref: 'main', - inputs: { - baseline_run_id: String(context.runId), - baseline_repository: 'ROCm/rocm-libraries', - baseline_release_type: 'nightly', - coverage_config_source: '${{ github.repository }}@${{ github.sha }}', - projects_to_test: '...', - amdgpu_families: '...', - }, - }); -``` - -Notes: - -- `baseline_run_id` must be a string (`String(context.runId)`). -- Both jobs share the same `baseline_run_id` but dispatch to different workflow - files and `amdgpu_families`. -- `ref` pins the TheRock workflow copy; update it when the coverage workflows - change in a breaking way. -- `baseline_release_type` and `projects_to_test` should match what the nightly - built. From 2c0c64056115ee565edf73f9b52743c34295d1e6 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 29 Sep 2026 04:29:04 -0500 Subject: [PATCH 50/70] docs(coverage): correct stale coverage CI descriptions - ci_overview.md: point at multi_arch_ci_coverage_linux.yml, the actual entry point; multi_arch_ci_coverage.yml does not exist. Note that an empty baseline_run_id also needs baseline_release_type=ci, since the run publishes to the ci bucket. - code_coverage.md: fix the local ctest path (build/math-libs/hipRAND/build). - code_coverage.md: drop the 60-minute test timeout floor, which is not implemented; coverage runs use each component's normal timeout. Scope the -O0 -g note to the upstream options that set it (hipRAND). - code_coverage.md: describe the swap as it works: the whole artifact is downloaded and only COMPONENT_MAP's library folder is extracted. A swap that matches nothing does not fail the test job; the report job fails on finding no profraw files. Co-authored-by: Cursor --- docs/development/ci_overview.md | 5 ++--- docs/development/code_coverage.md | 26 ++++++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index d01fdda0138..5d9b9e0b177 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -53,9 +53,8 @@ and upload steps inside a WSL Ubuntu shell. See Code coverage has its own entry point, because the projects it measures have to be rebuilt with instrumentation before their tests can be run: -- [`.github/workflows/multi_arch_ci_coverage.yml`](/.github/workflows/multi_arch_ci_coverage.yml) - thin top-level orchestrator (`setup_coverage_matrix`, `coverage_linux`, `ci_summary`). Dispatched by hand or by the nightly: pass the run id of a recent nightly as `baseline_run_id` to take the non-instrumented dependencies from it, or leave it empty to build the whole stack within the run - - [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - the per-platform build + fan-out layer: builds the instrumented compiler-runtime and math-libs, then fans out over projects (a future `multi_arch_ci_coverage_windows.yml` will mirror it) - - [`.github/workflows/multi_arch_ci_coverage_report.yml`](/.github/workflows/multi_arch_ci_coverage_report.yml) - runs one project's tests on one GPU family, then turns the collected profiles into an lcov file, a text summary and a browsable HTML report +- [`.github/workflows/multi_arch_ci_coverage_linux.yml`](/.github/workflows/multi_arch_ci_coverage_linux.yml) - the entry point (`setup_coverage_matrix`, the compiler-runtime and math-libs builds, `coverage_report`, `ci_summary`). Dispatched by hand, or called from another repository through `workflow_call`: pass the run id of a recent nightly as `baseline_run_id` to take the non-instrumented dependencies from it, or leave it empty and set `baseline_release_type` to `ci` to test this run's own instrumented stack. A future `multi_arch_ci_coverage_windows.yml` will mirror it + - [`.github/workflows/multi_arch_ci_coverage_report.yml`](/.github/workflows/multi_arch_ci_coverage_report.yml) - runs one project's tests on one GPU family, then turns the collected profiles into an lcov file, a text summary and a browsable HTML report See [Code Coverage](code_coverage.md) for the CMake options and how to onboard a project, and [Coverage CI](code_coverage.md#coverage-ci) for the end-to-end diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index a810b159fc8..3b34d0aab68 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -86,7 +86,7 @@ distinct, run the tests, then call `merge_coverage_report.py`: ```bash export LLVM_PROFILE_FILE="$PWD/coverage-report/profraw/%p-%m.profraw" -ctest --test-dir build/math-libs/hiprand +ctest --test-dir build/math-libs/hipRAND/build python build_tools/github_actions/merge_coverage_report.py \ --profraw-dir coverage-report/profraw \ @@ -162,10 +162,11 @@ Per shard, `test_code_coverage_component.yml` does three things: 1. **Run tests and upload profraw** under `always()` — a failing shard still exercised code. -`coverage_enabled` raises the test timeout to a 60-minute floor. Instrumented -libraries are `-O0 -g`; rocRAND tests that finish in milliseconds normally took -25–60 seconds each. A floor rather than a multiplier avoids over-extending -budgets that are already generous (rocBLAS: 288 min). +Coverage runs use each component's normal `timeout_minutes` from +`fetch_test_configurations.py`; nothing extends it for instrumented builds. +Instrumented tests run slower, and some upstream coverage options also lower the +optimization level (hipRAND's `BUILD_CODE_COVERAGE` adds `-O0 -g`), so a +component close to its limit may time out. #### Why there are two run ids @@ -175,11 +176,16 @@ its own profiles alongside hipRAND's, shifting hipRAND's numbers on unrelated changes. The baseline-install-then-overlay isolates exactly one project. The baseline is read from `baseline_release_type` (normally `nightly`), not the -coverage run's `ci` channel — artifacts are bucketed per channel. The overlay -targets `math-libs/hipRAND/stage` rather than the whole `rand` artifact (which -also contains rocRAND). The two runs write to separate S3 paths; no files -collide. An empty overlay fails the job rather than reporting coverage for -uninstrumented binaries. +coverage run's `ci` channel — artifacts are bucketed per channel. The swap +downloads the project's whole artifact (`rand`, which also contains rocRAND) +but extracts only the paths matching the project's library folder in +`COMPONENT_MAP` (`hipRAND`). The two runs write to separate S3 paths; no files +collide. + +A missing instrumented artifact fails the install step. A swap that matches no +files does not: the installer logs `Replaced 0` and the tests run against the +baseline's uninstrumented binaries. The failure then surfaces in the report +job, which finds no profraw files. #### What scopes a report to one project From 8f1bbb0c60f4a0889e7d356046903fc46a28ad31 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Tue, 29 Sep 2026 04:38:31 -0500 Subject: [PATCH 51/70] ci(coverage): rename build_instrumented_compiler_runtime to build_compiler_runtime The job gets no coverage_cmake_options, and no compiler-runtime project has a supported coverage option, so it is a regular build. Naming it "instrumented" suggested otherwise. code_coverage.md: follow the rename, and correct two claims next to it. coverage_cmake_options reaches the math-libs configure line only, not every stage's, and a whole-group selection in CI collapses to that group's option (THEROCK_COVERAGE_ROCM_LIBRARIES_ALL for the default), never THEROCK_COVERAGE_ALL, since rocm-systems selections are rejected. Co-authored-by: Cursor --- .../workflows/multi_arch_ci_coverage_linux.yml | 10 +++++----- docs/development/code_coverage.md | 16 +++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index e1afdce9c9e..7b8398d42df 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -158,8 +158,8 @@ jobs: # Built first so later stages have a compiler-runtime to link against and # fetch as an inbound artifact. - build_instrumented_compiler_runtime: - name: Build Instrumented Compiler Runtime + build_compiler_runtime: + name: Build Compiler Runtime needs: setup_coverage_matrix if: ${{ needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' }} uses: ./.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -185,7 +185,7 @@ jobs: # group, else individual project flags; unselected projects build normally. build_instrumented_math_libs: name: Build Instrumented Math Libs - needs: [setup_coverage_matrix, build_instrumented_compiler_runtime] + needs: [setup_coverage_matrix, build_compiler_runtime] if: >- ${{ !failure() && !cancelled() && @@ -224,7 +224,7 @@ jobs: needs: [ setup_coverage_matrix, - build_instrumented_compiler_runtime, + build_compiler_runtime, build_instrumented_math_libs, ] # Also guard the empty matrix: fromJSON('') would hard-error. @@ -264,7 +264,7 @@ jobs: if: always() needs: - setup_coverage_matrix - - build_instrumented_compiler_runtime + - build_compiler_runtime - build_instrumented_math_libs - coverage_report runs-on: ubuntu-24.04 diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 3b34d0aab68..d38d7dd7bcd 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -107,7 +107,7 @@ build. ```mermaid graph TD dispatch[Dispatch with projects_to_test] --> matrix[setup_coverage_matrix] - matrix --> compilerRuntime[Build instrumented compiler-runtime] + matrix --> compilerRuntime[Build compiler-runtime] compilerRuntime --> mathLibs[Build instrumented math-libs] mathLibs --> report[Per project: configure, test, report] report --> codecov[Codecov] @@ -135,13 +135,15 @@ nightly dispatch is a follow-up; `baseline_run_id` is already the right input. `coverage_cmake_options`, and `needs_math_libs`. Selections naming a project whose stage has no build job fail immediately. -`build_instrumented_compiler_runtime` always runs first — every other stage -pulls its inbound artifacts from it. `build_instrumented_math_libs` fans out -over GPU families and is skipped when `needs_math_libs` is false; 17 of the 19 -measurable projects live there. +`build_compiler_runtime` always runs first — every other stage pulls its +inbound artifacts from it. It is a regular, non-instrumented build: no +compiler-runtime project has a supported coverage option. +`build_instrumented_math_libs` fans out over GPU families and is skipped when +`needs_math_libs` is false; 17 of the 19 measurable projects live there. -`coverage_cmake_options` is appended to each stage's configure line. A -full-group selection collapses to `-DTHEROCK_COVERAGE_ALL=ON`; a narrow one +`coverage_cmake_options` is appended to the math-libs configure line only. A +selection covering a whole group collapses to that group's option (the default +selection becomes `-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON`); a narrow one names each project. `CMakeLists.txt` expands group options to `_ENABLE_COVERAGE` flags; `therock_subproject.cmake` translates each to the project's upstream name. Everything not selected builds normally. From 8523c5aeaee184746722bb0c0895df0901065fbf Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 5 Oct 2026 02:09:23 -0500 Subject: [PATCH 52/70] fix(coverage): keep coverage builds off ccache and in their own S3 namespace Fixes the failing Linux CodeCoverage jobs in the rocm-libraries nightly (run 36913152797): - Instrumented subprojects no longer compile through ccache. ccache drops every -Xarch_ pair, which removed the device-side negation (kernels were instrumented, and their colon-named profiles broke the profraw upload) and rocRAND's and hipRAND's -Xarch_host coverage flags (those libraries were not instrumented at all). - The instrumented stack publishes under -coverage. The nightly's regular build shares the workflow run, and the two overwrote each other's math-libs artifacts. WorkflowOutputRoot looks a suffixed run id up by its numeric part. - external_repo_config is forwarded to the coverage builds, which otherwise measured TheRock's pinned submodules rather than the commit under test. - rocWMMA's coverage option is ROCWMMA_CODE_COVERAGE. rocPRIM, hipCUB and rocThrust report on their own test binaries; a "!" glob excludes a sibling project's. - Coverage test timeouts are 4x the release ones, capped at 180 minutes. - Device-side profiles are dropped, with a warning, before upload. - merge_coverage_report.py fails a report whose profiles hold no counters, whose objects carry no coverage mapping, or in which no line ran, instead of publishing 0%. Co-authored-by: Cursor --- ...ti_arch_build_portable_linux_artifacts.yml | 15 +- .../multi_arch_ci_coverage_linux.yml | 18 +++ .../multi_arch_ci_coverage_report.yml | 9 +- .../test_code_coverage_component.yml | 25 +++- .../_therock_utils/workflow_outputs.py | 12 +- .../github_actions/configure_coverage_ci.py | 16 ++- .../fetch_test_configurations.py | 23 +++ .../github_actions/merge_coverage_report.py | 98 +++++++++++-- .../tests/configure_coverage_ci_test.py | 25 ++++ .../tests/fetch_test_configurations_test.py | 36 +++++ .../tests/merge_coverage_report_test.py | 132 +++++++++++++++++- build_tools/tests/workflow_outputs_test.py | 21 +++ cmake/therock_subproject.cmake | 13 ++ docs/development/code_coverage.md | 74 +++++++--- docs/development/workflow_outputs.md | 16 ++- 15 files changed, 485 insertions(+), 48 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 9e177204295..2b0078b0bfb 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -80,6 +80,14 @@ on: description: "Comma-separated artifact names to build (granular reuse). Empty = build all artifacts in stage." type: string default: "" + artifact_run_id: + description: >- + Run id to fetch inbound artifacts from and push this stage's artifacts + and logs under. Empty uses github.run_id. A variant build sharing a + workflow run with a regular one (coverage) passes its own id, since the + two produce artifacts with the same names. + type: string + default: "" quartz_tracking_id: type: string default: "" @@ -107,6 +115,7 @@ jobs: ROCM_PACKAGE_VERSION: ${{ inputs.rocm_package_version }} RELEASE_TYPE: ${{ inputs.release_type }} REBUILD_ARTIFACTS: ${{ inputs.rebuild_artifacts }} + ARTIFACT_RUN_ID: ${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }} STAGE_NAME: ${{ inputs.stage_name }} CCACHE_CONFIGPATH: ${{ github.workspace }}/.ccache/ccache.conf TEATIME_FORCE_INTERACTIVE: 0 @@ -169,7 +178,7 @@ jobs: - name: Fetch inbound artifacts run: | python build_tools/artifact_manager.py fetch \ - --run-id=${{ github.run_id }} \ + --run-id="${ARTIFACT_RUN_ID}" \ --stage="${STAGE_NAME}" \ --amdgpu-families="${AMDGPU_FAMILIES}" \ --output-dir="${BUILD_DIR}" \ @@ -272,7 +281,7 @@ jobs: - name: Push stage artifacts run: | python build_tools/artifact_manager.py push \ - --run-id ${{ github.run_id }} \ + --run-id "${ARTIFACT_RUN_ID}" \ --stage="${STAGE_NAME}" \ --build-dir="${BUILD_DIR}" @@ -280,7 +289,7 @@ jobs: if: ${{ always() }} run: | python build_tools/github_actions/post_stage_upload.py \ - --run-id=${{ github.run_id }} \ + --run-id="${ARTIFACT_RUN_ID}" \ --stage="${STAGE_NAME}" \ --build-dir="${BUILD_DIR}" \ --amdgpu-family="${AMDGPU_FAMILIES}" diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 7b8398d42df..4a08c37f8a5 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -46,6 +46,10 @@ on: type: string description: "Test type to run against the instrumented build." default: "standard" + external_repo_config: + type: string + description: "JSON from detect_external_repo_config.py naming the external checkout to build (e.g. a rocm-libraries commit). Empty builds TheRock's pinned submodules." + default: "" quartz_tracking_id: type: string description: "Leave empty. Reserved for release tracking." @@ -90,6 +94,10 @@ on: type: string description: "Test type to run against the instrumented build." default: "standard" + external_repo_config: + type: string + description: "JSON from detect_external_repo_config.py naming the external checkout to build. The rocm-libraries nightly passes its own, so coverage measures the commit under test; empty builds TheRock's pinned submodules." + default: "" quartz_tracking_id: type: string description: "Leave empty. Reserved for release tracking." @@ -126,6 +134,11 @@ jobs: coverage_cmake_options: ${{ steps.matrix.outputs.coverage_cmake_options }} needs_math_libs: ${{ steps.matrix.outputs.needs_math_libs }} rocm_package_version: ${{ steps.rocm_package_version.outputs.rocm_package_version }} + # S3 namespace for the instrumented stack. A caller that builds a regular + # stack in the same run (the rocm-libraries nightly does, and uses it as + # the baseline) publishes artifacts with the same names under + # github.run_id, so sharing that namespace overwrites them. + artifact_run_id: ${{ github.run_id }}-coverage steps: - name: Checking out repository id: checkout @@ -171,6 +184,8 @@ jobs: dist_amdgpu_families: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci + artifact_run_id: ${{ needs.setup_coverage_matrix.outputs.artifact_run_id }} + external_repo_config: ${{ inputs.external_repo_config }} quartz_tracking_id: ${{ inputs.quartz_tracking_id }} # TheRock scripts for the instrumented build; cross-repo callers pass ROCm/TheRock + ref. repository: ${{ inputs.repository }} @@ -206,6 +221,8 @@ jobs: extra_cmake_options: ${{ needs.setup_coverage_matrix.outputs.coverage_cmake_options }} rocm_package_version: ${{ needs.setup_coverage_matrix.outputs.rocm_package_version }} release_type: ci + artifact_run_id: ${{ needs.setup_coverage_matrix.outputs.artifact_run_id }} + external_repo_config: ${{ inputs.external_repo_config }} quartz_tracking_id: ${{ inputs.quartz_tracking_id }} # TheRock scripts for the instrumented build; cross-repo callers pass ROCm/TheRock + ref. repository: ${{ inputs.repository }} @@ -250,6 +267,7 @@ jobs: baseline_run_id: ${{ inputs.baseline_run_id }} baseline_release_type: ${{ inputs.baseline_release_type }} baseline_repository: ${{ inputs.baseline_repository }} + artifact_run_id: ${{ needs.setup_coverage_matrix.outputs.artifact_run_id }} test_type: ${{ inputs.test_type || 'standard' }} release_type: ci # Forwarded to L2's checkouts and the component jobs. diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index f9be1aa12b3..35aac6116a0 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -57,6 +57,12 @@ on: baseline_release_type: type: string default: nightly + artifact_run_id: + type: string + default: "" + description: >- + Run id the instrumented stack was pushed under, which the overlay and + the report read from. Empty uses github.run_id. codecov_flag: type: string default: "" @@ -152,6 +158,7 @@ jobs: # artifact name and folder from the component job_name. generic_run_id: ${{ inputs.baseline_run_id }} baseline_repository: ${{ inputs.baseline_repository }} + code_coverage_run_id: ${{ inputs.artifact_run_id }} artifact_group: multi-arch-coverage amdgpu_families: ${{ inputs.amdgpu_families }} amdgpu_targets: ${{ inputs.amdgpu_targets }} @@ -215,7 +222,7 @@ jobs: FETCH_ARTIFACT_ARGS: ${{ inputs.fetch_artifact_args }} run: | python ./build_tools/install_rocm_from_artifacts.py \ - --run-id=${{ github.run_id }} \ + --run-id=${{ inputs.artifact_run_id || github.run_id }} \ --artifact-group=${{ inputs.amdgpu_families }} \ --output-dir=${OUTPUT_ARTIFACTS_DIR} \ ${FETCH_ARTIFACT_ARGS} diff --git a/.github/workflows/test_code_coverage_component.yml b/.github/workflows/test_code_coverage_component.yml index 731192f5695..99a3ca3ed55 100644 --- a/.github/workflows/test_code_coverage_component.yml +++ b/.github/workflows/test_code_coverage_component.yml @@ -17,6 +17,12 @@ on: own repository. type: string default: "" + code_coverage_run_id: + description: >- + Run id the instrumented stack was pushed under, passed as + --code-coverage-run-id. Empty uses github.run_id. + type: string + default: "" artifact_group: type: string amdgpu_families: @@ -95,7 +101,7 @@ jobs: run: shell: bash env: - CODE_COVERAGE_RUN_ID: ${{ github.run_id }} + CODE_COVERAGE_RUN_ID: ${{ inputs.code_coverage_run_id || github.run_id }} OUTPUT_ARTIFACTS_DIR: "./build" THEROCK_BIN_DIR: "./build/bin" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} @@ -150,7 +156,7 @@ jobs: # --replace then re-extracts them, leaving the whole stack instrumented. generic_run_id="${{ inputs.generic_run_id }}" if [[ -z "${generic_run_id}" ]]; then - generic_run_id="${{ github.run_id }}" + generic_run_id="${CODE_COVERAGE_RUN_ID}" fi # Pass --run-github-repo only when set, so empty defaults to the # current repository. The generic baseline lives in a different repo @@ -255,6 +261,21 @@ jobs: run: | ${{ fromJSON(inputs.component).test_script }} + # A device-side profile is named after its GPU target + # (gfx942:sramecc+:xnack-.0..profraw). upload-artifact rejects the + # colons and with them every other file in the upload, and llvm-cov cannot + # use the profile against host objects anyway. Its presence means kernels + # were instrumented, which the build is meant to prevent, so say so. + - name: Drop device-side profiles + if: ${{ always() && inputs.coverage_enabled }} + run: | + shopt -s nullglob + device_profiles=("${GITHUB_WORKSPACE}"/coverage-report/profraw/*:*.profraw) + if (( ${#device_profiles[@]} )); then + echo "::warning::Kernels were instrumented; dropping ${#device_profiles[@]} device-side profile(s): ${device_profiles[*]##*/}" + rm -f -- "${device_profiles[@]}" + fi + # Upload even on test failure: partial coverage is useful and the # aggregation job tolerates missing shards. - name: Upload coverage profraw files diff --git a/build_tools/_therock_utils/workflow_outputs.py b/build_tools/_therock_utils/workflow_outputs.py index d1d312d3aa0..1422e468fda 100644 --- a/build_tools/_therock_utils/workflow_outputs.py +++ b/build_tools/_therock_utils/workflow_outputs.py @@ -295,7 +295,10 @@ def from_workflow_run( metadata and environment variables. Args: - run_id: GitHub Actions workflow run ID. + run_id: GitHub Actions workflow run ID. A variant build that shares + its workflow run with a regular one appends ``-`` + (``12345-coverage``), since both produce outputs with the same + names. platform: Platform name ('linux' or 'windows'). github_repository: Repository in 'owner/repo' format. If None, reads GITHUB_REPOSITORY env var (default: 'ROCm/TheRock'). @@ -303,7 +306,8 @@ def from_workflow_run( provided, uses it directly for fork detection and bucket cutover dating (no API call). lookup_workflow_run: If True and ``workflow_run`` is not provided, - fetches the workflow run from the GitHub API using ``run_id``. + fetches the workflow run from the GitHub API using ``run_id``, + less any ``-`` suffix the API would not recognize. Most callers running inside their own CI workflow do not need this — environment variables suffice. Set this when looking up another repository's workflow run (e.g. fetching artifacts). @@ -311,7 +315,9 @@ def from_workflow_run( None, falls back to the RELEASE_TYPE environment variable. """ workflow_run_id = ( - run_id if lookup_workflow_run and workflow_run is None else None + run_id.partition("-")[0] + if lookup_workflow_run and workflow_run is None + else None ) external_repo, bucket = _retrieve_bucket_info( github_repository=github_repository, diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 640e5d2628a..6e9fc26619a 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -108,7 +108,9 @@ class CoverageProject: coverage_config: Per-project coverage metadata file, relative to the root of the repository named by COVERAGE_CONFIG_SOURCE. object_globs: Globs, relative to the extracted artifact directory, - matching the instrumented binaries handed to `llvm-cov`. + matching the instrumented binaries handed to `llvm-cov`. A glob + prefixed with `!` removes its matches instead, for test binaries + that share an install directory with a sibling project's. fetch_artifact_args: Arguments to install_rocm_from_artifacts.py that pull the instrumented libraries into the report generation job. codecov_flag: Flag the report is filed under in Codecov. @@ -394,7 +396,8 @@ class CoverageProject: stage=STAGE_MATH_LIBS, test_component="rocprim", coverage_config="projects/rocprim/test_categories_coverage.yaml", - object_globs=["bin/test_*"], + # hipCUB's tests install into the same bin/ as test_hipcub_*. + object_globs=["bin/test_*", "!bin/test_hipcub_*"], fetch_artifact_args="--prim --tests", codecov_flag="rocPRIM", # rocPRIM's tests are a sibling subproject, so both stage dirs are @@ -409,7 +412,7 @@ class CoverageProject: stage=STAGE_MATH_LIBS, test_component="hipcub", coverage_config="projects/hipcub/test_categories_coverage.yaml", - object_globs=["bin/test_*"], + object_globs=["bin/test_hipcub_*"], fetch_artifact_args="--prim --tests", codecov_flag="hipCUB", ), @@ -421,7 +424,7 @@ class CoverageProject: stage=STAGE_MATH_LIBS, test_component="rocthrust", coverage_config="projects/rocthrust/test_categories_coverage.yaml", - object_globs=["bin/test_*"], + object_globs=["bin/*.hip"], fetch_artifact_args="--prim --tests", codecov_flag="rocThrust", ), @@ -429,11 +432,12 @@ class CoverageProject: cmake_target="rocWMMA", artifact_names=["rocwmma"], artifact_relpaths=["math-libs/rocWMMA/stage"], - coverage_option="CODE_COVERAGE", + # Prefixed, like hipTensor's above; a bare CODE_COVERAGE is never read. + coverage_option="ROCWMMA_CODE_COVERAGE", stage=STAGE_MATH_LIBS, test_component="rocwmma", coverage_config="projects/rocwmma/test_categories_coverage.yaml", - object_globs=["bin/*_test*"], + object_globs=["bin/*_test*", "bin/gemm_*-validate"], fetch_artifact_args="--rocwmma --tests", codecov_flag="rocWMMA", ), diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 19abfa5b3fc..b26ece0d8a7 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -51,6 +51,24 @@ def _get_artifact_path(artifact_path: str) -> str: return (OUTPUT_ARTIFACTS_DIR / artifact_path).as_posix() +# BUILD_VARIANT=coverage runs tests against instrumented libraries, which take +# far longer than the release timeouts below allow for: every executed region +# bumps a counter, and some upstream coverage options also drop optimization +# (hipRAND adds -O0) or compile in extra CPU-only test suites (rocRAND). +# The cap keeps the step inside test_code_coverage_component.yml's job +# timeout-minutes (210), which would otherwise cut it off first. +COVERAGE_TIMEOUT_MULTIPLIER = 4 +COVERAGE_TIMEOUT_CAP_MINUTES = 180 + + +def _coverage_timeout_minutes(timeout_minutes: int) -> int: + """Scales a component's timeout for an instrumented run, never shortening it.""" + scaled = min( + timeout_minutes * COVERAGE_TIMEOUT_MULTIPLIER, COVERAGE_TIMEOUT_CAP_MINUTES + ) + return max(timeout_minutes, scaled) + + # Maps a group label (the part after "test:") to the individual test matrix # keys it expands to. Use this when a single label should select multiple # related jobs without relying on name-prefix inference. @@ -1197,6 +1215,11 @@ def run(): job_config_data["timeout_minutes"] + 15 ) + if build_variant == "coverage": + job_config_data["timeout_minutes"] = _coverage_timeout_minutes( + job_config_data["timeout_minutes"] + ) + # For CI testing, we construct a shard array based on "total_shards" from "fetch_test_configurations.py" # This way, the test jobs will be split up into X shards. (ex: [1, 2, 3, 4] = 4 test shards) # For display purposes, we add "i + 1" for the job name (ex: 1 of 4). During the actual test sharding in the test executable, this array will become 0th index diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py index ec3b742d100..6f27d033725 100644 --- a/build_tools/github_actions/merge_coverage_report.py +++ b/build_tools/github_actions/merge_coverage_report.py @@ -18,10 +18,16 @@ that built the instrumented objects; a version mismatch produces an unhelpfully generic "malformed instrumentation profile data" error. Both live under `lib/llvm/bin` of an installed ROCm distribution. + +A report that measured nothing fails rather than publishing 0%. Most ways the +pipeline can break upstream of here still leave profraw files behind: an +uninstrumented library writes profiles with no counters, and a process that +crashes before its exit handler writes empty ones. """ import argparse import logging +import re import shutil import subprocess import sys @@ -64,10 +70,22 @@ def resolve_objects(rocm_dir: Path, object_globs: list[str]) -> list[Path]: Globs typically match a versioned family of symlinks (libfoo.so, libfoo.so.1, ...) that all resolve to one file, so results are deduplicated by real path to avoid handing llvm-cov the same object several times. + + A glob starting with '!' takes its matches back out. Header-only projects + report against their test binaries, and those can share a directory with + a sibling project's (rocPRIM's bin/test_* also matches hipCUB's tests). """ + excluded = { + match + for pattern in object_globs + if pattern.startswith("!") + for match in rocm_dir.glob(pattern[1:]) + } objects: dict[Path, Path] = {} for pattern in object_globs: - matches = sorted(rocm_dir.glob(pattern)) + if pattern.startswith("!"): + continue + matches = sorted(m for m in rocm_dir.glob(pattern) if m not in excluded) if not matches: logging.warning( "No files matched object glob '%s' under %s", pattern, rocm_dir @@ -93,6 +111,34 @@ def merge_profraw(llvm_profdata: Path, profraw_files: list[Path], output: Path) subprocess.run(command, check=True) +def count_profiled_functions(llvm_profdata: Path, profdata: Path) -> int | None: + """Returns how many functions the merged profile holds counters for. + + None if llvm-profdata's summary has no such line, so an unexpected format + skips the check rather than failing a good report. + """ + result = subprocess.run( + [str(llvm_profdata), "show", str(profdata)], + check=True, + capture_output=True, + text=True, + ) + match = re.search(r"^Total functions: (\d+)$", result.stdout, re.MULTILINE) + return int(match.group(1)) if match else None + + +def count_lcov_lines(lcov: Path) -> tuple[int, int]: + """Returns (lines found, lines hit) summed over every file in an lcov report.""" + found = hit = 0 + with open(lcov) as lcov_file: + for line in lcov_file: + if line.startswith("LF:"): + found += int(line[3:]) + elif line.startswith("LH:"): + hit += int(line[3:]) + return found, hit + + def build_cov_command( llvm_cov: Path, subcommand: str, @@ -250,7 +296,8 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--allow-empty", action="store_true", - help="Exit successfully when no profraw files were collected", + help="Exit successfully when no profraw files were collected, or " + "none of them hold any counters", ) args = parser.parse_args(argv) @@ -282,14 +329,37 @@ def main(argv: list[str] | None = None) -> int: return 1 merge_profraw(llvm_profdata, profraw_files, args.profdata_output) - export_lcov( - llvm_cov, - args.profdata_output, - objects, - args.lcov_output, - args.path_equivalence, - ) + if count_profiled_functions(llvm_profdata, args.profdata_output) == 0: + message = f"None of the {len(profraw_files)} profraw file(s) hold any counters" + if args.allow_empty: + logging.warning("%s, skipping report generation", message) + return 0 + logging.error( + "%s. The processes that wrote them loaded no instrumented code: " + "either the coverage flags never reached the compiler, or every " + "process died before writing its profile.", + message, + ) + return 1 + + try: + export_lcov( + llvm_cov, + args.profdata_output, + objects, + args.lcov_output, + args.path_equivalence, + ) + except subprocess.CalledProcessError: + logging.error( + "llvm-cov could not export the %d object(s). If it found no " + "coverage data, they carry no coverage mapping: the project's " + "coverage option did not take effect in its build.", + len(objects), + ) + return 1 logging.info("Wrote coverage report to %s", args.lcov_output) + lines_found, lines_hit = count_lcov_lines(args.lcov_output) if args.summary_output: write_summary( @@ -314,6 +384,16 @@ def main(argv: list[str] | None = None) -> int: demangler if demangler.is_file() else None, ) + # Checked after the other renderings so the report still shows what was + # measured. + if lines_found and not lines_hit: + logging.error( + "None of the %d instrumented line(s) in the reported objects ran. " + "The counters came from other binaries: the tests loaded a " + "different copy of the project, or never reached it.", + lines_found, + ) + return 1 return 0 diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index 478cd3d990b..bca44ccf46a 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -16,6 +16,7 @@ sys.path.insert(0, os.fspath(Path(__file__).parents[2])) import configure_coverage_ci +import merge_coverage_report from _therock_utils.build_topology import get_topology @@ -239,6 +240,30 @@ def test_object_globs_reach_the_matrix(self): self.assertEqual(entry["object_globs"], "lib/libhiprand.so*") self.assertEqual(entry["fetch_artifact_args"], "--rand") + def test_header_only_projects_report_on_their_own_test_binaries(self): + # rocPRIM, hipCUB and rocThrust install their tests into one flat bin/, + # and --prim --tests brings all three into every one of their reports. + installed = { + "rocprim": ["bin/test_basic", "bin/test_rocprim_tuple"], + "hipcub": ["bin/test_hipcub_basic"], + "rocthrust": ["bin/vector.hip"], + } + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "bin").mkdir() + for names in installed.values(): + for name in names: + (root / name).touch() + for project, names in installed.items(): + with self.subTest(project=project): + objects = merge_coverage_report.resolve_objects( + root, + configure_coverage_ci.COVERAGE_PROJECTS[project].object_globs, + ) + self.assertEqual( + [o.relative_to(root).as_posix() for o in objects], sorted(names) + ) + class BuildCoverageCmakeOptionsTest(unittest.TestCase): def test_full_selection_collapses_to_the_group_option(self): diff --git a/build_tools/github_actions/tests/fetch_test_configurations_test.py b/build_tools/github_actions/tests/fetch_test_configurations_test.py index d476af01a40..003eb57488a 100644 --- a/build_tools/github_actions/tests/fetch_test_configurations_test.py +++ b/build_tools/github_actions/tests/fetch_test_configurations_test.py @@ -201,6 +201,42 @@ def test_tensilelite_quick_omits_ctest_stage(self): ) self.assertEqual(tensilelite["timeout_minutes"], 15) + # ----------------------- + # Coverage variant timeouts + # ----------------------- + + def test_coverage_variant_scales_timeouts(self): + os.environ["PROJECTS_TO_TEST"] = "rocrand" + os.environ["BUILD_VARIANT"] = "coverage" + release = fetch_test_configurations.test_matrix["rocrand"]["timeout_minutes"] + + fetch_test_configurations.run() + + self.assertEqual( + self._get_components()[0]["timeout_minutes"], + fetch_test_configurations._coverage_timeout_minutes(release), + ) + + def test_release_variant_keeps_timeouts(self): + os.environ["PROJECTS_TO_TEST"] = "rocrand" + os.environ["BUILD_VARIANT"] = "release" + + fetch_test_configurations.run() + + self.assertEqual( + self._get_components()[0]["timeout_minutes"], + fetch_test_configurations.test_matrix["rocrand"]["timeout_minutes"], + ) + + def test_coverage_timeout_is_multiplied(self): + self.assertEqual(fetch_test_configurations._coverage_timeout_minutes(15), 60) + + def test_coverage_timeout_stays_inside_the_job_timeout(self): + self.assertEqual(fetch_test_configurations._coverage_timeout_minutes(90), 180) + + def test_coverage_timeout_never_shortens_a_longer_one(self): + self.assertEqual(fetch_test_configurations._coverage_timeout_minutes(288), 288) + # ----------------------- # tensilelite-common (Tensile/Tests/common on real hardware) # ----------------------- diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index 1272aff92b4..8ba388ffcd3 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -3,6 +3,7 @@ import os import platform +import subprocess import sys import tempfile import unittest @@ -115,6 +116,49 @@ def test_unmatched_glob_yields_nothing(self): merge_coverage_report.resolve_objects(self.root, ["lib/nothing*"]), [] ) + def test_bang_glob_takes_matches_back_out(self): + self.touch("bin/test_basic") + self.touch("bin/test_hipcub_basic") + + objects = merge_coverage_report.resolve_objects( + self.root, ["bin/test_*", "!bin/test_hipcub_*"] + ) + + self.assertEqual([o.name for o in objects], ["test_basic"]) + + +class CountProfiledFunctionsTest(unittest.TestCase): + def _count(self, stdout: str): + with mock.patch("subprocess.run", return_value=mock.Mock(stdout=stdout)) as run: + count = merge_coverage_report.count_profiled_functions( + Path("/llvm/llvm-profdata"), Path("/tmp/c.profdata") + ) + self.assertEqual(run.call_args.args[0][1], "show") + return count + + def test_reads_the_summary_line(self): + self.assertEqual( + self._count("Instrumentation level: Front-end\nTotal functions: 1305\n"), + 1305, + ) + + def test_profile_without_counters_reads_as_zero(self): + self.assertEqual(self._count("Total functions: 0\n"), 0) + + def test_unrecognized_output_skips_the_check(self): + self.assertIsNone(self._count("something else entirely\n")) + + +class CountLcovLinesTest(TempDirTestBase): + def test_sums_over_every_file(self): + lcov = self.root / "coverage.info" + lcov.write_text( + "SF:a.cpp\nDA:1,1\nLF:10\nLH:4\nend_of_record\n" + "SF:b.cpp\nDA:1,0\nLF:5\nLH:0\nend_of_record\n" + ) + + self.assertEqual(merge_coverage_report.count_lcov_lines(lcov), (15, 4)) + class CommandConstructionTest(TempDirTestBase): def test_merge_passes_every_profraw_file(self): @@ -242,7 +286,9 @@ def test_happy_path_invokes_both_llvm_tools(self): for tool in ("llvm-profdata", "llvm-cov"): (llvm_bin_dir / f"{tool}{merge_coverage_report.EXECUTABLE_SUFFIX}").touch() - with mock.patch("subprocess.run") as run: + with mock.patch( + "subprocess.run", return_value=mock.Mock(stdout="Total functions: 3\n") + ) as run: exit_code = merge_coverage_report.main( self._argv( "--profdata-output", @@ -253,11 +299,12 @@ def test_happy_path_invokes_both_llvm_tools(self): ) self.assertEqual(exit_code, 0) - self.assertEqual(run.call_count, 2) + self.assertEqual(run.call_count, 3) invoked = [Path(call.args[0][0]).name for call in run.call_args_list] self.assertEqual( invoked, [ + f"llvm-profdata{merge_coverage_report.EXECUTABLE_SUFFIX}", f"llvm-profdata{merge_coverage_report.EXECUTABLE_SUFFIX}", f"llvm-cov{merge_coverage_report.EXECUTABLE_SUFFIX}", ], @@ -286,9 +333,86 @@ def test_summary_and_html_are_opt_in(self): ) self.assertEqual(exit_code, 0) - # merge, then export, report and show off the one index. + # merge and check the index, then export, report and show off it. subcommands = [call.args[0][1] for call in run.call_args_list] - self.assertEqual(subcommands, ["merge", "export", "report", "show"]) + self.assertEqual(subcommands, ["merge", "show", "export", "report", "show"]) + + +class MainRejectsEmptyReportsTest(TempDirTestBase): + """Every way the pipeline breaks upstream still leaves profraw files behind.""" + + def setUp(self): + super().setUp() + self.touch("profraw/shard0/a.profraw") + self.touch("rocm/lib/libhiprand.so") + llvm_bin_dir = self.root / "rocm" / "lib" / "llvm" / "bin" + llvm_bin_dir.mkdir(parents=True) + for tool in ("llvm-profdata", "llvm-cov"): + (llvm_bin_dir / f"{tool}{merge_coverage_report.EXECUTABLE_SUFFIX}").touch() + + def _main( + self, *extra: str, functions: int = 3, lcov: str = "", export_fails=False + ): + def fake_run(command, **kwargs): + tool, subcommand = Path(command[0]).stem, command[1] + if (tool, subcommand) == ("llvm-profdata", "show"): + return mock.Mock(stdout=f"Total functions: {functions}\n") + if subcommand == "export": + if export_fails: + raise subprocess.CalledProcessError(1, command) + kwargs["stdout"].write(lcov) + return mock.Mock(stdout="") + + with mock.patch("subprocess.run", side_effect=fake_run) as run: + exit_code = merge_coverage_report.main( + [ + "--profraw-dir", + os.fspath(self.root / "profraw"), + "--rocm-dir", + os.fspath(self.root / "rocm"), + "--object-globs", + "lib/libhiprand.so*", + "--profdata-output", + os.fspath(self.root / "out" / "coverage.profdata"), + "--lcov-output", + os.fspath(self.root / "out" / "coverage.info"), + *extra, + ] + ) + return exit_code, [call.args[0][1] for call in run.call_args_list] + + def test_profiles_without_counters_fail_before_export(self): + # What an uninstrumented library writes: a header and nothing else. + exit_code, subcommands = self._main(functions=0) + + self.assertEqual(exit_code, 1) + self.assertEqual(subcommands, ["merge", "show"]) + + def test_profiles_without_counters_are_tolerated_with_allow_empty(self): + exit_code, _ = self._main("--allow-empty", functions=0) + + self.assertEqual(exit_code, 0) + + def test_objects_without_a_coverage_mapping_fail(self): + exit_code, _ = self._main(export_fails=True) + + self.assertEqual(exit_code, 1) + + def test_report_that_never_reaches_the_objects_fails(self): + exit_code, subcommands = self._main( + "--summary-output", + os.fspath(self.root / "out" / "coverage_summary.txt"), + lcov="SF:a.cpp\nLF:10\nLH:0\nend_of_record\n", + ) + + self.assertEqual(exit_code, 1) + # The summary is still written, so the job page shows the zero. + self.assertEqual(subcommands, ["merge", "show", "export", "report"]) + + def test_report_with_any_line_hit_passes(self): + exit_code, _ = self._main(lcov="SF:a.cpp\nLF:10\nLH:1\nend_of_record\n") + + self.assertEqual(exit_code, 0) if __name__ == "__main__": diff --git a/build_tools/tests/workflow_outputs_test.py b/build_tools/tests/workflow_outputs_test.py index 9a5bc26d845..eca90596a22 100644 --- a/build_tools/tests/workflow_outputs_test.py +++ b/build_tools/tests/workflow_outputs_test.py @@ -332,6 +332,27 @@ def test_lookup_workflow_run_triggers_api(self, mock_retrieve): release_type=None, ) + @mock.patch("_therock_utils.workflow_outputs._retrieve_bucket_info") + def test_lookup_finds_a_variant_run_by_its_workflow_run_id(self, mock_retrieve): + """A - suffix stays in the prefix but not in the API lookup.""" + mock_retrieve.return_value = ( + "ROCm-rocm-libraries/", + "therock-ci-artifacts-external", + ) + root = WorkflowOutputRoot.from_workflow_run( + run_id="99999-coverage", + platform="linux", + github_repository="ROCm/rocm-libraries", + lookup_workflow_run=True, + ) + self.assertEqual(root.prefix, "ROCm-rocm-libraries/99999-coverage-linux") + mock_retrieve.assert_called_once_with( + github_repository="ROCm/rocm-libraries", + workflow_run_id="99999", + workflow_run=None, + release_type=None, + ) + @mock.patch("_therock_utils.workflow_outputs._retrieve_bucket_info") def test_with_workflow_run_dict(self, mock_retrieve): """When workflow_run is provided, it's passed through (no API call).""" diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index e8d1c70c7b9..75afcc0ebcf 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -970,11 +970,24 @@ function(therock_cmake_subproject_activate target_name) # HIP sources in these projects are compiled -- as CXX, through the amd-hip # toolchain. A language brought up later by a bare enable_language() is not # covered, and would need the same rewrite from wherever that happens. + # + # The same file takes these projects off ccache. ccache (4.11 at least) + # removes every -Xarch_ pair unless also arrived as an + # Apple-style -arch, from the cache key and from the command it runs alike. + # That drops the negation below, so kernels end up instrumented after all, + # and drops the -Xarch_host spelling some upstream coverage options use + # (rocRAND, hipRAND), so those libraries end up not instrumented at all. + # Nothing fails at build time: either way the damage only shows up once the + # tests run. set(_coverage_project_include_arg) set(_coverage_configure_depends) if(NOT MSVC AND ${_coverage_var_name}) set(_coverage_host_only_contents "# Generated by therock_subproject.cmake; see the coverage comment there.\n") + foreach(_coverage_lang IN ITEMS C CXX HIP) + string(APPEND _coverage_host_only_contents + "set(CMAKE_${_coverage_lang}_COMPILER_LAUNCHER \"\")\n") + endforeach() # Runs on every project() call, so the rewrite has to be idempotent. # -Wno-unused-command-line-argument because -Xarch_device is unused for # the plain C++ sources that make up most of these projects, and several diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index d38d7dd7bcd..7ae04c76e0d 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -71,6 +71,16 @@ The later of a `-f`/`-fno-` pair wins; the project's own flags arrive after `CMAKE__COMPILE_OBJECT` via a generated file passed as `CMAKE_PROJECT_INCLUDE`. +The same file clears `CMAKE__COMPILER_LAUNCHER`, so instrumented +projects never compile through ccache. ccache (4.11 at least) removes every +`-Xarch_ ` pair whose arch did not also arrive as an Apple-style +`-arch`, from both the cache key and the command it runs. That drops the +negation above, so kernels are instrumented after all, and drops the +`-Xarch_host` spelling rocRAND's and hipRAND's own coverage options use, so +those libraries are not instrumented at all. Neither fails the build; both +only show up once the tests run. Projects that are not instrumented keep using +the cache. + ### The profile runtime's link dependencies `libclang_rt.profile_rocm.a` needs `-ldl` and `-lpthread`, which the driver @@ -148,9 +158,18 @@ names each project. `CMakeLists.txt` expands group options to `_ENABLE_COVERAGE` flags; `therock_subproject.cmake` translates each to the project's upstream name. Everything not selected builds normally. -Build jobs use `multi_arch_build_portable_linux_artifacts.yml` directly; -`extra_cmake_options` is the only build-side hook coverage adds. Artifacts -publish under `release_type: ci`. +Build jobs use `multi_arch_build_portable_linux_artifacts.yml` directly. +Coverage sets three of its inputs: + +- `extra_cmake_options` carries `coverage_cmake_options`. +- `artifact_run_id` is `-coverage`. Artifacts and logs publish there + under `release_type: ci`, not under the run id itself. A caller that builds a + regular stack in the same run (the rocm-libraries nightly does, and uses it + as the baseline) publishes artifacts with the same names, which a shared + namespace would overwrite. +- `external_repo_config` is forwarded from the caller, so a rocm-libraries + nightly measures the commit it is testing. Without it the build uses + TheRock's pinned submodules. ### Test execution @@ -158,17 +177,20 @@ Per shard, `test_code_coverage_component.yml` does three things: 1. **Install baseline + swap project.** `install_rocm_code_coverage_build.py` installs from `--run-id` (the nightly) and replaces only the - project-under-test from `--code-coverage-run-id`. + project-under-test from `--code-coverage-run-id` (`-coverage`). 1. **Set `LLVM_PROFILE_FILE`** to a per-shard path (`%p`/`%m`) so concurrent processes don't overwrite each other. 1. **Run tests and upload profraw** under `always()` — a failing shard still - exercised code. + exercised code. Device-side profiles (named after the GPU target, colons + included) are dropped with a warning first: one of them makes + `upload-artifact` reject the whole upload. -Coverage runs use each component's normal `timeout_minutes` from -`fetch_test_configurations.py`; nothing extends it for instrumented builds. -Instrumented tests run slower, and some upstream coverage options also lower the -optimization level (hipRAND's `BUILD_CODE_COVERAGE` adds `-O0 -g`), so a -component close to its limit may time out. +With `BUILD_VARIANT=coverage`, `fetch_test_configurations.py` multiplies each +component's `timeout_minutes` by 4, capped at 180 so the step ends inside the +job's 210-minute limit, and never lowered below the release value. +Instrumented tests run slower, and some upstream coverage options do more than +instrument: hipRAND's `BUILD_CODE_COVERAGE` adds `-O0 -g`, and rocRAND's +`CODE_COVERAGE` compiles in extra CPU-only test suites. #### Why there are two run ids @@ -181,8 +203,9 @@ The baseline is read from `baseline_release_type` (normally `nightly`), not the coverage run's `ci` channel — artifacts are bucketed per channel. The swap downloads the project's whole artifact (`rand`, which also contains rocRAND) but extracts only the paths matching the project's library folder in -`COMPONENT_MAP` (`hipRAND`). The two runs write to separate S3 paths; no files -collide. +`COMPONENT_MAP` (`hipRAND`). The instrumented stack lives under +`-coverage`, so even a baseline from the same workflow run is never +overwritten by it. A missing instrumented artifact fails the install step. A swap that matches no files does not: the installer logs `Replaced 0` and the tests run against the @@ -201,7 +224,7 @@ most for header-only projects (rocPRIM, hipCUB, rocThrust, rocWMMA). | Artifact | Where | Purpose | | ------------ | ----------------------------------- | ----------------------------- | -| Instrumented | CI bucket, this run | Stack under test + LLVM tools | +| Instrumented | CI bucket, `-coverage` | Stack under test + LLVM tools | | Profraw | GitHub Actions artifacts, per shard | Raw profiles for aggregation | Profraw names include the project, GPU family, and shard index. @@ -211,9 +234,19 @@ Profraw names include the project, GPU family, and shard index. `aggregate_coverage` runs under `if: !cancelled()` so partial reports survive shard failures. It downloads profraw artifacts, reinstalls the run's artifacts (LLVM tools must match the compiler that produced the profiles), then runs -`merge_coverage_report.py`. Two conditions are fatal: no profraw files (tests -didn't run or the wrong library was loaded) and no matching objects (`object_globs` -doesn't match the project's install layout). +`merge_coverage_report.py`. A report that measured nothing fails rather than +publishing 0%: + +- No profraw files: the tests didn't run, or the wrong library was loaded. +- No matching objects: `object_globs` doesn't match the project's install + layout. +- Profiles with no counters: the processes loaded no instrumented code. Either + the coverage flags never reached the compiler, or every process crashed + before writing its profile. +- Objects with no coverage mapping (`llvm-cov`: "no coverage data found"): the + project's coverage option did not take effect. +- No line in the reported objects ran: the tests loaded a different copy of the + project, or never reached it. The `coverage-report--` artifact: @@ -253,13 +286,18 @@ Add an entry to `COVERAGE_PROJECTS` in `configure_coverage_ci.py`. `coverage_option` must name the CMake option the project implements, and it must select LLVM instrumentation (not gcov, which writes `.gcda` files this pipeline can't read). Names vary: most use `BUILD_CODE_COVERAGE` or `CODE_COVERAGE`; -RCCL uses `ENABLE_CODE_COVERAGE`. +RCCL uses `ENABLE_CODE_COVERAGE`; rocWMMA and hipTensor prefix the project +(`ROCWMMA_CODE_COVERAGE`). A wrong name still configures and builds cleanly; +it only shows up as a report with no coverage data. Verify a local instrumented build produces a non-empty report before committing the entry. Set `source_repo` (`ROCM_LIBRARIES` or `ROCM_SYSTEMS`) to route the project into the correct group alias; membership lists are generated at configure time, so no CMake edit is needed. `object_globs` must match what the project -actually installs — a miss fails the report job rather than publishing zero. +actually installs — a miss fails the report job rather than publishing zero. A +glob prefixed with `!` removes its matches, for test binaries that share an +install directory with a sibling project's (rocPRIM's `bin/test_*` would +otherwise include hipCUB's `bin/test_hipcub_*`). ## Blocked projects diff --git a/docs/development/workflow_outputs.md b/docs/development/workflow_outputs.md index 94f5815000a..d6239f65025 100644 --- a/docs/development/workflow_outputs.md +++ b/docs/development/workflow_outputs.md @@ -37,9 +37,20 @@ s3://{bucket}/{external_repo}{run_id}-{platform}/ | --------------- | ------------------------------- | --------------------------------------------------------------------- | | `bucket` | `therock-ci-artifacts` | Selected based on repo, fork status, release type | | `external_repo` | `""` or `"githubuser-TheRock/"` | Non-empty for forks and non-TheRock repos (format: `{owner}-{repo}/`) | -| `run_id` | `12345678901` | GitHub Actions workflow run ID | +| `run_id` | `12345678901` | GitHub Actions workflow run ID; see [Variant builds](#variant-builds) | | `platform` | `linux` or `windows` | Build platform | +### Variant builds + +A variant build that shares its workflow run with a regular build publishes +under `{run_id}-{variant}` instead, because both produce artifacts with the same +names. A caller that builds a regular stack and a coverage-instrumented one in +the same run gets them under `12345678901-linux/` and +`12345678901-coverage-linux/`. Pass the suffixed id as `artifact_run_id` to +`multi_arch_build_portable_linux_artifacts.yml`, and as `--run-id` to anything +that reads the variant's outputs. A lookup through the GitHub API uses the id +without its suffix. + ### Directory structure There are two CI pipeline architectures with different log layouts: @@ -263,7 +274,8 @@ the GitHub API to fetch workflow run metadata (for fork detection and bucket cutover dating). Most callers running inside their own CI workflow do not need this — environment variables (`GITHUB_REPOSITORY`, `IS_PR_FROM_FORK`) suffice. Set `lookup_workflow_run=True` when looking up another repository's workflow -run, e.g. when fetching artifacts. +run, e.g. when fetching artifacts. A `-{variant}` suffix on `run_id` is +dropped for the lookup. ### StorageBackend From 21ddceb76097f6b207346ae6659cd295ada095f0 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 5 Oct 2026 09:43:44 -0500 Subject: [PATCH 53/70] ci(coverage): let a same-run baseline build in parallel with coverage The rocm-libraries nightly starts coverage only after its whole regular Linux build and test job finishes, though only the coverage tests need the baseline, and only its build stages. A reusable workflow cannot `needs:` a job inside another one, so wait_for_baseline polls the run's jobs instead and holds coverage_report until the regular build's math-libs stage has succeeded for every coverage family (compiler-runtime when no math-libs project is selected). The gate runs only when baseline_run_id is this run's id; a baseline from an earlier run skips it. It fails, skipping the tests, when the stage job fails, is skipped, or never runs, when the run builds the stage only for other families, or after 300 minutes. Jobs are matched by display name, ignoring the Windows build, which reuses the names; a test checks the names against multi_arch_build_portable_linux.yml. Listing a public repository's jobs needs no `actions` permission (the nightly's Coverage Summary already does it with contents: read), so callers need no new grant. With the nightly as it is, the gate finds the baseline already built and passes on its first poll. Co-authored-by: Cursor --- .../multi_arch_ci_coverage_linux.yml | 36 ++ .../tests/wait_for_baseline_stage_test.py | 358 ++++++++++++++++++ .../github_actions/wait_for_baseline_stage.py | 264 +++++++++++++ docs/development/code_coverage.md | 21 + 4 files changed, 679 insertions(+) create mode 100644 build_tools/github_actions/tests/wait_for_baseline_stage_test.py create mode 100644 build_tools/github_actions/wait_for_baseline_stage.py diff --git a/.github/workflows/multi_arch_ci_coverage_linux.yml b/.github/workflows/multi_arch_ci_coverage_linux.yml index 4a08c37f8a5..8b7790406c4 100644 --- a/.github/workflows/multi_arch_ci_coverage_linux.yml +++ b/.github/workflows/multi_arch_ci_coverage_linux.yml @@ -231,6 +231,40 @@ jobs: contents: read id-token: write + # A caller that builds the baseline in this same run (the rocm-libraries + # nightly) can start this workflow alongside that build. The tests install + # the baseline from S3, so hold them until that build's math-libs stage (which + # needs compiler-runtime) has succeeded, or its compiler-runtime stage when no + # math-libs project is selected. + wait_for_baseline: + name: Wait for Baseline + needs: setup_coverage_matrix + if: >- + ${{ + inputs.baseline_run_id == github.run_id && + needs.setup_coverage_matrix.outputs.coverage_matrix != '[]' + }} + runs-on: ubuntu-24.04 + # Above the script's own 300-minute limit, so it reports the timeout. + timeout-minutes: 330 + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 15 + with: + # Cross-repo callers pass ROCm/TheRock + ref; same-repo defaults to github.repository. + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref }} + - name: Wait for the baseline stage + env: + GITHUB_TOKEN: ${{ github.token }} + BASELINE_STAGE: ${{ needs.setup_coverage_matrix.outputs.needs_math_libs == 'true' && 'math-libs' || 'compiler-runtime' }} + AMDGPU_FAMILIES: ${{ needs.setup_coverage_matrix.outputs.dist_amdgpu_families }} + run: | + python build_tools/github_actions/wait_for_baseline_stage.py \ + --stage="${BASELINE_STAGE}" \ + --amdgpu-families="${AMDGPU_FAMILIES}" + # One test + report job per project. Each installs baseline_run_id and replaces # only its own project from this run's instrumented stack, so dependencies stay # non-instrumented even though the build instrumented them all at once. @@ -243,6 +277,7 @@ jobs: setup_coverage_matrix, build_compiler_runtime, build_instrumented_math_libs, + wait_for_baseline, ] # Also guard the empty matrix: fromJSON('') would hard-error. if: >- @@ -284,6 +319,7 @@ jobs: - setup_coverage_matrix - build_compiler_runtime - build_instrumented_math_libs + - wait_for_baseline - coverage_report runs-on: ubuntu-24.04 steps: diff --git a/build_tools/github_actions/tests/wait_for_baseline_stage_test.py b/build_tools/github_actions/tests/wait_for_baseline_stage_test.py new file mode 100644 index 00000000000..83190c317d2 --- /dev/null +++ b/build_tools/github_actions/tests/wait_for_baseline_stage_test.py @@ -0,0 +1,358 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import os +import sys +from pathlib import Path +from unittest.mock import call, patch + +import pytest + +# Add github_actions to path so wait_for_baseline_stage is importable. +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) + +from github_actions_api import GitHubAPIError +from wait_for_baseline_stage import ( + MAX_CONSECUTIVE_API_ERRORS, + MAX_PAGES, + STAGE_JOB_NAMES, + State, + evaluate, + expected_job_names, + list_run_jobs, + main, + parse_families, + wait_for_stage, +) +from workflow_utils import WORKFLOWS_DIR, load_workflow + +LINUX_STAGES = "Linux::release / Build Multi-Arch Stages" +WINDOWS_STAGES = "Windows::release / Build Multi-Arch Stages" +COVERAGE = "Linux::CodeCoverage" +GFX94X = "Stage - Math Libs (gfx94X-dcgpu)" +GFX950 = "Stage - Math Libs (gfx950-dcgpu)" + + +def job(name, status="completed", conclusion="success", job_id=1): + return { + "name": name, + "status": status, + "conclusion": conclusion if status == "completed" else None, + "html_url": f"https://github.com/ROCm/rocm-libraries/actions/runs/1/job/{job_id}", + } + + +def math_libs(family, prefix=LINUX_STAGES, **kwargs): + return job( + f"{prefix} / math-libs ({family}, linux-gfx942-1gpu) / Stage - Math Libs ({family})", + **kwargs, + ) + + +SETUP = job("setup / setup", job_id=100) +COMPILER_RUNTIME = job( + f"{LINUX_STAGES} / compiler-runtime / Stage - Compiler Runtime", job_id=101 +) + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + + +class TestParseFamilies: + def test_accepts_semicolons_and_commas(self): + assert parse_families("gfx94X-dcgpu; gfx950-dcgpu,gfx1151") == [ + "gfx94X-dcgpu", + "gfx950-dcgpu", + "gfx1151", + ] + + def test_empty(self): + assert parse_families(" ; ") == [] + + +class TestExpectedJobNames: + def test_math_libs_names_a_job_per_family(self): + assert expected_job_names("math-libs", ["gfx94X-dcgpu", "gfx950-dcgpu"]) == [ + GFX94X, + GFX950, + ] + + def test_compiler_runtime_is_one_job_for_every_family(self): + assert expected_job_names("compiler-runtime", ["gfx94X-dcgpu"]) == [ + "Stage - Compiler Runtime" + ] + + +# --------------------------------------------------------------------------- +# evaluate() +# --------------------------------------------------------------------------- + + +class TestEvaluate: + def test_ready_once_the_job_succeeds(self): + state, lines = evaluate( + [SETUP, COMPILER_RUNTIME, math_libs("gfx94X-dcgpu")], "math-libs", [GFX94X] + ) + assert state is State.READY + assert lines == [ + f"{GFX94X}: success https://github.com/ROCm/rocm-libraries/actions/runs/1/job/1" + ] + + def test_waits_while_the_job_runs(self): + jobs = [SETUP, math_libs("gfx94X-dcgpu", status="in_progress")] + state, lines = evaluate(jobs, "math-libs", [GFX94X]) + assert state is State.WAITING + assert lines[0].startswith(f"{GFX94X}: in_progress ") + + def test_waits_before_the_stage_starts(self): + state, lines = evaluate( + [SETUP, job(COMPILER_RUNTIME["name"], status="in_progress")], + "math-libs", + [GFX94X], + ) + assert state is State.WAITING + assert lines == [f"{GFX94X}: not started"] + + @pytest.mark.parametrize("conclusion", ["failure", "cancelled", "skipped"]) + def test_fails_when_the_job_does_not_succeed(self, conclusion): + jobs = [math_libs("gfx94X-dcgpu", conclusion=conclusion)] + state, lines = evaluate(jobs, "math-libs", [GFX94X]) + assert state is State.FAILED + assert lines[0].startswith(f"{GFX94X}: {conclusion} ") + + def test_ignores_windows_jobs_with_the_same_name(self): + windows_done = math_libs("gfx94X-dcgpu", prefix=WINDOWS_STAGES, job_id=2) + linux_running = math_libs("gfx94X-dcgpu", status="in_progress") + state, _ = evaluate([windows_done, linux_running], "math-libs", [GFX94X]) + assert state is State.WAITING + + windows_failed = math_libs( + "gfx94X-dcgpu", prefix=WINDOWS_STAGES, conclusion="failure", job_id=2 + ) + state, _ = evaluate( + [windows_failed, math_libs("gfx94X-dcgpu")], "math-libs", [GFX94X] + ) + assert state is State.READY + + @pytest.mark.parametrize( + "caller", + ["math-libs", "math-libs (gfx94X-dcgpu, linux-gfx942-1gpu)"], + ) + def test_fails_when_the_stage_did_not_run(self, caller): + skipped_call = job(f"{LINUX_STAGES} / {caller}", conclusion="skipped") + state, lines = evaluate( + [SETUP, job(COMPILER_RUNTIME["name"], conclusion="failure"), skipped_call], + "math-libs", + [GFX94X], + ) + assert state is State.FAILED + assert lines[0].startswith(f"{GFX94X}: will not run; the math-libs stage was") + + def test_fails_when_the_run_builds_only_other_families(self): + state, lines = evaluate( + [math_libs("gfx950-dcgpu", status="in_progress")], "math-libs", [GFX94X] + ) + assert state is State.FAILED + assert lines == [f"{GFX94X}: not built by this run, which builds {GFX950}"] + + def test_coverage_stage_jobs_are_not_the_baseline(self): + coverage_job = job( + f"{COVERAGE} / Build Instrumented Math Libs (gfx94X-dcgpu)" + " / Stage - Coverage Math Libs (gfx94X-dcgpu)", + job_id=3, + ) + state, lines = evaluate([coverage_job], "math-libs", [GFX94X]) + assert state is State.WAITING + assert lines == [f"{GFX94X}: not started"] + + def test_waits_for_every_family(self): + names = [GFX94X, GFX950] + partial = [ + math_libs("gfx94X-dcgpu"), + math_libs("gfx950-dcgpu", status="queued", job_id=2), + ] + assert evaluate(partial, "math-libs", names)[0] is State.WAITING + + done = [math_libs("gfx94X-dcgpu"), math_libs("gfx950-dcgpu", job_id=2)] + assert evaluate(done, "math-libs", names)[0] is State.READY + + def test_compiler_runtime_stage(self): + windows_running = job( + f"{WINDOWS_STAGES} / compiler-runtime / Stage - Compiler Runtime", + status="in_progress", + job_id=2, + ) + state, _ = evaluate( + [COMPILER_RUNTIME, windows_running], + "compiler-runtime", + ["Stage - Compiler Runtime"], + ) + assert state is State.READY + + +# --------------------------------------------------------------------------- +# list_run_jobs() +# --------------------------------------------------------------------------- + + +class TestListRunJobs: + def test_paginates(self): + page1 = {"jobs": [job(f"job {i}", job_id=i) for i in range(100)]} + page2 = {"jobs": [job("last", job_id=100)]} + with patch( + "wait_for_baseline_stage.gha_send_request", side_effect=[page1, page2] + ) as send: + jobs = list_run_jobs("ROCm/rocm-libraries", "123") + assert len(jobs) == 101 + urls = [c.args[0] for c in send.call_args_list] + assert urls == [ + "https://api.github.com/repos/ROCm/rocm-libraries/actions/runs/123/jobs" + f"?filter=latest&per_page=100&page={page}" + for page in (1, 2) + ] + + def test_raises_past_the_page_limit(self): + full_page = {"jobs": [job(f"job {i}", job_id=i) for i in range(100)]} + with patch( + "wait_for_baseline_stage.gha_send_request", return_value=full_page + ) as send: + with pytest.raises(GitHubAPIError): + list_run_jobs("ROCm/rocm-libraries", "123") + assert send.call_count == MAX_PAGES + + +# --------------------------------------------------------------------------- +# wait_for_stage() +# --------------------------------------------------------------------------- + +WAITING = [math_libs("gfx94X-dcgpu", status="in_progress")] +READY = [math_libs("gfx94X-dcgpu")] +FAILED = [math_libs("gfx94X-dcgpu", conclusion="failure")] + + +def run_wait(snapshots, monotonic=None, timeout_minutes=60): + clock = {"side_effect": monotonic} if monotonic else {"return_value": 0} + with ( + patch("wait_for_baseline_stage.list_run_jobs", side_effect=snapshots), + patch("wait_for_baseline_stage.time.monotonic", **clock), + patch("wait_for_baseline_stage.time.sleep") as sleep, + ): + result = wait_for_stage( + "ROCm/rocm-libraries", + "123", + "math-libs", + [GFX94X], + poll_seconds=180, + timeout_minutes=timeout_minutes, + ) + return result, sleep + + +class TestWaitForStage: + def test_returns_zero_once_ready(self, capsys): + result, sleep = run_wait([WAITING, WAITING, READY]) + assert result == 0 + assert sleep.call_args_list == [call(180), call(180)] + # Only status changes are logged. + assert capsys.readouterr().out.count("Baseline math-libs") == 2 + + def test_fails_once_the_next_poll_confirms_a_failure(self, capsys): + result, sleep = run_wait([FAILED, FAILED]) + assert result == 1 + assert sleep.call_count == 1 + assert "::error::The baseline math-libs build did not succeed" in ( + capsys.readouterr().out + ) + + def test_unconfirmed_failure_keeps_waiting(self): + other_family_only = [math_libs("gfx950-dcgpu", status="in_progress")] + result, _ = run_wait([other_family_only, WAITING, READY]) + assert result == 0 + + def test_times_out(self, capsys): + # Deadline at 600s; the first check passes, the second is past it. + result, sleep = run_wait( + [WAITING, WAITING], monotonic=[0, 0, 601], timeout_minutes=10 + ) + assert result == 1 + assert sleep.call_count == 1 + assert "::error::Timed out after 10 minutes" in capsys.readouterr().out + + def test_gives_up_after_consecutive_api_errors(self): + errors = [GitHubAPIError("boom")] * MAX_CONSECUTIVE_API_ERRORS + result, sleep = run_wait(errors) + assert result == 1 + assert sleep.call_count == MAX_CONSECUTIVE_API_ERRORS - 1 + + def test_recovers_from_api_errors(self): + result, _ = run_wait([GitHubAPIError("boom"), READY]) + assert result == 0 + + +# --------------------------------------------------------------------------- +# main() +# --------------------------------------------------------------------------- + + +class TestMain: + def test_math_libs_needs_families(self): + with pytest.raises(SystemExit) as exc: + main( + ["--stage=math-libs", "--repository=ROCm/rocm-libraries", "--run-id=1"] + ) + assert exc.value.code == 2 + + def test_needs_repository_and_run_id(self, monkeypatch): + monkeypatch.delenv("GITHUB_REPOSITORY", raising=False) + monkeypatch.delenv("GITHUB_RUN_ID", raising=False) + with pytest.raises(SystemExit) as exc: + main(["--stage=compiler-runtime"]) + assert exc.value.code == 2 + + def test_waits_for_the_named_jobs(self, monkeypatch): + monkeypatch.setenv("GITHUB_REPOSITORY", "ROCm/rocm-libraries") + monkeypatch.setenv("GITHUB_RUN_ID", "123") + with patch( + "wait_for_baseline_stage.wait_for_stage", return_value=0 + ) as wait_mock: + assert ( + main( + ["--stage=math-libs", "--amdgpu-families=gfx94X-dcgpu;gfx950-dcgpu"] + ) + == 0 + ) + wait_mock.assert_called_once_with( + "ROCm/rocm-libraries", "123", "math-libs", [GFX94X, GFX950], 180, 300 + ) + + +# --------------------------------------------------------------------------- +# Workflow drift +# --------------------------------------------------------------------------- + + +class TestStageJobNames: + def test_match_the_regular_linux_build(self): + jobs = load_workflow(WORKFLOWS_DIR / "multi_arch_build_portable_linux.yml")[ + "jobs" + ] + for stage, template in STAGE_JOB_NAMES.items(): + stage_inputs = jobs[stage]["with"] + assert stage_inputs["stage_name"] == stage + assert stage_inputs["stage_display_name"] == template.format( + amdgpu_family="${{ matrix.family_info.amdgpu_family }}" + ) + + def test_coverage_stage_jobs_are_named_apart(self): + jobs = load_workflow(WORKFLOWS_DIR / "multi_arch_ci_coverage_linux.yml")["jobs"] + prefixes = [template.split("{")[0] for template in STAGE_JOB_NAMES.values()] + coverage_names = [ + job_def["with"]["stage_display_name"] + for job_def in jobs.values() + if "stage_display_name" in job_def.get("with", {}) + ] + assert coverage_names + for name in coverage_names: + assert not any(name.startswith(prefix) for prefix in prefixes), name diff --git a/build_tools/github_actions/wait_for_baseline_stage.py b/build_tools/github_actions/wait_for_baseline_stage.py new file mode 100644 index 00000000000..88fa3f0c7cc --- /dev/null +++ b/build_tools/github_actions/wait_for_baseline_stage.py @@ -0,0 +1,264 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Waits for this run's regular Linux build to finish a stage. + +multi_arch_ci_coverage_linux.yml tests each instrumented project against a +non-instrumented baseline that it installs from S3. When the caller builds that +baseline in the same run (the rocm-libraries nightly does), coverage can start +alongside the regular build instead of after it, as long as its tests wait for +the baseline. A reusable workflow cannot `needs:` a job inside another one, so +this polls the run's jobs and exits 0 once the regular build's stage job has +succeeded for every requested family. + +It exits 1 once one of those jobs fails, is cancelled or is skipped, or once +the run shows the job will never start: the regular build skipped the stage +(an earlier stage failed, or the caller marked it prebuilt or skipped), or +built it only for other families. + +On a public repository, listing the jobs needs no `actions` permission. + +Usage: + + python build_tools/github_actions/wait_for_baseline_stage.py \\ + --stage=math-libs --amdgpu-families="gfx94X-dcgpu" +""" + +import argparse +from datetime import datetime, timezone +from enum import Enum +import os +import re +import sys +import time + +from github_actions_api import GitHubAPIError, gha_send_request + +# stage_display_name of each stage's job in multi_arch_build_portable_linux.yml, +# which keys every stage's job by the stage name. +STAGE_JOB_NAMES = { + "compiler-runtime": "Stage - Compiler Runtime", + "math-libs": "Stage - Math Libs ({amdgpu_family})", +} + +JOBS_PER_PAGE = 100 +MAX_PAGES = 20 +REQUEST_TIMEOUT_SECONDS = 60 +MAX_CONSECUTIVE_API_ERRORS = 5 + + +class State(Enum): + READY = "ready" + WAITING = "waiting" + FAILED = "failed" + + +def log(*args): + timestamp = datetime.now(timezone.utc).strftime("%H:%M:%SZ") + print(f"[{timestamp}]", *args, flush=True) + + +def parse_families(value: str) -> list[str]: + """Splits a comma- or semicolon-separated family list.""" + return [family.strip() for family in re.split(r"[,;]", value) if family.strip()] + + +def expected_job_names(stage: str, amdgpu_families: list[str]) -> list[str]: + template = STAGE_JOB_NAMES[stage] + if "{amdgpu_family}" not in template: + return [template] + return [template.format(amdgpu_family=family) for family in amdgpu_families] + + +def list_run_jobs(repository: str, run_id: str) -> list[dict]: + """Returns the latest attempt of every job in the run.""" + jobs: list[dict] = [] + for page in range(1, MAX_PAGES + 1): + url = ( + f"https://api.github.com/repos/{repository}/actions/runs/{run_id}/jobs" + f"?filter=latest&per_page={JOBS_PER_PAGE}&page={page}" + ) + response = gha_send_request(url, timeout_seconds=REQUEST_TIMEOUT_SECONDS) + page_jobs = response.get("jobs", []) + jobs.extend(page_jobs) + if len(page_jobs) < JOBS_PER_PAGE: + return jobs + raise GitHubAPIError(f"Run {run_id} has more than {len(jobs)} jobs") + + +def _name_parts(job: dict) -> list[str]: + # Jobs inside reusable workflows are named " / ... / ". + return job.get("name", "").split(" / ") + + +def _is_windows(job: dict) -> bool: + # Windows builds reuse the stage job names. multi_arch_ci.yml and the + # rocm-libraries nightly name the top-level job Windows::. + return _name_parts(job)[0].lower().startswith("windows") + + +def _describe(job: dict) -> str: + return f"{job.get('conclusion') or job.get('status')} {job.get('html_url', '')}" + + +def _did_not_succeed(job: dict) -> bool: + return job.get("status") == "completed" and job.get("conclusion") != "success" + + +def evaluate( + jobs: list[dict], stage: str, job_names: list[str] +) -> tuple[State, list[str]]: + """Returns whether every job in job_names succeeded, with a line per job.""" + jobs_by_leaf: dict[str, list[dict]] = {} + for job in jobs: + if not _is_windows(job): + jobs_by_leaf.setdefault(_name_parts(job)[-1], []).append(job) + + # A reusable-workflow call that did not run is listed as a single job named + # after the caller: the stage name, plus the matrix values once expanded. + stage_not_run = [ + job + for leaf, leaf_jobs in jobs_by_leaf.items() + if leaf == stage or leaf.startswith(f"{stage} (") + for job in leaf_jobs + if _did_not_succeed(job) + ] + template = STAGE_JOB_NAMES[stage] + family_prefix = template.split("{")[0] if "{" in template else None + built_names = sorted( + leaf + for leaf in jobs_by_leaf + if family_prefix and leaf.startswith(family_prefix) + ) + + lines = [] + failed = waiting = False + for name in job_names: + matches = jobs_by_leaf.get(name, []) + unsuccessful = [job for job in matches if _did_not_succeed(job)] + unfinished = [job for job in matches if job.get("status") != "completed"] + if unsuccessful: + failed = True + lines.extend(f"{name}: {_describe(job)}" for job in unsuccessful) + elif unfinished: + waiting = True + lines.extend(f"{name}: {_describe(job)}" for job in unfinished) + elif matches: + lines.extend(f"{name}: {_describe(job)}" for job in matches) + elif stage_not_run: + failed = True + lines.append( + f"{name}: will not run; the {stage} stage was " + f"{_describe(stage_not_run[0])}" + ) + elif built_names: + failed = True + lines.append( + f"{name}: not built by this run, which builds {', '.join(built_names)}" + ) + else: + waiting = True + lines.append(f"{name}: not started") + + if failed: + return State.FAILED, lines + if waiting: + return State.WAITING, lines + return State.READY, lines + + +def wait_for_stage( + repository: str, + run_id: str, + stage: str, + job_names: list[str], + poll_seconds: int, + timeout_minutes: int, +) -> int: + deadline = time.monotonic() + timeout_minutes * 60 + api_errors = 0 + last_lines: list[str] | None = None + unconfirmed_failure: list[str] | None = None + while True: + try: + jobs = list_run_jobs(repository, run_id) + except GitHubAPIError as e: + api_errors += 1 + log(f"Could not list jobs ({api_errors}/{MAX_CONSECUTIVE_API_ERRORS}): {e}") + if api_errors >= MAX_CONSECUTIVE_API_ERRORS: + print("::error::Gave up listing this run's jobs") + return 1 + else: + api_errors = 0 + state, lines = evaluate(jobs, stage, job_names) + if lines != last_lines: + log(f"Baseline {stage} {state.value}:") + for line in lines: + print(f" {line}", flush=True) + last_lines = lines + if state is State.READY: + return 0 + if state is State.FAILED: + # A snapshot spans several API pages, so act on a failure only + # once the next poll shows it too. + if lines == unconfirmed_failure: + print( + f"::error::The baseline {stage} build did not succeed;" + " coverage tests would run against missing artifacts" + ) + return 1 + unconfirmed_failure = lines + else: + unconfirmed_failure = None + + if time.monotonic() >= deadline: + print( + f"::error::Timed out after {timeout_minutes} minutes waiting for" + f" the baseline {stage} build" + ) + return 1 + time.sleep(poll_seconds) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Wait for this run's regular Linux build to finish a stage." + ) + parser.add_argument("--stage", required=True, choices=sorted(STAGE_JOB_NAMES)) + parser.add_argument( + "--amdgpu-families", + default="", + help="Comma- or semicolon-separated families; required for math-libs.", + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", ""), + help="Repository in owner/repo format (default: $GITHUB_REPOSITORY).", + ) + parser.add_argument( + "--run-id", + default=os.environ.get("GITHUB_RUN_ID", ""), + help="Workflow run ID (default: $GITHUB_RUN_ID).", + ) + parser.add_argument("--poll-seconds", type=int, default=180) + parser.add_argument("--timeout-minutes", type=int, default=300) + args = parser.parse_args(argv) + + if not args.repository or not args.run_id: + parser.error("--repository and --run-id are required outside GitHub Actions") + job_names = expected_job_names(args.stage, parse_families(args.amdgpu_families)) + if not job_names: + parser.error(f"--stage={args.stage} needs --amdgpu-families") + + return wait_for_stage( + args.repository, + args.run_id, + args.stage, + job_names, + args.poll_seconds, + args.timeout_minutes, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 7ae04c76e0d..433b7cff9a2 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -119,7 +119,9 @@ graph TD dispatch[Dispatch with projects_to_test] --> matrix[setup_coverage_matrix] matrix --> compilerRuntime[Build compiler-runtime] compilerRuntime --> mathLibs[Build instrumented math-libs] + matrix --> gate[Wait for a same-run baseline] mathLibs --> report[Per project: configure, test, report] + gate --> report report --> codecov[Codecov] ``` @@ -171,6 +173,25 @@ Coverage sets three of its inputs: nightly measures the commit it is testing. Without it the build uses TheRock's pinned submodules. +### Waiting for a same-run baseline + +Only the tests need the baseline, so a caller that builds it in the same run +(the rocm-libraries nightly) can start coverage alongside that build. When +`baseline_run_id` is this run's id, `wait_for_baseline` runs +`wait_for_baseline_stage.py`, which polls the run's jobs until the regular +Linux build's `Stage - Math Libs ()` job has succeeded for every family +(`Stage - Compiler Runtime` when no math-libs project is selected). +`coverage_report` needs it, so tests start once the baseline is in S3. A +baseline from an earlier run skips the wait. + +The wait fails, skipping the tests, when that job fails, when the stage never +runs (an earlier stage failed, or the caller marked it prebuilt or skipped), +when the run builds the stage only for other families, or after 300 minutes. +Jobs are matched by display name, ignoring those under the `Windows::` job, +which reuse the names; `wait_for_baseline_stage_test.py` checks the names +against `multi_arch_build_portable_linux.yml`. On a public repository, listing +the jobs needs no `actions` permission. + ### Test execution Per shard, `test_code_coverage_component.yml` does three things: From aaec6812a674474ad7a9d34b865506bedf9f3d7b Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 5 Oct 2026 14:25:37 -0500 Subject: [PATCH 54/70] fix(coverage): keep the profile runtime's GPU half out of instrumented links Instrumented tests in the rocm-libraries nightly (run 37288017642) abort at exit with "Cannot create GlobalVar Obj for symbol: __llvm_profile_sections_". Keeping kernels uninstrumented does not stop clang from registering a host-side shadow of every HIP translation unit's device counters, or the driver from force-linking InstrProfilingPlatformROCm.cpp to take the registrations. At exit that file asks the HIP runtime for each shadow's device side, which only exists when the device code has profile data, and the HIP runtime aborts once the first module has written its profile. Every other module loses its profile; with an instrumented test executable, that includes the library under test. Coverage subprojects now compile cmake/therock_coverage_profile_stub.c at configure time and link it into every executable and shared library. It defines the file's five entry points as hidden no-ops, so the real one, with its HIP interceptors and HSA drain, is never pulled in. A compiler that adds an entry point the stub lacks fails the link on duplicate symbols. -fuse-cuid=none would stop the shadows instead, but every translation unit then defines the same __hip_cuid_, and libraries with two HIP sources fail to link. Co-authored-by: Cursor --- cmake/therock_coverage_profile_stub.c | 28 +++++++++++++++++ cmake/therock_subproject.cmake | 44 +++++++++++++++++++++++++-- docs/development/code_coverage.md | 23 ++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 cmake/therock_coverage_profile_stub.c diff --git a/cmake/therock_coverage_profile_stub.c b/cmake/therock_coverage_profile_stub.c new file mode 100644 index 00000000000..caa16beb34c --- /dev/null +++ b/cmake/therock_coverage_profile_stub.c @@ -0,0 +1,28 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +// No-op definitions of every entry point into the GPU half of the LLVM profile +// runtime (InstrProfilingPlatformROCm.cpp in libclang_rt.profile_rocm.a). +// therock_subproject.cmake links this into each coverage-instrumented +// executable and shared library so that the real ones are never linked; see the +// coverage comments there. + +#pragma GCC visibility push(hidden) + +void __llvm_profile_offload_register_shadow_variable(void *Ptr) { (void)Ptr; } + +void __llvm_profile_offload_register_section_shadow_variable(void *Ptr) { + (void)Ptr; +} + +void __llvm_profile_offload_register_dynamic_module(int ModuleLoadRc, + void **Ptr, + const void *Image) { + (void)ModuleLoadRc; + (void)Ptr; + (void)Image; +} + +void __llvm_profile_offload_unregister_dynamic_module(void *Ptr) { (void)Ptr; } + +int __llvm_profile_hip_collect_device_data(void) { return 0; } diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 75afcc0ebcf..91533b765d0 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -999,12 +999,52 @@ function(therock_cmake_subproject_activate target_name) " CMAKE_${_coverage_lang}_COMPILE_OBJECT \"\${CMAKE_${_coverage_lang}_COMPILE_OBJECT}\")\n" "endif()\n") endforeach() + # Uninstrumented kernels don't stop clang from giving every HIP translation + # unit a host-side shadow of its device counters, registered from a + # constructor, or the driver from force-linking the GPU half of the profile + # runtime (InstrProfilingPlatformROCm.cpp in libclang_rt.profile_rocm.a) + # to take the registrations. At exit that half asks the HIP runtime for the + # device side of each shadow, which only exists when the device code has + # profile data. The HIP runtime aborts ("Cannot create GlobalVar Obj") right + # after the first module writes its profile, so every other module's is + # lost. The GPU half also wraps hipModuleLoad*, hipLaunchKernel and the + # like. Linking a stub that defines each of its entry points as a no-op into + # every executable and shared library leaves nothing to pull it in for. + # -fuse-cuid=none would stop the shadows instead, but then every translation + # unit defines the same __hip_cuid_ and any library with two HIP sources + # fails to link. Should a compiler update add an entry point the stub lacks, + # links fail on duplicate symbols rather than bringing the GPU half back. + set(_coverage_profile_stub_source + "${THEROCK_SOURCE_DIR}/cmake/therock_coverage_profile_stub.c") + string(APPEND _coverage_host_only_contents + "set(_therock_coverage_stub \"\${CMAKE_BINARY_DIR}/therock_coverage_profile_stub.o\")\n" + "if(\"${_coverage_profile_stub_source}\" IS_NEWER_THAN \"\${_therock_coverage_stub}\")\n" + " if(CMAKE_C_COMPILER)\n" + " set(_therock_coverage_cc \"\${CMAKE_C_COMPILER}\")\n" + " else()\n" + " set(_therock_coverage_cc \"\${CMAKE_CXX_COMPILER}\")\n" + " endif()\n" + " execute_process(\n" + " COMMAND \"\${_therock_coverage_cc}\" -x c -O2 -fPIC -c\n" + " \"${_coverage_profile_stub_source}\" -o \"\${_therock_coverage_stub}\"\n" + " RESULT_VARIABLE _therock_coverage_result)\n" + " if(NOT _therock_coverage_result EQUAL 0)\n" + " message(FATAL_ERROR \"Could not compile ${_coverage_profile_stub_source}\")\n" + " endif()\n" + "endif()\n" + "foreach(_therock_coverage_kind IN ITEMS EXE SHARED MODULE)\n" + " if(NOT CMAKE_\${_therock_coverage_kind}_LINKER_FLAGS MATCHES \"therock_coverage_profile_stub\")\n" + " string(APPEND CMAKE_\${_therock_coverage_kind}_LINKER_FLAGS \" \${_therock_coverage_stub}\")\n" + " endif()\n" + "endforeach()\n") file(CONFIGURE OUTPUT "${_cmake_project_coverage_file}" CONTENT "${_coverage_host_only_contents}" @ONLY ESCAPE_QUOTES) - list(APPEND _fprint_files "${_cmake_project_coverage_file}") + list(APPEND _fprint_files + "${_cmake_project_coverage_file}" "${_coverage_profile_stub_source}") set(_coverage_project_include_arg "-DCMAKE_PROJECT_INCLUDE=${_cmake_project_coverage_file}") - set(_coverage_configure_depends "${_cmake_project_coverage_file}") + set(_coverage_configure_depends + "${_cmake_project_coverage_file}" "${_coverage_profile_stub_source}") endif() file(CONFIGURE OUTPUT "${_cmake_project_init_file}" CONTENT "${_init_contents}" @ONLY ESCAPE_QUOTES) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 433b7cff9a2..4a1e9ccee34 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -89,6 +89,29 @@ shared library gets undefined references and any executable linking it fails `--no-allow-shlib-undefined`. `therock_subproject.cmake` adds both libraries automatically. +### The profile runtime's GPU half + +Keeping kernels uninstrumented does not stop the abort described above. Clang +still gives every HIP translation unit a host-side shadow of its device +counters, registered from a constructor. The driver still force-links +`InstrProfilingPlatformROCm.cpp` from `libclang_rt.profile_rocm.a` to take those +registrations. At exit, once a module has written its host profile, that file +asks the HIP runtime for the device side of each shadow. The device side only +exists when the device code has profile data, so the HIP runtime aborts +(`Cannot create GlobalVar Obj for symbol: __llvm_profile_sections_`). Every +module whose profile writer has not run yet loses its profile, which is how an +instrumented test executable costs the library under test its coverage. The same +file also wraps `hipModuleLoad*`, `hipLaunchKernel` and the like. + +`therock_subproject.cmake` therefore compiles +`cmake/therock_coverage_profile_stub.c`, which defines that file's five entry +points as hidden no-ops. It adds the object to `CMAKE_EXE_LINKER_FLAGS`, +`CMAKE_SHARED_LINKER_FLAGS` and `CMAKE_MODULE_LINKER_FLAGS`, so nothing pulls in +the real one. If a compiler update adds an entry point the stub lacks, links +fail on duplicate symbols. `-fuse-cuid=none` would remove the shadows instead, +but then every translation unit defines the same `__hip_cuid_`, and any library +with two HIP sources fails to link. + ## Producing a report locally Set `LLVM_PROFILE_FILE` with `%p`/`%m` substitutions to keep per-process files From 76a50258c3b69e91e858477dcd7e9786817d92cc Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Mon, 5 Oct 2026 14:25:46 -0500 Subject: [PATCH 55/70] fix(coverage): pin the program header table kpack moves in instrumented libraries kpack moves the program header table of each library it rewrites to the end of the file, mapped by a PT_LOAD of its own at an address only page-congruent with the file offset. compiler-rt's profile writer reads the table at __ehdr_start + e_phoff to record binary IDs, so instrumented rocBLAS and rocSPARSE segfault at exit before writing a counter, and rocRAND reads another segment's bytes instead. kpack already pins p_vaddr == p_offset for executables (normalize_phdr_vaddr). Until it does the same for shared libraries, install_rocm_code_coverage_build.py applies that edit to each replaced file with a __llvm_prf_cnts section once it is extracted (_therock_utils/elf_phdr.py). Uninstrumented libraries and the baseline install are left as they are. Co-authored-by: Cursor --- build_tools/_therock_utils/elf_phdr.py | 175 +++++++++++++ .../install_rocm_code_coverage_build.py | 13 +- build_tools/tests/elf_phdr_test.py | 247 ++++++++++++++++++ .../install_rocm_code_coverage_build_test.py | 48 +++- docs/development/code_coverage.md | 20 +- 5 files changed, 500 insertions(+), 3 deletions(-) create mode 100644 build_tools/_therock_utils/elf_phdr.py create mode 100644 build_tools/tests/elf_phdr_test.py diff --git a/build_tools/_therock_utils/elf_phdr.py b/build_tools/_therock_utils/elf_phdr.py new file mode 100644 index 00000000000..ebdb66ca4d4 --- /dev/null +++ b/build_tools/_therock_utils/elf_phdr.py @@ -0,0 +1,175 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Puts a relocated ELF program header table where in-process readers look. + +kpack moves the program header table of a binary it rewrites to the end of the +file, mapped by a PT_LOAD of its own whose p_vaddr is only page-congruent with +the table's file offset. ld.so copes, but code that finds the table at +__ehdr_start + e_phoff reads memory that is not mapped. compiler-rt's profile +writer does exactly that to record binary IDs, so an instrumented library that +kpack has rewritten segfaults at exit before writing a single counter. + +kpack pins p_vaddr == p_offset for executables (normalize_phdr_vaddr in +rocm_kpack/elf/phdr_manager.py) and leaves shared libraries alone. +pin_phdr_table() applies the same edit to a shared library. +""" + +import os +import struct +from pathlib import Path + +_EHDR = struct.Struct("<16sHHIQQQIHHHHHH") +_PHDR = struct.Struct(" bytes: + f.seek(offset) + data = f.read(size) + if len(data) != size: + raise ValueError(f"truncated ELF: wanted {size} bytes at {offset:#x}") + return data + + +def _round_up(value: int, alignment: int) -> int: + return (value + alignment - 1) // alignment * alignment + + +def _section_names(f, e_shoff: int, e_shentsize: int, e_shnum: int, e_shstrndx: int): + if not e_shoff or e_shentsize < _SHDR.size: + return set() + first = _SHDR.unpack(_read_at(f, e_shoff, _SHDR.size)) + if e_shnum == 0: + e_shnum = first[5] + if e_shstrndx == _SHN_XINDEX: + e_shstrndx = first[6] + if e_shstrndx >= e_shnum: + return set() + table = _read_at(f, e_shoff, e_shnum * e_shentsize) + sections = [_SHDR.unpack_from(table, i * e_shentsize) for i in range(e_shnum)] + strtab = sections[e_shstrndx] + names = _read_at(f, strtab[4], strtab[5]) + return { + names[s[0] : names.find(b"\0", s[0])].decode(errors="replace") + for s in sections + if s[0] < len(names) + } + + +def pin_phdr_table(path: Path, *, require_section: str | None = None) -> bool: + """Pins a shared library's relocated program header table to p_vaddr == p_offset. + + When the table already sits above every other PT_LOAD's address range this + only rewrites two program headers; otherwise the table is copied to a new + end-of-file offset above that range and e_phoff follows it, as kpack does + for executables. + + Leaves the file untouched, returning False, unless it is a little-endian + ELF64 x86-64 shared library without a PT_INTERP whose table lives in a + PT_LOAD of its own, the last one, starting at e_phoff and mapped somewhere + else, and, when require_section is given, it has a section of that name. + Returns True if the file was modified. + """ + with open(path, "r+b") as f: + header = f.read(_EHDR.size) + if len(header) < _EHDR.size or header[:4] != b"\x7fELF": + return False + if header[4] != 2 or header[5] != 1: # ELFCLASS64, ELFDATA2LSB + return False + ( + _, + e_type, + e_machine, + _, + _, + e_phoff, + e_shoff, + _, + _, + e_phentsize, + e_phnum, + e_shentsize, + e_shnum, + e_shstrndx, + ) = _EHDR.unpack(header) + if e_type != _ET_DYN or e_machine != _EM_X86_64: + return False + if e_phnum in (0, _PN_XNUM) or e_phentsize < _PHDR.size: + return False + + table = bytearray(_read_at(f, e_phoff, e_phnum * e_phentsize)) + phdrs = [ + list(_PHDR.unpack_from(table, i * e_phentsize)) for i in range(e_phnum) + ] + if any(p[_TYPE] == _PT_INTERP for p in phdrs): + return False + cover_index = next( + ( + i + for i, p in enumerate(phdrs) + if p[_TYPE] == _PT_LOAD + and p[_OFFSET] <= e_phoff < p[_OFFSET] + p[_FILESZ] + ), + None, + ) + if cover_index is None: + return False + cover = phdrs[cover_index] + if cover[_OFFSET] != e_phoff or cover[_VADDR] == cover[_OFFSET]: + return False + # The pinned table ends up above every other segment, so only the last + # PT_LOAD can take it without breaking their ascending p_vaddr order. + if any(p[_TYPE] == _PT_LOAD for p in phdrs[cover_index + 1 :]): + return False + if require_section is not None and require_section not in _section_names( + f, e_shoff, e_shentsize, e_shnum, e_shstrndx + ): + return False + + alignment = max(cover[_ALIGN], _PAGE_SIZE) + ceiling = _round_up( + max( + ( + p[_VADDR] + p[_MEMSZ] + for i, p in enumerate(phdrs) + if p[_TYPE] == _PT_LOAD and i != cover_index + ), + default=0, + ), + alignment, + ) + new_phoff = e_phoff + if e_phoff < ceiling: + # Pinning the table where it is would overlap another segment. + # Copy the whole covering segment, spare slots included, so the + # file holds every byte the PT_LOAD claims. + cover_bytes = _read_at(f, e_phoff, cover[_FILESZ]) + file_size = f.seek(0, os.SEEK_END) + new_phoff = _round_up(max(file_size, ceiling), alignment) + f.write(b"\0" * (new_phoff - file_size)) + f.write(cover_bytes) + + for i, p in enumerate(phdrs): + if i == cover_index or p[_TYPE] == _PT_PHDR: + p[_OFFSET] = p[_VADDR] = p[_PADDR] = new_phoff + _PHDR.pack_into(table, i * e_phentsize, *p) + f.seek(new_phoff) + f.write(table) + if new_phoff != e_phoff: + f.seek(_E_PHOFF_OFFSET) + f.write(struct.pack(" flag to its TheRock artifact and library folder. # e.g. rocBLAS ships in the 'blas' artifact (see BUILD_TOPOLOGY.toml). @@ -190,7 +191,10 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): Reads each archive's artifact_manifest.txt for relpath prefixes, then flattens members into output_dir -- but only those matching the artifact's library - folder (e.g. rocBLAS), leaving unrelated files in place. + folder (e.g. rocBLAS), leaving unrelated files in place. Instrumented shared + libraries whose program header table kpack moved get it pinned back (see + _therock_utils/elf_phdr.py); otherwise their profile writer can segfault + at exit before writing any counters. """ archives = sorted( p for p in dest_dir.iterdir() if p.name.endswith((".tar.zst", ".tar.xz")) @@ -199,6 +203,7 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): log(f"No replacement archives found in {dest_dir}") return + written_files = [] for archive in archives: an = ArtifactName.from_filename(archive.name) folders = artifacts.get(an.name) if an else None @@ -235,10 +240,16 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): _replace_scoped_member( tf, member, dest_path, output_dir, relpaths ) + if member.isfile(): + written_files.append(dest_path) replaced += 1 break log(f" Replaced {replaced} '{folder}' path(s) from {archive.name}") + for path in written_files: + if pin_phdr_table(path, require_section="__llvm_prf_cnts"): + log(f" Pinned the program header table kpack moved in {path}") + def main(argv): parser = argparse.ArgumentParser(prog="code-coverage-installer") diff --git a/build_tools/tests/elf_phdr_test.py b/build_tools/tests/elf_phdr_test.py new file mode 100644 index 00000000000..1726120f7df --- /dev/null +++ b/build_tools/tests/elf_phdr_test.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +import collections +import struct +import tempfile +import unittest +from pathlib import Path + +from _therock_utils.elf_phdr import pin_phdr_table + +_EHDR = struct.Struct("<16sHHIQQQIHHHHHH") +_PHDR = struct.Struct("-coverage`). + project-under-test from `--code-coverage-run-id` (`-coverage`). It + then moves each replaced library's program header table back to where the + profile writer looks for it (see + [Program headers kpack moved](#program-headers-kpack-moved)). 1. **Set `LLVM_PROFILE_FILE`** to a per-shard path (`%p`/`%m`) so concurrent processes don't overwrite each other. 1. **Run tests and upload profraw** under `always()` — a failing shard still @@ -251,6 +254,21 @@ but extracts only the paths matching the project's library folder in `-coverage`, so even a baseline from the same workflow run is never overwritten by it. +#### Program headers kpack moved + +kpack (`KPACK_SPLIT_ARTIFACTS`, on by default) moves the program header table +of each library it rewrites, the ones carrying device code, to the end of the +file. A `PT_LOAD` of its own maps the table there, at an address that is only +page-congruent with its file offset. ld.so copes. compiler-rt's profile writer +does not: to record binary IDs it reads the table at `__ehdr_start + e_phoff`. +For such a library that address is either unmapped, so rocBLAS and rocSPARSE +segfault at exit before writing a counter, or inside another segment, so +rocRAND reads unrelated bytes. kpack already pins `p_vaddr == p_offset` for +executables (`normalize_phdr_vaddr` in `rocm_kpack/elf/phdr_manager.py`). The +installer applies the same edit to every replaced shared library that has a +`__llvm_prf_cnts` section (`_therock_utils/elf_phdr.py`), growing the file when +the table has to move above the library's address range. + A missing instrumented artifact fails the install step. A swap that matches no files does not: the installer logs `Replaced 0` and the tests run against the baseline's uninstrumented binaries. The failure then surfaces in the report From f1aab3c165e525cd3a5b84c5e3032782d1d5da38 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 6 Oct 2026 14:43:28 +0530 Subject: [PATCH 56/70] ci(coverage): pushing coverage results to codecov.io --- .github/workflows/multi_arch_ci_coverage_report.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 35aac6116a0..08b4f1f7308 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -191,7 +191,6 @@ jobs: OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build PROFRAW_DIR: ${{ github.workspace }}/coverage-profraw REPORT_DIR: ${{ github.workspace }}/coverage-report - HAS_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN != '' }} steps: - name: Checking out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -300,10 +299,8 @@ jobs: # Skip (not fail) when there's no Codecov token, so forks and first-time # setups still get the lcov artifact above. - name: Upload report to Codecov - if: ${{ env.HAS_CODECOV_TOKEN == 'true' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: - token: ${{ secrets.CODECOV_TOKEN }} files: ${{ env.REPORT_DIR }}/coverage.info flags: ${{ inputs.codecov_flag || inputs.project_name }} name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} From f642523fb8c995fa195ad8fc2195f94b5ce93a14 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Wed, 7 Oct 2026 23:46:30 -0500 Subject: [PATCH 57/70] fix(coverage): resolve HIP calls before rocSPARSE's profile runtime archives The coverage math-libs build in rocm-libraries run 37441751375 fails to link librocsparse.so on duplicate symbols between the stub and InstrProfilingPlatformROCm.cpp, the GPU half of the profile runtime it stands in for. rocSPARSE names libclang_rt.profile.a and libclang_rt.profile_rocm.a on its own link line, ahead of the HIP runtime. The GPU half also defines weak hipLaunchKernel, hipModuleLoad and similar wrappers, so lld extracts it from libclang_rt.profile.a for the hipLaunchKernel references rocSPARSE's objects still have at that point. The generated coverage include now puts the toolchain's libamdhip64.so right after the stub, under --as-needed, so those names already resolve to the HIP runtime when any archive is scanned. Links that don't call into HIP gain no dependency on it, and subprojects without THEROCK_TOOLCHAIN_ROOT get the stub alone, as before. Co-authored-by: Cursor --- cmake/therock_coverage_profile_stub.c | 3 +- cmake/therock_subproject.cmake | 42 ++++++++++++++++++--------- docs/development/code_coverage.md | 26 ++++++++++------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/cmake/therock_coverage_profile_stub.c b/cmake/therock_coverage_profile_stub.c index caa16beb34c..3b4069f016c 100644 --- a/cmake/therock_coverage_profile_stub.c +++ b/cmake/therock_coverage_profile_stub.c @@ -2,7 +2,8 @@ // SPDX-License-Identifier: MIT // No-op definitions of every entry point into the GPU half of the LLVM profile -// runtime (InstrProfilingPlatformROCm.cpp in libclang_rt.profile_rocm.a). +// runtime (InstrProfilingPlatformROCm.cpp, in libclang_rt.profile.a and +// libclang_rt.profile_rocm.a). // therock_subproject.cmake links this into each coverage-instrumented // executable and shared library so that the real ones are never linked; see the // coverage comments there. diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 91533b765d0..eb797353581 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -1002,18 +1002,26 @@ function(therock_cmake_subproject_activate target_name) # Uninstrumented kernels don't stop clang from giving every HIP translation # unit a host-side shadow of its device counters, registered from a # constructor, or the driver from force-linking the GPU half of the profile - # runtime (InstrProfilingPlatformROCm.cpp in libclang_rt.profile_rocm.a) - # to take the registrations. At exit that half asks the HIP runtime for the - # device side of each shadow, which only exists when the device code has - # profile data. The HIP runtime aborts ("Cannot create GlobalVar Obj") right - # after the first module writes its profile, so every other module's is - # lost. The GPU half also wraps hipModuleLoad*, hipLaunchKernel and the - # like. Linking a stub that defines each of its entry points as a no-op into - # every executable and shared library leaves nothing to pull it in for. - # -fuse-cuid=none would stop the shadows instead, but then every translation - # unit defines the same __hip_cuid_ and any library with two HIP sources - # fails to link. Should a compiler update add an entry point the stub lacks, - # links fail on duplicate symbols rather than bringing the GPU half back. + # runtime (InstrProfilingPlatformROCm.cpp, in both libclang_rt.profile.a + # and libclang_rt.profile_rocm.a) to take the registrations. At exit that + # half asks the HIP runtime for the device side of each shadow, which only + # exists when the device code has profile data. The HIP runtime aborts + # ("Cannot create GlobalVar Obj") right after the first module writes its + # profile, so every other module's is lost. Linking a stub that defines + # each of its entry points as a no-op into every executable and shared + # library takes the registrations instead. -fuse-cuid=none would stop the + # shadows instead, but then every translation unit defines the same + # __hip_cuid_ and any library with two HIP sources fails to link. Should a + # compiler update add an entry point the stub lacks, links fail on + # duplicate symbols rather than bringing the GPU half back. + # + # The GPU half also wraps hipLaunchKernel, hipModuleLoad and the like, as + # weak definitions of those names, so it is extracted for them too by a + # link that reaches the profile runtime archives while they are still + # unresolved. The driver adds the archives after the HIP runtime, but + # rocSPARSE names them itself, ahead of it. Putting the HIP runtime in + # front of every object resolves those functions first; --as-needed keeps + # it from becoming a dependency of anything that does not call into it. set(_coverage_profile_stub_source "${THEROCK_SOURCE_DIR}/cmake/therock_coverage_profile_stub.c") string(APPEND _coverage_host_only_contents @@ -1032,9 +1040,17 @@ function(therock_cmake_subproject_activate target_name) " message(FATAL_ERROR \"Could not compile ${_coverage_profile_stub_source}\")\n" " endif()\n" "endif()\n" + "set(_therock_coverage_hip \"\")\n" + "if(THEROCK_TOOLCHAIN_ROOT)\n" + " file(GLOB _therock_coverage_hip_libs \"\${THEROCK_TOOLCHAIN_ROOT}/lib/libamdhip64.so*\")\n" + " if(_therock_coverage_hip_libs)\n" + " list(GET _therock_coverage_hip_libs 0 _therock_coverage_hip_lib)\n" + " set(_therock_coverage_hip \" -Wl,--push-state,--as-needed \${_therock_coverage_hip_lib} -Wl,--pop-state\")\n" + " endif()\n" + "endif()\n" "foreach(_therock_coverage_kind IN ITEMS EXE SHARED MODULE)\n" " if(NOT CMAKE_\${_therock_coverage_kind}_LINKER_FLAGS MATCHES \"therock_coverage_profile_stub\")\n" - " string(APPEND CMAKE_\${_therock_coverage_kind}_LINKER_FLAGS \" \${_therock_coverage_stub}\")\n" + " string(APPEND CMAKE_\${_therock_coverage_kind}_LINKER_FLAGS \" \${_therock_coverage_stub}\${_therock_coverage_hip}\")\n" " endif()\n" "endforeach()\n") file(CONFIGURE OUTPUT "${_cmake_project_coverage_file}" diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 6db2a116d44..42662002c6d 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -94,23 +94,29 @@ automatically. Keeping kernels uninstrumented does not stop the abort described above. Clang still gives every HIP translation unit a host-side shadow of its device counters, registered from a constructor. The driver still force-links -`InstrProfilingPlatformROCm.cpp` from `libclang_rt.profile_rocm.a` to take those -registrations. At exit, once a module has written its host profile, that file -asks the HIP runtime for the device side of each shadow. The device side only -exists when the device code has profile data, so the HIP runtime aborts +`InstrProfilingPlatformROCm.cpp`, which both `libclang_rt.profile.a` and +`libclang_rt.profile_rocm.a` carry, to take those registrations. At exit, once +a module has written its host profile, that file asks the HIP runtime for the +device side of each shadow. The device side only exists when the device code +has profile data, so the HIP runtime aborts (`Cannot create GlobalVar Obj for symbol: __llvm_profile_sections_`). Every module whose profile writer has not run yet loses its profile, which is how an instrumented test executable costs the library under test its coverage. The same -file also wraps `hipModuleLoad*`, `hipLaunchKernel` and the like. +file also wraps `hipLaunchKernel`, `hipModuleLoad` and the like, as weak +definitions of those names. `therock_subproject.cmake` therefore compiles `cmake/therock_coverage_profile_stub.c`, which defines that file's five entry points as hidden no-ops. It adds the object to `CMAKE_EXE_LINKER_FLAGS`, -`CMAKE_SHARED_LINKER_FLAGS` and `CMAKE_MODULE_LINKER_FLAGS`, so nothing pulls in -the real one. If a compiler update adds an entry point the stub lacks, links -fail on duplicate symbols. `-fuse-cuid=none` would remove the shadows instead, -but then every translation unit defines the same `__hip_cuid_`, and any library -with two HIP sources fails to link. +`CMAKE_SHARED_LINKER_FLAGS` and `CMAKE_MODULE_LINKER_FLAGS`, followed by the +HIP runtime under `--as-needed`, so nothing pulls in the real one. The HIP +runtime has to come first because a link that reaches the profile runtime +archives while `hipLaunchKernel` is still unresolved extracts the real file for +its wrappers. The driver adds the archives after the HIP runtime, but rocSPARSE +names them itself, ahead of it. If a compiler update adds an entry point the +stub lacks, links fail on duplicate symbols. `-fuse-cuid=none` would remove the +shadows instead, but then every translation unit defines the same +`__hip_cuid_`, and any library with two HIP sources fails to link. ## Producing a report locally From 80e80edc0e1c4818041220faad4b61a07cab749a Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Fri, 9 Oct 2026 01:44:06 -0500 Subject: [PATCH 58/70] fix(coverage): swap in every file of the project's stage directories rocThrust's coverage report in rocm-libraries run 37803818804 fails with none of its 67275 instrumented lines run. Its test job installed the baseline and replaced only files whose path names the project, which for rocThrust's tests left 2 of 372: bin/rocthrust/generate_resource_spec and CTestTestfile.cmake. ctest then ran the baseline's uninstrumented bin/*.hip, and the one profraw uploaded came from generate_resource_spec. rocPRIM's tests (bin/test_block_scan) and rocWMMA's (bin/gemm_*-validate) are missed the same way; their reports pass on the 1.72% and 2.20% that their few swapped binaries happen to share with them. A file under the project's own stage directory in the artifact manifest, math-libs/rocThrust/stage or math-libs/rocPRIM_tests/stage, is now replaced whatever its name. Paths elsewhere still have to name the project, and so does the device code under .kpack/, which holds every project's kernels in a multi-project artifact like prim; rocWMMA's, named after it, is replaced as before. Co-authored-by: Cursor --- .../install_rocm_code_coverage_build.py | 23 ++++- .../install_rocm_code_coverage_build_test.py | 90 ++++++++++++++++--- docs/development/code_coverage.md | 16 +++- 3 files changed, 110 insertions(+), 19 deletions(-) diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index 79ff3c6695d..f78c5694609 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -186,11 +186,28 @@ def _replace_scoped_member(tf, member, dest_path, output_dir, relpaths): raise IOError(f"Unhandled tar member: {member}") +def _is_component_path(folder, prefix, scoped_path): + """Whether a member under manifest relpath `prefix` belongs to `folder`. + + Everything under the component's own stage directories does, named after it + or not: the test binaries a header-only project's report measures are named + after what they test (rocThrust's bin/merge.hip, rocPRIM's + bin/test_block_scan). Elsewhere, and for the device code under .kpack/ that + every component in the artifact shares, only a path naming the component + does. + """ + names_folder = rf"{folder}[^a-zA-Z]" + if re.search(names_folder, scoped_path, flags=re.IGNORECASE): + return True + in_own_stage = re.search(rf"(^|/){names_folder}", prefix + "/", flags=re.IGNORECASE) + return bool(in_own_stage) and not scoped_path.startswith(".kpack/") + + def replace_instrumented_libraries(artifacts, dest_dir, output_dir): """Extract instrumented libs from downloaded archives into the install tree. Reads each archive's artifact_manifest.txt for relpath prefixes, then flattens - members into output_dir -- but only those matching the artifact's library + members into output_dir -- but only those belonging to the artifact's library folder (e.g. rocBLAS), leaving unrelated files in place. Instrumented shared libraries whose program header table kpack moved get it pinned back (see _therock_utils/elf_phdr.py); otherwise their profile writer can segfault @@ -232,9 +249,7 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): if not member.name.startswith(prefix_slash): continue scoped_path = member.name[len(prefix_slash) :] - if not re.search( - rf"{folder}[^a-zA-Z]", scoped_path, flags=re.IGNORECASE - ): + if not _is_component_path(folder, prefix, scoped_path): break dest_path = output_dir / PurePosixPath(scoped_path) _replace_scoped_member( diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py index 47a14474208..811e109658a 100644 --- a/build_tools/tests/install_rocm_code_coverage_build_test.py +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -100,29 +100,51 @@ def test_replacement_repo_defaults_to_github_repository_env(self): self.assertEqual(captured["github_repository"], "ROCm/TheRock") +def _write_archive(path, members): + """Write an artifact archive of `members`: name -> content, None for a dir.""" + with tarfile.open(path, "w:xz") as tf: + for name, content in members.items(): + info = tarfile.TarInfo(name) + if content is None: + info.type = tarfile.DIRTYPE + tf.addfile(info) + else: + info.size = len(content) + tf.addfile(info, io.BytesIO(content)) + + class TestReplaceInstrumentedLibraries(unittest.TestCase): + def _replaced_files(self, archive_name, artifacts, members): + """Run the replacement into an empty tree; return the files it wrote.""" + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + downloads = tmp / "downloads" + downloads.mkdir() + _write_archive(downloads / archive_name, members) + output_dir = tmp / "install" + with mock.patch.object(mod, "pin_phdr_table", return_value=False): + mod.replace_instrumented_libraries(artifacts, downloads, output_dir) + return sorted( + p.relative_to(output_dir).as_posix() + for p in output_dir.rglob("*") + if p.is_file() + ) + def test_pins_program_headers_of_every_replaced_file(self): with tempfile.TemporaryDirectory() as tmp: tmp = Path(tmp) downloads = tmp / "downloads" downloads.mkdir() prefix = "math-libs/BLAS/rocBLAS/stage" + other_prefix = "math-libs/BLAS/hipBLAS/stage" members = { - "artifact_manifest.txt": f"{prefix}\n".encode(), + "artifact_manifest.txt": f"{prefix}\n{other_prefix}\n".encode(), f"{prefix}/lib/librocblas.so.5.8": b"instrumented library", f"{prefix}/lib/rocblas/library": None, f"{prefix}/lib/rocblas/library/TensileLibrary.dat": b"data", - f"{prefix}/lib/libhipblas.so.3.8": b"some other library", + f"{other_prefix}/lib/libhipblas.so.3.8": b"some other library", } - with tarfile.open(downloads / "blas_lib_generic.tar.xz", "w:xz") as tf: - for name, content in members.items(): - info = tarfile.TarInfo(name) - if content is None: - info.type = tarfile.DIRTYPE - tf.addfile(info) - else: - info.size = len(content) - tf.addfile(info, io.BytesIO(content)) + _write_archive(downloads / "blas_lib_generic.tar.xz", members) output_dir = tmp / "install" with mock.patch.object(mod, "pin_phdr_table", return_value=False) as pin: @@ -142,6 +164,52 @@ def test_pins_program_headers_of_every_replaced_file(self): ], ) + def test_replaces_every_file_in_the_component_stage_dirs(self): + # Laid out like prim_test: tests named after what they test, and the + # device code of all three projects in hipCUB's stage directory. + stages = { + "hipCUB": "math-libs/hipCUB/stage", + "rocPRIM": "math-libs/rocPRIM_tests/stage", + "rocThrust": "math-libs/rocThrust/stage", + } + files = { + "hipCUB": ["bin/test_hipcub_block_scan"], + "rocPRIM": ["bin/test_block_scan"], + "rocThrust": ["bin/merge.hip", "bin/rocthrust/CTestTestfile.cmake"], + } + members = { + "artifact_manifest.txt": "".join( + f"{s}\n" for s in stages.values() + ).encode(), + f"{stages['hipCUB']}/.kpack/prim_test_gfx942.kpack": b"device code", + } + for folder, paths in files.items(): + for path in paths: + members[f"{stages[folder]}/{path}"] = b"instrumented test" + + for folder, paths in files.items(): + with self.subTest(folder=folder): + self.assertEqual( + self._replaced_files( + "prim_test_generic.tar.xz", {"prim": [folder]}, members + ), + paths, + ) + + def test_replaces_device_code_named_after_the_component(self): + prefix = "math-libs/rocWMMA/stage" + members = { + "artifact_manifest.txt": f"{prefix}\n".encode(), + f"{prefix}/.kpack/rocwmma_test_gfx942.kpack": b"device code", + f"{prefix}/bin/gemm_xdl-validate": b"instrumented test", + } + self.assertEqual( + self._replaced_files( + "rocwmma_test_gfx942.tar.xz", {"rocwmma": ["rocWMMA"]}, members + ), + [".kpack/rocwmma_test_gfx942.kpack", "bin/gemm_xdl-validate"], + ) + if __name__ == "__main__": unittest.main() diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 42662002c6d..31b24967580 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -24,7 +24,8 @@ rest come from the regular nightly build (the **baseline**): | **Coverage** | This run | Only the one instrumented project | The test runner installs the baseline, then replaces only the selected -project's libraries with instrumented ones. See [Coverage CI](#coverage-ci). +project's libraries and tests with instrumented ones. See +[Coverage CI](#coverage-ci). ## Enabling coverage for a local build @@ -255,8 +256,13 @@ changes. The baseline-install-then-overlay isolates exactly one project. The baseline is read from `baseline_release_type` (normally `nightly`), not the coverage run's `ci` channel — artifacts are bucketed per channel. The swap downloads the project's whole artifact (`rand`, which also contains rocRAND) -but extracts only the paths matching the project's library folder in -`COMPONENT_MAP` (`hipRAND`). The instrumented stack lives under +but extracts only the project's files: everything under its own stage +directories (`math-libs/hipRAND/stage`), and elsewhere only paths naming its +library folder in `COMPONENT_MAP` (`hipRAND`). The stage directories are what +bring in a header-only project's tests, which are named after what they test +(rocThrust's `bin/merge.hip`). The device code under `.kpack/` is shared by +every project in the artifact, so it is replaced only when its name names the +project, as rocWMMA's does. The instrumented stack lives under `-coverage`, so even a baseline from the same workflow run is never overwritten by it. @@ -278,7 +284,9 @@ the table has to move above the library's address range. A missing instrumented artifact fails the install step. A swap that matches no files does not: the installer logs `Replaced 0` and the tests run against the baseline's uninstrumented binaries. The failure then surfaces in the report -job, which finds no profraw files. +job, which finds no profraw files. A swap that misses just the binaries the +report measures leaves it counters from other programs: none of its +instrumented lines ran, or only the few those programs share. #### What scopes a report to one project From 05660fc84381d845c51977ec8370ebdd7fbb54e6 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Jain Date: Fri, 9 Oct 2026 09:04:26 -0500 Subject: [PATCH 59/70] fix(coverage): render each file once in the HTML report rocWMMA's report job died in every attempt of rocm-libraries run 37912746375: the hosted runner got a shutdown signal two to three minutes into llvm-cov show, after the lcov and the summary were written. llvm-cov's HTML repeats a header's source once per template instantiation, so rocWMMA's report came to 3.3GB with single pages near 1GB, and rendering it peaked at 17.9GB on four cores, past the 16GB of an ubuntu-24.04 runner. Most of that is the coverage mappings of the 29 test binaries, which the lcov export holds too (14.6GB); demangling every function name up front added 3GB, and rendering files in parallel another 2GB. Show each file once, its counts summed over instantiations as in the lcov. No function name is printed then, so drop the demangler, and render one file at a time. On the same inputs the report is 19MB, peaks at 14.1GB, and its totals are unchanged. Co-authored-by: Cursor --- .../github_actions/merge_coverage_report.py | 25 +++++++++++++------ .../tests/merge_coverage_report_test.py | 18 +++++++++++-- docs/development/code_coverage.md | 5 ++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py index 6f27d033725..14df7f1fc80 100644 --- a/build_tools/github_actions/merge_coverage_report.py +++ b/build_tools/github_actions/merge_coverage_report.py @@ -204,7 +204,6 @@ def write_html( output_dir: Path, path_equivalence: str | None = None, project_title: str | None = None, - demangler: Path | None = None, ) -> None: """Renders the browsable report into output_dir, entry point index.html. @@ -212,13 +211,27 @@ def write_html( machine is reported by llvm-cov as uncovered rather than annotated. Files generated into the build tree are never in a source checkout, so a few misses here are expected and not worth failing over. + + Each file is shown once, its counts summed over every template + instantiation as in the lcov. llvm-cov's default also repeats a header's + source for each instantiation, which made rocWMMA's report 3.3GB with + single pages near 1GB. Without those views no function name is printed, + so the report skips demangling, which llvm-cov does for every name up + front and which cost rocWMMA 3GB of the 16GB a hosted runner has. """ command = build_cov_command(llvm_cov, "show", profdata, objects, path_equivalence) - command.extend(["--format=html", f"-output-dir={output_dir}"]) + command.extend( + [ + "--format=html", + f"-output-dir={output_dir}", + "-show-instantiations=false", + # Every file rendered in parallel holds its own coverage on top of + # the mappings of all the objects, which are most of the memory. + "-num-threads=1", + ] + ) if project_title: command.append(f"--project-title={project_title}") - if demangler: - command.append(f"-Xdemangler={demangler}") output_dir.mkdir(parents=True, exist_ok=True) logging.info("Writing HTML report to %s", output_dir / "index.html") @@ -371,9 +384,6 @@ def main(argv: list[str] | None = None) -> int: ) if args.html_output: - # Demangling is what makes the C++ report readable; skip it rather - # than fail if this distribution has no llvm-cxxfilt. - demangler = llvm_bin_dir / f"llvm-cxxfilt{EXECUTABLE_SUFFIX}" write_html( llvm_cov, args.profdata_output, @@ -381,7 +391,6 @@ def main(argv: list[str] | None = None) -> int: args.html_output, args.path_equivalence, args.project_title, - demangler if demangler.is_file() else None, ) # Checked after the other renderings so the report still shows what was diff --git a/build_tools/github_actions/tests/merge_coverage_report_test.py b/build_tools/github_actions/tests/merge_coverage_report_test.py index 8ba388ffcd3..a94284f818d 100644 --- a/build_tools/github_actions/tests/merge_coverage_report_test.py +++ b/build_tools/github_actions/tests/merge_coverage_report_test.py @@ -212,7 +212,6 @@ def test_html_renders_into_a_directory(self): objects, output_dir, project_title="hiprand", - demangler=Path("/llvm/llvm-cxxfilt"), ) command = run.call_args.args[0] @@ -220,7 +219,22 @@ def test_html_renders_into_a_directory(self): self.assertIn("--format=html", command) self.assertIn(f"-output-dir={output_dir}", command) self.assertIn("--project-title=hiprand", command) - self.assertIn(f"-Xdemangler={Path('/llvm/llvm-cxxfilt')}", command) + + def test_html_shows_each_file_once_and_one_at_a_time(self): + # Per-instantiation views, demangling, and files rendered in parallel + # made rocWMMA's report 3.3GB and its rendering outgrow a 16GB runner. + with mock.patch("subprocess.run") as run: + merge_coverage_report.write_html( + Path("/llvm/llvm-cov"), + Path("/tmp/c.profdata"), + [self.touch("bin/gemm_xdl-validate")], + self.root / "out" / "html", + ) + + command = run.call_args.args[0] + self.assertIn("-show-instantiations=false", command) + self.assertIn("-num-threads=1", command) + self.assertEqual([arg for arg in command if arg.startswith("-Xdemangler")], []) def test_path_equivalence_reaches_every_rendering(self): objects = [self.touch("lib/a.so")] diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 31b24967580..14b421f3c24 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -337,6 +337,11 @@ Totals are also written to the job summary page. The HTML step runs `continue-on-error` (losing it costs only the annotated view). Codecov is skipped, not failed, when `CODECOV_TOKEN` is absent. +The annotated view shows each file once, its counts summed over every template +instantiation as in the lcov. With llvm-cov's per-instantiation views, demangled, +rocWMMA's report was 3.3GB and rendering it outgrew the 16GB hosted runner the +report job runs on. + ## Selecting projects to run `projects_to_test` takes comma-separated project names or group aliases From 8ad0f69766bd75394e2a648d4af37fdab862d15d Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Mon, 12 Oct 2026 19:12:10 +0530 Subject: [PATCH 60/70] ci(coverage): added token to push coverage results to codecov.io --- .github/workflows/multi_arch_ci_coverage_report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 08b4f1f7308..5d7d84c1143 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -301,6 +301,7 @@ jobs: - name: Upload report to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ${{ env.REPORT_DIR }}/coverage.info flags: ${{ inputs.codecov_flag || inputs.project_name }} name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} From 88f0ddf248546f37c4e76132ffe3da0b1b47f124 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Mon, 19 Oct 2026 20:01:47 +0530 Subject: [PATCH 61/70] Fixing - codecov fileNotFound issue --- .../multi_arch_ci_coverage_report.yml | 12 ++++-- .../github_actions/merge_coverage_report.py | 37 +++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 5d7d84c1143..6281e81dbe1 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -244,10 +244,14 @@ jobs: - name: Merge profraw and generate reports env: # Sources are recorded at the build container's path, not where they - # sit here, so tell llvm-cov how to map between them. Generated files - # under build/ are in neither and report as uncovered; the text - # summary still counts them. + # sit here. The HTML and text renderings need to reach them on disk, + # so they map the build workspace to this checkout. Codecov instead + # matches the lcov against the repository tree, so that export maps + # the build-time repo root to an empty prefix and emits repo-relative + # paths. The repo is a submodule one level below the superproject + # workspace, so its root is BUILD_WORKSPACE plus the repo name. BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + BUILD_REPO_ROOT: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/${{ github.event.repository.name }} HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} run: | html_args=() @@ -268,6 +272,8 @@ jobs: --lcov-output="${REPORT_DIR}/coverage.info" \ --summary-output="${REPORT_DIR}/coverage_summary.txt" \ --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ + --lcov-path-equivalence="${BUILD_REPO_ROOT}/," \ + --ignore-filename-regex=".*/build/.*" \ "${html_args[@]}" # Put totals on the run summary page, readable without downloading. diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py index 14df7f1fc80..34c36f4a8c7 100644 --- a/build_tools/github_actions/merge_coverage_report.py +++ b/build_tools/github_actions/merge_coverage_report.py @@ -163,9 +163,12 @@ def export_lcov( objects: list[Path], output: Path, path_equivalence: str | None = None, + ignore_filename_regex: str | None = None, ) -> None: command = build_cov_command(llvm_cov, "export", profdata, objects, path_equivalence) command.append("--format=lcov") + if ignore_filename_regex: + command.append(f"-ignore-filename-regex={ignore_filename_regex}") output.parent.mkdir(parents=True, exist_ok=True) logging.info("Exporting lcov for %d object(s) to %s", len(objects), output) @@ -204,13 +207,16 @@ def write_html( output_dir: Path, path_equivalence: str | None = None, project_title: str | None = None, + ignore_filename_regex: str | None = None, ) -> None: """Renders the browsable report into output_dir, entry point index.html. Unlike the other two this opens the source files, so anything not on the machine is reported by llvm-cov as uncovered rather than annotated. Files generated into the build tree are never in a source checkout, so a few - misses here are expected and not worth failing over. + misses here are expected; ignore_filename_regex drops them from this view so + they don't log an error per file. They stay in the lcov and text summary, + which count the mappings without opening the sources. Each file is shown once, its counts summed over every template instantiation as in the lcov. llvm-cov's default also repeats a header's @@ -232,6 +238,8 @@ def write_html( ) if project_title: command.append(f"--project-title={project_title}") + if ignore_filename_regex: + command.append(f"-ignore-filename-regex={ignore_filename_regex}") output_dir.mkdir(parents=True, exist_ok=True) logging.info("Writing HTML report to %s", output_dir / "index.html") @@ -298,7 +306,19 @@ def main(argv: list[str] | None = None) -> int: type=str, default=None, help="',' remapping for source paths recorded at build " - "time, e.g. '/__w/TheRock/TheRock,${GITHUB_WORKSPACE}'", + "time, e.g. '/__w/TheRock/TheRock,${GITHUB_WORKSPACE}'. Used by the " + "HTML and text renderings, which need to reach the sources on disk", + ) + parser.add_argument( + "--lcov-path-equivalence", + type=str, + default=None, + help="Like --path-equivalence but applied only to the lcov export, " + "where Codecov matches paths against the repository tree rather than " + "the local disk. Map the build-time repo root to an empty '' so " + "the emitted paths are repo-relative, e.g. " + "'/__w/rocm-libraries/rocm-libraries/rocm-libraries/,'. Defaults to " + "--path-equivalence", ) parser.add_argument( "--project-title", @@ -306,6 +326,15 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Heading for the HTML report's index page", ) + parser.add_argument( + "--ignore-filename-regex", + type=str, + default=None, + help="Regex of source paths to drop from the lcov export and the HTML " + "report, e.g. '.*/build/.*' to skip build-tree generated files that " + "aren't on the report machine and that Codecov can't map to the " + "repository. The text summary still counts them", + ) parser.add_argument( "--allow-empty", action="store_true", @@ -361,7 +390,8 @@ def main(argv: list[str] | None = None) -> int: args.profdata_output, objects, args.lcov_output, - args.path_equivalence, + args.lcov_path_equivalence or args.path_equivalence, + args.ignore_filename_regex, ) except subprocess.CalledProcessError: logging.error( @@ -391,6 +421,7 @@ def main(argv: list[str] | None = None) -> int: args.html_output, args.path_equivalence, args.project_title, + args.ignore_filename_regex, ) # Checked after the other renderings so the report still shows what was From 25a70b61acae631a9df91280caba229621008d8a Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Tue, 20 Oct 2026 09:09:08 +0530 Subject: [PATCH 62/70] Fixing - codecov fileNotFound issue --- .../multi_arch_ci_coverage_report.yml | 10 +++--- .../github_actions/merge_coverage_report.py | 35 ++++++++++++++----- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 6281e81dbe1..e85f1cf8883 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -246,10 +246,10 @@ jobs: # Sources are recorded at the build container's path, not where they # sit here. The HTML and text renderings need to reach them on disk, # so they map the build workspace to this checkout. Codecov instead - # matches the lcov against the repository tree, so that export maps - # the build-time repo root to an empty prefix and emits repo-relative - # paths. The repo is a submodule one level below the superproject - # workspace, so its root is BUILD_WORKSPACE plus the repo name. + # matches the lcov against the repository tree, so that export strips + # the build-time repo root to emit repo-relative paths. The repo is a + # submodule one level below the superproject workspace, so its root is + # BUILD_WORKSPACE plus the repo name. BUILD_WORKSPACE: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} BUILD_REPO_ROOT: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/${{ github.event.repository.name }} HTML_ENABLED: ${{ steps.fetch_sources.outcome == 'success' }} @@ -272,7 +272,7 @@ jobs: --lcov-output="${REPORT_DIR}/coverage.info" \ --summary-output="${REPORT_DIR}/coverage_summary.txt" \ --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ - --lcov-path-equivalence="${BUILD_REPO_ROOT}/," \ + --lcov-strip-prefix="${BUILD_REPO_ROOT}/" \ --ignore-filename-regex=".*/build/.*" \ "${html_args[@]}" diff --git a/build_tools/github_actions/merge_coverage_report.py b/build_tools/github_actions/merge_coverage_report.py index 34c36f4a8c7..92d0f8e12f2 100644 --- a/build_tools/github_actions/merge_coverage_report.py +++ b/build_tools/github_actions/merge_coverage_report.py @@ -127,6 +127,23 @@ def count_profiled_functions(llvm_profdata: Path, profdata: Path) -> int | None: return int(match.group(1)) if match else None +def strip_lcov_prefix(lcov: Path, prefix: str) -> None: + """Rewrites the lcov's SF: paths to drop a build-time prefix. + + llvm-cov records absolute paths from the build container, but Codecov + matches a report against the repository tree and needs repo-relative paths. + Its -path-equivalence can't map a prefix to an empty string (it rejects a + missing 'to'), so the leading repo root is stripped here instead. + """ + lines = lcov.read_text().splitlines(keepends=True) + rewritten = [] + for line in lines: + if line.startswith("SF:") and line[3:].startswith(prefix): + line = "SF:" + line[3 + len(prefix) :] + rewritten.append(line) + lcov.write_text("".join(rewritten)) + + def count_lcov_lines(lcov: Path) -> tuple[int, int]: """Returns (lines found, lines hit) summed over every file in an lcov report.""" found = hit = 0 @@ -310,15 +327,15 @@ def main(argv: list[str] | None = None) -> int: "HTML and text renderings, which need to reach the sources on disk", ) parser.add_argument( - "--lcov-path-equivalence", + "--lcov-strip-prefix", type=str, default=None, - help="Like --path-equivalence but applied only to the lcov export, " - "where Codecov matches paths against the repository tree rather than " - "the local disk. Map the build-time repo root to an empty '' so " - "the emitted paths are repo-relative, e.g. " - "'/__w/rocm-libraries/rocm-libraries/rocm-libraries/,'. Defaults to " - "--path-equivalence", + help="Prefix to drop from every SF: path in the lcov export so the " + "emitted paths are repo-relative and Codecov can match them against " + "the repository tree, e.g. " + "'/__w/rocm-libraries/rocm-libraries/rocm-libraries/'. llvm-cov's " + "-path-equivalence can't map a prefix to an empty string, so the strip " + "is a post-process step on the lcov instead", ) parser.add_argument( "--project-title", @@ -390,7 +407,7 @@ def main(argv: list[str] | None = None) -> int: args.profdata_output, objects, args.lcov_output, - args.lcov_path_equivalence or args.path_equivalence, + None, args.ignore_filename_regex, ) except subprocess.CalledProcessError: @@ -401,6 +418,8 @@ def main(argv: list[str] | None = None) -> int: len(objects), ) return 1 + if args.lcov_strip_prefix: + strip_lcov_prefix(args.lcov_output, args.lcov_strip_prefix) logging.info("Wrote coverage report to %s", args.lcov_output) lines_found, lines_hit = count_lcov_lines(args.lcov_output) From d9d3fef84b91b7efec9e74ce6e6ed3989e4a2c89 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Wed, 21 Oct 2026 08:31:02 +0530 Subject: [PATCH 63/70] correction to the cwd of codecov upload --- .github/workflows/multi_arch_ci_coverage_report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index e85f1cf8883..0b7a871a83d 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -308,6 +308,7 @@ jobs: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} + working-directory: /home/runner/work/rocm-libraries/rocm-libraries/ files: ${{ env.REPORT_DIR }}/coverage.info flags: ${{ inputs.codecov_flag || inputs.project_name }} name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} From 879322aaa38aee11c85446ae6c90fd330e48f692 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Wed, 21 Oct 2026 12:29:44 +0530 Subject: [PATCH 64/70] Options corrections to Codecov --- .github/workflows/multi_arch_ci_coverage_report.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 0b7a871a83d..44a7b0c0eee 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -310,6 +310,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} working-directory: /home/runner/work/rocm-libraries/rocm-libraries/ files: ${{ env.REPORT_DIR }}/coverage.info - flags: ${{ inputs.codecov_flag || inputs.project_name }} + flags: "${{ inputs.codecov_flag || inputs.project_name }} - Nightly" name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} + disable_search: true fail_ci_if_error: false From c3127afd30176d5861a174cf23b2170908eb553f Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Wed, 21 Oct 2026 15:35:59 +0530 Subject: [PATCH 65/70] codecov submodule fix --- .github/workflows/multi_arch_ci_coverage_report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 44a7b0c0eee..8f49e6db20b 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -313,4 +313,5 @@ jobs: flags: "${{ inputs.codecov_flag || inputs.project_name }} - Nightly" name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} disable_search: true + recurse_submodules: true fail_ci_if_error: false From 9e97c651aa74048bd3068a0212d2f2e42de4ef33 Mon Sep 17 00:00:00 2001 From: Abhilash Reddy Endurthi Date: Thu, 22 Oct 2026 11:36:31 -0700 Subject: [PATCH 66/70] Fix coverage lcov paths and Codecov working-directory for rocm-libraries submodule --- .github/workflows/multi_arch_ci_coverage_report.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/multi_arch_ci_coverage_report.yml b/.github/workflows/multi_arch_ci_coverage_report.yml index 8f49e6db20b..893f4fce9f5 100644 --- a/.github/workflows/multi_arch_ci_coverage_report.yml +++ b/.github/workflows/multi_arch_ci_coverage_report.yml @@ -264,6 +264,10 @@ jobs: echo "::warning::Sources unavailable; skipping the annotated HTML report." fi + # Strip through the submodule root so Codecov SF: paths are relative + # to rocm-libraries/ (matching its working-directory). path-equivalence + # maps the build-time submodule path to GITHUB_WORKSPACE for HTML. + BUILD_SUBMODULE_PATH="${BUILD_REPO_ROOT}/rocm-libraries" python ./build_tools/github_actions/merge_coverage_report.py \ --profraw-dir="${PROFRAW_DIR}" \ --rocm-dir="${OUTPUT_ARTIFACTS_DIR}" \ @@ -271,8 +275,8 @@ jobs: --profdata-output="${REPORT_DIR}/coverage.profdata" \ --lcov-output="${REPORT_DIR}/coverage.info" \ --summary-output="${REPORT_DIR}/coverage_summary.txt" \ - --path-equivalence="${BUILD_WORKSPACE},${GITHUB_WORKSPACE}" \ - --lcov-strip-prefix="${BUILD_REPO_ROOT}/" \ + --path-equivalence="${BUILD_SUBMODULE_PATH},${GITHUB_WORKSPACE}" \ + --lcov-strip-prefix="${BUILD_SUBMODULE_PATH}/" \ --ignore-filename-regex=".*/build/.*" \ "${html_args[@]}" @@ -308,7 +312,7 @@ jobs: uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} - working-directory: /home/runner/work/rocm-libraries/rocm-libraries/ + working-directory: ${{ github.workspace }}/rocm-libraries files: ${{ env.REPORT_DIR }}/coverage.info flags: "${{ inputs.codecov_flag || inputs.project_name }} - Nightly" name: ${{ inputs.project_name }}-${{ inputs.amdgpu_families }} From 538701c286ed650ed6e6e662cc4a14da90159074 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Thu, 5 Nov 2026 17:40:17 +0530 Subject: [PATCH 67/70] fix(coverage): replace shared kpack with the instrumented host binaries ## Motivation hipCUB coverage tests in rocm-libraries run 41136053832 (gfx94X-dcgpu) failed: 33 of 49 tests failed, 32 of them aborting with `Cannot find Symbol` for `hipcub::HIPCUB_400700_NS` kernels, and `GetVersionOnDevice` read 400701 instead of 400700. Since 80e80edc0, every file under the project's own stage directories is replaced, except the device code under `.kpack/` unless its name names the project. prim_test's kpack is `prim_test_gfx942.kpack`, shared by hipCUB, rocPRIM and rocThrust, so it was never replaced. The instrumented test binaries then looked up their kernels by name in the baseline's kpack, which was built at a different hipCUB version. ## Technical Details - `install_rocm_code_coverage_build.py`: replace the archive's `.kpack/` device code whenever the component has its own stage directory in the artifact's manifest (`owns_archive`), so the host binaries and their kernels come from the same build. Archives where the component has no stage directory still keep the baseline's kpack. Split the regexes into `_names_folder` and `_is_own_stage`. - Trade-off: other projects' baseline binaries in the same artifact (e.g. rocPRIM and rocThrust in prim_test) now load the instrumented kpack. A coverage job only runs the swapped project's tests. - Tests: the prim_test case now expects the shared kpack to be replaced for each project; added a case where the component has no stage directory in the artifact (rocBLAS vs a hipBLASLt-only blas_lib), which keeps the baseline kpack. - `docs/development/code_coverage.md`: describe the new `.kpack/` rule and the trade-off. --- .../install_rocm_code_coverage_build.py | 40 ++++++++++++++----- .../install_rocm_code_coverage_build_test.py | 18 ++++++++- docs/development/code_coverage.md | 8 +++- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index f78c5694609..acb4cf55166 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -16,6 +16,7 @@ [** all supported options of install_rocm_from_artifacts.py] """ + import os import re import sys @@ -186,21 +187,39 @@ def _replace_scoped_member(tf, member, dest_path, output_dir, relpaths): raise IOError(f"Unhandled tar member: {member}") -def _is_component_path(folder, prefix, scoped_path): +def _names_folder(folder, path): + """Whether `path` has a component naming `folder` (e.g. rocBLAS/, rocblas.so).""" + return re.search(rf"{folder}[^a-zA-Z]", path, flags=re.IGNORECASE) + + +def _is_own_stage(folder, prefix): + """Whether manifest relpath `prefix` is one of `folder`'s stage directories.""" + return bool( + re.search(rf"(^|/){folder}[^a-zA-Z]", prefix + "/", flags=re.IGNORECASE) + ) + + +def _is_component_path(folder, prefix, scoped_path, owns_archive): """Whether a member under manifest relpath `prefix` belongs to `folder`. Everything under the component's own stage directories does, named after it or not: the test binaries a header-only project's report measures are named after what they test (rocThrust's bin/merge.hip, rocPRIM's - bin/test_block_scan). Elsewhere, and for the device code under .kpack/ that - every component in the artifact shares, only a path naming the component - does. + bin/test_block_scan). Elsewhere only a path naming the component does -- + except the device code under .kpack/. Every component in the artifact shares + one kpack archive, stored under whichever stage directory comes first + (prim_test's sits in hipCUB's), and host binaries look their kernels up in + it by name. Swapping the host binaries without it leaves them loading the + baseline's kernels, which fail once the two builds differ (a bumped + HIPCUB__NS mangles to symbols the baseline kpack does not have). + So it follows the host binaries: it is replaced whenever `owns_archive`, + i.e. the component has a stage directory in this artifact. """ - names_folder = rf"{folder}[^a-zA-Z]" - if re.search(names_folder, scoped_path, flags=re.IGNORECASE): + if _names_folder(folder, scoped_path): return True - in_own_stage = re.search(rf"(^|/){names_folder}", prefix + "/", flags=re.IGNORECASE) - return bool(in_own_stage) and not scoped_path.startswith(".kpack/") + if scoped_path == ".kpack" or scoped_path.startswith(".kpack/"): + return owns_archive + return _is_own_stage(folder, prefix) def replace_instrumented_libraries(artifacts, dest_dir, output_dir): @@ -242,6 +261,7 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): ) with tf.extractfile(manifest_member) as mf_file: relpaths = [r for r in mf_file.read().decode().splitlines() if r] + owns_archive = any(_is_own_stage(folder, r) for r in relpaths) while member := tf.next(): for prefix in relpaths: @@ -249,7 +269,9 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): if not member.name.startswith(prefix_slash): continue scoped_path = member.name[len(prefix_slash) :] - if not _is_component_path(folder, prefix, scoped_path): + if not _is_component_path( + folder, prefix, scoped_path, owns_archive + ): break dest_path = output_dir / PurePosixPath(scoped_path) _replace_scoped_member( diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py index 811e109658a..f8eeafe200c 100644 --- a/build_tools/tests/install_rocm_code_coverage_build_test.py +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -187,15 +187,31 @@ def test_replaces_every_file_in_the_component_stage_dirs(self): for path in paths: members[f"{stages[folder]}/{path}"] = b"instrumented test" + # The shared kpack follows whichever project's host binaries are + # swapped in, or they look up kernels the baseline's kpack lacks. for folder, paths in files.items(): with self.subTest(folder=folder): self.assertEqual( self._replaced_files( "prim_test_generic.tar.xz", {"prim": [folder]}, members ), - paths, + sorted([".kpack/prim_test_gfx942.kpack", *paths]), ) + def test_keeps_device_code_of_artifacts_without_the_component_stage(self): + prefix = "math-libs/BLAS/hipBLASLt/stage" + members = { + "artifact_manifest.txt": f"{prefix}\n".encode(), + f"{prefix}/.kpack/blas_lib_gfx942.kpack": b"device code", + f"{prefix}/lib/libhipblaslt.so": b"some other library", + } + self.assertEqual( + self._replaced_files( + "blas_lib_gfx942.tar.xz", {"blas": ["rocBLAS"]}, members + ), + [], + ) + def test_replaces_device_code_named_after_the_component(self): prefix = "math-libs/rocWMMA/stage" members = { diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 14b421f3c24..96203c3eef4 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -261,8 +261,12 @@ directories (`math-libs/hipRAND/stage`), and elsewhere only paths naming its library folder in `COMPONENT_MAP` (`hipRAND`). The stage directories are what bring in a header-only project's tests, which are named after what they test (rocThrust's `bin/merge.hip`). The device code under `.kpack/` is shared by -every project in the artifact, so it is replaced only when its name names the -project, as rocWMMA's does. The instrumented stack lives under +every project in the artifact, but host binaries look their kernels up in it by +name, so it is replaced along with them whenever the project has a stage +directory in the artifact; keeping the baseline's would fail with `Cannot find Symbol` once the two builds' versions differ (`HIPCUB_400700_NS` vs +`HIPCUB_400701_NS`). Other projects' baseline binaries in that artifact then +load the instrumented kpack, so only the swapped project's tests are expected +to pass. The instrumented stack lives under `-coverage`, so even a baseline from the same workflow run is never overwritten by it. From a556b47d4203476857960d918d2f85ac0607a0dc Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Fri, 6 Nov 2026 15:26:53 +0530 Subject: [PATCH 68/70] fix(coverage): swap a wrapper's same-artifact backend too ## Motivation The coverage test job installs the baseline stack and overlays only the selected project's instrumented files. hip* wrappers call a backend at runtime that ships in the same artifact, so the instrumented wrapper ran against the baseline's backend, which can be a different version than the one it was built against. hipRAND's linkage test failed this way: hiprandGetVersion() (answered by the baseline librocrand, 500300) did not match the ROCRAND_VERSION it was compiled with (500100). ## Technical Details - Add SAME_ARTIFACT_BACKENDS to install_rocm_code_coverage_build.py: hipblas->rocBLAS, hipfft->rocFFT, hiprand->rocRAND, hipsolver->rocSOLVER, hipsparse->rocSPARSE. --replace- now also extracts the backend's folder from the same instrumented artifact. - Move folder collection out of main() into replacement_folders(), which de-duplicates folders when wrapper and backend are both selected. - Add tests for the mapping, for backends living in their wrapper's artifact, and for a rand archive swap installing both libhiprand and librocrand. - Document the behavior in docs/development/code_coverage.md. The backend's objects stay out of the wrapper's --object-globs, but header templates both instantiate share profile counters. Dependencies in other artifacts (fftw3, rocBLAS for hipSOLVER) still come from the baseline. --- .../install_rocm_code_coverage_build.py | 37 ++++++++++++-- .../install_rocm_code_coverage_build_test.py | 49 +++++++++++++++++++ docs/development/code_coverage.md | 13 ++++- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index acb4cf55166..f56f6a15bb1 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -22,6 +22,7 @@ import sys import argparse import platform +from collections.abc import Iterable from pathlib import Path, PurePosixPath from install_rocm_from_artifacts import main as install_from_artifacts_main @@ -66,6 +67,34 @@ "composablekernel": ("composable-kernel", "composable_kernel"), } +# Backend library folders a hip* wrapper calls at runtime that ship in the same +# artifact. They are swapped along with the wrapper so both come from one +# build: hipRAND's linkage test compares hiprandGetVersion() (answered by +# librocrand) against the ROCRAND_VERSION it was compiled with, and fails when +# the baseline's rocRAND is a different version than the coverage build's. +SAME_ARTIFACT_BACKENDS = { + "hipblas": ("rocBLAS",), + "hipfft": ("rocFFT",), + "hiprand": ("rocRAND",), + "hipsolver": ("rocSOLVER",), + "hipsparse": ("rocSPARSE",), +} + + +def replacement_folders(components: Iterable[str]) -> dict[str, list[str]]: + """Map each artifact to swap to the library folders to extract from it. + + A selected component brings its own folder plus its SAME_ARTIFACT_BACKENDS. + """ + artifacts: dict[str, list[str]] = {} + for comp in components: + name, folder = COMPONENT_MAP[comp] + folders = artifacts.setdefault(name, []) + for f in (folder, *SAME_ARTIFACT_BACKENDS.get(comp, ())): + if f not in folders: + folders.append(f) + return artifacts + def _read_passthrough_options(passthrough_argv): """Read options shared with install_rocm_from_artifacts without consuming them. @@ -336,11 +365,9 @@ def main(argv): install_from_artifacts_main(extra_args) # collect artifacts details as per the user options - artifacts = {} - for comp, (name, folder) in COMPONENT_MAP.items(): - if not getattr(args, f"replace_{comp}"): - continue # skip for non-selected components - artifacts.setdefault(name, []).append(folder) + artifacts = replacement_folders( + comp for comp in COMPONENT_MAP if getattr(args, f"replace_{comp}") + ) if not artifacts: log("No --replace-* components specified; nothing to download.") diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py index f8eeafe200c..c7e01cc7d3e 100644 --- a/build_tools/tests/install_rocm_code_coverage_build_test.py +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -226,6 +226,55 @@ def test_replaces_device_code_named_after_the_component(self): [".kpack/rocwmma_test_gfx942.kpack", "bin/gemm_xdl-validate"], ) + def test_wrapper_brings_its_backend_from_the_same_build(self): + # hipRAND's linkage test compares the librocrand version against the + # rocRAND headers it was built with, so rocRAND must be swapped too. + hiprand = "math-libs/hipRAND/stage" + rocrand = "math-libs/rocRAND/stage" + members = { + "artifact_manifest.txt": f"{hiprand}\n{rocrand}\n".encode(), + f"{hiprand}/lib/libhiprand.so.1.1": b"instrumented hiprand", + f"{rocrand}/lib/librocrand.so.1.1": b"instrumented rocrand", + } + self.assertEqual( + self._replaced_files( + "rand_lib_generic.tar.xz", + mod.replacement_folders(["hiprand"]), + members, + ), + ["lib/libhiprand.so.1.1", "lib/librocrand.so.1.1"], + ) + + +class TestReplacementFolders(unittest.TestCase): + def test_wrapper_adds_its_same_artifact_backend(self): + self.assertEqual( + mod.replacement_folders(["hiprand"]), {"rand": ["hipRAND", "rocRAND"]} + ) + + def test_backend_is_not_listed_twice(self): + self.assertEqual( + mod.replacement_folders(["hipfft", "rocfft"]), + {"fft": ["hipFFT", "rocFFT"]}, + ) + + def test_backend_alone_does_not_bring_its_wrapper(self): + self.assertEqual(mod.replacement_folders(["rocrand"]), {"rand": ["rocRAND"]}) + + def test_backends_ship_in_their_wrappers_artifact(self): + folders = {folder for _, folder in mod.COMPONENT_MAP.values()} + for comp, backends in mod.SAME_ARTIFACT_BACKENDS.items(): + artifact = mod.COMPONENT_MAP[comp][0] + for backend in backends: + with self.subTest(comp=comp, backend=backend): + self.assertIn(backend, folders) + owner = next( + name + for name, folder in mod.COMPONENT_MAP.values() + if folder == backend + ) + self.assertEqual(owner, artifact) + if __name__ == "__main__": unittest.main() diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 96203c3eef4..6f47ff03980 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -266,7 +266,18 @@ name, so it is replaced along with them whenever the project has a stage directory in the artifact; keeping the baseline's would fail with `Cannot find Symbol` once the two builds' versions differ (`HIPCUB_400700_NS` vs `HIPCUB_400701_NS`). Other projects' baseline binaries in that artifact then load the instrumented kpack, so only the swapped project's tests are expected -to pass. The instrumented stack lives under +to pass. + +A hip\* wrapper also brings the backend it calls at runtime from the same +artifact (`SAME_ARTIFACT_BACKENDS`: hipRAND brings rocRAND, hipFFT rocFFT, and +so on). Otherwise the instrumented wrapper runs against the baseline's backend, +which may be a different version than the one it was built with: hipRAND's +linkage test then fails comparing `hiprandGetVersion()` (answered by +librocrand) with the `ROCRAND_VERSION` it was compiled against. The backend's +objects are not in the wrapper's `--object-globs`, so its own code stays out of +the report, but header templates both instantiate share profile counters. +Dependencies in other artifacts (fftw3, rocBLAS for hipSOLVER) still come from +the baseline. The instrumented stack lives under `-coverage`, so even a baseline from the same workflow run is never overwritten by it. From ac9d49a8665252b06418a4d61902d0e28419529b Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Mon, 9 Nov 2026 11:20:57 +0530 Subject: [PATCH 69/70] fix(coverage): build prim projects without hipCCL in coverage runs ## Motivation Since the unified hipCCL superbuild became the default (THEROCK_FLAG_BUILD_HIPCCL=ON), coverage runs for rocPRIM, hipCUB and rocThrust broke. hipCCL has no rocPRIM/hipCUB/rocThrust subprojects for their _ENABLE_COVERAGE flags to reach, so the "instrumented" prim tests carried no profile sections. All of them also land in one hipCCL_tests stage directory, which the overlay does not recognize as any project's own: - hipCUB: the shared prim_test_gfx942.kpack was not replaced, so the swapped-in test binaries looked up their kernels in the baseline kpack and failed with "device kernel image is invalid". - rocThrust: only paths naming rocthrust were swapped, so the coverage build's CTestTestfile.cmake listed offset_iterator.hip and raw_reference_cast.hip, which the baseline does not ship ("Unable to find executable"), while every other test ran the baseline's uninstrumented binaries. ## Technical Details - configure_coverage_ci.py: add PRIM_PROJECTS (projects in the prim artifact). build_coverage_cmake_options() appends -DTHEROCK_FLAG_BUILD_HIPCCL=OFF when any is selected, including through the group options, so the coverage build uses the legacy subprojects the registry is written against. Installed paths match the hipCCL baseline once flattened. - Move the rocprofiler-compute entry to the profiler-apps stage, where BUILD_TOPOLOGY.toml now builds its artifact (fixes test_every_project_names_the_stage_that_builds_its_artifacts). - Tests: expect the flag in the group and rocPRIM option lists; check every prim project gets it and that PRIM_PROJECTS is rocprim/hipcub/rocthrust. - docs/development/code_coverage.md: explain why coverage builds prim without hipCCL. --- .../github_actions/configure_coverage_ci.py | 20 +++++++++--- .../tests/configure_coverage_ci_test.py | 31 ++++++++++++++++--- docs/development/code_coverage.md | 7 +++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/build_tools/github_actions/configure_coverage_ci.py b/build_tools/github_actions/configure_coverage_ci.py index 6e9fc26619a..727d10a48cd 100644 --- a/build_tools/github_actions/configure_coverage_ci.py +++ b/build_tools/github_actions/configure_coverage_ci.py @@ -514,12 +514,15 @@ class CoverageProject: codecov_flag="amdsmi", source_repo=ROCM_SYSTEMS, ), + # + # rocm-systems -- profiler-apps stage + # "rocprofiler-compute": CoverageProject( cmake_target="rocprofiler-compute", artifact_names=["rocprofiler-compute"], artifact_relpaths=["profiler/rocprofiler-compute/stage"], unsupported_reason="a Python tool with no native instrumentation option", - stage=STAGE_COMPILER_RUNTIME, + stage=STAGE_PROFILER_APPS, test_component="rocprofiler-compute", coverage_config="projects/rocprofiler-compute/test_categories_coverage.yaml", object_globs=["lib/librocprofiler-compute*.so*"], @@ -527,9 +530,6 @@ class CoverageProject: codecov_flag="rocprofiler-compute", source_repo=ROCM_SYSTEMS, ), - # - # rocm-systems -- profiler-apps stage - # "rocprofiler-systems": CoverageProject( cmake_target="rocprofiler-systems", artifact_names=["rocprofiler-systems"], @@ -597,6 +597,11 @@ class CoverageProject: for k, v in COVERAGE_PROJECTS.items() if v.source_repo == ROCM_SYSTEMS and v.coverage_option and not v.blocked_reason ) +# Projects shipped in the prim artifact, built by the legacy rocPRIM/hipCUB/ +# rocThrust subprojects or, by default, the unified hipCCL superbuild. +PRIM_PROJECTS: frozenset[str] = frozenset( + k for k, v in COVERAGE_PROJECTS.items() if "prim" in v.artifact_names +) # Group-alias tokens accepted by PROJECTS_TO_TEST (and the projects_to_test CI # input) that expand to a whole component group. Matched case-insensitively. @@ -767,6 +772,13 @@ def build_coverage_cmake_options(project_keys: list[str]) -> list[str]: project = COVERAGE_PROJECTS[key] for target in [project.cmake_target, *project.extra_cmake_targets]: options.append(f"-D{target.upper()}_ENABLE_COVERAGE=ON") + + # The unified hipCCL superbuild has no rocPRIM/hipCUB/rocThrust subprojects + # for their coverage flags to reach, so it ships uninstrumented tests whose + # stage dirs (hipCCL_tests) the replacement does not recognize. Build the + # legacy subprojects these entries are registered against instead. + if selected & PRIM_PROJECTS: + options.append("-DTHEROCK_FLAG_BUILD_HIPCCL=OFF") return options diff --git a/build_tools/github_actions/tests/configure_coverage_ci_test.py b/build_tools/github_actions/tests/configure_coverage_ci_test.py index bca44ccf46a..014010fbf23 100644 --- a/build_tools/github_actions/tests/configure_coverage_ci_test.py +++ b/build_tools/github_actions/tests/configure_coverage_ci_test.py @@ -272,7 +272,10 @@ def test_full_selection_collapses_to_the_group_option(self): configure_coverage_ci.build_coverage_cmake_options( sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) ), - ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"], + [ + "-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON", + "-DTHEROCK_FLAG_BUILD_HIPCCL=OFF", + ], ) def test_both_groups_collapse_to_the_combined_option(self): @@ -283,7 +286,7 @@ def test_both_groups_collapse_to_the_combined_option(self): - configure_coverage_ci.BLOCKED_PROJECTS ) ), - ["-DTHEROCK_COVERAGE_ALL=ON"], + ["-DTHEROCK_COVERAGE_ALL=ON", "-DTHEROCK_FLAG_BUILD_HIPCCL=OFF"], ) def test_blocked_project_is_named_rather_than_folded_into_a_group(self): @@ -306,14 +309,34 @@ def test_partial_selection_also_names_extra_targets(self): # rocPRIM alone would instrument nothing the report reads. self.assertEqual( configure_coverage_ci.build_coverage_cmake_options(["rocprim"]), - ["-DROCPRIM_ENABLE_COVERAGE=ON", "-DROCPRIM_TESTS_ENABLE_COVERAGE=ON"], + [ + "-DROCPRIM_ENABLE_COVERAGE=ON", + "-DROCPRIM_TESTS_ENABLE_COVERAGE=ON", + "-DTHEROCK_FLAG_BUILD_HIPCCL=OFF", + ], + ) + + def test_prim_projects_build_the_legacy_subprojects(self): + # hipCCL has no hipCUB subproject for HIPCUB_ENABLE_COVERAGE to reach. + for key in sorted(configure_coverage_ci.PRIM_PROJECTS): + with self.subTest(key=key): + self.assertIn( + "-DTHEROCK_FLAG_BUILD_HIPCCL=OFF", + configure_coverage_ci.build_coverage_cmake_options([key]), + ) + + def test_prim_projects_are_rocprim_hipcub_and_rocthrust(self): + self.assertEqual( + configure_coverage_ci.PRIM_PROJECTS, + {"rocprim", "hipcub", "rocthrust"}, ) def test_one_group_does_not_claim_to_cover_the_other(self): options = configure_coverage_ci.build_coverage_cmake_options( sorted(configure_coverage_ci.ROCM_LIBRARIES_PROJECTS) ) - self.assertEqual(options, ["-DTHEROCK_COVERAGE_ROCM_LIBRARIES_ALL=ON"]) + self.assertNotIn("-DTHEROCK_COVERAGE_ALL=ON", options) + self.assertNotIn("-DTHEROCK_COVERAGE_ROCM_SYSTEMS_ALL=ON", options) def test_no_selection_produces_no_options(self): self.assertEqual(configure_coverage_ci.build_coverage_cmake_options([]), []) diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 6f47ff03980..3e0ce021ad1 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -268,6 +268,13 @@ directory in the artifact; keeping the baseline's would fail with `Cannot find S load the instrumented kpack, so only the swapped project's tests are expected to pass. +The prim projects (rocPRIM, hipCUB, rocThrust) are built in the coverage run +with `-DTHEROCK_FLAG_BUILD_HIPCCL=OFF`. The default unified hipCCL superbuild +has no rocPRIM/hipCUB/rocThrust subprojects for their `_ENABLE_COVERAGE` +flags to reach, so it would ship uninstrumented tests, all under one +`hipCCL_tests` stage directory. With no stage directory of its own, hipCUB would +not take the shared kpack, and its host binaries would fail with `device kernel image is invalid` against the baseline's kernels. + A hip\* wrapper also brings the backend it calls at runtime from the same artifact (`SAME_ARTIFACT_BACKENDS`: hipRAND brings rocRAND, hipFFT rocFFT, and so on). Otherwise the instrumented wrapper runs against the baseline's backend, From c9a42ba62e6e7d09e6010ef6403a8fbb516e1630 Mon Sep 17 00:00:00 2001 From: "Kasula, Madhusudhan" Date: Wed, 11 Nov 2026 11:53:21 +0530 Subject: [PATCH 70/70] fix(coverage): swap hipSPARSELt's host libs from the blas artifact ## Motivation The instrumented libhipsparselt links libtensilelite-host (hipBLASLt) and liborigami, which ship in the blas artifact rather than sparse. The coverage test job overlaid only sparse, so hipSPARSELt ran against the baseline's copies, built from a different rocm-libraries commit. Their C++ interfaces did not match: in run 42687317066, 3082 hipsparselt-test cases failed with munmap_chunk() heap corruption, aborts and "No Tensile solution found". ## Technical Details - install_rocm_code_coverage_build.py: add OTHER_ARTIFACT_HOST_LIBS, mapping a component to (artifact, library folder, lib/ file prefix) entries; hipsparselt takes libtensilelite-host from hipBLASLt and liborigami from origami. host_library_deps() collects them for the selected components. - download_replacement_artifacts() also fetches only the generic lib archive of those artifacts (blas_lib_generic), skipping the large gfx942, test and dev archives, unless the artifact is already fetched whole. - New replace_host_libraries() extracts just the matching lib/* files (with their symlinks) from those stage folders, pins program headers like the other swapped libraries, and errors if an expected library is missing instead of silently testing against the baseline. - Factor the manifest read and phdr pinning into _read_manifest() and _pin_instrumented(). - Tests: mapping, generic-lib-only download, extraction limited to the named libraries, and the missing-library error. - docs/development/code_coverage.md: note the exception to other-artifact dependencies coming from the baseline. --- .../install_rocm_code_coverage_build.py | 135 ++++++++++++++++-- .../install_rocm_code_coverage_build_test.py | 103 +++++++++++++ docs/development/code_coverage.md | 6 +- 3 files changed, 228 insertions(+), 16 deletions(-) diff --git a/build_tools/install_rocm_code_coverage_build.py b/build_tools/install_rocm_code_coverage_build.py index f56f6a15bb1..5931695fb17 100644 --- a/build_tools/install_rocm_code_coverage_build.py +++ b/build_tools/install_rocm_code_coverage_build.py @@ -96,6 +96,34 @@ def replacement_folders(components: Iterable[str]) -> dict[str, list[str]]: return artifacts +# Host libraries a component links that ship in another artifact, as +# (artifact, library folder, file name prefix under lib/). Only these files are +# swapped, from the coverage run's generic lib archive: the baseline's copies +# come from a different source commit, and their C++ interfaces need not match +# the instrumented component's. hipSPARSELt picks and launches its kernels +# through hipBLASLt's tensilelite-host and origami; against the baseline's it +# aborts with heap corruption or "No Tensile solution found". +OTHER_ARTIFACT_HOST_LIBS = { + "hipsparselt": ( + ("blas", "hipBLASLt", "libtensilelite-host"), + ("blas", "origami", "liborigami"), + ), +} + + +def host_library_deps( + components: Iterable[str], +) -> dict[str, list[tuple[str, str]]]: + """Map each artifact to the (library folder, file prefix) pairs to take from it.""" + deps: dict[str, list[tuple[str, str]]] = {} + for comp in components: + for artifact, folder, lib_prefix in OTHER_ARTIFACT_HOST_LIBS.get(comp, ()): + entries = deps.setdefault(artifact, []) + if (folder, lib_prefix) not in entries: + entries.append((folder, lib_prefix)) + return deps + + def _read_passthrough_options(passthrough_argv): """Read options shared with install_rocm_from_artifacts without consuming them. @@ -132,10 +160,13 @@ def download_replacement_artifacts( opts, code_coverage_run_github_repo, code_coverage_release_type, + host_lib_artifacts=(), ): """Download instrumented replacement artifacts from the code-coverage run. - Fetches every component tar matching the requested artifact names and family. + Fetches every component tar matching the requested artifact names and family, + plus only the generic lib tar of each host_lib_artifacts entry not already + fetched whole. The coverage run lives in its own repo (--code-coverage-run-github-repo), separate from the generic baseline's --run-github-repo. Pass it explicitly: the S3 bucket lookup 404s if the run id is queried against the wrong repo. @@ -159,6 +190,22 @@ def download_replacement_artifacts( f"{backend.base_uri} for families {target_families}" ) + host_lib_names = set(host_lib_artifacts) - set(artifact_names) + if host_lib_names: + log(f"Matching host library artifacts {sorted(host_lib_names)} (generic lib)") + host_matched = find_available_artifacts( + host_lib_names, + ["generic"], + available, + excluded_components={"dbg", "dev", "doc", "run", "test"}, + ) + if not host_matched: + raise IOError( + f"ERROR: No host library artifacts {sorted(host_lib_names)} found " + f"in {backend.base_uri}" + ) + matched += host_matched + dest_dir = opts.output_dir / "code-coverage-replacements" dest_dir.mkdir(parents=True, exist_ok=True) @@ -279,17 +326,7 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): log(f"Replacing '{folder}' paths from {archive.name} into {output_dir}") replaced = 0 with open_archive_for_read(archive) as tf: - manifest_member = tf.next() - if ( - manifest_member is None - or manifest_member.name != "artifact_manifest.txt" - ): - raise IOError( - f"Artifact archive {archive} must have artifact_manifest.txt " - "as its first member" - ) - with tf.extractfile(manifest_member) as mf_file: - relpaths = [r for r in mf_file.read().decode().splitlines() if r] + relpaths = _read_manifest(tf, archive) owns_archive = any(_is_own_stage(folder, r) for r in relpaths) while member := tf.next(): @@ -312,6 +349,71 @@ def replace_instrumented_libraries(artifacts, dest_dir, output_dir): break log(f" Replaced {replaced} '{folder}' path(s) from {archive.name}") + _pin_instrumented(written_files) + + +def replace_host_libraries(host_libs, dest_dir, output_dir): + """Extract only the named host libraries from other artifacts' lib archives. + + host_libs maps an artifact to (library folder, file prefix) pairs, as built + by host_library_deps. A member is taken when it sits in that folder's stage + directory under lib/ and its file name starts with the prefix, so + libtensilelite-host.so* comes over without the rest of hipBLASLt. + """ + written_files = [] + for archive in sorted( + p for p in dest_dir.iterdir() if p.name.endswith((".tar.zst", ".tar.xz")) + ): + an = ArtifactName.from_filename(archive.name) + if not an or an.name not in host_libs: + continue + if an.component != "lib" or an.target_family != "generic": + continue + for folder, lib_prefix in host_libs[an.name]: + log(f"Replacing '{folder}' {lib_prefix}* from {archive.name}") + replaced = 0 + with open_archive_for_read(archive) as tf: + relpaths = _read_manifest(tf, archive) + while member := tf.next(): + for prefix in relpaths: + if not _is_own_stage(folder, prefix): + continue + prefix_slash = prefix + "/" + if not member.name.startswith(prefix_slash): + continue + scoped_path = member.name[len(prefix_slash) :] + if not scoped_path.startswith(f"lib/{lib_prefix}"): + break + dest_path = output_dir / PurePosixPath(scoped_path) + _replace_scoped_member( + tf, member, dest_path, output_dir, relpaths + ) + if member.isfile(): + written_files.append(dest_path) + replaced += 1 + break + if not replaced: + raise IOError( + f"ERROR: No {lib_prefix}* found under '{folder}' in {archive.name}" + ) + log(f" Replaced {replaced} path(s)") + + _pin_instrumented(written_files) + + +def _read_manifest(tf, archive): + """Return the relpath prefixes listed by the archive's leading manifest.""" + manifest_member = tf.next() + if manifest_member is None or manifest_member.name != "artifact_manifest.txt": + raise IOError( + f"Artifact archive {archive} must have artifact_manifest.txt " + "as its first member" + ) + with tf.extractfile(manifest_member) as mf_file: + return [r for r in mf_file.read().decode().splitlines() if r] + + +def _pin_instrumented(written_files): for path in written_files: if pin_phdr_table(path, require_section="__llvm_prf_cnts"): log(f" Pinned the program header table kpack moved in {path}") @@ -365,9 +467,9 @@ def main(argv): install_from_artifacts_main(extra_args) # collect artifacts details as per the user options - artifacts = replacement_folders( - comp for comp in COMPONENT_MAP if getattr(args, f"replace_{comp}") - ) + selected = [comp for comp in COMPONENT_MAP if getattr(args, f"replace_{comp}")] + artifacts = replacement_folders(selected) + host_libs = host_library_deps(selected) if not artifacts: log("No --replace-* components specified; nothing to download.") @@ -385,10 +487,13 @@ def main(argv): opts, args.code_coverage_run_github_repo, args.code_coverage_release_type, + host_lib_artifacts=host_libs.keys(), ) # replace selected library folder paths in selected component artifacts replace_instrumented_libraries(artifacts, dest_dir, opts.output_dir) + # and the host libraries they link from other artifacts + replace_host_libraries(host_libs, dest_dir, opts.output_dir) if __name__ == "__main__": diff --git a/build_tools/tests/install_rocm_code_coverage_build_test.py b/build_tools/tests/install_rocm_code_coverage_build_test.py index c7e01cc7d3e..55996cceaec 100644 --- a/build_tools/tests/install_rocm_code_coverage_build_test.py +++ b/build_tools/tests/install_rocm_code_coverage_build_test.py @@ -276,5 +276,108 @@ def test_backends_ship_in_their_wrappers_artifact(self): self.assertEqual(owner, artifact) +class TestHostLibraries(unittest.TestCase): + def test_hipsparselt_takes_tensilelite_host_and_origami_from_blas(self): + self.assertEqual( + mod.host_library_deps(["hipsparselt"]), + { + "blas": [ + ("hipBLASLt", "libtensilelite-host"), + ("origami", "liborigami"), + ] + }, + ) + + def test_component_without_host_libraries_needs_none(self): + self.assertEqual(mod.host_library_deps(["rocblas"]), {}) + + def test_downloads_only_the_generic_lib_archive_of_host_artifacts(self): + class _Backend: + base_uri = "s3://fake-bucket/fake" + + def list_artifacts(self, name_filter=None): + return [ + "sparse_lib_generic.tar.zst", + "sparse_test_gfx942.tar.zst", + "blas_lib_generic.tar.zst", + "blas_lib_gfx942.tar.zst", + "blas_dev_generic.tar.zst", + "blas_test_generic.tar.zst", + ] + + with tempfile.TemporaryDirectory() as tmp: + opts = mock.Mock( + artifact_group="gfx94X-dcgpu", amdgpu_targets="", output_dir=Path(tmp) + ) + with ( + mock.patch.object( + mod, "create_backend_from_env", return_value=_Backend() + ), + mock.patch.object( + mod, "download_artifact", return_value=Path(tmp) / "x" + ) as download, + ): + mod.download_replacement_artifacts( + "222", {"sparse"}, opts, "ROCm/TheRock", "ci", {"blas"} + ) + fetched = sorted(c.args[0].artifact_key for c in download.call_args_list) + self.assertEqual( + fetched, + [ + "blas_lib_generic.tar.zst", + "sparse_lib_generic.tar.zst", + "sparse_test_gfx942.tar.zst", + ], + ) + + def test_replaces_only_the_named_host_libraries(self): + hipblaslt = "math-libs/BLAS/hipBLASLt/stage" + origami = "math-libs/BLAS/origami/stage" + rocblas = "math-libs/BLAS/rocBLAS/stage" + members = { + "artifact_manifest.txt": f"{hipblaslt}\n{origami}\n{rocblas}\n".encode(), + f"{hipblaslt}/lib/libhipblaslt.so.1.0": b"hipblaslt", + f"{hipblaslt}/lib/libtensilelite-host.so.1.0": b"tensilelite-host", + f"{origami}/lib/liborigami.so.1.0": b"origami", + f"{rocblas}/lib/librocblas.so.5": b"rocblas", + } + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + downloads = tmp / "downloads" + downloads.mkdir() + _write_archive(downloads / "blas_lib_generic.tar.xz", members) + output_dir = tmp / "install" + with mock.patch.object(mod, "pin_phdr_table", return_value=False): + mod.replace_host_libraries( + mod.host_library_deps(["hipsparselt"]), downloads, output_dir + ) + written = sorted( + p.relative_to(output_dir).as_posix() + for p in output_dir.rglob("*") + if p.is_file() + ) + self.assertEqual( + written, ["lib/liborigami.so.1.0", "lib/libtensilelite-host.so.1.0"] + ) + + def test_missing_host_library_is_an_error(self): + prefix = "math-libs/BLAS/hipBLASLt/stage" + members = { + "artifact_manifest.txt": f"{prefix}\n".encode(), + f"{prefix}/lib/libhipblaslt.so.1.0": b"hipblaslt", + } + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + downloads = tmp / "downloads" + downloads.mkdir() + _write_archive(downloads / "blas_lib_generic.tar.xz", members) + with self.assertRaises(IOError): + mod.replace_host_libraries( + {"blas": [("hipBLASLt", "libtensilelite-host")]}, + downloads, + tmp / "install", + ) + + if __name__ == "__main__": unittest.main() diff --git a/docs/development/code_coverage.md b/docs/development/code_coverage.md index 3e0ce021ad1..30d3dc9ad0e 100644 --- a/docs/development/code_coverage.md +++ b/docs/development/code_coverage.md @@ -284,7 +284,11 @@ librocrand) with the `ROCRAND_VERSION` it was compiled against. The backend's objects are not in the wrapper's `--object-globs`, so its own code stays out of the report, but header templates both instantiate share profile counters. Dependencies in other artifacts (fftw3, rocBLAS for hipSOLVER) still come from -the baseline. The instrumented stack lives under +the baseline, except host libraries listed in `OTHER_ARTIFACT_HOST_LIBS`: +hipSPARSELt takes `libtensilelite-host` (hipBLASLt) and `liborigami` from the +coverage run's `blas_lib_generic`, because it calls their C++ interfaces and +aborts against the baseline's copies. Only those files are swapped. The +instrumented stack lives under `-coverage`, so even a baseline from the same workflow run is never overwritten by it.