feat(ci): add code coverage component test workflow and installer

.github/workflows/test_code_coverage_component.yml

@@ -0,0 +1,306 @@
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+name: Test Code Coverage Component
+
+on:
+  workflow_call:
+    inputs:
+      generic_run_id:
+        type: string
+        default: ""
+      code_coverage_run_id:
+        type: string
+        default: ""
+      artifact_group:
+        type: string
+      amdgpu_families:
+        type: string
+      amdgpu_targets:
+        type: string
+        default: ""
+      test_runs_on:
+        type: string
+      platform:
+        type: string
+      component:
+        type: string
+      release_type:
+        type: string
+        default: ci
+      repository:
+        description: "Repository to checkout. Defaults to github.repository."
+        type: string
+      ref:
+        description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref."
+        type: string
+      build_variant:
+        description: "Build variant (e.g., release, asan). Used to skip tests incompatible with certain builds."
+        type: string
+        default: "release"
+      coverage_enabled:
+        description: >-
+          Collect llvm profraw files from the test run and upload them as a
+          workflow artifact. Only meaningful against a coverage instrumented
+          build.
+        type: boolean
+        default: false
+      coverage_artifact_prefix:
+        description: >-
+          Prefix for the uploaded profraw artifact name. A coverage run tests
+          several projects in one workflow run, so the caller sets this to
+          something project-specific to keep each project profiles apart.
+        type: string
+        default: "coverage-profraw"
+      default_container_image:
+        type: string
+        default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98"
+      quartz_tracking_id:
+        type: string
+        default: ""
+
+
+permissions:
+  contents: read
+
+jobs:
+  test_component:
+    name: >-
+      Test ${{ fromJSON(inputs.component).job_name }}
+      (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }})
+      (${{ inputs.amdgpu_families }})
+      ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }}
+    runs-on: ${{ inputs.test_runs_on }}
+    continue-on-error: ${{ fromJSON(inputs.component).expect_failure == true }}
+    timeout-minutes: 210
+    outputs:
+      summary: ${{ steps.summary.outputs.summary }}
+    container:
+      # If the component has specified an alternate image, honor that option.
+      # Otherwise use the default image.
+      image: >-
+        ${{ inputs.platform == 'linux' &&
+            (fromJSON(inputs.component).container_image || inputs.default_container_image)
+            || null
+        }}
+      # Container options are configured in build_tools/github_actions/fetch_test_configurations.py
+      # and include base options (ipc, user, ulimit, seccomp), GPU options (devices, groups),
+      # and any job-specific options
+      options: ${{ fromJSON(inputs.component).container_options }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # The shard array is based on "total_shards" from "fetch_test_configurations.py"
+        # The test executable will shard based on the array. (ex: [1, 2, 3, 4] = four test shards)
+        shard: ${{ fromJSON(inputs.component).shard_arr }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      QUARTZ_REPORTING_WORKFLOW: test_component.yml
+      QUARTZ_WORKFLOW_POSTFIX: "- test ROCm component"
+      VENV_DIR: ${{ github.workspace }}/.venv
+      CODE_COVERAGE_RUN_ID: "${{ inputs.code_coverage_run_id != '' && inputs.code_coverage_run_id || github.run_id }}"
+      OUTPUT_ARTIFACTS_DIR: "./build"
+      THEROCK_BIN_DIR: "./build/bin"
+      AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
+      AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }}
+      ARTIFACT_GROUP: ${{ inputs.artifact_group }}
+      BUILD_VARIANT: ${{ inputs.build_variant }}
+      # Benchmark results database API endpoints for performance metrics collection
+      # NOTE: These secrets are only required for benchmark results submission in nightly CI runs.
+      # For PR/push workflows, secret retrieval will fail gracefully and benchmarks will skip API submission.
+      BENCHMARK_DB_URL: ${{ secrets.BENCHMARK_DB_URL }}
+      BENCHMARK_DB_FALLBACK_URL: ${{ secrets.BENCHMARK_DB_FALLBACK_URL }}
+    steps:
+      - name: "Quartz - started ${{ env.QUARTZ_WORKFLOW_POSTFIX }}"
+        continue-on-error: true
+        if: ${{ vars.NOTIFY_QUARTZ_ENABLED == 'true' }}
+        uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1
+        with:
+          run_phase: started
+          reporting_workflow: ${{ env.QUARTZ_REPORTING_WORKFLOW }}
+          workflow_inputs: ${{ toJSON(inputs) }}
+          gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }}
+          gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }}
+
+      - name: "Fetch 'build_tools' from repository"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ inputs.repository || github.repository }}
+          ref: ${{ inputs.ref }}
+          sparse-checkout: build_tools
+          path: "prejob"
+
+      - name: Pre-job cleanup processes on Windows
+        if: ${{ runner.os == 'Windows' }}
+        timeout-minutes: 5
+        shell: powershell
+        run: . '${{ github.workspace }}\prejob\build_tools\github_actions\cleanup_processes.ps1'
+
+      - name: Pre-job cleanup GPU processes on Linux
+        if: ${{ runner.os == 'Linux' }}
+        timeout-minutes: 5
+        run: bash ./prejob/build_tools/github_actions/cleanup_processes.sh
+
+      - name: Checking repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ inputs.repository || github.repository }}
+          ref: ${{ inputs.ref }}
+
+      #- name: Install Non-Instrumental TheRock Build with Replaced Instrumental Component
+      - name: Install TheRock Code Coverage Build
+        timeout-minutes: 15
+        env:
+          RELEASE_TYPE: ${{ inputs.release_type }}
+          FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }}
+        run: |
+          pip3 install -r requirements-test.txt
+          python3 build_tools/install_rocm_code_coverage_build.py \
+            --code-coverage-run-id=${{ env.CODE_COVERAGE_RUN_ID }} \
+            --run-id=${{ inputs.generic_run_id }} \
+            --artifact-group=${{ inputs.amdgpu_families }} \
+            --amdgpu-targets=${{ inputs.amdgpu_targets }} \
+            --tests \
+            --output-dir=${{ env.OUTPUT_ARTIFACTS_DIR }} \
+            ${FETCH_ARTIFACT_ARGS}
+
+      # safe.directory must be set before Runner Health Status
+      - name: Adjust git config
+        run: |
+          git config --global --add safe.directory $PWD
+          git config fetch.parallel 10
+
+      - name: Runner health status
+        run: |
+          python ./build_tools/health_status.py
+
+      - name: Driver / GPU sanity check
+        if: ${{ !fromJSON(inputs.component).linux_cpu_runner }}
+        timeout-minutes: 3
+        run: |
+          python ./build_tools/print_driver_gpu_info.py
+
+      - name: Setup Additional Requirements
+        if: ${{ fromJSON(inputs.component).additional_requirements_files != '' }}
+        run: |
+          python ./build_tools/install_additional_requirements.py \
+            --requirements-files ${{ join(fromJSON(inputs.component).additional_requirements_files, ',') }}
+
+        # TODO(#3755): Clean this up after final sanitizer environment variable flags are determined
+      - name: Enable sanitizer-specific test flags
+        if: ${{ contains(inputs.artifact_group, 'asan') }}
+        run: |
+          echo "ASAN_OPTIONS=detect_odr_violation=0:quarantine_size_mb=600" >> $GITHUB_ENV
+          echo "HSA_XNACK=1" >> $GITHUB_ENV
+          echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV
+
+      # Instrumented binaries write one profraw per process. %p (pid) and %m
+      # (binary signature) keep the files distinct so a shard that forks or
+      # loads several instrumented libraries does not overwrite itself.
+      - name: Prepare coverage profile directory
+        if: ${{ inputs.coverage_enabled }}
+        env:
+          TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }}
+          SHARD_INDEX: ${{ matrix.shard }}
+        run: |
+          mkdir -p "${GITHUB_WORKSPACE}/coverage-report/profraw"
+          echo "LLVM_PROFILE_FILE=${GITHUB_WORKSPACE}/coverage-report/profraw/${TEST_COMPONENT}-shard${SHARD_INDEX}-%p-%m.profraw" >> "${GITHUB_ENV}"
+
+      # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription.
+      # Must be done at runtime since container env vars aren't in GitHub's env context.
+      - name: Set thread limits
+        run: |
+          if [[ -n "${KUBE_CPU_REQUEST:-}" ]]; then
+            CPU_COUNT="${KUBE_CPU_REQUEST%.*}"
+            echo "OPENBLAS_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV
+            echo "OMP_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV
+            echo "Thread limits set: OPENBLAS_NUM_THREADS=${CPU_COUNT}, OMP_NUM_THREADS=${CPU_COUNT} (from KUBE_CPU_REQUEST=${KUBE_CPU_REQUEST})"
+          else
+            echo "KUBE_CPU_REQUEST not set, thread limits not configured"
+          fi
+
+      - name: Test
+        id: test
+        timeout-minutes: ${{ fromJSON(inputs.component).timeout_minutes }}
+        env:
+          SHARD_INDEX: ${{ matrix.shard }}
+          TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }}
+          TEST_TYPE: ${{ fromJSON(inputs.component).test_type }}
+          TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }}
+          ROCM_KPACK_DEBUG: "1"
+          # Windows hip-tests (PAL=1, ROCR=0) set this via matrix; other components
+          # leave it empty. #3587
+          # An empty GPU_ENABLE_PAL is misread as ROCr on Windows (atoi("")==0),
+          # whose Windows memory path OOMs some tests (e.g. rocsparse bsr*0 graph_test).
+          # Default unset components to the platform backend, mirroring the runtime fix
+          # in ROCm/rocm-systems#8050 (Windows -> PAL, Linux -> ROCr).
+          GPU_ENABLE_PAL: ${{ fromJSON(inputs.component).gpu_enable_pal || (inputs.platform == 'windows' && '1' || '0') }}
+          RETRY_THIS_STEP: "true"
+          RETRY_COUNT: "1"
+          RETRY_DELAY: "7"
+          STEP_TIMEOUT_MINUTES: ${{ fromJSON(inputs.component).timeout_minutes }}
+          STEP_NAME: >-
+            Test ${{ fromJSON(inputs.component).job_name }}
+            (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }})
+            (${{ inputs.amdgpu_families }})
+            ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }}
+        run: |
+          ${{ fromJSON(inputs.component).test_script }}
+
+      # Uploaded even when the test step failed: partial coverage from a failing
+      # shard is still useful, and the aggregation job tolerates missing shards.
+      - name: Upload coverage profraw files
+        if: ${{ always() && inputs.coverage_enabled }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: >-
+            ${{ inputs.coverage_artifact_prefix }}-${{ fromJSON(inputs.component).job_name }}-shard${{ matrix.shard }}
+          path: ${{ github.workspace }}/coverage-report/profraw/*.profraw
+          if-no-files-found: warn
+          retention-days: 3
+
+      - name: Print test reproduction command
+        if: ${{ failure() && steps.test.outcome == 'failure' }}
+        run: |
+          python ./build_tools/github_actions/reproduce_test_failure.py \
+            --run-id "${{ env.CODE_COVERAGE_RUN_ID }}" \
+            --repository "${{ github.repository }}" \
+            --amdgpu-family "${{ inputs.amdgpu_families }}" \
+            --amdgpu-targets "${{ inputs.amdgpu_targets }}" \
+            --test-script "${{ fromJSON(inputs.component).test_script }}" \
+            --shard-index "${{ matrix.shard }}" \
+            --total-shards "${{ fromJSON(inputs.component).total_shards }}" \
+            --test-type "${{ fromJSON(inputs.component).test_type }}" \
+            --print-cmd \
+            ${{ fromJSON(inputs.component).fetch_artifact_args && format('--fetch-artifact-args="{0}"', fromJSON(inputs.component).fetch_artifact_args) || '' }}
+
+      - name: Post-job cleanup GPU processes on Linux
+        if: ${{ always() && runner.os == 'Linux' }}
+        timeout-minutes: 5
+        run: bash ./build_tools/github_actions/cleanup_processes.sh
+
+      - name: Post-job cleanup processes on Windows
+        if: ${{ always() && runner.os == 'Windows' }}
+        timeout-minutes: 5
+        shell: powershell
+        run: . '${{ github.workspace }}\build_tools\github_actions\cleanup_processes.ps1'
+
+      - name: Capture job summary
+        id: summary
+        if: always()
+        run: python build_tools/github_actions/capture_job_summary.py
+
+      - name: "Quartz - completed ${{ env.QUARTZ_WORKFLOW_POSTFIX }}"
+        continue-on-error: true
+        if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' }}
+        uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1
+        with:
+          run_phase: completed
+          run_conclusion: ${{ job.status }}
+          reporting_workflow: ${{ env.QUARTZ_REPORTING_WORKFLOW }}
+          workflow_inputs: ${{ toJSON(inputs) }}
+          workflow_step_outputs: ${{ toJSON(steps) }}
+          gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }}
+          gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }}