[Proof of Concept ] code coverage for ROCm libraries

.github/scripts/coverage_runner.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""Run tests with coverage profiling and generate coverage reports.
+
+Invoked by the coverage ``Test`` step in
+``.github/workflows/therock-test-component.yml``: it sets ``LLVM_PROFILE_FILE``,
+runs the component's tests via ctest to produce profraw files, then merges them
+with ``llvm-profdata`` and renders text/HTML/LCOV reports with ``llvm-cov``. It is
+driven by the ``coverage_metadata.json`` produced by ``export_coverage_metadata.py``.
+
+Object and llvm tool locations are resolved by name within ``--build-dir`` so the
+script works whether it is pointed at a full build/dist tree or a staged artifact.
+"""
+import argparse
+import json
+import logging
+import os
+import subprocess
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+
+def find_tool(build_dir: Path, name: str) -> Path | None:
+    """Locate an executable ``name`` inside ``build_dir`` (prefer bin/ dirs)."""
+    candidates = [p for p in build_dir.rglob(name) if p.is_file()]
+    if not candidates:
+        return None
+    candidates.sort(key=lambda p: (0 if p.parent.name == "bin" else 1, len(str(p))))
+    return candidates[0]
+
+
+def resolve_objects(build_dir: Path, metadata: dict) -> list[Path]:
+    """Resolve coverage object files (shared libraries) within build_dir."""
+    objects: list[Path] = []
+
+    # Prefer the explicit relative paths recorded at export time, if present.
+    rel_objects = metadata.get("coverage_objects", {}).get("libraries", []) or []
+    for rel in rel_objects:
+        candidate = build_dir / rel
+        if candidate.is_file():
+            objects.append(candidate)
+
+    # Fall back to resolving by basename (handles a different layout than export).
+    if not objects:
+        basenames = metadata.get("object_basenames", {}).get("libraries", []) or []
+        for basename in basenames:
+            hits = [p for p in build_dir.rglob(f"{basename}*") if p.is_file()]
+            # Prefer concrete files (e.g. libhiprand.so.1.1) over bare symlinks,
+            # and the most specific (longest) name.
+            hits.sort(key=lambda p: (not p.is_symlink(), len(p.name)), reverse=True)
+            if hits:
+                objects.append(hits[0])
+
+    # De-duplicate while preserving order.
+    seen, unique = set(), []
+    for obj in objects:
+        key = obj.resolve()
+        if key not in seen:
+            seen.add(key)
+            unique.append(obj)
+    return unique
+
+
+def set_coverage_environment(metadata: dict, coverage_dir: Path):
+    pattern = metadata.get("llvm_profile_pattern", "%m")
+    profraw_dir = coverage_dir / "profraw"
+    profraw_dir.mkdir(parents=True, exist_ok=True)
+    profile_file = str(profraw_dir / f"{pattern}.profraw")
+    os.environ["LLVM_PROFILE_FILE"] = profile_file
+    logging.info("Set LLVM_PROFILE_FILE=%s", profile_file)
+
+
+def run_tests(test_dir: Path, metadata: dict) -> int:
+    category = metadata.get("test_category", "")
+    cmd = [
+        "ctest",
+        "--test-dir",
+        str(test_dir),
+        "--output-on-failure",
+        "--parallel",
+        "8",
+        "--timeout",
+        "7200",
+    ]
+    if category:
+        cmd += ["-L", category]
+    logging.info("Running tests: %s", " ".join(cmd))
+    return subprocess.run(cmd).returncode
+
+
+def merge_profraw_files(llvm_profdata: Path, profraw_dir: Path, out_path: Path) -> Path:
+    profraw_files = [str(p) for p in profraw_dir.rglob("*.profraw")]
+    if not profraw_files:
+        raise RuntimeError(f"No .profraw files found under {profraw_dir}")
+    logging.info("Merging %d profraw file(s)", len(profraw_files))
+    subprocess.run(
+        [str(llvm_profdata), "merge", "-sparse", "-o", str(out_path), *profraw_files],
+        check=True,
+    )
+    logging.info("Created %s", out_path)
+    return out_path
+
+
+def generate_reports(
+    llvm_cov: Path,
+    llvm_cxxfilt: Path | None,
+    objects: list[Path],
+    ignore_regex: str,
+    profdata: Path,
+    coverage_dir: Path,
+    project: str,
+    path_equivalence: str | None = None,
+):
+    object_args: list[str] = []
+    for obj in objects:
+        object_args += ["-object", str(obj)]
+    ignore_args = [f"-ignore-filename-regex={ignore_regex}"] if ignore_regex else []
+    # Map the build-node source prefix to the local checkout so llvm-cov can find
+    # sources for annotation (the build stages rocm-libraries at the workspace root;
+    # the test node checks it out under <workspace>/rocm-libraries).
+    patheq_args = [f"-path-equivalence={path_equivalence}"] if path_equivalence else []
+
+    text_report = coverage_dir / f"code_cov_{project}.report"
+    logging.info("Generating text report -> %s", text_report)
+    with open(text_report, "w") as f:
+        subprocess.run(
+            [
+                str(llvm_cov),
+                "report",
+                *object_args,
+                f"-instr-profile={profdata}",
+                *ignore_args,
+                *patheq_args,
+            ],
+            stdout=f,
+            check=True,
+        )
+    print(text_report.read_text())
+
+    logging.info("Generating HTML report -> %s", coverage_dir)
+    show_cmd = [
+        str(llvm_cov),
+        "show",
+        *object_args,
+        f"-instr-profile={profdata}",
+        *ignore_args,
+        *patheq_args,
+        "--format=html",
+        f"--output-dir={coverage_dir}",
+    ]
+    if llvm_cxxfilt is not None:
+        show_cmd.insert(2, f"-Xdemangler={llvm_cxxfilt}")
+    subprocess.run(show_cmd, check=True)
+
+    lcov_file = coverage_dir / "coverage.info"
+    logging.info("Generating LCOV export -> %s", lcov_file)
+    with open(lcov_file, "w") as f:
+        subprocess.run(
+            [
+                str(llvm_cov),
+                "export",
+                *object_args,
+                f"-instr-profile={profdata}",
+                *ignore_args,
+                *patheq_args,
+                "--format=lcov",
+            ],
+            stdout=f,
+            check=True,
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run/aggregate coverage and report")
+    parser.add_argument(
+        "--build-dir",
+        type=Path,
+        required=True,
+        help="Tree to resolve coverage objects and llvm tools from",
+    )
+    parser.add_argument(
+        "--metadata",
+        type=Path,
+        required=True,
+        help="coverage_metadata.json from export_coverage_metadata.py",
+    )
+    parser.add_argument(
+        "--coverage-dir", type=Path, required=True, help="Output directory for reports"
+    )
+    parser.add_argument(
+        "--test-dir",
+        type=Path,
+        default=None,
+        help="ctest directory (default: --build-dir) when running tests",
+    )
+    parser.add_argument(
+        "--path-equivalence",
+        type=str,
+        default=None,
+        help="Passed to llvm-cov as -path-equivalence=<from>,<to> so it can locate "
+        "sources when the build-time path differs from the local checkout",
+    )
+    args = parser.parse_args()
+
+    with open(args.metadata) as f:
+        metadata = json.load(f)
+    project = metadata.get("project", "project")
+    logging.info("Coverage for project: %s", project)
+
+    args.coverage_dir.mkdir(parents=True, exist_ok=True)
+    profraw_dir = args.coverage_dir / "profraw"
+
+    # Resolve tooling. Tools link libLLVM and bundled sysdeps that sit next to
+    # them, so add their directory to LD_LIBRARY_PATH.
+    llvm_profdata = find_tool(args.build_dir, "llvm-profdata")
+    llvm_cov = find_tool(args.build_dir, "llvm-cov")
+    llvm_cxxfilt = find_tool(args.build_dir, "llvm-cxxfilt")
+    if llvm_profdata is None or llvm_cov is None:
+        raise FileNotFoundError(
+            f"llvm-profdata/llvm-cov not found under {args.build_dir}"
+        )
+    tool_dirs = {str(t.parent) for t in (llvm_profdata, llvm_cov, llvm_cxxfilt) if t}
+    os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(
+        [*tool_dirs, os.environ.get("LD_LIBRARY_PATH", "")]
+    )
+    for t in (llvm_profdata, llvm_cov, llvm_cxxfilt):
+        if t:
+            t.chmod(0o755)
+
+    objects = resolve_objects(args.build_dir, metadata)
+    if not objects:
+        raise RuntimeError("No coverage object files could be resolved.")
+    logging.info("Coverage objects: %s", ", ".join(str(o) for o in objects))
+
+    set_coverage_environment(metadata, args.coverage_dir)
+    rc = run_tests(args.test_dir or args.build_dir, metadata)
+    if rc != 0:
+        logging.warning("Tests exited with %d; continuing with coverage.", rc)
+
+    # Coverage is a gating job: if the tests produced no profraw, fail loudly
+    # instead of passing as a silent no-op. An empty result means the library
+    # under test was not exercised or is not instrumented (e.g. the shipped
+    # artifact lacks instrumentation), which must surface rather than be hidden.
+    if not list(profraw_dir.rglob("*.profraw")):
+        logging.error(
+            "No .profraw files found under %s; the tests produced no coverage "
+            "data (the library under test is likely not instrumented). "
+            "Failing the coverage job.",
+            profraw_dir,
+        )
+        raise SystemExit(1)
+
+    profdata = merge_profraw_files(
+        llvm_profdata, profraw_dir, args.coverage_dir / f"{project}.profdata"
+    )
+    generate_reports(
+        llvm_cov,
+        llvm_cxxfilt,
+        objects,
+        metadata.get("ignore_filename_regex", ""),
+        profdata,
+        args.coverage_dir,
+        project,
+        path_equivalence=args.path_equivalence,
+    )
+    logging.info("Coverage generation complete.")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/export_coverage_metadata.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Export coverage metadata from a build tree to JSON for use in the report job.
+
+This is the build-time half of the coverage flow described in TheRock's coverage
+design docs. It reads ``test_categories_coverage.yaml``, verifies the configured
+coverage objects exist in the build/dist tree, and writes a ``coverage_metadata.json``
+that the coverage report job consumes later to merge profraw files and produce a report.
+
+Object/tool names are recorded by file name (basename). The report job locates the
+actual files in whatever layout it has (build dist or staged artifact), so version
+suffixes such as ``libhiprand.so.1.1`` are handled by the runner.
+"""
+import argparse
+import json
+import logging
+from pathlib import Path
+
+import yaml
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+
+def load_coverage_config(config_path: Path) -> dict:
+    with open(config_path) as f:
+        return yaml.safe_load(f)
+
+
+def find_by_basename(build_dir: Path, basename: str) -> list[str]:
+    """Find files in ``build_dir`` whose name starts with ``basename``.
+
+    Matches versioned shared libraries (e.g. libhiprand.so -> libhiprand.so.1.1).
+    Returns paths relative to ``build_dir`` so the metadata is location independent.
+    """
+    matches: list[str] = []
+    for path in build_dir.rglob(f"{basename}*"):
+        if path.is_file():
+            matches.append(str(path.relative_to(build_dir)))
+    return sorted(set(matches))
+
+
+def export_metadata(
+    build_dir: Path,
+    project: str,
+    config_path: Path,
+    output_path: Path,
+):
+    config = load_coverage_config(config_path)
+    # The coverage config is a per-project file: the document is the project's
+    # config directly, kept next to the project's test_categories.yaml.
+    key = project.lower()
+    project_config = config or {}
+
+    if not project_config.get("enabled", False):
+        logging.info("Coverage disabled for %s; nothing to export.", key)
+        return
+
+    objects = project_config.get("coverage_objects", {})
+
+    # Shared libraries are located by basename and staged separately (by the
+    # workflow) from dist/lib; the runner re-resolves them by basename.
+    found = {"libraries": []}
+    for basename in objects.get("libraries", []) or []:
+        hits = find_by_basename(build_dir, basename)
+        if hits:
+            found["libraries"].extend(hits)
+        else:
+            logging.warning("Coverage object not found for libraries: %s", basename)
+
+    metadata = {
+        "project": key,
+        "coverage_objects": found,
+        # Basenames are kept so the runner can re-resolve in a different layout.
+        "object_basenames": {
+            "libraries": objects.get("libraries", []) or [],
+        },
+        "ignore_filename_regex": project_config.get("ignore_filename_regex", ""),
+        "llvm_profile_pattern": project_config.get("llvm_profile_pattern", "%m"),
+        "test_category": project_config.get("test_category", ""),
+        "llvm_tools": {
+            "llvm_profdata": "llvm-profdata",
+            "llvm_cov": "llvm-cov",
+            "llvm_cxxfilt": "llvm-cxxfilt",
+        },
+    }
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+    logging.info("Exported coverage metadata to %s", output_path)
+    logging.info("  Libraries found:     %d", len(found["libraries"]))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Export coverage metadata to JSON")
+    parser.add_argument(
+        "--build-dir",
+        type=Path,
+        required=True,
+        help="Build/dist tree to search for coverage objects",
+    )
+    parser.add_argument(
+        "--project",
+        type=str,
+        required=True,
+        help="Project key or name (e.g. hiprand / HIPRAND)",
+    )
+    parser.add_argument(
+        "--config",
+        type=Path,
+        required=True,
+        help="Path to test_categories_coverage.yaml",
+    )
+    parser.add_argument(
+        "--output", type=Path, required=True, help="Output coverage_metadata.json path"
+    )
+    args = parser.parse_args()
+    export_metadata(
+        args.build_dir,
+        args.project,
+        args.config,
+        args.output,
+    )
+
+
+if __name__ == "__main__":
+    main()