Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,42 @@ schema_path = pkg_resources.files("rocinsight.ai_analysis") / "docs" / "analysis

---

## v0.5.0 — 2026-04-13

**ROCM-21553: Proposed improvements after first tests.**

**New optional fields in recommendation objects:**
- `confidence` (`number | null`, 0.0–1.0): Analysis confidence level for the
recommendation. Higher values indicate more data-driven confidence.

**New optional top-level fields:**
- `kernel_resources`: Per-kernel VGPR/SGPR/LDS/scratch usage and theoretical
occupancy (architecture-specific). Includes `arch`, `arch_specs`, and
`kernels` array with occupancy sub-objects.
- `api_breakdown`: Per-HIP-API call duration breakdown with `api_calls` array,
`launch_overhead_ns`, and `total_api_ns`.
- `roctx_regions`: roctx marker region analysis with per-region kernel/memcpy
correlation and unranged activity breakdown. `null` when no markers present.

**Changed recommendation categories:**
- `"Compute Bottleneck"` replaced by context-aware labels:
- `"Kernel Hotspot"` (Tier 1, no counters)
- `"Compute-Bound Kernel"` (Tier 2, GPU util > 90%)
- `"Mixed Bottleneck Kernel"` (Tier 2, GPU util 70–90%)
- `"Memory-Bound Kernel"` (Tier 2, GPU util < 70%)
- New categories: `"Warmup"`, `"Profiling Scope"`, `"GPU Utilization (Scoped)"`

**Changed `estimated_impact` values:** Now data-driven (reference actual metrics)
instead of generic percentage ranges.

**Tool command fixes:**
- `rocprof-sys` (deprecated wrapper) → `rocprof-sys-sample` (recommended)
- `--trace-gpu-memory` (nonexistent) → `--trace --device`
- Schema tool enum expanded: `rocprof-sys-sample`, `rocprof-sys-instrument`,
`rocprof-sys-run` added alongside existing `rocprofv3` and `rocprof-compute`

---

## v0.3.2 — 2026-03-25

**No schema changes.** Branding, output routing, and LLM hardening only.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -754,8 +754,8 @@
"properties": {
"tool": {
"type": "string",
"enum": ["rocprofv3", "rocprof-sys", "rocprof-compute"],
"description": "The ROCm profiling tool to invoke. 'rocprofv3' for trace/counter collection, 'rocprof-sys' for system-level profiling (Omnitrace), 'rocprof-compute' for kernel-level hardware counter analysis."
"enum": ["rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"],
"description": "The ROCm profiling tool to invoke. 'rocprofv3' for trace/counter collection, 'rocprof-sys-sample' for sampling-based system profiling, 'rocprof-sys-instrument' for binary instrumentation, 'rocprof-sys-run' for running instrumented binaries, 'rocprof-compute' for kernel-level hardware counter analysis."
},
"description": {
"type": "string",
Expand Down Expand Up @@ -805,8 +805,8 @@
"description": "Complete ready-to-run shell command including tool name, all flags, all args, and application placeholder '-- ./app'. Consumers can display this directly for copy-paste. Application arguments should be appended after '-- ./app'.",
"examples": [
"rocprofv3 --sys-trace --pmc SQ_WAVES GRBM_COUNT GRBM_GUI_ACTIVE -d ./output -o profile -- ./app",
"rocprof-sys --trace-gpu-memory -- ./app",
"rocprof-compute -i profile.db --filter kernel_name -- ./app"
"rocprof-sys-sample --trace --device -- ./app",
"rocprof-compute profile --kernel my_kernel -- ./app"
]
}
}
Expand Down Expand Up @@ -993,13 +993,11 @@
"full_command": "rocprofv3 --sys-trace --hsa-trace -d ./memcpy_output -o profile -- ./app"
},
{
"tool": "rocprof-sys",
"description": "Detailed memory transfer timeline with PCIe bandwidth and overlap analysis",
"flags": [],
"args": [
{"name": "--trace-gpu-memory", "value": null}
],
"full_command": "rocprof-sys --trace-gpu-memory -- ./app"
"tool": "rocprof-sys-sample",
"description": "System-level trace with device metrics for memory transfer overlap analysis",
"flags": ["--trace", "--device"],
"args": [],
"full_command": "rocprof-sys-sample --trace --device -- ./app"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1182,11 +1182,11 @@ def _generate_profiling_commands(
"full_command": "rocprofv3 --sys-trace -d ./sync_output -o sync_profile -- ./app",
},
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "System-level timeline showing CPU/GPU sync points and idle gaps",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand All @@ -1208,11 +1208,11 @@ def _generate_profiling_commands(
"estimated_impact": "Depends on call frequency; 5-30% improvement if in hot loop",
"commands": [
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "Timeline view to identify CPU/GPU synchronization points",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand Down Expand Up @@ -1273,11 +1273,11 @@ def _generate_profiling_commands(
"estimated_impact": "10-50% throughput improvement for workloads with independent work",
"commands": [
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "Visualize kernel concurrency gaps on the default stream",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand All @@ -1295,18 +1295,18 @@ def _generate_profiling_commands(
),
"suggestion": "Replace hipMallocManaged with explicit hipMalloc + hipMemcpy for predictable performance",
"actions": [
"Profile page-fault overhead with rocprof-sys --trace-gpu-memory",
"Profile page-fault overhead with rocprof-sys-sample --trace --device",
"Replace hipMallocManaged with hipMalloc (device) + hipHostMalloc (host) pairs",
"Use explicit hipMemcpy to control when data moves between host and device",
],
"estimated_impact": "Can eliminate page-migration stalls; 2-10x improvement in some cases",
"commands": [
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "Trace GPU memory page migrations and access patterns",
"flags": [],
"args": [{"name": "--trace-gpu-memory", "value": None}],
"full_command": "rocprof-sys --trace-gpu-memory -- ./app",
"flags": ["--trace", "--device"],
"args": [],
"full_command": "rocprof-sys-sample --trace --device -- ./app",
},
],
}
Expand Down Expand Up @@ -1418,11 +1418,11 @@ def _generate_profiling_commands(
"estimated_impact": "Framework profiler reveals op-level bottlenecks before HW counter collection",
"commands": [
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": f"System-level trace capturing {fw} GPU kernel timeline",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- python ./train.py",
"full_command": "rocprof-sys-sample --trace -- python ./train.py",
},
{
"tool": "rocprofv3",
Expand Down Expand Up @@ -1458,11 +1458,11 @@ def _generate_profiling_commands(
"estimated_impact": "Communication optimizations can yield 1.2-2x on multi-GPU workloads",
"commands": [
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "System timeline showing inter-GPU communication and kernel overlap",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,14 @@ def _filter_rec_commands(
- If after stripping, a rocprofv3 command has no remaining flags AND
its args contain only output-path or scope-filter entries (-d / -o /
--kernel-names / etc.), the command adds no new data and is dropped.
- ``rocprof-sys --trace`` alone is equivalent to ``rocprofv3 --sys-trace``
- ``rocprof-sys-sample --trace`` alone is equivalent to ``rocprofv3 --sys-trace``
(same HIP/HSA API data, just in Perfetto format instead of rocpd format)
and is dropped when sys-trace data is already present. ``rocprof-sys``
and is dropped when sys-trace data is already present. ``rocprof-sys-sample``
commands that carry *additional* flags beyond ``--trace`` (e.g.
``--trace-gpu-memory``, ``--call-stack-sampling``) are always kept
``--device``, ``--call-stack-sampling``) are always kept
because they collect data that rocprofv3 cannot.
- ``rocprof-compute`` commands are always kept -- they perform a deep
hardware counter analysis that neither rocprofv3 nor rocprof-sys covers.
hardware counter analysis that neither rocprofv3 nor rocprof-sys-sample covers.
- A short note is appended to ``description`` when flags/counters are
stripped so the user knows why the command looks different from the docs.
"""
Expand All @@ -170,16 +170,16 @@ def _filter_rec_commands(
flags = cmd.get("flags", [])
args = cmd.get("args", [])

# -- rocprof-sys -------------------------------------------------------
if tool == "rocprof-sys" and has_sys_trace:
# -- rocprof-sys-sample ------------------------------------------------
if tool == "rocprof-sys-sample" and has_sys_trace:
# --trace alone ~ rocprofv3 --sys-trace; drop if it adds nothing new
extra_flags = [f for f in flags if f != "--trace"]
meaningful_args = [
a for a in args if a.get("name", "") not in _OUTPUT_ONLY_ARGS
]
if not extra_flags and not meaningful_args:
continue # equivalent to already-collected sys-trace data
# Has meaningful extra flags (e.g. --trace-gpu-memory) -> keep as-is
# Has meaningful extra flags (e.g. --device) -> keep as-is
filtered.append(cmd)
continue

Expand Down Expand Up @@ -399,11 +399,11 @@ def generate_recommendations(
"full_command": "rocprofv3 --sys-trace --pmc GRBM_GUI_ACTIVE GRBM_COUNT -d ./utilization_output -o profile -- ./app",
},
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "System-level timeline: identify host/GPU idle gaps and synchronization stalls",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand Down Expand Up @@ -569,13 +569,11 @@ def generate_recommendations(
"full_command": "rocprofv3 --sys-trace --hsa-trace -d ./memcpy_output -o profile -- ./app",
},
{
"tool": "rocprof-sys",
"description": "Detailed memory transfer timeline with PCIe bandwidth and overlap analysis",
"flags": [],
"args": [
{"name": "--trace-gpu-memory", "value": None},
],
"full_command": "rocprof-sys --trace-gpu-memory -- ./app",
"tool": "rocprof-sys-sample",
"description": "System-level trace with device metrics for memory transfer overlap analysis",
"flags": ["--trace", "--device"],
"args": [],
"full_command": "rocprof-sys-sample --trace --device -- ./app",
},
],
}
Expand Down Expand Up @@ -635,14 +633,14 @@ def generate_recommendations(
{"name": "-d", "value": "./api_output"},
{"name": "-o", "value": "profile"},
],
"full_command": "rocprofv3 --hip-api-trace --hsa-trace -d ./api_output -o profile -- ./app",
"full_command": "rocprofv3 --hip-trace --hsa-trace -d ./api_output -o profile -- ./app",
},
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "System-level API call frequency and per-call latency breakdown",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand Down Expand Up @@ -793,11 +791,11 @@ def generate_recommendations(
"full_command": "rocprofv3 --sys-trace -d ./launch_output -o profile -- ./app",
},
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "Visualize kernel launch timeline and inter-launch gaps in a Perfetto trace",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
],
}
Expand Down Expand Up @@ -947,11 +945,11 @@ def generate_recommendations(
"full_command": "rocprofv3 --sys-trace --pmc GRBM_COUNT GRBM_GUI_ACTIVE SQ_WAVES -d ./counters_output -o profile -- ./app",
},
{
"tool": "rocprof-sys",
"tool": "rocprof-sys-sample",
"description": "Full system trace for comprehensive performance timeline",
"flags": ["--trace"],
"args": [],
"full_command": "rocprof-sys --trace -- ./app",
"full_command": "rocprof-sys-sample --trace -- ./app",
},
{
"tool": "rocprof-compute",
Expand Down
12 changes: 11 additions & 1 deletion experimental/python/rocinsight/rocinsight/formatters/json_fmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,17 +166,27 @@ def _format_as_json(
doc["metadata"]["analysis_version"] = "0.4.0"
doc["profiling_info"]["analysis_tier"] = 3

# ROCM-21553: kernel resources and API overhead
# ROCM-21553: kernel resources, API overhead, roctx regions
_has_rocm21553_fields = False
if kernel_resources and kernel_resources.get("kernels"):
doc["kernel_resources"] = kernel_resources
_has_rocm21553_fields = True
if api_overhead and api_overhead.get("has_api_data"):
doc["api_breakdown"] = {
"total_api_ns": api_overhead["total_api_ns"],
"launch_overhead_ns": api_overhead["launch_overhead_ns"],
"api_calls": api_overhead["api_calls"],
}
_has_rocm21553_fields = True
if roctx_regions and roctx_regions.get("has_markers"):
doc["roctx_regions"] = roctx_regions
_has_rocm21553_fields = True

# Recommendations always carry confidence now (added in ROCM-21553)
# Bump schema version to 0.5.0 when any ROCM-21553 field is present
if _has_rocm21553_fields:
doc["schema_version"] = "0.5.0"
doc["metadata"]["analysis_version"] = "0.5.0"

return _json.dumps(doc, indent=2)

Expand Down
2 changes: 1 addition & 1 deletion experimental/python/rocinsight/tests/test_analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ def test_tier2_commands_use_valid_tools():
"""Tier 2 recommendations include commands with valid tool names."""
from rocinsight.analyze import generate_recommendations

VALID_TOOLS = {"rocprofv3", "rocprof-sys", "rocprof-compute"}
VALID_TOOLS = {"rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"}
recs = generate_recommendations(
_empty_breakdown(),
[],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def files(package):
"errors",
]

COMMAND_TOOLS = {"rocprofv3", "rocprof-sys", "rocprof-compute"}
COMMAND_TOOLS = {"rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"}


def _load_schema():
Expand Down
Loading