From 5349dd3bda8fc02ea4336aece2e0f0c858a2b54c Mon Sep 17 00:00:00 2001 From: Ammar ELWazir Date: Mon, 13 Apr 2026 11:27:51 -0500 Subject: [PATCH 1/4] fix(rocm21553/pr4): correct rocprof-sys and rocprof-compute command syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified all recommended commands against actual tool CLIs and ROCm docs. rocprof-sys fixes: - "rocprof-sys" exists as a deprecated wrapper that forwards to rocprof-sys-instrument after a 60-second warning/acknowledgement prompt. Replaced with "rocprof-sys-sample" which is the recommended non-intrusive sampling tool (no binary rewrite needed, no deprecation warning). - "--trace-gpu-memory" flag does not exist on any rocprof-sys tool — replaced with "--trace --device" (-T -D: perfetto trace + device metrics sampling) - Updated schema tool enum to include all valid binaries: rocprof-sys-sample, rocprof-sys-instrument, rocprof-sys-run - Updated filter logic to match "rocprof-sys-sample" in command deduplication rocprof-compute fixes: - Schema example "rocprof-compute -i profile.db --filter kernel_name" had invalid flags (-i, --filter not recognized) — replaced with valid syntax - Verified: --block SQ/TD accepted as raw block names by arg parser - Verified: --kernel/-k valid for kernel filtering - Verified: profile subcommand with no flags runs full default collection 594 tests passing, 0 failures. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../docs/analysis-output.schema.json | 20 ++++----- .../rocinsight/ai_analysis/source_analyzer.py | 30 ++++++------- .../rocinsight/analysis/recommendations.py | 42 +++++++++---------- .../python/rocinsight/tests/test_analyze.py | 2 +- .../rocinsight/tests/test_analyze_schema.py | 2 +- 5 files changed, 46 insertions(+), 50 deletions(-) diff --git a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/analysis-output.schema.json b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/analysis-output.schema.json index 0ec04e42e5a5..65ea0e68c7c1 100644 --- a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/analysis-output.schema.json +++ b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/analysis-output.schema.json @@ -754,8 +754,8 @@ "properties": { "tool": { "type": "string", - "enum": ["rocprofv3", "rocprof-sys", "rocprof-compute"], - "description": "The ROCm profiling tool to invoke. 'rocprofv3' for trace/counter collection, 'rocprof-sys' for system-level profiling (Omnitrace), 'rocprof-compute' for kernel-level hardware counter analysis." + "enum": ["rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"], + "description": "The ROCm profiling tool to invoke. 'rocprofv3' for trace/counter collection, 'rocprof-sys-sample' for sampling-based system profiling, 'rocprof-sys-instrument' for binary instrumentation, 'rocprof-sys-run' for running instrumented binaries, 'rocprof-compute' for kernel-level hardware counter analysis." }, "description": { "type": "string", @@ -805,8 +805,8 @@ "description": "Complete ready-to-run shell command including tool name, all flags, all args, and application placeholder '-- ./app'. Consumers can display this directly for copy-paste. Application arguments should be appended after '-- ./app'.", "examples": [ "rocprofv3 --sys-trace --pmc SQ_WAVES GRBM_COUNT GRBM_GUI_ACTIVE -d ./output -o profile -- ./app", - "rocprof-sys --trace-gpu-memory -- ./app", - "rocprof-compute -i profile.db --filter kernel_name -- ./app" + "rocprof-sys-sample --trace --device -- ./app", + "rocprof-compute profile --kernel my_kernel -- ./app" ] } } @@ -993,13 +993,11 @@ "full_command": "rocprofv3 --sys-trace --hsa-trace -d ./memcpy_output -o profile -- ./app" }, { - "tool": "rocprof-sys", - "description": "Detailed memory transfer timeline with PCIe bandwidth and overlap analysis", - "flags": [], - "args": [ - {"name": "--trace-gpu-memory", "value": null} - ], - "full_command": "rocprof-sys --trace-gpu-memory -- ./app" + "tool": "rocprof-sys-sample", + "description": "System-level trace with device metrics for memory transfer overlap analysis", + "flags": ["--trace", "--device"], + "args": [], + "full_command": "rocprof-sys-sample --trace --device -- ./app" } ] } diff --git a/experimental/python/rocinsight/rocinsight/ai_analysis/source_analyzer.py b/experimental/python/rocinsight/rocinsight/ai_analysis/source_analyzer.py index 29c51bc81df4..17e6be052073 100644 --- a/experimental/python/rocinsight/rocinsight/ai_analysis/source_analyzer.py +++ b/experimental/python/rocinsight/rocinsight/ai_analysis/source_analyzer.py @@ -1182,11 +1182,11 @@ def _generate_profiling_commands( "full_command": "rocprofv3 --sys-trace -d ./sync_output -o sync_profile -- ./app", }, { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "System-level timeline showing CPU/GPU sync points and idle gaps", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -1208,11 +1208,11 @@ def _generate_profiling_commands( "estimated_impact": "Depends on call frequency; 5-30% improvement if in hot loop", "commands": [ { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "Timeline view to identify CPU/GPU synchronization points", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -1273,11 +1273,11 @@ def _generate_profiling_commands( "estimated_impact": "10-50% throughput improvement for workloads with independent work", "commands": [ { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "Visualize kernel concurrency gaps on the default stream", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -1295,18 +1295,18 @@ def _generate_profiling_commands( ), "suggestion": "Replace hipMallocManaged with explicit hipMalloc + hipMemcpy for predictable performance", "actions": [ - "Profile page-fault overhead with rocprof-sys --trace-gpu-memory", + "Profile page-fault overhead with rocprof-sys-sample --trace --device", "Replace hipMallocManaged with hipMalloc (device) + hipHostMalloc (host) pairs", "Use explicit hipMemcpy to control when data moves between host and device", ], "estimated_impact": "Can eliminate page-migration stalls; 2-10x improvement in some cases", "commands": [ { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "Trace GPU memory page migrations and access patterns", - "flags": [], - "args": [{"name": "--trace-gpu-memory", "value": None}], - "full_command": "rocprof-sys --trace-gpu-memory -- ./app", + "flags": ["--trace", "--device"], + "args": [], + "full_command": "rocprof-sys-sample --trace --device -- ./app", }, ], } @@ -1418,11 +1418,11 @@ def _generate_profiling_commands( "estimated_impact": "Framework profiler reveals op-level bottlenecks before HW counter collection", "commands": [ { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": f"System-level trace capturing {fw} GPU kernel timeline", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- python ./train.py", + "full_command": "rocprof-sys-sample --trace -- python ./train.py", }, { "tool": "rocprofv3", @@ -1458,11 +1458,11 @@ def _generate_profiling_commands( "estimated_impact": "Communication optimizations can yield 1.2-2x on multi-GPU workloads", "commands": [ { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "System timeline showing inter-GPU communication and kernel overlap", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } diff --git a/experimental/python/rocinsight/rocinsight/analysis/recommendations.py b/experimental/python/rocinsight/rocinsight/analysis/recommendations.py index 7a4cf934fc01..b53745885927 100644 --- a/experimental/python/rocinsight/rocinsight/analysis/recommendations.py +++ b/experimental/python/rocinsight/rocinsight/analysis/recommendations.py @@ -138,14 +138,14 @@ def _filter_rec_commands( - If after stripping, a rocprofv3 command has no remaining flags AND its args contain only output-path or scope-filter entries (-d / -o / --kernel-names / etc.), the command adds no new data and is dropped. - - ``rocprof-sys --trace`` alone is equivalent to ``rocprofv3 --sys-trace`` + - ``rocprof-sys-sample --trace`` alone is equivalent to ``rocprofv3 --sys-trace`` (same HIP/HSA API data, just in Perfetto format instead of rocpd format) - and is dropped when sys-trace data is already present. ``rocprof-sys`` + and is dropped when sys-trace data is already present. ``rocprof-sys-sample`` commands that carry *additional* flags beyond ``--trace`` (e.g. - ``--trace-gpu-memory``, ``--call-stack-sampling``) are always kept + ``--device``, ``--call-stack-sampling``) are always kept because they collect data that rocprofv3 cannot. - ``rocprof-compute`` commands are always kept -- they perform a deep - hardware counter analysis that neither rocprofv3 nor rocprof-sys covers. + hardware counter analysis that neither rocprofv3 nor rocprof-sys-sample covers. - A short note is appended to ``description`` when flags/counters are stripped so the user knows why the command looks different from the docs. """ @@ -170,8 +170,8 @@ def _filter_rec_commands( flags = cmd.get("flags", []) args = cmd.get("args", []) - # -- rocprof-sys ------------------------------------------------------- - if tool == "rocprof-sys" and has_sys_trace: + # -- rocprof-sys-sample ------------------------------------------------ + if tool == "rocprof-sys-sample" and has_sys_trace: # --trace alone ~ rocprofv3 --sys-trace; drop if it adds nothing new extra_flags = [f for f in flags if f != "--trace"] meaningful_args = [ @@ -179,7 +179,7 @@ def _filter_rec_commands( ] if not extra_flags and not meaningful_args: continue # equivalent to already-collected sys-trace data - # Has meaningful extra flags (e.g. --trace-gpu-memory) -> keep as-is + # Has meaningful extra flags (e.g. --device) -> keep as-is filtered.append(cmd) continue @@ -399,11 +399,11 @@ def generate_recommendations( "full_command": "rocprofv3 --sys-trace --pmc GRBM_GUI_ACTIVE GRBM_COUNT -d ./utilization_output -o profile -- ./app", }, { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "System-level timeline: identify host/GPU idle gaps and synchronization stalls", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -569,13 +569,11 @@ def generate_recommendations( "full_command": "rocprofv3 --sys-trace --hsa-trace -d ./memcpy_output -o profile -- ./app", }, { - "tool": "rocprof-sys", - "description": "Detailed memory transfer timeline with PCIe bandwidth and overlap analysis", - "flags": [], - "args": [ - {"name": "--trace-gpu-memory", "value": None}, - ], - "full_command": "rocprof-sys --trace-gpu-memory -- ./app", + "tool": "rocprof-sys-sample", + "description": "System-level trace with device metrics for memory transfer overlap analysis", + "flags": ["--trace", "--device"], + "args": [], + "full_command": "rocprof-sys-sample --trace --device -- ./app", }, ], } @@ -638,11 +636,11 @@ def generate_recommendations( "full_command": "rocprofv3 --hip-api-trace --hsa-trace -d ./api_output -o profile -- ./app", }, { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "System-level API call frequency and per-call latency breakdown", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -793,11 +791,11 @@ def generate_recommendations( "full_command": "rocprofv3 --sys-trace -d ./launch_output -o profile -- ./app", }, { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "Visualize kernel launch timeline and inter-launch gaps in a Perfetto trace", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, ], } @@ -947,11 +945,11 @@ def generate_recommendations( "full_command": "rocprofv3 --sys-trace --pmc GRBM_COUNT GRBM_GUI_ACTIVE SQ_WAVES -d ./counters_output -o profile -- ./app", }, { - "tool": "rocprof-sys", + "tool": "rocprof-sys-sample", "description": "Full system trace for comprehensive performance timeline", "flags": ["--trace"], "args": [], - "full_command": "rocprof-sys --trace -- ./app", + "full_command": "rocprof-sys-sample --trace -- ./app", }, { "tool": "rocprof-compute", diff --git a/experimental/python/rocinsight/tests/test_analyze.py b/experimental/python/rocinsight/tests/test_analyze.py index 3f7b43db69da..5be65949f56e 100644 --- a/experimental/python/rocinsight/tests/test_analyze.py +++ b/experimental/python/rocinsight/tests/test_analyze.py @@ -436,7 +436,7 @@ def test_tier2_commands_use_valid_tools(): """Tier 2 recommendations include commands with valid tool names.""" from rocinsight.analyze import generate_recommendations - VALID_TOOLS = {"rocprofv3", "rocprof-sys", "rocprof-compute"} + VALID_TOOLS = {"rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"} recs = generate_recommendations( _empty_breakdown(), [], diff --git a/experimental/python/rocinsight/tests/test_analyze_schema.py b/experimental/python/rocinsight/tests/test_analyze_schema.py index d2b095d52cc2..4574916dc8b4 100644 --- a/experimental/python/rocinsight/tests/test_analyze_schema.py +++ b/experimental/python/rocinsight/tests/test_analyze_schema.py @@ -91,7 +91,7 @@ def files(package): "errors", ] -COMMAND_TOOLS = {"rocprofv3", "rocprof-sys", "rocprof-compute"} +COMMAND_TOOLS = {"rocprofv3", "rocprof-sys-sample", "rocprof-sys-instrument", "rocprof-sys-run", "rocprof-compute"} def _load_schema(): From 7e9f2e90f3d708ee0f04b463a047a206d73ab9f6 Mon Sep 17 00:00:00 2001 From: Ammar ELWazir Date: Mon, 13 Apr 2026 12:15:25 -0500 Subject: [PATCH 2/4] docs(rocm21553/pr4): add v0.4.0 entry to SCHEMA_CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents all schema changes from ROCM-21553: - New confidence field on recommendations - New kernel_resources, api_breakdown, roctx_regions top-level fields - Changed recommendation categories (Compute Bottleneck → context-aware) - Data-driven estimated_impact values - Tool command corrections (rocprof-sys-sample, --trace --device) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ai_analysis/docs/SCHEMA_CHANGELOG.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md index e6112c598aa6..c44b12890ad5 100644 --- a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md +++ b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md @@ -78,6 +78,42 @@ schema_path = pkg_resources.files("rocinsight.ai_analysis") / "docs" / "analysis --- +## v0.4.0 — 2026-04-13 + +**ROCM-21553: Proposed improvements after first tests.** + +**New optional fields in recommendation objects:** +- `confidence` (`number | null`, 0.0–1.0): Analysis confidence level for the + recommendation. Higher values indicate more data-driven confidence. + +**New optional top-level fields:** +- `kernel_resources`: Per-kernel VGPR/SGPR/LDS/scratch usage and theoretical + occupancy (architecture-specific). Includes `arch`, `arch_specs`, and + `kernels` array with occupancy sub-objects. +- `api_breakdown`: Per-HIP-API call duration breakdown with `api_calls` array, + `launch_overhead_ns`, and `total_api_ns`. +- `roctx_regions`: roctx marker region analysis with per-region kernel/memcpy + correlation and unranged activity breakdown. `null` when no markers present. + +**Changed recommendation categories:** +- `"Compute Bottleneck"` replaced by context-aware labels: + - `"Kernel Hotspot"` (Tier 1, no counters) + - `"Compute-Bound Kernel"` (Tier 2, GPU util > 90%) + - `"Mixed Bottleneck Kernel"` (Tier 2, GPU util 70–90%) + - `"Memory-Bound Kernel"` (Tier 2, GPU util < 70%) +- New categories: `"Warmup"`, `"Profiling Scope"`, `"GPU Utilization (Scoped)"` + +**Changed `estimated_impact` values:** Now data-driven (reference actual metrics) +instead of generic percentage ranges. + +**Tool command fixes:** +- `rocprof-sys` (deprecated wrapper) → `rocprof-sys-sample` (recommended) +- `--trace-gpu-memory` (nonexistent) → `--trace --device` +- Schema tool enum expanded: `rocprof-sys-sample`, `rocprof-sys-instrument`, + `rocprof-sys-run` added alongside existing `rocprofv3` and `rocprof-compute` + +--- + ## v0.3.2 — 2026-03-25 **No schema changes.** Branding, output routing, and LLM hardening only. From 4db05cc12d0421c5eabb1d569164426fc9844aa9 Mon Sep 17 00:00:00 2001 From: Ammar ELWazir Date: Mon, 13 Apr 2026 14:11:04 -0500 Subject: [PATCH 3/4] fix: update stale --hip-api-trace in full_command to --hip-trace The structured flags were correctly updated to --hip-trace in PR4, but the full_command string in the API overhead recommendation was missed. This caused a copy-paste-ready command to still contain the invalid flag. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../python/rocinsight/rocinsight/analysis/recommendations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/python/rocinsight/rocinsight/analysis/recommendations.py b/experimental/python/rocinsight/rocinsight/analysis/recommendations.py index b53745885927..95a26a11c385 100644 --- a/experimental/python/rocinsight/rocinsight/analysis/recommendations.py +++ b/experimental/python/rocinsight/rocinsight/analysis/recommendations.py @@ -633,7 +633,7 @@ def generate_recommendations( {"name": "-d", "value": "./api_output"}, {"name": "-o", "value": "profile"}, ], - "full_command": "rocprofv3 --hip-api-trace --hsa-trace -d ./api_output -o profile -- ./app", + "full_command": "rocprofv3 --hip-trace --hsa-trace -d ./api_output -o profile -- ./app", }, { "tool": "rocprof-sys-sample", From e0ac9a31a155eaca7d9a342e9226eaa2310ee08e Mon Sep 17 00:00:00 2001 From: Ammar ELWazir Date: Mon, 13 Apr 2026 15:38:07 -0500 Subject: [PATCH 4/4] fix: bump schema_version to 0.5.0 for ROCM-21553 fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JSON schema version was not bumped when ROCM-21553 fields (kernel_resources, api_breakdown, roctx_regions, confidence) were present. v0.4.0 was already used by ATT Tier 3, so our changes are v0.5.0. Updated SCHEMA_CHANGELOG to match. Version progression: 0.1.0 (base) → 0.3.0 (TraceLens) → 0.4.0 (ATT) → 0.5.0 (ROCM-21553). Highest present version wins. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md | 2 +- .../rocinsight/rocinsight/formatters/json_fmt.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md index c44b12890ad5..f9a3c87fc9ef 100644 --- a/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md +++ b/experimental/python/rocinsight/rocinsight/ai_analysis/docs/SCHEMA_CHANGELOG.md @@ -78,7 +78,7 @@ schema_path = pkg_resources.files("rocinsight.ai_analysis") / "docs" / "analysis --- -## v0.4.0 — 2026-04-13 +## v0.5.0 — 2026-04-13 **ROCM-21553: Proposed improvements after first tests.** diff --git a/experimental/python/rocinsight/rocinsight/formatters/json_fmt.py b/experimental/python/rocinsight/rocinsight/formatters/json_fmt.py index a5fa9d89ea98..823b63ff83a9 100644 --- a/experimental/python/rocinsight/rocinsight/formatters/json_fmt.py +++ b/experimental/python/rocinsight/rocinsight/formatters/json_fmt.py @@ -166,17 +166,27 @@ def _format_as_json( doc["metadata"]["analysis_version"] = "0.4.0" doc["profiling_info"]["analysis_tier"] = 3 - # ROCM-21553: kernel resources and API overhead + # ROCM-21553: kernel resources, API overhead, roctx regions + _has_rocm21553_fields = False if kernel_resources and kernel_resources.get("kernels"): doc["kernel_resources"] = kernel_resources + _has_rocm21553_fields = True if api_overhead and api_overhead.get("has_api_data"): doc["api_breakdown"] = { "total_api_ns": api_overhead["total_api_ns"], "launch_overhead_ns": api_overhead["launch_overhead_ns"], "api_calls": api_overhead["api_calls"], } + _has_rocm21553_fields = True if roctx_regions and roctx_regions.get("has_markers"): doc["roctx_regions"] = roctx_regions + _has_rocm21553_fields = True + + # Recommendations always carry confidence now (added in ROCM-21553) + # Bump schema version to 0.5.0 when any ROCM-21553 field is present + if _has_rocm21553_fields: + doc["schema_version"] = "0.5.0" + doc["metadata"]["analysis_version"] = "0.5.0" return _json.dumps(doc, indent=2)