amd-vkale · Oct 6, 2026 · Oct 9, 2026 · Oct 9, 2026 · Oct 14, 2026 · Oct 15, 2026
@@ -10,6 +10,8 @@ Full documentation for ROCprofiler-SDK is available at [rocm.docs.amd.com/projec
 
 ### Resolved issues
 
+  - Fixed rocpd reporting a counter at N times its value under `rocprofv3 --replay-mode kernel` when N `--pmc` groups list it, for example a sanity counter such as `SQ_WAVES` in every group. Every pass of a replayed dispatch shares the dispatch's record, so the `counters_collection` view, and the counter CSV and Perfetto output that `rocpd convert` builds from it, summed the passes. Passes after the first are now tagged with `replay_pass` in `rocpd_pmc_event.extdata`, the view averages a counter over the passes that collected it, and the `pmc_events` view reports each row's pass in a new `replay_pass` column.
+
 ### Known issues
 
 ### Removed

@@ -139,6 +139,15 @@ Default ``rocpd``
 The default output format is ``rocpd``. Convert with ``rocpd convert`` as described in
 :ref:`using-rocpd-output-format`.
 
+The ``counters_collection`` view, which ``rocpd convert`` uses for counter CSV and Perfetto output,
+has one row per dispatch and counter. A counter repeated across groups is collected by every pass
+whose group lists it, and the view reports the mean of those passes, so a sanity counter such as
+``SQ_WAVES`` in every group reads the same as it does in a single pass. The individual passes stay
+in the database: a ``rocpd_pmc_event`` row from a pass after the first carries the pass index as
+``replay_pass`` in its ``extdata`` column, and a row without it belongs to pass 0. The
+``pmc_events`` view, which has one row per counter instance and pass, reports it as a
+``replay_pass`` column.
+
 What is snapshotted
 ===================
 

@@ -1689,6 +1689,14 @@ write_rocpd(
                 auto        dispatch_id = info.dispatch_id;
 
                 auto evt_id = dispatch_evt_ids.at(dispatch_id);
+                // Every kernel-replay pass of a dispatch shares its event, so counters_collection
+                // needs replay_pass to average a counter collected by several passes instead of
+                // summing them. Pass 0 keeps the column default, so non-replay rows are unchanged;
+                // extdata is always bound so the insert batch keeps a single column list.
+                const auto extdata =
+                    (record.replay_pass == 0)
+                        ? std::string{"{}"}
+                        : fmt::format(R"({{"replay_pass":{}}})", record.replay_pass);
                 for(const auto& count : record.read())
                 {
                     get_insert_statement(db,
@@ -1698,6 +1706,7 @@ write_rocpd(
                                              insert_value("event_id", evt_id),
                                              insert_value("pmc_id", count.id.handle),
                                              insert_value("value", count.value),
+                                             insert_value("extdata", extdata),
                                          });
                 }
             }

@@ -161,8 +161,8 @@ using RocpdImportData with multiple databases, these views query across the atta
 | memory_allocations | Alloc/free with type, level, agent, size, queue/stream. | Memory usage. |
 | scratch_memory | Scratch-only allocations (subset of memory_allocate). | GPU scratch analysis. |
 | pmc_info | PMC counter metadata. | Discover available counters. |
-| pmc_events | PMC values per kernel (counter name, value, duration). | Counter time series. |
-| counters_collection | Aggregated PMC per dispatch/kernel/counter (e.g. SUM of value). | Counter summaries per kernel. |
+| pmc_events | PMC values per kernel (counter name, value, duration, kernel-replay pass). | Counter time series. |
+| counters_collection | Aggregated PMC per dispatch/kernel/counter: instances are summed, kernel-replay passes are averaged. | Counter summaries per kernel. |
 | events_args | Events with argument rows (category, arg name/value). | API or event arguments. |
 | stream_args | Stream arguments with stream description. | Stream-related API analysis. |
 

@@ -400,7 +400,9 @@ SELECT
     K.end,
     (K.end - K.start) AS duration,
     PMC_I.name AS counter_name,
-    PMC_E.value AS counter_value
+    PMC_E.value AS counter_value,
+    -- Kernel-replay pass that collected the value. Rows without the tag are pass 0.
+    COALESCE(json_extract(PMC_E.extdata, '$.replay_pass'), 0) AS replay_pass
 FROM
     `rocpd_pmc_event` PMC_E
     INNER JOIN `rocpd_info_pmc` PMC_I ON PMC_I.id = PMC_E.pmc_id
@@ -784,7 +786,12 @@ SELECT
     PMC_I.expression,
     PMC_I.value_type,
     PMC_I.id AS counter_id,
-    SUM(PMC_E.value) AS value,
+    -- Kernel replay records every pass of a dispatch under the same event and tags passes after
+    -- the first with extdata.replay_pass. Sum the instances of each pass, then average over the
+    -- passes that collected the counter instead of adding the passes together.
+    SUM(PMC_E.value) / COUNT(
+        DISTINCT COALESCE(json_extract(PMC_E.extdata, '$.replay_pass'), 0)
+    ) AS value,
     K.start,
     K.end,
     PMC_I.is_constant,

@@ -59,3 +59,6 @@ rocprofiler-sdk-rocpd:
 # Version 3.0.4
 # - Added rocpd_hip_event table for HIP event barrier records (hipEventRecord, hipStreamWaitEvent)
 # - Added hip_events data view
+# - Fix (no table change): kernel-replay passes after the first tag their rocpd_pmc_event rows with
+#   replay_pass in extdata, counters_collection averages a counter over the passes that collected
+#   it instead of summing them, and pmc_events reports the pass in a replay_pass column
@@ -74,6 +74,8 @@
 
 _PMC_NO_SPM_ID = 1
 _PMC_SPM_ID = 2
+_PMC_GRBM_COUNT_ID = 3
+_PMC_SQ_INSTS_SALU_ID = 4
 _TRACK_SPM_ID = 1
 _SAMPLE_SPM_ID = 1
 
@@ -544,6 +546,82 @@ def tbl(name: str) -> str:
             ),
         )
 
+        # Kernel-replay dispatch counters. Dispatch 1 is replayed three times: SQ_WAVES has two
+        # instances in every pass, with per-pass sums 16, 18 and 20; GRBM_COUNT is collected by
+        # pass 0 only; SQ_INSTS_SALU by passes 1 and 2 only, with per-pass sums 5 and 6. Passes
+        # after the first carry replay_pass in extdata, as rocprofv3 writes them. Dispatch 2 has
+        # untagged SQ_WAVES rows, like a run without kernel replay.
+        conn.executemany(
+            f"INSERT INTO {tbl('rocpd_info_pmc')} "
+            "(id, guid, nid, pid, agent_id, target_arch, name, symbol, description, "
+            "component, value_type, block, expression, is_constant, is_derived, "
+            "spm_support, extdata) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            [
+                (
+                    _PMC_GRBM_COUNT_ID,
+                    guid,
+                    _NID,
+                    _PID,
+                    _AGENT_ID,
+                    "GPU",
+                    "GRBM_COUNT",
+                    "GRBM_COUNT",
+                    "Tie High - Count Number of Clocks",
+                    "rocm",
+                    "ABS",
+                    "GRBM",
+                    "",
+                    0,
+                    0,
+                    None,
+                    "{}",
+                ),
+                (
+                    _PMC_SQ_INSTS_SALU_ID,
+                    guid,
+                    _NID,
+                    _PID,
+                    _AGENT_ID,
+                    "GPU",
+                    "SQ_INSTS_SALU",
+                    "SQ_INSTS_SALU",
+                    "Number of SALU instructions issued",
+                    "rocm",
+                    "ABS",
+                    "SQ",
+                    "",
+                    0,
+                    0,
+                    None,
+                    "{}",
+                ),
+            ],
+        )
+
+        def replay_pass(index):
+            return "{}" if index == 0 else f'{{"replay_pass":{index}}}'
+
+        conn.executemany(
+            f"INSERT INTO {tbl('rocpd_pmc_event')} "
+            "(id, guid, event_id, pmc_id, value, extdata) VALUES (?,?,?,?,?,?)",
+            [
+                (3, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 8.0, replay_pass(0)),
+                (4, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 8.0, replay_pass(0)),
+                (5, guid, _EVT_KERNEL_1, _PMC_GRBM_COUNT_ID, 1000.0, replay_pass(0)),
+                (6, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 9.0, replay_pass(1)),
+                (7, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 9.0, replay_pass(1)),
+                (8, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 10.0, replay_pass(2)),
+                (9, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 10.0, replay_pass(2)),
+                (10, guid, _EVT_KERNEL_2, _PMC_NO_SPM_ID, 4.0, "{}"),
+                (11, guid, _EVT_KERNEL_2, _PMC_NO_SPM_ID, 4.0, "{}"),
+                (12, guid, _EVT_KERNEL_1, _PMC_SQ_INSTS_SALU_ID, 2.0, replay_pass(1)),
+                (13, guid, _EVT_KERNEL_1, _PMC_SQ_INSTS_SALU_ID, 3.0, replay_pass(1)),
+                (14, guid, _EVT_KERNEL_1, _PMC_SQ_INSTS_SALU_ID, 3.0, replay_pass(2)),
+                (15, guid, _EVT_KERNEL_1, _PMC_SQ_INSTS_SALU_ID, 3.0, replay_pass(2)),
+            ],
+        )
+
 
 # ---------------------------------------------------------------------------
 # Entry point

@@ -38,6 +38,9 @@
   * CSV  — graph_exec_id / graph_node_id columns absent for pre-latest schemas;
            out_graph_launch_trace.csv not created for pre-latest schemas.
   * CSV  — above columns / file ARE present for the latest schema.
+  * CSV  — counters_collection averages a counter over kernel-replay passes for the
+           latest schema instead of summing them, and pmc_events reports each pass.
+  * Perfetto — counter values for the latest schema are those pass means.
   * Perfetto — .pftrace exists and is non-empty for every schema.
   * OTF2 — "HIP Graph Launch" locations absent for pre-latest schemas;
             present for the latest schema (requires the ``otf2`` package).
@@ -47,6 +50,7 @@
 """
 
 import csv as csv_mod
+import sqlite3
 import sys
 from pathlib import Path
 
@@ -254,6 +258,58 @@ def test_csv_for_schema_3_0_4_changes_present(output_root, latest_schema):
         ), f"Column '{expected}' missing from hip_event CSV for schema {latest_schema}"
 
 
+def test_csv_counter_collection_averages_replay_passes(output_root, latest_schema):
+    """counters_collection must average a counter over kernel-replay passes, not sum them.
+
+    make_db.py gives dispatch 1 SQ_WAVES in three replay passes with per-pass sums 16, 18
+    and 20, GRBM_COUNT in pass 0 only, and SQ_INSTS_SALU in passes 1 and 2 only with sums
+    5 and 6, and gives dispatch 2 untagged SQ_WAVES rows that sum to 8.
+    """
+    counter_csv = output_root / latest_schema / "csv" / "out_counter_collection_trace.csv"
+    assert (
+        counter_csv.exists()
+    ), f"out_counter_collection_trace.csv not found for schema {latest_schema}: {counter_csv}"
+    with open(counter_csv) as fh:
+        values = {
+            (int(row["Dispatch_Id"]), row["Counter_Name"]): float(row["Counter_Value"])
+            for row in csv_mod.DictReader(fh)
+        }
+    assert values == {
+        (1, "SQ_WAVES"): 18.0,
+        (1, "GRBM_COUNT"): 1000.0,
+        (1, "SQ_INSTS_SALU"): 5.5,
+        (2, "SQ_WAVES"): 8.0,
+    }, f"unexpected counter_collection values for schema {latest_schema}: {values}"
+
+
+def test_pmc_events_report_replay_pass(output_root, latest_schema):
+    """pmc_events must report each row's kernel-replay pass, and pass 0 for untagged rows.
+
+    SQ_WAVES of dispatch 1 is left out because the view also lists that dispatch's SPM
+    samples, which have no pass.
+    """
+    db_path = (
+        output_root
+        / latest_schema
+        / "db"
+        / f"schema_{latest_schema.replace('.', '_')}.db"
+    )
+    assert db_path.exists(), f"database not found for schema {latest_schema}: {db_path}"
+    with sqlite3.connect(db_path) as conn:
+        rows = conn.execute(
+            "SELECT dispatch_id, counter_name, replay_pass, SUM(counter_value) "
+            "FROM pmc_events "
+            "WHERE counter_name IN ('GRBM_COUNT', 'SQ_INSTS_SALU') OR dispatch_id = 2 "
+            "GROUP BY dispatch_id, counter_name, replay_pass"
+        ).fetchall()
+    assert sorted(rows) == [
+        (1, "GRBM_COUNT", 0, 1000.0),
+        (1, "SQ_INSTS_SALU", 1, 5.0),
+        (1, "SQ_INSTS_SALU", 2, 6.0),
+        (2, "SQ_WAVES", 0, 8.0),
+    ], f"unexpected pmc_events passes for schema {latest_schema}: {rows}"
+
+
 # ---------------------------------------------------------------------------
 # Perfetto tests — old schemas
 # ---------------------------------------------------------------------------
@@ -278,6 +334,33 @@ def test_pftrace_completes_without_error_latest(output_root, latest_schema):
     assert pftrace.stat().st_size > 0, f".pftrace is empty for schema {latest_schema}"
 
 
+def test_pftrace_counter_values_average_replay_passes(output_root, latest_schema):
+    """Perfetto counter values must be the kernel-replay pass means, not the pass sums.
+
+    The writer annotates every kernel slice with each counter's total over all dispatches,
+    so only counters that make_db.py collects on a single dispatch are checked: GRBM_COUNT
+    (pass 0 only) and SQ_INSTS_SALU (passes 1 and 2, with sums 5 and 6).
+    """
+    pytest.importorskip("perfetto", reason="perfetto package not installed")
+    from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
+
+    pftrace = output_root / latest_schema / "pftrace" / "out_results.pftrace"
+    assert pftrace.exists(), f".pftrace not found for schema {latest_schema}: {pftrace}"
+    with PerfettoReader(str(pftrace)) as reader:
+        rows = reader.query_tp(
+            "SELECT args.key AS key, COALESCE(args.real_value, args.int_value) AS value "
+            "FROM slice JOIN args USING (arg_set_id) "
+            "WHERE args.key IN ('debug.GRBM_COUNT', 'debug.SQ_INSTS_SALU')"
+        )
+    values = {}
+    for key, value in zip(rows["key"], rows["value"]):
+        values.setdefault(key, set()).add(float(value))
+    assert values == {
+        "debug.GRBM_COUNT": {1000.0},
+        "debug.SQ_INSTS_SALU": {5.5},
+    }, f"unexpected Perfetto counter values for schema {latest_schema}: {values}"
+
+
 # ---------------------------------------------------------------------------
 # OTF2 tests — old schemas
 # ---------------------------------------------------------------------------

@@ -55,8 +55,8 @@ add_test(
         --pmc SQ_WAVES SQ_INSTS_VALU GRBM_GUI_ACTIVE --pmc SQ_WAVES SQ_INSTS_VALU
         SQ_INSTS_SALU --pmc SQ_WAVES SQ_INSTS_VALU SQ_INSTS_SMEM --pmc SQ_WAVES
         SQ_INSTS_VALU SQ_INSTS_LDS --replay-mode kernel --kernel-replay-beta-enabled
-        --output-format csv json -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay -o out --
-        $<TARGET_FILE:kernel-replay> 1048576 1)
+        --output-format csv json rocpd -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay -o
+        out -- $<TARGET_FILE:kernel-replay> 1048576 1)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-generate
@@ -74,14 +74,16 @@ set_tests_properties(
                rocprofv3-test-kernel-replay-data)
 
 # Validate replay structure: each dispatch replayed REPLAY_PASSES times, the shared sanity
-# counters (SQ_WAVES, SQ_INSTS_VALU) are constant across a dispatch's passes, and the
-# kernels are present.
+# counters (SQ_WAVES, SQ_INSTS_VALU) are constant across a dispatch's passes, the kernels
+# are present, and rocpd's counters_collection averages the shared counters over the
+# passes rather than summing them.
 add_test(
     NAME rocprofv3-test-kernel-replay-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
         ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay/out_results.json --passes
-        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU)
+        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU --rocpd-input
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay/out_results.db)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-validate
@@ -107,8 +109,9 @@ add_test(
         --pmc SQ_WAVES SQ_INSTS_VALU GRBM_GUI_ACTIVE --pmc SQ_WAVES SQ_INSTS_VALU
         SQ_INSTS_SALU --pmc SQ_WAVES SQ_INSTS_VALU SQ_INSTS_SMEM --pmc SQ_WAVES
         SQ_INSTS_VALU SQ_INSTS_LDS --replay-mode kernel --kernel-replay-beta-enabled
-        --output-format csv json -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval -o
-        out -- $<TARGET_FILE:kernel-replay> 1048576 1)
+        --output-format csv json rocpd -d
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval -o out --
+        $<TARGET_FILE:kernel-replay> 1048576 1)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-interval-generate
@@ -130,7 +133,8 @@ add_test(
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
         ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval/out_results.json --passes
-        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU)
+        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU --rocpd-input
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval/out_results.db)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-interval-validate

@@ -22,6 +22,7 @@
 
 import json
 import os
+import sqlite3
 
 import pytest
 
@@ -33,6 +34,12 @@ def pytest_addoption(parser):
         required=True,
         help="rocprofv3 results JSON produced by a --replay-mode kernel --kernel-replay-beta-enabled run",
     )
+    parser.addoption(
+        "--rocpd-input",
+        action="store",
+        default=None,
+        help="rocpd database written by the same run as --json-input",
+    )
     parser.addoption(
         "--passes",
         action="store",
@@ -57,6 +64,17 @@ def json_data(request):
         return json.load(f)
 
 
+@pytest.fixture
+def rocpd_db(request):
+    path = request.config.getoption("--rocpd-input")
+    if path is None:
+        pytest.skip("no --rocpd-input given")
+    assert os.path.isfile(path), f"missing rocpd input: {path}"
+    connection = sqlite3.connect(path)
+    yield connection
+    connection.close()
+
+
 @pytest.fixture
 def expected_passes(request):
     return request.config.getoption("--passes")