amd-vkale · Oct 6, 2026 · Oct 9, 2026 · Oct 9, 2026 · Oct 14, 2026 · Oct 15, 2026
@@ -10,6 +10,8 @@ Full documentation for ROCprofiler-SDK is available at [rocm.docs.amd.com/projec
 
 ### Resolved issues
 
+  - Fixed rocpd reporting a counter at N times its value under `rocprofv3 --replay-mode kernel` when N `--pmc` groups list it, for example a sanity counter such as `SQ_WAVES` in every group. Every pass of a replayed dispatch shares the dispatch's record, so the `counters_collection` view, and the counter CSV and Perfetto output that `rocpd convert` builds from it, summed the passes. Passes after the first are now tagged with `replay_pass` in `rocpd_pmc_event.extdata`, and the view averages a counter over the passes that collected it.
+
 ### Known issues
 
 ### Removed

@@ -139,6 +139,13 @@ Default ``rocpd``
 The default output format is ``rocpd``. Convert with ``rocpd convert`` as described in
 :ref:`using-rocpd-output-format`.
 
+The ``counters_collection`` view, which ``rocpd convert`` uses for counter CSV and Perfetto output,
+has one row per dispatch and counter. A counter repeated across groups is collected by every pass
+whose group lists it, and the view reports the mean of those passes, so a sanity counter such as
+``SQ_WAVES`` in every group reads the same as it does in a single pass. The individual passes stay
+in the database: a ``rocpd_pmc_event`` row from a pass after the first carries the pass index as
+``replay_pass`` in its ``extdata`` column, and a row without it belongs to pass 0.
+
 What is snapshotted
 ===================
 

@@ -1689,6 +1689,14 @@ write_rocpd(
                 auto        dispatch_id = info.dispatch_id;
 
                 auto evt_id = dispatch_evt_ids.at(dispatch_id);
+                // Every kernel-replay pass of a dispatch shares its event, so counters_collection
+                // needs replay_pass to average a counter collected by several passes instead of
+                // summing them. Pass 0 keeps the column default, so non-replay rows are unchanged;
+                // extdata is always bound so the insert batch keeps a single column list.
+                const auto extdata =
+                    (record.replay_pass == 0)
+                        ? std::string{"{}"}
+                        : fmt::format(R"({{"replay_pass":{}}})", record.replay_pass);
                 for(const auto& count : record.read())
                 {
                     get_insert_statement(db,
@@ -1698,6 +1706,7 @@ write_rocpd(
                                              insert_value("event_id", evt_id),
                                              insert_value("pmc_id", count.id.handle),
                                              insert_value("value", count.value),
+                                             insert_value("extdata", extdata),
                                          });
                 }
             }

@@ -162,7 +162,7 @@ using RocpdImportData with multiple databases, these views query across the atta
 | scratch_memory | Scratch-only allocations (subset of memory_allocate). | GPU scratch analysis. |
 | pmc_info | PMC counter metadata. | Discover available counters. |
 | pmc_events | PMC values per kernel (counter name, value, duration). | Counter time series. |
-| counters_collection | Aggregated PMC per dispatch/kernel/counter (e.g. SUM of value). | Counter summaries per kernel. |
+| counters_collection | Aggregated PMC per dispatch/kernel/counter: instances are summed, kernel-replay passes are averaged. | Counter summaries per kernel. |
 | events_args | Events with argument rows (category, arg name/value). | API or event arguments. |
 | stream_args | Stream arguments with stream description. | Stream-related API analysis. |
 

@@ -784,7 +784,12 @@ SELECT
     PMC_I.expression,
     PMC_I.value_type,
     PMC_I.id AS counter_id,
-    SUM(PMC_E.value) AS value,
+    -- Kernel replay records every pass of a dispatch under the same event and tags passes after
+    -- the first with extdata.replay_pass. Sum the instances of each pass, then average over the
+    -- passes that collected the counter instead of adding the passes together.
+    SUM(PMC_E.value) / COUNT(
+        DISTINCT COALESCE(json_extract(PMC_E.extdata, '$.replay_pass'), 0)
+    ) AS value,
     K.start,
     K.end,
     PMC_I.is_constant,

@@ -59,3 +59,6 @@ rocprofiler-sdk-rocpd:
 # Version 3.0.4
 # - Added rocpd_hip_event table for HIP event barrier records (hipEventRecord, hipStreamWaitEvent)
 # - Added hip_events data view
+# - Fix (no table change): kernel-replay passes after the first tag their rocpd_pmc_event rows with
+#   replay_pass in extdata, and counters_collection averages a counter over the passes that
+#   collected it instead of summing them
@@ -74,6 +74,7 @@
 
 _PMC_NO_SPM_ID = 1
 _PMC_SPM_ID = 2
+_PMC_GRBM_COUNT_ID = 3
 _TRACK_SPM_ID = 1
 _SAMPLE_SPM_ID = 1
 
@@ -544,6 +545,56 @@ def tbl(name: str) -> str:
             ),
         )
 
+        # Kernel-replay dispatch counters. Dispatch 1 is replayed three times: SQ_WAVES has two
+        # instances in every pass, with per-pass sums 16, 18 and 20, and GRBM_COUNT is collected
+        # by pass 0 only. Passes after the first carry replay_pass in extdata, as rocprofv3
+        # writes them. Dispatch 2 has untagged SQ_WAVES rows, like a run without kernel replay.
+        conn.execute(
+            f"INSERT INTO {tbl('rocpd_info_pmc')} "
+            "(id, guid, nid, pid, agent_id, target_arch, name, symbol, description, "
+            "component, value_type, block, expression, is_constant, is_derived, "
+            "spm_support, extdata) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            (
+                _PMC_GRBM_COUNT_ID,
+                guid,
+                _NID,
+                _PID,
+                _AGENT_ID,
+                "GPU",
+                "GRBM_COUNT",
+                "GRBM_COUNT",
+                "Tie High - Count Number of Clocks",
+                "rocm",
+                "ABS",
+                "GRBM",
+                "",
+                0,
+                0,
+                None,
+                "{}",
+            ),
+        )
+
+        def replay_pass(index):
+            return "{}" if index == 0 else f'{{"replay_pass":{index}}}'
+
+        conn.executemany(
+            f"INSERT INTO {tbl('rocpd_pmc_event')} "
+            "(id, guid, event_id, pmc_id, value, extdata) VALUES (?,?,?,?,?,?)",
+            [
+                (3, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 8.0, replay_pass(0)),
+                (4, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 8.0, replay_pass(0)),
+                (5, guid, _EVT_KERNEL_1, _PMC_GRBM_COUNT_ID, 1000.0, replay_pass(0)),
+                (6, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 9.0, replay_pass(1)),
+                (7, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 9.0, replay_pass(1)),
+                (8, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 10.0, replay_pass(2)),
+                (9, guid, _EVT_KERNEL_1, _PMC_NO_SPM_ID, 10.0, replay_pass(2)),
+                (10, guid, _EVT_KERNEL_2, _PMC_NO_SPM_ID, 4.0, "{}"),
+                (11, guid, _EVT_KERNEL_2, _PMC_NO_SPM_ID, 4.0, "{}"),
+            ],
+        )
+
 
 # ---------------------------------------------------------------------------
 # Entry point

@@ -38,6 +38,8 @@
   * CSV  — graph_exec_id / graph_node_id columns absent for pre-latest schemas;
            out_graph_launch_trace.csv not created for pre-latest schemas.
   * CSV  — above columns / file ARE present for the latest schema.
+  * CSV  — counters_collection averages a counter over kernel-replay passes for the
+           latest schema instead of summing them.
   * Perfetto — .pftrace exists and is non-empty for every schema.
   * OTF2 — "HIP Graph Launch" locations absent for pre-latest schemas;
             present for the latest schema (requires the ``otf2`` package).
@@ -254,6 +256,29 @@ def test_csv_for_schema_3_0_4_changes_present(output_root, latest_schema):
         ), f"Column '{expected}' missing from hip_event CSV for schema {latest_schema}"
 
 
+def test_csv_counter_collection_averages_replay_passes(output_root, latest_schema):
+    """counters_collection must average a counter over kernel-replay passes, not sum them.
+
+    make_db.py gives dispatch 1 SQ_WAVES in three replay passes with per-pass sums 16, 18
+    and 20, plus GRBM_COUNT in pass 0 only, and gives dispatch 2 untagged SQ_WAVES rows
+    that sum to 8.
+    """
+    counter_csv = output_root / latest_schema / "csv" / "out_counter_collection_trace.csv"
+    assert (
+        counter_csv.exists()
+    ), f"out_counter_collection_trace.csv not found for schema {latest_schema}: {counter_csv}"
+    with open(counter_csv) as fh:
+        values = {
+            (int(row["Dispatch_Id"]), row["Counter_Name"]): float(row["Counter_Value"])
+            for row in csv_mod.DictReader(fh)
+        }
+    assert values == {
+        (1, "SQ_WAVES"): 18.0,
+        (1, "GRBM_COUNT"): 1000.0,
+        (2, "SQ_WAVES"): 8.0,
+    }, f"unexpected counter_collection values for schema {latest_schema}: {values}"
+
+
 # ---------------------------------------------------------------------------
 # Perfetto tests — old schemas
 # ---------------------------------------------------------------------------

@@ -55,8 +55,8 @@ add_test(
         --pmc SQ_WAVES SQ_INSTS_VALU GRBM_GUI_ACTIVE --pmc SQ_WAVES SQ_INSTS_VALU
         SQ_INSTS_SALU --pmc SQ_WAVES SQ_INSTS_VALU SQ_INSTS_SMEM --pmc SQ_WAVES
         SQ_INSTS_VALU SQ_INSTS_LDS --replay-mode kernel --kernel-replay-beta-enabled
-        --output-format csv json -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay -o out --
-        $<TARGET_FILE:kernel-replay> 1048576 1)
+        --output-format csv json rocpd -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay -o
+        out -- $<TARGET_FILE:kernel-replay> 1048576 1)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-generate
@@ -74,14 +74,16 @@ set_tests_properties(
                rocprofv3-test-kernel-replay-data)
 
 # Validate replay structure: each dispatch replayed REPLAY_PASSES times, the shared sanity
-# counters (SQ_WAVES, SQ_INSTS_VALU) are constant across a dispatch's passes, and the
-# kernels are present.
+# counters (SQ_WAVES, SQ_INSTS_VALU) are constant across a dispatch's passes, the kernels
+# are present, and rocpd's counters_collection averages the shared counters over the
+# passes rather than summing them.
 add_test(
     NAME rocprofv3-test-kernel-replay-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
         ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay/out_results.json --passes
-        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU)
+        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU --rocpd-input
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay/out_results.db)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-validate
@@ -107,8 +109,9 @@ add_test(
         --pmc SQ_WAVES SQ_INSTS_VALU GRBM_GUI_ACTIVE --pmc SQ_WAVES SQ_INSTS_VALU
         SQ_INSTS_SALU --pmc SQ_WAVES SQ_INSTS_VALU SQ_INSTS_SMEM --pmc SQ_WAVES
         SQ_INSTS_VALU SQ_INSTS_LDS --replay-mode kernel --kernel-replay-beta-enabled
-        --output-format csv json -d ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval -o
-        out -- $<TARGET_FILE:kernel-replay> 1048576 1)
+        --output-format csv json rocpd -d
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval -o out --
+        $<TARGET_FILE:kernel-replay> 1048576 1)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-interval-generate
@@ -130,7 +133,8 @@ add_test(
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
         ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval/out_results.json --passes
-        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU)
+        ${REPLAY_PASSES} --common-counters SQ_WAVES SQ_INSTS_VALU --rocpd-input
+        ${CMAKE_CURRENT_BINARY_DIR}/kernel-replay-interval/out_results.db)
 
 set_tests_properties(
     rocprofv3-test-kernel-replay-interval-validate

@@ -22,6 +22,7 @@
 
 import json
 import os
+import sqlite3
 
 import pytest
 
@@ -47,6 +48,12 @@ def pytest_addoption(parser):
         default=["SQ_WAVES", "SQ_INSTS_VALU"],
         help="counters shared by every --pmc group; must be constant across a kernel's passes",
     )
+    parser.addoption(
+        "--rocpd-input",
+        action="store",
+        default=None,
+        help="rocpd database written by the same run as --json-input",
+    )
 
 
 @pytest.fixture
@@ -57,6 +64,17 @@ def json_data(request):
         return json.load(f)
 
 
+@pytest.fixture
+def rocpd_db(request):
+    path = request.config.getoption("--rocpd-input")
+    if path is None:
+        pytest.skip("no --rocpd-input given")
+    assert os.path.isfile(path), f"missing rocpd input: {path}"
+    connection = sqlite3.connect(path)
+    yield connection
+    connection.close()
+
+
 @pytest.fixture
 def expected_passes(request):
     return request.config.getoption("--passes")

@@ -37,6 +37,8 @@
 #   5. dispatch_id identity: a replayed dispatch keeps ONE dispatch_id across its passes (only the
 #      replay_pass index advances), and dispatch_id increments sequentially across dispatches -- one
 #      id minted per logical dispatch, reused by every pass (never one id per pass).
+#   6. With --rocpd-input, the rocpd counters_collection view reports each (dispatch, counter) as
+#      the mean of the passes that collected it, so the common counters are not multiplied by N.
 # (The app verifies its own results in the generate step, guarding the restored data itself.)
 
 import collections
@@ -391,6 +393,39 @@ def test_common_counters_constant_across_passes(json_data, common_counters):
             )
 
 
+def test_rocpd_counters_collection_averages_passes(json_data, rocpd_db):
+    # rocpd stores every pass of a replayed dispatch under the dispatch's one event, so the
+    # counters_collection view must average a counter over the passes that collected it. Summing
+    # them reported each common counter, which every group lists, at N times its value.
+    per_pass = {}
+    for dispatch_id, entry in _records_by_dispatch(_sdk(json_data)).items():
+        for batch in entry["passes"].values():
+            for counter, value in batch.items():
+                per_pass.setdefault((dispatch_id, counter), []).append(value)
+    assert any(
+        len(values) > 1 for values in per_pass.values()
+    ), "no counter was collected by more than one pass, so the run cannot show summed passes"
+
+    actual = {
+        (int(dispatch_id), counter): float(value)
+        for dispatch_id, counter, value in rocpd_db.execute(
+            "SELECT dispatch_id, counter_name, value FROM counters_collection"
+        )
+    }
+    assert sorted(actual) == sorted(per_pass), (
+        "rocpd counters_collection and JSON disagree on the (dispatch, counter) pairs: "
+        f"only in rocpd {sorted(set(actual) - set(per_pass))}, "
+        f"only in JSON {sorted(set(per_pass) - set(actual))}"
+    )
+    for (dispatch_id, counter), values in sorted(per_pass.items()):
+        mean = sum(values) / len(values)
+        assert actual[(dispatch_id, counter)] == pytest.approx(mean, rel=1e-9), (
+            f"dispatch {dispatch_id} counter {counter}: rocpd counters_collection reports "
+            f"{actual[(dispatch_id, counter)]}, but its replay passes in JSON are {values} "
+            f"(mean {mean})"
+        )
+
+
 def test_each_pass_collects_distinct_batch(json_data, expected_passes, common_counters):
     # Metric 3: each pass collects exactly one unique (non-common) counter, and those per-pass
     # counters differ — proving real multi-pass batch rotation rather than one pass carrying