amd-vkale · Oct 12, 2026 · Oct 12, 2026
@@ -13,16 +13,17 @@ sitting and can be merged or dropped on its own. The table below lists what has
 | Samples, GPU | `samples/range_replay/` | a replayed range with `divergence_count == 0`, a tool that opts out, a multi-queue decline |
 | Integration, GPU | `tests/range-replay-local-context/` | every per-dispatch service sees the replayed dispatches and honors per-pass toggles |
 | Declines, GPU | `tests/range-replay-declines/` | a device-writing copy between a range's kernels, through `hipMemcpyAsync` (C1) or `hsa_amd_memory_async_batch_copy` (C2), declines with `MEMORY_COPY_IN_RANGE` while nothing traces memory copies; no pass runs and the application's result is intact |
+| Concurrency, GPU | `tests/range-replay-concurrency/` | another thread's HIP graph launches on the agent, timed into the replay window, wait for it to close: the range replays with `divergence_count == 0`, its result is intact, and every launch's increment survives (C8) |
 | Performance, GPU, nightly | `tests/range-replay-perf/` | pass-count scaling, and amortization of the window cost over dispatches |
 
 ## What is missing
 
 Most decline reasons are asserted only as bookkeeping: nothing on hardware drives an application
-into them and checks that the range is declined and the application is unaffected. Of the three
-bugs fixed in #10864, C1 and C2 now guard the first two:
+into them and checks that the range is declined and the application is unaffected. The three bugs
+fixed in #10864 now each have a regression test:
 
-- device-writing copies went unseen when nothing traced memory copies;
-- `hsa_amd_memory_async_batch_copy` bypassed the decline;
+- device-writing copies went unseen when nothing traced memory copies (C1);
+- `hsa_amd_memory_async_batch_copy` bypassed the decline (C2);
 - a graph launched from another thread during a replay window could corrupt application data
   (C8).
 

@@ -100,6 +100,7 @@ add_subdirectory(kernel-replay-perf)
 add_subdirectory(range-replay-perf)
 add_subdirectory(range-replay-local-context)
 add_subdirectory(range-replay-declines)
+add_subdirectory(range-replay-concurrency)
 add_subdirectory(queue-hooks-perf)
 add_subdirectory(openmp-tools)
 add_subdirectory(openmp-tools-coexist)

@@ -0,0 +1,81 @@
+#
+# Range replay against another thread's work on the same agent. The replay window restores
+# the agent's whole tracked inventory before each pass and at its end, so work from another
+# thread must wait for the window instead of running inside it, or its writes are restored
+# away. The tool times a second thread's HIP graph launches into the window, and the
+# application checks that its range and every launch left the values its own run computed.
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-range-replay-concurrency LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+find_package(rocprofiler-sdk REQUIRED COMPONENTS headers shared-library)
+find_package(Threads REQUIRED)
+
+add_library(range-replay-concurrency-client SHARED)
+target_sources(range-replay-concurrency-client PRIVATE client.cpp range.hpp)
+target_include_directories(range-replay-concurrency-client
+                           PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(
+    range-replay-concurrency-client PRIVATE rocprofiler-sdk::rocprofiler-sdk
+                                            rocprofiler-sdk::tests-build-flags)
+
+# The application resolves the range replay entry points and the tool's rr_coord_get
+# handshake with dlsym instead of linking either: linking rocprofiler-sdk into a HIP
+# executable makes HIP report hipErrorInvalidDeviceFunction on the first launch on gfx942,
+# the same reason the range replay samples do it that way.
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+add_executable(range-replay-concurrency-testapp)
+target_sources(range-replay-concurrency-testapp PRIVATE main.cpp range.hpp)
+target_include_directories(range-replay-concurrency-testapp
+                           PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(
+    range-replay-concurrency-testapp
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk-headers rocprofiler-sdk::tests-build-flags
+            Threads::Threads ${CMAKE_DL_LIBS})
+add_dependencies(range-replay-concurrency-testapp range-replay-concurrency-client)
+
+# C8: a second thread launches graphs on the agent while the first thread's range is being
+# replayed. Divergence checking is on, so a launch that ran inside the window also shows
+# up as a region the final pass changed. The tool's verdict alone would let a run pass
+# while the application reported a wrong buffer, so the application's failure line fails
+# the test too.
+rocprofiler_add_integration_execute_test(
+    test-range-replay-concurrent-graph-launch
+    COMMAND $<TARGET_FILE:range-replay-concurrency-testapp>
+    DEPENDS range-replay-concurrency-client
+    TIMEOUT 180
+    LABELS "integration-tests;range-replay"
+    PRELOAD "$<TARGET_FILE:range-replay-concurrency-client>"
+    ENVIRONMENT "ROCPROF_RANGE_REPLAY_VERIFY=1"
+    PASS_REGULAR_EXPRESSION "\\[rr-concurrency\\] PASS"
+    FAIL_REGULAR_EXPRESSION
+        "\\[rr-concurrency\\] FAIL|\\[rr-concurrency\\] error|\\[app\\] FAIL|${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+    SKIP_REGULAR_EXPRESSION "\\[app\\] SKIP")
@@ -0,0 +1,222 @@
+// MIT License
+//
+// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+/**
+ * @file tests/range-replay-concurrency/client.cpp
+ *
+ * @brief LD_PRELOAD tool that replays the application's range and times a second thread's graph
+ * launches into the replay window.
+ *
+ * The window restores the agent's whole tracked inventory before every pass and once more at the
+ * end, so the second thread's buffers are rewound with everything else. Its graphs must therefore
+ * wait for the window to close rather than run inside it: a launch that ran inside the window would
+ * have its writes restored away, and its buffer would end short of the count the application
+ * expects. The handshake makes that overlap happen on every run:
+ *   - PASS enter, pass 1: the window is open (the range was taken off the thread, so a launch can
+ * no longer decline it as a concurrent dispatch); tell the graph thread to start.
+ *   - PASS enter, last pass: wait until the graph thread has begun launching, then give the launch
+ *     time to reach the queue while the window is still held.
+ *   - CLOSE: tell the graph thread the range is over, whether or not it was replayed.
+ */
+
+#include "range.hpp"
+
+#include <rocprofiler-sdk/callback_tracing.h>
+#include <rocprofiler-sdk/experimental/range_replay.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+
+#define RC(call)                                                                                   \
+    do                                                                                             \
+    {                                                                                              \
+        rocprofiler_status_t _s = (call);                                                          \
+        if(_s != ROCPROFILER_STATUS_SUCCESS)                                                       \
+        {                                                                                          \
+            fprintf(stderr,                                                                        \
+                    "[rr-concurrency] error '%s' @%d: %s\n",                                       \
+                    #call,                                                                         \
+                    __LINE__,                                                                      \
+                    rocprofiler_get_status_string(_s));                                            \
+            std::abort();                                                                          \
+        }                                                                                          \
+    } while(0)
+
+// Exported with default visibility on purpose: the tests build compiles with hidden visibility,
+// but the application resolves this symbol with dlsym(RTLD_DEFAULT, "rr_coord_get").
+extern "C" __attribute__((visibility("default"))) rr_coord_t*
+rr_coord_get()
+{
+    static rr_coord_t coord{};
+    return &coord;
+}
+
+namespace
+{
+rocprofiler_context_id_t g_ctx{0};
+
+std::atomic<int>      g_configs{0};
+std::atomic<int>      g_passes{0};
+std::atomic<int>      g_closes{0};
+std::atomic<int>      g_status{ROCPROFILER_RANGE_REPLAY_STATUS_LAST};
+std::atomic<uint64_t> g_dispatches{0};
+std::atomic<uint64_t> g_divergence{0};
+std::atomic<int>      g_overlapped{0};
+
+uint64_t pass_count(uint64_t, rocprofiler_user_data_t) { return kPasses; }
+
+void
+await_first_launch()
+{
+    auto*      coord    = rr_coord_get();
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{10};
+    while(coord->launches_started.load() == 0 && std::chrono::steady_clock::now() < deadline)
+        std::this_thread::sleep_for(std::chrono::milliseconds{1});
+
+    if(coord->launches_started.load() == 0) return;
+
+    // The flag is raised just before the first hipGraphLaunch; this is the time that launch has to
+    // reach the queue and wait on the agent's replay lock while the window still holds it.
+    std::this_thread::sleep_for(std::chrono::milliseconds{100});
+    g_overlapped.store(1);
+}
+
+void
+range_replay_cb(rocprofiler_callback_tracing_record_t record, rocprofiler_user_data_t*, void*)
+{
+    if(record.kind != ROCPROFILER_CALLBACK_TRACING_RANGE_REPLAY) return;
+    if(record.phase != ROCPROFILER_CALLBACK_PHASE_ENTER) return;
+
+    auto* data = static_cast<rocprofiler_callback_tracing_range_replay_data_t*>(record.payload);
+    if(data->range_id != kRangeId) return;
+
+    switch(record.operation)
+    {
+        case ROCPROFILER_RANGE_REPLAY_CONFIG:
+            data->pass_count_cb = pass_count;
+            g_configs.fetch_add(1);
+            break;
+        case ROCPROFILER_RANGE_REPLAY_PASS:
+            g_passes.fetch_add(1);
+            if(data->current_pass == 1) rr_coord_get()->window_open.store(1);
+            if(data->current_pass == kPasses - 1) await_first_launch();
+            break;
+        case ROCPROFILER_RANGE_REPLAY_CLOSE:
+            g_status.store(data->status);
+            g_dispatches.store(data->dispatch_count);
+            g_divergence.store(data->divergence_count);
+            g_closes.fetch_add(1);
+            rr_coord_get()->range_closed.store(1);
+            break;
+        default: break;
+    }
+}
+
+int
+tool_init(rocprofiler_client_finalize_t, void*)
+{
+    RC(rocprofiler_create_context(&g_ctx));
+    RC(rocprofiler_configure_callback_tracing_service(
+        g_ctx, ROCPROFILER_CALLBACK_TRACING_RANGE_REPLAY, nullptr, 0, range_replay_cb, nullptr));
+    RC(rocprofiler_start_context(g_ctx));
+    return 0;
+}
+
+void
+tool_fini(void*)
+{
+    if(const char* reason = std::getenv(kSkipEnv); reason != nullptr)
+    {
+        fprintf(stderr, "[rr-concurrency] SKIP: %s\n", reason);
+        return;
+    }
+
+    fprintf(stderr,
+            "[rr-concurrency] configs=%d passes=%d closes=%d status=%d dispatches=%lu "
+            "divergence=%lu overlapped=%d\n",
+            g_configs.load(),
+            g_passes.load(),
+            g_closes.load(),
+            g_status.load(),
+            static_cast<unsigned long>(g_dispatches.load()),
+            static_cast<unsigned long>(g_divergence.load()),
+            g_overlapped.load());
+
+    auto ok = true;
+    if(g_configs.load() != 1 || g_closes.load() != 1)
+    {
+        fprintf(stderr, "[rr-concurrency] FAIL: expected exactly one CONFIG and one CLOSE\n");
+        ok = false;
+    }
+    if(g_status.load() != static_cast<int>(ROCPROFILER_RANGE_REPLAY_STATUS_REPLAYED))
+    {
+        fprintf(stderr, "[rr-concurrency] FAIL: the range was not replayed\n");
+        ok = false;
+    }
+    if(g_passes.load() != static_cast<int>(kPasses - 1))
+    {
+        fprintf(stderr,
+                "[rr-concurrency] FAIL: expected %lu replayed passes\n",
+                static_cast<unsigned long>(kPasses - 1));
+        ok = false;
+    }
+    if(g_dispatches.load() != kRangeDispatches)
+    {
+        fprintf(stderr, "[rr-concurrency] FAIL: CLOSE reported the wrong dispatch count\n");
+        ok = false;
+    }
+    // A graph that ran inside the window would have written between the exit snapshot and the
+    // final pass's snapshot.
+    if(std::getenv("ROCPROF_RANGE_REPLAY_VERIFY") != nullptr && g_divergence.load() != 0)
+    {
+        fprintf(stderr,
+                "[rr-concurrency] FAIL: the final pass diverged from the application's run\n");
+        ok = false;
+    }
+    if(g_overlapped.load() == 0)
+    {
+        fprintf(stderr,
+                "[rr-concurrency] FAIL: the graph thread never started launching inside the replay "
+                "window, so nothing was tested\n");
+        ok = false;
+    }
+
+    fprintf(stderr, ok ? "[rr-concurrency] PASS\n" : "[rr-concurrency] FAIL\n");
+}
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t, const char*, uint32_t priority, rocprofiler_client_id_t* id)
+{
+    if(priority > 0) return nullptr;
+    id->name        = "rr-concurrency";
+    static auto cfg = rocprofiler_tool_configure_result_t{
+        sizeof(rocprofiler_tool_configure_result_t), &tool_init, &tool_fini, nullptr};
+    return &cfg;
+}