From 80ccdb062c07b4fc2c09d64b15c5a6e3b2874594 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Fri, 9 Oct 2026 10:26:55 +0200 Subject: [PATCH 01/12] hrr: snapshot pinned host memory kernels read, and restore it at replay --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 194 +++++++++++++++++++- projects/clr/rocclr/utils/flags.hpp | 5 + projects/hrr/include/hrr/hrr_api_args.h | 6 +- projects/hrr/playback/hip_playback.cpp | 49 ++++- projects/hrr/playback/hip_playback.h | 45 +++++ projects/hrr/playback/hrr_playback.cpp | 4 + projects/hrr/playback/hrr_reader.cpp | 5 +- projects/hrr/playback/hrr_reader.h | 3 +- projects/hrr/tools/gen_hrr_api_args.py | 6 +- 9 files changed, 306 insertions(+), 11 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 07e26718bec5..a1f10303464f 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -161,6 +161,121 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s return out_buf != nullptr && out_size > 0; } +// --------------------------------------------------------------------------- +// Pinned host snapshots +// +// A kernel can read pinned host memory the host filled with plain stores, which +// no HIP call records. Before each launch, the 8-byte-aligned words of its +// arguments are looked up in the tracked pinned allocations (hipHostMalloc, +// hipHostRegister). Each allocation hit is written whole as a +// blob, and named by a record in the launch event unless its hash equals that +// of the last record for it. Replay copies the blob back before the launch. +// See DESIGN.md, "Pinned host snapshots". +// --------------------------------------------------------------------------- + +namespace { +struct PinnedAlloc { + size_t size = 0; + hrr_cap::Hash128 last{}; // blob of the last record, {0,0} before the first +}; +struct HostSnap { + uint64_t base = 0, size = 0; + hrr_cap::Hash128 hash{}; + bool record = false; // contents differ from the last record +}; +std::mutex g_pinned_mu; // also held while a snapshot reads +std::map g_pinned; +std::atomic g_pinned_count{0}; +// The pinned allocations the launch this thread is making points into. +thread_local std::vector t_snaps; +} // namespace + +static void pinned_track(void* p, size_t size) { + if (!p || size == 0) return; + std::lock_guard lk(g_pinned_mu); + g_pinned[reinterpret_cast(p)] = PinnedAlloc{size, {}}; + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); +} + +// Called before the memory is released, so no snapshot reads it afterwards. +static void pinned_untrack(void* p) { + if (!p || g_pinned_count.load(std::memory_order_relaxed) == 0) return; + std::lock_guard lk(g_pinned_mu); + g_pinned.erase(reinterpret_cast(p)); + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); +} + +static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { + const size_t cap = static_cast(HIP_HRR_HOST_SNAPSHOT_MAX_MB) << 20; + for (size_t j = 0; bytes && j + sizeof(uint64_t) <= sz; j += sizeof(uint64_t)) { + uint64_t v; + std::memcpy(&v, bytes + j, sizeof(v)); + auto it = g_pinned.upper_bound(static_cast(v)); + if (it == g_pinned.begin()) continue; + --it; + if (v - it->first >= it->second.size || + std::any_of(t_snaps.begin(), t_snaps.end(), + [&](const HostSnap& s) { return s.base == it->first; })) + continue; + HostSnap s; + s.base = it->first; + s.size = it->second.size; + if (s.size > cap) { + hrr_cap::writer::note_unreplayable( + "pinned host snapshot", + "a pinned allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB was not recorded"); + } else { + s.hash = hrr_cap::writer::write_blob(reinterpret_cast(s.base), s.size); + PinnedAlloc& a = it->second; + s.record = (s.hash.lo || s.hash.hi) && (s.hash.lo != a.last.lo || s.hash.hi != a.last.hi); + if (s.record) a.last = s.hash; + } + t_snaps.push_back(s); + } +} + +// Takes the snapshot of one launch before the real call and drops it after, +// whether or not the launch was recorded. +struct PinnedSnapshot { + PinnedSnapshot(hipFunction_t f, void** params, void** extra, + const void* kbuf = nullptr, size_t ksz = 0) { + t_snaps.clear(); + if (!f || !HIP_HRR_HOST_SNAPSHOTS || g_pinned_count.load(std::memory_order_relaxed) == 0) + return; + if (!params && !kbuf && extra) parse_kernel_extra(extra, kbuf, ksz); + const amd::KernelSignature& sig = hip::asKernel(f)->signature(); + std::lock_guard lk(g_pinned_mu); + uint32_t param_idx = 0; + for (uint32_t i = 0; i < sig.numParametersAll(); i++) { + const auto& d = sig.at(i); + if (d.info_.hidden_) continue; + const uint8_t* bytes = nullptr; + if (kbuf && d.offset_ + d.size_ <= ksz) + bytes = static_cast(kbuf) + d.offset_; + else if (!kbuf && params) + bytes = static_cast(params[param_idx]); + param_idx++; + if (d.offset_ % sizeof(uint64_t) == 0) pinned_scan_locked(bytes, d.size_); + } + } + ~PinnedSnapshot() { t_snaps.clear(); } +}; + +// The kernel behind a host stub, looked up before the launch only when there +// is pinned memory to snapshot, without changing the application's last error. +static hipFunction_t pinned_stub_fn(const void* stub) { + if (!HIP_HRR_HOST_SNAPSHOTS || g_pinned_count.load(std::memory_order_relaxed) == 0 || + !g_real_table.hipGetFuncBySymbol_fn) + return nullptr; + const hipError_t saved_cmd = hip::tls.last_command_error_; + const hipError_t saved_err = hip::tls.last_error_; + hipFunction_t f = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn(&f, stub) != hipSuccess) f = nullptr; + hip::tls.last_command_error_ = saved_cmd; + hip::tls.last_error_ = saved_err; + return f; +} + // --------------------------------------------------------------------------- // Kernel launch event serialization // @@ -175,7 +290,7 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // u32[3] block // u32 shared_mem // u16 num_args -// u16 num_snapshots (always 0) +// u16 num_snapshots // for each arg: // u8 value_kind (0=scalar, 1=pointer/gpu addr, 2=hidden, // 3=scalar/struct with embedded gpu pointer(s)) @@ -183,7 +298,14 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // u8[] data (size bytes) // if value_kind == 3: // u16 n_ptrs -// u16[] ptr_offset (n_ptrs entries: byte offset of each 8-byte gpu ptr) +// u16[] ptr_offset (n_ptrs entries: byte offset of each 8-byte gpu ptr, +// or pointer into a pinned allocation in t_snaps) +// for each snapshot (v8): +// u64 ptr_handle (capture-time base of the pinned allocation) +// u64 offset (0), u64 length (the allocation size) +// u64 hash_lo, u64 hash_hi (blob with the bytes before the launch) +// u8 direction (0) +// u32 num_attrs, u32 attr stride, num_attrs * stride bytes // --------------------------------------------------------------------------- // Locate device pointers embedded inside a by-value kernel argument. @@ -375,7 +497,8 @@ static void serialize_kernel_launch( if (!sig.at(i).info_.hidden_) num_args++; } push_u16(num_args); - push_u16(0); // num_snapshots + push_u16(static_cast(std::count_if( + t_snaps.begin(), t_snaps.end(), [](const HostSnap& s) { return s.record; }))); // HIP_HRR_DEBUG_ARGS dumps every captured arg (kind, size, full bytes, // detected embedded-pointer offsets) — used to confirm pointer layout. Use the @@ -401,6 +524,17 @@ static void serialize_kernel_launch( } std::vector offs; scan_embedded_ptr_offsets(func_key, idx, bytes, sz, offs); + // Words pointing into a pinned allocation this launch reads, so replay + // translates them too. + for (size_t j = 0; bytes && !t_snaps.empty() && j + 8 <= sz; j += 8) { + uint64_t v; + std::memcpy(&v, bytes + j, 8); + if (std::any_of(t_snaps.begin(), t_snaps.end(), + [&](const HostSnap& s) { return v - s.base < s.size; }) && + std::find(offs.begin(), offs.end(), j) == offs.end()) + offs.push_back(static_cast(j)); + } + std::sort(offs.begin(), offs.end()); uint8_t kind = offs.empty() ? 0 : 3; push_u8(kind); push_u16(sz); if (bytes) push_bytes(bytes, sz); @@ -464,6 +598,12 @@ static void serialize_kernel_launch( } } + for (const HostSnap& s : t_snaps) { + if (!s.record) continue; + push_u64(s.base); push_u64(0); push_u64(s.size); + push_u64(s.hash.lo); push_u64(s.hash.hi); push_u8(0); + } + // Launch-attribute tail. Written for every launch so the payload has one // shape: a plain launch records a count of zero. The stride lets replay // reject a recording made against a different hipLaunchAttribute layout @@ -1075,6 +1215,7 @@ hipError_t capture_hipModuleLaunchKernel( unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams, void** extra) { + PinnedSnapshot snap(f, kernelParams, extra); hipError_t r = g_real_table.hipModuleLaunchKernel_fn( f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, @@ -1096,6 +1237,7 @@ hipError_t capture_hipModuleLaunchCooperativeKernel( unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams) { + PinnedSnapshot snap(f, kernelParams, nullptr); hipError_t r = g_real_table.hipModuleLaunchCooperativeKernel_fn( f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, @@ -1116,6 +1258,7 @@ hipError_t capture_hipExtModuleLaunchKernel( size_t sharedMemBytes, hipStream_t stream, void** kernelParams, void** extra, hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags) { + PinnedSnapshot snap(f, kernelParams, extra); hipError_t r = g_real_table.hipExtModuleLaunchKernel_fn( f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, @@ -1145,6 +1288,7 @@ hipError_t capture_hipLaunchKernel(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(function_address), args, nullptr); hipError_t r = g_real_table.hipLaunchKernel_fn( function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1171,6 +1315,7 @@ hipError_t capture_hipLaunchKernel_spt(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(function_address), args, nullptr); hipError_t r = g_real_table.hipLaunchKernel_spt_fn( function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1195,6 +1340,7 @@ hipError_t capture_hipLaunchCooperativeKernel(const void* f, void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(f), kernelParams, nullptr); hipError_t r = g_real_table.hipLaunchCooperativeKernel_fn( f, gridDim, blockDimX, kernelParams, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1216,6 +1362,7 @@ hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, void** kernelParams, uint32_t sharedMemBytes, hipStream_t hStream) { + PinnedSnapshot snap(pinned_stub_fn(f), kernelParams, nullptr); hipError_t r = g_real_table.hipLaunchCooperativeKernel_spt_fn( f, gridDim, blockDim, kernelParams, sharedMemBytes, hStream); if (r == hipSuccess) { @@ -1246,6 +1393,7 @@ hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** params, void** extra) { + PinnedSnapshot snap(config ? f : nullptr, params, extra); hipError_t r = g_real_table.hipDrvLaunchKernelEx_fn(config, f, params, extra); if (r == hipSuccess && config) { record_launch(f, @@ -1259,6 +1407,7 @@ hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipError_t capture_hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args) { + PinnedSnapshot snap(config ? pinned_stub_fn(fPtr) : nullptr, args, nullptr); hipError_t r = g_real_table.hipLaunchKernelExC_fn(config, fPtr, args); if (r == hipSuccess && config) { // fPtr is a host stub address, not a hipFunction_t: the same resolution @@ -1334,6 +1483,8 @@ hipError_t capture_hipLaunchByPtr(const void* func) { kargs = e.arguments_; // copy (not move) to leave arguments_ intact for the real launch } + PinnedSnapshot snap(kargs.empty() ? nullptr : pinned_stub_fn(func), nullptr, nullptr, + kargs.data(), kargs.size()); hipError_t r = g_real_table.hipLaunchByPtr_fn(func); if (r == hipSuccess) { // func is a host stub pointer — resolve to real hipFunction_t first @@ -1514,11 +1665,13 @@ hipError_t capture_hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int std::lock_guard lk(g_pinned_reg_mu); g_pinned_reg_map[hostPtr] = sizeBytes; } + pinned_track(hostPtr, sizeBytes); } return r; } hipError_t capture_hipHostUnregister(void* hostPtr) { + pinned_untrack(hostPtr); hipError_t r = g_real_table.hipHostUnregister_fn(hostPtr); if (r == hipSuccess) { hrr_args_hipHostUnregister a{}; @@ -1533,6 +1686,40 @@ hipError_t capture_hipHostUnregister(void* hostPtr) { return r; } +// hipHostMalloc and the frees are recorded by generated shims. These wrappers +// sit in front of them in g_cap_table and keep g_pinned current; hipFree can +// release pinned memory too. The other pinned allocators are not tracked: +// replay makes no buffer for them. +namespace { +decltype(HipDispatchTable::hipHostMalloc_fn) g_next_hipHostMalloc = nullptr; +decltype(HipDispatchTable::hipHostFree_fn) g_next_hipHostFree = nullptr; +decltype(HipDispatchTable::hipFreeHost_fn) g_next_hipFreeHost = nullptr; +decltype(HipDispatchTable::hipFree_fn) g_next_hipFree = nullptr; +} // namespace + +static hipError_t pinned_hipHostMalloc(void** ptr, size_t size, unsigned int flags) { + hipError_t r = g_next_hipHostMalloc(ptr, size, flags); + if (r == hipSuccess && ptr) pinned_track(*ptr, size); + return r; +} +static hipError_t pinned_hipHostFree(void* p) { pinned_untrack(p); return g_next_hipHostFree(p); } +static hipError_t pinned_hipFreeHost(void* p) { pinned_untrack(p); return g_next_hipFreeHost(p); } +static hipError_t pinned_hipFree(void* p) { pinned_untrack(p); return g_next_hipFree(p); } + +static void pinned_wrap_table() { + static bool done = false; + if (done) return; + done = true; +#define HRR_PINNED_WRAP(api) \ + g_next_##api = g_cap_table.api##_fn; \ + g_cap_table.api##_fn = pinned_##api; + HRR_PINNED_WRAP(hipHostMalloc) + HRR_PINNED_WRAP(hipHostFree) + HRR_PINNED_WRAP(hipFreeHost) + HRR_PINNED_WRAP(hipFree) +#undef HRR_PINNED_WRAP +} + // --------------------------------------------------------------------------- // hipMemPoolSetAttribute: value is void* to a scalar, stored inline in the low // bytes of value_u64 (an int32_t for the three reuse policies, uint64_t otherwise). @@ -2643,6 +2830,7 @@ void hip_capture_init() { // only (see comment above — no static-init capture hook). if (!g_installed) { hip_capture_build_table(); + pinned_wrap_table(); hip_capture_install(); } diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp index f8cad268285c..ce588742a799 100644 --- a/projects/clr/rocclr/utils/flags.hpp +++ b/projects/clr/rocclr/utils/flags.hpp @@ -278,6 +278,11 @@ release(cstring, HIP_HRR_CAPTURE_OUTPUT, "", \ release(bool, HIP_HRR_DEBUG_ARGS, false, \ "Enable HRR capture argument-provenance tracing (per-arg dumps and " \ "H2D destination logging) at LOG_INFO level") \ +release(bool, HIP_HRR_HOST_SNAPSHOTS, true, \ + "HRR capture: record the pinned host memory a kernel reads, before " \ + "each launch") \ +release(uint, HIP_HRR_HOST_SNAPSHOT_MAX_MB, 64, \ + "HRR capture: largest pinned host allocation snapshotted, in MiB") \ release(uint, DEBUG_CLR_DOORBELL_SKIP, 16, \ "Number of consecutive dispatches that may skip the doorbell flush.") \ release(bool, DEBUG_CLR_DISABLE_FALLBACK, false, \ diff --git a/projects/hrr/include/hrr/hrr_api_args.h b/projects/hrr/include/hrr/hrr_api_args.h index ea80e8a71a1f..be3058917dbc 100644 --- a/projects/hrr/include/hrr/hrr_api_args.h +++ b/projects/hrr/include/hrr/hrr_api_args.h @@ -76,8 +76,10 @@ * old recording. * v7: the host blobs of the pitched copies are packed, and the writer sets * HRR_FILE_FLAG_PACKED_HOST_RECTS. A v6 reader ignored the flag and replayed - * a packed blob with the recorded pitch, reading past its end. */ -#define HRR_VERSION ((uint16_t)7u) + * a packed blob with the recorded pitch, reading past its end. + * v8: a kernel launch event carries pinned host snapshot records after its + * arguments (num_snapshots was always 0). A v7 reader would not restore them. */ +#define HRR_VERSION ((uint16_t)8u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1) diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index 4dfbb83aeb69..1bd4a7332b85 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -422,11 +422,47 @@ hipFunction_t PlaybackContext::resolve_replacement(const std::string& kernel_nam // [+12..23] block[3] (uint32_t[3]) // [+24..27] shared_mem (uint32_t) // [+28..29] num_args (uint16_t) -// [+30..31] num_snapshots (uint16_t, always 0) +// [+30..31] num_snapshots (uint16_t) // per arg: u8 value_kind, u16 size, bytes data // value_kind: 0=scalar, 1=gpu-pointer, 2=hidden, // 3=scalar/struct with embedded gpu pointer(s); // kind 3 appends u16 n_ptrs then n_ptrs * u16 byte offsets. +// per snapshot (v8): u64 ptr_handle, u64 offset, u64 length, u64 hash_lo, +// u64 hash_hi, u8 direction (0) +// u32 num_attrs, u32 attr stride, num_attrs * stride bytes + +// Write a launch's pinned host snapshots back before it runs, once the work +// already queued on its stream is done. Not inside a graph capture, where the +// kernel runs only when the graph is launched. +static void restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, + uint16_t n, hipStream_t stream, + const std::string& kernel_name) { + if (ctx.in_graph_capture) { + ctx.host_snapshots_skipped += n; + return; + } + bool synced = false; + for (uint16_t i = 0; i < n; i++, rec += kHostSnapRecordSize) { + uint64_t hash[2]; + memcpy(hash, rec + 24, 16); + size_t blob_len = 0; + const void* blob = ctx.load_blob(hash[0], hash[1], &blob_len); + const char* why = nullptr; + void* dst = hrr_host_snapshot_target(ctx, rec, blob, blob_len, &why); + if (!dst) { + ctx.host_snapshots_refused++; + fprintf(stderr, "[HRR] '%s': pinned host snapshot %u %s — refused\n", + compact_kernel_name(kernel_name).c_str(), i, why); + continue; + } + if (!synced) (void)HRR_HIP_CHECK(hipStreamSynchronize(stream)); + synced = true; + uint64_t len; + memcpy(&len, rec + 16, 8); + memcpy(dst, blob, len); + ctx.host_snapshots_restored++; + } +} // ext_global_worksize: the captured grid[] holds *global work-item counts* // (HSA/OpenCL semantics, as passed to hipExtModuleLaunchKernel), NOT workgroup @@ -1001,6 +1037,15 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, arg_storage, &rls, dbg_dump_ptrs ? &dbg_ptrs : nullptr, dbg_dump_ptrs ? &dbg_args : nullptr); + const uint8_t* snapshots = p; + if (p > end || static_cast(end - p) / kHostSnapRecordSize < num_snapshots) { + fprintf(stderr, "[HRR] '%s': %u pinned host snapshot records run past the " + "event — refused\n", kernel_name.c_str(), num_snapshots); + ctx.host_snapshots_refused += num_snapshots; + num_snapshots = 0; + } + p += static_cast(num_snapshots) * kHostSnapRecordSize; + // Launch-attribute tail: u32 count, u32 per-entry stride, then the // entries. Every launch payload carries it (count 0 for a plain launch). std::vector launch_attrs; @@ -1048,6 +1093,8 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, hipStream_t stream = ctx.translate_stream(stream_rec); const size_t kernel_ordinal = ctx.kernels_launched.load(std::memory_order_relaxed) + 1; + if (num_snapshots) + restore_host_snapshots(ctx, snapshots, num_snapshots, stream, kernel_name); if (ctx.audit_host_args) { for (auto& [idx, rec, live] : dbg_ptrs) { diff --git a/projects/hrr/playback/hip_playback.h b/projects/hrr/playback/hip_playback.h index 54f084253005..844b668aec5f 100644 --- a/projects/hrr/playback/hip_playback.h +++ b/projects/hrr/playback/hip_playback.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -234,6 +235,11 @@ struct PlaybackContext { // If d2h_attempted > 0 but d2h_pass == 0 && d2h_fail == 0, every check was // skipped — pointer translation or blob loading failed for all D2H events. std::atomic d2h_attempted{0}; + // Pinned host snapshot records written back, refused, and not applied + // because the launch was replayed into a graph capture. + std::atomic host_snapshots_restored{0}; + std::atomic host_snapshots_refused{0}; + std::atomic host_snapshots_skipped{0}; // Set true by note_d2h_fail when the running D2H-failure fraction crosses the // configured divergence-abort threshold. Distinguishes a clean "replay @@ -833,6 +839,45 @@ extern thread_local uint64_t hrr_dispatch_seq; // completion — including a genuine GPU fault — is returned to the caller. hipError_t hrr_watchdog_device_sync(PlaybackContext& ctx, const char* what); +// One pinned host snapshot record of a kernel launch event: u64 ptr_handle, +// u64 offset, u64 length, u64 hash_lo, u64 hash_hi, u8 direction (0). +inline constexpr size_t kHostSnapRecordSize = 41; + +// Where replay writes the blob of the record at `rec`, or nullptr with *why +// saying why it refuses the record. The archive is input like any other: a +// record must name the base of a pinned host allocation replay made, lie +// inside it, and match its blob's length. +inline void* hrr_host_snapshot_target(const PlaybackContext& ctx, const uint8_t* rec, + const void* blob, size_t blob_len, + const char** why) { + uint64_t ptr, off, len; + memcpy(&ptr, rec, 8); + memcpy(&off, rec + 8, 8); + memcpy(&len, rec + 16, 8); + void* live = nullptr; + size_t size = 0; + { + std::shared_lock lk(ctx.map_mutex); + auto it = ctx.alloc_map.find(ptr); + if (it != ctx.alloc_map.end() && (it->second.kind == AllocKind::HostMalloc || + it->second.kind == AllocKind::HostRegister)) { + live = it->second.live_ptr; + size = it->second.size; + } + } + if (rec[40] != 0) + *why = "has an unknown direction"; + else if (!live) + *why = "names no pinned host allocation replay made"; + else if (off > size || len > size - off) + *why = "lies outside its allocation"; + else if (!blob || blob_len != len) + *why = "does not match its blob"; + else + return static_cast(live) + off; + return nullptr; +} + // --------------------------------------------------------------------------- // Playback function signature and dispatch table // --------------------------------------------------------------------------- diff --git a/projects/hrr/playback/hrr_playback.cpp b/projects/hrr/playback/hrr_playback.cpp index 02b590ac38e9..06fb419ec968 100644 --- a/projects/hrr/playback/hrr_playback.cpp +++ b/projects/hrr/playback/hrr_playback.cpp @@ -1654,6 +1654,10 @@ int main(int argc, char** argv) { ctx.d2h_pass.load() - ctx.d2h_pass_tol.load(), ctx.d2h_pass_tol.load(), ctx.d2h_fail.load(), ctx.d2h_attempted.load() - ctx.d2h_pass.load() - ctx.d2h_fail.load()); + if (ctx.host_snapshots_restored + ctx.host_snapshots_refused + ctx.host_snapshots_skipped) + printf("[HRR] Pinned host snapshots: %zu restored, %zu refused, %zu skipped " + "(graph capture)\n", ctx.host_snapshots_restored.load(), + ctx.host_snapshots_refused.load(), ctx.host_snapshots_skipped.load()); bool ok = (ctx.d2h_fail == 0); diff --git a/projects/hrr/playback/hrr_reader.cpp b/projects/hrr/playback/hrr_reader.cpp index 71d68e9def63..fac473effa56 100644 --- a/projects/hrr/playback/hrr_reader.cpp +++ b/projects/hrr/playback/hrr_reader.cpp @@ -227,8 +227,9 @@ static bool resolve_archive_path(const std::string& input, std::string& resolved // [+12..23] block[3] (uint32_t[3]) // [+24..27] shared_mem (uint32_t) // [+28..29] num_args (uint16_t) -// [+30..31] num_snapshots (uint16_t, always 0) +// [+30..31] num_snapshots (uint16_t) // per arg: u8 value_kind, u16 size, bytes data +// per snapshot (v8, 41 bytes): see BufferSnapshot; the attribute tail follows // --------------------------------------------------------------------------- static bool parse_kernel_launch(const uint8_t* data, size_t len, @@ -287,7 +288,7 @@ static bool parse_kernel_launch(const uint8_t* data, size_t len, kl.args.push_back(std::move(arg)); } - // buffer snapshots (always 0 in in-tree captures) + // pinned host snapshots for (uint16_t i = 0; i < num_snapshots; i++) { if (p + 41 > end) return false; BufferSnapshot snap; diff --git a/projects/hrr/playback/hrr_reader.h b/projects/hrr/playback/hrr_reader.h index 3e2b19c52e94..29013347d32e 100644 --- a/projects/hrr/playback/hrr_reader.h +++ b/projects/hrr/playback/hrr_reader.h @@ -92,7 +92,8 @@ struct KernelArg { std::vector ptr_offsets; // only for value_kind == 3 }; -// Buffer snapshot (always empty in in-tree captures) +// Pinned host snapshot (v8): the blob replay writes to [offset, offset + +// length) of the pinned allocation recorded at ptr_handle before the launch. struct BufferSnapshot { uint64_t ptr_handle; uint64_t offset; diff --git a/projects/hrr/tools/gen_hrr_api_args.py b/projects/hrr/tools/gen_hrr_api_args.py index cd9a342bf5dd..869b5947f381 100644 --- a/projects/hrr/tools/gen_hrr_api_args.py +++ b/projects/hrr/tools/gen_hrr_api_args.py @@ -1960,8 +1960,10 @@ def validate_capture_coverage(entries: List[ApiEntry], * old recording. * v7: the host blobs of the pitched copies are packed, and the writer sets * HRR_FILE_FLAG_PACKED_HOST_RECTS. A v6 reader ignored the flag and replayed - * a packed blob with the recorded pitch, reading past its end. */ -#define HRR_VERSION ((uint16_t)7u) + * a packed blob with the recorded pitch, reading past its end. + * v8: a kernel launch event carries pinned host snapshot records after its + * arguments (num_snapshots was always 0). A v7 reader would not restore them. */ +#define HRR_VERSION ((uint16_t)8u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1) From 4e60f35b0d4858f7ce70b5c6aa6f5fd2e701d059 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Fri, 9 Oct 2026 10:26:56 +0200 Subject: [PATCH 02/12] hrr: test and document pinned host snapshots --- projects/hrr/DESIGN.md | 92 ++++++++++-------- projects/hrr/README.md | 1 + projects/hrr/tests/hrr_test_common.hh | 5 +- projects/hrr/tests/integration/CMakeLists.txt | 5 + .../tests/integration/hrr_pinned_host_test.cc | 97 +++++++++++++++++++ .../tests/integration/hrr_roundtrip_test.cc | 8 +- projects/hrr/tests/unit/hrr_format_test.cc | 80 +++++++++++++++ 7 files changed, 244 insertions(+), 44 deletions(-) create mode 100644 projects/hrr/tests/integration/hrr_pinned_host_test.cc diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index bf981a61158c..3051efe9c395 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -148,7 +148,7 @@ precisely the ones left without a trailer and absent from the root index, while the parent that exited cleanly needs no repair. Sub-archives that already carry a clean trailer are skipped without being read. -### Archive Format (v7) +### Archive Format (v8) ``` capture.hrr/ manifest.json { version, capture_mode, owner_pid, processes[] } @@ -260,7 +260,7 @@ projects/hrr/ — standalone HRR project (portable layer) Not used by capture — it is the format an out-of-tree producer writes and playback reads playback/ - hrr_reader.h/.cpp — archive loader, v7 format; record framing + hrr_reader.h/.cpp — archive loader, v8 format; record framing (read_raw_record / open_record_stream) shared with the region sidecars hrr_region_map.h/.cpp — region timeline: merge, cursor, live block set, @@ -339,13 +339,13 @@ The generator classifies each API: Generated capture shims for manual APIs are pass-throughs (no `write_event()`). When adding HIP API support, update this script to classify the API in the appropriate capture and playback policy sets. APIs requiring non-trivial serialization or replay belong in `MANUAL_CAPTURE_APIS` and/or `MANUAL_PLAYBACK_APIS`; intentionally unsupported replay APIs belong in `NOOP_PLAYBACK_APIS`. -## Archive Format (v7) +## Archive Format (v8) Single-authority definition in `hrr_api_args.h` (auto-generated): ``` HRR_MAGIC = 0x52524845 ("HRRE") -HRR_VERSION = 7 +HRR_VERSION = 8 ``` Version history, so an archive written by an older runtime can be placed: @@ -374,12 +374,15 @@ Version history, so an archive written by an older runtime can be placed: `HRR_FILE_FLAG_PACKED_HOST_RECTS` in `hrr_file_header.reserved`. A v6 reader ignores that field and would replay a packed blob with the recorded pitch, reading past its end. See 2D/3D Memcpy and Memset below. +- **v8** puts pinned host snapshot records after a kernel launch's arguments + (`num_snapshots` was always 0). A v7 reader would not restore them. See + Pinned Host Snapshots below. ``` / manifest.json { version, capture_mode, owner_pid, processes[] } (version here is the manifest schema = 1, distinct from the - events.bin HRR_VERSION = 7) + events.bin HRR_VERSION = 8) pid-/ manifest.json { pid, parent_pid, complete, event_count, blob_count } writer_state.json checkpoint cursor (next_seq, event/blob counts, events file @@ -497,41 +500,47 @@ At playback: - `playback_hipHostUnregister` translates the pointer, calls `hipHostUnregister`, frees the backing buffer, and removes both map entries. -### Sysmem Update Tracking (planned — not yet implemented) - -**Problem:** `hipHostRegister` captures the buffer contents at registration time only. -After that, the CPU may modify the buffer without going through any HIP API: - -``` -hipHostRegister(ptr, sz) ← blob captured (initial state) -// CPU writes to ptr[] ← invisible to capture layer -hipMemcpyAsync(d, ptr, sz, H2D, stream) ← H2D blob captures current contents ✓ -// CPU writes to ptr[] again -hipModuleLaunchKernel(...) ← kernel reads ptr directly via mapped flag ✗ stale -``` - -H2D memcpy is already handled — the src buffer is re-snapshotted at each call regardless -of registration. The gap is direct GPU reads from registered host memory (mapped flag) -after a CPU write that was not routed through a memcpy. - -**Planned design:** - -1. New synthetic event `HRR_SYSMEM_UPDATE` (not a real HIP API): - fields: `hostPtr u64`, `sizeBytes u64`, `blob_hash_lo u64`, `blob_hash_hi u64`. - -2. Per-region `last_hash` stored alongside `g_pinned_reg_map`. Before each H2D memcpy src - check and before each kernel launch for pointer args in registered ranges: hash the - current contents, compare with `last_hash`. Emit `HRR_SYSMEM_UPDATE` + write blob only - if hash changed. Content-addressed storage deduplicates unchanged regions automatically. - -3. Optional sync-gated dirty flag: set `dirty=true` for all registered regions after any - `hipStreamSynchronize` / `hipDeviceSynchronize` / `hipEventSynchronize`. Only hash - (step 2) if `dirty==true`. Avoids hashing in pure-GPU loops where the CPU never - touches the buffer between launches. Falls back to always-hash if sync events are - not captured. - -4. At replay: `HRR_SYSMEM_UPDATE` handler `memcpy`s the blob into the live registered - buffer, ordered by sequence ID like all other events. +### Pinned Host Snapshots + +A kernel can read pinned host memory that the host filled with plain stores, which no HIP +call records; replay would run it on whatever its fresh buffer holds. Capture tracks the +ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, +`hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of +the arguments, by-value structs included; for each tracked allocation a word points into, +it writes the whole allocation as a blob and adds a record (base, offset 0, length, blob +hash, direction 0) to the launch event, unless the hash equals that of the allocation's +last record. Struct words pointing into such an allocation are flagged for translation like +embedded device pointers. Replay checks each record names the base of a pinned host +allocation it made, lies inside it and matches its blob's length, refusing and counting it +otherwise; then it synchronizes the launch stream, copies the blob in, and launches. The +summary prints restored, refused and skipped counts. + +Kept deliberately simple: + +- Capture does not wait for the stream: the bytes are those at enqueue time, which is what + the kernel reads unless earlier queued work still changes them. +- `HIP_HRR_HOST_SNAPSHOTS=0` turns snapshots off; an allocation larger than + `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under + `unreplayable_apis`. +- Replay restores nothing for a launch replayed into a graph capture, and capture does not + snapshot at graph launch. +- `hipHostAlloc`, `hipMallocHost` and `hipMemAllocHost` are not tracked, since replay makes + no buffer for them, nor is a `hipHostRegister` device alias that differs from the host + address. +- An unchanged allocation gets no record, so device writes into it between two launches + stay in replay's buffer as they did at capture; a host store that puts back exactly the + last recorded bytes after such a device write is missed. +- Only the launch stream is synchronized before a restore; work on other streams reading + the buffer is not waited for. A launch that fails after its snapshot still counts as the + last record, and a `fork()` while another thread snapshots can leave the child's tracking + lock held. + +**Threat model.** The snapshots put into the archive host memory the application never +handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, +including stale bytes from earlier requests in a reused buffer. The archive must be +handled as at least as sensitive as the workload's data. Replay treats every record as +untrusted input and writes only inside a pinned host allocation it made itself, never +through a recorded address. ## Kernel Argument Capture @@ -1170,7 +1179,8 @@ Sysmem capture is a fundamentally hard problem. Two approaches exist: and emit `HRR_SYSMEM_UPDATE` synthetic events only when content changes. This is significantly more complex to implement and may not be fully feasible on all platforms. -Neither approach is currently implemented. +Pinned host memory a kernel reads is now snapshotted before each launch; see Pinned Host +Snapshots above. ### Partial-Replay APIs — H2D and D2H Work, Kernel Launches Do Not diff --git a/projects/hrr/README.md b/projects/hrr/README.md index b1e1d65bc395..5c21af990a22 100644 --- a/projects/hrr/README.md +++ b/projects/hrr/README.md @@ -92,6 +92,7 @@ User-facing capture, replay, and validation knobs. Implementation details can be |----------|---------|---------| | `HIP_HRR_CAPTURE_OUTPUT` | *(unset)* | Enable capture; path to the `.hrr` archive directory. An empty or blank value leaves capture off. Ignored, with one line on stderr, when Linux starts the program in secure-execution mode (set-user-ID, set-group-ID, file capabilities or an LSM transition) | | `HIP_HRR_DEBUG_ARGS` | off | Dump every captured kernel arg to the log (debug / provenance) | +| `HIP_HRR_HOST_SNAPSHOTS` | on | Record the pinned host memory each kernel reads before the launch, so replay can write it back. `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (default `64`) caps the allocation size. The archive then holds that host memory (see DESIGN.md, Pinned Host Snapshots) | On Linux the archive is readable only by the user who captured it (directories 0700, files 0600). If the archive cannot be set up, capture is disabled with a `[HRR capture] Capture disabled` message on stderr and the application runs normally. diff --git a/projects/hrr/tests/hrr_test_common.hh b/projects/hrr/tests/hrr_test_common.hh index 3896042eb391..9d626ed90f7f 100644 --- a/projects/hrr/tests/hrr_test_common.hh +++ b/projects/hrr/tests/hrr_test_common.hh @@ -281,6 +281,7 @@ inline void hrr_run_roundtrip(const std::string& direct_case, // // hrr_capture_direct: spawn a hidden _Direct workload with HIP_HRR_CAPTURE_OUTPUT // set, REQUIRE a clean capture, and assert the archive has >= min_events. +// Pairs in env are set in the workload's environment as well. // // hrr_playback_env: run hrr-playback with arbitrary extra environment pairs // (e.g. HIP_HRR_REPLAY_ZERO_INIT / HIP_HRR_REPLAY_DIVERGENCE_ABORT) and return @@ -292,9 +293,11 @@ inline void hrr_run_roundtrip(const std::string& direct_case, // --------------------------------------------------------------------------- inline void hrr_capture_direct(const std::string& direct_case, const fs::path& cap_path, - size_t min_events = 5) { + size_t min_events = 5, + const std::vector>& env = {}) { { hrr::test::SpawnProc proc(HRR_TEST_EXE); proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap_path.string()); + for (const auto& kv : env) proc.setEnv(kv.first, kv.second); { set_proc_search_path(proc); } int ret = proc.run("\"" + direct_case + "\""); INFO("Capture exit: " << ret); REQUIRE(ret == 0); } diff --git a/projects/hrr/tests/integration/CMakeLists.txt b/projects/hrr/tests/integration/CMakeLists.txt index 03f5beb639f8..eee77270a8dd 100644 --- a/projects/hrr/tests/integration/CMakeLists.txt +++ b/projects/hrr/tests/integration/CMakeLists.txt @@ -41,6 +41,10 @@ # address-shaped lost pointer becomes null. Contains its own hidden # [.][hrr-direct] workload. # +# hrr_pinned_host_test.cc covers pinned host memory a kernel reads directly, +# which capture snapshots before each launch and replay writes back. Contains +# its own hidden [.][hrr-direct] workloads. +# # hrr_fatbin_header_test.cc covers which registered fat binaries capture stores: # its hidden [.][hrr-direct] workload registers valid and altered bundle # headers itself, before and after HIP initialises. It compiles its kernel with @@ -69,6 +73,7 @@ set(HRR_GPU_WORKLOAD_SOURCES hrr_regions_test.cc hrr_api_matrix_workload_test.cc hrr_kernel_args_test.cc + hrr_pinned_host_test.cc hrr_disk_space_test.cc ) diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc new file mode 100644 index 000000000000..00114e8b5016 --- /dev/null +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * @addtogroup HRR HRR pinned host snapshots + * @{ + * @ingroup HRRTest + * A kernel reads pinned host memory that the host refills with plain stores + * between launches, which no HIP call records. Capture snapshots it before + * each launch and replay writes it back, so the replayed D2H checks match. + */ + +#include "hrr_test_common.hh" + +#include +#include + +namespace { +constexpr int kPinnedN = 256; +constexpr int kPinnedIters = 4; +// The input changes on every second launch: two snapshot records, four checks. +int pinned_value(int it, int i) { return (it / 2) * 1000 + i; } +} // namespace + +__global__ void hrr_pinned_triple(int* out, const int* in, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) out[i] = in[i] * 3 + 1; +} + +static void hrr_pinned_workload(int* in) { + int* dev_in = nullptr; + HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); + int* dout = nullptr; + HRR_HIP_CHECK(hipMalloc(&dout, kPinnedN * sizeof(int))); + std::vector hout(kPinnedN); + for (int it = 0; it < kPinnedIters; ++it) { + for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, nullptr, + dout, dev_in, kPinnedN); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, kPinnedN * sizeof(int), hipMemcpyDeviceToHost)); + for (int i = 0; i < kPinnedN; ++i) REQUIRE(hout[i] == pinned_value(it, i) * 3 + 1); + } + HRR_HIP_CHECK(hipFree(dout)); +} + +TEST_CASE("Unit_HRR_PinnedHostMalloc_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipHostFree(in)); +} + +TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + std::vector in(kPinnedN); + HRR_HIP_CHECK(hipHostRegister(in.data(), kPinnedN * sizeof(int), hipHostRegisterDefault)); + hrr_pinned_workload(in.data()); + HRR_HIP_CHECK(hipHostUnregister(in.data())); +} + +#if defined(HRR_PLAYBACK_EXE) && defined(HRR_TEST_EXE) +// Replay matches the recording only if it wrote each launch's input back; its +// own buffer never sees the host's stores. With HIP_HRR_HOST_SNAPSHOTS=0 there +// is nothing to write back and the checks fail. +HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { + for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", + "Unit_HRR_PinnedHostRegister_Direct"}) { + for (const char* snapshots : {"1", "0"}) { + INFO(direct << " HIP_HRR_HOST_SNAPSHOTS=" << snapshots); + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host"}; + hrr_capture_direct(direct, cap.path, 5, {{"HIP_HRR_HOST_SNAPSHOTS", snapshots}}); + auto [ret, out] = hrr_playback_env(cap.path, {{"HIP_HRR_D2H_EXACT", "1"}}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + int pass = 0, fail = 0; + REQUIRE(hrr_parse_d2h_summary(out, pass, fail)); + if (std::string(snapshots) == "1") { + CHECK(ret == 0); + CHECK(pass == kPinnedIters); + CHECK(fail == 0); + CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused") != std::string::npos); + } else { + CHECK(ret != 0); + CHECK(fail > 0); + } + } + } +} +#endif // HRR_PLAYBACK_EXE && HRR_TEST_EXE + +/** + * @} + */ diff --git a/projects/hrr/tests/integration/hrr_roundtrip_test.cc b/projects/hrr/tests/integration/hrr_roundtrip_test.cc index 06e4d440f31c..cc8efab4abc7 100644 --- a/projects/hrr/tests/integration/hrr_roundtrip_test.cc +++ b/projects/hrr/tests/integration/hrr_roundtrip_test.cc @@ -673,7 +673,10 @@ HRR_TEST_CASE(Unit_HRR_ZeroInitRoundtrip) { */ HRR_TEST_CASE(Unit_HRR_DivergenceAbortRoundtrip) { ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_divergence"}; - hrr_capture_direct("Unit_HRR_UncapturedHostWrite_Direct", cap.path); + // Pinned host snapshots would record the flag, so the write stays unseen only + // without them. + hrr_capture_direct("Unit_HRR_UncapturedHostWrite_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOTS", "0"}}); SECTION("guard ON -> clean exit 2") { auto [ret, out] = hrr_playback_env( @@ -723,7 +726,8 @@ HRR_TEST_CASE(Unit_HRR_DivergenceAbortRoundtrip) { */ TEST_CASE("Unit_HRR_NullOptionalPtrRoundtrip", "[.][hrr-repro]") { ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_nulloptional"}; - hrr_capture_direct("Unit_HRR_NullOptionalPtr_Direct", cap.path); + hrr_capture_direct("Unit_HRR_NullOptionalPtr_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOTS", "0"}}); auto [ret, out] = hrr_playback_env( cap.path, diff --git a/projects/hrr/tests/unit/hrr_format_test.cc b/projects/hrr/tests/unit/hrr_format_test.cc index 2359df2be75c..d97a18551b08 100644 --- a/projects/hrr/tests/unit/hrr_format_test.cc +++ b/projects/hrr/tests/unit/hrr_format_test.cc @@ -467,3 +467,83 @@ HRR_TEST_CASE(Unit_HRR_Format_UnknownEventType) { CHECK(archive.events[0].header().event_type == 0xFFFFu); CHECK(archive.events[0].malloc_ev.ptr_handle == 0u); } + +// A kernel launch event with no arguments and one pinned host snapshot record, +// cut `cut` bytes short of a whole record (v8 layout, as capture writes it). +static std::vector make_launch_with_snapshot(uint64_t seq, + const std::vector& rec, + size_t cut = 0) { + std::vector p(sizeof(hrr_event_header), 0); + auto put = [&](const void* d, size_t n) { + p.insert(p.end(), static_cast(d), static_cast(d) + n); + }; + const uint64_t stream = 0, co_hash[2] = {0, 0}; + const uint16_t name_len = 1, counts[2] = {0, 1}; // num_args, num_snapshots + const uint32_t dims[7] = {1, 1, 1, 1, 1, 1, 0}, attrs[2] = {0, 0}; + put(&stream, 8); put(&name_len, 2); put("k", 1); put(co_hash, 16); + put(dims, sizeof(dims)); put(counts, sizeof(counts)); + put(rec.data(), rec.size() - cut); + if (!cut) put(attrs, sizeof(attrs)); + auto* h = reinterpret_cast(p.data()); + h->event_type = static_cast(HRR_API_HIPMODULELAUNCHKERNEL); + h->sequence_id = seq; + h->payload_length = static_cast(p.size()); + return p; +} + +/** + * Test Description + * ---------------- + * - A pinned host snapshot record round-trips through the reader, and replay + * points it at its own pinned buffer at the recorded offset. + * - A launch whose record runs past the event is not decoded, and replay + * refuses a record that names a device allocation or the inside of one, + * runs past its allocation, differs from its blob in length, or has an + * unknown direction. + */ +HRR_TEST_CASE(Unit_HRR_Format_HostSnapshotRecord) { + auto record = [](uint64_t ptr, uint64_t off, uint64_t len, uint8_t direction) { + const uint64_t f[5] = {ptr, off, len, 0x1122334455667788ULL, 0x99aabbccddeeff00ULL}; + std::vector r(kHostSnapRecordSize); + std::memcpy(r.data(), f, sizeof(f)); + r[40] = direction; + return r; + }; + const auto good = record(0x2000, 16, 64, 0); + TmpArchive arc("host_snapshot"); + auto body = make_launch_with_snapshot(0, good); + const auto torn = make_launch_with_snapshot(1, good, 1); + body.insert(body.end(), torn.begin(), torn.end()); + arc.write_events(body); + + hrr::Archive archive; + REQUIRE(hrr::load_archive(arc.path(), archive)); + REQUIRE(archive.events.size() == 2); + REQUIRE(archive.events[0].kernel_launch != nullptr); + REQUIRE(archive.events[0].kernel_launch->snapshots.size() == 1); + const hrr::BufferSnapshot& s = archive.events[0].kernel_launch->snapshots[0]; + CHECK(s.ptr_handle == 0x2000); + CHECK(s.offset == 16); + CHECK(s.length == 64); + CHECK(s.hash_lo == 0x1122334455667788ULL); + CHECK(s.hash_hi == 0x99aabbccddeeff00ULL); + CHECK(s.direction == 0); + CHECK(archive.events[1].kernel_launch == nullptr); + + PlaybackContext ctx; + std::vector pinned(128), device(128), blob(64); + ctx.record_alloc(0x2000, pinned.data(), pinned.size(), AllocKind::HostMalloc); + ctx.record_alloc(0x3000, device.data(), device.size(), AllocKind::Device); + const char* why = nullptr; + CHECK(hrr_host_snapshot_target(ctx, good.data(), blob.data(), blob.size(), &why) == + pinned.data() + 16); + CHECK(why == nullptr); + CHECK(hrr_host_snapshot_target(ctx, good.data(), nullptr, 0, &why) == nullptr); + for (const auto& bad : {record(0x3000, 0, 64, 0), record(0x2008, 0, 64, 0), + record(0x2000, 80, 64, 0), record(0x2000, 16, 32, 0), + record(0x2000, 16, 64, 1)}) { + why = nullptr; + CHECK(hrr_host_snapshot_target(ctx, bad.data(), blob.data(), blob.size(), &why) == nullptr); + CHECK(why != nullptr); + } +} From b6583ca06c38fb1922a13bcf06d63f2bb479a7d2 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Fri, 9 Oct 2026 10:46:29 +0200 Subject: [PATCH 03/12] hrr: mark a pinned snapshot as last only after its event is written; more tests --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 9 ++- projects/hrr/DESIGN.md | 12 ++-- projects/hrr/playback/hip_playback.cpp | 2 +- projects/hrr/playback/hip_playback.h | 5 ++ .../tests/integration/hrr_pinned_host_test.cc | 57 +++++++++++++++++-- projects/hrr/tests/unit/hrr_format_test.cc | 6 +- 6 files changed, 76 insertions(+), 15 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index a1f10303464f..3f84e200849f 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -228,7 +228,6 @@ static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { s.hash = hrr_cap::writer::write_blob(reinterpret_cast(s.base), s.size); PinnedAlloc& a = it->second; s.record = (s.hash.lo || s.hash.hi) && (s.hash.lo != a.last.lo || s.hash.hi != a.last.hi); - if (s.record) a.last = s.hash; } t_snaps.push_back(s); } @@ -633,9 +632,17 @@ static void serialize_kernel_launch( hrr_cap::writer::mark_incomplete(reason); return; } + // A record becomes the allocation's last only once its event is written, in + // the same order as the events. + std::unique_lock lk(g_pinned_mu, std::defer_lock); + if (!t_snaps.empty()) lk.lock(); hrr_cap::writer::write_event_raw(api_id, reinterpret_cast(payload.data()), static_cast(payload.size())); + for (const HostSnap& s : t_snaps) { + auto it = g_pinned.find(s.base); + if (s.record && it != g_pinned.end()) it->second.last = s.hash; + } } static hrr_cap::Hash128 kernel_code_object_hash(amd::Kernel* kernel); diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 3051efe9c395..56eba527dba4 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -531,9 +531,8 @@ Kept deliberately simple: stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. - Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. A launch that fails after its snapshot still counts as the - last record, and a `fork()` while another thread snapshots can leave the child's tracking - lock held. + the buffer is not waited for. A `fork()` while another thread snapshots can leave the + child's tracking lock held. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, @@ -628,10 +627,9 @@ This detector is a value-based heuristic with three deliberate properties: previously caused occasional replay faults on ATen elementwise kernels. `HIP_HRR_PTR_RELAX=1` disables the replay-side guard for debugging. -**Scope:** only `hipMemoryTypeDevice`/`Unified` words are flagged. A pinned/host -(`hipMemoryTypeHost`) pointer embedded by value keeps its capture-time host VA at -replay (invalid in the replay process); translating embedded host pointers is -intentionally out of scope. +**Scope:** `hipMemoryTypeDevice`/`Unified` words are flagged, and so are words +pointing into a pinned allocation the launch snapshots (see Pinned Host Snapshots). +Any other embedded host pointer keeps its capture-time VA at replay. A per-launch `co_hash` (the FNV-1a-128 hash of the owning code object) **is** recorded in kernel launch events. Playback resolves kernels by `(co_hash, name)`: it first looks diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index 1bd4a7332b85..d98c71ddd2ad 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -1038,7 +1038,7 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, dbg_dump_ptrs ? &dbg_args : nullptr); const uint8_t* snapshots = p; - if (p > end || static_cast(end - p) / kHostSnapRecordSize < num_snapshots) { + if (!hrr_host_snapshots_fit(p, end, num_snapshots)) { fprintf(stderr, "[HRR] '%s': %u pinned host snapshot records run past the " "event — refused\n", kernel_name.c_str(), num_snapshots); ctx.host_snapshots_refused += num_snapshots; diff --git a/projects/hrr/playback/hip_playback.h b/projects/hrr/playback/hip_playback.h index 844b668aec5f..e4eaf6c470df 100644 --- a/projects/hrr/playback/hip_playback.h +++ b/projects/hrr/playback/hip_playback.h @@ -843,6 +843,11 @@ hipError_t hrr_watchdog_device_sync(PlaybackContext& ctx, const char* what); // u64 offset, u64 length, u64 hash_lo, u64 hash_hi, u8 direction (0). inline constexpr size_t kHostSnapRecordSize = 41; +// Whether n whole records fit between p and the end of the event. +inline bool hrr_host_snapshots_fit(const uint8_t* p, const uint8_t* end, uint16_t n) { + return p <= end && static_cast(end - p) / kHostSnapRecordSize >= n; +} + // Where replay writes the blob of the record at `rec`, or nullptr with *why // saying why it refuses the record. The archive is input like any other: a // record must name the base of a pinned host allocation replay made, lie diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index 00114e8b5016..ece0aef41029 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -30,7 +30,16 @@ __global__ void hrr_pinned_triple(int* out, const int* in, int n) { if (i < n) out[i] = in[i] * 3 + 1; } -static void hrr_pinned_workload(int* in) { +// The same kernel with the pinned pointer inside a by-value struct. +struct PinnedIn { const int* in; int n; }; +__global__ void hrr_pinned_triple_struct(int* out, PinnedIn s) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < s.n) out[i] = s.in[i] * 3 + 1; +} + +__global__ void hrr_pinned_ignore(uint64_t) {} + +static void hrr_pinned_workload(int* in, bool by_struct = false) { int* dev_in = nullptr; HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); int* dout = nullptr; @@ -38,8 +47,12 @@ static void hrr_pinned_workload(int* in) { std::vector hout(kPinnedN); for (int it = 0; it < kPinnedIters; ++it) { for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores - hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, nullptr, - dout, dev_in, kPinnedN); + if (by_struct) + hipLaunchKernelGGL(hrr_pinned_triple_struct, dim3(1), dim3(kPinnedN), 0, nullptr, + dout, PinnedIn{dev_in, kPinnedN}); + else + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, nullptr, + dout, dev_in, kPinnedN); HRR_HIP_CHECK(hipGetLastError()); HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, kPinnedN * sizeof(int), hipMemcpyDeviceToHost)); for (int i = 0; i < kPinnedN; ++i) REQUIRE(hout[i] == pinned_value(it, i) * 3 + 1); @@ -55,6 +68,27 @@ TEST_CASE("Unit_HRR_PinnedHostMalloc_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostFree(in)); } +TEST_CASE("Unit_HRR_PinnedHostStruct_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in, true); + HRR_HIP_CHECK(hipHostFree(in)); +} + +// A freed allocation's address passed on as a plain number: capture must not +// snapshot memory that is gone. +TEST_CASE("Unit_HRR_PinnedHostFreed_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + void* p = nullptr; + HRR_HIP_CHECK(hipHostMalloc(&p, kPinnedN * sizeof(int), 0)); + HRR_HIP_CHECK(hipHostFree(p)); + hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, + reinterpret_cast(p)); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipDeviceSynchronize()); +} + TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); std::vector in(kPinnedN); @@ -69,8 +103,12 @@ TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { // is nothing to write back and the checks fail. HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", - "Unit_HRR_PinnedHostRegister_Direct"}) { + "Unit_HRR_PinnedHostRegister_Direct", + "Unit_HRR_PinnedHostStruct_Direct"}) { for (const char* snapshots : {"1", "0"}) { + // Without snapshots the struct would carry a host address replay never mapped. + if (std::string(direct) == "Unit_HRR_PinnedHostStruct_Direct" && *snapshots == '0') + continue; INFO(direct << " HIP_HRR_HOST_SNAPSHOTS=" << snapshots); ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host"}; hrr_capture_direct(direct, cap.path, 5, {{"HIP_HRR_HOST_SNAPSHOTS", snapshots}}); @@ -90,6 +128,17 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { } } } + +// Capture stops tracking an allocation when it is freed: had it snapshotted the +// stale address, replay would refuse the record for an allocation it freed. +HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_freed"}; + hrr_capture_direct("Unit_HRR_PinnedHostFreed_Direct", cap.path, 3); + auto [ret, out] = hrr_playback_env(cap.path, {}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + CHECK(ret == 0); + CHECK(out.find("Pinned host snapshots") == std::string::npos); +} #endif // HRR_PLAYBACK_EXE && HRR_TEST_EXE /** diff --git a/projects/hrr/tests/unit/hrr_format_test.cc b/projects/hrr/tests/unit/hrr_format_test.cc index d97a18551b08..d4db66e4d01b 100644 --- a/projects/hrr/tests/unit/hrr_format_test.cc +++ b/projects/hrr/tests/unit/hrr_format_test.cc @@ -496,8 +496,8 @@ static std::vector make_launch_with_snapshot(uint64_t seq, * ---------------- * - A pinned host snapshot record round-trips through the reader, and replay * points it at its own pinned buffer at the recorded offset. - * - A launch whose record runs past the event is not decoded, and replay - * refuses a record that names a device allocation or the inside of one, + * - A launch whose record runs past the event is not decoded by the reader + * nor by replay, and replay refuses a record that names a device allocation or the inside of one, * runs past its allocation, differs from its blob in length, or has an * unknown direction. */ @@ -529,6 +529,8 @@ HRR_TEST_CASE(Unit_HRR_Format_HostSnapshotRecord) { CHECK(s.hash_hi == 0x99aabbccddeeff00ULL); CHECK(s.direction == 0); CHECK(archive.events[1].kernel_launch == nullptr); + CHECK(hrr_host_snapshots_fit(good.data(), good.data() + good.size(), 1)); + CHECK_FALSE(hrr_host_snapshots_fit(good.data(), good.data() + good.size() - 1, 1)); PlaybackContext ctx; std::vector pinned(128), device(128), blob(64); From b3bd5dade1dff51759e0c5432929b45f7d58c307 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Mon, 12 Oct 2026 10:45:08 +0200 Subject: [PATCH 04/12] fix(hrr): hold pinned tracking across fork and failed frees, translate low pinned addresses The pinned tracking mutex joins the writer's fork protocol, taken first in prepare. A release call keeps the allocation tracked unless it succeeds, with snapshots held off for its duration. Replay translates a struct word capture flagged inside a pinned host allocation even when its low 32 bits fail the device-address shape check. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 44 ++++++++++++++----- projects/hrr/DESIGN.md | 9 ++-- projects/hrr/playback/hip_playback.cpp | 14 +++--- projects/hrr/playback/hip_playback.h | 10 +++++ .../tests/integration/hrr_pinned_host_test.cc | 35 ++++++++++++++- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 6d5a7856d0bc..c4cef30427f0 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -198,12 +198,34 @@ static void pinned_track(void* p, size_t size) { g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); } -// Called before the memory is released, so no snapshot reads it afterwards. -static void pinned_untrack(void* p) { - if (!p || g_pinned_count.load(std::memory_order_relaxed) == 0) return; - std::lock_guard lk(g_pinned_mu); - g_pinned.erase(reinterpret_cast(p)); - g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); +// Runs the call that releases p with no snapshot of it reading meanwhile, and +// stops tracking p only if the call succeeded. +static hipError_t pinned_release(hipError_t (*release)(void*), void* p) { + std::unique_lock lk(g_pinned_mu, std::defer_lock); + if (p && g_pinned_count.load(std::memory_order_relaxed) != 0) { + lk.lock(); + if (!g_pinned.count(reinterpret_cast(p))) lk.unlock(); + } + const hipError_t r = release(p); + if (r == hipSuccess && lk.owns_lock()) { + g_pinned.erase(reinterpret_cast(p)); + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); + } + return r; +} + +// fork() takes g_pinned_mu before the writer's locks, the order a launch takes +// them in, so a child never inherits it held by a snapshot on a thread it does +// not have. Registered after writer::open() registers the writer's handlers: +// prepare handlers run in reverse order of registration. +static void pinned_install_atfork() { +#ifndef _WIN32 + static std::once_flag once; + std::call_once(once, [] { + pthread_atfork([] { g_pinned_mu.lock(); }, [] { g_pinned_mu.unlock(); }, + [] { g_pinned_mu.unlock(); }); + }); +#endif } static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { @@ -1679,8 +1701,7 @@ hipError_t capture_hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int } hipError_t capture_hipHostUnregister(void* hostPtr) { - pinned_untrack(hostPtr); - hipError_t r = g_real_table.hipHostUnregister_fn(hostPtr); + hipError_t r = pinned_release(g_real_table.hipHostUnregister_fn, hostPtr); if (r == hipSuccess) { hrr_args_hipHostUnregister a{}; a.ret = static_cast(r); @@ -1710,9 +1731,9 @@ static hipError_t pinned_hipHostMalloc(void** ptr, size_t size, unsigned int fla if (r == hipSuccess && ptr) pinned_track(*ptr, size); return r; } -static hipError_t pinned_hipHostFree(void* p) { pinned_untrack(p); return g_next_hipHostFree(p); } -static hipError_t pinned_hipFreeHost(void* p) { pinned_untrack(p); return g_next_hipFreeHost(p); } -static hipError_t pinned_hipFree(void* p) { pinned_untrack(p); return g_next_hipFree(p); } +static hipError_t pinned_hipHostFree(void* p) { return pinned_release(g_next_hipHostFree, p); } +static hipError_t pinned_hipFreeHost(void* p) { return pinned_release(g_next_hipFreeHost, p); } +static hipError_t pinned_hipFree(void* p) { return pinned_release(g_next_hipFree, p); } static void pinned_wrap_table() { static bool done = false; @@ -2929,6 +2950,7 @@ void hip_capture_init() { hip_capture_uninstall(); return; } + pinned_install_atfork(); hrr_cap::writer::set_capture_metadata_json( hrr_cap::metadata::collect_json()); diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 16a1e73f698d..45487c9f1df7 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -515,12 +515,14 @@ At playback: A kernel can read pinned host memory that the host filled with plain stores, which no HIP call records; replay would run it on whatever its fresh buffer holds. Capture tracks the ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, -`hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of +`hipFree` and `hipHostUnregister` once the call succeeds; no snapshot reads it during the +call). Before each launch it reads the 8-byte-aligned words of the arguments, by-value structs included; for each tracked allocation a word points into, it writes the whole allocation as a blob and adds a record (base, offset 0, length, blob hash, direction 0) to the launch event, unless the hash equals that of the allocation's last record. Struct words pointing into such an allocation are flagged for translation like -embedded device pointers. Replay checks each record names the base of a pinned host +embedded device pointers, and replay translates them without the device-address shape +check. Replay checks each record names the base of a pinned host allocation it made, lies inside it and matches its blob's length, refusing and counting it otherwise; then it synchronizes the launch stream, copies the blob in, and launches. The summary prints restored, refused and skipped counts. @@ -541,8 +543,7 @@ Kept deliberately simple: stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. - Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. A `fork()` while another thread snapshots can leave the - child's tracking lock held. + the buffer is not waited for. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index d98c71ddd2ad..0c7f39cb9188 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -834,11 +834,15 @@ static void decode_kernel_args( // may be a genuine scalar (a large count, a double, a packed value) // that was mis-flagged — overwriting it with null would silently // corrupt it. Only rewrite the word when it actually resolves. - auto try_translate_word = [&](size_t off, const char* src) -> bool { + auto try_translate_word = [&](size_t off, const char* src, bool flagged) -> bool { if (off + 8 > arg_size) return false; uint64_t rec_ptr; memcpy(&rec_ptr, data + off, 8); - // null, small, or a packed-integer false positive - if (!va_shaped(rec_ptr)) return false; + // null, small, or a packed-integer false positive. A word capture + // flagged inside a pinned host allocation need not look like a + // device VA: a host address can have small low 32 bits. + if (!va_shaped(rec_ptr) && + !(flagged && ctx.in_pinned_host(rec_ptr))) + return false; void* live = ctx.translate_ptr(rec_ptr); // Same bypassed-segment materialisation as a whole-pointer // argument: a struct field pointing into memory HIP never saw @@ -863,7 +867,7 @@ static void decode_kernel_args( // First honor the capture-recorded offsets (these may be unaligned). std::vector handled(arg_size, 0); for (uint16_t off : ptr_offsets) { - if (try_translate_word(off, "captured")) + if (try_translate_word(off, "captured", true)) for (int b = 0; b < 8 && static_cast(off) + b < arg_size; b++) handled[off + b] = 1; else if (ctx.verbose) @@ -886,7 +890,7 @@ static void decode_kernel_args( (std::getenv("HIP_HRR_REPLAY_NO_RESCAN") != nullptr); for (size_t off = 0; !no_rescan && off + 8 <= arg_size; ) { if (handled[off]) { off += 1; continue; } - if (try_translate_word(off, "rescan")) off += 8; + if (try_translate_word(off, "rescan", false)) off += 8; else off += 1; } } else { diff --git a/projects/hrr/playback/hip_playback.h b/projects/hrr/playback/hip_playback.h index e4eaf6c470df..7a660ff8b041 100644 --- a/projects/hrr/playback/hip_playback.h +++ b/projects/hrr/playback/hip_playback.h @@ -440,6 +440,16 @@ struct PlaybackContext { return nullptr; } + // Does v fall inside a pinned host allocation replay made? + bool in_pinned_host(uint64_t v) const { + std::shared_lock lk(map_mutex); + for (auto& [base, e] : alloc_map) + if ((e.kind == AllocKind::HostMalloc || e.kind == AllocKind::HostRegister) && + v - base < e.size) + return true; + return false; + } + // Is this value an address from the capturing process — i.e. does it fall // inside an allocation as *recorded*, before translation? Reports the // enclosing recorded base so a hit can be traced back to the event that diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index ece0aef41029..9e901901e4e2 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -17,6 +17,9 @@ #include #include +#ifndef _WIN32 +#include +#endif namespace { constexpr int kPinnedN = 256; @@ -97,6 +100,32 @@ TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostUnregister(in.data())); } +// A hipHostFree that fails, as on registered memory, leaves it tracked. +TEST_CASE("Unit_HRR_PinnedHostFailedFree_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + std::vector in(kPinnedN); + HRR_HIP_CHECK(hipHostRegister(in.data(), kPinnedN * sizeof(int), hipHostRegisterDefault)); + REQUIRE(hipHostFree(in.data()) != hipSuccess); + (void)hipGetLastError(); + hrr_pinned_workload(in.data()); + HRR_HIP_CHECK(hipHostUnregister(in.data())); +} + +#ifndef _WIN32 +// The struct case from pinned memory whose low 32 bits are zero, an address +// replay's device-pointer shape check rejects. +TEST_CASE("Unit_HRR_PinnedHostStructLowBits_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + void* at = reinterpret_cast(uintptr_t{0x7e0000000000}); + void* p = mmap(at, 1 << 16, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + REQUIRE(p == at); + HRR_HIP_CHECK(hipHostRegister(p, kPinnedN * sizeof(int), hipHostRegisterDefault)); + hrr_pinned_workload(static_cast(p), true); + HRR_HIP_CHECK(hipHostUnregister(p)); + munmap(p, 1 << 16); +} +#endif + #if defined(HRR_PLAYBACK_EXE) && defined(HRR_TEST_EXE) // Replay matches the recording only if it wrote each launch's input back; its // own buffer never sees the host's stores. With HIP_HRR_HOST_SNAPSHOTS=0 there @@ -104,10 +133,14 @@ TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", "Unit_HRR_PinnedHostRegister_Direct", + "Unit_HRR_PinnedHostFailedFree_Direct", +#ifndef _WIN32 + "Unit_HRR_PinnedHostStructLowBits_Direct", +#endif "Unit_HRR_PinnedHostStruct_Direct"}) { for (const char* snapshots : {"1", "0"}) { // Without snapshots the struct would carry a host address replay never mapped. - if (std::string(direct) == "Unit_HRR_PinnedHostStruct_Direct" && *snapshots == '0') + if (std::string(direct).find("Struct") != std::string::npos && *snapshots == '0') continue; INFO(direct << " HIP_HRR_HOST_SNAPSHOTS=" << snapshots); ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host"}; From fa1ffc5c648bef5268a2e4644dcc4b18beac1fba Mon Sep 17 00:00:00 2001 From: andraghetti Date: Tue, 13 Oct 2026 15:49:12 +0200 Subject: [PATCH 05/12] fix(hrr): keep the pinned last record out of graph captures, drop fork and failed-free handling Replay restores no pinned snapshot while a graph capture is open, but capture still made such a record the allocation's last, so the next direct launch on the same bytes got no record and replay ran it on stale memory. A record written while any stream in the process captures no longer becomes the last. The pinned workload now launches into a discarded graph capture before its third launch, and the roundtrip checks that launch is still restored. The pinned mutex no longer joins the fork handlers, and a free or unregister stops tracking the allocation before the call again, whatever it returns. DESIGN.md lists a fork during a snapshot as a known limit again. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 54 +++++++------------ projects/hrr/DESIGN.md | 9 ++-- projects/hrr/tests/config/integration.yaml | 1 - .../tests/integration/hrr_pinned_host_test.cc | 37 ++++++------- 4 files changed, 43 insertions(+), 58 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index efc760be2cf7..328b7f62a5db 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -198,34 +198,12 @@ static void pinned_track(void* p, size_t size) { g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); } -// Runs the call that releases p with no snapshot of it reading meanwhile, and -// stops tracking p only if the call succeeded. -static hipError_t pinned_release(hipError_t (*release)(void*), void* p) { - std::unique_lock lk(g_pinned_mu, std::defer_lock); - if (p && g_pinned_count.load(std::memory_order_relaxed) != 0) { - lk.lock(); - if (!g_pinned.count(reinterpret_cast(p))) lk.unlock(); - } - const hipError_t r = release(p); - if (r == hipSuccess && lk.owns_lock()) { - g_pinned.erase(reinterpret_cast(p)); - g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); - } - return r; -} - -// fork() takes g_pinned_mu before the writer's locks, the order a launch takes -// them in, so a child never inherits it held by a snapshot on a thread it does -// not have. Registered after writer::open() registers the writer's handlers: -// prepare handlers run in reverse order of registration. -static void pinned_install_atfork() { -#ifndef _WIN32 - static std::once_flag once; - std::call_once(once, [] { - pthread_atfork([] { g_pinned_mu.lock(); }, [] { g_pinned_mu.unlock(); }, - [] { g_pinned_mu.unlock(); }); - }); -#endif +// Called before the memory is released, so no snapshot reads it afterwards. +static void pinned_untrack(void* p) { + if (!p || g_pinned_count.load(std::memory_order_relaxed) == 0) return; + std::lock_guard lk(g_pinned_mu); + g_pinned.erase(reinterpret_cast(p)); + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); } static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { @@ -656,7 +634,13 @@ static void serialize_kernel_launch( return; } // A record becomes the allocation's last only once its event is written, in - // the same order as the events. + // the same order as the events. Replay restores nothing while any graph + // capture is open, so a record written then does not become the last. + bool capturing = false; + if (!t_snaps.empty()) { + amd::ScopedLock cl(hip::g_streamSetLock); + capturing = !hip::g_allCapturingStreams.empty(); + } std::unique_lock lk(g_pinned_mu, std::defer_lock); if (!t_snaps.empty()) lk.lock(); hrr_cap::writer::write_event_raw(api_id, @@ -664,7 +648,7 @@ static void serialize_kernel_launch( static_cast(payload.size())); for (const HostSnap& s : t_snaps) { auto it = g_pinned.find(s.base); - if (s.record && it != g_pinned.end()) it->second.last = s.hash; + if (s.record && !capturing && it != g_pinned.end()) it->second.last = s.hash; } } @@ -1701,7 +1685,8 @@ hipError_t capture_hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int } hipError_t capture_hipHostUnregister(void* hostPtr) { - hipError_t r = pinned_release(g_real_table.hipHostUnregister_fn, hostPtr); + pinned_untrack(hostPtr); + hipError_t r = g_real_table.hipHostUnregister_fn(hostPtr); if (r == hipSuccess) { hrr_args_hipHostUnregister a{}; a.ret = static_cast(r); @@ -1731,9 +1716,9 @@ static hipError_t pinned_hipHostMalloc(void** ptr, size_t size, unsigned int fla if (r == hipSuccess && ptr) pinned_track(*ptr, size); return r; } -static hipError_t pinned_hipHostFree(void* p) { return pinned_release(g_next_hipHostFree, p); } -static hipError_t pinned_hipFreeHost(void* p) { return pinned_release(g_next_hipFreeHost, p); } -static hipError_t pinned_hipFree(void* p) { return pinned_release(g_next_hipFree, p); } +static hipError_t pinned_hipHostFree(void* p) { pinned_untrack(p); return g_next_hipHostFree(p); } +static hipError_t pinned_hipFreeHost(void* p) { pinned_untrack(p); return g_next_hipFreeHost(p); } +static hipError_t pinned_hipFree(void* p) { pinned_untrack(p); return g_next_hipFree(p); } static void pinned_wrap_table() { static bool done = false; @@ -2950,7 +2935,6 @@ void hip_capture_init() { hip_capture_uninstall(); return; } - pinned_install_atfork(); hrr_cap::writer::set_capture_metadata_json( hrr_cap::metadata::collect_json()); diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 4bd8801b031e..80538accca42 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -525,8 +525,7 @@ At playback: A kernel can read pinned host memory that the host filled with plain stores, which no HIP call records; replay would run it on whatever its fresh buffer holds. Capture tracks the ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, -`hipFree` and `hipHostUnregister` once the call succeeds; no snapshot reads it during the -call). Before each launch it reads the 8-byte-aligned words of +`hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of the arguments, by-value structs included; for each tracked allocation a word points into, it writes the whole allocation as a blob and adds a record (base, offset 0, length, blob hash, direction 0) to the launch event, unless the hash equals that of the allocation's @@ -545,7 +544,8 @@ Kept deliberately simple: `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under `unreplayable_apis`. - Replay restores nothing for a launch replayed into a graph capture, and capture does not - snapshot at graph launch. + snapshot at graph launch. A record written while any stream captures does not become the + allocation's last, so the next launch outside the capture records again. - `hipHostAlloc`, `hipMallocHost` and `hipMemAllocHost` are not tracked, since replay makes no buffer for them, nor is a `hipHostRegister` device alias that differs from the host address. @@ -553,7 +553,8 @@ Kept deliberately simple: stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. - Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. + the buffer is not waited for. A `fork()` while another thread snapshots can leave the + child's tracking lock held. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index b2c063c7e65c..27cba3f7ccd7 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -208,7 +208,6 @@ cases: Unit_HRR_Occupancy_Direct: {} Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} - Unit_HRR_PinnedHostFailedFree_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} Unit_HRR_PinnedHostFreed_Direct: {} Unit_HRR_PinnedHostMalloc_Direct: {} diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index 9e901901e4e2..1c8ebd528ff8 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -48,15 +48,27 @@ static void hrr_pinned_workload(int* in, bool by_struct = false) { int* dout = nullptr; HRR_HIP_CHECK(hipMalloc(&dout, kPinnedN * sizeof(int))); std::vector hout(kPinnedN); - for (int it = 0; it < kPinnedIters; ++it) { - for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores + auto launch = [&](hipStream_t s) { if (by_struct) - hipLaunchKernelGGL(hrr_pinned_triple_struct, dim3(1), dim3(kPinnedN), 0, nullptr, + hipLaunchKernelGGL(hrr_pinned_triple_struct, dim3(1), dim3(kPinnedN), 0, s, dout, PinnedIn{dev_in, kPinnedN}); else - hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, nullptr, - dout, dev_in, kPinnedN); + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, s, dout, dev_in, kPinnedN); HRR_HIP_CHECK(hipGetLastError()); + }; + for (int it = 0; it < kPinnedIters; ++it) { + for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores + if (it == 2) { // replay restores nothing in a graph capture; the next launch still must + hipStream_t s = nullptr; + hipGraph_t g = nullptr; + HRR_HIP_CHECK(hipStreamCreate(&s)); + HRR_HIP_CHECK(hipStreamBeginCapture(s, hipStreamCaptureModeGlobal)); + launch(s); + HRR_HIP_CHECK(hipStreamEndCapture(s, &g)); + HRR_HIP_CHECK(hipGraphDestroy(g)); + HRR_HIP_CHECK(hipStreamDestroy(s)); + } + launch(nullptr); HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, kPinnedN * sizeof(int), hipMemcpyDeviceToHost)); for (int i = 0; i < kPinnedN; ++i) REQUIRE(hout[i] == pinned_value(it, i) * 3 + 1); } @@ -100,17 +112,6 @@ TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostUnregister(in.data())); } -// A hipHostFree that fails, as on registered memory, leaves it tracked. -TEST_CASE("Unit_HRR_PinnedHostFailedFree_Direct", "[.][hrr-direct]") { - HRR_HIP_CHECK(hipSetDevice(0)); - std::vector in(kPinnedN); - HRR_HIP_CHECK(hipHostRegister(in.data(), kPinnedN * sizeof(int), hipHostRegisterDefault)); - REQUIRE(hipHostFree(in.data()) != hipSuccess); - (void)hipGetLastError(); - hrr_pinned_workload(in.data()); - HRR_HIP_CHECK(hipHostUnregister(in.data())); -} - #ifndef _WIN32 // The struct case from pinned memory whose low 32 bits are zero, an address // replay's device-pointer shape check rejects. @@ -133,7 +134,6 @@ TEST_CASE("Unit_HRR_PinnedHostStructLowBits_Direct", "[.][hrr-direct]") { HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", "Unit_HRR_PinnedHostRegister_Direct", - "Unit_HRR_PinnedHostFailedFree_Direct", #ifndef _WIN32 "Unit_HRR_PinnedHostStructLowBits_Direct", #endif @@ -153,7 +153,8 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { CHECK(ret == 0); CHECK(pass == kPinnedIters); CHECK(fail == 0); - CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused") != std::string::npos); + CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused, 1 skipped") != + std::string::npos); } else { CHECK(ret != 0); CHECK(fail > 0); From aa91d3ffedb8217e62e8ab38c268066ba08b6b48 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Wed, 14 Oct 2026 14:04:20 +0200 Subject: [PATCH 06/12] docs(hrr): drop the lock-order note for the removed pinned fork handler --- projects/clr/hipamd/src/hrr/hip_capture_writer.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp index d27677e709ac..7598e202ef0f 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp @@ -172,10 +172,8 @@ static constexpr size_t kPathMax = 4096; static constexpr size_t kMetadataJsonMax = 128u * 1024u; static constexpr size_t kEmergencyManifestMax = kMetadataJsonMax + 1024u; -// Lock order: g_pinned_mu (hip_capture.cpp, held while a launch snapshots -// pinned host memory), then g_reopen_mu, then g_file_mu, then g_blob_mu, then -// g_unreplayable_mu. Only fork() holds them all: the pinned prepare handler -// takes g_pinned_mu, then atfork_prepare takes the rest. +// Lock order: g_reopen_mu, then g_file_mu, then g_blob_mu, then +// g_unreplayable_mu. atfork_prepare is the only path that holds them all. static std::mutex g_file_mu; static int g_events_fd = -1; // g_base_dir is the archive path requested via HIP_HRR_CAPTURE_OUTPUT. From 9e2389599815af6723ae6328b31d53414bbaed3e Mon Sep 17 00:00:00 2001 From: andraghetti Date: Thu, 15 Oct 2026 16:50:04 +0200 Subject: [PATCH 07/12] fix(hrr): copy pinned host memory before hashing it, hold its lock across fork A pinned snapshot hashed the application's memory and then read it again to write the blob, so a host thread or queued GPU work could change the bytes in between. The blob was then filed under a hash that did not name its contents, and the blob store's dedup served those bytes for any later blob with that hash. The snapshot now copies each allocation into a buffer under the tracking lock, releases the lock, and hashes and writes the copy. The buffer is allocated per snapshot and freed once written, before the launch; a failed allocation is listed under unreplayable_apis like an oversized one. The tracking lock is still held while a launch writes its event, so a fork() from another thread can still leave a child with it locked. Its fork handler is back: fa1ffc5c64 dropped it only to keep this change small, not over lock order. It is registered after the writer's handlers, so fork takes the pinned lock first, then the writer's, the order a launch takes them in. Unit_HRR_PinnedHostFork forks while another thread snapshots a 32 MiB allocation; each child takes the lock through a failing hipFree, and one that does not exit within 10 s fails the case. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 44 ++++++++++-- .../clr/hipamd/src/hrr/hip_capture_writer.cpp | 6 +- projects/hrr/DESIGN.md | 13 ++-- projects/hrr/tests/config/integration.yaml | 2 + .../tests/integration/hrr_pinned_host_test.cc | 67 +++++++++++++++++++ 5 files changed, 117 insertions(+), 15 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 328b7f62a5db..dad008ca3d22 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -181,10 +181,11 @@ struct PinnedAlloc { }; struct HostSnap { uint64_t base = 0, size = 0; - hrr_cap::Hash128 hash{}; + hrr_cap::Hash128 hash{}, last{}; bool record = false; // contents differ from the last record + std::unique_ptr copy; // the bytes, until hashed and written }; -std::mutex g_pinned_mu; // also held while a snapshot reads +std::mutex g_pinned_mu; // also held while a snapshot copies std::map g_pinned; std::atomic g_pinned_count{0}; // The pinned allocations the launch this thread is making points into. @@ -206,6 +207,20 @@ static void pinned_untrack(void* p) { g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); } +// fork() takes g_pinned_mu before the writer's locks, the order a launch takes +// them in, so a child never inherits it held by a snapshot on a thread it does +// not have. Registered after writer::open() registers the writer's handlers: +// prepare handlers run in reverse order of registration. +static void pinned_install_atfork() { +#ifndef _WIN32 + static std::once_flag once; + std::call_once(once, [] { + pthread_atfork([] { g_pinned_mu.lock(); }, [] { g_pinned_mu.unlock(); }, + [] { g_pinned_mu.unlock(); }); + }); +#endif +} + static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { const size_t cap = static_cast(HIP_HRR_HOST_SNAPSHOT_MAX_MB) << 20; for (size_t j = 0; bytes && j + sizeof(uint64_t) <= sz; j += sizeof(uint64_t)) { @@ -226,11 +241,18 @@ static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { "pinned host snapshot", "a pinned allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB was not recorded"); } else { - s.hash = hrr_cap::writer::write_blob(reinterpret_cast(s.base), s.size); - PinnedAlloc& a = it->second; - s.record = (s.hash.lo || s.hash.hi) && (s.hash.lo != a.last.lo || s.hash.hi != a.last.hi); + // Hashed and written from the copy, after the lock, so the blob holds + // the bytes its hash names. + s.copy.reset(new (std::nothrow) uint8_t[s.size]); + if (s.copy) { + std::memcpy(s.copy.get(), reinterpret_cast(s.base), s.size); + s.last = it->second.last; + } else { + hrr_cap::writer::note_unreplayable("pinned host snapshot", + "no memory to copy a pinned allocation"); + } } - t_snaps.push_back(s); + t_snaps.push_back(std::move(s)); } } @@ -244,7 +266,7 @@ struct PinnedSnapshot { return; if (!params && !kbuf && extra) parse_kernel_extra(extra, kbuf, ksz); const amd::KernelSignature& sig = hip::asKernel(f)->signature(); - std::lock_guard lk(g_pinned_mu); + std::unique_lock lk(g_pinned_mu); uint32_t param_idx = 0; for (uint32_t i = 0; i < sig.numParametersAll(); i++) { const auto& d = sig.at(i); @@ -257,6 +279,13 @@ struct PinnedSnapshot { param_idx++; if (d.offset_ % sizeof(uint64_t) == 0) pinned_scan_locked(bytes, d.size_); } + lk.unlock(); + for (HostSnap& s : t_snaps) { + if (!s.copy) continue; + s.hash = hrr_cap::writer::write_blob(s.copy.get(), s.size); + s.copy.reset(); + s.record = (s.hash.lo || s.hash.hi) && (s.hash.lo != s.last.lo || s.hash.hi != s.last.hi); + } } ~PinnedSnapshot() { t_snaps.clear(); } }; @@ -2935,6 +2964,7 @@ void hip_capture_init() { hip_capture_uninstall(); return; } + pinned_install_atfork(); hrr_cap::writer::set_capture_metadata_json( hrr_cap::metadata::collect_json()); diff --git a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp index 7598e202ef0f..1dd2de9a9622 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp @@ -172,8 +172,10 @@ static constexpr size_t kPathMax = 4096; static constexpr size_t kMetadataJsonMax = 128u * 1024u; static constexpr size_t kEmergencyManifestMax = kMetadataJsonMax + 1024u; -// Lock order: g_reopen_mu, then g_file_mu, then g_blob_mu, then -// g_unreplayable_mu. atfork_prepare is the only path that holds them all. +// Lock order: g_pinned_mu (hip_capture.cpp, held while a launch copies pinned +// host memory or writes its event), then g_reopen_mu, then g_file_mu, then +// g_blob_mu, then g_unreplayable_mu. Only fork() holds them all: the pinned +// prepare handler takes g_pinned_mu, then atfork_prepare takes the rest. static std::mutex g_file_mu; static int g_events_fd = -1; // g_base_dir is the archive path requested via HIP_HRR_CAPTURE_OUTPUT. diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 81cf6e3da67a..47e2bdce9a28 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -541,9 +541,10 @@ call records; replay would run it on whatever its fresh buffer holds. Capture tr ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, `hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of the arguments, by-value structs included; for each tracked allocation a word points into, -it writes the whole allocation as a blob and adds a record (base, offset 0, length, blob -hash, direction 0) to the launch event, unless the hash equals that of the allocation's -last record. Struct words pointing into such an allocation are flagged for translation like +it copies the whole allocation under the tracking lock, writes the copy as a blob and adds a +record (base, offset 0, length, blob hash, direction 0) to the launch event, unless the hash +equals that of the allocation's last record. Struct words pointing into such an allocation +are flagged for translation like embedded device pointers, and replay translates them without the device-address shape check. Replay checks each record names the base of a pinned host allocation it made, lies inside it and matches its blob's length, refusing and counting it @@ -553,7 +554,8 @@ summary prints restored, refused and skipped counts. Kept deliberately simple: - Capture does not wait for the stream: the bytes are those at enqueue time, which is what - the kernel reads unless earlier queued work still changes them. + the kernel reads unless earlier queued work still changes them. If that work is writing + the allocation during the copy, replay restores the partial bytes over its output. - `HIP_HRR_HOST_SNAPSHOTS=0` turns snapshots off; an allocation larger than `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under `unreplayable_apis`. @@ -567,8 +569,7 @@ Kept deliberately simple: stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. - Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. A `fork()` while another thread snapshots can leave the - child's tracking lock held. + the buffer is not waited for. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 2a4df73c41bb..96462b5721c7 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -209,6 +209,8 @@ cases: Unit_HRR_Occupancy_Direct: {} Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} + Unit_HRR_PinnedHostFork: {} + Unit_HRR_PinnedHostFork_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} Unit_HRR_PinnedHostFreed_Direct: {} Unit_HRR_PinnedHostMalloc_Direct: {} diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index 1c8ebd528ff8..cf0f3f0b38c9 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -19,6 +19,13 @@ #include #ifndef _WIN32 #include +#include +#include + +#include +#include +#include +#include #endif namespace { @@ -125,6 +132,53 @@ TEST_CASE("Unit_HRR_PinnedHostStructLowBits_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostUnregister(p)); munmap(p, 1 << 16); } + +// Forks while another thread snapshots a 32 MiB pinned allocation, so most forks +// land while it holds the pinned tracking lock. Each child frees an address +// capture does not track, which takes that lock and fails without device work. +// A child that inherited the lock held never exits. +TEST_CASE("Unit_HRR_PinnedHostFork_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + constexpr size_t kBig = 32u << 20; + void* p = nullptr; + HRR_HIP_CHECK(hipHostMalloc(&p, kBig, 0)); + std::memset(p, 1, kBig); + // Nothing may REQUIRE before the join. + std::atomic stop{false}; + std::thread launcher([&] { + while (!stop.load()) { + hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, + reinterpret_cast(p)); + (void)hipStreamSynchronize(nullptr); + } + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + int hung = 0, failed = 0; + for (int i = 0; i < 20 && !hung; ++i) { + const pid_t pid = fork(); + if (pid == 0) { + int x = 0; + _exit(hipFree(&x) == hipSuccess); + } + int status = 0; + pid_t got = 0; + for (int ms = 0; pid > 0 && ms < 10000 && (got = waitpid(pid, &status, WNOHANG)) == 0; ++ms) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (pid > 0 && got == 0) { + kill(pid, SIGKILL); + waitpid(pid, &status, 0); + ++hung; + } else if (got != pid || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + ++failed; + } + } + stop = true; + launcher.join(); + HRR_HIP_CHECK(hipHostFree(p)); + INFO("hung " << hung << ", failed " << failed); + REQUIRE(hung == 0); + REQUIRE(failed == 0); +} #endif #if defined(HRR_PLAYBACK_EXE) && defined(HRR_TEST_EXE) @@ -173,6 +227,19 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { CHECK(ret == 0); CHECK(out.find("Pinned host snapshots") == std::string::npos); } + +#ifndef _WIN32 +// The pinned tracking lock is held across fork(), so no child inherits it locked. +HRR_TEST_CASE(Unit_HRR_PinnedHostFork) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_fork"}; + hrr::test::SpawnProc proc(hrr_self_exe()); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.runWithTimeout("\"Unit_HRR_PinnedHostFork_Direct\"", 300); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); +} +#endif #endif // HRR_PLAYBACK_EXE && HRR_TEST_EXE /** From 499c82b63d4cb516aeee1004f9c316a72c3e5399 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Thu, 15 Oct 2026 16:50:59 +0200 Subject: [PATCH 08/12] fix(hrr): restore pinned snapshots by the launch stream's capture state Replay skipped every pinned restore while any graph capture was open, and capture kept every record made meanwhile from becoming the allocation's last. A direct launch on a stream that was not capturing lost its restore. Both sides now look at the launch stream only: replay asks hipStreamIsCapturing, capture reads the stream's capture status. The pinned workload launches on a second stream during its graph capture; replay must restore that launch, and the next launch then has nothing new to record. Unit_HRR_PinnedHostStructLowBits_Direct maps its fixed address with MAP_FIXED_NOREPLACE and skips when the range is taken, or when an older kernel treats the address as a hint and maps elsewhere. It runs last in Unit_HRR_PinnedHostRoundtrip, since a skip ends that case. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 13 ++++----- projects/hrr/DESIGN.md | 6 ++-- projects/hrr/playback/hip_playback.cpp | 7 +++-- .../tests/integration/hrr_pinned_host_test.cc | 28 +++++++++++++++---- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index dad008ca3d22..2fb7d177d6ab 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -663,13 +663,12 @@ static void serialize_kernel_launch( return; } // A record becomes the allocation's last only once its event is written, in - // the same order as the events. Replay restores nothing while any graph - // capture is open, so a record written then does not become the last. - bool capturing = false; - if (!t_snaps.empty()) { - amd::ScopedLock cl(hip::g_streamSetLock); - capturing = !hip::g_allCapturingStreams.empty(); - } + // the same order as the events. Replay restores nothing for a launch into a + // stream capture, so a record written for one does not become the last. + hipStream_t cs = stream; + hip::getStreamPerThread(cs); + const bool capturing = cs && cs != hipStreamLegacy && + reinterpret_cast(cs)->GetCaptureStatus() != hipStreamCaptureStatusNone; std::unique_lock lk(g_pinned_mu, std::defer_lock); if (!t_snaps.empty()) lk.lock(); hrr_cap::writer::write_event_raw(api_id, diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 47e2bdce9a28..80a641ef64dd 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -559,9 +559,9 @@ Kept deliberately simple: - `HIP_HRR_HOST_SNAPSHOTS=0` turns snapshots off; an allocation larger than `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under `unreplayable_apis`. -- Replay restores nothing for a launch replayed into a graph capture, and capture does not - snapshot at graph launch. A record written while any stream captures does not become the - allocation's last, so the next launch outside the capture records again. +- Replay restores nothing for a launch replayed into a capturing stream, and capture does + not snapshot at graph launch. A record for a launch into a capturing stream does not + become the allocation's last, so the next launch outside the capture records again. - `hipHostAlloc`, `hipMallocHost` and `hipMemAllocHost` are not tracked, since replay makes no buffer for them, nor is a `hipHostRegister` device alias that differs from the host address. diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index 34055dfc362b..dcf93b861228 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -458,12 +458,13 @@ hipFunction_t PlaybackContext::resolve_replacement(const std::string& kernel_nam // u32 num_attrs, u32 attr stride, num_attrs * stride bytes // Write a launch's pinned host snapshots back before it runs, once the work -// already queued on its stream is done. Not inside a graph capture, where the -// kernel runs only when the graph is launched. +// already queued on its stream is done. Not when that stream captures into a +// graph, where the kernel runs only when the graph is launched. static void restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, uint16_t n, hipStream_t stream, const std::string& kernel_name) { - if (ctx.in_graph_capture) { + hipStreamCaptureStatus cs = hipStreamCaptureStatusNone; + if (hipStreamIsCapturing(stream, &cs) != hipSuccess || cs != hipStreamCaptureStatusNone) { ctx.host_snapshots_skipped += n; return; } diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index cf0f3f0b38c9..e06b98f85204 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -23,9 +23,13 @@ #include #include +#include #include #include #include +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE 0x100000 +#endif #endif namespace { @@ -65,15 +69,20 @@ static void hrr_pinned_workload(int* in, bool by_struct = false) { }; for (int it = 0; it < kPinnedIters; ++it) { for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores - if (it == 2) { // replay restores nothing in a graph capture; the next launch still must - hipStream_t s = nullptr; + // Replay restores nothing in a graph capture. A launch on another stream + // meanwhile is restored, and leaves the next launch nothing to record. + if (it == 2) { + hipStream_t s = nullptr, s2 = nullptr; hipGraph_t g = nullptr; HRR_HIP_CHECK(hipStreamCreate(&s)); + HRR_HIP_CHECK(hipStreamCreate(&s2)); HRR_HIP_CHECK(hipStreamBeginCapture(s, hipStreamCaptureModeGlobal)); launch(s); + launch(s2); HRR_HIP_CHECK(hipStreamEndCapture(s, &g)); HRR_HIP_CHECK(hipGraphDestroy(g)); HRR_HIP_CHECK(hipStreamDestroy(s)); + HRR_HIP_CHECK(hipStreamDestroy(s2)); } launch(nullptr); HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, kPinnedN * sizeof(int), hipMemcpyDeviceToHost)); @@ -125,8 +134,15 @@ TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { TEST_CASE("Unit_HRR_PinnedHostStructLowBits_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); void* at = reinterpret_cast(uintptr_t{0x7e0000000000}); - void* p = mmap(at, 1 << 16, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - REQUIRE(p == at); + void* p = mmap(at, 1 << 16, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p != at) { + const int err = errno; + // A kernel older than 4.17 takes the address as a hint only. + if (p != MAP_FAILED) munmap(p, 1 << 16); + if (p != MAP_FAILED || err == EEXIST) HRR_SKIP("0x7e0000000000 is already mapped"); + FAIL("mmap: " << std::strerror(err)); + } HRR_HIP_CHECK(hipHostRegister(p, kPinnedN * sizeof(int), hipHostRegisterDefault)); hrr_pinned_workload(static_cast(p), true); HRR_HIP_CHECK(hipHostUnregister(p)); @@ -188,10 +204,12 @@ TEST_CASE("Unit_HRR_PinnedHostFork_Direct", "[.][hrr-direct]") { HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", "Unit_HRR_PinnedHostRegister_Direct", + "Unit_HRR_PinnedHostStruct_Direct", #ifndef _WIN32 + // Last, since a skip ends the case. "Unit_HRR_PinnedHostStructLowBits_Direct", #endif - "Unit_HRR_PinnedHostStruct_Direct"}) { + }) { for (const char* snapshots : {"1", "0"}) { // Without snapshots the struct would carry a host address replay never mapped. if (std::string(direct).find("Struct") != std::string::npos && *snapshots == '0') From 69d5514e36bcb2f06b23a8a0e4540206e1f9c17d Mon Sep 17 00:00:00 2001 From: andraghetti Date: Thu, 15 Oct 2026 18:33:20 +0200 Subject: [PATCH 09/12] fix(hrr): sync a pinned restore's stream without invalidating a capture Replay synchronizes the launch stream before it writes a pinned snapshot back. When another stream was capturing in global mode, that sync invalidated the capture, and its hipStreamEndCapture failed with 901. The pinned workload hits this: it launches on a second stream while its graph capture is open, and Unit_HRR_PinnedHostRoundtrip failed on MI350. The launch stream is not capturing, so waiting for it is safe. Replay now switches the thread to relaxed capture mode for that one sync and puts the old mode back. --- projects/clr/rocclr/utils/flags.hpp | 5 ----- projects/hrr/DESIGN.md | 3 ++- projects/hrr/playback/hip_playback.cpp | 10 +++++++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp index ce588742a799..f8cad268285c 100644 --- a/projects/clr/rocclr/utils/flags.hpp +++ b/projects/clr/rocclr/utils/flags.hpp @@ -278,11 +278,6 @@ release(cstring, HIP_HRR_CAPTURE_OUTPUT, "", \ release(bool, HIP_HRR_DEBUG_ARGS, false, \ "Enable HRR capture argument-provenance tracing (per-arg dumps and " \ "H2D destination logging) at LOG_INFO level") \ -release(bool, HIP_HRR_HOST_SNAPSHOTS, true, \ - "HRR capture: record the pinned host memory a kernel reads, before " \ - "each launch") \ -release(uint, HIP_HRR_HOST_SNAPSHOT_MAX_MB, 64, \ - "HRR capture: largest pinned host allocation snapshotted, in MiB") \ release(uint, DEBUG_CLR_DOORBELL_SKIP, 16, \ "Number of consecutive dispatches that may skip the doorbell flush.") \ release(bool, DEBUG_CLR_DISABLE_FALLBACK, false, \ diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index e33de11d853c..7af4fe8962dd 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -572,7 +572,8 @@ Kept deliberately simple: stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. - Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. + the buffer is not waited for. The sync runs in relaxed capture mode, so a global-mode + capture open on another stream is not invalidated by it. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index 01d8930f02b7..8578bf74279d 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -501,7 +501,15 @@ static void restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, compact_kernel_name(kernel_name).c_str(), i, why); continue; } - if (!synced) (void)HRR_HIP_CHECK(hipStreamSynchronize(stream)); + if (!synced) { + // In global mode a sync during another stream's capture + // invalidates that capture. This stream is not capturing, so + // waiting for it is safe: relax the mode for the call only. + hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed; + (void)hipThreadExchangeStreamCaptureMode(&mode); + (void)HRR_HIP_CHECK(hipStreamSynchronize(stream)); + (void)hipThreadExchangeStreamCaptureMode(&mode); + } synced = true; uint64_t len; memcpy(&len, rec + 16, 8); From 9ddccdde1bdfaddc79432c6a34078c8cc9cac985 Mon Sep 17 00:00:00 2001 From: andraghetti Date: Thu, 15 Oct 2026 18:33:26 +0200 Subject: [PATCH 10/12] refactor(hrr): read the pinned snapshot settings in the HRR capture code HIP_HRR_HOST_SNAPSHOTS and HIP_HRR_HOST_SNAPSHOT_MAX_MB were release flags in rocclr/utils/flags.hpp. They only tune capture, so they now live in hip_capture.cpp, next to the code that uses them, and flags.hpp is back to develop's. The names, defaults and parsing are unchanged: the snapshot switch is on unless set to something other than "true" or a non-zero number, and the cap is 64 MiB unless set. Each is read once, since every launch asks. Unit_HRR_PinnedHostSnapshotCap captures a 2 MiB pinned allocation with the cap at 2 and at 1 MiB. Within the cap, replay restores it and matches. Over it, the manifest lists it as unreplayable, replay restores nothing and the device-to-host checks fail. The case set in integration.yaml gains it and Unit_HRR_PinnedHostLarge_Direct. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 34 ++++++++++++--- projects/hrr/tests/config/integration.yaml | 2 + .../tests/integration/hrr_pinned_host_test.cc | 41 +++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 2fb7d177d6ab..4dfa3b1aee64 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -130,8 +130,11 @@ const char* hip_capture_output_dir() { // HIP_HRR_DEBUG_ARGS also enables provenance tracing: every H2D memcpy // destination is logged so a kernel pointer arg can be matched to the copy that -// filled it. Routed through the CLR flags registry (flags.hpp) like every other -// HRR flag, so it is discoverable and honors AMD_LOG_LEVEL log routing. +// filled it. Routed through the CLR flags registry (flags.hpp), so it is +// discoverable and honors AMD_LOG_LEVEL log routing. HIP_HRR_HOST_SNAPSHOTS and +// HIP_HRR_HOST_SNAPSHOT_MAX_MB are the exception: they only tune capture, so they +// are read here (hrr_host_snapshots_enabled, hrr_host_snapshot_cap_bytes) and +// stay with the HRR code and its reviewers rather than in flags.hpp. static bool hrr_dbg_args_enabled() { return HIP_HRR_DEBUG_ARGS; } @@ -221,8 +224,29 @@ static void pinned_install_atfork() { #endif } +// HIP_HRR_HOST_SNAPSHOTS, on unless set to something other than "true" or a +// non-zero number, and HIP_HRR_HOST_SNAPSHOT_MAX_MB, 64 unless set. Parsed as +// the CLR flags registry parses a bool and a uint, and read once: every launch +// asks. +static bool hrr_host_snapshots_enabled() { + static const bool on = [] { + const char* v = std::getenv("HIP_HRR_HOST_SNAPSHOTS"); + return !v || std::strcmp(v, "true") == 0 || std::atoi(v) != 0; + }(); + return on; +} + +static size_t hrr_host_snapshot_cap_bytes() { + static const size_t cap = [] { + const char* v = std::getenv("HIP_HRR_HOST_SNAPSHOT_MAX_MB"); + const unsigned int mb = v ? static_cast(std::atoi(v)) : 64u; + return static_cast(mb) << 20; + }(); + return cap; +} + static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { - const size_t cap = static_cast(HIP_HRR_HOST_SNAPSHOT_MAX_MB) << 20; + const size_t cap = hrr_host_snapshot_cap_bytes(); for (size_t j = 0; bytes && j + sizeof(uint64_t) <= sz; j += sizeof(uint64_t)) { uint64_t v; std::memcpy(&v, bytes + j, sizeof(v)); @@ -262,7 +286,7 @@ struct PinnedSnapshot { PinnedSnapshot(hipFunction_t f, void** params, void** extra, const void* kbuf = nullptr, size_t ksz = 0) { t_snaps.clear(); - if (!f || !HIP_HRR_HOST_SNAPSHOTS || g_pinned_count.load(std::memory_order_relaxed) == 0) + if (!f || !hrr_host_snapshots_enabled() || g_pinned_count.load(std::memory_order_relaxed) == 0) return; if (!params && !kbuf && extra) parse_kernel_extra(extra, kbuf, ksz); const amd::KernelSignature& sig = hip::asKernel(f)->signature(); @@ -293,7 +317,7 @@ struct PinnedSnapshot { // The kernel behind a host stub, looked up before the launch only when there // is pinned memory to snapshot, without changing the application's last error. static hipFunction_t pinned_stub_fn(const void* stub) { - if (!HIP_HRR_HOST_SNAPSHOTS || g_pinned_count.load(std::memory_order_relaxed) == 0 || + if (!hrr_host_snapshots_enabled() || g_pinned_count.load(std::memory_order_relaxed) == 0 || !g_real_table.hipGetFuncBySymbol_fn) return nullptr; const hipError_t saved_cmd = hip::tls.last_command_error_; diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 3cfdc171368b..924c9bc0eefd 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -214,9 +214,11 @@ cases: Unit_HRR_PinnedHostFork_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} Unit_HRR_PinnedHostFreed_Direct: {} + Unit_HRR_PinnedHostLarge_Direct: {} Unit_HRR_PinnedHostMalloc_Direct: {} Unit_HRR_PinnedHostRegister_Direct: {} Unit_HRR_PinnedHostRoundtrip: {} + Unit_HRR_PinnedHostSnapshotCap: {} Unit_HRR_PinnedHostStructLowBits_Direct: {} Unit_HRR_PinnedHostStruct_Direct: {} Unit_HRR_PitchedD2HBlobEdits: {} diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index e06b98f85204..b459eedb113a 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -107,6 +107,15 @@ TEST_CASE("Unit_HRR_PinnedHostStruct_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostFree(in)); } +// A 2 MiB allocation: over HIP_HRR_HOST_SNAPSHOT_MAX_MB=1, within 2. +TEST_CASE("Unit_HRR_PinnedHostLarge_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), size_t{2} << 20, 0)); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipHostFree(in)); +} + // A freed allocation's address passed on as a plain number: capture must not // snapshot memory that is gone. TEST_CASE("Unit_HRR_PinnedHostFreed_Direct", "[.][hrr-direct]") { @@ -235,6 +244,38 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { } } +// An allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB is not snapshotted and +// is listed as unreplayable, so replay cannot match. Within the cap it is. +HRR_TEST_CASE(Unit_HRR_PinnedHostSnapshotCap) { + for (const char* max_mb : {"2", "1"}) { + const bool over = *max_mb == '1'; + INFO("HIP_HRR_HOST_SNAPSHOT_MAX_MB=" << max_mb); + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_cap"}; + hrr_capture_direct("Unit_HRR_PinnedHostLarge_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOT_MAX_MB", max_mb}}); + const std::string manifest = + read_text_file(hrr_single_process_archive(cap.path) / "manifest.json"); + INFO("manifest.json:\n" << manifest); + CHECK((manifest.find("larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB") != std::string::npos) == + over); + auto [ret, out] = hrr_playback_env(cap.path, {{"HIP_HRR_D2H_EXACT", "1"}}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + int pass = 0, fail = 0; + REQUIRE(hrr_parse_d2h_summary(out, pass, fail)); + if (over) { + CHECK(ret != 0); + CHECK(fail > 0); + CHECK(out.find("Pinned host snapshots") == std::string::npos); + } else { + CHECK(ret == 0); + CHECK(pass == kPinnedIters); + CHECK(fail == 0); + CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused, 1 skipped") != + std::string::npos); + } + } +} + // Capture stops tracking an allocation when it is freed: had it snapshotted the // stale address, replay would refuse the record for an allocation it freed. HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { From d6b99b1ccc6a2ada1ac5242c29076977f5225c2c Mon Sep 17 00:00:00 2001 From: andraghetti Date: Mon, 19 Oct 2026 09:40:40 +0200 Subject: [PATCH 11/12] fix(hrr): queue pinned restores on the launch stream instead of waiting for it Replay copied a launch's pinned snapshots in after hipStreamSynchronize, which hangs when the stream waits on work the replay thread submits later, and left the copy to race a free on another replay thread. The copy is now a host function queued ahead of the kernel; hipHostFree and hipHostUnregister wait for every stream. A blob the host memory cap refuses or a failed enqueue ends the launch before the kernel runs. Since a restore is ordered on its own stream only, capture records again for a launch on another stream than the allocation's last record. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 21 +++-- projects/hrr/DESIGN.md | 13 ++-- projects/hrr/playback/hip_playback.cpp | 61 +++++++++------ projects/hrr/tests/config/integration.yaml | 3 + .../tests/integration/hrr_pinned_host_test.cc | 77 ++++++++++++++++++- 5 files changed, 137 insertions(+), 38 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 07dd8a85a37b..9b915bee068f 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -179,8 +179,9 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // no HIP call records. Before each launch, the 8-byte-aligned words of its // arguments are looked up in the tracked pinned allocations (hipHostMalloc, // hipHostRegister). Each allocation hit is written whole as a -// blob, and named by a record in the launch event unless its hash equals that -// of the last record for it. Replay copies the blob back before the launch. +// blob, and named by a record in the launch event unless its hash and stream +// equal those of the last record for it. Replay copies the blob back when the +// stream reaches the launch. // See DESIGN.md, "Pinned host snapshots". // --------------------------------------------------------------------------- @@ -188,11 +189,13 @@ namespace { struct PinnedAlloc { size_t size = 0; hrr_cap::Hash128 last{}; // blob of the last record, {0,0} before the first + hipStream_t last_stream = nullptr; // and its launch stream }; struct HostSnap { uint64_t base = 0, size = 0; hrr_cap::Hash128 hash{}, last{}; - bool record = false; // contents differ from the last record + hipStream_t last_stream = nullptr; + bool record = false; // contents or stream differ from the last record std::unique_ptr copy; // the bytes, until hashed and written }; std::mutex g_pinned_mu; // also held while a snapshot copies @@ -277,7 +280,8 @@ static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { s.copy.reset(new (std::nothrow) uint8_t[s.size]); if (s.copy) { std::memcpy(s.copy.get(), reinterpret_cast(s.base), s.size); - s.last = it->second.last; + s.last = it->second.last; + s.last_stream = it->second.last_stream; } else { hrr_cap::writer::note_unreplayable("pinned host snapshot", "no memory to copy a pinned allocation"); @@ -557,6 +561,10 @@ static void serialize_kernel_launch( if (!sig.at(i).info_.hidden_) num_args++; } push_u16(num_args); + // Replay restores a record on its launch's stream, so a launch on another + // stream than the last record's needs its own. + for (HostSnap& s : t_snaps) + if (s.last_stream != stream && (s.hash.lo || s.hash.hi)) s.record = true; push_u16(static_cast(std::count_if( t_snaps.begin(), t_snaps.end(), [](const HostSnap& s) { return s.record; }))); @@ -707,7 +715,10 @@ static void serialize_kernel_launch( static_cast(payload.size())); for (const HostSnap& s : t_snaps) { auto it = g_pinned.find(s.base); - if (s.record && !capturing && it != g_pinned.end()) it->second.last = s.hash; + if (s.record && !capturing && it != g_pinned.end()) { + it->second.last = s.hash; + it->second.last_stream = stream; + } } } diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 861f2021a953..95653ce4da7d 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -549,13 +549,15 @@ ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipF the arguments, by-value structs included; for each tracked allocation a word points into, it copies the whole allocation under the tracking lock, writes the copy as a blob and adds a record (base, offset 0, length, blob hash, direction 0) to the launch event, unless the hash -equals that of the allocation's last record. Struct words pointing into such an allocation +and the launch stream equal those of the allocation's last record. Struct words pointing into such an allocation are flagged for translation like embedded device pointers, and replay translates them without the device-address shape check. Replay checks each record names the base of a pinned host allocation it made, lies inside it and matches its blob's length, refusing and counting it -otherwise; then it synchronizes the launch stream, copies the blob in, and launches. The -summary prints restored, refused and skipped counts. +otherwise; then it queues a host function that copies the blobs in on the launch stream, +ahead of the kernel, without waiting for the stream. `hipHostFree` and `hipHostUnregister` +wait for every stream, so the allocation outlives a queued copy. The summary prints +restored, refused and skipped counts. Kept deliberately simple: @@ -574,9 +576,8 @@ Kept deliberately simple: - An unchanged allocation gets no record, so device writes into it between two launches stay in replay's buffer as they did at capture; a host store that puts back exactly the last recorded bytes after such a device write is missed. -- Only the launch stream is synchronized before a restore; work on other streams reading - the buffer is not waited for. The sync runs in relaxed capture mode, so a global-mode - capture open on another stream is not invalidated by it. +- A restore is ordered on the launch stream only; work on other streams reading the buffer + is not waited for, which is why a launch on another stream records again. **Threat model.** The snapshots put into the archive host memory the application never handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index afb2b67bab97..4c73bc92a5b8 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -476,23 +476,37 @@ hipFunction_t PlaybackContext::resolve_replacement(const std::string& kernel_nam // u64 hash_hi, u8 direction (0) // u32 num_attrs, u32 attr stride, num_attrs * stride bytes -// Write a launch's pinned host snapshots back before it runs, once the work -// already queued on its stream is done. Not when that stream captures into a -// graph, where the kernel runs only when the graph is launched. -static void restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, - uint16_t n, hipStream_t stream, - const std::string& kernel_name) { +// (destination, blob, length) of each copy; blob_cache_ is never evicted. +using HostSnapCopies = std::vector>; +static void host_snapshot_copy(void* data) { + auto* copies = static_cast(data); + for (auto& [dst, src, len] : *copies) memcpy(dst, src, len); + delete copies; +} + +// Write a launch's pinned host snapshots back when its stream reaches it, by a +// host function queued ahead of the kernel. The replay thread does not wait +// for the stream, which may wait on work it has yet to submit. hipHostFree and +// hipHostUnregister wait for every stream, so the allocation outlives the copy. +// Nothing is restored when the stream captures into a graph, where the kernel +// runs only when the graph is launched. +static hipError_t restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, + uint16_t n, hipStream_t stream, + const std::string& kernel_name) { hipStreamCaptureStatus cs = hipStreamCaptureStatusNone; if (hipStreamIsCapturing(stream, &cs) != hipSuccess || cs != hipStreamCaptureStatusNone) { ctx.host_snapshots_skipped += n; - return; + return hipSuccess; } - bool synced = false; + auto copies = std::make_unique(); for (uint16_t i = 0; i < n; i++, rec += kHostSnapRecordSize) { uint64_t hash[2]; memcpy(hash, rec + 24, 16); size_t blob_len = 0; const void* blob = ctx.load_blob(hash[0], hash[1], &blob_len); + // Refused for the host memory cap or a read error, not merely absent. + if (!blob && ctx.fatal_error.load(std::memory_order_acquire)) + return hipErrorUnknown; const char* why = nullptr; void* dst = hrr_host_snapshot_target(ctx, rec, blob, blob_len, &why); if (!dst) { @@ -501,21 +515,15 @@ static void restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, compact_kernel_name(kernel_name).c_str(), i, why); continue; } - if (!synced) { - // In global mode a sync during another stream's capture - // invalidates that capture. This stream is not capturing, so - // waiting for it is safe: relax the mode for the call only. - hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed; - (void)hipThreadExchangeStreamCaptureMode(&mode); - (void)HRR_HIP_CHECK(hipStreamSynchronize(stream)); - (void)hipThreadExchangeStreamCaptureMode(&mode); - } - synced = true; - uint64_t len; - memcpy(&len, rec + 16, 8); - memcpy(dst, blob, len); - ctx.host_snapshots_restored++; + copies->emplace_back(dst, blob, blob_len); } + if (copies->empty()) return hipSuccess; + const size_t k = copies->size(); + hipError_t r = HRR_HIP_CHECK(hipLaunchHostFunc(stream, host_snapshot_copy, copies.get())); + if (r != hipSuccess) return r; + copies.release(); + ctx.host_snapshots_restored += k; + return hipSuccess; } // ext_global_worksize: the captured grid[] holds *global work-item counts* @@ -1151,8 +1159,13 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, hipStream_t stream = ctx.translate_stream(stream_rec); const size_t kernel_ordinal = ctx.kernels_launched.load(std::memory_order_relaxed) + 1; - if (num_snapshots) - restore_host_snapshots(ctx, snapshots, num_snapshots, stream, kernel_name); + if (hipError_t sr = num_snapshots ? restore_host_snapshots(ctx, snapshots, num_snapshots, + stream, kernel_name) + : hipSuccess; + sr != hipSuccess) { + for (const auto& g : rls.guards) hrr_block_guard_teardown(g); + return sr; + } if (ctx.audit_host_args) { for (auto& [idx, rec, live] : dbg_ptrs) { diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 208e74437022..b490d9c711a3 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -211,6 +211,7 @@ cases: Unit_HRR_Occupancy_Direct: {} Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} + Unit_HRR_PinnedHostBlobRefused: {} Unit_HRR_PinnedHostFork: {} Unit_HRR_PinnedHostFork_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} @@ -220,6 +221,8 @@ cases: Unit_HRR_PinnedHostRegister_Direct: {} Unit_HRR_PinnedHostRoundtrip: {} Unit_HRR_PinnedHostSnapshotCap: {} + Unit_HRR_PinnedHostStreamsRoundtrip: {} + Unit_HRR_PinnedHostStreams_Direct: {} Unit_HRR_PinnedHostStructLowBits_Direct: {} Unit_HRR_PinnedHostStruct_Direct: {} Unit_HRR_PitchedD2HBlobEdits: {} diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index b459eedb113a..9bc158702f67 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -70,7 +70,7 @@ static void hrr_pinned_workload(int* in, bool by_struct = false) { for (int it = 0; it < kPinnedIters; ++it) { for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores // Replay restores nothing in a graph capture. A launch on another stream - // meanwhile is restored, and leaves the next launch nothing to record. + // meanwhile is restored on that stream, so the next launch records again. if (it == 2) { hipStream_t s = nullptr, s2 = nullptr; hipGraph_t g = nullptr; @@ -91,6 +91,49 @@ static void hrr_pinned_workload(int* in, bool by_struct = false) { HRR_HIP_CHECK(hipFree(dout)); } +__global__ void hrr_pinned_wait(int* flag) { + while (atomicAdd(flag, 0) == 0) __builtin_amdgcn_s_sleep(8); +} +__global__ void hrr_pinned_signal(int* flag) { atomicExch(flag, 1); } + +// The pinned-input kernel on stream a queues behind a kernel waiting for a flag +// that b sets later, after a launch on b that reads the same unchanged input. +TEST_CASE("Unit_HRR_PinnedHostStreams_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int *in = nullptr, *dev_in = nullptr, *flag = nullptr, *dout = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); + HRR_HIP_CHECK(hipMalloc(&flag, sizeof(int))); + HRR_HIP_CHECK(hipMemset(flag, 0, sizeof(int))); + HRR_HIP_CHECK(hipMalloc(&dout, 2 * kPinnedN * sizeof(int))); + hipStream_t a = nullptr, b = nullptr; + HRR_HIP_CHECK(hipStreamCreateWithFlags(&a, hipStreamNonBlocking)); + HRR_HIP_CHECK(hipStreamCreateWithFlags(&b, hipStreamNonBlocking)); + std::vector hout(2 * kPinnedN); + for (int it = 0; it < 2; ++it) { + for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(2 * it, i); + if (it == 1) { + hipLaunchKernelGGL(hrr_pinned_wait, dim3(1), dim3(1), 0, a, flag); + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, a, + dout + kPinnedN, dev_in, kPinnedN); + } + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, b, dout, dev_in, kPinnedN); + if (it == 1) hipLaunchKernelGGL(hrr_pinned_signal, dim3(1), dim3(1), 0, b, flag); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipStreamSynchronize(a)); + HRR_HIP_CHECK(hipStreamSynchronize(b)); + HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, (it + 1) * kPinnedN * sizeof(int), + hipMemcpyDeviceToHost)); + for (int i = 0; i < (it + 1) * kPinnedN; ++i) + REQUIRE(hout[i] == pinned_value(2 * it, i % kPinnedN) * 3 + 1); + } + HRR_HIP_CHECK(hipStreamDestroy(a)); + HRR_HIP_CHECK(hipStreamDestroy(b)); + HRR_HIP_CHECK(hipFree(dout)); + HRR_HIP_CHECK(hipFree(flag)); + HRR_HIP_CHECK(hipHostFree(in)); +} + TEST_CASE("Unit_HRR_PinnedHostMalloc_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); int* in = nullptr; @@ -234,7 +277,7 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { CHECK(ret == 0); CHECK(pass == kPinnedIters); CHECK(fail == 0); - CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused, 1 skipped") != + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 1 skipped") != std::string::npos); } else { CHECK(ret != 0); @@ -244,6 +287,22 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { } } +// Replay queues each restore on its launch's stream rather than waiting for +// the stream, here held by a kernel that waits for one replay submits later. +// The launch on b records the input too, since replay restores it on a. +HRR_TEST_CASE(Unit_HRR_PinnedHostStreamsRoundtrip) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_streams"}; + hrr_capture_direct("Unit_HRR_PinnedHostStreams_Direct", cap.path, 5); + auto [ret, out] = hrr_playback_watchdog(cap.path, 120); + INFO("Playback exit " << ret << ", output:\n" << out); + REQUIRE(ret != kHrrWatchdogKilled); + CHECK(ret == 0); + // Exact, since the tolerance takes these ints for tiny floats. + CHECK(out.find("2 pass (2 exact, 0 within tol), 0 fail") != std::string::npos); + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 0 skipped") != + std::string::npos); +} + // An allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB is not snapshotted and // is listed as unreplayable, so replay cannot match. Within the cap it is. HRR_TEST_CASE(Unit_HRR_PinnedHostSnapshotCap) { @@ -270,12 +329,24 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostSnapshotCap) { CHECK(ret == 0); CHECK(pass == kPinnedIters); CHECK(fail == 0); - CHECK(out.find("Pinned host snapshots: 2 restored, 0 refused, 1 skipped") != + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 1 skipped") != std::string::npos); } } } +// A snapshot blob over --max-file-bytes ends the replay before the kernel that +// reads it is launched on input replay never restored. +HRR_TEST_CASE(Unit_HRR_PinnedHostBlobRefused) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_refused"}; + hrr_capture_direct("Unit_HRR_PinnedHostLarge_Direct", cap.path, 5); + auto [ret, out] = hrr_playback_merged(cap.path, "--trace-kernels --max-file-bytes 2097151"); + INFO("Playback exit " << ret << ", output:\n" << out); + CHECK(ret != 0); + CHECK(out.find("refusing to read") != std::string::npos); + CHECK(out.find("[HRR launch]") == std::string::npos); +} + // Capture stops tracking an allocation when it is freed: had it snapshotted the // stale address, replay would refuse the record for an allocation it freed. HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { From a7b0018c17252dcbe160c6e7541617219523710e Mon Sep 17 00:00:00 2001 From: andraghetti Date: Mon, 19 Oct 2026 11:59:48 +0200 Subject: [PATCH 12/12] fix(hrr): track hipMallocHost for pinned snapshots, test the other frees and the extra[] launch hipMallocHost was replayed into a pinned buffer but never tracked, so a kernel reading it ran on replay's empty buffer. Test that hipFree, hipFreeHost and hipHostUnregister stop the tracking, and a hipModuleLaunchKernel launch whose arguments come in extra[]. DESIGN.md now says what the enqueue-time copy does to queued device writes into the allocation. --- projects/clr/hipamd/src/hrr/hip_capture.cpp | 20 ++++-- projects/hrr/DESIGN.md | 15 ++-- projects/hrr/tests/config/integration.yaml | 2 + .../tests/integration/hrr_pinned_host_test.cc | 70 +++++++++++++++---- 4 files changed, 84 insertions(+), 23 deletions(-) diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 9b915bee068f..d31d32785f81 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -181,8 +181,9 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // hipHostRegister). Each allocation hit is written whole as a // blob, and named by a record in the launch event unless its hash and stream // equal those of the last record for it. Replay copies the blob back when the -// stream reaches the launch. -// See DESIGN.md, "Pinned host snapshots". +// stream reaches the launch. The copy is taken at enqueue, without waiting for +// the stream: bytes that queued GPU work has yet to write are restored as they +// were before it. See DESIGN.md, "Pinned host snapshots". // --------------------------------------------------------------------------- namespace { @@ -1770,12 +1771,13 @@ hipError_t capture_hipHostUnregister(void* hostPtr) { return r; } -// hipHostMalloc and the frees are recorded by generated shims. These wrappers -// sit in front of them in g_cap_table and keep g_pinned current; hipFree can -// release pinned memory too. The other pinned allocators are not tracked: -// replay makes no buffer for them. +// hipHostMalloc, hipMallocHost and the frees are recorded by generated shims. +// These wrappers sit in front of them in g_cap_table and keep g_pinned current; +// hipFree can release pinned memory too. hipHostAlloc and hipMemAllocHost are +// not tracked: replay makes no buffer for them. namespace { decltype(HipDispatchTable::hipHostMalloc_fn) g_next_hipHostMalloc = nullptr; +decltype(HipDispatchTable::hipMallocHost_fn) g_next_hipMallocHost = nullptr; decltype(HipDispatchTable::hipHostFree_fn) g_next_hipHostFree = nullptr; decltype(HipDispatchTable::hipFreeHost_fn) g_next_hipFreeHost = nullptr; decltype(HipDispatchTable::hipFree_fn) g_next_hipFree = nullptr; @@ -1786,6 +1788,11 @@ static hipError_t pinned_hipHostMalloc(void** ptr, size_t size, unsigned int fla if (r == hipSuccess && ptr) pinned_track(*ptr, size); return r; } +static hipError_t pinned_hipMallocHost(void** ptr, size_t size) { + hipError_t r = g_next_hipMallocHost(ptr, size); + if (r == hipSuccess && ptr) pinned_track(*ptr, size); + return r; +} static hipError_t pinned_hipHostFree(void* p) { pinned_untrack(p); return g_next_hipHostFree(p); } static hipError_t pinned_hipFreeHost(void* p) { pinned_untrack(p); return g_next_hipFreeHost(p); } static hipError_t pinned_hipFree(void* p) { pinned_untrack(p); return g_next_hipFree(p); } @@ -1798,6 +1805,7 @@ static void pinned_wrap_table() { g_next_##api = g_cap_table.api##_fn; \ g_cap_table.api##_fn = pinned_##api; HRR_PINNED_WRAP(hipHostMalloc) + HRR_PINNED_WRAP(hipMallocHost) HRR_PINNED_WRAP(hipHostFree) HRR_PINNED_WRAP(hipFreeHost) HRR_PINNED_WRAP(hipFree) diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 95653ce4da7d..7b3cdecebdd1 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -544,7 +544,7 @@ At playback: A kernel can read pinned host memory that the host filled with plain stores, which no HIP call records; replay would run it on whatever its fresh buffer holds. Capture tracks the -ranges of `hipHostMalloc` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, +ranges of `hipHostMalloc`, `hipMallocHost` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, `hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of the arguments, by-value structs included; for each tracked allocation a word points into, it copies the whole allocation under the tracking lock, writes the copy as a blob and adds a @@ -561,16 +561,21 @@ restored, refused and skipped counts. Kept deliberately simple: -- Capture does not wait for the stream: the bytes are those at enqueue time, which is what - the kernel reads unless earlier queued work still changes them. If that work is writing - the allocation during the copy, replay restores the partial bytes over its output. +- Capture does not wait for the stream: the bytes are those at enqueue time, the host's + stores up to the launch. Queued work that writes the allocation (a kernel, an async copy + into it) and has not finished by then is not in them, and replay, which restores right + before the kernel, puts the older bytes back over its output. A kernel reading device + writes to pinned memory that the host did not wait for replays wrong. Waiting for the + stream would deadlock a stream that waits on work the host submits later, and a + stream-ordered copy would stall the stream on every such launch and put a host node in + the application's graphs. - `HIP_HRR_HOST_SNAPSHOTS=0` turns snapshots off; an allocation larger than `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under `unreplayable_apis`. - Replay restores nothing for a launch replayed into a capturing stream, and capture does not snapshot at graph launch. A record for a launch into a capturing stream does not become the allocation's last, so the next launch outside the capture records again. -- `hipHostAlloc`, `hipMallocHost` and `hipMemAllocHost` are not tracked, since replay makes +- `hipHostAlloc` and `hipMemAllocHost` are not tracked, since replay makes no buffer for them, nor is a `hipHostRegister` device alias that differs from the host address. - An unchanged allocation gets no record, so device writes into it between two launches diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index b490d9c711a3..a0b3c3a5f2a0 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -218,6 +218,7 @@ cases: Unit_HRR_PinnedHostFreed_Direct: {} Unit_HRR_PinnedHostLarge_Direct: {} Unit_HRR_PinnedHostMalloc_Direct: {} + Unit_HRR_PinnedHostModule_Direct: {} Unit_HRR_PinnedHostRegister_Direct: {} Unit_HRR_PinnedHostRoundtrip: {} Unit_HRR_PinnedHostSnapshotCap: {} @@ -225,6 +226,7 @@ cases: Unit_HRR_PinnedHostStreams_Direct: {} Unit_HRR_PinnedHostStructLowBits_Direct: {} Unit_HRR_PinnedHostStruct_Direct: {} + Unit_HRR_PinnedMallocHost_Direct: {} Unit_HRR_PitchedD2HBlobEdits: {} Unit_HRR_PitchedD2HGapRoundtrip: {} Unit_HRR_PitchedD2HGap_Direct: {} diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc index 9bc158702f67..d8cd0853e5ec 100644 --- a/projects/hrr/tests/integration/hrr_pinned_host_test.cc +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -53,14 +53,25 @@ __global__ void hrr_pinned_triple_struct(int* out, PinnedIn s) { __global__ void hrr_pinned_ignore(uint64_t) {} -static void hrr_pinned_workload(int* in, bool by_struct = false) { +// by_module launches through hipModuleLaunchKernel with an extra[] buffer, the +// path where capture finds the arguments by the kernel's offsets. +static void hrr_pinned_workload(int* in, bool by_struct = false, bool by_module = false) { int* dev_in = nullptr; HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); int* dout = nullptr; HRR_HIP_CHECK(hipMalloc(&dout, kPinnedN * sizeof(int))); std::vector hout(kPinnedN); + hipFunction_t f = nullptr; + if (by_module) + HRR_HIP_CHECK(hipGetFuncBySymbol(&f, reinterpret_cast(hrr_pinned_triple))); auto launch = [&](hipStream_t s) { - if (by_struct) + if (by_module) { + struct { int* out; const int* in; int n; } a{dout, dev_in, kPinnedN}; + size_t sz = sizeof(a); + void* extra[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, &a, HIP_LAUNCH_PARAM_BUFFER_SIZE, &sz, + HIP_LAUNCH_PARAM_END}; + HRR_HIP_CHECK(hipModuleLaunchKernel(f, 1, 1, 1, kPinnedN, 1, 1, 0, s, nullptr, extra)); + } else if (by_struct) hipLaunchKernelGGL(hrr_pinned_triple_struct, dim3(1), dim3(kPinnedN), 0, s, dout, PinnedIn{dev_in, kPinnedN}); else @@ -150,6 +161,26 @@ TEST_CASE("Unit_HRR_PinnedHostStruct_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostFree(in)); } +TEST_CASE("Unit_HRR_PinnedHostModule_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in, false, true); + HRR_HIP_CHECK(hipHostFree(in)); +} + +// The deprecated hipMallocHost and hipFreeHost, which replay still replays. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +TEST_CASE("Unit_HRR_PinnedMallocHost_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipMallocHost(reinterpret_cast(&in), kPinnedN * sizeof(int))); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipFreeHost(in)); +} +#pragma clang diagnostic pop + // A 2 MiB allocation: over HIP_HRR_HOST_SNAPSHOT_MAX_MB=1, within 2. TEST_CASE("Unit_HRR_PinnedHostLarge_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); @@ -159,18 +190,30 @@ TEST_CASE("Unit_HRR_PinnedHostLarge_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipHostFree(in)); } -// A freed allocation's address passed on as a plain number: capture must not -// snapshot memory that is gone. +// A released allocation's address passed on as a plain number, once for each +// way to release it: capture must not snapshot memory that is no longer pinned. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" TEST_CASE("Unit_HRR_PinnedHostFreed_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); - void* p = nullptr; - HRR_HIP_CHECK(hipHostMalloc(&p, kPinnedN * sizeof(int), 0)); - HRR_HIP_CHECK(hipHostFree(p)); - hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, - reinterpret_cast(p)); - HRR_HIP_CHECK(hipGetLastError()); + std::vector reg(kPinnedN); + for (int how = 0; how < 4; ++how) { + void* p = reg.data(); + if (how == 3) + HRR_HIP_CHECK(hipHostRegister(p, kPinnedN * sizeof(int), hipHostRegisterDefault)); + else + HRR_HIP_CHECK(hipHostMalloc(&p, kPinnedN * sizeof(int), 0)); + if (how == 0) HRR_HIP_CHECK(hipHostFree(p)); + if (how == 1) HRR_HIP_CHECK(hipFree(p)); + if (how == 2) HRR_HIP_CHECK(hipFreeHost(p)); + if (how == 3) HRR_HIP_CHECK(hipHostUnregister(p)); + hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, + reinterpret_cast(p)); + HRR_HIP_CHECK(hipGetLastError()); + } HRR_HIP_CHECK(hipDeviceSynchronize()); } +#pragma clang diagnostic pop TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); @@ -255,6 +298,8 @@ TEST_CASE("Unit_HRR_PinnedHostFork_Direct", "[.][hrr-direct]") { // is nothing to write back and the checks fail. HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", + "Unit_HRR_PinnedMallocHost_Direct", + "Unit_HRR_PinnedHostModule_Direct", "Unit_HRR_PinnedHostRegister_Direct", "Unit_HRR_PinnedHostStruct_Direct", #ifndef _WIN32 @@ -347,8 +392,9 @@ HRR_TEST_CASE(Unit_HRR_PinnedHostBlobRefused) { CHECK(out.find("[HRR launch]") == std::string::npos); } -// Capture stops tracking an allocation when it is freed: had it snapshotted the -// stale address, replay would refuse the record for an allocation it freed. +// Capture stops tracking an allocation on hipHostFree, hipFree, hipFreeHost and +// hipHostUnregister: had it snapshotted the stale address, replay would refuse +// the record for an allocation it released. HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_freed"}; hrr_capture_direct("Unit_HRR_PinnedHostFreed_Direct", cap.path, 3);