diff --git a/projects/clr/hipamd/src/hrr/hip_capture.cpp b/projects/clr/hipamd/src/hrr/hip_capture.cpp index 96576df72231..d31d32785f81 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture.cpp @@ -134,7 +134,10 @@ const char* hip_capture_output_dir() { // build (DEBUG from rocclr/utils/macros.hpp, the macro that picks debug() over // release() there) honours it; any other build ignores it, because the dumps put // a second copy of the workload's kernel arguments in the log, where log -// collection picks them up. +// collection picks them up. HIP_HRR_HOST_SNAPSHOTS and +// HIP_HRR_HOST_SNAPSHOT_MAX_MB are not in flags.hpp: they only tune capture, so +// they are read here (hrr_host_snapshots_enabled, hrr_host_snapshot_cap_bytes) +// and stay with the HRR code and its reviewers. static bool hrr_dbg_args_enabled() { #if defined(DEBUG) return HIP_HRR_DEBUG_ARGS; @@ -169,6 +172,175 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s return out_buf != nullptr && out_size > 0; } +// --------------------------------------------------------------------------- +// Pinned host snapshots +// +// A kernel can read pinned host memory the host filled with plain stores, which +// no HIP call records. Before each launch, the 8-byte-aligned words of its +// arguments are looked up in the tracked pinned allocations (hipHostMalloc, +// hipHostRegister). Each allocation hit is written whole as a +// blob, and named by a record in the launch event unless its hash and stream +// equal those of the last record for it. Replay copies the blob back when the +// stream reaches the launch. The copy is taken at enqueue, without waiting for +// the stream: bytes that queued GPU work has yet to write are restored as they +// were before it. See DESIGN.md, "Pinned host snapshots". +// --------------------------------------------------------------------------- + +namespace { +struct PinnedAlloc { + size_t size = 0; + hrr_cap::Hash128 last{}; // blob of the last record, {0,0} before the first + hipStream_t last_stream = nullptr; // and its launch stream +}; +struct HostSnap { + uint64_t base = 0, size = 0; + hrr_cap::Hash128 hash{}, last{}; + hipStream_t last_stream = nullptr; + bool record = false; // contents or stream differ from the last record + std::unique_ptr copy; // the bytes, until hashed and written +}; +std::mutex g_pinned_mu; // also held while a snapshot copies +std::map g_pinned; +std::atomic g_pinned_count{0}; +// The pinned allocations the launch this thread is making points into. +thread_local std::vector t_snaps; +} // namespace + +static void pinned_track(void* p, size_t size) { + if (!p || size == 0) return; + std::lock_guard lk(g_pinned_mu); + g_pinned[reinterpret_cast(p)] = PinnedAlloc{size, {}}; + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); +} + +// Called before the memory is released, so no snapshot reads it afterwards. +static void pinned_untrack(void* p) { + if (!p || g_pinned_count.load(std::memory_order_relaxed) == 0) return; + std::lock_guard lk(g_pinned_mu); + g_pinned.erase(reinterpret_cast(p)); + g_pinned_count.store(g_pinned.size(), std::memory_order_relaxed); +} + +// fork() takes g_pinned_mu before the writer's locks, the order a launch takes +// them in, so a child never inherits it held by a snapshot on a thread it does +// not have. Registered after writer::open() registers the writer's handlers: +// prepare handlers run in reverse order of registration. +static void pinned_install_atfork() { +#ifndef _WIN32 + static std::once_flag once; + std::call_once(once, [] { + pthread_atfork([] { g_pinned_mu.lock(); }, [] { g_pinned_mu.unlock(); }, + [] { g_pinned_mu.unlock(); }); + }); +#endif +} + +// HIP_HRR_HOST_SNAPSHOTS, on unless set to something other than "true" or a +// non-zero number, and HIP_HRR_HOST_SNAPSHOT_MAX_MB, 64 unless set. Parsed as +// the CLR flags registry parses a bool and a uint, and read once: every launch +// asks. +static bool hrr_host_snapshots_enabled() { + static const bool on = [] { + const char* v = std::getenv("HIP_HRR_HOST_SNAPSHOTS"); + return !v || std::strcmp(v, "true") == 0 || std::atoi(v) != 0; + }(); + return on; +} + +static size_t hrr_host_snapshot_cap_bytes() { + static const size_t cap = [] { + const char* v = std::getenv("HIP_HRR_HOST_SNAPSHOT_MAX_MB"); + const unsigned int mb = v ? static_cast(std::atoi(v)) : 64u; + return static_cast(mb) << 20; + }(); + return cap; +} + +static void pinned_scan_locked(const uint8_t* bytes, size_t sz) { + const size_t cap = hrr_host_snapshot_cap_bytes(); + for (size_t j = 0; bytes && j + sizeof(uint64_t) <= sz; j += sizeof(uint64_t)) { + uint64_t v; + std::memcpy(&v, bytes + j, sizeof(v)); + auto it = g_pinned.upper_bound(static_cast(v)); + if (it == g_pinned.begin()) continue; + --it; + if (v - it->first >= it->second.size || + std::any_of(t_snaps.begin(), t_snaps.end(), + [&](const HostSnap& s) { return s.base == it->first; })) + continue; + HostSnap s; + s.base = it->first; + s.size = it->second.size; + if (s.size > cap) { + hrr_cap::writer::note_unreplayable( + "pinned host snapshot", + "a pinned allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB was not recorded"); + } else { + // Hashed and written from the copy, after the lock, so the blob holds + // the bytes its hash names. + s.copy.reset(new (std::nothrow) uint8_t[s.size]); + if (s.copy) { + std::memcpy(s.copy.get(), reinterpret_cast(s.base), s.size); + s.last = it->second.last; + s.last_stream = it->second.last_stream; + } else { + hrr_cap::writer::note_unreplayable("pinned host snapshot", + "no memory to copy a pinned allocation"); + } + } + t_snaps.push_back(std::move(s)); + } +} + +// Takes the snapshot of one launch before the real call and drops it after, +// whether or not the launch was recorded. +struct PinnedSnapshot { + PinnedSnapshot(hipFunction_t f, void** params, void** extra, + const void* kbuf = nullptr, size_t ksz = 0) { + t_snaps.clear(); + if (!f || !hrr_host_snapshots_enabled() || g_pinned_count.load(std::memory_order_relaxed) == 0) + return; + if (!params && !kbuf && extra) parse_kernel_extra(extra, kbuf, ksz); + const amd::KernelSignature& sig = hip::asKernel(f)->signature(); + std::unique_lock lk(g_pinned_mu); + uint32_t param_idx = 0; + for (uint32_t i = 0; i < sig.numParametersAll(); i++) { + const auto& d = sig.at(i); + if (d.info_.hidden_) continue; + const uint8_t* bytes = nullptr; + if (kbuf && d.offset_ + d.size_ <= ksz) + bytes = static_cast(kbuf) + d.offset_; + else if (!kbuf && params) + bytes = static_cast(params[param_idx]); + param_idx++; + if (d.offset_ % sizeof(uint64_t) == 0) pinned_scan_locked(bytes, d.size_); + } + lk.unlock(); + for (HostSnap& s : t_snaps) { + if (!s.copy) continue; + s.hash = hrr_cap::writer::write_blob(s.copy.get(), s.size); + s.copy.reset(); + s.record = (s.hash.lo || s.hash.hi) && (s.hash.lo != s.last.lo || s.hash.hi != s.last.hi); + } + } + ~PinnedSnapshot() { t_snaps.clear(); } +}; + +// The kernel behind a host stub, looked up before the launch only when there +// is pinned memory to snapshot, without changing the application's last error. +static hipFunction_t pinned_stub_fn(const void* stub) { + if (!hrr_host_snapshots_enabled() || g_pinned_count.load(std::memory_order_relaxed) == 0 || + !g_real_table.hipGetFuncBySymbol_fn) + return nullptr; + const hipError_t saved_cmd = hip::tls.last_command_error_; + const hipError_t saved_err = hip::tls.last_error_; + hipFunction_t f = nullptr; + if (g_real_table.hipGetFuncBySymbol_fn(&f, stub) != hipSuccess) f = nullptr; + hip::tls.last_command_error_ = saved_cmd; + hip::tls.last_error_ = saved_err; + return f; +} + // --------------------------------------------------------------------------- // Kernel launch event serialization // @@ -183,7 +355,7 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // u32[3] block // u32 shared_mem // u16 num_args -// u16 num_snapshots (always 0) +// u16 num_snapshots // for each arg: // u8 value_kind (0=scalar, 1=pointer/gpu addr, 2=hidden, // 3=scalar/struct with embedded gpu pointer(s)) @@ -191,7 +363,14 @@ static bool parse_kernel_extra(void** extra, const void*& out_buf, size_t& out_s // u8[] data (size bytes) // if value_kind == 3: // u16 n_ptrs -// u16[] ptr_offset (n_ptrs entries: byte offset of each 8-byte gpu ptr) +// u16[] ptr_offset (n_ptrs entries: byte offset of each 8-byte gpu ptr, +// or pointer into a pinned allocation in t_snaps) +// for each snapshot (v10): +// u64 ptr_handle (capture-time base of the pinned allocation) +// u64 offset (0), u64 length (the allocation size) +// u64 hash_lo, u64 hash_hi (blob with the bytes before the launch) +// u8 direction (0) +// u32 num_attrs, u32 attr stride, num_attrs * stride bytes // --------------------------------------------------------------------------- // Locate device pointers embedded inside a by-value kernel argument. @@ -383,7 +562,12 @@ static void serialize_kernel_launch( if (!sig.at(i).info_.hidden_) num_args++; } push_u16(num_args); - push_u16(0); // num_snapshots + // Replay restores a record on its launch's stream, so a launch on another + // stream than the last record's needs its own. + for (HostSnap& s : t_snaps) + if (s.last_stream != stream && (s.hash.lo || s.hash.hi)) s.record = true; + push_u16(static_cast(std::count_if( + t_snaps.begin(), t_snaps.end(), [](const HostSnap& s) { return s.record; }))); // HIP_HRR_DEBUG_ARGS dumps every captured arg (kind, size, full bytes, // detected embedded-pointer offsets) — used to confirm pointer layout. Use the @@ -409,6 +593,17 @@ static void serialize_kernel_launch( } std::vector offs; scan_embedded_ptr_offsets(func_key, idx, bytes, sz, offs); + // Words pointing into a pinned allocation this launch reads, so replay + // translates them too. + for (size_t j = 0; bytes && !t_snaps.empty() && j + 8 <= sz; j += 8) { + uint64_t v; + std::memcpy(&v, bytes + j, 8); + if (std::any_of(t_snaps.begin(), t_snaps.end(), + [&](const HostSnap& s) { return v - s.base < s.size; }) && + std::find(offs.begin(), offs.end(), j) == offs.end()) + offs.push_back(static_cast(j)); + } + std::sort(offs.begin(), offs.end()); uint8_t kind = offs.empty() ? 0 : 3; push_u8(kind); push_u16(sz); if (bytes) push_bytes(bytes, sz); @@ -472,6 +667,12 @@ static void serialize_kernel_launch( } } + for (const HostSnap& s : t_snaps) { + if (!s.record) continue; + push_u64(s.base); push_u64(0); push_u64(s.size); + push_u64(s.hash.lo); push_u64(s.hash.hi); push_u8(0); + } + // Launch-attribute tail. Written for every launch so the payload has one // shape: a plain launch records a count of zero. The stride lets replay // reject a recording made against a different hipLaunchAttribute layout @@ -501,9 +702,25 @@ static void serialize_kernel_launch( hrr_cap::writer::mark_incomplete(reason); return; } + // A record becomes the allocation's last only once its event is written, in + // the same order as the events. Replay restores nothing for a launch into a + // stream capture, so a record written for one does not become the last. + hipStream_t cs = stream; + hip::getStreamPerThread(cs); + const bool capturing = cs && cs != hipStreamLegacy && + reinterpret_cast(cs)->GetCaptureStatus() != hipStreamCaptureStatusNone; + std::unique_lock lk(g_pinned_mu, std::defer_lock); + if (!t_snaps.empty()) lk.lock(); hrr_cap::writer::write_event_raw(api_id, reinterpret_cast(payload.data()), static_cast(payload.size())); + for (const HostSnap& s : t_snaps) { + auto it = g_pinned.find(s.base); + if (s.record && !capturing && it != g_pinned.end()) { + it->second.last = s.hash; + it->second.last_stream = stream; + } + } } static hrr_cap::Hash128 kernel_code_object_hash(amd::Kernel* kernel); @@ -1083,6 +1300,7 @@ hipError_t capture_hipModuleLaunchKernel( unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams, void** extra) { + PinnedSnapshot snap(f, kernelParams, extra); hipError_t r = g_real_table.hipModuleLaunchKernel_fn( f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, @@ -1104,6 +1322,7 @@ hipError_t capture_hipModuleLaunchCooperativeKernel( unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, hipStream_t stream, void** kernelParams) { + PinnedSnapshot snap(f, kernelParams, nullptr); hipError_t r = g_real_table.hipModuleLaunchCooperativeKernel_fn( f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, @@ -1124,6 +1343,7 @@ hipError_t capture_hipExtModuleLaunchKernel( size_t sharedMemBytes, hipStream_t stream, void** kernelParams, void** extra, hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags) { + PinnedSnapshot snap(f, kernelParams, extra); hipError_t r = g_real_table.hipExtModuleLaunchKernel_fn( f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, @@ -1153,6 +1373,7 @@ hipError_t capture_hipLaunchKernel(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(function_address), args, nullptr); hipError_t r = g_real_table.hipLaunchKernel_fn( function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1179,6 +1400,7 @@ hipError_t capture_hipLaunchKernel_spt(const void* function_address, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(function_address), args, nullptr); hipError_t r = g_real_table.hipLaunchKernel_spt_fn( function_address, numBlocks, dimBlocks, args, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1203,6 +1425,7 @@ hipError_t capture_hipLaunchCooperativeKernel(const void* f, void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream) { + PinnedSnapshot snap(pinned_stub_fn(f), kernelParams, nullptr); hipError_t r = g_real_table.hipLaunchCooperativeKernel_fn( f, gridDim, blockDimX, kernelParams, sharedMemBytes, stream); if (r == hipSuccess) { @@ -1224,6 +1447,7 @@ hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, void** kernelParams, uint32_t sharedMemBytes, hipStream_t hStream) { + PinnedSnapshot snap(pinned_stub_fn(f), kernelParams, nullptr); hipError_t r = g_real_table.hipLaunchCooperativeKernel_spt_fn( f, gridDim, blockDim, kernelParams, sharedMemBytes, hStream); if (r == hipSuccess) { @@ -1254,6 +1478,7 @@ hipError_t capture_hipLaunchCooperativeKernel_spt(const void* f, hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** params, void** extra) { + PinnedSnapshot snap(config ? f : nullptr, params, extra); hipError_t r = g_real_table.hipDrvLaunchKernelEx_fn(config, f, params, extra); if (r == hipSuccess && config) { record_launch(f, @@ -1267,6 +1492,7 @@ hipError_t capture_hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipError_t capture_hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args) { + PinnedSnapshot snap(config ? pinned_stub_fn(fPtr) : nullptr, args, nullptr); hipError_t r = g_real_table.hipLaunchKernelExC_fn(config, fPtr, args); if (r == hipSuccess && config) { // fPtr is a host stub address, not a hipFunction_t: the same resolution @@ -1342,6 +1568,8 @@ hipError_t capture_hipLaunchByPtr(const void* func) { kargs = e.arguments_; // copy (not move) to leave arguments_ intact for the real launch } + PinnedSnapshot snap(kargs.empty() ? nullptr : pinned_stub_fn(func), nullptr, nullptr, + kargs.data(), kargs.size()); hipError_t r = g_real_table.hipLaunchByPtr_fn(func); if (r == hipSuccess) { // func is a host stub pointer — resolve to real hipFunction_t first @@ -1522,11 +1750,13 @@ hipError_t capture_hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int std::lock_guard lk(g_pinned_reg_mu); g_pinned_reg_map[hostPtr] = sizeBytes; } + pinned_track(hostPtr, sizeBytes); } return r; } hipError_t capture_hipHostUnregister(void* hostPtr) { + pinned_untrack(hostPtr); hipError_t r = g_real_table.hipHostUnregister_fn(hostPtr); if (r == hipSuccess) { hrr_args_hipHostUnregister a{}; @@ -1541,6 +1771,47 @@ hipError_t capture_hipHostUnregister(void* hostPtr) { return r; } +// hipHostMalloc, hipMallocHost and the frees are recorded by generated shims. +// These wrappers sit in front of them in g_cap_table and keep g_pinned current; +// hipFree can release pinned memory too. hipHostAlloc and hipMemAllocHost are +// not tracked: replay makes no buffer for them. +namespace { +decltype(HipDispatchTable::hipHostMalloc_fn) g_next_hipHostMalloc = nullptr; +decltype(HipDispatchTable::hipMallocHost_fn) g_next_hipMallocHost = nullptr; +decltype(HipDispatchTable::hipHostFree_fn) g_next_hipHostFree = nullptr; +decltype(HipDispatchTable::hipFreeHost_fn) g_next_hipFreeHost = nullptr; +decltype(HipDispatchTable::hipFree_fn) g_next_hipFree = nullptr; +} // namespace + +static hipError_t pinned_hipHostMalloc(void** ptr, size_t size, unsigned int flags) { + hipError_t r = g_next_hipHostMalloc(ptr, size, flags); + if (r == hipSuccess && ptr) pinned_track(*ptr, size); + return r; +} +static hipError_t pinned_hipMallocHost(void** ptr, size_t size) { + hipError_t r = g_next_hipMallocHost(ptr, size); + if (r == hipSuccess && ptr) pinned_track(*ptr, size); + return r; +} +static hipError_t pinned_hipHostFree(void* p) { pinned_untrack(p); return g_next_hipHostFree(p); } +static hipError_t pinned_hipFreeHost(void* p) { pinned_untrack(p); return g_next_hipFreeHost(p); } +static hipError_t pinned_hipFree(void* p) { pinned_untrack(p); return g_next_hipFree(p); } + +static void pinned_wrap_table() { + static bool done = false; + if (done) return; + done = true; +#define HRR_PINNED_WRAP(api) \ + g_next_##api = g_cap_table.api##_fn; \ + g_cap_table.api##_fn = pinned_##api; + HRR_PINNED_WRAP(hipHostMalloc) + HRR_PINNED_WRAP(hipMallocHost) + HRR_PINNED_WRAP(hipHostFree) + HRR_PINNED_WRAP(hipFreeHost) + HRR_PINNED_WRAP(hipFree) +#undef HRR_PINNED_WRAP +} + // --------------------------------------------------------------------------- // hipMemPoolSetAttribute: value is void* to a scalar, stored inline in the low // bytes of value_u64 (an int32_t for the three reuse policies, uint64_t otherwise). @@ -2717,6 +2988,7 @@ void hip_capture_init() { // only (see comment above — no static-init capture hook). if (!g_installed) { hip_capture_build_table(); + pinned_wrap_table(); hip_capture_install(); } @@ -2728,6 +3000,7 @@ void hip_capture_init() { hip_capture_uninstall(); return; } + pinned_install_atfork(); // HIP_HRR_DEBUG_ARGS traces are emitted via LogPrintfInfo (amd::LOG_INFO). // ClPrint filters anything above AMD_LOG_LEVEL, so a user who set the trace diff --git a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp index 6a4cff183160..18a1c781217e 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_writer.cpp @@ -153,8 +153,10 @@ static constexpr size_t kPathMax = 4096; static constexpr size_t kMetadataJsonMax = 128u * 1024u; static constexpr size_t kEmergencyManifestMax = kMetadataJsonMax + 1024u; -// Lock order: g_reopen_mu, then g_file_mu, then g_blob_mu, then -// g_unreplayable_mu. atfork_prepare is the only path that holds them all. +// Lock order: g_pinned_mu (hip_capture.cpp, held while a launch copies pinned +// host memory or writes its event), then g_reopen_mu, then g_file_mu, then +// g_blob_mu, then g_unreplayable_mu. Only fork() holds them all: the pinned +// prepare handler takes g_pinned_mu, then atfork_prepare takes the rest. static std::mutex g_file_mu; static int g_events_fd = -1; // g_base_dir is the archive path requested via HIP_HRR_CAPTURE_OUTPUT. diff --git a/projects/hrr/DESIGN.md b/projects/hrr/DESIGN.md index 32380b54afc3..7b3cdecebdd1 100644 --- a/projects/hrr/DESIGN.md +++ b/projects/hrr/DESIGN.md @@ -168,7 +168,7 @@ the parent that exited cleanly needs no repair. A sub-archive is skipped when payloads, finds it complete in the sense `load_archive` uses. `--info` on a root applies the same check before it shows a process as complete. -### Archive Format (v9) +### Archive Format (v10) ``` capture.hrr/ manifest.json { version, capture_mode, owner_pid, processes[] } @@ -286,7 +286,7 @@ projects/hrr/ — standalone HRR project (portable layer) Not used by capture — it is the format an out-of-tree producer writes and playback reads playback/ - hrr_reader.h/.cpp — archive loader, v7 format; record framing + hrr_reader.h/.cpp — archive loader, v10 format; record framing (read_raw_record / open_record_stream) shared with the region sidecars hrr_region_map.h/.cpp — region timeline: merge, cursor, live block set, @@ -371,13 +371,13 @@ The generator classifies each API: Generated capture shims for manual APIs are pass-throughs (no `write_event()`). When adding HIP API support, update this script to classify the API in the appropriate capture and playback policy sets. APIs requiring non-trivial serialization or replay belong in `MANUAL_CAPTURE_APIS` and/or `MANUAL_PLAYBACK_APIS`; intentionally unsupported replay APIs belong in `NOOP_PLAYBACK_APIS`. -## Archive Format (v9) +## Archive Format (v10) Single-authority definition in `hrr_api_args.h` (auto-generated): ``` HRR_MAGIC = 0x52524845 ("HRRE") -HRR_VERSION = 9 +HRR_VERSION = 10 ``` Version history, so an archive written by an older runtime can be placed: @@ -414,12 +414,15 @@ Version history, so an archive written by an older runtime can be placed: - **v9** records `hipKernelSetAttributeForDevice`, appended to `HipDispatchTable`, which shifts the compiler API IDs up by one again. A v8 reader would decode those events as the wrong API. +- **v10** puts pinned host snapshot records after a kernel launch's arguments + (`num_snapshots` was always 0). A v9 reader would not restore them. See + Pinned Host Snapshots below. ``` / manifest.json { version, capture_mode, owner_pid, processes[] } (version here is the manifest schema = 1, distinct from the - events.bin HRR_VERSION = 9) + events.bin HRR_VERSION = 10) pid-/ manifest.json { pid, parent_pid, complete, event_count, blob_count } active one line naming the process instance (boot id and @@ -537,41 +540,56 @@ At playback: - `playback_hipHostUnregister` translates the pointer, calls `hipHostUnregister`, frees the backing buffer, and removes both map entries. -### Sysmem Update Tracking (planned — not yet implemented) - -**Problem:** `hipHostRegister` captures the buffer contents at registration time only. -After that, the CPU may modify the buffer without going through any HIP API: - -``` -hipHostRegister(ptr, sz) ← blob captured (initial state) -// CPU writes to ptr[] ← invisible to capture layer -hipMemcpyAsync(d, ptr, sz, H2D, stream) ← H2D blob captures current contents ✓ -// CPU writes to ptr[] again -hipModuleLaunchKernel(...) ← kernel reads ptr directly via mapped flag ✗ stale -``` - -H2D memcpy is already handled — the src buffer is re-snapshotted at each call regardless -of registration. The gap is direct GPU reads from registered host memory (mapped flag) -after a CPU write that was not routed through a memcpy. - -**Planned design:** - -1. New synthetic event `HRR_SYSMEM_UPDATE` (not a real HIP API): - fields: `hostPtr u64`, `sizeBytes u64`, `blob_hash_lo u64`, `blob_hash_hi u64`. - -2. Per-region `last_hash` stored alongside `g_pinned_reg_map`. Before each H2D memcpy src - check and before each kernel launch for pointer args in registered ranges: hash the - current contents, compare with `last_hash`. Emit `HRR_SYSMEM_UPDATE` + write blob only - if hash changed. Content-addressed storage deduplicates unchanged regions automatically. - -3. Optional sync-gated dirty flag: set `dirty=true` for all registered regions after any - `hipStreamSynchronize` / `hipDeviceSynchronize` / `hipEventSynchronize`. Only hash - (step 2) if `dirty==true`. Avoids hashing in pure-GPU loops where the CPU never - touches the buffer between launches. Falls back to always-hash if sync events are - not captured. - -4. At replay: `HRR_SYSMEM_UPDATE` handler `memcpy`s the blob into the live registered - buffer, ordered by sequence ID like all other events. +### Pinned Host Snapshots + +A kernel can read pinned host memory that the host filled with plain stores, which no HIP +call records; replay would run it on whatever its fresh buffer holds. Capture tracks the +ranges of `hipHostMalloc`, `hipMallocHost` and `hipHostRegister` (dropped on `hipHostFree`, `hipFreeHost`, +`hipFree` and `hipHostUnregister`). Before each launch it reads the 8-byte-aligned words of +the arguments, by-value structs included; for each tracked allocation a word points into, +it copies the whole allocation under the tracking lock, writes the copy as a blob and adds a +record (base, offset 0, length, blob hash, direction 0) to the launch event, unless the hash +and the launch stream equal those of the allocation's last record. Struct words pointing into such an allocation +are flagged for translation like +embedded device pointers, and replay translates them without the device-address shape +check. Replay checks each record names the base of a pinned host +allocation it made, lies inside it and matches its blob's length, refusing and counting it +otherwise; then it queues a host function that copies the blobs in on the launch stream, +ahead of the kernel, without waiting for the stream. `hipHostFree` and `hipHostUnregister` +wait for every stream, so the allocation outlives a queued copy. The summary prints +restored, refused and skipped counts. + +Kept deliberately simple: + +- Capture does not wait for the stream: the bytes are those at enqueue time, the host's + stores up to the launch. Queued work that writes the allocation (a kernel, an async copy + into it) and has not finished by then is not in them, and replay, which restores right + before the kernel, puts the older bytes back over its output. A kernel reading device + writes to pinned memory that the host did not wait for replays wrong. Waiting for the + stream would deadlock a stream that waits on work the host submits later, and a + stream-ordered copy would stall the stream on every such launch and put a host node in + the application's graphs. +- `HIP_HRR_HOST_SNAPSHOTS=0` turns snapshots off; an allocation larger than + `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (64) is not snapshotted and is listed under + `unreplayable_apis`. +- Replay restores nothing for a launch replayed into a capturing stream, and capture does + not snapshot at graph launch. A record for a launch into a capturing stream does not + become the allocation's last, so the next launch outside the capture records again. +- `hipHostAlloc` and `hipMemAllocHost` are not tracked, since replay makes + no buffer for them, nor is a `hipHostRegister` device alias that differs from the host + address. +- An unchanged allocation gets no record, so device writes into it between two launches + stay in replay's buffer as they did at capture; a host store that puts back exactly the + last recorded bytes after such a device write is missed. +- A restore is ordered on the launch stream only; work on other streams reading the buffer + is not waited for, which is why a launch on another stream records again. + +**Threat model.** The snapshots put into the archive host memory the application never +handed to a HIP call: for a serving stack, token IDs, sampling state and request metadata, +including stale bytes from earlier requests in a reused buffer. The archive must be +handled as at least as sensitive as the workload's data. Replay treats every record as +untrusted input and writes only inside a pinned host allocation it made itself, never +through a recorded address. ## Kernel Argument Capture @@ -659,10 +677,9 @@ This detector is a value-based heuristic with three deliberate properties: previously caused occasional replay faults on ATen elementwise kernels. `HIP_HRR_PTR_RELAX=1` disables the replay-side guard for debugging. -**Scope:** only `hipMemoryTypeDevice`/`Unified` words are flagged. A pinned/host -(`hipMemoryTypeHost`) pointer embedded by value keeps its capture-time host VA at -replay (invalid in the replay process); translating embedded host pointers is -intentionally out of scope. +**Scope:** `hipMemoryTypeDevice`/`Unified` words are flagged, and so are words +pointing into a pinned allocation the launch snapshots (see Pinned Host Snapshots). +Any other embedded host pointer keeps its capture-time VA at replay. A per-launch `co_hash` (the FNV-1a-128 hash of the owning code object) **is** recorded in kernel launch events. Playback resolves kernels by `(co_hash, name)`: it first looks @@ -1273,7 +1290,8 @@ Sysmem capture is a fundamentally hard problem. Two approaches exist: and emit `HRR_SYSMEM_UPDATE` synthetic events only when content changes. This is significantly more complex to implement and may not be fully feasible on all platforms. -Neither approach is currently implemented. +Pinned host memory a kernel reads is now snapshotted before each launch; see Pinned Host +Snapshots above. ### Partial-Replay APIs — H2D and D2H Work, Kernel Launches Do Not @@ -1491,7 +1509,7 @@ The event wire format (finding H5): shrinking `reserved` to 2 bytes), so kernel launches with large serialized payloads (many args / long mangled names / large by-value structs) up to ~4 GiB are recorded normally instead of being dropped at 65535 bytes. This is the change that bumped - `HRR_VERSION` to 4; the current version is 9, see Archive Format above. The + `HRR_VERSION` to 4; the current version is 10, see Archive Format above. The writer's single-record buffer path now writes any oversized record straight through. - **Per-argument size limit (64 KiB) now fails loudly.** Each kernel arg's size is still a `uint16_t`. A by-value struct argument ≥ 64 KiB cannot be represented, so the launch diff --git a/projects/hrr/README.md b/projects/hrr/README.md index 864d464f0e61..ac9fefa59a53 100644 --- a/projects/hrr/README.md +++ b/projects/hrr/README.md @@ -92,6 +92,7 @@ User-facing capture, replay, and validation knobs. Implementation details can be |----------|---------|---------| | `HIP_HRR_CAPTURE_OUTPUT` | *(unset)* | Enable capture; path to the `.hrr` archive directory. An empty or blank value leaves capture off. Ignored, with one line on stderr, when Linux starts the program in secure-execution mode (set-user-ID, set-group-ID, file capabilities or an LSM transition) | | `HIP_HRR_DEBUG_ARGS` | off | Dump every captured kernel arg to the log (debug / provenance). Only a Debug build of the HIP runtime (no `NDEBUG`) reads it; a release build ignores it. Either way capture says on stderr whether it reads it, whatever the log level | +| `HIP_HRR_HOST_SNAPSHOTS` | on | Record the pinned host memory each kernel reads before the launch, so replay can write it back. `HIP_HRR_HOST_SNAPSHOT_MAX_MB` (default `64`) caps the allocation size. The archive then holds that host memory (see DESIGN.md, Pinned Host Snapshots) | On Linux the archive is readable only by the user who captured it (directories 0700, files 0600). If the archive cannot be set up, capture is disabled with a `[HRR capture] Capture disabled` message on stderr and the application runs normally. diff --git a/projects/hrr/include/hrr/hrr_api_args.h b/projects/hrr/include/hrr/hrr_api_args.h index 2f2bde533848..87a9bd6cd851 100644 --- a/projects/hrr/include/hrr/hrr_api_args.h +++ b/projects/hrr/include/hrr/hrr_api_args.h @@ -83,8 +83,10 @@ * reader decodes those events as the wrong API. * v9: hipKernelSetAttributeForDevice was appended to HipDispatchTable, * taking the first compiler ID and shifting the compiler API IDs up by one. - * A v8 reader would decode those events as the wrong API. */ -#define HRR_VERSION ((uint16_t)9u) + * A v8 reader would decode those events as the wrong API. + * v10: a kernel launch event carries pinned host snapshot records after its + * arguments (num_snapshots was always 0). A v9 reader would not restore them. */ +#define HRR_VERSION ((uint16_t)10u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1) diff --git a/projects/hrr/playback/hip_playback.cpp b/projects/hrr/playback/hip_playback.cpp index e0abb3d99921..4c73bc92a5b8 100644 --- a/projects/hrr/playback/hip_playback.cpp +++ b/projects/hrr/playback/hip_playback.cpp @@ -467,11 +467,64 @@ hipFunction_t PlaybackContext::resolve_replacement(const std::string& kernel_nam // [+12..23] block[3] (uint32_t[3]) // [+24..27] shared_mem (uint32_t) // [+28..29] num_args (uint16_t) -// [+30..31] num_snapshots (uint16_t, always 0) +// [+30..31] num_snapshots (uint16_t) // per arg: u8 value_kind, u16 size, bytes data // value_kind: 0=scalar, 1=gpu-pointer, 2=hidden, // 3=scalar/struct with embedded gpu pointer(s); // kind 3 appends u16 n_ptrs then n_ptrs * u16 byte offsets. +// per snapshot (v10): u64 ptr_handle, u64 offset, u64 length, u64 hash_lo, +// u64 hash_hi, u8 direction (0) +// u32 num_attrs, u32 attr stride, num_attrs * stride bytes + +// (destination, blob, length) of each copy; blob_cache_ is never evicted. +using HostSnapCopies = std::vector>; +static void host_snapshot_copy(void* data) { + auto* copies = static_cast(data); + for (auto& [dst, src, len] : *copies) memcpy(dst, src, len); + delete copies; +} + +// Write a launch's pinned host snapshots back when its stream reaches it, by a +// host function queued ahead of the kernel. The replay thread does not wait +// for the stream, which may wait on work it has yet to submit. hipHostFree and +// hipHostUnregister wait for every stream, so the allocation outlives the copy. +// Nothing is restored when the stream captures into a graph, where the kernel +// runs only when the graph is launched. +static hipError_t restore_host_snapshots(PlaybackContext& ctx, const uint8_t* rec, + uint16_t n, hipStream_t stream, + const std::string& kernel_name) { + hipStreamCaptureStatus cs = hipStreamCaptureStatusNone; + if (hipStreamIsCapturing(stream, &cs) != hipSuccess || cs != hipStreamCaptureStatusNone) { + ctx.host_snapshots_skipped += n; + return hipSuccess; + } + auto copies = std::make_unique(); + for (uint16_t i = 0; i < n; i++, rec += kHostSnapRecordSize) { + uint64_t hash[2]; + memcpy(hash, rec + 24, 16); + size_t blob_len = 0; + const void* blob = ctx.load_blob(hash[0], hash[1], &blob_len); + // Refused for the host memory cap or a read error, not merely absent. + if (!blob && ctx.fatal_error.load(std::memory_order_acquire)) + return hipErrorUnknown; + const char* why = nullptr; + void* dst = hrr_host_snapshot_target(ctx, rec, blob, blob_len, &why); + if (!dst) { + ctx.host_snapshots_refused++; + fprintf(stderr, "[HRR] '%s': pinned host snapshot %u %s — refused\n", + compact_kernel_name(kernel_name).c_str(), i, why); + continue; + } + copies->emplace_back(dst, blob, blob_len); + } + if (copies->empty()) return hipSuccess; + const size_t k = copies->size(); + hipError_t r = HRR_HIP_CHECK(hipLaunchHostFunc(stream, host_snapshot_copy, copies.get())); + if (r != hipSuccess) return r; + copies.release(); + ctx.host_snapshots_restored += k; + return hipSuccess; +} // ext_global_worksize: the captured grid[] holds *global work-item counts* // (HSA/OpenCL semantics, as passed to hipExtModuleLaunchKernel), NOT workgroup @@ -843,11 +896,15 @@ static void decode_kernel_args( // may be a genuine scalar (a large count, a double, a packed value) // that was mis-flagged — overwriting it with null would silently // corrupt it. Only rewrite the word when it actually resolves. - auto try_translate_word = [&](size_t off, const char* src) -> bool { + auto try_translate_word = [&](size_t off, const char* src, bool flagged) -> bool { if (off + 8 > arg_size) return false; uint64_t rec_ptr; memcpy(&rec_ptr, data + off, 8); - // null, small, or a packed-integer false positive - if (!va_shaped(rec_ptr)) return false; + // null, small, or a packed-integer false positive. A word capture + // flagged inside a pinned host allocation need not look like a + // device VA: a host address can have small low 32 bits. + if (!va_shaped(rec_ptr) && + !(flagged && ctx.in_pinned_host(rec_ptr))) + return false; void* live = ctx.translate_ptr(rec_ptr); // Same bypassed-segment materialisation as a whole-pointer // argument: a struct field pointing into memory HIP never saw @@ -872,7 +929,7 @@ static void decode_kernel_args( // First honor the capture-recorded offsets (these may be unaligned). std::vector handled(arg_size, 0); for (uint16_t off : ptr_offsets) { - if (try_translate_word(off, "captured")) + if (try_translate_word(off, "captured", true)) for (int b = 0; b < 8 && static_cast(off) + b < arg_size; b++) handled[off + b] = 1; else if (ctx.verbose) @@ -895,7 +952,7 @@ static void decode_kernel_args( (std::getenv("HIP_HRR_REPLAY_NO_RESCAN") != nullptr); for (size_t off = 0; !no_rescan && off + 8 <= arg_size; ) { if (handled[off]) { off += 1; continue; } - if (try_translate_word(off, "rescan")) off += 8; + if (try_translate_word(off, "rescan", false)) off += 8; else off += 1; } } else { @@ -1046,6 +1103,15 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, arg_storage, &rls, dbg_dump_ptrs ? &dbg_ptrs : nullptr, dbg_dump_ptrs ? &dbg_args : nullptr); + const uint8_t* snapshots = p; + if (!hrr_host_snapshots_fit(p, end, num_snapshots)) { + fprintf(stderr, "[HRR] '%s': %u pinned host snapshot records run past the " + "event — refused\n", kernel_name.c_str(), num_snapshots); + ctx.host_snapshots_refused += num_snapshots; + num_snapshots = 0; + } + p += static_cast(num_snapshots) * kHostSnapRecordSize; + // Launch-attribute tail: u32 count, u32 per-entry stride, then the // entries. Every launch payload carries it (count 0 for a plain launch). std::vector launch_attrs; @@ -1093,6 +1159,13 @@ static hipError_t replay_kernel_launch(PlaybackContext& ctx, const uint8_t* pl, hipStream_t stream = ctx.translate_stream(stream_rec); const size_t kernel_ordinal = ctx.kernels_launched.load(std::memory_order_relaxed) + 1; + if (hipError_t sr = num_snapshots ? restore_host_snapshots(ctx, snapshots, num_snapshots, + stream, kernel_name) + : hipSuccess; + sr != hipSuccess) { + for (const auto& g : rls.guards) hrr_block_guard_teardown(g); + return sr; + } if (ctx.audit_host_args) { for (auto& [idx, rec, live] : dbg_ptrs) { diff --git a/projects/hrr/playback/hip_playback.h b/projects/hrr/playback/hip_playback.h index 10974a48d824..35072c5d5f63 100644 --- a/projects/hrr/playback/hip_playback.h +++ b/projects/hrr/playback/hip_playback.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,11 @@ struct PlaybackContext { // If d2h_attempted > 0 but d2h_pass == 0 && d2h_fail == 0, every check was // skipped — pointer translation or blob loading failed for all D2H events. std::atomic d2h_attempted{0}; + // Pinned host snapshot records written back, refused, and not applied + // because the launch was replayed into a graph capture. + std::atomic host_snapshots_restored{0}; + std::atomic host_snapshots_refused{0}; + std::atomic host_snapshots_skipped{0}; // Set true by note_d2h_fail when the running D2H-failure fraction crosses the // configured divergence-abort threshold. Distinguishes a clean "replay @@ -549,6 +555,16 @@ struct PlaybackContext { return nullptr; } + // Does v fall inside a pinned host allocation replay made? + bool in_pinned_host(uint64_t v) const { + std::shared_lock lk(map_mutex); + for (auto& [base, e] : alloc_map) + if ((e.kind == AllocKind::HostMalloc || e.kind == AllocKind::HostRegister) && + v - base < e.size) + return true; + return false; + } + // Is this value an address from the capturing process — i.e. does it fall // inside an allocation as *recorded*, before translation? Reports the // enclosing recorded base so a hit can be traced back to the event that @@ -989,6 +1005,50 @@ extern thread_local uint64_t hrr_dispatch_seq; // completion — including a genuine GPU fault — is returned to the caller. hipError_t hrr_watchdog_device_sync(PlaybackContext& ctx, const char* what); +// One pinned host snapshot record of a kernel launch event: u64 ptr_handle, +// u64 offset, u64 length, u64 hash_lo, u64 hash_hi, u8 direction (0). +inline constexpr size_t kHostSnapRecordSize = 41; + +// Whether n whole records fit between p and the end of the event. +inline bool hrr_host_snapshots_fit(const uint8_t* p, const uint8_t* end, uint16_t n) { + return p <= end && static_cast(end - p) / kHostSnapRecordSize >= n; +} + +// Where replay writes the blob of the record at `rec`, or nullptr with *why +// saying why it refuses the record. The archive is input like any other: a +// record must name the base of a pinned host allocation replay made, lie +// inside it, and match its blob's length. +inline void* hrr_host_snapshot_target(const PlaybackContext& ctx, const uint8_t* rec, + const void* blob, size_t blob_len, + const char** why) { + uint64_t ptr, off, len; + memcpy(&ptr, rec, 8); + memcpy(&off, rec + 8, 8); + memcpy(&len, rec + 16, 8); + void* live = nullptr; + size_t size = 0; + { + std::shared_lock lk(ctx.map_mutex); + auto it = ctx.alloc_map.find(ptr); + if (it != ctx.alloc_map.end() && (it->second.kind == AllocKind::HostMalloc || + it->second.kind == AllocKind::HostRegister)) { + live = it->second.live_ptr; + size = it->second.size; + } + } + if (rec[40] != 0) + *why = "has an unknown direction"; + else if (!live) + *why = "names no pinned host allocation replay made"; + else if (off > size || len > size - off) + *why = "lies outside its allocation"; + else if (!blob || blob_len != len) + *why = "does not match its blob"; + else + return static_cast(live) + off; + return nullptr; +} + // --------------------------------------------------------------------------- // Playback function signature and dispatch table // --------------------------------------------------------------------------- diff --git a/projects/hrr/playback/hrr_playback.cpp b/projects/hrr/playback/hrr_playback.cpp index d75005d324f5..f254baed2dae 100644 --- a/projects/hrr/playback/hrr_playback.cpp +++ b/projects/hrr/playback/hrr_playback.cpp @@ -2013,6 +2013,10 @@ int main(int argc, char** argv) { ctx.d2h_pass.load() - ctx.d2h_pass_tol.load(), ctx.d2h_pass_tol.load(), ctx.d2h_fail.load(), ctx.d2h_attempted.load() - ctx.d2h_pass.load() - ctx.d2h_fail.load()); + if (ctx.host_snapshots_restored + ctx.host_snapshots_refused + ctx.host_snapshots_skipped) + printf("[HRR] Pinned host snapshots: %zu restored, %zu refused, %zu skipped " + "(graph capture)\n", ctx.host_snapshots_restored.load(), + ctx.host_snapshots_refused.load(), ctx.host_snapshots_skipped.load()); bool ok = (ctx.d2h_fail == 0); diff --git a/projects/hrr/playback/hrr_reader.cpp b/projects/hrr/playback/hrr_reader.cpp index 14dabe4a1771..21ea753e5e0e 100644 --- a/projects/hrr/playback/hrr_reader.cpp +++ b/projects/hrr/playback/hrr_reader.cpp @@ -231,8 +231,9 @@ static bool resolve_archive_path(const std::string& input, std::string& resolved // [+12..23] block[3] (uint32_t[3]) // [+24..27] shared_mem (uint32_t) // [+28..29] num_args (uint16_t) -// [+30..31] num_snapshots (uint16_t, always 0) +// [+30..31] num_snapshots (uint16_t) // per arg: u8 value_kind, u16 size, bytes data +// per snapshot (v10, 41 bytes): see BufferSnapshot; the attribute tail follows // --------------------------------------------------------------------------- static bool parse_kernel_launch(const uint8_t* data, size_t len, @@ -291,7 +292,7 @@ static bool parse_kernel_launch(const uint8_t* data, size_t len, kl.args.push_back(std::move(arg)); } - // buffer snapshots (always 0 in in-tree captures) + // pinned host snapshots for (uint16_t i = 0; i < num_snapshots; i++) { if (p + 41 > end) return false; BufferSnapshot snap; diff --git a/projects/hrr/playback/hrr_reader.h b/projects/hrr/playback/hrr_reader.h index 70f59261a676..27fe794a7f44 100644 --- a/projects/hrr/playback/hrr_reader.h +++ b/projects/hrr/playback/hrr_reader.h @@ -93,7 +93,8 @@ struct KernelArg { std::vector ptr_offsets; // only for value_kind == 3 }; -// Buffer snapshot (always empty in in-tree captures) +// Pinned host snapshot (v10): the blob replay writes to [offset, offset + +// length) of the pinned allocation recorded at ptr_handle before the launch. struct BufferSnapshot { uint64_t ptr_handle; uint64_t offset; diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 0c5138ab759f..a0b3c3a5f2a0 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -211,6 +211,22 @@ cases: Unit_HRR_Occupancy_Direct: {} Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} + Unit_HRR_PinnedHostBlobRefused: {} + Unit_HRR_PinnedHostFork: {} + Unit_HRR_PinnedHostFork_Direct: {} + Unit_HRR_PinnedHostFreedRoundtrip: {} + Unit_HRR_PinnedHostFreed_Direct: {} + Unit_HRR_PinnedHostLarge_Direct: {} + Unit_HRR_PinnedHostMalloc_Direct: {} + Unit_HRR_PinnedHostModule_Direct: {} + Unit_HRR_PinnedHostRegister_Direct: {} + Unit_HRR_PinnedHostRoundtrip: {} + Unit_HRR_PinnedHostSnapshotCap: {} + Unit_HRR_PinnedHostStreamsRoundtrip: {} + Unit_HRR_PinnedHostStreams_Direct: {} + Unit_HRR_PinnedHostStructLowBits_Direct: {} + Unit_HRR_PinnedHostStruct_Direct: {} + Unit_HRR_PinnedMallocHost_Direct: {} Unit_HRR_PitchedD2HBlobEdits: {} Unit_HRR_PitchedD2HGapRoundtrip: {} Unit_HRR_PitchedD2HGap_Direct: {} diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index 4360a3b598cb..9690060e8775 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -21,6 +21,7 @@ cases: Unit_HRR_Format_BadMagic: {} Unit_HRR_Format_EventTypeName: {} Unit_HRR_Format_HashHex: {} + Unit_HRR_Format_HostSnapshotRecord: {} Unit_HRR_Format_Roundtrip: {} Unit_HRR_Format_SmallPayloadForType: {} Unit_HRR_Format_TruncatedEvent: {} diff --git a/projects/hrr/tests/include/hrr_test_common.hh b/projects/hrr/tests/include/hrr_test_common.hh index ef0d4f378ab1..8f2314b89419 100644 --- a/projects/hrr/tests/include/hrr_test_common.hh +++ b/projects/hrr/tests/include/hrr_test_common.hh @@ -324,6 +324,7 @@ inline void hrr_run_roundtrip(const std::string& direct_case, // // hrr_capture_direct: spawn a hidden _Direct workload with HIP_HRR_CAPTURE_OUTPUT // set, REQUIRE a clean capture, and assert the archive has >= min_events. +// Pairs in env are set in the workload's environment as well. // // hrr_playback_env: run hrr-playback with arbitrary extra environment pairs // (e.g. HIP_HRR_REPLAY_ZERO_INIT / HIP_HRR_REPLAY_DIVERGENCE_ABORT) and return @@ -335,9 +336,11 @@ inline void hrr_run_roundtrip(const std::string& direct_case, // --------------------------------------------------------------------------- inline void hrr_capture_direct(const std::string& direct_case, const fs::path& cap_path, - size_t min_events = 5) { + size_t min_events = 5, + const std::vector>& env = {}) { { hrr::test::SpawnProc proc(hrr_self_exe()); proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap_path.string()); + for (const auto& kv : env) proc.setEnv(kv.first, kv.second); { set_proc_search_path(proc); } int ret = proc.run("\"" + direct_case + "\""); INFO("Capture exit: " << ret); diff --git a/projects/hrr/tests/integration/CMakeLists.txt b/projects/hrr/tests/integration/CMakeLists.txt index e87f128aa713..ddc8624851dc 100644 --- a/projects/hrr/tests/integration/CMakeLists.txt +++ b/projects/hrr/tests/integration/CMakeLists.txt @@ -49,6 +49,10 @@ # argument bytes for HIP_HRR_DEBUG_ARGS only in a Debug runtime. Contains its # own hidden [.][hrr-direct] workload. # +# hrr_pinned_host_test.cc covers pinned host memory a kernel reads directly, +# which capture snapshots before each launch and replay writes back. Contains +# its own hidden [.][hrr-direct] workloads. +# # hrr_va_placement_test.cc covers capture-address placement: a device pointer # stored in device memory by an H2D copy (ROCM-31827). Contains its own hidden # [.][hrr-direct] workload. @@ -89,6 +93,7 @@ set(HRR_GPU_WORKLOAD_SOURCES hrr_regions_test.cc hrr_api_matrix_workload_test.cc hrr_kernel_args_test.cc + hrr_pinned_host_test.cc hrr_va_placement_test.cc hrr_disk_space_test.cc ) diff --git a/projects/hrr/tests/integration/hrr_pinned_host_test.cc b/projects/hrr/tests/integration/hrr_pinned_host_test.cc new file mode 100644 index 000000000000..d8cd0853e5ec --- /dev/null +++ b/projects/hrr/tests/integration/hrr_pinned_host_test.cc @@ -0,0 +1,423 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +/** + * @addtogroup HRR HRR pinned host snapshots + * @{ + * @ingroup HRRTest + * A kernel reads pinned host memory that the host refills with plain stores + * between launches, which no HIP call records. Capture snapshots it before + * each launch and replay writes it back, so the replayed D2H checks match. + */ + +#include "hrr_test_common.hh" + +#include +#include +#ifndef _WIN32 +#include +#include +#include + +#include +#include +#include +#include +#include +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE 0x100000 +#endif +#endif + +namespace { +constexpr int kPinnedN = 256; +constexpr int kPinnedIters = 4; +// The input changes on every second launch: two snapshot records, four checks. +int pinned_value(int it, int i) { return (it / 2) * 1000 + i; } +} // namespace + +__global__ void hrr_pinned_triple(int* out, const int* in, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) out[i] = in[i] * 3 + 1; +} + +// The same kernel with the pinned pointer inside a by-value struct. +struct PinnedIn { const int* in; int n; }; +__global__ void hrr_pinned_triple_struct(int* out, PinnedIn s) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < s.n) out[i] = s.in[i] * 3 + 1; +} + +__global__ void hrr_pinned_ignore(uint64_t) {} + +// by_module launches through hipModuleLaunchKernel with an extra[] buffer, the +// path where capture finds the arguments by the kernel's offsets. +static void hrr_pinned_workload(int* in, bool by_struct = false, bool by_module = false) { + int* dev_in = nullptr; + HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); + int* dout = nullptr; + HRR_HIP_CHECK(hipMalloc(&dout, kPinnedN * sizeof(int))); + std::vector hout(kPinnedN); + hipFunction_t f = nullptr; + if (by_module) + HRR_HIP_CHECK(hipGetFuncBySymbol(&f, reinterpret_cast(hrr_pinned_triple))); + auto launch = [&](hipStream_t s) { + if (by_module) { + struct { int* out; const int* in; int n; } a{dout, dev_in, kPinnedN}; + size_t sz = sizeof(a); + void* extra[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, &a, HIP_LAUNCH_PARAM_BUFFER_SIZE, &sz, + HIP_LAUNCH_PARAM_END}; + HRR_HIP_CHECK(hipModuleLaunchKernel(f, 1, 1, 1, kPinnedN, 1, 1, 0, s, nullptr, extra)); + } else if (by_struct) + hipLaunchKernelGGL(hrr_pinned_triple_struct, dim3(1), dim3(kPinnedN), 0, s, + dout, PinnedIn{dev_in, kPinnedN}); + else + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, s, dout, dev_in, kPinnedN); + HRR_HIP_CHECK(hipGetLastError()); + }; + for (int it = 0; it < kPinnedIters; ++it) { + for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(it, i); // uncaptured stores + // Replay restores nothing in a graph capture. A launch on another stream + // meanwhile is restored on that stream, so the next launch records again. + if (it == 2) { + hipStream_t s = nullptr, s2 = nullptr; + hipGraph_t g = nullptr; + HRR_HIP_CHECK(hipStreamCreate(&s)); + HRR_HIP_CHECK(hipStreamCreate(&s2)); + HRR_HIP_CHECK(hipStreamBeginCapture(s, hipStreamCaptureModeGlobal)); + launch(s); + launch(s2); + HRR_HIP_CHECK(hipStreamEndCapture(s, &g)); + HRR_HIP_CHECK(hipGraphDestroy(g)); + HRR_HIP_CHECK(hipStreamDestroy(s)); + HRR_HIP_CHECK(hipStreamDestroy(s2)); + } + launch(nullptr); + HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, kPinnedN * sizeof(int), hipMemcpyDeviceToHost)); + for (int i = 0; i < kPinnedN; ++i) REQUIRE(hout[i] == pinned_value(it, i) * 3 + 1); + } + HRR_HIP_CHECK(hipFree(dout)); +} + +__global__ void hrr_pinned_wait(int* flag) { + while (atomicAdd(flag, 0) == 0) __builtin_amdgcn_s_sleep(8); +} +__global__ void hrr_pinned_signal(int* flag) { atomicExch(flag, 1); } + +// The pinned-input kernel on stream a queues behind a kernel waiting for a flag +// that b sets later, after a launch on b that reads the same unchanged input. +TEST_CASE("Unit_HRR_PinnedHostStreams_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int *in = nullptr, *dev_in = nullptr, *flag = nullptr, *dout = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + HRR_HIP_CHECK(hipHostGetDevicePointer(reinterpret_cast(&dev_in), in, 0)); + HRR_HIP_CHECK(hipMalloc(&flag, sizeof(int))); + HRR_HIP_CHECK(hipMemset(flag, 0, sizeof(int))); + HRR_HIP_CHECK(hipMalloc(&dout, 2 * kPinnedN * sizeof(int))); + hipStream_t a = nullptr, b = nullptr; + HRR_HIP_CHECK(hipStreamCreateWithFlags(&a, hipStreamNonBlocking)); + HRR_HIP_CHECK(hipStreamCreateWithFlags(&b, hipStreamNonBlocking)); + std::vector hout(2 * kPinnedN); + for (int it = 0; it < 2; ++it) { + for (int i = 0; i < kPinnedN; ++i) in[i] = pinned_value(2 * it, i); + if (it == 1) { + hipLaunchKernelGGL(hrr_pinned_wait, dim3(1), dim3(1), 0, a, flag); + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, a, + dout + kPinnedN, dev_in, kPinnedN); + } + hipLaunchKernelGGL(hrr_pinned_triple, dim3(1), dim3(kPinnedN), 0, b, dout, dev_in, kPinnedN); + if (it == 1) hipLaunchKernelGGL(hrr_pinned_signal, dim3(1), dim3(1), 0, b, flag); + HRR_HIP_CHECK(hipGetLastError()); + HRR_HIP_CHECK(hipStreamSynchronize(a)); + HRR_HIP_CHECK(hipStreamSynchronize(b)); + HRR_HIP_CHECK(hipMemcpy(hout.data(), dout, (it + 1) * kPinnedN * sizeof(int), + hipMemcpyDeviceToHost)); + for (int i = 0; i < (it + 1) * kPinnedN; ++i) + REQUIRE(hout[i] == pinned_value(2 * it, i % kPinnedN) * 3 + 1); + } + HRR_HIP_CHECK(hipStreamDestroy(a)); + HRR_HIP_CHECK(hipStreamDestroy(b)); + HRR_HIP_CHECK(hipFree(dout)); + HRR_HIP_CHECK(hipFree(flag)); + HRR_HIP_CHECK(hipHostFree(in)); +} + +TEST_CASE("Unit_HRR_PinnedHostMalloc_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipHostFree(in)); +} + +TEST_CASE("Unit_HRR_PinnedHostStruct_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in, true); + HRR_HIP_CHECK(hipHostFree(in)); +} + +TEST_CASE("Unit_HRR_PinnedHostModule_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), kPinnedN * sizeof(int), 0)); + hrr_pinned_workload(in, false, true); + HRR_HIP_CHECK(hipHostFree(in)); +} + +// The deprecated hipMallocHost and hipFreeHost, which replay still replays. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +TEST_CASE("Unit_HRR_PinnedMallocHost_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipMallocHost(reinterpret_cast(&in), kPinnedN * sizeof(int))); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipFreeHost(in)); +} +#pragma clang diagnostic pop + +// A 2 MiB allocation: over HIP_HRR_HOST_SNAPSHOT_MAX_MB=1, within 2. +TEST_CASE("Unit_HRR_PinnedHostLarge_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + int* in = nullptr; + HRR_HIP_CHECK(hipHostMalloc(reinterpret_cast(&in), size_t{2} << 20, 0)); + hrr_pinned_workload(in); + HRR_HIP_CHECK(hipHostFree(in)); +} + +// A released allocation's address passed on as a plain number, once for each +// way to release it: capture must not snapshot memory that is no longer pinned. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +TEST_CASE("Unit_HRR_PinnedHostFreed_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + std::vector reg(kPinnedN); + for (int how = 0; how < 4; ++how) { + void* p = reg.data(); + if (how == 3) + HRR_HIP_CHECK(hipHostRegister(p, kPinnedN * sizeof(int), hipHostRegisterDefault)); + else + HRR_HIP_CHECK(hipHostMalloc(&p, kPinnedN * sizeof(int), 0)); + if (how == 0) HRR_HIP_CHECK(hipHostFree(p)); + if (how == 1) HRR_HIP_CHECK(hipFree(p)); + if (how == 2) HRR_HIP_CHECK(hipFreeHost(p)); + if (how == 3) HRR_HIP_CHECK(hipHostUnregister(p)); + hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, + reinterpret_cast(p)); + HRR_HIP_CHECK(hipGetLastError()); + } + HRR_HIP_CHECK(hipDeviceSynchronize()); +} +#pragma clang diagnostic pop + +TEST_CASE("Unit_HRR_PinnedHostRegister_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + std::vector in(kPinnedN); + HRR_HIP_CHECK(hipHostRegister(in.data(), kPinnedN * sizeof(int), hipHostRegisterDefault)); + hrr_pinned_workload(in.data()); + HRR_HIP_CHECK(hipHostUnregister(in.data())); +} + +#ifndef _WIN32 +// The struct case from pinned memory whose low 32 bits are zero, an address +// replay's device-pointer shape check rejects. +TEST_CASE("Unit_HRR_PinnedHostStructLowBits_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + void* at = reinterpret_cast(uintptr_t{0x7e0000000000}); + void* p = mmap(at, 1 << 16, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p != at) { + const int err = errno; + // A kernel older than 4.17 takes the address as a hint only. + if (p != MAP_FAILED) munmap(p, 1 << 16); + if (p != MAP_FAILED || err == EEXIST) HRR_SKIP("0x7e0000000000 is already mapped"); + FAIL("mmap: " << std::strerror(err)); + } + HRR_HIP_CHECK(hipHostRegister(p, kPinnedN * sizeof(int), hipHostRegisterDefault)); + hrr_pinned_workload(static_cast(p), true); + HRR_HIP_CHECK(hipHostUnregister(p)); + munmap(p, 1 << 16); +} + +// Forks while another thread snapshots a 32 MiB pinned allocation, so most forks +// land while it holds the pinned tracking lock. Each child frees an address +// capture does not track, which takes that lock and fails without device work. +// A child that inherited the lock held never exits. +TEST_CASE("Unit_HRR_PinnedHostFork_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + constexpr size_t kBig = 32u << 20; + void* p = nullptr; + HRR_HIP_CHECK(hipHostMalloc(&p, kBig, 0)); + std::memset(p, 1, kBig); + // Nothing may REQUIRE before the join. + std::atomic stop{false}; + std::thread launcher([&] { + while (!stop.load()) { + hipLaunchKernelGGL(hrr_pinned_ignore, dim3(1), dim3(1), 0, nullptr, + reinterpret_cast(p)); + (void)hipStreamSynchronize(nullptr); + } + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + int hung = 0, failed = 0; + for (int i = 0; i < 20 && !hung; ++i) { + const pid_t pid = fork(); + if (pid == 0) { + int x = 0; + _exit(hipFree(&x) == hipSuccess); + } + int status = 0; + pid_t got = 0; + for (int ms = 0; pid > 0 && ms < 10000 && (got = waitpid(pid, &status, WNOHANG)) == 0; ++ms) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (pid > 0 && got == 0) { + kill(pid, SIGKILL); + waitpid(pid, &status, 0); + ++hung; + } else if (got != pid || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + ++failed; + } + } + stop = true; + launcher.join(); + HRR_HIP_CHECK(hipHostFree(p)); + INFO("hung " << hung << ", failed " << failed); + REQUIRE(hung == 0); + REQUIRE(failed == 0); +} +#endif + +#if defined(HRR_PLAYBACK_EXE) && defined(HRR_TEST_EXE) +// Replay matches the recording only if it wrote each launch's input back; its +// own buffer never sees the host's stores. With HIP_HRR_HOST_SNAPSHOTS=0 there +// is nothing to write back and the checks fail. +HRR_TEST_CASE(Unit_HRR_PinnedHostRoundtrip) { + for (const char* direct : {"Unit_HRR_PinnedHostMalloc_Direct", + "Unit_HRR_PinnedMallocHost_Direct", + "Unit_HRR_PinnedHostModule_Direct", + "Unit_HRR_PinnedHostRegister_Direct", + "Unit_HRR_PinnedHostStruct_Direct", +#ifndef _WIN32 + // Last, since a skip ends the case. + "Unit_HRR_PinnedHostStructLowBits_Direct", +#endif + }) { + for (const char* snapshots : {"1", "0"}) { + // Without snapshots the struct would carry a host address replay never mapped. + if (std::string(direct).find("Struct") != std::string::npos && *snapshots == '0') + continue; + INFO(direct << " HIP_HRR_HOST_SNAPSHOTS=" << snapshots); + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host"}; + hrr_capture_direct(direct, cap.path, 5, {{"HIP_HRR_HOST_SNAPSHOTS", snapshots}}); + auto [ret, out] = hrr_playback_env(cap.path, {{"HIP_HRR_D2H_EXACT", "1"}}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + int pass = 0, fail = 0; + REQUIRE(hrr_parse_d2h_summary(out, pass, fail)); + if (std::string(snapshots) == "1") { + CHECK(ret == 0); + CHECK(pass == kPinnedIters); + CHECK(fail == 0); + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 1 skipped") != + std::string::npos); + } else { + CHECK(ret != 0); + CHECK(fail > 0); + } + } + } +} + +// Replay queues each restore on its launch's stream rather than waiting for +// the stream, here held by a kernel that waits for one replay submits later. +// The launch on b records the input too, since replay restores it on a. +HRR_TEST_CASE(Unit_HRR_PinnedHostStreamsRoundtrip) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_streams"}; + hrr_capture_direct("Unit_HRR_PinnedHostStreams_Direct", cap.path, 5); + auto [ret, out] = hrr_playback_watchdog(cap.path, 120); + INFO("Playback exit " << ret << ", output:\n" << out); + REQUIRE(ret != kHrrWatchdogKilled); + CHECK(ret == 0); + // Exact, since the tolerance takes these ints for tiny floats. + CHECK(out.find("2 pass (2 exact, 0 within tol), 0 fail") != std::string::npos); + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 0 skipped") != + std::string::npos); +} + +// An allocation larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB is not snapshotted and +// is listed as unreplayable, so replay cannot match. Within the cap it is. +HRR_TEST_CASE(Unit_HRR_PinnedHostSnapshotCap) { + for (const char* max_mb : {"2", "1"}) { + const bool over = *max_mb == '1'; + INFO("HIP_HRR_HOST_SNAPSHOT_MAX_MB=" << max_mb); + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_cap"}; + hrr_capture_direct("Unit_HRR_PinnedHostLarge_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOT_MAX_MB", max_mb}}); + const std::string manifest = + read_text_file(hrr_single_process_archive(cap.path) / "manifest.json"); + INFO("manifest.json:\n" << manifest); + CHECK((manifest.find("larger than HIP_HRR_HOST_SNAPSHOT_MAX_MB") != std::string::npos) == + over); + auto [ret, out] = hrr_playback_env(cap.path, {{"HIP_HRR_D2H_EXACT", "1"}}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + int pass = 0, fail = 0; + REQUIRE(hrr_parse_d2h_summary(out, pass, fail)); + if (over) { + CHECK(ret != 0); + CHECK(fail > 0); + CHECK(out.find("Pinned host snapshots") == std::string::npos); + } else { + CHECK(ret == 0); + CHECK(pass == kPinnedIters); + CHECK(fail == 0); + CHECK(out.find("Pinned host snapshots: 3 restored, 0 refused, 1 skipped") != + std::string::npos); + } + } +} + +// A snapshot blob over --max-file-bytes ends the replay before the kernel that +// reads it is launched on input replay never restored. +HRR_TEST_CASE(Unit_HRR_PinnedHostBlobRefused) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_refused"}; + hrr_capture_direct("Unit_HRR_PinnedHostLarge_Direct", cap.path, 5); + auto [ret, out] = hrr_playback_merged(cap.path, "--trace-kernels --max-file-bytes 2097151"); + INFO("Playback exit " << ret << ", output:\n" << out); + CHECK(ret != 0); + CHECK(out.find("refusing to read") != std::string::npos); + CHECK(out.find("[HRR launch]") == std::string::npos); +} + +// Capture stops tracking an allocation on hipHostFree, hipFree, hipFreeHost and +// hipHostUnregister: had it snapshotted the stale address, replay would refuse +// the record for an allocation it released. +HRR_TEST_CASE(Unit_HRR_PinnedHostFreedRoundtrip) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_freed"}; + hrr_capture_direct("Unit_HRR_PinnedHostFreed_Direct", cap.path, 3); + auto [ret, out] = hrr_playback_env(cap.path, {}); + INFO("Playback exit " << ret << ", stdout:\n" << out); + CHECK(ret == 0); + CHECK(out.find("Pinned host snapshots") == std::string::npos); +} + +#ifndef _WIN32 +// The pinned tracking lock is held across fork(), so no child inherits it locked. +HRR_TEST_CASE(Unit_HRR_PinnedHostFork) { + ScopedDir cap{fs::temp_directory_path() / "hrr_pinned_host_fork"}; + hrr::test::SpawnProc proc(hrr_self_exe()); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + const int ret = proc.runWithTimeout("\"Unit_HRR_PinnedHostFork_Direct\"", 300); + INFO("Capture exit code: " << ret); + REQUIRE(ret == 0); +} +#endif +#endif // HRR_PLAYBACK_EXE && HRR_TEST_EXE + +/** + * @} + */ diff --git a/projects/hrr/tests/integration/hrr_roundtrip_test.cc b/projects/hrr/tests/integration/hrr_roundtrip_test.cc index 92f06a26e904..79ce3e400d63 100644 --- a/projects/hrr/tests/integration/hrr_roundtrip_test.cc +++ b/projects/hrr/tests/integration/hrr_roundtrip_test.cc @@ -673,7 +673,10 @@ HRR_TEST_CASE(Unit_HRR_ZeroInitRoundtrip) { */ HRR_TEST_CASE(Unit_HRR_DivergenceAbortRoundtrip) { ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_divergence"}; - hrr_capture_direct("Unit_HRR_UncapturedHostWrite_Direct", cap.path); + // Pinned host snapshots would record the flag, so the write stays unseen only + // without them. + hrr_capture_direct("Unit_HRR_UncapturedHostWrite_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOTS", "0"}}); SECTION("guard ON -> clean exit 2") { auto [ret, out] = hrr_playback_env( @@ -723,7 +726,8 @@ HRR_TEST_CASE(Unit_HRR_DivergenceAbortRoundtrip) { */ TEST_CASE("Unit_HRR_NullOptionalPtrRoundtrip", "[.][hrr-repro]") { ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_nulloptional"}; - hrr_capture_direct("Unit_HRR_NullOptionalPtr_Direct", cap.path); + hrr_capture_direct("Unit_HRR_NullOptionalPtr_Direct", cap.path, 5, + {{"HIP_HRR_HOST_SNAPSHOTS", "0"}}); auto [ret, out] = hrr_playback_env( cap.path, diff --git a/projects/hrr/tests/unit/hrr_format_test.cc b/projects/hrr/tests/unit/hrr_format_test.cc index 2359df2be75c..711440d011b9 100644 --- a/projects/hrr/tests/unit/hrr_format_test.cc +++ b/projects/hrr/tests/unit/hrr_format_test.cc @@ -467,3 +467,85 @@ HRR_TEST_CASE(Unit_HRR_Format_UnknownEventType) { CHECK(archive.events[0].header().event_type == 0xFFFFu); CHECK(archive.events[0].malloc_ev.ptr_handle == 0u); } + +// A kernel launch event with no arguments and one pinned host snapshot record, +// cut `cut` bytes short of a whole record (v10 layout, as capture writes it). +static std::vector make_launch_with_snapshot(uint64_t seq, + const std::vector& rec, + size_t cut = 0) { + std::vector p(sizeof(hrr_event_header), 0); + auto put = [&](const void* d, size_t n) { + p.insert(p.end(), static_cast(d), static_cast(d) + n); + }; + const uint64_t stream = 0, co_hash[2] = {0, 0}; + const uint16_t name_len = 1, counts[2] = {0, 1}; // num_args, num_snapshots + const uint32_t dims[7] = {1, 1, 1, 1, 1, 1, 0}, attrs[2] = {0, 0}; + put(&stream, 8); put(&name_len, 2); put("k", 1); put(co_hash, 16); + put(dims, sizeof(dims)); put(counts, sizeof(counts)); + put(rec.data(), rec.size() - cut); + if (!cut) put(attrs, sizeof(attrs)); + auto* h = reinterpret_cast(p.data()); + h->event_type = static_cast(HRR_API_HIPMODULELAUNCHKERNEL); + h->sequence_id = seq; + h->payload_length = static_cast(p.size()); + return p; +} + +/** + * Test Description + * ---------------- + * - A pinned host snapshot record round-trips through the reader, and replay + * points it at its own pinned buffer at the recorded offset. + * - A launch whose record runs past the event is not decoded by the reader + * nor by replay, and replay refuses a record that names a device allocation or the inside of one, + * runs past its allocation, differs from its blob in length, or has an + * unknown direction. + */ +HRR_TEST_CASE(Unit_HRR_Format_HostSnapshotRecord) { + auto record = [](uint64_t ptr, uint64_t off, uint64_t len, uint8_t direction) { + const uint64_t f[5] = {ptr, off, len, 0x1122334455667788ULL, 0x99aabbccddeeff00ULL}; + std::vector r(kHostSnapRecordSize); + std::memcpy(r.data(), f, sizeof(f)); + r[40] = direction; + return r; + }; + const auto good = record(0x2000, 16, 64, 0); + TmpArchive arc("host_snapshot"); + auto body = make_launch_with_snapshot(0, good); + const auto torn = make_launch_with_snapshot(1, good, 1); + body.insert(body.end(), torn.begin(), torn.end()); + arc.write_events(body); + + hrr::Archive archive; + REQUIRE(hrr::load_archive(arc.path(), archive)); + REQUIRE(archive.events.size() == 2); + REQUIRE(archive.events[0].kernel_launch != nullptr); + REQUIRE(archive.events[0].kernel_launch->snapshots.size() == 1); + const hrr::BufferSnapshot& s = archive.events[0].kernel_launch->snapshots[0]; + CHECK(s.ptr_handle == 0x2000); + CHECK(s.offset == 16); + CHECK(s.length == 64); + CHECK(s.hash_lo == 0x1122334455667788ULL); + CHECK(s.hash_hi == 0x99aabbccddeeff00ULL); + CHECK(s.direction == 0); + CHECK(archive.events[1].kernel_launch == nullptr); + CHECK(hrr_host_snapshots_fit(good.data(), good.data() + good.size(), 1)); + CHECK_FALSE(hrr_host_snapshots_fit(good.data(), good.data() + good.size() - 1, 1)); + + PlaybackContext ctx; + std::vector pinned(128), device(128), blob(64); + ctx.record_alloc(0x2000, pinned.data(), pinned.size(), AllocKind::HostMalloc); + ctx.record_alloc(0x3000, device.data(), device.size(), AllocKind::Device); + const char* why = nullptr; + CHECK(hrr_host_snapshot_target(ctx, good.data(), blob.data(), blob.size(), &why) == + pinned.data() + 16); + CHECK(why == nullptr); + CHECK(hrr_host_snapshot_target(ctx, good.data(), nullptr, 0, &why) == nullptr); + for (const auto& bad : {record(0x3000, 0, 64, 0), record(0x2008, 0, 64, 0), + record(0x2000, 80, 64, 0), record(0x2000, 16, 32, 0), + record(0x2000, 16, 64, 1)}) { + why = nullptr; + CHECK(hrr_host_snapshot_target(ctx, bad.data(), blob.data(), blob.size(), &why) == nullptr); + CHECK(why != nullptr); + } +} diff --git a/projects/hrr/tools/gen_hrr_api_args.py b/projects/hrr/tools/gen_hrr_api_args.py index 2dbbd07189bc..7188b0c8a94e 100644 --- a/projects/hrr/tools/gen_hrr_api_args.py +++ b/projects/hrr/tools/gen_hrr_api_args.py @@ -2073,8 +2073,10 @@ def validate_capture_coverage(entries: List[ApiEntry], * reader decodes those events as the wrong API. * v9: hipKernelSetAttributeForDevice was appended to HipDispatchTable, * taking the first compiler ID and shifting the compiler API IDs up by one. - * A v8 reader would decode those events as the wrong API. */ -#define HRR_VERSION ((uint16_t)9u) + * A v8 reader would decode those events as the wrong API. + * v10: a kernel launch event carries pinned host snapshot records after its + * arguments (num_snapshots was always 0). A v9 reader would not restore them. */ +#define HRR_VERSION ((uint16_t)10u) /* Written once at byte 0 of events.bin. */ #pragma pack(push, 1)