danielbolsson · Sep 30, 2026 · Aug 24, 2026 · Aug 27, 2026 · Aug 24, 2026 · Aug 24, 2026
@@ -24,7 +24,6 @@ extern hipError_t ihipGetDeviceProperties(hipDeviceProp_t* props, hipDevice_t de
 
 namespace hrr_cap {
 namespace metadata {
-namespace {
 
 std::string json_escape(const char* s) {
   std::string out;
@@ -54,7 +53,9 @@ std::string json_escape(const char* s) {
   return out;
 }
 
-std::string json_escape(const std::string& s) { return json_escape(s.c_str()); }
+namespace {
+
+std::string json_escape(const std::string& s) { return metadata::json_escape(s.c_str()); }
 
 std::string quote(const std::string& s) { return "\"" + json_escape(s) + "\""; }
 

@@ -16,5 +16,10 @@ namespace metadata {
 // APIs that would re-enter hip::init().
 std::string collect_json();
 
+// Escapes `s` for use inside a JSON string literal (quotes not included):
+// backslash, double quote, the short control escapes and any other byte below
+// 0x20 as \u00XX. A null `s` yields "".
+std::string json_escape(const char* s);
+
 }  // namespace metadata
 }  // namespace hrr_cap
@@ -35,7 +35,9 @@
 #include <cstring>
 #include <cerrno>
 #include <filesystem>
+#include <map>
 #include <mutex>
+#include <set>
 #include <string>
 #include <unordered_set>
 #include <algorithm>
@@ -216,6 +218,15 @@ static std::atomic<uint64_t> g_blob_count{0};
 static std::mutex                      g_blob_mu;
 static std::unordered_set<std::string> g_written_blobs;
 
+// APIs recorded in this archive that replay cannot reproduce (note_unreplayable).
+// Listed in manifest.json so the gap is a property of the archive rather than
+// something only visible in a replay log.
+static std::mutex                      g_unreplayable_mu;
+// api -> distinct reasons it was noted for. One API can be unreplayable for
+// several reasons (e.g. a different truncated argument on different calls),
+// and each is worth reporting.
+static std::map<std::string, std::set<std::string>> g_unreplayable_apis;
+
 // ---------------------------------------------------------------------------
 // Low-level fd helpers
 // ---------------------------------------------------------------------------
@@ -499,6 +510,26 @@ static void write_manifest_stdio(const char* output_dir, bool complete) {
           complete ? "true" : "false",
           static_cast<unsigned long long>(g_event_count.load()),
           static_cast<unsigned long long>(g_blob_count.load()));
+  {
+    std::lock_guard<std::mutex> lk(g_unreplayable_mu);
+    if (!g_unreplayable_apis.empty()) {
+      fprintf(mf, ",\n  \"unreplayable_apis\": {");
+      bool first_api = true;
+      for (const auto& [api, reasons] : g_unreplayable_apis) {
+        fprintf(mf, "%s\n    \"%s\": [", first_api ? "" : ",",
+                metadata::json_escape(api.c_str()).c_str());
+        bool first_reason = true;
+        for (const auto& reason : reasons) {
+          fprintf(mf, "%s\"%s\"", first_reason ? "" : ", ",
+                  metadata::json_escape(reason.c_str()).c_str());
+          first_reason = false;
+        }
+        fprintf(mf, "]");
+        first_api = false;
+      }
+      fprintf(mf, "\n  }");
+    }
+  }
   if (g_metadata_json_len > 0) {
     fprintf(mf, ",\n  \"metadata\": %.*s\n",
             static_cast<int>(g_metadata_json_len), g_metadata_json);
@@ -751,6 +782,23 @@ void mark_incomplete(const char* reason) {
 
 bool is_incomplete() { return g_capture_incomplete.load(std::memory_order_relaxed); }
 
+void note_unreplayable(const char* api, const char* reason) {
+  if (!api) return;
+  if (!reason) reason = "(unspecified)";
+  {
+    std::lock_guard<std::mutex> lk(g_unreplayable_mu);
+    if (!g_unreplayable_apis[api].insert(reason).second) return;
+  }
+  // Warning, not Error: unlike mark_incomplete() the archive is well-formed and
+  // every event is present — only the ability to re-execute this one call is
+  // lost. That is a degradation, not a failure.
+  // log_printf appends its own newline, so the format string omits it.
+  LogPrintfWarning(
+      "[HRR capture] %s cannot be replayed: %s. The call is recorded, but "
+      "replay will report it as unreplayable rather than reproduce it",
+      api, reason);
+}
+
 void flush(const char* /*output_dir*/) {
   // Always finalize the *effective* directory this process actually wrote to
   // (g_output_dir), which is always a pid-<pid> sub-archive. The caller passes

@@ -71,6 +71,22 @@ void mark_incomplete(const char* reason);
 // Returns true if mark_incomplete() has been called.
 bool is_incomplete();
 
+// Record that `api` was captured but cannot be replayed faithfully — the call
+// is in the archive, its effect is not reproducible (a host callback belonging
+// to this process, a handle only meaningful to another rank, an argument
+// truncated at capture, ...). Warns once per distinct (api, reason) at capture
+// time, so the gap is visible when the recording is made rather than only when
+// someone tries to replay it, and lists it in manifest.json as
+// "unreplayable_apis": {"<api>": ["<reason>", ...]}. Both strings are
+// JSON-escaped on write, so any text is safe to pass.
+//
+// Deliberately NOT mark_incomplete(): that means an event was dropped or torn
+// and suppresses the clean-shutdown trailer. Here every event is present and
+// the archive is well-formed; what is missing is the ability to re-execute
+// this particular call. Thread-safe; the warning fires once per (api, reason)
+// per process.
+void note_unreplayable(const char* api, const char* reason);
+
 // Write a buffer as a content-addressed blob. Returns hash.
 // Thread-safe. Skips write if blob already exists on disk.
 Hash128 write_blob(const void* data, size_t len);

@@ -1426,11 +1426,37 @@ manifest classifies it as payload loss and `derive_manifest.py` fails the build
 count drifts from its recorded baseline, so a newly-captured struct-input API cannot
 lose its payload unnoticed.
 
-### Recorded Failures Are Not Replay Failures
-
-A recorded call that *failed* is reproduced, not repaired: when the archived return is
-non-zero and replay reproduces the same error, that is the faithful outcome and the
-dispatcher reports it as such instead of as a handler failure.
+### Fail-Loud Scope Exclusions
+
+Some calls cannot be reproduced in a different process no matter how much of the
+argument is recorded. Rather than pass a null or a stale value and let the runtime
+report something unattributable, these are `UNREPLAYABLE_PLAYBACK_APIS`: the handler
+returns `hipErrorNotSupported` (fatal unless `--continue-on-error`), names itself and
+its reason on stderr, and the replay summary lists the archive as incomplete. Capture
+warns once at record time as well, so the incompleteness is visible when the recording
+is made and not only when someone tries to replay it. The exclusions are:
+
+- **Host callbacks** — `hipLaunchHostFunc`, `hipLaunchHostFunc_spt`,
+  `hipStreamAddCallback`, `hipStreamAddCallback_spt`. The callback is a function pointer
+  in the recording process; there is nothing to call at replay.
+- **Host nodes in a graph** — `hipGraphAddHostNode`, `hipGraphHostNodeSetParams`,
+  `hipGraphExecHostNodeSetParams`: the same function pointer reached through the graph
+  API. Like every exclusion these stop the replay at the call itself. Under
+  `--continue-on-error` the replay goes on, so the handler also marks the owning graph
+  incomplete, and instantiating that graph is then refused rather than run short a node.
+- **Cross-process handle import** — `hipMemImportFromShareableHandle`,
+  `hipMemPoolImportFromShareableHandle`. The exported fd or HANDLE is meaningful only
+  inside the exporting process and its peers, which a later replay is not.
+- **Multi-device launch by host function address** —
+  `hipLaunchCooperativeKernelMultiDevice`, `hipExtLaunchMultiKernelMultiDevice`.
+  `hipLaunchParams` names each kernel by a host function address and, unlike the
+  single-device spellings, has no entry point that takes a `hipFunction_t` instead.
+- **Host-object lifetime callbacks** — `hipUserObjectCreate`, whose destructor is a host
+  function pointer with the same problem as a stream callback.
+
+A recorded call that *failed* is a separate case and is not an exclusion: when the
+archived return is non-zero and replay reproduces the same error, that is the faithful
+outcome and the dispatcher reports it as such instead of as a handler failure.
 
 ## Relationship to Original HRR Code
 

@@ -162,7 +162,13 @@ the next event, so the divergence is confined to the launch under examination.
 | `HIP_HRR_REPLAY_NO_RESCAN` | off | Disable suballoc pointer rescan at replay |
 | `HIP_HRR_PTR_RELAX` | off | Disable replay-side stale-pointer guard (debug only) |
 | `HIP_HRR_REPLAY_FORCE_EXT_CIJK` | off | Force external Cijk kernel binding workaround (debug) |
-| `HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL` | `0` | Dump pointer translation map at event ordinal `N` (debug) |
+| `HIP_HRR_REPLAY_DUMP_PTRS_ORDINAL` | `0` | Dump pointer translation map, and the allocation each argument lands in, at kernel ordinal `N` (debug) |
+| `HIP_HRR_REPLAY_SCAN_ARGS_ORDINAL` | `0` | Before kernel `N`, read back each pointer argument's allocation and report words that are recorded addresses (debug) |
+| `HIP_HRR_REPLAY_SCAN_ARGS_BYTES` | `4096` | Per-allocation cap for the argument scan |
+| `HIP_HRR_REPLAY_SCAN_H2D` | off | Report recorded addresses inside replayed H2D payloads (debug) |
+| `HIP_HRR_REPLAY_AUDIT_HOST_ARGS` | off | Report kernels taking a pointer into host memory, whose contents replay cannot restore (debug) |
+| `HIP_HRR_REPLAY_FILL_BYTE` | `0` | Byte to fill fresh allocations with; set it to e.g. `0xa5` to expose kernels reading memory nothing wrote (debug) |
+| `HIP_HRR_REPLAY_EXPLAIN_ADDR` | unset | At the scan ordinal, report whether an address is recorded, live, or neither, and whether the GPU can read it (debug) |
 
 ### D2H validation