balajinvda · Jul 22, 2026 · Jul 20, 2026 · Jul 21, 2026 · Jul 21, 2026 · Jul 21, 2026
@@ -48,7 +48,7 @@ spec:
         - name: nvsnap-pull-secret
       containers:
         - name: agent
-          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
           imagePullPolicy: Always
           args:
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint

@@ -107,7 +107,7 @@ spec:
         - name: nvsnap-pull-secret
       containers:
         - name: agent
-          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
           imagePullPolicy: Always
           args:
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint
@@ -140,7 +140,7 @@ spec:
             - nvcr.io/0651155215864979/ncp-dev/uvloop-builder:v0.0.1
             - nvcr.io/0651155215864979/ncp-dev/libuv-builder:v0.0.1
             - nvcr.io/0651155215864979/ncp-dev/libzmq-builder:v0.0.1
-            - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+            - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
           securityContext:
             privileged: true  # Required for CRIU and containerd access
           env:

@@ -50,7 +50,7 @@ spec:
         - { name: nvsnap-lib, mountPath: /nvsnap-lib }
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -69,7 +69,7 @@ spec:
         - { name: nvsnap-lib, mountPath: /nvsnap-lib }
 
     - name: get-nvsnap
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -23,7 +23,7 @@ spec:
 
   initContainers:
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -46,7 +46,7 @@ spec:
     # in place. Source image is the agent v0.0.20 — same restore-entrypoint
     # binary the production deploy will use.
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -68,7 +68,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -87,7 +87,7 @@ spec:
     # Drops both libnvsnap_intercept.so and the runtime sitecustomize.py.
     # Renamed from get-intercept-lib to get-nvsnap to reflect the broader job.
     - name: get-nvsnap
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -67,7 +67,7 @@ spec:
 
   # Copy CRIU bundle, intercept library, and debug tools from agent image
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -72,7 +72,7 @@ spec:
 
   # Copy intercept lib from agent
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -72,7 +72,7 @@ spec:
 
   # Copy CRIU bundle, intercept library, and debug tools from agent image
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -71,7 +71,7 @@ spec:
           mountPath: /nvsnap-lib
   # Copy intercept lib from agent
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -62,7 +62,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -75,7 +75,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

@@ -1,7 +1,4 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
  * nvsnap-cuda-checkpoint: a drop-in replacement for NVIDIA's `cuda-checkpoint`
  * binary, built directly on the CUDA driver checkpoint API (the CUDA_CHECKPOINT
  * group: cuCheckpointProcess{GetState,GetRestoreThreadId,Lock,Checkpoint,
@@ -11,7 +8,7 @@
  * by name on PATH) can call this instead, with no other changes:
  *
  *   --get-state       --pid <pid>
- *   --action lock|checkpoint|restore|unlock --pid <pid> [--timeout <ms>]
+ *   --action lock|checkpoint|restore|unlock|resume --pid <pid> [--timeout <ms>]
  *   --toggle          --pid <pid>
  *   --get-restore-tid --pid <pid>
  *
@@ -83,6 +80,15 @@ static int do_unlock(int pid)
     return r == CUDA_SUCCESS ? 0 : err_action("unlock", pid, r);
 }
 
+/* resume: restore then unlock in one process, so cuInit (the ~2.7s driver
+ * attach) is paid once instead of once per action. Used by the CRIU cuda
+ * plugin on the common restore path (process was running at checkpoint). */
+static int do_resume(int pid)
+{
+    int rc = do_restore(pid);
+    return rc ? rc : do_unlock(pid);
+}
+
 static int do_get_state(int pid)
 {
     CUprocessState s;
@@ -134,7 +140,7 @@ static void usage(const char *p)
         "nvsnap-cuda-checkpoint: CUDA checkpoint/restore via the driver API.\n"
         "Operations:\n"
         "  --get-state       --pid <pid>\n"
-        "  --action lock|checkpoint|restore|unlock --pid <pid> [--timeout <ms>]\n"
+        "  --action lock|checkpoint|restore|unlock|resume --pid <pid> [--timeout <ms>]\n"
         "  --toggle          --pid <pid>\n"
         "  --get-restore-tid --pid <pid>\n"
         "Options:\n"
@@ -201,6 +207,7 @@ int main(int argc, char **argv)
     if (!strcmp(action, "checkpoint")) return do_checkpoint(pid);
     if (!strcmp(action, "restore"))    return do_restore(pid);
     if (!strcmp(action, "unlock"))     return do_unlock(pid);
+    if (!strcmp(action, "resume"))     return do_resume(pid);
 
     fprintf(stderr, "error: unknown action '%s'\n", action);
     usage(argv[0]);

@@ -1,29 +1,36 @@
 # NVSNAP Checkpoint/Restore Benchmarks
 
-## Huge-page CRIU restore (Lever A), July 2026
+## Faster criu-v2 restore (Levers A + B), July 2026
 
 Cluster: aws-dev1, H100 80GB (single GPU), driver 580.126, criu-v2 engine.
-The CRIU fork now `madvise(MADV_HUGEPAGE)`s the premapped private-VMA arena
-before faulting in the process image, so the CPU-side memory installs as 2MB
-pages instead of ~800k 4KB pages. Matched agent-restore time, before (app
-v0.2.20) vs after (app v0.2.21, base v0.0.11, criu fork `1ddd5c9c3`):
-
-| Workload | Checkpoint | Before | After | Delta |
-|---|---:|---:|---:|---:|
-| vllm-small (TinyLlama 1.1B) | 3.2G | 44.3 s | 32.2 s | -27% |
-| vllm-8b (Llama-3.1-8B) | 18G | 58.9 s | 38.5 s | -35% |
-| e5-mistral-7b | 15G | 84.0 s | 40.8 s | -51% |
-| vllm-qwen32b (Qwen2.5-32B) | 64G | 77.4 s | 48.5 s | -37% |
-
-All four tested single-GPU workloads restore faster; larger CPU-side footprints
-generally benefit, although the improvement is not monotonic in this sample.
-Phase split from the vllm-small `-v4` restore.log:
-the CPU pages-restore phase dropped 13.5 s to 3.8 s (-72%); the cuda_plugin GPU
-restore phase (~27 s) is unchanged. No lasting memory cost: `AnonHugePages` is 0
-in the final restored process because the huge pages exist only transiently in
-the CRIU staging arena during population and split on the remap to final
-addresses. Best-effort and safe: on a non-THP node the madvise is a no-op and
-restore proceeds unchanged. Design: `docs/proposals/single-gpu-restore-speedup.md`.
+Two restore-path optimizations, measured as matched agent-restore time.
+
+Lever A (base v0.0.11 / app v0.2.21, criu fork `1ddd5c9c3`): the CRIU fork
+`madvise(MADV_HUGEPAGE)`s the premapped private-VMA arena before faulting in the
+process image, so the CPU-side memory installs as 2MB pages instead of ~800k 4KB
+pages.
+
+Lever B (base v0.0.12 / app v0.2.22, criu fork `1e926fa4d`): each cuda-checkpoint
+spawn pays a fixed ~2.7 s cuInit driver-attach. On the common restore path a
+process needs both a restore and an unlock; the plugin now issues them as one
+combined "resume" invocation (one cuInit instead of two), so the win scales with
+the GPU pid count.
+
+| Workload | Checkpoint | Baseline (v0.2.20) | + A (v0.2.21) | + A+B (v0.2.22) | Total |
+|---|---:|---:|---:|---:|---:|
+| vllm-small (TinyLlama 1.1B) | 3.2G | 44.3 s | 32.2 s | 25.8 s | -42% |
+| vllm-8b (Llama-3.1-8B) | 18G | 58.9 s | 38.5 s | 32.4 s | -45% |
+| e5-mistral-7b | 15G | 84.0 s | 40.8 s | 36.6 s | -56% |
+| vllm-qwen32b (Qwen2.5-32B) | 64G | 77.4 s | 48.5 s | 44.3 s | -43% |
+
+Every single-GPU workload restores faster at both steps; the A win grows with
+the process's CPU-side memory footprint, the B win with the GPU pid count.
+Phase split from the vllm-small `-v4` restore.log: A cut the CPU pages-restore
+phase 13.5 s to 3.8 s (-72%); B cut the cuda_plugin GPU restore phase by one
+cuInit per pid (~2.7 s x 2 pids). Both are best-effort and correctness-neutral:
+A is a no-op on a non-THP node, B issues the same driver operations in the same
+order. 5/5 single-GPU e2e PASS on the A+B build. Design:
+`docs/proposals/single-gpu-restore-speedup.md`.
 
 ## Warm cache restore (cachedir), June 2026
 

@@ -93,26 +93,29 @@ Degradation by node THP state:
 Worst case is "no gain on that node", never "broken restore". Safe to ship to
 arbitrary customer clusters.
 
-### B. Parallelize the per-process CUDA restore (attacks the 27s serial phase) -- PROPOSED
-
-cuda_plugin restores GPU processes sequentially: pid 53 (13.5->22s) then pid 453
-(22->38s). On one physical GPU the context rebuilds may not fully overlap, but
-the ~3s per-proc "find restore thread" handshake gaps (spawn cuda-checkpoint +
-cuInit + query) are pure latency that can overlap. We already parallelized
-cuda-checkpoint loops in the legacy restore.go (#216); this is the criu-v2
-equivalent inside the plugin.
-
-```text
-today:   [ctx rebuild 53][gap][ctx rebuild 453][gap]      ~27s
-target:  [ctx rebuild 53          ]
-         [gap][ctx rebuild 453     ]  (overlapped)         ~15s
-```
-
-Ceiling: the ~27s cuda phase -> ~15s (bounded by the single slowest context +
-GPU serialization of concurrent context creation).
-Risk: two contexts building on one GPU may serialize in the driver anyway;
-measure the real overlap before committing. Handshake-latency removal is the
-safe portion.
+### B. Coalesce cuda-checkpoint spawns to cut cuInit cost -- EASY HALF DONE
+
+Each cuda-checkpoint invocation pays a flat ~2.7s cuInit driver-attach (measured:
+6 successive --get-state calls on a running process, all ~2.7s, so it is driver
+attach, not a wait). The plugin spawns cuda-checkpoint ~3x per pid on the restore
+path (get-restore-tid, restore, unlock). Two collapses:
+
+- DONE (easy, shipped): on the common restore path a process needs both a restore
+  and an unlock; the plugin now issues them as one combined "resume" invocation
+  (one cuInit instead of two). Same driver operations, same order, one fewer
+  spawn per pid. Measured -4 to -6s per single-GPU workload; the win scales with
+  the GPU pid count.
+- NOT done (medium): eliminate the get-restore-tid spawn by scanning
+  /proc/<pid>/task for the restore-thread name instead of spawning
+  cuda-checkpoint --get-restore-tid. Riskier: a wrong tid signals the wrong
+  thread.
+
+The HARD half is truly parallelizing the per-pid context rebuilds. CRIU calls the
+cuda_plugin resume hook one pid at a time as it restores the task tree serially,
+and a UVM-ordering constraint ("the process must stay frozen or UVM pointer access
+crashes") means parallelizing needs a CRIU-core change and risks a crash-on-restore.
+Deferred: high risk, and much of the residual is the context rebuild's module/kernel
+reload, an NVIDIA-side cost.
 
 ### C. Warm-standby: never release the context (attacks the ENTIRE 27s) -- OUT OF SCOPE
 

@@ -42,8 +42,8 @@ NVSNAP_CRIU_SRC="${NVSNAP_CRIU_SRC:-}"
 
 # Core images — all on a fresh v0.0.1 baseline. Bump per CLAUDE.md
 # rule 19 (never reuse a tag on rebuild).
-NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.11}"
-NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.21}"
+NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.12}"
+NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.22}"
 NVSNAP_SERVER_VERSION="${NVSNAP_SERVER_VERSION:-v0.0.31}"
 NVSNAP_BLOBSTORE_VERSION="${NVSNAP_BLOBSTORE_VERSION:-v0.0.1}"
 NVSNAP_L2WAIT_VERSION="${NVSNAP_L2WAIT_VERSION:-v0.0.1}"
@@ -67,7 +67,7 @@ NVSNAP_CRIU_REPO="${NVSNAP_CRIU_REPO:-https://github.com/balajinvda/criu.git}"
 # rings survive C/R, so libuv/uvloop servers restore with UV_USE_IO_URING=1)
 # for reproducible OSS builds. Bump when the fork advances.
 # Consumed by build-agent.sh (clean-checkout auto-clone) and ci/build-image.sh.
-NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-1ddd5c9c3951f994598b540917be462c338b518d}"
+NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-1e926fa4da5c1309c3029bedbeef103c1459438d}"
 NVSNAP_LIBZMQ_REPO="${NVSNAP_LIBZMQ_REPO:-https://github.com/balajinvda/libzmq.git}"
 NVSNAP_LIBZMQ_REF="${NVSNAP_LIBZMQ_REF:-checkpoint-restore-v1}"
 NVSNAP_LIBUV_REPO="${NVSNAP_LIBUV_REPO:-https://github.com/balajinvda/libuv.git}"