From 9fc43843322b788601638e3ce7e955473793eb6c Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:59:53 +0800 Subject: [PATCH 1/3] [Bugfix][Weight Cache] Fingerprint sampled tensor bytes, not headers only Fixes #59647. hash_checkpoint() hashed only each shard's safetensors header (names, dtypes, shapes, offsets), so two checkpoints with the same layout but different weight values, e.g. a base model and its fine-tune, produced the same WeightCacheKey. An engine started with --load-format ipc_cache against a daemon holding the other checkpoint then mapped the stale weights with no warning, while the CLI and API reported the new path. Mix the file size and the head (1 KiB) of every tensor's data region into the per-shard hash. The key stays stable for byte-identical copies in different directories (the #54921 property), while any real fine-tune moves it. Reads stay at a few hundred KB per shard, negligible next to weight loading. Co-authored-by: Kimi Code Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> --- docs/features/preload.md | 5 +- .../model_loader/test_weight_cache.py | 51 +++++++++++++++++++ .../model_loader/weight_cache/protocol.py | 50 +++++++++++++++--- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/docs/features/preload.md b/docs/features/preload.md index 31aa9e2db3ec..e05fe07d23c7 100644 --- a/docs/features/preload.md +++ b/docs/features/preload.md @@ -79,8 +79,9 @@ vllm preload --model /path/to/model --tensor-parallel-size 4 meta device (no storage) and requests the tensors from the daemon on its GPU. 3. Before serving anything, the engine and daemon compare a fingerprint of the - cached weights: checkpoint content (hashed from safetensors metadata, so - identical weights in different directories still match), model + cached weights: checkpoint content (hashed from safetensors metadata plus + sampled tensor bytes, so identical weights in different directories still + match, while a checkpoint whose weight values changed does not), model architecture, TP/DP size/rank, dtype, quantization method and config, text RoPE parameters, model revision, KV cache dtype and quantization skip layers, and vLLM version. On any mismatch the engine falls back to disk loading diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 0d27d9ccd37b..a7b09c147c3b 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -641,6 +641,57 @@ def test_ipc_rope_matching( server_task.result(timeout=5) +def test_hash_checkpoint_distinguishes_weight_values(tmp_path): + # Same layout, different values: a base checkpoint and its fine-tune must + # not share a key, or an ipc_cache engine silently maps the daemon's stale + # weights (#59647). + import torch + from safetensors.torch import save_file + + from vllm.model_executor.model_loader.weight_cache.protocol import ( + hash_checkpoint, + ) + + for name, value in (("base", 0.0), ("finetuned", 1.0)): + (tmp_path / name).mkdir() + save_file( + {"w": torch.full((4, 4), value)}, + str(tmp_path / name / "model.safetensors"), + ) + assert hash_checkpoint(str(tmp_path / "base")) != hash_checkpoint( + str(tmp_path / "finetuned") + ) + + +def test_hash_checkpoint_stable_across_directory_copies(tmp_path): + # The #54921 property: byte-identical checkpoints under different paths + # share a key, so a daemon preloaded from one directory can serve an + # engine started on the copy. + import torch + from safetensors.torch import save_file + + from vllm.model_executor.model_loader.weight_cache.protocol import ( + hash_checkpoint, + ) + + (tmp_path / "a").mkdir() + save_file( + {"w": torch.arange(4096, dtype=torch.float32).reshape(64, 64)}, + str(tmp_path / "a" / "model.safetensors"), + ) + shutil.copytree(tmp_path / "a", tmp_path / "b") + assert hash_checkpoint(str(tmp_path / "a")) == hash_checkpoint(str(tmp_path / "b")) + + +def test_hash_checkpoint_without_safetensors_returns_none(tmp_path): + from vllm.model_executor.model_loader.weight_cache.protocol import ( + hash_checkpoint, + ) + + assert hash_checkpoint(str(tmp_path)) is None + assert hash_checkpoint(str(tmp_path / "not-a-dir")) is None + + def test_ipc_loader_copy_mode_reports_no_external_weight_memory(): """Copy mode clones the weights into the engine, so nothing is external (vllm_config is never touched).""" diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index b2bca701ea57..bbfa4195f79c 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -276,13 +276,47 @@ def _safetensors_header(path: str) -> bytes: return size_bytes + f.read(header_len) -def hash_checkpoint(model: str) -> str | None: - """Fingerprint checkpoint content from local safetensors metadata. +_TENSOR_HEAD_BYTES = 1024 + + +def _mix_safetensors_fingerprint(path: str, hasher: Any) -> None: + """Mix one shard's layout and sampled weight bytes into ``hasher``. + + The header carries tensor names, dtypes, shapes and offsets but no weight + values, so hashing it alone gives a base model and its fine-tune the same + key (#59647). Sampling the head of every tensor's data region catches a + value-only change at the cost of one small read per tensor, and stays + stable for byte-identical copies in different directories. + """ + hasher.update(str(os.path.getsize(path)).encode()) + with open(path, "rb") as f: + size_bytes = f.read(8) + (header_len,) = struct.unpack(" str | None: + """Fingerprint checkpoint content from local safetensors files. + + Hashes each shard's metadata plus sampled tensor bytes so a daemon and an + engine pointing at identical weights in different directories produce the + same key, while checkpoints that differ only in weight values do not. + Returns None when local safetensors files can't be located (e.g. an + undownloaded Hugging Face repo id), leaving the caller to fall back to the + model path. """ if not os.path.isdir(model): return None @@ -296,7 +330,7 @@ def hash_checkpoint(model: str) -> str | None: hasher = safe_hash(b"", usedforsecurity=False) for path in sorted(files, key=os.path.basename): hasher.update(os.path.basename(path).encode()) - hasher.update(_safetensors_header(path)) + _mix_safetensors_fingerprint(path, hasher) return hasher.hexdigest() @@ -351,7 +385,7 @@ def from_model_config( may mutate hf_config.quantization_config, which would change the hash between the daemon and the engine. - The checkpoint is identified by a hash of its safetensors metadata when + The checkpoint is identified by a hash of its safetensors files when the weights are available locally, so a daemon and engine referencing identical weights in different directories still match; otherwise it falls back to the model path. From 80a015effa522eadb8862cc6d8ae65a6279d1fbd Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Tue, 6 Oct 2026 05:36:48 +0800 Subject: [PATCH 2/3] [Perf][Weight Cache] Read fingerprint samples concurrently On NFS each sampled pread costs one RPC RTT, so sequential per-tensor sampling of a K3-scale checkpoint (~70k tensors) takes over a minute at 1 ms RTT. Issue the sampled reads through a small thread pool (32 in flight) and mix them in sorted-name order as before. pool.map preserves order, so the digest is byte-identical to sequential sampling and keys already in the wild stay valid; wall time drops to a few seconds. Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Kimi Code --- .../model_loader/test_weight_cache.py | 24 +++++++++++++ .../model_loader/weight_cache/protocol.py | 34 +++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index a7b09c147c3b..9f8a93124b95 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -683,6 +683,30 @@ def test_hash_checkpoint_stable_across_directory_copies(tmp_path): assert hash_checkpoint(str(tmp_path / "a")) == hash_checkpoint(str(tmp_path / "b")) +def test_hash_checkpoint_catches_single_tensor_change_among_many(tmp_path): + # Every tensor's head contributes to the key, so changing one tensor out + # of 64 flips it. The sampled reads run through a thread pool, and 64 + # tensors exceed the worker count, so this also pins down that the digest + # does not depend on read scheduling. + import torch + from safetensors.torch import save_file + + from vllm.model_executor.model_loader.weight_cache.protocol import ( + hash_checkpoint, + ) + + tensors = {f"w{i:03d}": torch.full((8, 8), float(i)) for i in range(64)} + (tmp_path / "base").mkdir() + save_file(tensors, str(tmp_path / "base" / "model.safetensors")) + changed = dict(tensors) + changed["w037"] = torch.full((8, 8), -1.0) + (tmp_path / "changed").mkdir() + save_file(changed, str(tmp_path / "changed" / "model.safetensors")) + assert hash_checkpoint(str(tmp_path / "base")) != hash_checkpoint( + str(tmp_path / "changed") + ) + + def test_hash_checkpoint_without_safetensors_returns_none(tmp_path): from vllm.model_executor.model_loader.weight_cache.protocol import ( hash_checkpoint, diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index bbfa4195f79c..aabab6ccb15e 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -19,6 +19,7 @@ import stat import struct import tempfile +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, fields from typing import Any, NamedTuple @@ -277,16 +278,25 @@ def _safetensors_header(path: str) -> bytes: _TENSOR_HEAD_BYTES = 1024 +# Bounded concurrency for the sampled reads. On NFS each pread costs one RPC +# RTT, so sequential sampling of a K3-scale checkpoint (~70k tensors) would +# take over a minute at 1 ms RTT; 32 in-flight reads keep that at a few +# seconds without hammering the server. +_FINGERPRINT_READ_WORKERS = 32 -def _mix_safetensors_fingerprint(path: str, hasher: Any) -> None: +def _mix_safetensors_fingerprint( + path: str, hasher: Any, pool: ThreadPoolExecutor +) -> None: """Mix one shard's layout and sampled weight bytes into ``hasher``. The header carries tensor names, dtypes, shapes and offsets but no weight values, so hashing it alone gives a base model and its fine-tune the same key (#59647). Sampling the head of every tensor's data region catches a value-only change at the cost of one small read per tensor, and stays - stable for byte-identical copies in different directories. + stable for byte-identical copies in different directories. The reads go + through ``pool`` but are mixed in sorted-name order, so the digest is + identical to sequential sampling. """ hasher.update(str(os.path.getsize(path)).encode()) with open(path, "rb") as f: @@ -300,12 +310,21 @@ def _mix_safetensors_fingerprint(path: str, hasher: Any) -> None: except json.JSONDecodeError: return data_start = 8 + header_len + plan = [] for name in sorted(entries): offsets = entries[name].get("data_offsets") if not offsets: continue - f.seek(data_start + offsets[0]) - hasher.update(f.read(min(_TENSOR_HEAD_BYTES, offsets[1] - offsets[0]))) + plan.append( + ( + data_start + offsets[0], + min(_TENSOR_HEAD_BYTES, offsets[1] - offsets[0]), + ) + ) + fd = f.fileno() + samples = pool.map(lambda spec: os.pread(fd, spec[1], spec[0]), plan) + for sample in samples: + hasher.update(sample) def hash_checkpoint(model: str) -> str | None: @@ -328,9 +347,10 @@ def hash_checkpoint(model: str) -> str | None: if not files: return None hasher = safe_hash(b"", usedforsecurity=False) - for path in sorted(files, key=os.path.basename): - hasher.update(os.path.basename(path).encode()) - _mix_safetensors_fingerprint(path, hasher) + with ThreadPoolExecutor(max_workers=_FINGERPRINT_READ_WORKERS) as pool: + for path in sorted(files, key=os.path.basename): + hasher.update(os.path.basename(path).encode()) + _mix_safetensors_fingerprint(path, hasher, pool) return hasher.hexdigest() From c4bf23eeba642b60a47b939ef3d3b632526329e1 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:25:03 +0800 Subject: [PATCH 3/3] [Docs][Weight Cache] Qualify sampled fingerprint coverage Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> --- docs/features/preload.md | 5 +++-- .../model_loader/weight_cache/protocol.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/features/preload.md b/docs/features/preload.md index 3ddf5c64c877..f8a51bb08fbe 100644 --- a/docs/features/preload.md +++ b/docs/features/preload.md @@ -129,8 +129,9 @@ state rides the data plane. GPU. 3. Before serving anything, the engine and daemon compare a fingerprint of the cached weights: checkpoint content (hashed from safetensors metadata plus - sampled tensor bytes, so identical weights in different directories still - match, while a checkpoint whose weight values changed does not), model + the first 1 KiB of each tensor, so identical weights in different + directories still match and a change in any sampled region also changes + the key), model architecture, TP/DP size/rank, dtype, quantization method and config, text RoPE parameters, model revision, KV cache dtype and quantization skip layers, and vLLM version. On any mismatch the engine falls back to disk loading diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index 941e996dcbe8..5161686f5fff 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -296,9 +296,10 @@ def _mix_safetensors_fingerprint( The header carries tensor names, dtypes, shapes and offsets but no weight values, so hashing it alone gives a base model and its fine-tune the same - key (#59647). Sampling the head of every tensor's data region catches a - value-only change at the cost of one small read per tensor, and stays - stable for byte-identical copies in different directories. The reads go + key (#59647). Sampling the head of every tensor's data region catches + value-only changes within that region at the cost of one small read per + tensor, and stays stable for byte-identical copies in different + directories. The reads go through ``pool`` but are mixed in sorted-name order, so the digest is identical to sequential sampling. """ @@ -336,7 +337,8 @@ def hash_checkpoint(model: str) -> str | None: Hashes each shard's metadata plus sampled tensor bytes so a daemon and an engine pointing at identical weights in different directories produce the - same key, while checkpoints that differ only in weight values do not. + same key, while a difference within any tensor's sampled head produces a + different key. Returns None when local safetensors files can't be located (e.g. an undownloaded Hugging Face repo id), leaving the caller to fall back to the model path.