diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index 09e83581fab1..f19c0af72007 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -18,6 +18,10 @@ default_hasher = blake3 _INT32_MAX = 2**31 - 1 +# Versioned tag prefixed to every content hash so the canonical, self-describing +# serialization scheme can evolve without silently reusing stale cache keys. +_HASH_SCHEME_TAG = b"trtllm.mm.hash.v1" + def strip_mm_data_for_generation(mm_data: Dict[str, Any]) -> None: """Clear `mm_data` in place, retaining only `mrope_config.mrope_position_deltas`. @@ -666,21 +670,10 @@ class MultimodalServerConfig(): def _update_hash(hasher, item: object) -> None: """Hash the content of a multimodal item into the provided hasher.""" + hasher.update(_HASH_SCHEME_TAG) if isinstance(item, BaseModalityData): item.update_hash(hasher) return - if isinstance(item, torch.Tensor): - item = item.detach().cpu().contiguous() - hasher.update(serialize_item(item)) - return - if isinstance(item, list): - for element in item: - hasher.update(b"") - if isinstance(element, torch.Tensor): - element = element.detach().cpu().contiguous() - hasher.update(serialize_item(element)) - return - hasher.update(serialize_item(item)) @@ -711,7 +704,6 @@ def apply_mm_hashes( def _hash_item(item): """Hash only the content of a multimodal item (no UUID).""" - # TODO: possible hash collision w/ this simplified version (vllm/PR/17378) hasher = hash_lib() _update_hash(hasher, item) return hasher.hexdigest() diff --git a/tensorrt_llm/inputs/multimodal_data.py b/tensorrt_llm/inputs/multimodal_data.py index 6cee4ff199be..4c8e468e1f00 100644 --- a/tensorrt_llm/inputs/multimodal_data.py +++ b/tensorrt_llm/inputs/multimodal_data.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import struct from dataclasses import dataclass from typing import Any, Protocol @@ -8,6 +9,15 @@ import torch from PIL import Image +# Video metadata fields that participate in the cache-key hash. These describe +# how frames were sampled and therefore change the model-visible content. +_VIDEO_HASH_METADATA_FIELDS = ( + "frames_indices", + "fps", + "duration", + "total_num_frames", +) + class ContentHasher(Protocol): """Hash object that accepts bytes.""" @@ -16,30 +26,93 @@ def update(self, data: bytes) -> None: """Update the hash with raw bytes.""" +def _u8(value: int) -> bytes: + """Encode an unsigned 8-bit integer.""" + return value.to_bytes(1, "big", signed=False) + + +def _u32(value: int) -> bytes: + """Encode an unsigned 32-bit big-endian integer.""" + return value.to_bytes(4, "big", signed=False) + + +def _u64(value: int) -> bytes: + """Encode an unsigned 64-bit big-endian integer.""" + return value.to_bytes(8, "big", signed=False) + + +def _len_prefixed(payload: bytes) -> bytes: + """Encode a byte payload prefixed with its u64 length.""" + return _u64(len(payload)) + payload + + def serialize_item(obj: object) -> bytes: - """Serialize a supported multimodal hash leaf to bytes.""" + """Serialize a supported multimodal hash leaf to bytes. + + The encoding is canonical and self-describing: every value is + `[1-byte type tag][typed metadata][length-prefixed payload]` with all + multi-byte integers big-endian. This prevents cache-key hash collisions + between distinct values that happen to share a raw byte payload (for + example transposed image dimensions or reshaped arrays). + """ if isinstance(obj, str): - return obj.encode("utf-8") + return _u8(0x01) + _len_prefixed(obj.encode("utf-8")) if isinstance(obj, bytes): - return obj - if isinstance(obj, (int, float)): - return np.array(obj).tobytes() + return _u8(0x02) + _len_prefixed(obj) + # bool must be checked before int: bool is a subclass of int. + if isinstance(obj, bool): + return _u8(0x05) + _u8(1 if obj else 0) + if isinstance(obj, int): + nbytes = (obj.bit_length() + 8) // 8 # +1 sign bit, then ceil-divide. + return _u8(0x03) + _u8(nbytes) + obj.to_bytes(nbytes, "big", signed=True) + if isinstance(obj, float): + return _u8(0x04) + struct.pack(">d", obj) if isinstance(obj, Image.Image): - return np.array(obj.convert("RGBA")).tobytes() - if isinstance(obj, torch.Tensor): - return obj.numpy().tobytes() - if isinstance(obj, np.ndarray): - return obj.tobytes() + width, height = obj.size + payload = np.array(obj.convert("RGBA")).tobytes() + return ( + _u8(0x10) + + _len_prefixed(obj.mode.encode("utf-8")) + + _u32(width) + + _u32(height) + + _len_prefixed(payload) + ) + if isinstance(obj, (torch.Tensor, np.ndarray)): + # The container (torch.Tensor vs np.ndarray) is not part of the content + # identity -- only dtype, shape, and raw bytes are. Normalize both to a + # contiguous NumPy array so identical content hashes identically. + if isinstance(obj, torch.Tensor): + obj = obj.detach().cpu().contiguous().numpy() + array = np.ascontiguousarray(obj) + parts = [ + _u8(0x11), + _len_prefixed(array.dtype.str.encode("utf-8")), + _u8(array.ndim), + ] + parts.extend(_u64(dim) for dim in array.shape) + parts.append(_len_prefixed(array.tobytes())) + return b"".join(parts) if isinstance(obj, (tuple, list)): - container_tag = b"T" if isinstance(obj, tuple) else b"L" - parts = [container_tag, len(obj).to_bytes(8, "big", signed=False)] - for item in obj: - payload = serialize_item(item) - parts.append(len(payload).to_bytes(8, "big", signed=False)) - parts.append(payload) + # Ordered sequence; the container (tuple vs list) is not part of the + # content identity. + parts = [_u8(0x20), _u64(len(obj))] + parts.extend(serialize_item(item) for item in obj) + return b"".join(parts) + if isinstance(obj, dict): + parts = [_u8(0x22), _u64(len(obj))] + for key in sorted(obj): + parts.append(serialize_item(key)) + parts.append(serialize_item(obj[key])) return b"".join(parts) + if isinstance(obj, np.generic): + # numpy scalar (e.g. np.int64 / np.float32 / np.bool_): normalize to the + # equivalent Python scalar and recurse, so numpy-typed values hash + # identically to their Python counterparts. In numpy 2.x these are not + # subclasses of Python int/float/bool, so they bypass the checks above. + return serialize_item(obj.item()) + raise ValueError(f"Unsupported object type: {type(obj)}") @@ -65,11 +138,8 @@ def __post_init__(self) -> None: self.sample_rate = int(self.sample_rate) def update_hash(self, hasher: ContentHasher) -> None: - samples = self.samples - if isinstance(samples, torch.Tensor): - samples = samples.detach().cpu().contiguous() hasher.update(b"