diff --git a/.gitignore b/.gitignore index a175e19e31b..d9e8bd68b22 100644 --- a/.gitignore +++ b/.gitignore @@ -208,6 +208,7 @@ cache/ .cache/ diffusion_cache/ kv_cache/ +!vllm_omni/experimental/ar_diffusion/kv_cache/ # Logs logs/ diff --git a/tests/ar_diffusion/__init__.py b/tests/ar_diffusion/__init__.py new file mode 100644 index 00000000000..9881313609a --- /dev/null +++ b/tests/ar_diffusion/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/ar_diffusion/test_engine_routing.py b/tests/ar_diffusion/test_engine_routing.py new file mode 100644 index 00000000000..fc59e54ac64 --- /dev/null +++ b/tests/ar_diffusion/test_engine_routing.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the generic engine-backend dispatcher + AR-Diffusion runner wiring. + +``DiffusionEngine.resolve_engine_class`` is a generic dispatcher (``"default"`` / a +``DiffusionEngine`` subclass / an import-path string). DreamZero selects the +AR-Diffusion engine via its deploy config's ``engine_backend`` qualname — no +DreamZero/ar_diffusion knowledge lives in the public base, so the routing check +here is simply that the ``ARDiffusionEngine`` qualname resolves correctly. +""" + +from dataclasses import fields +from types import SimpleNamespace + +import pytest +from vllm.utils.import_utils import resolve_obj_by_qualname + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine +from vllm_omni.experimental.ar_diffusion.engine import ( + AR_DIFFUSION_MODEL_RUNNER_CLS, + ARDiffusionEngine, +) + + +def _cfg(backend): + return SimpleNamespace(engine_backend=backend) + + +def test_default_resolves_to_diffusion_engine(): + assert DiffusionEngine.resolve_engine_class(_cfg("default")) is DiffusionEngine + + +def test_subclass_type_is_returned(): + assert DiffusionEngine.resolve_engine_class(_cfg(ARDiffusionEngine)) is ARDiffusionEngine + + +def test_ar_diffusion_qualname_resolves(): + # How DreamZero's deploy config selects the engine: a full import-path string. + cls = DiffusionEngine.resolve_engine_class(_cfg("vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine")) + assert cls is ARDiffusionEngine + + +def test_non_engine_type_raises(): + with pytest.raises(TypeError): + DiffusionEngine.resolve_engine_class(_cfg(dict)) + + +def test_qualname_not_an_engine_raises(): + with pytest.raises(TypeError): + DiffusionEngine.resolve_engine_class(_cfg("builtins.dict")) + + +def test_bad_qualname_raises(): + with pytest.raises(ValueError): + DiffusionEngine.resolve_engine_class(_cfg("not.a.real.module.NoSuchEngine")) + + +def test_ar_diffusion_engine_is_diffusion_engine_subclass(): + assert issubclass(ARDiffusionEngine, DiffusionEngine) + + +def test_config_engine_backend_field_defaults_to_default(): + field = {f.name: f for f in fields(OmniDiffusionConfig)}["engine_backend"] + assert field.default == "default" + + +# --- AR-Diffusion worker/runner wiring ----------------------------------------------- + + +def test_ar_diffusion_model_runner_cls_resolves(): + cls = resolve_obj_by_qualname(AR_DIFFUSION_MODEL_RUNNER_CLS) + assert cls.__name__ == "ARDiffusionModelRunner" + + +def test_engine_class_declares_its_runner(): + """Routing lives on the engine class (review: hsliuustc0106) — engines + never mutate od_config; the worker resolves the runner off the class.""" + assert ARDiffusionEngine.default_diffusion_model_runner_cls == AR_DIFFUSION_MODEL_RUNNER_CLS + assert DiffusionEngine.default_diffusion_model_runner_cls is None + + +def _select_runner(od_config, platform_default="PLATFORM"): + """Mirror of the worker's runner-selection chain (override > engine > platform).""" + runner_override = getattr(od_config, "diffusion_model_runner_cls", None) + if isinstance(runner_override, str) and runner_override: + return runner_override + engine_cls = DiffusionEngine.resolve_engine_class(od_config) + engine_runner = getattr(engine_cls, "default_diffusion_model_runner_cls", None) + if isinstance(engine_runner, str) and engine_runner: + return engine_runner + return platform_default + + +def test_runner_selection_chain(): + ar = "vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine" + # engine-declared runner wins over platform default + assert ( + _select_runner(SimpleNamespace(diffusion_model_runner_cls=None, engine_backend=ar)) + == AR_DIFFUSION_MODEL_RUNNER_CLS + ) + # explicit override wins over the engine's declaration + assert ( + _select_runner(SimpleNamespace(diffusion_model_runner_cls="my.custom.Runner", engine_backend=ar)) + == "my.custom.Runner" + ) + # default engine -> platform default + assert _select_runner(SimpleNamespace(diffusion_model_runner_cls=None, engine_backend="default")) == "PLATFORM" + + +def test_dreamzero_pipeline_rejects_non_ar_diffusion_engine(): + """Review (hsliuustc0106): a stale config with engine_backend='default' + must fail at init, not mid-forward on the first KV access.""" + from types import SimpleNamespace + + import pytest + + from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline + + with pytest.raises(ValueError, match="requires the AR-Diffusion engine"): + DreamZeroPipeline(od_config=SimpleNamespace(engine_backend="default")) + with pytest.raises(ValueError, match="requires the AR-Diffusion engine"): + DreamZeroPipeline(od_config=SimpleNamespace()) + + +def test_runner_rejects_multi_sequence_config(): + """Review (hsliuustc0106): the paged path writes batch index 0 only, so + max_num_seqs > 1 must be rejected at init.""" + from types import SimpleNamespace + + import pytest + + from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner + + fake = SimpleNamespace(od_config=SimpleNamespace(max_num_seqs=4)) + with pytest.raises(ValueError, match="max_num_seqs=1"): + ARDiffusionModelRunner._preallocate_kv_cache(fake) diff --git a/tests/ar_diffusion/test_kv_cache.py b/tests/ar_diffusion/test_kv_cache.py new file mode 100644 index 00000000000..cbb9656e54e --- /dev/null +++ b/tests/ar_diffusion/test_kv_cache.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the AR-Diffusion KV cache helpers (Phase 1, PR-2). + +Covers the request adapter, the chunk-window spec/manager (registration + the +eviction policy), and the pool builder — exercised against the installed vLLM +KV stack on CPU (block bookkeeping only, no GPU tensors). +""" + +import pytest +import torch +from vllm.v1.kv_cache_interface import KVCacheSpecKind, get_kv_cache_spec_kind +from vllm.v1.request import RequestStatus + +try: + from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry +except ModuleNotFoundError: + KVCacheSpecRegistry = None + from vllm.v1.core.single_type_kv_cache_manager import spec_manager_map + +from vllm_omni.experimental.ar_diffusion.kv_cache import ( + ARDiffusionKVCache, + ARDiffusionKVConfig, + ARDiffusionRequestAdapter, + ChunkWindowManager, + ChunkWindowSpec, + allocate_kv_pool_with_views, + build_kv_manager, + compute_num_blocks, +) +from vllm_omni.experimental.ar_diffusion.kv_cache.paged import chunk_window_skipped_tokens + +BLOCK = 16 + + +def make_spec(*, chunk_size=BLOCK, window_chunks=2, sink_chunks=0, reset_at_boundary=False): + return ChunkWindowSpec( + block_size=BLOCK, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + sliding_window=window_chunks * chunk_size, + chunk_size=chunk_size, + window_chunks=window_chunks, + sink_chunks=sink_chunks, + reset_at_boundary=reset_at_boundary, + ) + + +# --- ChunkWindowSpec registration ------------------------------------------- + + +def test_spec_registration_resolves_to_chunk_window_manager(): + # Without explicit registration the MRO walk would fall back to the parent + # SlidingWindowManager; assert the subclass manager wins. + spec = make_spec() + if KVCacheSpecRegistry is not None: + assert KVCacheSpecRegistry.get_manager_class(spec) is ChunkWindowManager + else: + assert spec_manager_map[type(spec)] is ChunkWindowManager + + +def test_spec_kind_is_sliding_window(): + assert get_kv_cache_spec_kind(make_spec()) == KVCacheSpecKind.SLIDING_WINDOW + + +def test_spec_rejects_inconsistent_window(): + with pytest.raises(ValueError): + ChunkWindowSpec( + block_size=BLOCK, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + sliding_window=99, # != window_chunks * chunk_size + chunk_size=BLOCK, + window_chunks=2, + ) + + +# --- eviction policy (pure) ------------------------------------------------- + + +def test_sliding_replace_keeps_window(): + # window = 2 chunks * 16 = 32. Base sliding formula keeps `window` tokens; + # the snap is to chunk boundaries. + def skip(n): + return chunk_window_skipped_tokens(n, chunk_size=16, sliding_window=32, sink_chunks=0, reset_at_boundary=False) + + assert skip(32) == 0 # nothing past the window yet + assert skip(48) == 16 # one chunk fell out of the window + assert skip(64) == 32 + + +def test_sliding_replace_snaps_to_chunk_boundary(): + # A non-chunk-aligned overflow must snap down so a chunk is never half-evicted. + skip = chunk_window_skipped_tokens(50, chunk_size=16, sliding_window=32, sink_chunks=0, reset_at_boundary=False) + assert skip % 16 == 0 and skip == 16 + + +def test_sink_chunks_protected(): + # sink = 1 chunk (16 tokens) is never skipped. + skip = chunk_window_skipped_tokens(80, chunk_size=16, sliding_window=32, sink_chunks=1, reset_at_boundary=False) + # raw overflow = 80 - 32 - 16 = 32 -> snapped 32 + assert skip == 32 + + +def test_reset_at_boundary_drops_completed_past_sink(): + skip = chunk_window_skipped_tokens(48, chunk_size=16, sliding_window=32, sink_chunks=1, reset_at_boundary=True) + # completed = 48; sink = 16 -> drop 32 + assert skip == 32 + + +# --- ARDiffusionKVConfig ------------------------------------------------------------ + + +def test_kv_config_sliding_window_property(): + assert ARDiffusionKVConfig(chunk_size=16, window_chunks=3).sliding_window == 48 + assert ARDiffusionKVConfig(chunk_size=16, window_chunks=None).sliding_window is None + + +# --- ARDiffusionRequestAdapter ------------------------------------------------------ + + +def test_adapter_advances_per_chunk_not_per_step(): + a = ARDiffusionRequestAdapter("r0", chunk_size=16) + assert a.num_computed_tokens == 0 + assert a.num_tokens == 16 # in-flight chunk + a.on_chunk_committed() + assert a.num_computed_tokens == 16 + assert a.num_tokens == 32 + + +def test_adapter_accounts_for_prefill_prefix(): + a = ARDiffusionRequestAdapter("r0", chunk_size=16, prefill_prefix_tokens=4) + assert a.num_computed_tokens == 4 + assert a.num_prompt_tokens == 4 + assert a.num_tokens == 20 + + +def test_adapter_status_is_vllm_enum(): + assert isinstance(ARDiffusionRequestAdapter("r0", chunk_size=16).status, RequestStatus) + + +# --- pool / manager --------------------------------------------------------- + + +def test_compute_num_blocks(): + # 1 MiB budget at fraction 0.5 with 16 KiB pages -> 32 blocks. + assert compute_num_blocks(1 << 20, 0.5, 16 << 10) == 32 + + +def test_paged_pool_layout_exposes_flat_slot_views(): + kv_pools, k_pools, v_pools = allocate_kv_pool_with_views( + num_blocks=4, + block_size=BLOCK, + num_layers=1, + num_kv_heads=4, + head_dim=64, + dtype=torch.float32, + device=torch.device("cpu"), + ) + assert kv_pools[0].shape == (2, 4, BLOCK, 4, 64) + assert k_pools[0].shape == (4 * BLOCK, 4, 64) + assert v_pools[0].shape == (4 * BLOCK, 4, 64) + + k_pools[0][BLOCK + 3].fill_(7) + v_pools[0][2 * BLOCK + 5].fill_(11) + assert torch.equal(kv_pools[0][0, 1, 3], k_pools[0][BLOCK + 3]) + assert torch.equal(kv_pools[0][1, 2, 5], v_pools[0][2 * BLOCK + 5]) + + +def test_build_manager_allocate_free_roundtrip(): + """End-to-end: a ARDiffusionRequestAdapter drives a real KVCacheManager. + + This is the adapter conformance check — if the adapter were missing an + attribute the manager reads, allocate_slots/free would raise here. + """ + spec = make_spec() + mgr = build_kv_manager(spec, ["layer0"], num_blocks=16, max_model_len=1024) + free_before = mgr.block_pool.get_num_free_blocks() + + adapter = ARDiffusionRequestAdapter("req-0", chunk_size=BLOCK) + blocks = mgr.allocate_slots(adapter, num_new_tokens=BLOCK, full_sequence_must_fit=True) + assert blocks is not None + assert mgr.block_pool.get_num_free_blocks() < free_before + + mgr.free(adapter) + assert mgr.block_pool.get_num_free_blocks() == free_before + + +def test_cross_attn_pool_deducted_from_self_attn_budget(): + """The cross-attn pool is allocated directly; its bytes are subtracted from + the self-attn paged-pool budget so the two together stay within the free + memory budget (review: zwhzzz0821).""" + L = 512 + avail = 1 << 30 # 1 GiB + kv = ARDiffusionKVCache( + ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=2, gpu_memory_fraction=0.5), + num_layers=2, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + block_size=BLOCK, + max_model_len=4096, + available_bytes=avail, + cross_attn_length=L, + device=torch.device("cpu"), + ) + cross_bytes = 2 * 2 * L * 4 * 64 * torch.float16.itemsize * 2 # K+V, pos+neg, layers + expected = compute_num_blocks(avail - cross_bytes, 0.5, kv.spec.page_size_bytes * 2) + assert kv.num_blocks == expected + assert expected > 2 * (2 + 1) + 2 # above the min-blocks floor, so the deduction is what's tested + + +def _make_kv(*, local_branches, num_frame_per_block=2, window_chunks=9): + return ARDiffusionKVCache( + ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks), + num_layers=1, + num_kv_heads=4, + head_size=64, + dtype=torch.float32, + block_size=BLOCK, + max_model_len=4096, + available_bytes=1 << 16, # tiny -> the floor binds + device=torch.device("cpu"), + local_branches=local_branches, + num_frame_per_block=num_frame_per_block, + ) + + +def test_pool_floor_is_branch_aware(): + """CFG-parallel rank (one local branch) sizes for one window + in-flight chunk; + a single-process run (both branches) sizes for two. Scratch scales the same way.""" + one = _make_kv(local_branches=1) + two = _make_kv(local_branches=2) + + assert one.managed_num_blocks == 1 * (9 + 2) + 2 # 13 + assert two.managed_num_blocks == 2 * (9 + 2) + 2 # 24 + assert one.scratch_num_blocks == one.scratch_blocks_per_branch + assert two.scratch_num_blocks == 2 * two.scratch_blocks_per_branch + assert one.num_blocks_total == 13 + one.scratch_blocks_per_branch + + +def test_scratch_maps_to_slot_zero_with_one_local_branch(): + """A CFG-parallel rank runs exactly one branch: whichever CFG side it is, + its scratch lands in the rank's single slot (no dead second slot).""" + one = _make_kv(local_branches=1) + assert one.scratch_block_ids(True, 0, 2) == one.scratch_block_ids(False, 0, 2) + + two = _make_kv(local_branches=2) + assert two.scratch_block_ids(True, 0, 2) != two.scratch_block_ids(False, 0, 2) + + +def test_scratch_exhaustion_still_raises(): + one = _make_kv(local_branches=1) + cap = one.scratch_blocks_per_branch + with pytest.raises(RuntimeError, match="scratch blocks exhausted"): + one.scratch_block_ids(False, 0, cap + 1) + + +def test_invalid_local_branches_rejected(): + with pytest.raises(ValueError, match="local_branches"): + _make_kv(local_branches=3) diff --git a/tests/ar_diffusion/test_kv_orchestration.py b/tests/ar_diffusion/test_kv_orchestration.py new file mode 100644 index 00000000000..4ba5cb49bf1 --- /dev/null +++ b/tests/ar_diffusion/test_kv_orchestration.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for ARDiffusionKVCache — the engine-level KV orchestration body (Phase 1).""" + +import pytest +import torch + +from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig + +DIMS = dict(num_layers=2, num_kv_heads=4, head_size=64, dtype=torch.float16, block_size=16) + + +def make_cache(*, chunk_size=16, window_chunks=2, available_bytes=1 << 24): + cfg = ARDiffusionKVConfig(enable=True, chunk_size=chunk_size, window_chunks=window_chunks) + return ARDiffusionKVCache(cfg, max_model_len=4096, available_bytes=available_bytes, **DIMS) + + +def test_requires_enabled_config(): + with pytest.raises(ValueError): + ARDiffusionKVCache(ARDiffusionKVConfig(enable=False), max_model_len=256, available_bytes=1 << 20, **DIMS) + + +def test_requires_bounded_window(): + cfg = ARDiffusionKVConfig(enable=True, chunk_size=16, window_chunks=None) + with pytest.raises(ValueError): + ARDiffusionKVCache(cfg, max_model_len=256, available_bytes=1 << 20, **DIMS) + + +def test_full_request_lifecycle_and_eviction(): + """begin -> per-chunk allocate/slots/commit over a long rollout -> free. + + Exercises the orchestrator end-to-end and asserts the chunk window bounds + memory (pool plateaus) and frees cleanly. + """ + kv = make_cache(chunk_size=16, window_chunks=2) + free_total = kv.manager.block_pool.get_num_free_blocks() + + adapter = kv.begin_request("req-0") + free_after = [] + for k in range(10): + block_table = kv.allocate_chunk(adapter) + slots = kv.chunk_write_slots(adapter) + # Slots target real blocks in this chunk's table; length == chunk_size. + assert len(slots) == kv.spec.chunk_size + used = {int(s) // kv.block_size for s in slots} + assert kv.null_block_id not in used + assert used <= set(block_table) + # Resident window stays bounded by window + the in-flight chunk. + assert len(kv.window_block_ids(adapter)) <= kv.spec.window_chunks + 1 + free_after.append(kv.manager.block_pool.get_num_free_blocks()) + kv.commit_chunk(adapter) + + # Pool memory plateaus once the window is full (eviction recycles blocks). + assert free_after[-1] == free_after[kv.spec.window_chunks] + + kv.end_request(adapter) + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_num_computed_advances_per_chunk(): + kv = make_cache(chunk_size=16, window_chunks=2) + a = kv.begin_request("r") + assert a.num_computed_tokens == 0 + kv.allocate_chunk(a) + kv.commit_chunk(a) + assert a.num_computed_tokens == 16 + kv.end_request(a) + + +def test_state_close_frees_both_branch_blocks(): + """ARDiffusionKVState.close() returns both CFG branches' pool blocks to the pool. + + This is the primitive the runner's LRU eviction relies on: when a session is + evicted, close() must free the blocks both adapters hold, or session churn + leaks pool ownership (review P1). + """ + from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + + kv = make_cache(chunk_size=16, window_chunks=2) + free_total = kv.manager.block_pool.get_num_free_blocks() + + pos = kv.begin_request("bde__s") + neg = kv.begin_request("bde__s__neg") + state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) + for _ in range(3): + for adapter in (pos, neg): + kv.allocate_chunk(adapter) + kv.commit_chunk(adapter) + # Both branches hold resident blocks now. + assert kv.manager.block_pool.get_num_free_blocks() < free_total + + state.close() + # All blocks returned — no leak across an evicted session. + assert kv.manager.block_pool.get_num_free_blocks() == free_total diff --git a/tests/ar_diffusion/test_paged_attention.py b/tests/ar_diffusion/test_paged_attention.py new file mode 100644 index 00000000000..1836656f940 --- /dev/null +++ b/tests/ar_diffusion/test_paged_attention.py @@ -0,0 +1,394 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for AR-Diffusion paged self-attention contexts.""" + +from __future__ import annotations + +import subprocess +from importlib.util import find_spec +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import torch + +from vllm_omni.experimental.ar_diffusion.kv_cache import ( + ARDiffusionKVCache, + ARDiffusionKVConfig, + ARDiffusionPagedLayerContext, + ARDiffusionPagedLayerInputs, + ar_diffusion_paged_attention, + paged_write_attn, +) +from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + +BLOCK = 16 +N_HEADS = 4 +HEAD_DIM = 64 + + +def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu")): + cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks) + kv = ARDiffusionKVCache( + cfg, + num_layers=num_layers, + num_kv_heads=N_HEADS, + head_size=HEAD_DIM, + dtype=dtype, + block_size=BLOCK, + max_model_len=4096, + available_bytes=1 << 26, + device=device, + ) + pos = kv.begin_request("r-pos") + neg = kv.begin_request("r-neg") + return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + + +def _dense_attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor: + scores = torch.einsum("bqhd,bkhd->bhqk", query.float(), key.float()) * (HEAD_DIM**-0.5) + probs = torch.softmax(scores, dim=-1).to(value.dtype) + return torch.einsum("bhqk,bkhd->bqhd", probs, value) + + +def _cuda_flash_attn_usable() -> bool: + if not torch.cuda.is_available(): + return False + try: + spec = find_spec("vllm.vllm_flash_attn") + if spec is None or spec.origin is None: + return True + fa2_so = Path(spec.origin).parent / "_vllm_fa2_C.abi3.so" + linked = subprocess.check_output(["ldd", str(fa2_so)], text=True, timeout=5) + except Exception: + return True + if "libcudart.so.13" not in linked: + return True + try: + out = subprocess.check_output( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + text=True, + timeout=5, + ) + driver_major = int(out.splitlines()[0].split(".")[0]) + except Exception: + return True + return driver_major >= 580 + + +def _commit_video_span( + kv: ARDiffusionKVCache, + st: ARDiffusionKVState, + *, + is_negative: bool, + n_chunks: int, + dtype: torch.dtype, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor]: + ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(device) + k = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + v = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + kv._k_pools[0][ctx.current_video_slot_mapping] = k[0] + kv._v_pools[0][ctx.current_video_slot_mapping] = v[0] + st.commit_paged_context(is_negative) + return k, v + + +def test_paged_context_allocates_lazily_and_commits_after_forward(): + _, st = make_state() + + contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True) + ctx = contexts[0].forward_ctx + assert isinstance(contexts[0], ARDiffusionPagedLayerContext) + assert st.pos.completed_chunks == 0 + assert ctx.current_video_slot_mapping is None + + ctx.ensure_video_slots(torch.device("cpu")) + assert st.pos.completed_chunks == 0 + assert len(ctx.current_video_block_ids) == 1 + + st.commit_paged_context(False) + assert st.pos.completed_chunks == 1 + assert st._committed[False] == BLOCK + + +def test_scratch_video_and_action_blocks_do_not_commit(): + kv, st = make_state() + + ctx = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=False)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + ctx.ensure_action_slots(3, torch.device("cpu")) + + assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 2) + assert ctx.action_scratch_block_ids == kv.scratch_block_ids(False, 2, 1) + st.commit_paged_context(False) + assert st.pos.completed_chunks == 0 + assert st._committed[False] == 0 + + +def test_pipeline_kv_get_paged_path_has_no_gather_backend(): + kv, st = make_state() + assert not hasattr(kv, "gather_window_all_layers") + + from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline + + pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline) + pipeline._ar_diffusion_kv_state = st + contexts = pipeline._kv_get(MagicMock(), False, seq_len=BLOCK, update_kv_cache=False) + + assert len(contexts) == 1 + assert isinstance(contexts[0], ARDiffusionPagedLayerContext) + + +@pytest.mark.parametrize("history_chunks", [0, 1, 3]) +@pytest.mark.parametrize("action_len", [0, 3]) +@pytest.mark.parametrize("commit_current", [False, True]) +def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, commit_current): + torch.manual_seed(0) + device = torch.device("cpu") + dtype = torch.float32 + kv, st = make_state(dtype=dtype, device=device, window_chunks=2) + + history_k_parts: list[torch.Tensor] = [] + history_v_parts: list[torch.Tensor] = [] + if history_chunks: + k, v = _commit_video_span( + kv, + st, + is_negative=False, + n_chunks=history_chunks, + dtype=dtype, + device=device, + ) + history_k_parts.append(k) + history_v_parts.append(v) + + ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0].forward_ctx + ctx.ensure_video_slots(device) + current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + kv._k_pools[0][ctx.current_video_slot_mapping] = current_k[0] + kv._v_pools[0][ctx.current_video_slot_mapping] = current_v[0] + + action_k = action_v = None + if action_len: + ctx.ensure_action_slots(action_len, device) + action_k = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + action_v = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + kv._k_pools[0][ctx.action_slot_mapping] = action_k[0] + kv._v_pools[0][ctx.action_slot_mapping] = action_v[0] + + query = torch.randn(1, BLOCK + action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + block_table, query_start_loc, seq_lens, max_query_len, max_seq_len = ctx.build_block_table( + action_len=action_len, + query_len=query.shape[1], + device=device, + ) + paged = ar_diffusion_paged_attention( + query, + kv.key_cache(0), + kv.value_cache(0), + block_table=block_table, + query_start_loc=query_start_loc, + seq_lens=seq_lens, + max_query_len=max_query_len, + max_seq_len=max_seq_len, + softmax_scale=HEAD_DIM**-0.5, + causal=False, + ) + + if history_k_parts: + history_k = torch.cat(history_k_parts, dim=1) + history_v = torch.cat(history_v_parts, dim=1) + else: + history_k = torch.empty(1, 0, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + history_v = torch.empty(1, 0, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + new_k = torch.cat([history_k, current_k], dim=1)[:, -kv.spec.sliding_window :] + new_v = torch.cat([history_v, current_v], dim=1)[:, -kv.spec.sliding_window :] + if action_len: + new_k = torch.cat([new_k, action_k], dim=1) + new_v = torch.cat([new_v, action_v], dim=1) + ref = _dense_attention(query, new_k, new_v) + + torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5) + + before = st.pos.completed_chunks + st.commit_paged_context(False) + assert st.pos.completed_chunks == before + (1 if commit_current else 0) + + +@pytest.mark.skipif(not _cuda_flash_attn_usable(), reason="usable CUDA FlashAttention is required") +@pytest.mark.parametrize("history_chunks", [1, 3]) +@pytest.mark.parametrize("action_len", [0, 3]) +@pytest.mark.parametrize("commit_current", [False, True]) +def test_paged_attention_matches_dense_reference_gpu(history_chunks, action_len, commit_current): + pytest.importorskip("vllm.vllm_flash_attn") + torch.manual_seed(0) + device = torch.device("cuda") + dtype = torch.float16 + kv, st = make_state(dtype=dtype, device=device, window_chunks=2) + + history_k, history_v = _commit_video_span( + kv, + st, + is_negative=False, + n_chunks=history_chunks, + dtype=dtype, + device=device, + ) + + layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0] + ctx = layer_ctx.forward_ctx + current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + action_k = action_v = None + if action_len: + action_k = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + action_v = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + + query = torch.randn(1, BLOCK + action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device) + + # The production path: once-per-forward host prep, then the fused + # write+attend custom op consuming the NamedTuple payload. + ctx.prepare(device=device, action_len=action_len, query_len=query.shape[1]) + inputs = layer_ctx.to_layer_inputs() + assert isinstance(inputs, ARDiffusionPagedLayerInputs) + paged = paged_write_attn( + inputs, + query[0], + current_k[0], + current_v[0], + action_k[0] if action_len else None, + action_v[0] if action_len else None, + HEAD_DIM**-0.5, + ).unsqueeze(0) + + # Direct python-fn call on the same (already written) pools must be + # bit-exact: identical kernel, identical inputs. + direct = ar_diffusion_paged_attention( + query, + kv.key_cache(0), + kv.value_cache(0), + block_table=ctx.block_table, + query_start_loc=ctx.query_start_loc, + seq_lens=ctx.seq_lens, + max_query_len=ctx.max_query_len, + max_seq_len=ctx.max_seq_len, + softmax_scale=HEAD_DIM**-0.5, + causal=False, + ) + assert torch.equal(paged, direct) + + new_k = torch.cat([history_k, current_k], dim=1)[:, -kv.spec.sliding_window :] + new_v = torch.cat([history_v, current_v], dim=1)[:, -kv.spec.sliding_window :] + if action_len: + new_k = torch.cat([new_k, action_k], dim=1) + new_v = torch.cat([new_v, action_v], dim=1) + ref = _dense_attention(query, new_k, new_v) + + torch.testing.assert_close(paged, ref, rtol=2e-2, atol=2e-2) + + +def test_block_table_padded_to_fixed_width(): + """Shapes must be constant across window growth: only values change.""" + device = torch.device("cpu") + kv, st = make_state(window_chunks=2) + + ctx1 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx1.prepare(device=device, action_len=0, query_len=BLOCK) + st.commit_paged_context(False) + + ctx2 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx2.prepare(device=device, action_len=0, query_len=BLOCK) + st.commit_paged_context(False) + + # 1-block vs 2-block visible history: same table width, same max_seq_len. + assert ctx1.block_table.shape == ctx2.block_table.shape + assert ctx1.max_seq_len == ctx2.max_seq_len + expected_width = kv.spec.sliding_window // kv.block_size + 1 + assert ctx1.block_table.shape == (1, expected_width) + # Real lengths live in seq_lens, not the padded table. + assert int(ctx1.seq_lens[0]) == BLOCK + assert int(ctx2.seq_lens[0]) == 2 * BLOCK + + +def test_prepare_is_idempotent_and_layers_share_metadata(): + device = torch.device("cpu") + _, st = make_state(num_layers=2) + contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + fctx = contexts[0].forward_ctx + fctx.prepare(device=device, action_len=0, query_len=BLOCK) + table = fctx.block_table + fctx.prepare(device=device, action_len=0, query_len=BLOCK) + assert fctx.block_table is table # memoized, not rebuilt + + i0, i1 = contexts[0].to_layer_inputs(), contexts[1].to_layer_inputs() + # 0-dim tensors (NOT python ints) so dynamo doesn't install per-layer + # value guards on the shared block code object. + assert isinstance(i0.layer_idx, torch.Tensor) and int(i0.layer_idx) == 0 + assert isinstance(i1.layer_idx, torch.Tensor) and int(i1.layer_idx) == 1 + # All layers share the same metadata tensor objects. + assert i0.block_table is i1.block_table + assert i0.seq_lens is i1.seq_lens + assert i0.video_slots is i1.video_slots + + +def test_layer_inputs_before_prepare_raises(): + _, st = make_state() + layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0] + with pytest.raises(RuntimeError, match="before prepare"): + layer_ctx.to_layer_inputs() + + +def test_custom_op_registration_idempotent(): + import importlib + import sys + + assert hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn") + mod = "vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention" + saved = sys.modules.pop(mod) + try: + importlib.import_module(mod) # re-registration must not raise + finally: + sys.modules[mod] = saved + assert hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn") + + +def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): + """The op must trace as one opaque node: fullgraph OK, and changed tensor + VALUES (new slots / block ids) must not trigger recompilation.""" + import torch._dynamo + + device = torch.device("cpu") + kv, st = make_state(num_layers=2, window_chunks=2) + + def run_one_forward(commit): + contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit) + fctx = contexts[0].forward_ctx + fctx.prepare(device=device, action_len=0, query_len=BLOCK) + q = torch.randn(BLOCK, N_HEADS, HEAD_DIM) + k = torch.randn(BLOCK, N_HEADS, HEAD_DIM) + v = torch.randn(BLOCK, N_HEADS, HEAD_DIM) + # Both layers through ONE compiled fn: layer_idx is a tensor, so a + # different layer must NOT recompile (all 40 DiT blocks share the + # block-forward code object in production). + for layer_ctx in contexts: + out = compiled(layer_ctx.to_layer_inputs(), q, k, v) + st.commit_paged_context(False) + return out + + torch._dynamo.reset() + from torch._dynamo.testing import CompileCounter + + counter = CompileCounter() + + def fn(inputs, q, k, v): + return paged_write_attn(inputs, q, k, v, None, None, HEAD_DIM**-0.5) * 1.0 + + compiled = torch.compile(fn, backend=counter, fullgraph=True) + + run_one_forward(commit=True) # history grows between calls -> + run_one_forward(commit=True) # block-table VALUES change, shapes don't + run_one_forward(commit=False) + + assert counter.frame_count == 1, f"recompiled: frame_count={counter.frame_count}" diff --git a/tests/ar_diffusion/test_pipeline_bridge.py b/tests/ar_diffusion/test_pipeline_bridge.py new file mode 100644 index 00000000000..83c710efa8e --- /dev/null +++ b/tests/ar_diffusion/test_pipeline_bridge.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the ARDiffusionKVState paged-attention pipeline bridge.""" + +import pytest +import torch + +from vllm_omni.experimental.ar_diffusion.kv_cache import ( + ARDiffusionKVCache, + ARDiffusionKVConfig, + ARDiffusionPagedLayerContext, +) +from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + +BLOCK = 16 +N_HEADS = 4 +HEAD_DIM = 64 + + +def make_state(num_layers=1, window_chunks=4, cross_attn_length=0): + cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks) + kv = ARDiffusionKVCache( + cfg, + num_layers=num_layers, + num_kv_heads=N_HEADS, + head_size=HEAD_DIM, + dtype=torch.float32, + block_size=BLOCK, + max_model_len=4096, + available_bytes=1 << 24, + cross_attn_length=cross_attn_length, + device=torch.device("cpu"), + ) + pos = kv.begin_request("r-pos") + neg = kv.begin_request("r-neg") + return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + + +def _prepare_and_commit(st: ARDiffusionKVState, is_negative: bool, n_chunks: int) -> None: + ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + st.commit_paged_context(is_negative) + + +def test_get_returns_paged_layer_contexts_when_nothing_committed(): + _, st = make_state(num_layers=2) + for neg in (False, True): + contexts = st.get_kv_caches(neg, seq_len=BLOCK, commit_current=False) + assert len(contexts) == 2 + assert all(isinstance(ctx, ARDiffusionPagedLayerContext) for ctx in contexts) + assert contexts[0].history_block_ids == [] + + +def test_managed_paged_context_commits_only_after_forward(): + kv, st = make_state(num_layers=1) + contexts = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=True) + ctx = contexts[0].forward_ctx + + assert st.pos.completed_chunks == 0 + assert kv.window_block_ids(st.pos) == [] + ctx.ensure_video_slots(torch.device("cpu")) + assert st.pos.completed_chunks == 0 + assert len(ctx.current_video_block_ids) == 2 + + st.commit_paged_context(False) + + assert st.pos.completed_chunks == 2 + assert st._committed[False] == 2 * BLOCK + assert len(kv.window_block_ids(st.pos)) == 2 + + +def test_scratch_paged_context_does_not_commit(): + kv, st = make_state(num_layers=1) + contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + ctx = contexts[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + + assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 1) + st.commit_paged_context(False) + + assert st.pos.completed_chunks == 0 + assert st._committed[False] == 0 + assert kv.window_block_ids(st.pos) == [] + + +def test_eviction_bounds_resident_window(): + kv, st = make_state(num_layers=1, window_chunks=3) + _prepare_and_commit(st, False, 3) + assert len(kv.window_block_ids(st.pos)) == 3 + + _prepare_and_commit(st, False, 2) + ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0].forward_ctx + visible_blocks, video_len = ctx.video_block_table(torch.device("cpu")) + assert len(visible_blocks) == 3 + assert video_len == 3 * BLOCK + + +def test_branches_are_independent(): + kv, st = make_state() + _prepare_and_commit(st, False, 1) + assert len(kv.window_block_ids(st.pos)) == 1 + assert kv.window_block_ids(st.neg) == [] + + +def test_reset_clears_session_window(): + kv, st = make_state(num_layers=1, window_chunks=4) + _prepare_and_commit(st, False, 2) + _prepare_and_commit(st, True, 1) + assert st._committed[False] == 2 * BLOCK and st._committed[True] == BLOCK + free_before = kv.manager.block_pool.get_num_free_blocks() + + st.reset() + + assert st._committed == {False: 0, True: 0} + assert st._paged_pending == {False: None, True: None} + assert kv.window_block_ids(st.pos) == [] + assert kv.window_block_ids(st.neg) == [] + assert kv.manager.block_pool.get_num_free_blocks() > free_before + _prepare_and_commit(st, False, 1) + assert len(kv.window_block_ids(st.pos)) == 1 + + +def test_get_cross_kv_caches_raises_before_populate(): + """Engine ownership guard: a cross-attn read before _kv_populate_cross must fail loud.""" + _, st = make_state() # cross_attn_length == 0, nothing populated + for neg in (False, True): + with pytest.raises(RuntimeError, match="cross-attn read before"): + st.get_cross_kv_caches(neg) + # Guards the full AND: the populated flag alone is not enough without a pool. + st._cross_text_populated[False] = True + with pytest.raises(RuntimeError, match="cross-attn read before"): + st.get_cross_kv_caches(False) + + +def test_get_cross_kv_caches_returns_pool_dicts_when_populated(): + L = 8 + kv, st = make_state(num_layers=2, cross_attn_length=L) + written = [] + for i in range(2): + k = torch.randn(1, L, N_HEADS, HEAD_DIM) + v = torch.randn(1, L, N_HEADS, HEAD_DIM) + kv.write_cross_kv(i, False, k, v) + written.append((k, v)) + st._cross_text_populated[False] = True + + out = st.get_cross_kv_caches(False) + assert len(out) == 2 + for i, (k, v) in enumerate(written): + assert out[i]["is_init"] is True + assert out[i]["k"].shape == (1, L, N_HEADS, HEAD_DIM) + assert torch.equal(out[i]["k"], k) + assert torch.equal(out[i]["v"], v) + + +def test_kv_create_is_noop_engine_owns_allocation(): + from unittest.mock import MagicMock + + from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline + + p = DreamZeroPipeline.__new__(DreamZeroPipeline) + p._ar_diffusion_kv_state = object() + state = MagicMock() + p._kv_create(state, 1, "float32", "cpu", 24, 4, 64) + state.create_kv_caches.assert_not_called() + + +def test_get_kv_cache_requires_frame_aligned_seqlen(): + _, st = make_state() + with pytest.raises(AssertionError, match="frame-aligned"): + st.get_kv_caches(False, seq_len=BLOCK + 1, commit_current=True) + + +def test_prepare_one_branch_allocates_nothing_for_the_other(): + """CFG-parallel laziness: a rank prepares only the branch it runs.""" + kv, st = make_state(num_layers=1, window_chunks=4) + pos_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + neg_ctx = st.get_kv_caches(True, seq_len=BLOCK, commit_current=True)[0].forward_ctx + + pos_ctx.prepare(device=torch.device("cpu"), action_len=0, query_len=BLOCK) + + assert pos_ctx._allocated_video + assert not neg_ctx._allocated_video + assert kv.window_block_ids(st.neg) == [] + + +def test_failed_forward_tears_down_session(monkeypatch): + """Review follow-up (hsliuustc0106): a forward that dies partway must not + leave allocated-but-uncommitted KV behind — the session is torn down (pool + blocks freed, model-local state dropped) and the error propagates.""" + from types import SimpleNamespace + + from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner + from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner + + kv, st = make_state(num_layers=1, window_chunks=4) + _prepare_and_commit(st, False, 2) # session owns pool blocks + free_before_failure = kv.manager.block_pool.get_num_free_blocks() + + runner = object.__new__(ARDiffusionModelRunner) + runner.kv_cache = kv + runner._ar_diffusion_states = __import__("collections").OrderedDict({"s1": st}) + runner._max_ar_diffusion_states = 4 + runner.pipeline = SimpleNamespace(_states={"s1": object()}, _ar_diffusion_kv_state=None) + runner.device = None + runner._perf_e2e_times = [] + + def boom(self, req): + raise RuntimeError("layer 17 exploded") + + monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) + req = SimpleNamespace( + request_id="r0", + sampling_params=SimpleNamespace(extra_args={"session_id": "s1"}), + ) + + with pytest.raises(RuntimeError, match="layer 17 exploded"): + ARDiffusionModelRunner.execute_model(runner, req) + + assert "s1" not in runner._ar_diffusion_states + assert "s1" not in runner.pipeline._states + assert runner.pipeline._ar_diffusion_kv_state is None + # close() freed the session's resident blocks. + assert kv.manager.block_pool.get_num_free_blocks() > free_before_failure diff --git a/tests/ar_diffusion/test_slot_mapping.py b/tests/ar_diffusion/test_slot_mapping.py new file mode 100644 index 00000000000..061df4d9c64 --- /dev/null +++ b/tests/ar_diffusion/test_slot_mapping.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Slot-mapping / block-table tests for the AR-Diffusion engine (Phase 1, Step 3).""" + +import pytest +import torch + +from vllm_omni.experimental.ar_diffusion.kv_cache import ( + ARDiffusionRequestAdapter, + ChunkWindowSpec, + build_kv_manager, + chunk_slot_mapping, + compute_slot_mapping, + resident_block_ids, +) + +BLOCK = 16 + + +def make_spec(*, window_chunks=2): + return ChunkWindowSpec( + block_size=BLOCK, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + sliding_window=window_chunks * BLOCK, + chunk_size=BLOCK, + window_chunks=window_chunks, + ) + + +def test_slot_mapping_matches_block_offsets(): + # block table [5, 2, 9], block_size 4. slot = block*4 + (pos % 4). + slots = compute_slot_mapping([5, 2, 9], [0, 1, 4, 5, 8, 9], block_size=4) + assert slots.tolist() == [20, 21, 8, 9, 36, 37] + + +def test_slot_mapping_rejects_bad_block_size(): + with pytest.raises(ValueError): + compute_slot_mapping([1], [0], block_size=0) + + +def test_chunk_slot_mapping_targets_current_chunk(): + # After 2 committed chunks (num_computed=32), the in-flight chunk maps to the + # block covering positions 32..47 — i.e. block table index 2. + block_ids = [7, 4, 3] # block index 2 -> physical block 3 + slots = chunk_slot_mapping(block_ids, num_computed_tokens=32, chunk_size=4, block_size=16) + # positions 32..35 -> block_index 2 -> physical 3 -> slots 48..51 + assert slots.tolist() == [48, 49, 50, 51] + + +def test_resident_block_ids_excludes_null(): + assert resident_block_ids([0, 1, 0, 2, 3], null_block_id=0) == [1, 2, 3] + + +def test_blocktable_build_single_request(): + """Integration: a real KVCacheManager block table -> a valid slot mapping. + + Every slot for the in-flight chunk must land inside a real (non-null) block + that the manager actually allocated for the request. + """ + spec = make_spec() + mgr = build_kv_manager(spec, ["l0"], num_blocks=16, max_model_len=1024) + null_id = mgr.block_pool.null_block.block_id + adapter = ARDiffusionRequestAdapter("req", chunk_size=BLOCK) + + mgr.allocate_slots(adapter, num_new_tokens=BLOCK) + block_ids = mgr.get_block_ids(adapter.request_id)[0] + real = set(resident_block_ids(block_ids, null_id)) + assert real, "expected at least one real block after allocation" + + slots = chunk_slot_mapping(block_ids, adapter.num_computed_tokens, spec.chunk_size, spec.block_size) + # Each slot resolves to a real allocated block, none to null_block. + blocks_used = {int(s) // spec.block_size for s in slots} + assert blocks_used <= real + assert null_id not in blocks_used + assert len(slots) == spec.chunk_size diff --git a/tests/dreamzero/test_pipeline_state.py b/tests/dreamzero/test_pipeline_state.py index a3a5aaec03c..f00caf53ca7 100644 --- a/tests/dreamzero/test_pipeline_state.py +++ b/tests/dreamzero/test_pipeline_state.py @@ -4,7 +4,6 @@ from collections import OrderedDict import pytest -import torch from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState @@ -46,14 +45,11 @@ def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None: assert "session-b" not in pipeline._states -def test_dreamzero_state_cache_access_requires_initialization() -> None: +def test_dreamzero_state_owns_no_kv_caches() -> None: + """KV (self- and cross-attention) is engine-owned since the AR-Diffusion + paged backend: the model-local cache accessors must be gone so nothing can + silently bypass the engine pool.""" state = DreamZeroState() - with pytest.raises(RuntimeError, match="KV caches not initialized"): - state.get_kv_caches() - - with pytest.raises(RuntimeError, match="Cross-attn caches not initialized"): - state.get_crossattn_caches() - - with pytest.raises(RuntimeError, match="create_kv_caches first"): - state.update_kv_cache(0, torch.empty(0)) + for removed in ("get_kv_caches", "create_kv_caches", "update_kv_cache", "get_crossattn_caches"): + assert not hasattr(state, removed) diff --git a/vllm_omni/deploy/dreamzero.yaml b/vllm_omni/deploy/dreamzero.yaml index 079101ba0e0..62b3a5eed9b 100644 --- a/vllm_omni/deploy/dreamzero.yaml +++ b/vllm_omni/deploy/dreamzero.yaml @@ -14,6 +14,8 @@ stages: max_num_seqs: 1 enforce_eager: false model_class_name: DreamZeroPipeline + # DreamZero runs on the AR-Diffusion engine (engine-level paged KV cache). + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine model_config: default_robot_embodiment: roboarena policy_server_config: diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index f5fe4c30aeb..7362e5cf766 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -613,6 +613,20 @@ class OmniDiffusionConfig: distributed_executor_backend: str = "mp" nccl_port: int | None = None + # Engine backend selection, resolved by ``DiffusionEngine.resolve_engine_class`` + # (mirrors ``DiffusionExecutor.get_class``). Config files use a string: + # "default" -> DiffusionEngine, or an import path (set e.g. by a deploy + # config). Programmatic callers may pass a DiffusionEngine subclass + # directly — hence ``str | type`` (structured-config mirrors should expose + # the string form only). + engine_backend: str | type = "default" + + # Optional override for the diffusion model runner class (import path). + # Precedence in the worker: this override > the runner declared by the + # selected engine class (``default_diffusion_model_runner_cls``) > the + # platform default. Never mutated by engines. + diffusion_model_runner_cls: str | None = None + # HuggingFace specific parameters trust_remote_code: bool = False revision: str | None = None @@ -1173,7 +1187,8 @@ def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig": valid_fields = {f.name for f in fields(cls)} filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_fields} - return cls(**filtered_kwargs) + instance = cls(**filtered_kwargs) + return instance @dataclass diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 2554099e760..f552ae2936c 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -134,6 +134,14 @@ class _RpcTask: class DiffusionEngine: """The diffusion engine for vLLM-Omni diffusion models.""" + #: Import path of the model runner this engine's workers should build, or + #: ``None`` for the platform default. Subclasses declare their runner here + #: (e.g. the AR-Diffusion engine), the worker resolves it through + #: :meth:`resolve_engine_class` — so engine->runner routing lives on the + #: engine class itself, and ``od_config.diffusion_model_runner_cls`` + #: remains a pure explicit user override (never mutated by engines). + default_diffusion_model_runner_cls: str | None = None + def __init__( self, od_config: OmniDiffusionConfig, @@ -608,20 +616,56 @@ def _handle_step_streaming_runner_output( ) self._put_streaming_output_with_cv(request_id, out) + @staticmethod + def resolve_engine_class(config: OmniDiffusionConfig) -> type[DiffusionEngine]: + """Resolve the engine class selected by ``config.engine_backend``. + + Mirrors ``DiffusionExecutor.get_class``: accepts ``"default"``, a + ``DiffusionEngine`` subclass, or an import-path string (e.g. a deploy + config's ``engine_backend``). Kept separate from :meth:`make_engine` so the + selection is testable without constructing an engine (which runs a dummy forward). + + Args: + config: The configuration for the diffusion engine. + + Returns: + The ``DiffusionEngine`` (sub)class to instantiate. + """ + backend = getattr(config, "engine_backend", "default") or "default" + + if isinstance(backend, type): + if not issubclass(backend, DiffusionEngine): + raise TypeError(f"engine_backend must be a DiffusionEngine subclass. Got {backend}.") + return backend + if backend == "default": + return DiffusionEngine + if isinstance(backend, str): + try: + engine_class = resolve_obj_by_qualname(backend) + except (ImportError, ValueError) as e: + raise ValueError( + f"Failed to load engine_backend '{backend}'. Ensure it is a valid python path. Error: {e}" + ) from e + if not issubclass(engine_class, DiffusionEngine): + raise TypeError(f"engine_backend must resolve to a DiffusionEngine subclass. Got {engine_class}.") + return engine_class + raise ValueError(f"Unknown engine_backend: {backend!r}") + @staticmethod def make_engine( config: OmniDiffusionConfig, scheduler: SchedulerInterface | None = None, ) -> DiffusionEngine: - """Factory method to create a DiffusionEngine instance. + """Factory method to create the engine selected by ``config.engine_backend``. Args: config: The configuration for the diffusion engine. Returns: - An instance of DiffusionEngine. + An instance of the resolved ``DiffusionEngine`` (sub)class. """ - return DiffusionEngine(config, scheduler=scheduler) + engine_class = DiffusionEngine.resolve_engine_class(config) + return engine_class(config, scheduler=scheduler) def add_request(self, request: OmniDiffusionRequest) -> str: with self._cv: diff --git a/vllm_omni/diffusion/models/dreamzero/causal_wan_model.py b/vllm_omni/diffusion/models/dreamzero/causal_wan_model.py index ad3175e1ed5..ca92538918f 100644 --- a/vllm_omni/diffusion/models/dreamzero/causal_wan_model.py +++ b/vllm_omni/diffusion/models/dreamzero/causal_wan_model.py @@ -37,6 +37,20 @@ MultiEmbodimentActionEncoder, ) +# AR-Diffusion paged self-attention (in-tree experimental engine). Import at +# module level so the isinstance check + custom-op call trace cleanly inside +# the fullgraph-compiled DiT block (an import inside the traced region would +# graph-break). The model still works without the engine: the payload type is +# only ever constructed by the AR-Diffusion runner. +try: + from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( + ARDiffusionPagedLayerInputs, + paged_write_attn, + ) +except ImportError: # pragma: no cover - experimental package always ships in-tree + ARDiffusionPagedLayerInputs = None + paged_write_attn = None + # ── RoPE utilities ────────────────────────────────────────────────── @@ -510,7 +524,7 @@ def forward( freqs_action: torch.Tensor, freqs_state: torch.Tensor, action_register_length: int | None, - kv_cache: torch.Tensor | None = None, + kv_cache: torch.Tensor | Any | None = None, current_start_frame: int = 0, is_tf: bool = True, ) -> tuple[torch.Tensor, torch.Tensor | None]: @@ -567,24 +581,44 @@ def forward( action_v = v[:, -action_register_length:] v = v[:, :-action_register_length] - updated_k = kv_cache[0] - updated_v = kv_cache[1] - new_k = torch.cat([updated_k, roped_key], dim=1) - new_v = torch.cat([updated_v, v], dim=1) - new_k = new_k[:, -self.max_attention_size :] - new_v = new_v[:, -self.max_attention_size :] - - if action_register_length is not None: - q_cat = torch.cat([roped_query, roped_action_query], dim=1) - k_cat = torch.cat([new_k, roped_action_key], dim=1) - v_cat = torch.cat([new_v, action_v], dim=1) + if ARDiffusionPagedLayerInputs is not None and isinstance(kv_cache, ARDiffusionPagedLayerInputs): + # Fused write+attend custom op: one opaque node in the compiled + # graph (slot writes + FlashAttention block-table kernel inside). + # Metadata tensors were prepared once per forward in _forward_blocks. + if action_register_length is not None: + q_cat = torch.cat([roped_query, roped_action_query], dim=1) + k_act, v_act = roped_action_key[0], action_v[0] + else: + q_cat = roped_query + k_act = v_act = None + x = paged_write_attn( + kv_cache, + q_cat[0], + roped_key[0], + v[0], + k_act, + v_act, + self.head_dim**-0.5, + ).unsqueeze(0) else: - q_cat = roped_query - k_cat = new_k - v_cat = new_v + updated_k = kv_cache[0] + updated_v = kv_cache[1] + new_k = torch.cat([updated_k, roped_key], dim=1) + new_v = torch.cat([updated_v, v], dim=1) + new_k = new_k[:, -self.max_attention_size :] + new_v = new_v[:, -self.max_attention_size :] + + if action_register_length is not None: + q_cat = torch.cat([roped_query, roped_action_query], dim=1) + k_cat = torch.cat([new_k, roped_action_key], dim=1) + v_cat = torch.cat([new_v, action_v], dim=1) + else: + q_cat = roped_query + k_cat = new_k + v_cat = new_v - x = self.attn(q_cat, k_cat, v_cat) - updated_kv_cache = torch.stack([new_k, new_v], dim=0) + x = self.attn(q_cat, k_cat, v_cat) + updated_kv_cache = torch.stack([new_k, new_v], dim=0) x = x.flatten(2) x = self.o(x) @@ -646,7 +680,7 @@ def forward( freqs_state: torch.Tensor, context: torch.Tensor, action_register_length: int | None = None, - kv_cache: torch.Tensor | None = None, + kv_cache: torch.Tensor | Any | None = None, crossattn_cache: dict | None = None, current_start_frame: int = 0, is_tf: bool = True, @@ -925,10 +959,10 @@ def _forward_blocks( action: torch.Tensor | None, timestep_action: torch.Tensor | None, state: torch.Tensor | None, - kv_cache: list[torch.Tensor], - crossattn_cache: list[dict] | None, + kv_cache: list[Any], current_start_frame: int, - ) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]: + crossattn_cache: list[dict] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor | None]]: x = x.flatten(start_dim=2).transpose(1, 2) B = x.shape[0] F_t = timestep.shape[1] @@ -966,7 +1000,28 @@ def _forward_blocks( clip_embedding = self.img_emb(clip_feature) context = torch.cat([clip_embedding, context], dim=1) - updated_kv_caches: list[torch.Tensor] = [] + # AR-Diffusion paged path: host-side prep ONCE per branch forward, outside + # the compiled blocks — allocate current video/action slots, build the + # padded block-table metadata all 40 layers share, and hand the compiled + # region plain tensors (ARDiffusionPagedLayerInputs) instead of a Python + # context object. Lazy-allocation contract preserved: only the branch this + # CFG-parallel rank executes reaches its _forward_blocks. + if kv_cache and getattr(kv_cache[0], "is_ar_diffusion_paged_context", False): + if B != 1: + raise RuntimeError("AR-Diffusion paged self-attention currently expects batch_size=1") + fctx = kv_cache[0].forward_ctx + if seq_len != fctx.seq_len: + raise RuntimeError( + f"AR-Diffusion paged context seq_len={fctx.seq_len} but current video KV has {seq_len} tokens" + ) + fctx.prepare( + device=x.device, + action_len=int(action_register_length or 0), + query_len=int(x.shape[1]), + ) + kv_cache = [c.to_layer_inputs() for c in kv_cache] + + updated_kv_caches: list[torch.Tensor | None] = [] for block_index, block in enumerate(self.blocks): x, updated_kv_cache = block( x=x, @@ -1000,9 +1055,9 @@ def _forward_inference( timestep: torch.Tensor, context: torch.Tensor, seq_len: int, - kv_cache: list[torch.Tensor], - crossattn_cache: list[torch.Tensor], + kv_cache: list[Any], current_start_frame: int, + crossattn_cache: list[dict] | None = None, y: torch.Tensor | None = None, clip_feature: torch.Tensor | None = None, action: torch.Tensor | None = None, diff --git a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py index 1c1902a7151..107638f8179 100644 --- a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py @@ -3,7 +3,7 @@ """DreamZero pipeline for vllm-omni. -Entry point for DiffusionEngine.step() → pipeline.forward(req) +Entry point for DiffusionEngine.step() -> pipeline.forward(req) """ from __future__ import annotations @@ -93,17 +93,102 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): Multi-output: predict_noise() returns (video_pred, action_pred). CFG: video gets standard CFG, action takes positive branch only. - State: DreamZeroState manages KV cache + frame buffer across forward() calls. + + KV is managed by the AR-Diffusion engine: ``self._ar_diffusion_kv_state`` is set by the + runner before ``forward()`` and the pipeline routes all KV access (get / update / + commit / reset) through the pool-backed state. Purely duck-typed (no engine import). """ + _ar_diffusion_kv_state = None # set by the runner before each forward + + def _kv_get(self, state, is_negative, seq_len=None, update_kv_cache=False): + return self._ar_diffusion_kv_state.get_kv_caches( + is_negative, + seq_len=seq_len, + commit_current=update_kv_cache, + ) + + def _kv_create(self, state, batch_size, dtype, device, num_layers, num_heads, head_dim): + # The engine owns all KV allocation: self-attn is allocated lazily from + # paged contexts, and cross-attn is populated eagerly in _kv_populate_cross. + # Nothing is created model-side. + return + + def _kv_commit(self, is_negative: bool): + self._ar_diffusion_kv_state.commit_paged_context(is_negative) + + def _kv_get_cross(self, state, is_negative): + """Cross-attn cache from the engine pool (text k/v + I2V image k_img/v_img).""" + return self._ar_diffusion_kv_state.get_cross_kv_caches(is_negative) + + def _kv_populate_cross(self, context: torch.Tensor, clip_feature, is_negative: bool) -> None: + """Eagerly project cross-attn K/V for all layers into the AR-Diffusion pool. + + Caches the session-invariant cross-attn projections once, per half: the text + ``k``/``v`` from ``text_embedding(context)`` survive window-boundary resets + (prompt unchanged within a session — only session resets clear them), while + the I2V image-token ``k_img``/``v_img`` from ``img_emb(clip_feature)`` (the + 257 image tokens the forward splits off, cached model-side by #4154) are + re-projected on every window restart from the fresh CLIP features. Must run + after the image is encoded so ``clip_feature`` is available. + """ + s = self._ar_diffusion_kv_state + need_text = not s._cross_text_populated.get(is_negative, False) + need_img = ( + clip_feature is not None + and getattr(self.transformer, "model_type", "t2v") == "i2v" + and not s._cross_img_populated.get(is_negative, False) + ) + if not need_text and not need_img: + return + projected = self.transformer.text_embedding(context) if need_text else None + img_ctx = self.transformer.img_emb(clip_feature) if need_img else None + for i, block in enumerate(self.transformer.blocks): + ca = block.cross_attn + n, d = ca.tp_num_heads, ca.head_dim + k = v = None + if projected is not None: + k = ca.norm_k(ca.k(projected)).unflatten(2, (n, d)) + v = ca.v(projected).unflatten(2, (n, d)) + k_img = v_img = None + if img_ctx is not None: + k_img = ca.norm_k_img(ca.k_img(img_ctx)).unflatten(2, (n, d)) + v_img = ca.v_img(img_ctx).unflatten(2, (n, d)) + s.kv_cache.write_cross_kv(i, is_negative, k, v, k_img, v_img) + if need_text: + s._cross_text_populated[is_negative] = True + if need_img: + s._cross_img_populated[is_negative] = True + logger.info( + "AR-Diffusion CROSS POPULATE [%s]: %d layers, text=%s img=%s", + "neg" if is_negative else "pos", + len(self.transformer.blocks), + "kept" if projected is None else tuple(context.shape), + None if img_ctx is None else tuple(img_ctx.shape), + ) + + def _kv_reset(self, state, *, clear_video_latents: bool = True): + """Reset the engine's pooled session window plus the model's non-KV state. + + DreamZero resets at the attention-window boundary; the engine pool drops the + same window so the next forward starts fresh. ``clear_video_latents=False`` + keeps the accumulated video latents for export. + + ``clear_video_latents=False`` also marks a window ("inference") reset: the + prompt is unchanged, so the pool keeps the text cross-attn K/V and only the + image half repopulates on the restart forward. + """ + state.reset(clear_video_latents=clear_video_latents) + self._ar_diffusion_kv_state.reset(keep_cross_text=not clear_video_latents) + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: """Initialize pipeline components. DreamZero root checkpoint layout (GEAR-Dreams/DreamZero-DROID): - config.json — root config (action_head_cfg, architectures, etc.) - model-*.safetensors — all learned weights (action_head.{model,text_encoder,image_encoder,vae}.*) - experiment_cfg/metadata.json — per-embodiment action normalization stats - vae/ — symlink to Wan2.1 VAE (diffusers-compatible) + config.json -- root config (action_head_cfg, architectures, etc.) + model-*.safetensors -- all learned weights (action_head.{model,text_encoder,image_encoder,vae}.*) + experiment_cfg/metadata.json -- per-embodiment action normalization stats + vae/ -- symlink to Wan2.1 VAE (diffusers-compatible) Components are instantiated from config (not from_pretrained), then filled by load_weights() which reads root safetensors and remaps key prefixes. @@ -118,6 +203,18 @@ def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: """ super().__init__() + # DreamZero is engine-only: every KV access in forward() routes through + # the AR-Diffusion engine's pool-backed state. Fail fast here — a stale + # or programmatic config that leaves engine_backend="default" would + # otherwise only crash mid-forward on the first KV access. + engine_backend = str(getattr(od_config, "engine_backend", "") or "") + if "ar_diffusion" not in engine_backend.lower().replace("-", "_"): + raise ValueError( + "DreamZeroPipeline requires the AR-Diffusion engine; set " + "engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine " + f"in the deploy config (got engine_backend={engine_backend!r})." + ) + model_path = od_config.model model_config = od_config.model_config local_files_only = os.path.exists(model_path) @@ -228,8 +325,11 @@ def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: self.max_action_dim: int = ah_config["max_action_dim"] self.negative_prompt: str = model_config.get("negative_prompt", DEFAULT_NEGATIVE_PROMPT) + # The negative prompt is a model constant: encode it once, lazily, and + # reuse across every forward/session (UMT5 encode is deterministic). + self._negative_prompt_embeds_cache: torch.Tensor | None = None - # Embodiment name → numeric ID mapping (model knowledge) + # Embodiment name -> numeric ID mapping (model knowledge) self.embodiment_name_to_id: dict[str, int] = model_config.get( "embodiment_name_to_id", DEFAULT_EMBODIMENT_NAME_TO_ID, @@ -344,11 +444,17 @@ def _predict_noise_eager(self, kwargs: dict) -> tuple[torch.Tensor, torch.Tensor state=kwargs.get("state_features"), embodiment_id=kwargs.get("embodiment_id"), ) - if kwargs.get("update_kv_cache", False) and updated_kv_caches: - state = kwargs.get("dreamzero_state", self.state) + if kwargs.get("update_kv_cache", False): is_neg = kwargs.get("is_negative", False) - for i, kv in enumerate(updated_kv_caches): - state.update_kv_cache(i, kv, is_negative=is_neg) + logger.debug( + "AR-Diffusion pipeline predict_noise -> commit paged context: " + "is_neg=%s seq_len=%s current_start_frame=%s layers=%d", + is_neg, + kwargs.get("seq_len"), + kwargs.get("current_start_frame"), + len(updated_kv_caches), + ) + self._kv_commit(is_neg) return video_pred, action_pred @@ -362,6 +468,8 @@ def setup_compile(self) -> None: Paper D.2 uses ``mode=reduce-overhead``, ``fullgraph=True``, ``dynamic=False`` on text/image/VAE and DiT. VAE decode uses a tensor feat_cache patch and compiles ``decoder.forward`` (not ``_decode``, which has a Python frame loop). + Incremental VAE encode (``_vae_encode_encoder_chunk``) stays eager because + Wan ``feat_cache`` mutation is incompatible with CUDAGraph capture. DiT blocks use per-block ``fullgraph=True``. """ if not torch.cuda.is_available(): @@ -376,6 +484,8 @@ def setup_compile(self) -> None: compile_ro = {"mode": "reduce-overhead", "fullgraph": True, "dynamic": False} # DiT blocks: default avoids CUDAGraph overwrite on modulation tensors; encoders use reduce-overhead. + # The AR-Diffusion paged self-attention is a registered custom op, so the + # block stays fullgraph even on that path. dit_compile = {"mode": "default", "fullgraph": True, "dynamic": False} logger.info( @@ -396,9 +506,9 @@ def setup_compile(self) -> None: logger.warning("DreamZero: image_encoder compile failed (%s); skipping.", exc) try: - self.vae.encode = torch.compile(self.vae.encode, **compile_ro) + self.vae._encode = torch.compile(self.vae._encode, **compile_ro) except Exception as exc: - logger.warning("DreamZero: vae.encode compile failed (%s); skipping.", exc) + logger.warning("DreamZero: vae._encode compile failed (%s); skipping.", exc) compiled_blocks = 0 for block in self.transformer.blocks: @@ -437,10 +547,24 @@ def warmup_compile(self) -> None: try: image = torch.zeros(1, 1, 3, 180, 320, dtype=torch.bfloat16, device=device) - self._encode_image(image, self.num_frames, 180, 320) + self._encode_image(image, self.num_frames, 180, 320, state=self.state) except Exception as exc: logger.warning("DreamZero compile warmup (image_encoder) skipped: %s", exc) + try: + self.state.reset_vae_encoder_stream() + dummy_video = torch.zeros(1, 3, 1, 180, 320, dtype=torch.bfloat16, device=device) + self._vae_stream_seed(self.state, dummy_video[:, :, :1]) + for _ in range(4): + self._vae_stream_append_frame(self.state, dummy_video[:, :, :1]) + self._vae_stream_get_observation_latents( + self.state, + self.num_frame_per_block, + dtype=torch.bfloat16, + ) + except Exception as exc: + logger.warning("DreamZero compile warmup (vae encode stream) skipped: %s", exc) + try: latent_h, latent_w = 180 // 8, 320 // 8 dummy_latent = torch.zeros( @@ -499,7 +623,7 @@ def _synchronize_cfg_parallel_step_output( # ----------------------------------------------------------------------- def _preprocess_video(self, videos: torch.Tensor) -> torch.Tensor: - """uint8 [B,T,H,W,C] → bfloat16 [B,C,T,H,W] normalized to [-1,1].""" + """uint8 [B,T,H,W,C] -> bfloat16 [B,C,T,H,W] normalized to [-1,1].""" videos = videos.permute(0, 4, 1, 2, 3) if videos.dtype == torch.uint8: videos = videos.float() / 255.0 @@ -546,6 +670,8 @@ def _encode_image( num_frames: int, height: int, width: int, + *, + state: DreamZeroState | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Encode first frame via CLIP + VAE. Returns: (clip_feas, ys, image_latent) @@ -581,11 +707,147 @@ def _encode_image( new_image = y[:, :, 0:1] y = torch.concat([msk, y], dim=1) + if state is not None: + if not state.vae_stream_initialized: + # Seed the AR streaming encoder after the compiled full encode above; + # ``cudagraph_mark_step_begin`` isolates eager feat_cache work from CUDAGraph. + self._vae_stream_seed(state, image_input[:, :, :1]) + else: + # Window ("inference") restart with a live stream: keep the real + # frame history in the Wan feat_cache and append the restart + # observation instead of reseeding from scratch. + self._cudagraph_mark_step_begin() + self._vae_stream_append_frame(state, image_input[:, :, :1]) + return clip_context, y, new_image + def _vae_patchify(self, videos: torch.Tensor) -> torch.Tensor: + if self.vae.config.patch_size is not None: + from diffusers.models.autoencoders.autoencoder_kl_wan import patchify + + return patchify(videos, patch_size=self.vae.config.patch_size) + return videos + + @staticmethod + def _vae_clone_feat_map(feat_map: list[torch.Tensor | None]) -> list[torch.Tensor | None]: + return [entry.clone() if isinstance(entry, torch.Tensor) else entry for entry in feat_map] + + def _vae_init_enc_feat_map(self) -> list[torch.Tensor | None]: + self.vae.clear_cache() + return self._vae_clone_feat_map(self.vae._enc_feat_map) + + def _vae_encode_encoder_chunk( + self, + chunk: torch.Tensor, + feat_map: list[torch.Tensor | None], + ) -> tuple[torch.Tensor, list[torch.Tensor | None]]: + """Run one Wan encoder chunk while mutating ``feat_map`` in place. + + Must stay eager (not ``torch.compile``): Wan causal ``feat_cache`` updates + conflict with ``reduce-overhead`` CUDAGraph capture. + """ + self.vae._enc_feat_map = feat_map + self.vae._enc_conv_idx = [0] + out = self.vae.encoder( + chunk, + feat_cache=self.vae._enc_feat_map, + feat_idx=self.vae._enc_conv_idx, + ) + return out, self._vae_clone_feat_map(self.vae._enc_feat_map) + + def _vae_quantize_encoder_out(self, encoder_out: torch.Tensor) -> torch.Tensor: + enc = self.vae.quant_conv(encoder_out) + mu, _ = enc.chunk(2, dim=1) + mean = self.vae_latents_mean.to(device=mu.device, dtype=mu.dtype) + inv_std = self.vae_latents_inv_std.to(device=mu.device, dtype=mu.dtype) + return (mu - mean) * inv_std + + def _vae_stream_seed(self, state: DreamZeroState, first_frame: torch.Tensor) -> None: + """Seed incremental VAE encode with the first observation frame.""" + state.reset_vae_encoder_stream() + feat_map = self._vae_init_enc_feat_map() + chunk = self._vae_patchify(first_frame.to(dtype=self.vae.dtype)) + self._cudagraph_mark_step_begin() + encoder_out, feat_map = self._vae_encode_encoder_chunk(chunk[:, :, :1], feat_map) + state.vae_enc_feat_map = feat_map + state.vae_encoder_out = encoder_out + state.vae_pending_body_frames = None + state.vae_stream_initialized = True + + def _vae_stream_append_frame(self, state: DreamZeroState, new_frame: torch.Tensor) -> None: + """Append one pixel frame and encode a 4-frame body chunk when ready.""" + if not state.vae_stream_initialized or state.vae_enc_feat_map is None or state.vae_encoder_out is None: + raise RuntimeError("VAE encoder stream is not initialized.") + + frame = new_frame.to(dtype=self.vae.dtype) + if state.vae_pending_body_frames is None: + state.vae_pending_body_frames = frame + else: + state.vae_pending_body_frames = torch.cat([state.vae_pending_body_frames, frame], dim=2) + + while state.vae_pending_body_frames is not None and state.vae_pending_body_frames.shape[2] >= 4: + body = state.vae_pending_body_frames[:, :, :4] + if state.vae_pending_body_frames.shape[2] > 4: + state.vae_pending_body_frames = state.vae_pending_body_frames[:, :, 4:] + else: + state.vae_pending_body_frames = None + chunk = self._vae_patchify(body) + encoder_chunk, feat_map = self._vae_encode_encoder_chunk(chunk, state.vae_enc_feat_map) + state.vae_encoder_out = torch.cat([state.vae_encoder_out, encoder_chunk], dim=2) + state.vae_enc_feat_map = feat_map + + def _vae_stream_get_observation_latents( + self, + state: DreamZeroState, + num_latent_frames: int, + *, + dtype: torch.dtype, + ) -> torch.Tensor: + if state.vae_encoder_out is None: + raise RuntimeError("VAE encoder stream has no accumulated encoder output.") + latents = self._vae_quantize_encoder_out(state.vae_encoder_out).to(dtype=dtype) + if latents.shape[2] >= num_latent_frames: + return latents[:, :, -num_latent_frames:] + pad_count = num_latent_frames - latents.shape[2] + pad = latents[:, :, -1:].expand(-1, -1, pad_count, -1, -1) + return torch.cat([pad, latents], dim=2) + + def _preprocess_vae_observation_window(self, videos: torch.Tensor) -> torch.Tensor: + _, _, num_frames_raw, _, _ = videos.shape + if (num_frames_raw - 1) // 4 == self.num_frame_per_block: + return videos + if num_frames_raw // 4 != self.num_frame_per_block: + repeat_factor = self.num_frame_per_block // (num_frames_raw // 4) + videos = torch.repeat_interleave(videos, repeat_factor, dim=2) + first_frame = videos[:, :, 0:1] + return torch.cat([first_frame, videos], dim=2) + first_frame = videos[:, :, 0:1] + return torch.cat([first_frame, videos], dim=2) + + def _encode_observation_latents( + self, + state: DreamZeroState, + videos: torch.Tensor, + *, + latent_dtype: torch.dtype, + ) -> torch.Tensor: + """Encode current robot observation into normalized VAE latents.""" + if state.vae_stream_initialized: + self._cudagraph_mark_step_begin() + self._vae_stream_append_frame(state, videos[:, :, -1:]) + return self._vae_stream_get_observation_latents( + state, + self.num_frame_per_block, + dtype=latent_dtype, + ) + + videos = self._preprocess_vae_observation_window(videos) + return self._encode_vae_latents(videos).to(dtype=latent_dtype) + def _encode_vae_latents(self, videos: torch.Tensor) -> torch.Tensor: """Encode videos into normalized VAE latents.""" input_dtype = videos.dtype + self._cudagraph_mark_step_begin() hidden = self.vae._encode(videos.to(dtype=self.vae.dtype)) mu, _ = hidden.chunk(2, dim=1) mean = self.vae_latents_mean.to(device=mu.device, dtype=mu.dtype) @@ -635,7 +897,7 @@ def _prefill_kv_cache( ) -> None: """Prefill KV cache with first frame and/or current observation. - Uses predict_noise_maybe_with_cfg() for CFG parallel — same path as + Uses predict_noise_maybe_with_cfg() for CFG parallel -- same path as the denoise loop. The mixin handles rank dispatch automatically. KV cache update happens as a side effect inside predict_noise(). """ @@ -646,7 +908,8 @@ def _prefill_kv_cache( head_dim = self.transformer.dim // self.transformer.num_heads if state.current_start_frame == 0: - state.create_kv_caches( + self._kv_create( + state, batch_size, dtype, device, @@ -671,16 +934,16 @@ def _prefill_kv_cache( ) positive_kwargs = dict( encoder_hidden_states=prompt_embeds, - kv_cache=state.get_kv_caches(False), - crossattn_cache=state.get_crossattn_caches(False), + kv_cache=self._kv_get(state, False, seq_len=frame_seqlen, update_kv_cache=True), + crossattn_cache=self._kv_get_cross(state, False), is_negative=False, **common, ) negative_kwargs = ( dict( encoder_hidden_states=negative_prompt_embeds, - kv_cache=state.get_kv_caches(True), - crossattn_cache=state.get_crossattn_caches(True), + kv_cache=self._kv_get(state, True, seq_len=frame_seqlen, update_kv_cache=True), + crossattn_cache=self._kv_get_cross(state, True), is_negative=True, **common, ) @@ -721,16 +984,16 @@ def _prefill_kv_cache( ) positive_kwargs = dict( encoder_hidden_states=prompt_embeds, - kv_cache=state.get_kv_caches(False), - crossattn_cache=state.get_crossattn_caches(False), + kv_cache=self._kv_get(state, False, seq_len=seq_len, update_kv_cache=True), + crossattn_cache=self._kv_get_cross(state, False), is_negative=False, **common, ) negative_kwargs = ( dict( encoder_hidden_states=negative_prompt_embeds, - kv_cache=state.get_kv_caches(True), - crossattn_cache=state.get_crossattn_caches(True), + kv_cache=self._kv_get(state, True, seq_len=seq_len, update_kv_cache=True), + crossattn_cache=self._kv_get_cross(state, True), is_negative=True, **common, ) @@ -763,8 +1026,8 @@ def diffuse( For each timestep: 1. Build positive_kwargs / negative_kwargs - 2. predict_noise_maybe_with_cfg() → (video_pred, action_pred) - 3. scheduler_step_maybe_with_cfg() → VideoActionScheduler + 2. predict_noise_maybe_with_cfg() -> (video_pred, action_pred) + 3. scheduler_step_maybe_with_cfg() -> VideoActionScheduler 4. _synchronize_cfg_parallel_step_output() """ seq_len = kwargs["seq_len"] @@ -823,8 +1086,8 @@ def diffuse( hidden_states=noisy_input.transpose(1, 2), timestep_video=timestep, encoder_hidden_states=prompt_embeds, - kv_cache=state.get_kv_caches(False), - crossattn_cache=state.get_crossattn_caches(False), + kv_cache=self._kv_get(state, False, seq_len=seq_len, update_kv_cache=False), + crossattn_cache=self._kv_get_cross(state, False), y=y, clip_feature=state.clip_feas, action=noisy_input_action, @@ -838,8 +1101,8 @@ def diffuse( hidden_states=noisy_input.transpose(1, 2), timestep_video=timestep, encoder_hidden_states=negative_prompt_embeds, - kv_cache=state.get_kv_caches(True), - crossattn_cache=state.get_crossattn_caches(True), + kv_cache=self._kv_get(state, True, seq_len=seq_len, update_kv_cache=False), + crossattn_cache=self._kv_get_cross(state, True), y=y, clip_feature=state.clip_feas, action=noisy_input_action, @@ -936,7 +1199,7 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: device=device, ) - # State: raw from transform → pad to (B, state_horizon=1, max_state_dim) + # State: raw from transform -> pad to (B, state_horizon=1, max_state_dim) raw_state = unified_obs["state"] state_for_postprocess = None if raw_state is not None: @@ -976,38 +1239,66 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: # Explicit reset from OpenPI serving is carried by `extra_args["reset"]` # on the next inference request after websocket reset/session switch. if extra_args.get("reset", False): - state.reset() + self._kv_reset(state) else: + # Auto-reset based on model state (before accumulation). Both a "session" + # reset and an "inference" (window-boundary) reset clear the model-local + # KV window, so route both through _kv_reset to drop the AR-Diffusion pool window + # too and stay parity-exact; "inference" keeps the accumulated video + # latents (reset_inference_state == reset(clear_video_latents=False)). reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size) if reset_reason == "session": - state.reset() + self._kv_reset(state) elif reset_reason == "inference": - state.reset_inference_state() + self._kv_reset(state, clear_video_latents=False) state.language = text_tokens - # Frame accumulation: stitched single frame → multi-frame video + # Frame accumulation: stitched single frame -> multi-frame video video_frames = state.accumulate_frames(stitched) # (T, H, W, C) videos = torch.from_numpy(video_frames).unsqueeze(0).to(device) # (B=1, T, H, W, C) - videos = self._preprocess_video(videos) # → [B,C,T,H,W] bf16 + videos = self._preprocess_video(videos) # -> [B,C,T,H,W] bf16 _, _, num_frames_raw, height, width = videos.shape - prompt_embeds = self._encode_text(text_tokens, attention_mask) + # Optional phase timing (DZ_PHASE_TIMING=1): logs per-forward stage costs + # (text encode / obs VAE encode / KV prefill / denoise) at INFO. Each mark + # synchronizes CUDA, so leave it off for timed benchmark runs. + _pt = None + if os.environ.get("DZ_PHASE_TIMING"): + import time as _time + + torch.accelerator.synchronize() + _pt = {"time": _time, "t0": _time.perf_counter(), "marks": []} + + def _pt_mark(name: str) -> None: + if _pt is not None: + torch.accelerator.synchronize() + _pt["marks"].append((name, _pt["time"].perf_counter())) + + # Prompt embeds are constant within a session (a prompt change triggers a + # "session" reset above, which clears this cache alongside state.language). + if state.prompt_embeds is None: + state.prompt_embeds = self._encode_text(text_tokens, attention_mask) + prompt_embeds = state.prompt_embeds # Negative prompt for CFG uncond branch (model constant) negative_prompt_embeds = None if self.cfg_scale > 1.0: - neg_inputs = self.tokenizer( - self.negative_prompt, - max_length=512, - padding="max_length", - truncation=True, - return_tensors="pt", - add_special_tokens=True, - ) - negative_prompt_embeds = self._encode_text( - neg_inputs["input_ids"].to(device), - neg_inputs["attention_mask"].to(device), - ) + if self._negative_prompt_embeds_cache is None: + neg_inputs = self.tokenizer( + self.negative_prompt, + max_length=512, + padding="max_length", + truncation=True, + return_tensors="pt", + add_special_tokens=True, + ) + self._negative_prompt_embeds_cache = self._encode_text( + neg_inputs["input_ids"].to(device), + neg_inputs["attention_mask"].to(device), + ) + negative_prompt_embeds = self._negative_prompt_embeds_cache + + _pt_mark("text_encode") # Extract first/last frame for CLIP + VAE encoding if num_frames_raw == 4 or num_frames_raw == 9: @@ -1021,27 +1312,24 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: self.num_frames, height, width, + state=state, ) state.clip_feas = clip_feas.to(dtype=image.dtype) state.ys = ys.to(dtype=image.dtype) - if state.current_start_frame != 0: - # Subsequent calls: encode current observation via VAE - if (num_frames_raw - 1) // 4 == self.num_frame_per_block: - pass - elif num_frames_raw // 4 != self.num_frame_per_block: - repeat_factor = self.num_frame_per_block // (num_frames_raw // 4) - videos = torch.repeat_interleave(videos, repeat_factor, dim=2) - first_frame = videos[:, :, 0:1] - videos = torch.cat([first_frame, videos], dim=2) - else: - first_frame = videos[:, :, 0:1] - videos = torch.cat([first_frame, videos], dim=2) + # Eager cross-attn population (AR-Diffusion only): cache text + image-token K/V + # into the pool now that the image is encoded (clip_feas available). + # Runs on the first forward of a session and after each window-boundary + # reset (current_start_frame returns to 0); the populate guards (text/img halves) gate re-entry. + self._kv_populate_cross(prompt_embeds, state.clip_feas, is_negative=False) + if negative_prompt_embeds is not None: + self._kv_populate_cross(negative_prompt_embeds, state.clip_feas, is_negative=True) + if state.current_start_frame != 0: latent_dtype = videos.dtype with torch.no_grad(): - image = self._encode_vae_latents(videos) - image = image.to(dtype=latent_dtype) + image = self._encode_observation_latents(state, videos, latent_dtype=latent_dtype) + _pt_mark("obs_vae_encode") batch_size = image.shape[0] generator = torch.Generator(device=device).manual_seed(self.seed) @@ -1082,6 +1370,7 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: do_true_cfg, state, ) + _pt_mark("prefill_kv") sample_scheduler = copy.deepcopy(self.scheduler) sample_scheduler_action = copy.deepcopy(self.scheduler) @@ -1123,6 +1412,19 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: state_features=state_features, embodiment_id=embodiment_id, ) + if _pt is not None: + _pt_mark("diffuse") + prev_t = _pt["t0"] + parts = [] + for name, t in _pt["marks"]: + parts.append(f"{name}={1000 * (t - prev_t):.1f}ms") + prev_t = t + logger.info( + "DZ_PHASE_TIMING csf=%s total=%.1fms %s", + state.current_start_frame, + 1000 * (prev_t - _pt["t0"]), + " ".join(parts), + ) if state.current_start_frame == 1: video_out = torch.cat([image, video_out], dim=1) @@ -1133,7 +1435,7 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: # q99 denorm: [-1,1] → real values action_out = self._denormalize_action(action_out.float(), embodiment_name) - # Relative → absolute: only for relative_action_keys (joint_position only) + # Relative -> absolute: only for relative_action_keys (joint_position only) # gripper_position is NOT relative, so don't add state back to it if self.relative_action and state_for_postprocess is not None: n_relative = self.relative_action_dim # 7 for DROID (joint only) @@ -1145,7 +1447,7 @@ def forward(self, req: DiffusionRequestBatch, **kwargs) -> DiffusionOutput: action_out[..., :n_relative] + last_state.unsqueeze(1) # broadcast over horizon ) - # Squeeze batch dim for output: (B, horizon, dim) → (horizon, dim) + # Squeeze batch dim for output: (B, horizon, dim) -> (horizon, dim) actions_np = action_out.squeeze(0).float().cpu().numpy() # (horizon, max_action_dim) actions_np = transform.transform_action_output(actions_np) @@ -1245,7 +1547,7 @@ def _denormalize_action( q01 = stats["q01"].to(device=action.device, dtype=action.dtype) q99 = stats["q99"].to(device=action.device, dtype=action.dtype) # action shape: (B, horizon, action_dim) or (B, horizon, max_action_dim) - # q01/q99 shape: (actual_action_dim,) — only denorm actual dims + # q01/q99 shape: (actual_action_dim,) -- only denorm actual dims actual_dim = q01.shape[0] action_real = action.clone() action_real[..., :actual_dim] = (action[..., :actual_dim] + 1) / 2 * (q99 - q01) + q01 diff --git a/vllm_omni/diffusion/models/dreamzero/state_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/state_dreamzero.py index 5815d93d4b1..4c9209808a4 100644 --- a/vllm_omni/diffusion/models/dreamzero/state_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/state_dreamzero.py @@ -115,17 +115,24 @@ def reset(self, *, clear_video_latents: bool = True) -> None: # Concatenated along T before decode (matches upstream video_across_time). self.video_latents_across_time = saved_video_latents - # KV cache once robot-policy diffusion supports that integration. - self.kv_cache = None - self.kv_cache_neg = None - self.crossattn_cache = None - self.crossattn_cache_neg = None self.current_start_frame = 0 self.clip_feas = None self.ys = None if clear_video_latents: + # Session reset: drop the prompt-embed cache and VAE encoder stream. self.language = None + self.prompt_embeds = None + self.reset_vae_encoder_stream() + # Window ("inference") resets keep both: the prompt is unchanged, and the + # Wan feat_cache history is independent of the DiT attention window. + + def reset_vae_encoder_stream(self) -> None: + """Clear incremental Wan VAE encoder state used across AR steps.""" + self.vae_stream_initialized = False + self.vae_enc_feat_map: list[torch.Tensor | None] | None = None + self.vae_encoder_out: torch.Tensor | None = None + self.vae_pending_body_frames: torch.Tensor | None = None def reset_inference_state(self) -> None: """Reset KV/frame state after local attention rolls without dropping video latents.""" @@ -166,57 +173,3 @@ def reset_reason( def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool: """Determine if state should be reset before this forward().""" return self.reset_reason(text_tokens, num_video_frames, local_attn_size) is not None - - # ------------------------------------------------------------------ - # KV cache management - # ------------------------------------------------------------------ - - def create_kv_caches( - self, - batch_size: int, - dtype: torch.dtype, - device: torch.device, - num_layers: int, - num_heads: int, - head_dim: int, - ) -> None: - """Initialize empty KV caches and cross-attention caches.""" - self.kv_cache = [ - torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers) - ] - self.kv_cache_neg = [ - torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers) - ] - - self.crossattn_cache = [ - {"is_init": False, "k": None, "v": None, "k_img": None, "v_img": None} for _ in range(num_layers) - ] - self.crossattn_cache_neg = [ - {"is_init": False, "k": None, "v": None, "k_img": None, "v_img": None} for _ in range(num_layers) - ] - - def update_kv_cache( - self, - layer_index: int, - updated_kv: torch.Tensor, - is_negative: bool = False, - ) -> None: - """Update a single layer's KV cache after prefill.""" - cache = self.kv_cache_neg if is_negative else self.kv_cache - if cache is None: - raise RuntimeError("KV caches not initialized, call create_kv_caches first.") - cache[layer_index] = updated_kv.clone() - - def get_kv_caches(self, is_negative: bool = False) -> list[torch.Tensor]: - """Get KV caches for the specified branch.""" - cache = self.kv_cache_neg if is_negative else self.kv_cache - if cache is None: - raise RuntimeError("KV caches not initialized.") - return cache - - def get_crossattn_caches(self, is_negative: bool = False) -> list[dict[str, bool | torch.Tensor | None]]: - """Get cross-attention caches for the specified branch.""" - cache = self.crossattn_cache_neg if is_negative else self.crossattn_cache - if cache is None: - raise RuntimeError("Cross-attn caches not initialized.") - return cache diff --git a/vllm_omni/diffusion/models/dreamzero/video_export_worker.py b/vllm_omni/diffusion/models/dreamzero/video_export_worker.py index d039e53c816..ba980a33e08 100644 --- a/vllm_omni/diffusion/models/dreamzero/video_export_worker.py +++ b/vllm_omni/diffusion/models/dreamzero/video_export_worker.py @@ -9,6 +9,35 @@ class DreamZeroVideoExportWorkerExtension: """DreamZero worker RPCs used by offline example video export.""" + def gpu_mem_stats(self) -> dict: + """Peak GPU memory (GiB) for this worker's device, for profiling. + + ``max_memory_reserved`` is the caching-allocator high-water mark (the best + in-process proxy for "peak VRAM"); ``max_memory_allocated`` is the live + tensor high-water mark. Both are monotonic from process start, so the + end-of-run values capture the whole rollout's peak. + """ + dev = torch.accelerator.current_device_index() + return { + "device": int(dev), + "peak_reserved_gib": torch.accelerator.max_memory_reserved(dev) / (1024**3), + "peak_allocated_gib": torch.accelerator.max_memory_allocated(dev) / (1024**3), + } + + def ar_diffusion_perf_stats(self, reset: bool = True) -> dict: + """Per-request server-side forward E2E times (s) recorded by the AR-Diffusion runner. + + The worker-side compute time per ``execute_model`` — the analog of the + upstream server's per-request E2E — which excludes the engine<->worker IPC + that the client's ``omni.generate`` wall time includes. Returns ``[]`` when + the runner is the base (KV-disabled) runner that does not record timings. + """ + runner = self.model_runner + times = list(getattr(runner, "_perf_e2e_times", []) or []) + if reset and hasattr(runner, "_perf_e2e_times"): + runner._perf_e2e_times = [] + return {"server_e2e_s": times} + @staticmethod def _latents_to_uint8_frames(decoded: torch.Tensor) -> torch.Tensor: decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous() diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 172f48cf15c..8cce478976d 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -213,8 +213,32 @@ def __init__( self._step_lora_state: dict[str, tuple[LoRARequest | None, float]] = {} self.stage_id = getattr(od_config, "stage_id", 0) self.init_device() - # Create model runner using the platform-specified class - model_runner_cls_path = current_omni_platform.get_diffusion_model_runner_cls() + # Create model runner — one decision chain, in precedence order: + # 1. explicit od_config.diffusion_model_runner_cls (user override), + # 2. the runner declared by the engine class that engine_backend + # selects (e.g. ARDiffusionEngine -> ARDiffusionModelRunner), + # 3. the platform default. + # Routing policy therefore lives on the engine class / config surface; + # engines never mutate od_config. Overrides must be import-path + # strings — guard with isinstance so a non-string (e.g. a Mock + # od_config in tests) doesn't shadow the platform hook. + runner_override = getattr(self.od_config, "diffusion_model_runner_cls", None) + engine_runner = None + if not (isinstance(runner_override, str) and runner_override): + try: + from vllm_omni.diffusion.diffusion_engine import DiffusionEngine + + engine_cls = DiffusionEngine.resolve_engine_class(self.od_config) + engine_runner = getattr(engine_cls, "default_diffusion_model_runner_cls", None) + except Exception: + logger.warning("Worker %s: engine_backend resolution failed; using platform runner", self.rank) + engine_runner = None + if isinstance(runner_override, str) and runner_override: + model_runner_cls_path = runner_override + elif isinstance(engine_runner, str) and engine_runner: + model_runner_cls_path = engine_runner + else: + model_runner_cls_path = current_omni_platform.get_diffusion_model_runner_cls() model_runner_cls = resolve_obj_by_qualname(model_runner_cls_path) self.model_runner = model_runner_cls( vllm_config=self.vllm_config, diff --git a/vllm_omni/experimental/__init__.py b/vllm_omni/experimental/__init__.py index f8b58d229b1..c7d59e2e262 100644 --- a/vllm_omni/experimental/__init__.py +++ b/vllm_omni/experimental/__init__.py @@ -1,3 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Experimental vLLM-Omni features (APIs may change without notice).""" +"""Experimental vLLM-Omni subsystems. + +Modules under ``vllm_omni.experimental`` are in active development and not yet +covered by the stability guarantees of the rest of the package — their APIs, +config surfaces, and defaults may change without notice. The AR-Diffusion +Engine (:mod:`vllm_omni.experimental.ar_diffusion`) lives here. +""" diff --git a/vllm_omni/experimental/ar_diffusion/__init__.py b/vllm_omni/experimental/ar_diffusion/__init__.py new file mode 100644 index 00000000000..b312660ec72 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/__init__.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +"""AR-Diffusion Engine (AR-Diffusion). + +The AR-Diffusion engine: a ``DiffusionEngine`` subclass that adds engine-level KV +cache management for autoregressive / chunked "world-model" diffusion models +(DreamZero and the AR-DiT family). Selected via +``OmniDiffusionConfig.engine_backend = "ar_diffusion"``. +""" + +from vllm_omni.experimental.ar_diffusion.engine import ARDiffusionEngine + +__all__ = ["ARDiffusionEngine"] diff --git a/vllm_omni/experimental/ar_diffusion/engine.py b/vllm_omni/experimental/ar_diffusion/engine.py new file mode 100644 index 00000000000..6fb6ee7c940 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/engine.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The AR-Diffusion Engine (AR-Diffusion).""" + +from __future__ import annotations + +from vllm.logger import init_logger + +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine + +logger = init_logger(__name__) + +#: Import path of the runner the AR-Diffusion engine routes its workers to. +AR_DIFFUSION_MODEL_RUNNER_CLS = "vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner" + + +class ARDiffusionEngine(DiffusionEngine): + """AR-Diffusion engine with engine-level KV cache management. + + AR-Diffusion serves autoregressive / chunked blockwise-causal diffusion models + (world models, AR-DiT) that materialize persistent attention KV. It reuses + vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / ``BlockTables``) + as a library, driven from the engine rather than hand-rolled inside each + model. See ``BDE_doc/diffusion_kv_cache_management.md`` for the design and + ``BDE_doc/dreamzero_kv_phase1_plan.md`` for the rollout. + + It is selected per model via ``OmniDiffusionConfig.engine_backend = "ar_diffusion"`` + (resolved by :meth:`DiffusionEngine.make_engine`), so models that do not opt + in keep using the base ``DiffusionEngine`` unchanged. + + Architecture note: in the multiproc setup the KV cache lives in the worker / + runner process (GPU side), co-located with the model and KV tensors — so the + actual KV *body* is :class:`~vllm_omni.experimental.ar_diffusion.kv_cache.manager.ARDiffusionKVCache`, owned + by :class:`~vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner`. ``ARDiffusionEngine`` itself is the + thin selection / injection seam; it wires the AR-Diffusion executor → worker → runner + so DreamZero's rollout runs against the runner-owned KV cache. + """ + + #: Workers of this engine build the AR-Diffusion runner (unless the config + #: sets an explicit ``diffusion_model_runner_cls`` override). Declared on + #: the class so routing needs no od_config mutation at engine init. + default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py new file mode 100644 index 00000000000..ab29f6093ab --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +"""AR-Diffusion engine-level KV cache helpers. + +Thin glue over vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / +``SlidingWindowManager``), used by the AR-Diffusion Engine to manage KV for +AR-diffusion models. See ``BDE_doc/dreamzero_kv_phase1_plan.md``. + +Layout: ``config`` (public knob) · ``paged`` (engine-generic paging mechanics + +chunk-window eviction spec) · ``manager`` (the ARDiffusionKVCache orchestrator + its +request adapter / pool builders) · ``state`` (the model-facing ARDiffusionKVState bridge). +""" + +from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig +from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ( + ARDiffusionKVCache, + ARDiffusionRequestAdapter, + build_kv_manager, + compute_num_blocks, +) +from vllm_omni.experimental.ar_diffusion.kv_cache.paged import ( + ChunkWindowManager, + ChunkWindowSpec, + allocate_kv_pool_with_views, + chunk_slot_mapping, + compute_slot_mapping, + pool_write_chunk, + resident_block_ids, +) +from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( + ARDiffusionPagedForwardContext, + ARDiffusionPagedLayerContext, + ARDiffusionPagedLayerInputs, + ar_diffusion_paged_attention, + paged_write_attn, + set_current_paged_kv_cache, +) + +__all__ = [ + "ARDiffusionKVCache", + "ARDiffusionKVConfig", + "ARDiffusionPagedForwardContext", + "ARDiffusionPagedLayerContext", + "ARDiffusionPagedLayerInputs", + "ARDiffusionRequestAdapter", + "ChunkWindowManager", + "ChunkWindowSpec", + "allocate_kv_pool_with_views", + "ar_diffusion_paged_attention", + "build_kv_manager", + "chunk_slot_mapping", + "compute_num_blocks", + "compute_slot_mapping", + "paged_write_attn", + "pool_write_chunk", + "resident_block_ids", + "set_current_paged_kv_cache", +] diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py new file mode 100644 index 00000000000..4b3c737751d --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Configuration for AR-Diffusion engine-level KV cache management.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class ARDiffusionKVConfig: + """Settings for the AR-Diffusion paged KV cache. + + Disabled by default: when ``enable`` is False the AR-Diffusion engine behaves exactly + like the base ``DiffusionEngine`` (no pool, no paged KV). + """ + + enable: bool = False + # Persistent KV tokens materialized per AR chunk. For DreamZero this is + # ``num_frame_per_block * frame_seqlen``. + chunk_size: int = 0 + # Resident window in chunks. ``None`` means full attention (no eviction). + window_chunks: int | None = None + # Protected leading chunks (attention sink); never evicted. + sink_chunks: int = 0 + # DreamZero-style window reset vs. VGGT-style sliding replace. + reset_at_boundary: bool = False + # Fraction of free device memory budgeted for the AR-Diffusion KV pool. + gpu_memory_fraction: float = 0.1 + # When CUDA graph / torch.compile is on (not enforce_eager), pre-capture the + # DiT graphs for every window-fill shape at load time via a synthetic rollout, + # so the serving run is fast from the first chunk. No effect when eager. + warmup_cudagraph: bool = True + # Also capture the post-window-boundary (reset-cycle) forward during warm-up. + warmup_capture_reset: bool = False + + @property + def sliding_window(self) -> int | None: + """Window size in tokens, or ``None`` for full attention.""" + if self.window_chunks is None: + return None + return self.window_chunks * self.chunk_size diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py new file mode 100644 index 00000000000..4b3b5d788d4 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ARDiffusionKVCache — the engine-level KV cache orchestrator for one AR-Diffusion model. + +This is the *body* of AR-Diffusion's KV management: it owns a vLLM ``KVCacheManager`` (a +single chunk-window group) and the per-request adapter lifecycle, and exposes the +per-chunk operations a rollout needs — allocate, slot mapping, commit, window +lookup, free. It lives in the model runner (worker / GPU side), co-located with +the model and the KV tensors; the DreamZero pipeline calls these methods during a +rollout. The main-process ``ARDiffusionEngine`` only selects the engine and is otherwise +thin. +""" + +from __future__ import annotations + +import inspect +import os +from collections.abc import Sequence + +import torch +from vllm.logger import init_logger +from vllm.v1.core.kv_cache_manager import KVCacheManager +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + KVCacheSpec, + KVCacheTensor, +) +from vllm.v1.request import RequestStatus + +from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig +from vllm_omni.experimental.ar_diffusion.kv_cache.paged import ( + ChunkWindowSpec, + allocate_kv_pool_with_views, + chunk_slot_mapping, + pool_write_chunk, + resident_block_ids, +) + +_log = init_logger(__name__) + + +class ARDiffusionRequestAdapter: + """Duck-types the subset of ``vllm.v1.request.Request`` that the + ``KVCacheManager`` reads (``allocate_slots`` / ``get_computed_blocks`` / + ``free`` and the coordinator they call into). + + It is intentionally NOT a full ``Request``. The conformance test exercises a + real ``KVCacheManager`` against this adapter so the surface cannot silently + drift across vLLM versions. + + A AR-Diffusion request advances one *chunk* at a time: ``allocate_slots`` is called + once per chunk and ``num_computed_tokens`` advances only when a chunk is + committed (:meth:`on_chunk_committed`), so the ``T`` denoise steps of a chunk + reuse the same slots. + """ + + def __init__( + self, + request_id: str, + *, + chunk_size: int, + prefill_prefix_tokens: int = 0, + ) -> None: + self.request_id = request_id + self._chunk_size = chunk_size + self._prefill = prefill_prefix_tokens + self._completed_chunks = 0 + # Filled only when cross-request prefix reuse is enabled (Phase 3). + self.block_hashes: list = [] + self.skip_reading_prefix_cache = True + self.num_preemptions = 0 + # vLLM watermark gate reads this; map the request lifecycle onto it. + self.status = RequestStatus.WAITING + + @property + def num_computed_tokens(self) -> int: + """Persistent KV already materialized (committed chunks + prefill).""" + return self._prefill + self._completed_chunks * self._chunk_size + + @property + def num_tokens(self) -> int: + """Total tokens once the in-flight chunk is committed.""" + return self._prefill + (self._completed_chunks + 1) * self._chunk_size + + @property + def num_prompt_tokens(self) -> int: + """The prefill prefix length (read by ``cache_blocks`` when caching).""" + return self._prefill + + @property + def completed_chunks(self) -> int: + return self._completed_chunks + + def on_chunk_committed(self) -> None: + """Advance by one chunk. Call once per chunk, not per denoise step.""" + self._completed_chunks += 1 + + +def compute_num_blocks( + available_bytes: int, + gpu_memory_fraction: float, + page_size_bytes: int, +) -> int: + """Number of KV blocks that fit in ``fraction`` of the memory budget.""" + if page_size_bytes <= 0: + raise ValueError(f"page_size_bytes must be positive, got {page_size_bytes}") + if not 0.0 < gpu_memory_fraction <= 1.0: + raise ValueError(f"gpu_memory_fraction must be in (0, 1], got {gpu_memory_fraction}") + budget = int(available_bytes * gpu_memory_fraction) + return max(0, budget // page_size_bytes) + + +def build_kv_manager( + spec: KVCacheSpec, + layer_names: Sequence[str], + num_blocks: int, + max_model_len: int, + *, + enable_caching: bool = False, +) -> KVCacheManager: + """Build a ``KVCacheManager`` with a single KV cache group for ``spec``. + + Args: + spec: The KV cache spec for the group (e.g. a ``ChunkWindowSpec``). + layer_names: Attention layers sharing this group's block table. + num_blocks: Total physical blocks in the pool. + max_model_len: Upper bound on a request's sequence length. + enable_caching: Cross-request prefix caching (Phase 3); off in Phase 1. + """ + layer_names = list(layer_names) + group = KVCacheGroupSpec(layer_names=layer_names, kv_cache_spec=spec) + tensors = [KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=layer_names)] + config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=tensors, + kv_cache_groups=[group], + ) + kwargs = dict(max_model_len=max_model_len, hash_block_size=spec.block_size, enable_caching=enable_caching) + params = inspect.signature(KVCacheManager).parameters + if "scheduler_block_size" in params: + kwargs["scheduler_block_size"] = spec.block_size + if "max_num_batched_tokens" in params: + kwargs["max_num_batched_tokens"] = max_model_len + return KVCacheManager(config, **kwargs) + + +class ARDiffusionKVCache: + """Owns the paged KV pool + per-request lifecycle for a AR-Diffusion model. + + Build once per loaded model (dimensions known); then per request: + ``begin_request`` → per chunk (``allocate_chunk`` → ``chunk_write_slots`` → + [model writes K/V] → ``commit_chunk``) → ``end_request``. + """ + + def __init__( + self, + config: ARDiffusionKVConfig, + *, + num_layers: int, + num_kv_heads: int, + head_size: int, + dtype: torch.dtype, + block_size: int, + max_model_len: int, + available_bytes: int, + cross_attn_length: int = 0, + cross_attn_img_length: int = 0, + device: torch.device | None = None, + local_branches: int = 2, + num_frame_per_block: int = 1, + ) -> None: + if not config.enable: + raise ValueError("ARDiffusionKVCache built with a disabled ARDiffusionKVConfig") + if config.window_chunks is None: + raise ValueError("Phase 1 requires a bounded window (window_chunks)") + if config.chunk_size <= 0: + raise ValueError("ARDiffusionKVConfig.chunk_size must be set (> 0)") + if local_branches not in (1, 2): + raise ValueError(f"local_branches must be 1 or 2, got {local_branches}") + + self.config = config + # How many CFG branches allocate from THIS rank's pool. Under + # CFG-parallel x2 each rank executes exactly one branch (rank0 pos, + # rank1 neg) and the other branch's lazy contexts never allocate, so + # sizing for both would leave ~half the pool as idle capacity. A + # single-process run (TP1 / offline) executes both branches -> 2. + self.local_branches = local_branches + self.num_frame_per_block = max(1, int(num_frame_per_block)) + self.block_size = block_size + self.num_layers = num_layers + self.num_kv_heads = num_kv_heads + self.head_size = head_size + self.dtype = dtype + self.cross_attn_length = cross_attn_length + self.device = device or torch.device("cpu") + self._adapters: dict[str, ARDiffusionRequestAdapter] = {} + + # -- cross-attention pool (static, one-time-fill) ----------------------- + # Cross-attn KV is computed once from the text encoder and never changes. + # Allocate a per-layer contiguous tensor for each branch: (2, text_len, + # kv_heads, head_dim) where dim-0 = [pos, neg]. Allocated separately + # from the self-attn block pool (static, small — ~400 MiB for + # DreamZero I2V). Both pools draw from the same GPU free memory but + # the cross-attn pool is sized directly rather than via the block pool + # budget (no eviction/chunk lifecycle needed). + self.cross_attn_img_length = cross_attn_img_length + self._cross_k: list[torch.Tensor] = [] + self._cross_v: list[torch.Tensor] = [] + # I2V image-token cross-attn pool. Like the text k/v, the image-token + # k_img/v_img are session-invariant (the conditioning image doesn't change), + # so they are cached once and read every denoise step — see + # WanI2VCrossAttention (#4154 caches these model-side too). Empty for T2V. + self._cross_k_img: list[torch.Tensor] = [] + self._cross_v_img: list[torch.Tensor] = [] + + # Bytes the cross-attn pools consume directly (K+V, pos+neg, all layers). + # Deducted from the self-attn paged-pool budget below so the two + # allocations together stay within the GPU free-memory budget. + def _cross_pool_bytes(length: int) -> int: + return 2 * 2 * length * num_kv_heads * head_size * dtype.itemsize * num_layers + + cross_total_bytes = 0 + if device is not None and cross_attn_length > 0: + cross_shape = (2, cross_attn_length, num_kv_heads, head_size) + cross_bytes = _cross_pool_bytes(cross_attn_length) + cross_total_bytes += cross_bytes + for _ in range(num_layers): + self._cross_k.append(torch.empty(cross_shape, dtype=dtype, device=device)) + self._cross_v.append(torch.empty(cross_shape, dtype=dtype, device=device)) + _log.info( + "AR-Diffusion cross-attn pool: %d layers × (%d tok × %d heads × %d) = %.1f MiB", + num_layers, + cross_attn_length, + num_kv_heads, + head_size, + cross_bytes / (1024 * 1024), + ) + if cross_attn_img_length > 0: + img_shape = (2, cross_attn_img_length, num_kv_heads, head_size) + cross_total_bytes += _cross_pool_bytes(cross_attn_img_length) + for _ in range(num_layers): + self._cross_k_img.append(torch.empty(img_shape, dtype=dtype, device=device)) + self._cross_v_img.append(torch.empty(img_shape, dtype=dtype, device=device)) + _log.info( + "AR-Diffusion cross-attn IMG pool: %d layers × %d img-tok (I2V)", + num_layers, + cross_attn_img_length, + ) + + self.spec = ChunkWindowSpec( + block_size=block_size, + num_kv_heads=num_kv_heads, + head_size=head_size, + dtype=dtype, + sliding_window=config.window_chunks * config.chunk_size, + chunk_size=config.chunk_size, + window_chunks=config.window_chunks, + sink_chunks=config.sink_chunks, + reset_at_boundary=config.reset_at_boundary, + ) + # Each pool block spans all layers' K/V, so size against the per-layer + # page size times the layer count. + # Size the self-attn pool against the memory left after the cross-attn + # pools (allocated above), so cross + self-attn stays within the budget. + num_blocks = compute_num_blocks( + max(0, available_bytes - cross_total_bytes), + config.gpu_memory_fraction, + self.spec.page_size_bytes * num_layers, + ) + # Floor: one forward needs the resident window plus the in-flight chunk + # (num_frame_per_block frame-blocks) for every branch THIS rank runs, + # with a little eviction-transient headroom. The memory-fraction + # heuristic can under-size this once block_size grows — e.g. frame-granular + # paging at the true frame_seqlen makes each block larger and the pool + # fewer-blocks — so guarantee the minimum the rollout cannot run without, + # otherwise allocate_chunk hits an exhausted pool mid-forward. + min_blocks = self.local_branches * (config.window_chunks + self.num_frame_per_block) + 2 + if num_blocks < min_blocks: + _log.warning( + "AR-Diffusion KV pool: memory-fraction sizing gave %d blocks; raising to the %d-block " + "floor (%d local CFG branch(es) x (window_chunks=%d + num_frame_per_block=%d) + 2 headroom)", + num_blocks, + min_blocks, + self.local_branches, + config.window_chunks, + self.num_frame_per_block, + ) + num_blocks = min_blocks + layer_names = [f"ar_diffusion.layer.{i}" for i in range(num_layers)] + self.manager = build_kv_manager(self.spec, layer_names, num_blocks, max_model_len) + self.managed_num_blocks = num_blocks + self.num_blocks = num_blocks + # Scratch blocks are outside KVCacheManager ownership. They hold current + # denoise-step video KV when update_kv_cache=False and action/state KV in + # both modes; they are reused every forward and never committed. + # Default 4: measured high-water is num_frame_per_block video blocks (2) + # + 1 action block per branch per forward, +1 margin. + scratch_per_branch = int(os.environ.get("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", "4")) + if scratch_per_branch <= 0: + raise ValueError("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH must be positive") + self.scratch_blocks_per_branch = scratch_per_branch + self.scratch_num_blocks = self.local_branches * scratch_per_branch + self.num_blocks_total = self.managed_num_blocks + self.scratch_num_blocks + self.null_block_id = self.manager.block_pool.null_block.block_id + + # Allocate the per-layer paged K/V pools on the given device. + self._kv_pools: list[torch.Tensor] = [] + self._k_pools: list[torch.Tensor] = [] + self._v_pools: list[torch.Tensor] = [] + if device is not None: + self._kv_pools, self._k_pools, self._v_pools = allocate_kv_pool_with_views( + self.num_blocks_total, + block_size, + num_layers, + num_kv_heads, + head_size, + dtype, + device, + ) + + # -- cross-attention pool access ------------------------------------------- + # Cross-attn KV is static once populated — write once (from text encoder), + # read many (every denoising step). Not managed through the paged block pool. + + def write_cross_kv( + self, + layer_idx: int, + is_negative: bool, + k: torch.Tensor | None, + v: torch.Tensor | None, + k_img: torch.Tensor | None = None, + v_img: torch.Tensor | None = None, + ) -> None: + """Write one layer's cross-attn K/V into the pool. + + ``k`` / ``v``: ``(B, text_len, tp_num_heads, head_dim)``; only batch-0 + is copied (B=1 for inference). ``None`` skips the text half (window + restarts keep the still-valid text K/V and rewrite only the image half). + The ``is_negative`` flag selects the correct CFG branch slot. ``k_img`` / + ``v_img`` (I2V image tokens, ``(B, 257, ...)``) are written when the + image pool is allocated. + """ + branch = 1 if is_negative else 0 + if k is not None: + self._cross_k[layer_idx][branch].copy_(k[0]) + self._cross_v[layer_idx][branch].copy_(v[0]) + if k_img is not None and self._cross_k_img: + self._cross_k_img[layer_idx][branch].copy_(k_img[0]) + self._cross_v_img[layer_idx][branch].copy_(v_img[0]) + + def read_cross_kv(self, layer_idx: int, is_negative: bool) -> dict: + """Return a pool-backed cross-attn cache dict for one layer. + + The dict matches the ``{"is_init": True, "k": Tensor, "v": Tensor}`` + convention the cross-attention module expects — it reads from the pool + slice rather than from the lazy-initialised model-local dict. For I2V, + ``k_img`` / ``v_img`` are added so the image-token cache (added by #4154) + reads from the pool too. + """ + branch = 1 if is_negative else 0 + k = self._cross_k[layer_idx][branch].unsqueeze(0) # (1, L, heads, dim) + v = self._cross_v[layer_idx][branch].unsqueeze(0) + cache = {"is_init": True, "k": k, "v": v} + if self._cross_k_img: + cache["k_img"] = self._cross_k_img[layer_idx][branch].unsqueeze(0) + cache["v_img"] = self._cross_v_img[layer_idx][branch].unsqueeze(0) + return cache + + # -- request lifecycle --------------------------------------------------- + + def begin_request(self, request_id: str, *, prefill_prefix_tokens: int = 0) -> ARDiffusionRequestAdapter: + adapter = ARDiffusionRequestAdapter( + request_id, + chunk_size=self.spec.chunk_size, + prefill_prefix_tokens=prefill_prefix_tokens, + ) + self._adapters[request_id] = adapter + _log.debug("AR-Diffusion begin_request: req=%s prefill=%d", request_id, prefill_prefix_tokens) + return adapter + + def end_request(self, adapter: ARDiffusionRequestAdapter) -> None: + _log.debug( + "AR-Diffusion end_request: req=%s chunks=%d free=%d", + adapter.request_id, + adapter.completed_chunks, + self.manager.block_pool.get_num_free_blocks(), + ) + self.manager.free(adapter) + self._adapters.pop(adapter.request_id, None) + + # -- per-chunk operations ------------------------------------------------ + + def allocate_chunk(self, adapter: ARDiffusionRequestAdapter) -> list[int]: + """Allocate a chunk's blocks (evicting out-of-window blocks first). + + Returns the request's full block table (incl. null_block placeholders). + """ + blocks = self.manager.allocate_slots(adapter, num_new_tokens=self.spec.chunk_size) + if blocks is None: + raise RuntimeError("AR-Diffusion KV pool exhausted while allocating a chunk") + table = self.block_table(adapter) + resident = resident_block_ids(table, self.null_block_id) + _log.debug( + "AR-Diffusion allocate_chunk: req=%s chunk=%d table_len=%d resident=%d free=%d", + adapter.request_id, + adapter.completed_chunks, + len(table), + len(resident), + self.manager.block_pool.get_num_free_blocks(), + ) + return table + + def allocate_token_slots(self, adapter: ARDiffusionRequestAdapter, num_tokens: int) -> list[int]: + """Allocate managed blocks for an in-flight video span without committing it.""" + if num_tokens <= 0: + raise ValueError(f"num_tokens must be positive, got {num_tokens}") + blocks = self.manager.allocate_slots(adapter, num_new_tokens=num_tokens) + if blocks is None: + raise RuntimeError("AR-Diffusion KV pool exhausted while allocating paged attention slots") + return self.block_table(adapter) + + def block_table(self, adapter: ARDiffusionRequestAdapter) -> list[int]: + return list(self.manager.get_block_ids(adapter.request_id)[0]) + + def chunk_write_slots(self, adapter: ARDiffusionRequestAdapter) -> torch.Tensor: + """Slot mapping for the in-flight chunk — the K/V write target.""" + return chunk_slot_mapping( + self.block_table(adapter), + adapter.num_computed_tokens, + self.spec.chunk_size, + self.block_size, + ) + + def scratch_block_ids(self, is_negative: bool, start: int, count: int) -> list[int]: + """Return branch-local scratch block ids outside manager ownership.""" + if count < 0 or start < 0: + raise ValueError(f"scratch start/count must be non-negative, got start={start}, count={count}") + if start + count > self.scratch_blocks_per_branch: + raise RuntimeError( + "AR-Diffusion paged attention scratch blocks exhausted: " + f"need [{start}, {start + count}) of {self.scratch_blocks_per_branch}. " + "Increase AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH." + ) + # With one local branch (CFG-parallel x2) the rank's only branch maps to + # slot 0 whichever CFG side it is; the other branch never allocates here. + branch_offset = self.scratch_blocks_per_branch if (is_negative and self.local_branches == 2) else 0 + base = self.managed_num_blocks + branch_offset + start + return list(range(base, base + count)) + + def key_cache(self, layer_idx: int) -> torch.Tensor: + return self._kv_pools[layer_idx][0] + + def value_cache(self, layer_idx: int) -> torch.Tensor: + return self._kv_pools[layer_idx][1] + + def window_block_ids(self, adapter: ARDiffusionRequestAdapter) -> list[int]: + """Resident (non-null) managed blocks visible to paged attention.""" + return resident_block_ids(self.block_table(adapter), self.null_block_id) + + def commit_chunk(self, adapter: ARDiffusionRequestAdapter) -> None: + """Advance the adapter by one chunk after its K/V is written. + + This standalone primitive is used by low-level manager tests. In the + DreamZero paged-attention path, :meth:`ARDiffusionKVState.commit_paged_context` + advances the adapter only after the forward succeeds. Call once per + committed chunk, not per denoise step. + """ + _log.debug("AR-Diffusion commit: req=%s before=%d", adapter.request_id, adapter.completed_chunks) + adapter.on_chunk_committed() + _log.debug("AR-Diffusion commit: req=%s after=%d", adapter.request_id, adapter.completed_chunks) + + # -- pool-backed K/V access -------------------------------------------- + + def write_chunk_kv( + self, + layer_index: int, + new_k: torch.Tensor, + new_v: torch.Tensor, + adapter: ARDiffusionRequestAdapter, + ) -> None: + """Write one layer's committed-chunk K/V into the pool.""" + slots = self.chunk_write_slots(adapter) + _log.debug( + "AR-Diffusion write: req=%s layer=%d chunk=%d shapes=%s dev=%s", + adapter.request_id, + layer_index, + adapter.completed_chunks, + (tuple(new_k.shape), tuple(new_v.shape)), + slots.device, + ) + pool_write_chunk( + self._k_pools[layer_index], + self._v_pools[layer_index], + new_k, + new_v, + slots, + ) diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py new file mode 100644 index 00000000000..cb84467a3e4 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Generic paging mechanics + chunk-window eviction spec for the AR-Diffusion engine. + +Engine-generic, model-agnostic primitives — the layer a second model (e.g. the +Cosmos port) reuses unchanged. Three concerns live here: + +* **Slot mapping** — absolute token positions → physical KV-cache slots, the + standard PagedAttention layout ``slot(pos) = block_id(pos) * block_size + + (pos % block_size)``. +* **Pool I/O** — allocate per-layer FlashAttention-compatible paged K/V pools + plus flat compatibility views for direct slot writes. +* **Chunk-window eviction** — a ``SlidingWindowSpec`` subclass whose unit is a + *chunk* (``sliding_window = window_chunks * chunk_size``) plus a manager that + evicts at chunk boundaries. Memory policy / refcounting / ``null_block`` + replacement stay in vLLM's ``BlockPool``; only the token-skip math is here. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import torch +from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager +from vllm.v1.kv_cache_interface import SlidingWindowSpec + +try: + from vllm.v1.kv_cache_spec_registry import register_kv_cache_spec +except ModuleNotFoundError: + + def register_kv_cache_spec(*, manager_class, uniform_type_base_spec=None): + """Compatibility shim for vLLM versions without kv_cache_spec_registry.""" + + def decorator(spec_class): + from vllm.v1.core.single_type_kv_cache_manager import spec_manager_map + + spec_manager_map[spec_class] = manager_class + return spec_class + + return decorator + + +# ── Slot mapping ──────────────────────────────────────────────────────────── + + +def compute_slot_mapping( + block_ids: Sequence[int], + positions: torch.Tensor | Sequence[int], + block_size: int, +) -> torch.Tensor: + """Map absolute token positions to physical KV-cache slots. + + Args: + block_ids: physical block id per block index (the request's block table). + positions: absolute token positions to map (1-D). + block_size: tokens per block. + + Returns: + ``LongTensor`` of physical slots, one per position. + """ + if block_size <= 0: + raise ValueError(f"block_size must be positive, got {block_size}") + table = torch.as_tensor(block_ids, dtype=torch.long) + pos = torch.as_tensor(positions, dtype=torch.long) + block_index = torch.div(pos, block_size, rounding_mode="floor") + offset = pos % block_size + return table[block_index] * block_size + offset + + +def chunk_slot_mapping( + block_ids: Sequence[int], + num_computed_tokens: int, + chunk_size: int, + block_size: int, +) -> torch.Tensor: + """Slot mapping for the in-flight chunk's tokens (the commit write target). + + The chunk occupies absolute positions + ``[num_computed_tokens, num_computed_tokens + chunk_size)``. + """ + positions = torch.arange( + num_computed_tokens, + num_computed_tokens + chunk_size, + dtype=torch.long, + ) + return compute_slot_mapping(block_ids, positions, block_size) + + +def resident_block_ids(block_ids: Sequence[int], null_block_id: int) -> list[int]: + """Real (non-null) blocks currently resident, in table order. + + These are the blocks the read path gathers the attention window from; + out-of-window positions are the shared ``null_block`` and are excluded. + """ + return [int(b) for b in block_ids if int(b) != null_block_id] + + +# ── Pool I/O (allocate / write) ───────────────────────────────────────────── + + +def allocate_kv_pool_with_views( + num_blocks: int, + block_size: int, + num_layers: int, + num_kv_heads: int, + head_dim: int, + dtype: torch.dtype, + device: torch.device, +) -> tuple[list[torch.Tensor], list[torch.Tensor], list[torch.Tensor]]: + """Allocate vLLM-style paged KV pools plus flat slot-write views. + + The owning tensor follows FlashAttention's block-table cache layout: + ``(2, num_blocks, block_size, num_kv_heads, head_dim)``, where dim-0 is + ``[K, V]``. The flat K/V views keep ``slot = block_id * block_size + + offset`` writes simple without changing the kernel-facing cache layout. + """ + kv_pools: list[torch.Tensor] = [] + k_pools: list[torch.Tensor] = [] + v_pools: list[torch.Tensor] = [] + for _ in range(num_layers): + kv = torch.empty(2, num_blocks, block_size, num_kv_heads, head_dim, dtype=dtype, device=device) + kv_pools.append(kv) + k_pools.append(kv[0].reshape(num_blocks * block_size, num_kv_heads, head_dim)) + v_pools.append(kv[1].reshape(num_blocks * block_size, num_kv_heads, head_dim)) + return kv_pools, k_pools, v_pools + + +def pool_write_chunk( + k_pool: torch.Tensor, + v_pool: torch.Tensor, + new_k: torch.Tensor, + new_v: torch.Tensor, + slot_mapping: torch.Tensor, +) -> None: + """Write the committed chunk's K/V into pool slots (in place). + + ``new_k`` / ``new_v`` shape: ``(batch, chunk_size, num_kv_heads, head_dim)``. + ``slot_mapping`` shape: ``(chunk_size,)`` — one physical slot per token position. + The batch dim is written to the same slots (multi-batch writes the same + positions; the caller is responsible for per-sequence bookkeeping). + + This is the *commit* write — called once per chunk, after the last denoise step. + """ + k_pool[slot_mapping] = new_k[0] # batch index 0 + v_pool[slot_mapping] = new_v[0] + + +def _drop_batch(t: torch.Tensor) -> torch.Tensor: + """``(1, L, n_heads, head_dim)`` -> ``(L, n_heads, head_dim)``; pass 3-D through.""" + if t.dim() == 4 and t.shape[0] == 1: + return t[0] + return t + + +# ── Chunk-window eviction (spec + manager) ────────────────────────────────── + + +def chunk_window_skipped_tokens( + num_computed_tokens: int, + *, + chunk_size: int, + sliding_window: int, + sink_chunks: int, + reset_at_boundary: bool, +) -> int: + """Tokens outside the resident chunk window, snapped to a chunk boundary. + + Pure function so the eviction policy is unit-testable without constructing a + manager. Two strategies: + + - ``reset_at_boundary`` (DreamZero): at each chunk boundary everything past + the sink is dropped. + - otherwise (VGGT-style sliding replace): keep the last ``window`` tokens + (plus the sink); the skip count snaps down to a chunk boundary so a chunk + is never half-evicted. + """ + sink = sink_chunks * chunk_size + if reset_at_boundary: + completed = (num_computed_tokens // chunk_size) * chunk_size + return max(0, completed - sink) + skipped = max(0, num_computed_tokens - sliding_window - sink) + return (skipped // chunk_size) * chunk_size + + +class ChunkWindowManager(SlidingWindowManager): + """``SlidingWindowManager`` that evicts at chunk boundaries. + + ``self.sliding_window`` is set by the base ``__init__``; the chunk fields are + read from ``self.kv_cache_spec`` (a :class:`ChunkWindowSpec`). + """ + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + spec = self.kv_cache_spec + return chunk_window_skipped_tokens( + num_computed_tokens, + chunk_size=spec.chunk_size, + sliding_window=self.sliding_window, + sink_chunks=spec.sink_chunks, + reset_at_boundary=spec.reset_at_boundary, + ) + + +# Register so KVCacheManager resolves ChunkWindowSpec to ChunkWindowManager. +# Dispatch walks the spec's MRO, so without explicit registration the subclass +# would silently fall back to the parent SlidingWindowManager (override ignored). +# uniform_type_base_spec=None => its own KV cache group. +@register_kv_cache_spec(manager_class=ChunkWindowManager, uniform_type_base_spec=None) +@dataclass(frozen=True, kw_only=True) +class ChunkWindowSpec(SlidingWindowSpec): + # sliding_window (inherited) MUST equal window_chunks * chunk_size. + chunk_size: int + window_chunks: int + sink_chunks: int = 0 + reset_at_boundary: bool = False + + def __post_init__(self): + super().__post_init__() + if self.sliding_window != self.window_chunks * self.chunk_size: + raise ValueError( + "ChunkWindowSpec.sliding_window must equal " + f"window_chunks * chunk_size ({self.window_chunks} * " + f"{self.chunk_size}), got {self.sliding_window}" + ) diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py new file mode 100644 index 00000000000..2038e496d31 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py @@ -0,0 +1,550 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Paged self-attention helpers for AR-Diffusion DreamZero KV reuse.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, ClassVar, NamedTuple + +import torch + +from vllm_omni.experimental.ar_diffusion.kv_cache.paged import compute_slot_mapping + +# Set by ARDiffusionPagedForwardContext.prepare() before each branch forward and +# read by the fused write+attend custom op below. The pools are process-lifetime +# allocations owned by one ARDiffusionKVCache per worker process (CFG-parallel / +# TP ranks are separate processes), mirroring vLLM's forward-context pattern for +# keeping multi-GiB cache tensors out of the compiled graph's inputs. +_CURRENT_PAGED_KV_CACHE: Any = None + + +def set_current_paged_kv_cache(kv_cache: Any) -> None: + global _CURRENT_PAGED_KV_CACHE + _CURRENT_PAGED_KV_CACHE = kv_cache + + +_LAYER_IDX_TENSORS: dict[int, torch.Tensor] = {} + + +def _layer_idx_tensor(layer_idx: int) -> torch.Tensor: + t = _LAYER_IDX_TENSORS.get(layer_idx) + if t is None: + t = torch.tensor(layer_idx, dtype=torch.int64) + _LAYER_IDX_TENSORS[layer_idx] = t + return t + + +class ARDiffusionPagedLayerInputs(NamedTuple): + """Compiled-region payload for one layer's paged self-attention. + + A NamedTuple of plain tensors + ints so ``torch.compile`` treats every field + as a pytree graph input (no object-attribute guards, no recompiles when only + tensor *values* change). All layers of one branch forward share the same + metadata tensor objects, built once by ``prepare()``. + + ``layer_idx`` is a 0-dim CPU tensor, NOT a python int: all 40 DiT blocks + share one compiled code object, and an int here becomes a per-layer dynamo + value guard (``layer_idx == k``) — 40 cache variants that blow the + recompile limit. A tensor input guards on shape/dtype only, so one graph + serves every layer. + """ + + layer_idx: torch.Tensor + seq_len: int + video_slots: torch.Tensor + action_slots: torch.Tensor + block_table: torch.Tensor + query_start_loc: torch.Tensor + seq_lens: torch.Tensor + max_query_len: int + max_seq_len: int + + +@dataclass +class ARDiffusionPagedForwardContext: + """Mutable branch-level state shared by all layer contexts in one forward.""" + + kv_cache: Any + adapter: Any + is_negative: bool + history_block_ids: list[int] + seq_len: int + commit_current: bool + max_video_tokens: int + current_video_block_ids: list[int] = field(default_factory=list) + current_video_slot_mapping: torch.Tensor | None = None + action_scratch_block_ids: list[int] = field(default_factory=list) + action_slot_mapping: torch.Tensor | None = None + query_len: int = 0 + kv_len: int = 0 + _allocated_video: bool = False + _committed: bool = False + _action_len: int = 0 + # Set once by prepare(); shared by all layers of the branch forward. + block_table: torch.Tensor | None = None + query_start_loc: torch.Tensor | None = None + seq_lens: torch.Tensor | None = None + max_query_len: int = 0 + max_seq_len: int = 0 + _prepared: bool = False + + @property + def block_size(self) -> int: + return int(self.kv_cache.block_size) + + @property + def num_current_video_blocks(self) -> int: + if self.seq_len % self.block_size != 0: + raise AssertionError( + "AR-Diffusion paged attention expects frame-aligned seq_len " + f"(multiple of block_size={self.block_size}), got {self.seq_len}" + ) + return self.seq_len // self.block_size + + def ensure_video_slots(self, device: torch.device) -> None: + """Allocate/write targets for the current video tokens, once per branch.""" + if self._allocated_video: + return + + n_blocks = self.num_current_video_blocks + if self.commit_current: + start = int(self.adapter.num_computed_tokens) + self.kv_cache.allocate_token_slots(self.adapter, self.seq_len) + table = self.kv_cache.block_table(self.adapter) + start_block = start // self.block_size + self.current_video_block_ids = [int(b) for b in table[start_block : start_block + n_blocks]] + positions = torch.arange(start, start + self.seq_len, dtype=torch.long) + self.current_video_slot_mapping = compute_slot_mapping(table, positions, self.block_size).to(device=device) + else: + self.current_video_block_ids = self.kv_cache.scratch_block_ids(self.is_negative, 0, n_blocks) + positions = torch.arange(self.seq_len, dtype=torch.long) + self.current_video_slot_mapping = compute_slot_mapping( + self.current_video_block_ids, + positions, + self.block_size, + ).to(device=device) + self._allocated_video = True + + def ensure_action_slots(self, action_len: int, device: torch.device) -> None: + """Reserve scratch slots for action/state K/V, if present.""" + if action_len <= 0: + self.action_scratch_block_ids = [] + self.action_slot_mapping = torch.empty(0, dtype=torch.long, device=device) + self._action_len = 0 + return + + self.ensure_video_slots(device) + if self.action_slot_mapping is not None and self._action_len == action_len: + return + + action_blocks = (action_len + self.block_size - 1) // self.block_size + scratch_offset = 0 if self.commit_current else len(self.current_video_block_ids) + self.action_scratch_block_ids = self.kv_cache.scratch_block_ids( + self.is_negative, + scratch_offset, + action_blocks, + ) + positions = torch.arange(action_len, dtype=torch.long) + self.action_slot_mapping = compute_slot_mapping( + self.action_scratch_block_ids, + positions, + self.block_size, + ).to(device=device) + self._action_len = action_len + + def video_block_table(self, device: torch.device) -> tuple[list[int], int]: + self.ensure_video_slots(device) + if self.max_video_tokens % self.block_size != 0: + raise AssertionError( + "AR-Diffusion paged attention requires max_video_tokens to be block-aligned, " + f"got max_video_tokens={self.max_video_tokens}, block_size={self.block_size}" + ) + all_video_blocks = self.history_block_ids + self.current_video_block_ids + max_video_blocks = self.max_video_tokens // self.block_size + visible_video_blocks = all_video_blocks[-max_video_blocks:] if max_video_blocks else [] + video_len = min(len(all_video_blocks) * self.block_size, self.max_video_tokens) + return visible_video_blocks, video_len + + def build_block_table( + self, + *, + action_len: int, + query_len: int, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]: + """Build FlashAttention block-table metadata for one self-attn call. + + The block table is tail-padded to a fixed width and ``max_seq_len`` is a + constant upper bound, so across window growth only tensor *values* change + — tensor shapes and the int consts stay stable for ``torch.compile`` + (and, later, CUDA-graph capture). The kernel only dereferences the first + ``ceil(seq_lens/block_size)`` entries, so padding is never read. + """ + video_blocks, video_len = self.video_block_table(device) + self.ensure_action_slots(action_len, device) + action_blocks = self.action_scratch_block_ids if action_len > 0 else [] + block_ids = video_blocks + action_blocks + if not block_ids: + raise RuntimeError("AR-Diffusion paged attention needs at least current video KV blocks") + + # Fixed capacity: full visible video window + one action-capacity block. + action_capacity_blocks = max(1, (action_len + self.block_size - 1) // self.block_size) + width = max(self.max_video_tokens // self.block_size + action_capacity_blocks, len(block_ids)) + padded = block_ids + [0] * (width - len(block_ids)) + + self.query_len = int(query_len) + self.kv_len = int(video_len + action_len) + max_seq_len = int(self.max_video_tokens + action_capacity_blocks * self.block_size) + block_table = torch.tensor([padded], dtype=torch.int32, device=device) + query_start_loc = torch.tensor([0, self.query_len], dtype=torch.int32, device=device) + seq_lens = torch.tensor([self.kv_len], dtype=torch.int32, device=device) + return block_table, query_start_loc, seq_lens, self.query_len, max_seq_len + + def prepare(self, device: torch.device, action_len: int, query_len: int) -> None: + """Host-side, once-per-branch-forward setup (called OUTSIDE torch.compile). + + Allocates the current video/action slots (still lazy: only the branch a + CFG-parallel rank actually runs reaches its ``_forward_blocks``), builds + the padded block-table metadata ONCE for all layers, and publishes the + pool registry for the fused custom op. The compiled per-layer code then + only consumes prebuilt tensors via ``ARDiffusionPagedLayerInputs``. + """ + if getattr(self, "_prepared", False): + return + self.ensure_video_slots(device) + ( + self.block_table, + self.query_start_loc, + self.seq_lens, + self.max_query_len, + self.max_seq_len, + ) = self.build_block_table(action_len=action_len, query_len=query_len, device=device) + if self.action_slot_mapping is None: + self.action_slot_mapping = torch.empty(0, dtype=torch.long, device=device) + set_current_paged_kv_cache(self.kv_cache) + self._prepared = True + + def layer_inputs(self, layer_idx: int) -> ARDiffusionPagedLayerInputs: + if not getattr(self, "_prepared", False): + raise RuntimeError("ARDiffusionPagedForwardContext.layer_inputs() before prepare()") + return ARDiffusionPagedLayerInputs( + layer_idx=_layer_idx_tensor(layer_idx), + seq_len=int(self.seq_len), + video_slots=self.current_video_slot_mapping, + action_slots=self.action_slot_mapping, + block_table=self.block_table, + query_start_loc=self.query_start_loc, + seq_lens=self.seq_lens, + max_query_len=int(self.max_query_len), + max_seq_len=int(self.max_seq_len), + ) + + def mark_committed(self) -> None: + self._committed = True + + +@dataclass +class ARDiffusionPagedLayerContext: + """Layer-specific handle passed through DreamZero's existing ``kv_cache`` slot.""" + + is_ar_diffusion_paged_context: ClassVar[bool] = True + layer_idx: int + forward_ctx: ARDiffusionPagedForwardContext + + @property + def kv_cache(self): + return self.forward_ctx.kv_cache + + @property + def adapter(self): + return self.forward_ctx.adapter + + @property + def is_negative(self) -> bool: + return self.forward_ctx.is_negative + + @property + def history_block_ids(self) -> list[int]: + return self.forward_ctx.history_block_ids + + @property + def current_video_block_ids(self) -> list[int]: + return self.forward_ctx.current_video_block_ids + + @property + def current_video_slot_mapping(self) -> torch.Tensor | None: + return self.forward_ctx.current_video_slot_mapping + + @property + def action_scratch_block_ids(self) -> list[int]: + return self.forward_ctx.action_scratch_block_ids + + @property + def action_slot_mapping(self) -> torch.Tensor | None: + return self.forward_ctx.action_slot_mapping + + @property + def seq_len(self) -> int: + return self.forward_ctx.seq_len + + @property + def query_len(self) -> int: + return self.forward_ctx.query_len + + @property + def kv_len(self) -> int: + return self.forward_ctx.kv_len + + @property + def commit_current(self) -> bool: + return self.forward_ctx.commit_current + + def to_layer_inputs(self) -> ARDiffusionPagedLayerInputs: + """Compiled-region payload; requires ``forward_ctx.prepare()`` first.""" + return self.forward_ctx.layer_inputs(self.layer_idx) + + +def is_ar_diffusion_paged_context(value: object) -> bool: + return isinstance(value, ARDiffusionPagedLayerContext) + + +def _reference_paged_attention( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + softmax_scale: float, + *, + causal: bool, +) -> torch.Tensor: + if causal: + raise NotImplementedError("AR-Diffusion paged self-attention uses causal=False") + outs: list[torch.Tensor] = [] + block_size = key_cache.shape[1] + for i in range(seq_lens.shape[0]): + q_start = int(query_start_loc[i].item()) + q_end = int(query_start_loc[i + 1].item()) + kv_len = int(seq_lens[i].item()) + q = query[q_start:q_end] + positions = torch.arange(kv_len, device=query.device) + logical_blocks = torch.div(positions, block_size, rounding_mode="floor") + offsets = positions % block_size + physical_blocks = block_table[i, logical_blocks].long() + k = key_cache[physical_blocks, offsets] + v = value_cache[physical_blocks, offsets] + scores = torch.einsum("qhd,khd->hqk", q.float(), k.float()) * float(softmax_scale) + probs = torch.softmax(scores, dim=-1).to(v.dtype) + outs.append(torch.einsum("hqk,khd->qhd", probs, v)) + return torch.cat(outs, dim=0) + + +_FA_VERSION_BY_HEAD_SIZE: dict[int, int] = {} + + +def _resolve_fa_version(head_size: int) -> int: + # get_flash_attn_version -> current_platform.get_device_capability() is not + # dynamo-traceable, and the answer is fixed per head size for the process. + version = _FA_VERSION_BY_HEAD_SIZE.get(head_size) + if version is None: + try: + from vllm.v1.attention.backends.fa_utils import get_flash_attn_version + + version = int(get_flash_attn_version(requires_alibi=False, head_size=head_size) or 2) + except Exception: + version = 2 + _FA_VERSION_BY_HEAD_SIZE[head_size] = version + return version + + +def ar_diffusion_paged_attention( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + *, + block_table: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + max_query_len: int, + max_seq_len: int, + softmax_scale: float, + causal: bool = False, +) -> torch.Tensor: + """Run non-causal paged attention over a vLLM block table. + + ``query`` may be ``(B, L, H, D)`` or already flattened as ``(T, H, D)``. + ``key_cache`` / ``value_cache`` are ``(num_blocks, block_size, H, D)``. + """ + batched = query.dim() == 4 + if batched: + batch, q_len = query.shape[:2] + query_flat = query.reshape(batch * q_len, *query.shape[2:]) + else: + query_flat = query + + if not query_flat.is_cuda: + out = _reference_paged_attention( + query_flat, + key_cache, + value_cache, + block_table, + query_start_loc, + seq_lens, + softmax_scale, + causal=causal, + ) + else: + from vllm.vllm_flash_attn import flash_attn_varlen_func + + fa_version = _resolve_fa_version(query_flat.shape[-1]) + + out = torch.empty_like(query_flat) + flash_attn_varlen_func( + q=query_flat, + k=key_cache, + v=value_cache, + out=out, + cu_seqlens_q=query_start_loc, + max_seqlen_q=int(max_query_len), + seqused_k=seq_lens, + max_seqlen_k=int(max_seq_len), + softmax_scale=float(softmax_scale), + causal=causal, + block_table=block_table, + fa_version=fa_version, + ) + + if batched: + return out.reshape(query.shape) + return out + + +# ── Fused write+attend custom op (torch.compile-safe) ────────────────────── +# +# One opaque op per layer keeps the compiled DiT block fullgraph: dynamo treats +# it as a single graph node (no eager island, no graph breaks), and the K/V slot +# writes happen inside the op so write→read ordering with the block-table kernel +# is internal — no hidden-mutation ordering hazards against a separate reader op +# and no multi-GiB pool tensors as graph inputs (the flat and paged pool views +# alias the same storage, which AOTAutograd handles poorly as inputs). Pattern +# follows sage_attn3.py in-repo and vLLM's own unified attention ops. +def _paged_write_attn_impl( + query: torch.Tensor, + k_curr: torch.Tensor, + v_curr: torch.Tensor, + k_act: torch.Tensor | None, + v_act: torch.Tensor | None, + layer_idx: torch.Tensor, + video_slots: torch.Tensor, + action_slots: torch.Tensor, + block_table: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + max_query_len: int, + max_seq_len: int, + softmax_scale: float, +) -> torch.Tensor: + kv = _CURRENT_PAGED_KV_CACHE + if kv is None: + raise RuntimeError("ar_diffusion_paged_write_attn called before prepare() set the KV pool registry") + layer_idx = int(layer_idx) + k_pool = kv._k_pools[layer_idx] + v_pool = kv._v_pools[layer_idx] + k_pool[video_slots] = k_curr.to(k_pool.dtype) + v_pool[video_slots] = v_curr.to(v_pool.dtype) + if k_act is not None and v_act is not None and k_act.shape[0] > 0: + k_pool[action_slots] = k_act.to(k_pool.dtype) + v_pool[action_slots] = v_act.to(v_pool.dtype) + return ar_diffusion_paged_attention( + query, + kv.key_cache(layer_idx), + kv.value_cache(layer_idx), + block_table=block_table, + query_start_loc=query_start_loc, + seq_lens=seq_lens, + max_query_len=max_query_len, + max_seq_len=max_seq_len, + softmax_scale=softmax_scale, + causal=False, + ) + + +# hasattr guard keeps registration idempotent across test re-imports that pop +# the module from sys.modules (same as sage_attn3.py). +if not hasattr(torch.ops.vllm_omni, "ar_diffusion_paged_write_attn"): + + @torch.library.custom_op("vllm_omni::ar_diffusion_paged_write_attn", mutates_args=()) + def _paged_write_attn_op( + query: torch.Tensor, + k_curr: torch.Tensor, + v_curr: torch.Tensor, + k_act: torch.Tensor | None, + v_act: torch.Tensor | None, + layer_idx: torch.Tensor, + video_slots: torch.Tensor, + action_slots: torch.Tensor, + block_table: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + max_query_len: int, + max_seq_len: int, + softmax_scale: float, + ) -> torch.Tensor: + return _paged_write_attn_impl( + query, + k_curr, + v_curr, + k_act, + v_act, + layer_idx, + video_slots, + action_slots, + block_table, + query_start_loc, + seq_lens, + max_query_len, + max_seq_len, + softmax_scale, + ) + + @_paged_write_attn_op.register_fake + def _( + query, + k_curr, + v_curr, + k_act, + v_act, + layer_idx, + video_slots, + action_slots, + block_table, + query_start_loc, + seq_lens, + max_query_len, + max_seq_len, + softmax_scale, + ): + return torch.empty_like(query) + + +def paged_write_attn( + inputs: ARDiffusionPagedLayerInputs, query, k_curr, v_curr, k_act, v_act, softmax_scale: float +) -> torch.Tensor: + """Model-facing entry: routes through the custom op (traceable in fullgraph).""" + return torch.ops.vllm_omni.ar_diffusion_paged_write_attn( + query, + k_curr, + v_curr, + k_act, + v_act, + inputs.layer_idx, + inputs.video_slots, + inputs.action_slots, + inputs.block_table, + inputs.query_start_loc, + inputs.seq_lens, + inputs.max_query_len, + inputs.max_seq_len, + softmax_scale, + ) diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py new file mode 100644 index 00000000000..510bbe043a0 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +"""DreamZero-facing AR-Diffusion KV bridge backed by paged self-attention.""" + +from __future__ import annotations + +from vllm.logger import init_logger + +from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( + ARDiffusionPagedForwardContext, + ARDiffusionPagedLayerContext, +) + +_log = init_logger(__name__) + + +class ARDiffusionKVState: + """Per-request bridge between a DreamZero rollout and the AR-Diffusion KV pool. + + The self-attention path is paged-only: ``get_kv_caches`` returns lightweight + per-layer contexts, and each attention layer writes current K/V into the + pool before calling the FlashAttention block-table kernel. No resident + window is gathered into a contiguous tensor. + """ + + def __init__(self, kv_cache, pos_adapter, neg_adapter, num_layers: int) -> None: + """One state per DreamZero session, holding BOTH CFG branches' adapters. + + ``pos``/``neg`` are the classifier-free-guidance branches: ``pos`` is + the conditional pass (real prompt), ``neg`` the unconditional pass + (negative prompt). Each owns an independent request adapter — and + therefore independent pool blocks — because the two branches produce + different K/V for the same frames. Every ``is_negative`` flag in this + module selects between them (``False`` -> ``pos``, ``True`` -> ``neg``). + """ + self.kv_cache = kv_cache + self.pos = pos_adapter + self.neg = neg_adapter + self.num_layers = num_layers + self._committed: dict[bool, int] = {False: 0, True: 0} + self._paged_pending: dict[bool, ARDiffusionPagedForwardContext | None] = {False: None, True: None} + # Cross-attn KV is populated once after text encoding (eager), then + # read every denoising step. Tracked per half: the text K/V depend only + # on the session prompt (survive window-boundary resets), while the I2V + # image-token K/V come from the current observation's CLIP features and + # must be re-projected on every window restart. + self._cross_text_populated: dict[bool, bool] = {False: False, True: False} + self._cross_img_populated: dict[bool, bool] = {False: False, True: False} + + def _adapter(self, is_negative: bool): + return self.neg if is_negative else self.pos + + def get_kv_caches( + self, + is_negative: bool, + seq_len: int | None = None, + commit_current: bool = False, + ) -> list[ARDiffusionPagedLayerContext]: + if seq_len is None: + raise ValueError("AR-Diffusion paged self-attention requires seq_len in get_kv_caches()") + return self.prepare_paged_context(is_negative, seq_len, commit_current) + + def prepare_paged_context( + self, + is_negative: bool, + seq_len: int, + commit_current: bool, + ) -> list[ARDiffusionPagedLayerContext]: + """Return per-layer paged attention contexts for one branch forward. + + Allocation is lazy: constructing both CFG branches' kwargs must not + allocate manager blocks for a branch that a CFG-parallel rank will not run. + The first self-attn layer that consumes the context allocates the current + video slots, then all layers share that branch-level state. + """ + cs = self.kv_cache.spec.chunk_size + if int(seq_len) % cs != 0: + raise AssertionError( + f"AR-Diffusion expects frame-aligned seq_len (multiple of chunk_size={cs}), got {seq_len}" + ) + + pending = self._paged_pending.get(is_negative) + if pending is not None and pending.commit_current and pending._allocated_video and not pending._committed: + raise RuntimeError("AR-Diffusion paged context replaced before its managed current chunk was committed") + + adapter = self._adapter(is_negative) + branch = "neg" if is_negative else "pos" + forward_ctx = ARDiffusionPagedForwardContext( + kv_cache=self.kv_cache, + adapter=adapter, + is_negative=is_negative, + history_block_ids=self.kv_cache.window_block_ids(adapter), + seq_len=int(seq_len), + commit_current=bool(commit_current), + max_video_tokens=int(self.kv_cache.spec.sliding_window), + ) + self._paged_pending[is_negative] = forward_ctx + _log.debug( + "AR-Diffusion GET [%s] source=paged-attn layers=%d history_blocks=%d seq_len=%d commit_current=%s", + branch, + self.num_layers, + len(forward_ctx.history_block_ids), + int(seq_len), + bool(commit_current), + ) + return [ARDiffusionPagedLayerContext(layer_idx=i, forward_ctx=forward_ctx) for i in range(self.num_layers)] + + def commit_paged_context(self, is_negative: bool) -> None: + """Commit the managed current video blocks after a successful forward.""" + ctx = self._paged_pending.get(is_negative) + if ctx is None: + return + branch = "neg" if is_negative else "pos" + if ctx.commit_current and ctx._allocated_video: + n_chunks = ctx.seq_len // self.kv_cache.spec.chunk_size + for _ in range(n_chunks): + ctx.adapter.on_chunk_committed() + self._committed[is_negative] += ctx.seq_len + _log.debug( + "AR-Diffusion COMMIT [%s] paged-attn new_tokens=%d chunks=%d resident=%d/%d", + branch, + ctx.seq_len, + n_chunks, + len(self.kv_cache.window_block_ids(ctx.adapter)), + self.kv_cache.spec.window_chunks, + ) + ctx.mark_committed() + self._paged_pending[is_negative] = None + + def get_cross_kv_caches(self, is_negative: bool) -> list[dict]: + """Return pool-backed cross-attn cache dicts. + + Under AR-Diffusion the cross-attn pool is always populated by ``_kv_populate_cross`` + before the first read, so this must never be reached unpopulated — if it + were, the model would lazily project and write cross KV itself, violating + engine ownership. Fail loud rather than fall back to the model. + """ + img_ok = not self.kv_cache._cross_k_img or self._cross_img_populated[is_negative] + if not (self._cross_text_populated[is_negative] and img_ok and self.kv_cache.cross_attn_length > 0): + raise RuntimeError( + f"AR-Diffusion cross-attn read before _kv_populate_cross (neg={is_negative}, " + f"cross_attn_length={self.kv_cache.cross_attn_length}, " + f"text={self._cross_text_populated[is_negative]} " + f"img={self._cross_img_populated[is_negative]}) — the engine must own all cross KV" + ) + return [self.kv_cache.read_cross_kv(i, is_negative) for i in range(self.num_layers)] + + def close(self) -> None: + """Final teardown: free both branches' resident pool blocks.""" + self.kv_cache.end_request(self.pos) + self.kv_cache.end_request(self.neg) + self._paged_pending = {False: None, True: None} + + def reset(self, *, keep_cross_text: bool = False) -> None: + """Drop this session's KV — mirrors the model-local ``state.reset()``. + + DreamZero resets at the attention-window boundary (``should_reset``); the + model starts a fresh sliding window, so AR-Diffusion must free the resident pool + blocks and start a fresh adapter for each branch. The ``ARDiffusionKVState`` + object (and thus ``pipeline._ar_diffusion_kv_state``) is preserved across the reset + so the runner's session mapping stays valid — only the pool-backed state + is recycled. + + Args: + keep_cross_text: On window ("inference") resets the session prompt is + unchanged, so the text cross-attn K/V in the static pool buffers are + still valid — keep them and only force the image half to repopulate. + Session resets (prompt change / explicit reset) pass ``False``. + """ + pos_id, neg_id = self.pos.request_id, self.neg.request_id + self.close() + self.pos = self.kv_cache.begin_request(pos_id) + self.neg = self.kv_cache.begin_request(neg_id) + self._committed = {False: 0, True: 0} + self._paged_pending = {False: None, True: None} + if not keep_cross_text: + self._cross_text_populated = {False: False, True: False} + self._cross_img_populated = {False: False, True: False} + _log.info( + "AR-Diffusion RESET [%s/%s] session KV cleared (window boundary; cross text %s)", + pos_id, + neg_id, + "kept" if keep_cross_text else "cleared", + ) diff --git a/vllm_omni/experimental/ar_diffusion/runner.py b/vllm_omni/experimental/ar_diffusion/runner.py new file mode 100644 index 00000000000..c8b8e93e1d8 --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/runner.py @@ -0,0 +1,409 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ARDiffusionModelRunner — the diffusion model runner for the AR-Diffusion engine. + +Subclasses ``DiffusionModelRunner`` and owns the engine-level KV cache +(:class:`ARDiffusionKVCache`). It brackets a request's rollout with the KV lifecycle +(``begin_request`` / ``end_request``) and exposes the live ``ARDiffusionKVCache`` so the +model pipeline (DreamZero) can drive the per-chunk allocate / slot-mapping / +gather / commit operations during ``pipeline.forward``. + +Selecting the AR-Diffusion engine enables KV management (no env gate); optional KV +parameters come from the deploy yaml's ``model_config["ar_diffusion_kv_config"]``. +""" + +from __future__ import annotations + +import dataclasses +import math +import time +from collections import OrderedDict + +import numpy as np +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import MAX_DREAMZERO_SESSIONS +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig +from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache +from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + +logger = init_logger(__name__) + + +def resolve_ar_diffusion_kv_config(od_config: OmniDiffusionConfig) -> ARDiffusionKVConfig: + """Resolve the AR-Diffusion KV config from the deploy/command arg. + + Selecting the AR-Diffusion engine implies KV management, so the result is always + enabled. Optional overrides (``window_chunks`` / ``gpu_memory_fraction`` / + ``sink_chunks`` / ``reset_at_boundary``) may be supplied via the deploy yaml as + ``model_config["ar_diffusion_kv_config"]``. ``chunk_size`` / ``window_chunks`` are + finalized from the model geometry at load (see ``_preallocate_kv_cache``). + """ + raw = getattr(od_config, "ar_diffusion_kv_config", None) + if raw is None: + model_config = getattr(od_config, "model_config", None) + if isinstance(model_config, dict): + raw = model_config.get("ar_diffusion_kv_config") + if isinstance(raw, ARDiffusionKVConfig): + return dataclasses.replace(raw, enable=True) + if isinstance(raw, dict): + return ARDiffusionKVConfig(**{**raw, "enable": True}) + return ARDiffusionKVConfig(enable=True) + + +class ARDiffusionModelRunner(DiffusionModelRunner): + def __init__(self, vllm_config, od_config: OmniDiffusionConfig, device) -> None: + super().__init__(vllm_config, od_config, device) + self.ar_diffusion_kv_config = resolve_ar_diffusion_kv_config(od_config) + # Built after the model is loaded (dimensions known); stays None while KV + # management is disabled. + self.kv_cache: ARDiffusionKVCache | None = None + # DreamZero KV is session-scoped (persists across a session's forwards), + # so AR-Diffusion KV state is keyed by session_id and reused, not created per request. + # Bounded by the same LRU cap as DreamZeroPipeline._states: an evicted + # session's ARDiffusionKVState owns pool blocks (two CFG adapters), so without a + # bound, session-id churn in a long-running server would leak pool ownership + # until the KV pool is exhausted. Evicting frees those blocks (state.close()). + self._ar_diffusion_states: OrderedDict[str, ARDiffusionKVState] = OrderedDict() + self._max_ar_diffusion_states = MAX_DREAMZERO_SESSIONS + # Per-request server-side forward E2E times (seconds), recorded in + # execute_model. This is the worker-side compute time — the analog of the + # upstream server's per-request E2E — and excludes the engine<->worker IPC + # that the client's omni.generate wall time includes. Read (and cleared) + # by the ar_diffusion_perf_stats worker RPC for the perf-compare summary. + self._perf_e2e_times: list[float] = [] + + def build_kv_cache( + self, + *, + num_layers: int, + num_kv_heads: int, + head_size: int, + dtype, + block_size: int, + max_model_len: int, + available_bytes: int, + cross_attn_length: int = 0, + cross_attn_img_length: int = 0, + local_branches: int = 2, + num_frame_per_block: int = 1, + ) -> None: + """Construct the ARDiffusionKVCache (preallocating GPU pools on ``self.device``). + + No-op when KV management is disabled. + """ + if not self.ar_diffusion_kv_config.enable: + return + self.kv_cache = ARDiffusionKVCache( + self.ar_diffusion_kv_config, + num_layers=num_layers, + num_kv_heads=num_kv_heads, + head_size=head_size, + dtype=dtype, + block_size=block_size, + max_model_len=max_model_len, + available_bytes=available_bytes, + cross_attn_length=cross_attn_length, + cross_attn_img_length=cross_attn_img_length, + local_branches=local_branches, + num_frame_per_block=num_frame_per_block, + device=self.device, + ) + logger.info( + "AR-Diffusion KV cache enabled: %d blocks, chunk_size=%d, window_chunks=%s, " + "layers=%d kv_heads=%d head_dim=%d block_size=%d cross_attn_len=%d device=%s", + self.kv_cache.num_blocks, + self.ar_diffusion_kv_config.chunk_size, + self.ar_diffusion_kv_config.window_chunks, + num_layers, + num_kv_heads, + head_size, + block_size, + cross_attn_length, + self.device, + ) + + # -- preallocation at load ------------------------------------------------- + + def load_model(self, *args, **kwargs): + super().load_model(*args, **kwargs) + if self.ar_diffusion_kv_config.enable and self.pipeline is not None: + self._preallocate_kv_cache() + # Pre-capture the CUDA graphs for every window-fill shape at load time + # (only when compiling/cuda-graph is on) so serving is fast from chunk 0. + if not self.od_config.enforce_eager and self.ar_diffusion_kv_config.warmup_cudagraph: + self._warmup_ar_rollout() + + def _infer_frame_seqlen(self) -> int: + """frame_seqlen = (H//8)*(W//8)//4 from the configured image_resolution.""" + mc = getattr(self.od_config, "model_config", None) or {} + psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} + res = psc.get("image_resolution", [180, 320]) + h, w = int(res[0]), int(res[1]) + return (h // 8) * (w // 8) // 4 + + def _preallocate_kv_cache(self) -> None: + """Build the KV pool once, from the loaded DreamZero transformer geometry.""" + # The paged self-attention path carries one rollout per request: slot + # mappings/adapters are single-sequence and K/V writes address batch + # index 0 only. Reject a multi-sequence config at init instead of + # failing (or silently dropping sequences) mid-forward. + max_num_seqs = int(getattr(self.od_config, "max_num_seqs", 1) or 1) + if max_num_seqs > 1: + raise ValueError( + "AR-Diffusion paged KV supports max_num_seqs=1 (single-sequence " + f"rollouts); got max_num_seqs={max_num_seqs}. Per-sequence " + "adapters/slot maps are not implemented." + ) + t = self.pipeline.transformer + num_layers = int(t.num_layers) + num_kv_heads = int(getattr(t.blocks[0].self_attn, "tp_num_heads", t.num_heads)) + head_size = int(t.dim // t.num_heads) + num_frame_per_block = int(t.num_frame_per_block) + local_attn_size = int(t.local_attn_size) + # The loaded transformer is the source of truth for frame_seqlen (it derives + # max_attention_size from it). The deploy-config inference is only a + # cross-check — warn on drift rather than silently sizing chunks wrong. + frame_seqlen = int(getattr(t, "frame_seqlen", 0)) or self._infer_frame_seqlen() + inferred = self._infer_frame_seqlen() + if frame_seqlen != inferred: + logger.warning( + "AR-Diffusion frame_seqlen mismatch: transformer=%d deploy-config=%d; using transformer", + frame_seqlen, + inferred, + ) + # The model's own attention window, in tokens (= the [-max_attention_size:] + # slice it applies). Read it directly rather than recomputing. + max_attention_size = int(t.blocks[0].self_attn.max_attention_size) + # Cross-attn text length — static pool for the text-encoder output. + # The cached k/v spans the full text sequence (the image tokens + # prepended in _forward_blocks are stripped inside the cross-attn + # forward but are not part of the cached k/v). + cross_attn_length = int(getattr(t, "text_len", 0)) + # I2V image-token cross-attn length: the model splits context[:, :257] as the + # image tokens (the img_emb(clip_feature) output), caching k_img/v_img. T2V: 0. + cross_attn_img_length = 257 if getattr(t, "model_type", "t2v") == "i2v" else 0 + + # Frame-granular paging: 1 block = 1 frame = frame_seqlen tokens, so the + # resident window matches max_attention_size exactly (it need not be a whole + # number of num_frame_per_block causal blocks). + chunk_size = frame_seqlen + window_chunks = self.ar_diffusion_kv_config.window_chunks or (max_attention_size // frame_seqlen) + + self.ar_diffusion_kv_config = dataclasses.replace( + self.ar_diffusion_kv_config, chunk_size=chunk_size, window_chunks=window_chunks + ) + free_bytes = torch.cuda.mem_get_info(self.device)[0] + # Under CFG-parallel each rank executes exactly ONE branch (rank0 pos, + # rank1 neg; the other branch's lazy contexts never allocate on this + # rank), so its pool only needs one branch's capacity. A single-process + # run (cfg world 1) executes both branches from one pool. + try: + from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_world_size, + ) + + cfg_world = int(get_classifier_free_guidance_world_size()) + except Exception: + cfg_world = 1 + local_branches = 1 if cfg_world >= 2 else 2 + logger.info( + "AR-Diffusion preallocating (paged): frame_seqlen=%d num_frame_per_block=%d " + "local_attn_size=%d -> chunk_size=%d window_chunks=%d (window=%d tokens)", + frame_seqlen, + num_frame_per_block, + local_attn_size, + chunk_size, + window_chunks, + window_chunks * chunk_size, + ) + self.build_kv_cache( + num_layers=num_layers, + num_kv_heads=num_kv_heads, + head_size=head_size, + dtype=self.od_config.dtype, + block_size=chunk_size, # one pool block per chunk + max_model_len=1 << 20, + available_bytes=free_bytes, + cross_attn_length=cross_attn_length, + cross_attn_img_length=cross_attn_img_length, + local_branches=local_branches, + num_frame_per_block=num_frame_per_block, + ) + + def execute_model(self, req: OmniDiffusionRequest) -> DiffusionOutput: + # KV disabled -> base behavior, unchanged. + if self.kv_cache is None: + return super().execute_model(req) + + kv = self.kv_cache + # DreamZero KV is session-scoped (the model state persists across a + # session's forwards), so the AR-Diffusion KV state is keyed by session_id and + # reused — matching how pipeline.forward resolves the model-local state. + extra_args = req.sampling_params.extra_args or {} + session_id = str(extra_args.get("session_id") or "default") + state = self._ar_diffusion_states.get(session_id) + if state is None: + pos = kv.begin_request(f"bde__{session_id}") + neg = kv.begin_request(f"bde__{session_id}__neg") + state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) + self._ar_diffusion_states[session_id] = state + # Evict the least-recently-used session(s) past the cap, freeing their + # pool blocks — mirrors DreamZeroPipeline._states so the AR-Diffusion pool can't + # outlive the model-local state map it shadows. + while len(self._ar_diffusion_states) > self._max_ar_diffusion_states: + old_id, old_state = self._ar_diffusion_states.popitem(last=False) + old_state.close() + logger.debug("AR-Diffusion evicted session=%s (LRU); freed pool blocks", old_id) + # Track recency on every access (hit or miss) so the LRU order is correct. + self._ar_diffusion_states.move_to_end(session_id) + logger.debug( + "AR-Diffusion execute_model: req=%s session=%s chunk_size=%d window_chunks=%s num_blocks=%d", + req.request_id, + session_id, + kv.spec.chunk_size, + kv.config.window_chunks, + kv.num_blocks, + ) + self.pipeline._ar_diffusion_kv_state = state + # Time the worker-side forward (true per-request compute E2E). The CUDA + # sync below makes the stop reflect completed GPU work, not just dispatch. + _e2e_t0 = time.perf_counter() + try: + out = super().execute_model(req) + except Exception: + # Transactional containment: a forward that died partway may have + # written some layers' K/V into allocated-but-uncommitted blocks + # and advanced model-local state. Tear the whole session down + # (engine pool blocks freed via close(); model-local state dropped) + # so the next request with this session id starts clean instead of + # tripping the pending-commit guard or reading half-written KV. + broken = self._ar_diffusion_states.pop(session_id, None) + if broken is not None: + broken.close() + states = getattr(self.pipeline, "_states", None) + if states is not None: + states.pop(session_id, None) + logger.warning( + "AR-Diffusion execute_model failed for session=%s; session state torn down " + "(pool blocks freed) — the next request starts a fresh session", + session_id, + ) + raise + finally: + self.pipeline._ar_diffusion_kv_state = None + if self.device is not None and torch.cuda.is_available(): + torch.accelerator.synchronize(self.device) + self._perf_e2e_times.append(time.perf_counter() - _e2e_t0) + return out + + # -- cuda-graph warm-up ---------------------------------------------------- + + #: Dedicated session id for the warm-up rollout; cleaned from both state maps. + _WARMUP_SID = "__ardiffusion_warmup__" + + def _synth_robot_obs(self, h: int, w: int, n_frames: int) -> dict: + """Zero-filled roboarena observation matching the client's obs layout. + + Mirrors ``examples/.../client_schedule.make_obs_from_video`` (2 ext + wrist + cameras, 7/6/1 state vectors) so it survives ``_transform_robot_obs`` exactly + like a real request. One frame -> ``(H,W,3)``; chunk -> ``(n_frames,H,W,3)``. + """ + img = np.zeros((h, w, 3), dtype=np.uint8) if n_frames == 1 else np.zeros((n_frames, h, w, 3), dtype=np.uint8) + return { + "observation/exterior_image_0_left": img, + "observation/exterior_image_1_left": img, + "observation/wrist_image_left": img, + "observation/joint_position": np.zeros(7, dtype=np.float32), + "observation/cartesian_position": np.zeros(6, dtype=np.float32), + "observation/gripper_position": np.zeros(1, dtype=np.float32), + "prompt": "warmup", + "session_id": self._WARMUP_SID, + } + + def _warmup_ar_rollout(self) -> None: + """Drive a synthetic rollout through one window-fill so every DiT graph is + captured at load (off the serving hot path). Frees the synthetic session and + leaves the KV pool exactly as found. Never raises — falls back to lazy capture. + """ + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + kv = self.kv_cache + if kv is None: + return + sid = self._WARMUP_SID + t = self.pipeline.transformer + nfpb = int(t.num_frame_per_block) + window_chunks = int(kv.config.window_chunks or 0) + if window_chunks <= 0: + return + # Distinct DiT window shapes are walked by one fill: prefill + ceil((W-1)/nfpb) + # chunk forwards. +1 optionally to also capture the reset-cycle forward. + n_forwards = 1 + math.ceil(max(0, window_chunks - 1) / nfpb) + if self.ar_diffusion_kv_config.warmup_capture_reset: + n_forwards += 1 + res = self._image_resolution() + h, w = int(res[0]), int(res[1]) + + free_before = kv.manager.block_pool.get_num_free_blocks() + # The step cache is left active: the DiT input shape is per-chunk (the resident + # window length), and the first denoise step of every chunk always computes, so + # every window shape is captured regardless of how many later steps are skipped. + logger.info( + "AR-Diffusion cudagraph warm-up: up to %d synthetic forwards (window_chunks=%d, nfpb=%d)", + n_forwards, + window_chunks, + nfpb, + ) + try: + for i in range(n_forwards): + n_frames = 1 if i == 0 else 4 # client convention: 1-frame prefill, 4-frame chunks + obs = self._synth_robot_obs(h, w, n_frames) + sp = OmniDiffusionSamplingParams(extra_args={"reset": i == 0, "session_id": sid, "robot_obs": obs}) + req = OmniDiffusionRequest(prompts=["warmup"], sampling_params=sp, request_id=f"ardiffusion-warmup-{i}") + self.execute_model(req) + # Stop once the resident window is full — the remaining shapes are + # already captured (the window caps/resets through the same set). + ar_state = self._ar_diffusion_states.get(sid) + if ( + not self.ar_diffusion_kv_config.warmup_capture_reset + and ar_state is not None + and len(kv.window_block_ids(ar_state.pos)) >= window_chunks + ): + break + except Exception as e: # noqa: BLE001 — warm-up must never break model load + logger.warning("AR-Diffusion cudagraph warm-up failed (%s); using lazy capture.", e) + finally: + # Free the synthetic session from BOTH maps (KV pool + pipeline state). + ar_state = self._ar_diffusion_states.pop(sid, None) + if ar_state is not None: + try: + ar_state.close() # return KV blocks to the BlockPool + except Exception as e: # noqa: BLE001 + logger.warning("AR-Diffusion warm-up: state.close() failed (%s)", e) + try: + self.pipeline._states.pop(sid, None) + except Exception: # noqa: BLE001 + pass + # Drop any per-forward timings the warm-up recorded so serving stats start clean. + self._perf_e2e_times.clear() + free_after = kv.manager.block_pool.get_num_free_blocks() + if free_after != free_before: + logger.warning( + "AR-Diffusion warm-up: KV pool not restored (free %d -> %d) — possible leak", + free_before, + free_after, + ) + else: + logger.info( + "AR-Diffusion cudagraph warm-up complete; KV pool restored (%d free blocks)", + free_after, + ) + + def _image_resolution(self) -> list[int]: + mc = getattr(self.od_config, "model_config", None) or {} + psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} + return psc.get("image_resolution", [180, 320])