From 96235da4682614bc01ed7c66ba8c42c60a8a4942 Mon Sep 17 00:00:00 2001 From: Jack Chen Date: Thu, 23 Jul 2026 12:37:32 +0800 Subject: [PATCH] Generalize AR-Diffusion KV session capabilities (#5271) Signed-off-by: Jack47 --- .../ar_diffusion_pipeline_capability.md | 165 +++++ tests/ar_diffusion/test_capability_runner.py | 317 ++++++++++ tests/ar_diffusion/test_kv_cache.py | 84 ++- tests/ar_diffusion/test_kv_orchestration.py | 22 +- tests/ar_diffusion/test_paged_attention.py | 69 ++- tests/ar_diffusion/test_pipeline_bridge.py | 251 +++++--- tests/dreamzero/test_pipeline_state.py | 56 +- .../models/dreamzero/pipeline_dreamzero.py | 235 ++++++-- .../experimental/ar_diffusion/__init__.py | 19 +- .../experimental/ar_diffusion/capability.py | 169 ++++++ vllm_omni/experimental/ar_diffusion/engine.py | 5 +- .../ar_diffusion/kv_cache/__init__.py | 2 +- .../ar_diffusion/kv_cache/config.py | 5 +- .../ar_diffusion/kv_cache/manager.py | 355 +++++++---- .../ar_diffusion/kv_cache/paged.py | 21 +- .../ar_diffusion/kv_cache/paged_attention.py | 39 +- .../ar_diffusion/kv_cache/state.py | 225 ++++--- vllm_omni/experimental/ar_diffusion/runner.py | 569 ++++++++---------- 18 files changed, 1808 insertions(+), 800 deletions(-) create mode 100644 docs/design/ar_diffusion_pipeline_capability.md create mode 100644 tests/ar_diffusion/test_capability_runner.py create mode 100644 vllm_omni/experimental/ar_diffusion/capability.py diff --git a/docs/design/ar_diffusion_pipeline_capability.md b/docs/design/ar_diffusion_pipeline_capability.md new file mode 100644 index 000000000..c9e05c8e8 --- /dev/null +++ b/docs/design/ar_diffusion_pipeline_capability.md @@ -0,0 +1,165 @@ +# AR-Diffusion pipeline capability + +> **Status:** experimental and non-normative. This note documents the current +> `experimental/ar_diffusion` contract. It follows the ownership and lifecycle +> direction in [#5137](https://github.com/vllm-project/vllm-omni/issues/5137), +> but it is not yet part of the stable module-design hierarchy. + +`ARDiffusionEngine` is an opt-in diffusion backend for pipelines that keep +autoregressive attention KV across requests. The engine only selects +`ARDiffusionModelRunner`; the runner owns paged KV pools, session lookup, LRU +eviction, and failure cleanup. + +A pipeline opts in by implementing `SupportsARDiffusionPipeline`. Its immutable +`ARDiffusionKVCacheSpec` reports worker-local KV geometry and policy: + +- self-attention layers, TP-local KV heads, head size, and tokens per latent frame; +- latent frames committed by one model block, sliding window, and attention sink; +- logical KV branches mapped to worker-local storage slots; +- named fixed-length cross-attention KV, model-specific scratch geometry, and + the pipeline's requested session-capacity upper bound. + +The runner binds an `ARDiffusionKVState` only around one `forward()` call. The +pipeline may use it during that context but must not retain it. The session +survives subsequent requests with the same `session_id`. Request reset, explicit +close, LRU eviction, and forward exceptions all release runner-owned KV and call +the pipeline's reset/close lifecycle hook. Cross-attention KV is allocated lazily +per session and logical KV branch, and is released on the same path. A pipeline +publishes one named cache transactionally by yielding exactly one `(k, v)` pair +per layer: + +```python +state.populate_cross_attention("main", "text", project_text_kv_by_layer()) +``` + +The cache becomes readable only after the iterable completes successfully. +Logical branches remain independent even when they share a worker-local +`local_index`; that index only maps lazily executed self-attention and scratch +work to worker-local capacity slots. It does not define cross-attention cache +ownership. + +Requests select the session with `extra_args["session_id"]`; setting `reset=True` +replaces it before the forward, while `close_session=True` releases it after a +successful forward. The runner also exposes `reset_session()` and +`close_session()` for an owning serving layer. + +## Current execution limits + +The experimental runner supports request-mode execution with +`max_num_seqs=1`. It rejects step execution and request-batch execution at load +time, and its inherited batch/step entry points also fail explicitly. Supporting +those paths requires a batch-aware state-binding contract rather than silently +bypassing `bind_ar_diffusion_state()`. + +The current implementation retains one session per model instance. A request +for a different `session_id` closes the resident session through the same LRU +lifecycle used by explicit close and failure cleanup before allocating the new +one. Self-attention and cross-attention memory are therefore sized for one +resident session even if the pipeline declares a larger `session_capacity`. +This prioritizes rollout throughput and prevents block-pool exhaustion before a +count-based eviction can run. Multi-session residency can be added later with a +memory-backed capacity calculation or allocation-pressure eviction. + +Scratch storage is also capability-driven. `frames_per_block` reserves space for +an uncommitted current video block, while `max_scratch_tokens_per_branch` +declares the maximum model-specific tokens, such as action/state registers, that +must coexist with it. `AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH` may increase +that derived minimum for deployment experiments but cannot reduce it. + +## DreamZero adapter + +DreamZero implements the capability directly on `DreamZeroPipeline`. Its adapter +reads DreamZero transformer geometry, describes the positive and negative CFG +branches, binds the state with a context manager, and translates the existing +boolean CFG calls to the explicit `"positive"` and `"negative"` KV branch names. +Its optional `ar_diffusion_warmup_requests()` provider owns the robot observation +schema and the one-frame/four-frame warmup convention; none of those details are +present in the generic runner. + +Code migrating from the former experimental KV state API should use the +mechanical mapping below: + +```python +# Before: state.pos, state.neg, state.get_kv_caches(is_negative, ...) +positive = state.adapter("positive") +negative = state.adapter("negative") +contexts = state.get_kv_caches("negative", seq_len=frame_tokens, commit_current=True) +``` + +Direct experimental `ARDiffusionKVCache` construction also migrates +mechanically: replace `local_branches` with explicit `kv_branches`, +`num_frame_per_block` with `frames_per_block`, and the fixed text/image length +arguments with a named `cross_attention_lengths` mapping plus +`session_capacity`. Normal engine users do not construct this object directly. + +Cross-attention writers migrate from incremental publication: + +```python +# Before +for layer_index, (key, value) in enumerate(project_text_kv_by_layer()): + state.write_cross_attention_kv("text", layer_index, branch, key, value) +state.mark_cross_attention_populated(branch, "text") + +# After +state.populate_cross_attention(branch, "text", project_text_kv_by_layer()) +``` + +The iterable is consumed once and must yield exactly `num_layers` pairs. Failed +or incomplete projection leaves the previous complete cache unchanged, or +leaves the cache absent on its first population. + +## Single-KV-branch pipeline sketch + +A future single-branch causal DMD pipeline can opt in without changes to the +runner or KV core: + +```python +class WorldPipeline: + def ar_diffusion_kv_cache_spec(self): + return ARDiffusionKVCacheSpec( + num_layers=self.transformer.num_layers, + num_kv_heads=self.transformer.tp_local_kv_heads, + head_size=self.transformer.head_size, + tokens_per_frame=self.transformer.tokens_per_latent_frame, + frames_per_block=3, + window_frames=self.transformer.sliding_window_frames, + sink_frames=self.transformer.sink_frames, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", self.text_length),), + max_scratch_tokens_per_branch=self.transformer.max_register_tokens, + session_capacity=1, + ) + + @contextmanager + def bind_ar_diffusion_state(self, session_id, state): + self.current_ar_state = state + try: + yield + finally: + self.current_ar_state = None + + def reset_ar_diffusion_session(self, session_id): + self.model_states.pop(session_id, None) + + def close_ar_diffusion_session(self, session_id): + self.model_states.pop(session_id, None) +``` + +The model writes/reads named cross-attention KV through the bound state and uses +the `"main"` KV branch for paged self-attention. Pipelines without a warmup +provider are loaded normally and skip AR rollout warmup. + +## Relationship to the multi-stage KV manager RFC + +[#5244](https://github.com/vllm-project/vllm-omni/issues/5244) describes a +future multi-stage KV transfer manager, connector integration, and cache-aware +scheduling. This capability has a narrower scope: it owns model-local paged KV +and session lifecycle inside one AR-Diffusion runner. It does not define +cross-stage transfer, connector behavior, or scheduler-visible cache affinity. + +When the experimental runtime migrates toward #5244, the implementation should +reconcile its current full `KVCacheManager` use with the RFC's proposed +`BlockPool`-oriented DiT manager. The pipeline capability should remain the +model-facing geometry and state-binding boundary where possible, while the +experimental `ARDiffusionKVCache` class itself should not be treated as a stable +public API. diff --git a/tests/ar_diffusion/test_capability_runner.py b/tests/ar_diffusion/test_capability_runner.py new file mode 100644 index 000000000..7b94fe80f --- /dev/null +++ b/tests/ar_diffusion/test_capability_runner.py @@ -0,0 +1,317 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU contracts for capability-driven AR-Diffusion sessions.""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, +) +from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVConfig +from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner + +BLOCK = 16 +POS = "positive" +NEG = "negative" + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def lingbot_like_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: + """Single-kv_branch causal DMD: 3 latent frames/block + sink/window + text KV.""" + return ARDiffusionKVCacheSpec( + num_layers=2, + num_kv_heads=4, # already TP-local + head_size=64, + tokens_per_frame=BLOCK, + frames_per_block=3, + window_frames=5, + sink_frames=1, + kv_branches=(ARDiffusionKVBranchSpec("main", 0),), + session_capacity=capacity, + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", 8),), + ) + + +def dreamzero_like_spec(*, capacity: int = 2) -> ARDiffusionKVCacheSpec: + return ARDiffusionKVCacheSpec( + num_layers=2, + num_kv_heads=4, + head_size=64, + tokens_per_frame=BLOCK, + frames_per_block=4, + window_frames=6, + kv_branches=(ARDiffusionKVBranchSpec(POS, 0), ARDiffusionKVBranchSpec(NEG, 1)), + session_capacity=capacity, + cross_attention=(ARDiffusionCrossAttentionKVSpec("text", 8),), + ) + + +class CapablePipeline: + def __init__(self, spec: ARDiffusionKVCacheSpec) -> None: + self.spec = spec + self.bound_state = None + self.binds: list[str] = [] + self.resets: list[str] = [] + self.closes: list[str] = [] + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + return self.spec + + @contextmanager + def bind_ar_diffusion_state(self, session_id: str, state): + assert self.bound_state is None + self.bound_state = state + self.binds.append(session_id) + try: + yield + finally: + self.bound_state = None + + def reset_ar_diffusion_session(self, session_id: str) -> None: + self.resets.append(session_id) + + def close_ar_diffusion_session(self, session_id: str) -> None: + self.closes.append(session_id) + + +class WarmupPipeline(CapablePipeline): + def __init__(self, spec: ARDiffusionKVCacheSpec, requests: list[object]) -> None: + super().__init__(spec) + self.requests = requests + + def ar_diffusion_warmup_requests(self, session_id: str): + assert session_id == ARDiffusionModelRunner._WARMUP_SID + return iter(self.requests) + + +class BatchCapablePipeline(CapablePipeline): + supports_request_batch = True + + +def make_runner( + pipeline: object, + *, + available_bytes: int = 1 << 28, + step_execution: bool = False, +) -> ARDiffusionModelRunner: + runner = object.__new__(ARDiffusionModelRunner) + runner.od_config = SimpleNamespace( + max_num_seqs=1, + dtype=torch.float32, + enforce_eager=True, + step_execution=step_execution, + ) + runner.device = torch.device("cpu") + runner.pipeline = pipeline + runner.ar_diffusion_kv_config = ARDiffusionKVConfig(enable=True) + runner.kv_cache = None + runner._ar_diffusion_capability = None + runner._ar_diffusion_kv_cache_spec = None + runner._sessions = OrderedDict() + runner._session_capacity = 0 + runner._perf_e2e_times = [] + runner._preallocate_kv_cache(available_bytes=available_bytes) + return runner + + +def commit_one_frame(runner: ARDiffusionModelRunner, session_id: str, kv_branch: str): + state = runner._get_or_create_session(session_id) + ctx = state.get_kv_caches(kv_branch, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + state.commit_paged_context(kv_branch) + return state + + +def test_ar_runner_rejects_pipeline_without_capability(): + runner = object.__new__(ARDiffusionModelRunner) + runner.od_config = SimpleNamespace(max_num_seqs=1) + runner.pipeline = object() + with pytest.raises(TypeError, match="SupportsARDiffusionPipeline"): + runner._preallocate_kv_cache(available_bytes=1 << 20) + + +def test_lingbot_like_single_branch_session_reuse_reset_and_close(): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + assert kv.num_local_kv_branches == 1 + assert kv.frames_per_block == 3 + assert kv.spec.window_chunks == 5 + assert kv.spec.sink_chunks == 1 + assert kv.cross_attention_lengths == {"text": 8} + + first = commit_one_frame(runner, "s1", "main") + assert runner._get_or_create_session("s1") is first + k = torch.randn(1, 8, 4, 64) + v = torch.randn(1, 8, 4, 64) + first.populate_cross_attention("main", "text", [(k, v)] * first.num_layers) + assert first.get_cross_attention_kv("main", "text")[0]["k"].shape == k.shape + + runner.reset_session("s1") + assert "s1" not in runner._sessions + assert "s1" not in kv._cross_sessions + assert pipeline.resets == ["s1"] + second = runner._get_or_create_session("s1") + assert second is not first + + runner.close_session("s1") + assert "s1" not in runner._sessions + assert pipeline.closes == ["s1"] + + +def test_lingbot_like_sink_survives_sliding_window_eviction(): + runner = make_runner(CapablePipeline(lingbot_like_spec())) + kv = runner.kv_cache + assert kv is not None + + for _ in range(8): + state = commit_one_frame(runner, "s1", "main") + + table = kv.block_table(state.adapter("main")) + assert table[0] != kv.null_block_id + assert table[1] == kv.null_block_id + + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=False)[0].forward_ctx + visible, _ = ctx.video_block_table(torch.device("cpu")) + assert visible[0] == table[0] + assert len(visible) == 6 # sink + recent window, including current scratch + + +def test_dreamzero_like_two_branches_are_independent(): + runner = make_runner(CapablePipeline(dreamzero_like_spec())) + kv = runner.kv_cache + assert kv is not None and kv.num_local_kv_branches == 2 + state = commit_one_frame(runner, "s1", POS) + assert len(kv.window_block_ids(state.adapter(POS))) == 1 + assert kv.window_block_ids(state.adapter(NEG)) == [] + + +def test_lru_eviction_releases_blocks_and_notifies_pipeline(): + pipeline = CapablePipeline(lingbot_like_spec(capacity=8)) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + assert runner._session_capacity == 1 + assert kv.session_capacity == 1 + free_total = kv.manager.block_pool.get_num_free_blocks() + old = commit_one_frame(runner, "old", "main") + k = torch.randn(1, 8, 4, 64) + old.populate_cross_attention("main", "text", [(k, k)] * old.num_layers) + assert kv.manager.block_pool.get_num_free_blocks() < free_total + + runner._get_or_create_session("new") + + assert tuple(runner._sessions) == ("new",) + assert pipeline.closes == ["old"] + assert "old" not in kv._cross_sessions + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_forward_exception_releases_pending_allocation_and_model_state(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + free_total = kv.manager.block_pool.get_num_free_blocks() + + def boom(self, req, kv_prefetch_job=None): + state = pipeline.bound_state + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + raise RuntimeError("layer exploded") + + monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) + request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + + with pytest.raises(RuntimeError, match="layer exploded"): + runner.execute_model(request) + + assert pipeline.bound_state is None + assert pipeline.closes == ["broken"] + assert not runner._sessions + assert not kv._adapters + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_synchronize_exception_uses_forward_cleanup_path(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + kv = runner.kv_cache + assert kv is not None + free_total = kv.manager.block_pool.get_num_free_blocks() + + def return_after_allocation(self, req, kv_prefetch_job=None): + state = pipeline.bound_state + ctx = state.get_kv_caches("main", seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx.ensure_video_slots(torch.device("cpu")) + return object() + + def synchronize_boom(device): + raise RuntimeError("asynchronous kernel failed") + + monkeypatch.setattr(DiffusionModelRunner, "execute_model", return_after_allocation) + monkeypatch.setattr(torch.accelerator, "synchronize", synchronize_boom) + runner.device = torch.device("cuda") + request = SimpleNamespace(sampling_params=SimpleNamespace(extra_args={"session_id": "broken"})) + + with pytest.raises(RuntimeError, match="asynchronous kernel failed"): + runner.execute_model(request) + + assert pipeline.bound_state is None + assert pipeline.closes == ["broken"] + assert not runner._sessions + assert not kv._adapters + assert not runner._perf_e2e_times + assert kv.manager.block_pool.get_num_free_blocks() == free_total + + +def test_ar_runner_rejects_step_and_request_batch_modes(): + with pytest.raises(ValueError, match="step_execution=True"): + make_runner(CapablePipeline(lingbot_like_spec()), step_execution=True) + with pytest.raises(ValueError, match="request-batch execution"): + make_runner(BatchCapablePipeline(lingbot_like_spec())) + + +def test_ar_runner_defensively_rejects_inherited_batch_and_step_entrypoints(): + runner = object.__new__(ARDiffusionModelRunner) + with pytest.raises(RuntimeError, match="request-batch execution"): + runner.execute_model_batch(None, None) + with pytest.raises(RuntimeError, match="step execution"): + runner.execute_stepwise(None) + + +def test_model_specific_warmup_provider_is_consumed(monkeypatch): + requests = [object(), object()] + pipeline = WarmupPipeline(lingbot_like_spec(), requests) + runner = make_runner(pipeline) + seen: list[object] = [] + monkeypatch.setattr(runner, "execute_model", seen.append) + + runner._warmup_ar_rollout() + + assert seen == requests + assert pipeline.closes == [runner._WARMUP_SID] + + +def test_pipeline_without_warmup_provider_is_safely_skipped(monkeypatch): + pipeline = CapablePipeline(lingbot_like_spec()) + runner = make_runner(pipeline) + execute = SimpleNamespace(called=False) + + def fail_if_called(request): + execute.called = True + + monkeypatch.setattr(runner, "execute_model", fail_if_called) + runner._warmup_ar_rollout() + assert execute.called is False diff --git a/tests/ar_diffusion/test_kv_cache.py b/tests/ar_diffusion/test_kv_cache.py index beb575a0b..9ca0e547d 100644 --- a/tests/ar_diffusion/test_kv_cache.py +++ b/tests/ar_diffusion/test_kv_cache.py @@ -12,6 +12,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind, get_kv_cache_spec_kind from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry from vllm.v1.request import RequestStatus +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -194,16 +195,29 @@ def test_cross_attn_pool_deducted_from_self_attn_budget(): block_size=BLOCK, max_model_len=4096, available_bytes=avail, - cross_attn_length=L, + kv_branches=(ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 0)), + session_capacity=1, + cross_attention_lengths={"text": L}, device=torch.device("cpu"), ) cross_bytes = 2 * 2 * L * 4 * 64 * torch.float16.itemsize * 2 # K+V, pos+neg, layers expected = compute_num_blocks(avail - cross_bytes, 0.5, kv.spec.page_size_bytes * 2) assert kv.num_blocks == expected - assert expected > 2 * (2 + 1) + 2 # above the min-blocks floor, so the deduction is what's tested + assert expected > 1 * (2 + 1) + 2 # above the local-slot floor, so the cross deduction is what's tested -def _make_kv(*, local_branches, num_frame_per_block=2, window_chunks=9): +def _make_kv( + *, + local_branches, + num_frame_per_block=2, + window_chunks=9, + max_scratch_tokens_per_branch=0, +): + kv_branches = ( + (ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 0)) + if local_branches == 1 + else (ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 1)) + ) return ARDiffusionKVCache( ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks), num_layers=1, @@ -214,41 +228,75 @@ def _make_kv(*, local_branches, num_frame_per_block=2, window_chunks=9): max_model_len=4096, available_bytes=1 << 16, # tiny -> the floor binds device=torch.device("cpu"), - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, + kv_branches=kv_branches, + session_capacity=1, + frames_per_block=num_frame_per_block, + max_scratch_tokens_per_branch=max_scratch_tokens_per_branch, ) def test_pool_floor_is_branch_aware(): - """CFG-parallel rank (one local branch) sizes for one window + in-flight chunk; - a single-process run (both branches) sizes for two. Scratch scales the same way.""" + """CFG-parallel rank (one local kv_branch) sizes for one window + in-flight chunk; + a single-process run (both kv_branches) sizes for two. Scratch scales the same way.""" one = _make_kv(local_branches=1) two = _make_kv(local_branches=2) assert one.managed_num_blocks == 1 * (9 + 2) + 2 # 13 assert two.managed_num_blocks == 2 * (9 + 2) + 2 # 24 - assert one.scratch_num_blocks == one.scratch_blocks_per_branch - assert two.scratch_num_blocks == 2 * two.scratch_blocks_per_branch - assert one.num_blocks_total == 13 + one.scratch_blocks_per_branch + assert one.scratch_num_blocks == one.scratch_blocks_per_kv_branch + assert two.scratch_num_blocks == 2 * two.scratch_blocks_per_kv_branch + assert one.scratch_blocks_per_kv_branch == 2 + assert one.num_blocks_total == 13 + one.scratch_blocks_per_kv_branch + + +def test_scratch_capacity_is_derived_from_declared_geometry(monkeypatch): + monkeypatch.delenv("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", raising=False) + kv = _make_kv( + local_branches=1, + num_frame_per_block=6, + max_scratch_tokens_per_branch=BLOCK + 1, + ) + assert kv.scratch_blocks_per_kv_branch == 8 # six video blocks + two auxiliary blocks + + +def test_scratch_env_override_cannot_reduce_declared_minimum(monkeypatch): + monkeypatch.setenv("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", "1") + kv = _make_kv( + local_branches=1, + num_frame_per_block=6, + max_scratch_tokens_per_branch=BLOCK + 1, + ) + assert kv.scratch_blocks_per_kv_branch == 8 def test_scratch_maps_to_slot_zero_with_one_local_branch(): - """A CFG-parallel rank runs exactly one branch: whichever CFG side it is, + """A CFG-parallel rank runs exactly one kv_branch: whichever CFG side it is, its scratch lands in the rank's single slot (no dead second slot).""" one = _make_kv(local_branches=1) - assert one.scratch_block_ids(True, 0, 2) == one.scratch_block_ids(False, 0, 2) + assert one.scratch_block_ids("positive", 0, 2) == one.scratch_block_ids("negative", 0, 2) two = _make_kv(local_branches=2) - assert two.scratch_block_ids(True, 0, 2) != two.scratch_block_ids(False, 0, 2) + assert two.scratch_block_ids("positive", 0, 2) != two.scratch_block_ids("negative", 0, 2) def test_scratch_exhaustion_still_raises(): one = _make_kv(local_branches=1) - cap = one.scratch_blocks_per_branch + cap = one.scratch_blocks_per_kv_branch with pytest.raises(RuntimeError, match="scratch blocks exhausted"): - one.scratch_block_ids(False, 0, cap + 1) + one.scratch_block_ids("positive", 0, cap + 1) -def test_invalid_local_branches_rejected(): - with pytest.raises(ValueError, match="local_branches"): - _make_kv(local_branches=3) +def test_non_contiguous_branch_indices_rejected(): + with pytest.raises(ValueError, match="contiguous"): + ARDiffusionKVCache( + ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=2), + num_layers=1, + num_kv_heads=4, + head_size=64, + dtype=torch.float32, + block_size=BLOCK, + max_model_len=4096, + available_bytes=1 << 16, + kv_branches=(ARDiffusionKVBranchSpec("main", 1),), + session_capacity=1, + ) diff --git a/tests/ar_diffusion/test_kv_orchestration.py b/tests/ar_diffusion/test_kv_orchestration.py index 4ba5cb49b..b7c77390c 100644 --- a/tests/ar_diffusion/test_kv_orchestration.py +++ b/tests/ar_diffusion/test_kv_orchestration.py @@ -4,9 +4,18 @@ import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig -DIMS = dict(num_layers=2, num_kv_heads=4, head_size=64, dtype=torch.float16, block_size=16) +DIMS = dict( + num_layers=2, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + block_size=16, + kv_branches=(ARDiffusionKVBranchSpec("positive", 0), ARDiffusionKVBranchSpec("negative", 1)), + session_capacity=2, +) def make_cache(*, chunk_size=16, window_chunks=2, available_bytes=1 << 24): @@ -67,7 +76,7 @@ def test_num_computed_advances_per_chunk(): def test_state_close_frees_both_branch_blocks(): - """ARDiffusionKVState.close() returns both CFG branches' pool blocks to the pool. + """ARDiffusionKVState.close() returns both CFG kv_branches' pool blocks to the pool. This is the primitive the runner's LRU eviction relies on: when a session is evicted, close() must free the blocks both adapters hold, or session churn @@ -80,12 +89,17 @@ def test_state_close_frees_both_branch_blocks(): pos = kv.begin_request("bde__s") neg = kv.begin_request("bde__s__neg") - state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) + state = ARDiffusionKVState( + kv, + "s", + {"positive": pos, "negative": neg}, + num_layers=kv.num_layers, + ) for _ in range(3): for adapter in (pos, neg): kv.allocate_chunk(adapter) kv.commit_chunk(adapter) - # Both branches hold resident blocks now. + # Both kv_branches hold resident blocks now. assert kv.manager.block_pool.get_num_free_blocks() < free_total state.close() diff --git a/tests/ar_diffusion/test_paged_attention.py b/tests/ar_diffusion/test_paged_attention.py index 1836656f9..b5c5e5870 100644 --- a/tests/ar_diffusion/test_paged_attention.py +++ b/tests/ar_diffusion/test_paged_attention.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -24,6 +25,8 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVStat BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 +POS = "positive" +NEG = "negative" def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu")): @@ -37,11 +40,15 @@ def make_state(*, num_layers=1, window_chunks=2, dtype=torch.float32, device=tor block_size=BLOCK, max_model_len=4096, available_bytes=1 << 26, + kv_branches=(ARDiffusionKVBranchSpec(POS, 0), ARDiffusionKVBranchSpec(NEG, 1)), + session_capacity=2, + frames_per_block=2, + max_scratch_tokens_per_branch=BLOCK, device=device, ) pos = kv.begin_request("r-pos") neg = kv.begin_request("r-neg") - return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + return kv, ARDiffusionKVState(kv, "s1", {POS: pos, NEG: neg}, num_layers=num_layers) def _dense_attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor: @@ -79,51 +86,51 @@ def _commit_video_span( kv: ARDiffusionKVCache, st: ARDiffusionKVState, *, - is_negative: bool, + kv_branch: str, n_chunks: int, dtype: torch.dtype, device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor]: - ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx + ctx = st.get_kv_caches(kv_branch, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx ctx.ensure_video_slots(device) k = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) v = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) kv._k_pools[0][ctx.current_video_slot_mapping] = k[0] kv._v_pools[0][ctx.current_video_slot_mapping] = v[0] - st.commit_paged_context(is_negative) + st.commit_paged_context(kv_branch) return k, v def test_paged_context_allocates_lazily_and_commits_after_forward(): _, st = make_state() - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True) ctx = contexts[0].forward_ctx assert isinstance(contexts[0], ARDiffusionPagedLayerContext) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert ctx.current_video_slot_mapping is None ctx.ensure_video_slots(torch.device("cpu")) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert len(ctx.current_video_block_ids) == 1 - st.commit_paged_context(False) - assert st.pos.completed_chunks == 1 - assert st._committed[False] == BLOCK + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == 1 + assert st._committed[POS] == BLOCK def test_scratch_video_and_action_blocks_do_not_commit(): kv, st = make_state() - ctx = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=False)[0].forward_ctx + ctx = st.get_kv_caches(POS, seq_len=2 * BLOCK, commit_current=False)[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) ctx.ensure_action_slots(3, torch.device("cpu")) - assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 2) - assert ctx.action_scratch_block_ids == kv.scratch_block_ids(False, 2, 1) - st.commit_paged_context(False) - assert st.pos.completed_chunks == 0 - assert st._committed[False] == 0 + assert ctx.current_video_block_ids == kv.scratch_block_ids(POS, 0, 2) + assert ctx.action_scratch_block_ids == kv.scratch_block_ids(POS, 2, 1) + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == 0 + assert st._committed[POS] == 0 def test_pipeline_kv_get_paged_path_has_no_gather_backend(): @@ -155,7 +162,7 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, k, v = _commit_video_span( kv, st, - is_negative=False, + kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device, @@ -163,7 +170,7 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, history_k_parts.append(k) history_v_parts.append(v) - ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0].forward_ctx + ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit_current)[0].forward_ctx ctx.ensure_video_slots(device) current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) @@ -212,9 +219,9 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len, torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5) - before = st.pos.completed_chunks - st.commit_paged_context(False) - assert st.pos.completed_chunks == before + (1 if commit_current else 0) + before = st.adapter(POS).completed_chunks + st.commit_paged_context(POS) + assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0) @pytest.mark.skipif(not _cuda_flash_attn_usable(), reason="usable CUDA FlashAttention is required") @@ -231,13 +238,13 @@ def test_paged_attention_matches_dense_reference_gpu(history_chunks, action_len, history_k, history_v = _commit_video_span( kv, st, - is_negative=False, + kv_branch=POS, n_chunks=history_chunks, dtype=dtype, device=device, ) - layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit_current)[0] + layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit_current)[0] ctx = layer_ctx.forward_ctx current_k = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) current_v = torch.randn(1, BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device) @@ -294,13 +301,13 @@ def test_block_table_padded_to_fixed_width(): device = torch.device("cpu") kv, st = make_state(window_chunks=2) - ctx1 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx1 = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx ctx1.prepare(device=device, action_len=0, query_len=BLOCK) - st.commit_paged_context(False) + st.commit_paged_context(POS) - ctx2 = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx + ctx2 = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx ctx2.prepare(device=device, action_len=0, query_len=BLOCK) - st.commit_paged_context(False) + st.commit_paged_context(POS) # 1-block vs 2-block visible history: same table width, same max_seq_len. assert ctx1.block_table.shape == ctx2.block_table.shape @@ -315,7 +322,7 @@ def test_block_table_padded_to_fixed_width(): def test_prepare_is_idempotent_and_layers_share_metadata(): device = torch.device("cpu") _, st = make_state(num_layers=2) - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False) fctx = contexts[0].forward_ctx fctx.prepare(device=device, action_len=0, query_len=BLOCK) table = fctx.block_table @@ -335,7 +342,7 @@ def test_prepare_is_idempotent_and_layers_share_metadata(): def test_layer_inputs_before_prepare_raises(): _, st = make_state() - layer_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0] + layer_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0] with pytest.raises(RuntimeError, match="before prepare"): layer_ctx.to_layer_inputs() @@ -363,7 +370,7 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): kv, st = make_state(num_layers=2, window_chunks=2) def run_one_forward(commit): - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=commit) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=commit) fctx = contexts[0].forward_ctx fctx.prepare(device=device, action_len=0, query_len=BLOCK) q = torch.randn(BLOCK, N_HEADS, HEAD_DIM) @@ -374,7 +381,7 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): # block-forward code object in production). for layer_ctx in contexts: out = compiled(layer_ctx.to_layer_inputs(), q, k, v) - st.commit_paged_context(False) + st.commit_paged_context(POS) return out torch._dynamo.reset() diff --git a/tests/ar_diffusion/test_pipeline_bridge.py b/tests/ar_diffusion/test_pipeline_bridge.py index 2c4284531..92d490b92 100644 --- a/tests/ar_diffusion/test_pipeline_bridge.py +++ b/tests/ar_diffusion/test_pipeline_bridge.py @@ -4,6 +4,7 @@ import pytest import torch +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ( ARDiffusionKVCache, ARDiffusionKVConfig, @@ -14,9 +15,11 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVStat BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 +POS = "positive" +NEG = "negative" -def make_state(num_layers=1, window_chunks=4, cross_attn_length=0): +def make_state(num_layers=1, window_chunks=4, cross_attn_length=0, shared_local_index=False): cfg = ARDiffusionKVConfig(enable=True, chunk_size=BLOCK, window_chunks=window_chunks) kv = ARDiffusionKVCache( cfg, @@ -27,24 +30,29 @@ def make_state(num_layers=1, window_chunks=4, cross_attn_length=0): block_size=BLOCK, max_model_len=4096, available_bytes=1 << 24, - cross_attn_length=cross_attn_length, + kv_branches=( + ARDiffusionKVBranchSpec(POS, 0), + ARDiffusionKVBranchSpec(NEG, 0 if shared_local_index else 1), + ), + session_capacity=2, + cross_attention_lengths={"text": cross_attn_length} if cross_attn_length else None, device=torch.device("cpu"), ) pos = kv.begin_request("r-pos") neg = kv.begin_request("r-neg") - return kv, ARDiffusionKVState(kv, pos, neg, num_layers=num_layers) + return kv, ARDiffusionKVState(kv, "s1", {POS: pos, NEG: neg}, num_layers=num_layers) -def _prepare_and_commit(st: ARDiffusionKVState, is_negative: bool, n_chunks: int) -> None: - ctx = st.get_kv_caches(is_negative, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx +def _prepare_and_commit(st: ARDiffusionKVState, kv_branch: str, n_chunks: int) -> None: + ctx = st.get_kv_caches(kv_branch, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) - st.commit_paged_context(is_negative) + st.commit_paged_context(kv_branch) def test_get_returns_paged_layer_contexts_when_nothing_committed(): _, st = make_state(num_layers=2) - for neg in (False, True): - contexts = st.get_kv_caches(neg, seq_len=BLOCK, commit_current=False) + for kv_branch in (POS, NEG): + contexts = st.get_kv_caches(kv_branch, seq_len=BLOCK, commit_current=False) assert len(contexts) == 2 assert all(isinstance(ctx, ARDiffusionPagedLayerContext) for ctx in contexts) assert contexts[0].history_block_ids == [] @@ -52,43 +60,43 @@ def test_get_returns_paged_layer_contexts_when_nothing_committed(): def test_managed_paged_context_commits_only_after_forward(): kv, st = make_state(num_layers=1) - contexts = st.get_kv_caches(False, seq_len=2 * BLOCK, commit_current=True) + contexts = st.get_kv_caches(POS, seq_len=2 * BLOCK, commit_current=True) ctx = contexts[0].forward_ctx - assert st.pos.completed_chunks == 0 - assert kv.window_block_ids(st.pos) == [] + assert st.adapter(POS).completed_chunks == 0 + assert kv.window_block_ids(st.adapter(POS)) == [] ctx.ensure_video_slots(torch.device("cpu")) - assert st.pos.completed_chunks == 0 + assert st.adapter(POS).completed_chunks == 0 assert len(ctx.current_video_block_ids) == 2 - st.commit_paged_context(False) + st.commit_paged_context(POS) - assert st.pos.completed_chunks == 2 - assert st._committed[False] == 2 * BLOCK - assert len(kv.window_block_ids(st.pos)) == 2 + assert st.adapter(POS).completed_chunks == 2 + assert st._committed[POS] == 2 * BLOCK + assert len(kv.window_block_ids(st.adapter(POS))) == 2 def test_scratch_paged_context_does_not_commit(): kv, st = make_state(num_layers=1) - contexts = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False) + contexts = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False) ctx = contexts[0].forward_ctx ctx.ensure_video_slots(torch.device("cpu")) - assert ctx.current_video_block_ids == kv.scratch_block_ids(False, 0, 1) - st.commit_paged_context(False) + assert ctx.current_video_block_ids == kv.scratch_block_ids(POS, 0, 1) + st.commit_paged_context(POS) - assert st.pos.completed_chunks == 0 - assert st._committed[False] == 0 - assert kv.window_block_ids(st.pos) == [] + assert st.adapter(POS).completed_chunks == 0 + assert st._committed[POS] == 0 + assert kv.window_block_ids(st.adapter(POS)) == [] def test_eviction_bounds_resident_window(): kv, st = make_state(num_layers=1, window_chunks=3) - _prepare_and_commit(st, False, 3) - assert len(kv.window_block_ids(st.pos)) == 3 + _prepare_and_commit(st, POS, 3) + assert len(kv.window_block_ids(st.adapter(POS))) == 3 - _prepare_and_commit(st, False, 2) - ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=False)[0].forward_ctx + _prepare_and_commit(st, POS, 2) + ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=False)[0].forward_ctx visible_blocks, video_len = ctx.video_block_table(torch.device("cpu")) assert len(visible_blocks) == 3 assert video_len == 3 * BLOCK @@ -96,53 +104,47 @@ def test_eviction_bounds_resident_window(): def test_branches_are_independent(): kv, st = make_state() - _prepare_and_commit(st, False, 1) - assert len(kv.window_block_ids(st.pos)) == 1 - assert kv.window_block_ids(st.neg) == [] + _prepare_and_commit(st, POS, 1) + assert len(kv.window_block_ids(st.adapter(POS))) == 1 + assert kv.window_block_ids(st.adapter(NEG)) == [] def test_reset_clears_session_window(): kv, st = make_state(num_layers=1, window_chunks=4) - _prepare_and_commit(st, False, 2) - _prepare_and_commit(st, True, 1) - assert st._committed[False] == 2 * BLOCK and st._committed[True] == BLOCK + _prepare_and_commit(st, POS, 2) + _prepare_and_commit(st, NEG, 1) + assert st._committed[POS] == 2 * BLOCK and st._committed[NEG] == BLOCK free_before = kv.manager.block_pool.get_num_free_blocks() st.reset() - assert st._committed == {False: 0, True: 0} - assert st._paged_pending == {False: None, True: None} - assert kv.window_block_ids(st.pos) == [] - assert kv.window_block_ids(st.neg) == [] + assert st._committed == {POS: 0, NEG: 0} + assert st._paged_pending == {POS: None, NEG: None} + assert kv.window_block_ids(st.adapter(POS)) == [] + assert kv.window_block_ids(st.adapter(NEG)) == [] assert kv.manager.block_pool.get_num_free_blocks() > free_before - _prepare_and_commit(st, False, 1) - assert len(kv.window_block_ids(st.pos)) == 1 + _prepare_and_commit(st, POS, 1) + assert len(kv.window_block_ids(st.adapter(POS))) == 1 -def test_get_cross_kv_caches_raises_before_populate(): - """Engine ownership guard: a cross-attn read before _kv_populate_cross must fail loud.""" - _, st = make_state() # cross_attn_length == 0, nothing populated - for neg in (False, True): - with pytest.raises(RuntimeError, match="cross-attn read before"): - st.get_cross_kv_caches(neg) - # Guards the full AND: the populated flag alone is not enough without a pool. - st._cross_text_populated[False] = True - with pytest.raises(RuntimeError, match="cross-attn read before"): - st.get_cross_kv_caches(False) +def test_get_cross_attention_kv_raises_before_populate(): + _, st = make_state(cross_attn_length=8) + for kv_branch in (POS, NEG): + with pytest.raises(RuntimeError, match="before it was populated"): + st.get_cross_attention_kv(kv_branch, "text") -def test_get_cross_kv_caches_returns_pool_dicts_when_populated(): +def test_get_cross_attention_kv_returns_session_pool_dicts_when_populated(): L = 8 kv, st = make_state(num_layers=2, cross_attn_length=L) written = [] - for i in range(2): + for _ in range(2): k = torch.randn(1, L, N_HEADS, HEAD_DIM) v = torch.randn(1, L, N_HEADS, HEAD_DIM) - kv.write_cross_kv(i, False, k, v) written.append((k, v)) - st._cross_text_populated[False] = True + st.populate_cross_attention(POS, "text", written) - out = st.get_cross_kv_caches(False) + out = st.get_cross_attention_kv(POS, "text") assert len(out) == 2 for i, (k, v) in enumerate(written): assert out[i]["is_init"] is True @@ -151,6 +153,99 @@ def test_get_cross_kv_caches_returns_pool_dicts_when_populated(): assert torch.equal(out[i]["v"], v) +def test_window_reset_can_keep_named_cross_attention_cache(): + kv, st = make_state(num_layers=1, cross_attn_length=8) + k = torch.randn(1, 8, N_HEADS, HEAD_DIM) + v = torch.randn(1, 8, N_HEADS, HEAD_DIM) + st.populate_cross_attention(POS, "text", [(k, v)]) + old_adapter = st.adapter(POS) + + st.reset(keep_cross_attention=("text",)) + + assert st.adapter(POS) is not old_adapter + assert torch.equal(st.get_cross_attention_kv(POS, "text")[0]["k"], k) + assert "s1" in kv._cross_sessions + + st.reset() + assert "s1" not in kv._cross_sessions + assert not st.is_cross_attention_populated(POS, "text") + + +def test_cross_attention_uses_logical_branch_when_local_slot_is_shared(): + _, st = make_state(num_layers=2, cross_attn_length=8, shared_local_index=True) + assert st.kv_cache.num_local_kv_branches == 1 + pos_layers = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), 1.0), torch.full((1, 8, N_HEADS, HEAD_DIM), 2.0)) for _ in range(2) + ] + neg_layers = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), 3.0), torch.full((1, 8, N_HEADS, HEAD_DIM), 4.0)) for _ in range(2) + ] + + st.populate_cross_attention(POS, "text", pos_layers) + st.populate_cross_attention(NEG, "text", neg_layers) + + for layer in st.get_cross_attention_kv(POS, "text"): + assert torch.count_nonzero(layer["k"] != 1.0) == 0 + assert torch.count_nonzero(layer["v"] != 2.0) == 0 + for layer in st.get_cross_attention_kv(NEG, "text"): + assert torch.count_nonzero(layer["k"] != 3.0) == 0 + assert torch.count_nonzero(layer["v"] != 4.0) == 0 + + +def test_cross_attention_population_is_transactional(): + kv, st = make_state(num_layers=2, cross_attn_length=8) + original = [ + (torch.full((1, 8, N_HEADS, HEAD_DIM), value), torch.full((1, 8, N_HEADS, HEAD_DIM), -value)) + for value in (1.0, 2.0) + ] + replacement = ( + torch.full((1, 8, N_HEADS, HEAD_DIM), 9.0), + torch.full((1, 8, N_HEADS, HEAD_DIM), -9.0), + ) + + def interrupted_population(): + yield replacement + raise RuntimeError("projection failed") + + with pytest.raises(RuntimeError, match="projection failed"): + st.populate_cross_attention(POS, "text", interrupted_population()) + assert not st.is_cross_attention_populated(POS, "text") + assert "s1" not in kv._cross_sessions + + st.populate_cross_attention(POS, "text", original) + + with pytest.raises(RuntimeError, match="projection failed"): + st.populate_cross_attention(POS, "text", interrupted_population()) + out = st.get_cross_attention_kv(POS, "text") + for layer_index, layer in enumerate(out): + assert torch.equal(layer["k"], original[layer_index][0]) + assert torch.equal(layer["v"], original[layer_index][1]) + + for invalid_layers in (original[:1], [*original, replacement]): + with pytest.raises(ValueError, match="expected 2 layers"): + st.populate_cross_attention(POS, "text", invalid_layers) + out = st.get_cross_attention_kv(POS, "text") + for layer_index, layer in enumerate(out): + assert torch.equal(layer["k"], original[layer_index][0]) + assert torch.equal(layer["v"], original[layer_index][1]) + + invalid_shape = (torch.empty(1, 7, N_HEADS, HEAD_DIM), original[0][1]) + with pytest.raises(ValueError, match="expected k/v shape"): + st.populate_cross_attention(POS, "text", [invalid_shape, original[1]]) + out = st.get_cross_attention_kv(POS, "text") + assert torch.equal(out[0]["k"], original[0][0]) + + +def test_closed_state_cannot_repopulate_cross_attention(): + kv, st = make_state(num_layers=1, cross_attn_length=8) + layer = (torch.ones(1, 8, N_HEADS, HEAD_DIM), torch.ones(1, 8, N_HEADS, HEAD_DIM)) + st.close() + + with pytest.raises(RuntimeError, match="is closed"): + st.populate_cross_attention(POS, "text", [layer]) + assert "s1" not in kv._cross_sessions + + def test_kv_create_is_noop_engine_owns_allocation(): from unittest.mock import MagicMock @@ -166,57 +261,17 @@ def test_kv_create_is_noop_engine_owns_allocation(): def test_get_kv_cache_requires_frame_aligned_seqlen(): _, st = make_state() with pytest.raises(AssertionError, match="frame-aligned"): - st.get_kv_caches(False, seq_len=BLOCK + 1, commit_current=True) + st.get_kv_caches(POS, seq_len=BLOCK + 1, commit_current=True) def test_prepare_one_branch_allocates_nothing_for_the_other(): - """CFG-parallel laziness: a rank prepares only the branch it runs.""" + """CFG-parallel laziness: a rank prepares only the kv_branch it runs.""" kv, st = make_state(num_layers=1, window_chunks=4) - pos_ctx = st.get_kv_caches(False, seq_len=BLOCK, commit_current=True)[0].forward_ctx - neg_ctx = st.get_kv_caches(True, seq_len=BLOCK, commit_current=True)[0].forward_ctx + pos_ctx = st.get_kv_caches(POS, seq_len=BLOCK, commit_current=True)[0].forward_ctx + neg_ctx = st.get_kv_caches(NEG, seq_len=BLOCK, commit_current=True)[0].forward_ctx pos_ctx.prepare(device=torch.device("cpu"), action_len=0, query_len=BLOCK) assert pos_ctx._allocated_video assert not neg_ctx._allocated_video - assert kv.window_block_ids(st.neg) == [] - - -def test_failed_forward_tears_down_session(monkeypatch): - """Review follow-up (hsliuustc0106): a forward that dies partway must not - leave allocated-but-uncommitted KV behind — the session is torn down (pool - blocks freed, model-local state dropped) and the error propagates.""" - from types import SimpleNamespace - - from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner - from vllm_omni.experimental.ar_diffusion.runner import ARDiffusionModelRunner - - kv, st = make_state(num_layers=1, window_chunks=4) - _prepare_and_commit(st, False, 2) # session owns pool blocks - free_before_failure = kv.manager.block_pool.get_num_free_blocks() - - runner = object.__new__(ARDiffusionModelRunner) - runner.kv_cache = kv - runner._ar_diffusion_states = __import__("collections").OrderedDict({"s1": st}) - runner._max_ar_diffusion_states = 4 - runner.pipeline = SimpleNamespace(_states={"s1": object()}, _ar_diffusion_kv_state=None) - runner.device = None - runner._perf_e2e_times = [] - - def boom(self, req, kv_prefetch_job=None): - raise RuntimeError("layer 17 exploded") - - monkeypatch.setattr(DiffusionModelRunner, "execute_model", boom) - req = SimpleNamespace( - request_id="r0", - sampling_params=SimpleNamespace(extra_args={"session_id": "s1"}), - ) - - with pytest.raises(RuntimeError, match="layer 17 exploded"): - ARDiffusionModelRunner.execute_model(runner, req) - - assert "s1" not in runner._ar_diffusion_states - assert "s1" not in runner.pipeline._states - assert runner.pipeline._ar_diffusion_kv_state is None - # close() freed the session's resident blocks. - assert kv.manager.block_pool.get_num_free_blocks() > free_before_failure + assert kv.window_block_ids(st.adapter(NEG)) == [] diff --git a/tests/dreamzero/test_pipeline_state.py b/tests/dreamzero/test_pipeline_state.py index f00caf53c..366876ef7 100644 --- a/tests/dreamzero/test_pipeline_state.py +++ b/tests/dreamzero/test_pipeline_state.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections import OrderedDict +from types import SimpleNamespace import pytest @@ -14,7 +15,6 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _empty_pipeline() -> DreamZeroPipeline: pipeline = DreamZeroPipeline.__new__(DreamZeroPipeline) pipeline._states = OrderedDict() - pipeline._max_session_states = 2 return pipeline @@ -32,17 +32,59 @@ def test_dreamzero_pipeline_state_is_session_keyed() -> None: assert session_b.call_count == 3 -def test_dreamzero_pipeline_state_lru_caps_retained_sessions() -> None: +def test_dreamzero_pipeline_state_follows_runner_lifecycle_notifications() -> None: pipeline = _empty_pipeline() session_a = pipeline._get_or_create_state("session-a") - pipeline._get_or_create_state("session-b") - assert pipeline._get_or_create_state("session-a") is session_a + session_b = pipeline._get_or_create_state("session-b") + pipeline.state = session_b - pipeline._get_or_create_state("session-c") + pipeline.close_ar_diffusion_session("session-a") + assert pipeline.state is session_b + pipeline.reset_ar_diffusion_session("session-b") - assert list(pipeline._states) == ["session-a", "session-c"] - assert "session-b" not in pipeline._states + assert not pipeline._states + assert pipeline.state is None + assert pipeline._get_or_create_state("session-a") is not session_a + assert pipeline._get_or_create_state("session-b") is not session_b + + +def test_close_session_clears_active_state_alias() -> None: + pipeline = _empty_pipeline() + session_a = pipeline._get_or_create_state("session-a") + pipeline.state = session_a + + pipeline.close_ar_diffusion_session("session-a") + + assert not pipeline._states + assert pipeline.state is None + + +def test_dreamzero_warmup_provider_builds_session_scoped_requests() -> None: + provider = SimpleNamespace( + ar_diffusion_kv_cache_spec=lambda: SimpleNamespace(window_frames=5, frames_per_block=4), + od_config=SimpleNamespace( + ar_diffusion_kv_config=None, + model_config={"policy_server_config": {"image_resolution": [8, 16]}}, + ), + _ar_warmup_robot_obs=DreamZeroPipeline._ar_warmup_robot_obs, + ) + + requests = list(DreamZeroPipeline.ar_diffusion_warmup_requests(provider, "warmup-session")) + + assert [request.prompt for request in requests] == ["warmup", "warmup"] + assert all(request.sampling_params.extra_args["session_id"] == "warmup-session" for request in requests) + assert requests[0].sampling_params.extra_args["robot_obs"]["observation/exterior_image_0_left"].shape == ( + 8, + 16, + 3, + ) + assert requests[1].sampling_params.extra_args["robot_obs"]["observation/exterior_image_0_left"].shape == ( + 4, + 8, + 16, + 3, + ) def test_dreamzero_state_owns_no_kv_caches() -> None: diff --git a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py index d43693b0d..711b345d2 100644 --- a/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py +++ b/vllm_omni/diffusion/models/dreamzero/pipeline_dreamzero.py @@ -11,10 +11,12 @@ from __future__ import annotations import copy import json import logging +import math import os import re as re_module from collections import OrderedDict from collections.abc import Iterable +from contextlib import contextmanager import numpy as np import torch @@ -52,7 +54,14 @@ from vllm_omni.diffusion.models.dreamzero.utils import ( DEFAULT_SIGMA_SHIFT, ) from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import FlowUniPCMultistepScheduler +from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, +) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams logger = logging.getLogger(__name__) MAX_DREAMZERO_SESSIONS = 64 @@ -92,18 +101,128 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): """DreamZero world model pipeline. Multi-output: predict_noise() returns (video_pred, action_pred). - CFG: video gets standard CFG, action takes positive branch only. + CFG: video gets standard CFG, action takes the positive KV branch only. - KV is managed by the AR-Diffusion engine: ``self._ar_diffusion_kv_state`` is set by the - runner before ``forward()`` and the pipeline routes all KV access (get / update / - commit / reset) through the pool-backed state. Purely duck-typed (no engine import). + KV is managed by the AR-Diffusion engine through the explicit capability + methods below. The runner binds one session state only for ``forward()``. """ - _ar_diffusion_kv_state = None # set by the runner before each forward + _POSITIVE_BRANCH = "positive" + _NEGATIVE_BRANCH = "negative" + _ar_diffusion_kv_state = None + state: DreamZeroState | None + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + """Describe DreamZero's local KV geometry to the generic runner.""" + transformer = self.transformer + frame_tokens = int(transformer.frame_seqlen) + max_attention_tokens = int(transformer.blocks[0].self_attn.max_attention_size) + cfg_world = int(get_classifier_free_guidance_world_size()) + negative_local_index = 0 if cfg_world >= 2 else 1 + cross_attention = [ + ARDiffusionCrossAttentionKVSpec("text", int(transformer.text_len)), + ] + if transformer.model_type == "i2v": + cross_attention.append(ARDiffusionCrossAttentionKVSpec("image", 257)) + return ARDiffusionKVCacheSpec( + num_layers=int(transformer.num_layers), + num_kv_heads=int(transformer.blocks[0].self_attn.tp_num_heads), + head_size=int(transformer.dim // transformer.num_heads), + tokens_per_frame=frame_tokens, + frames_per_block=int(transformer.num_frame_per_block), + window_frames=max_attention_tokens // frame_tokens, + kv_branches=( + ARDiffusionKVBranchSpec(self._POSITIVE_BRANCH, 0), + ARDiffusionKVBranchSpec(self._NEGATIVE_BRANCH, negative_local_index), + ), + session_capacity=MAX_DREAMZERO_SESSIONS, + cross_attention=tuple(cross_attention), + max_scratch_tokens_per_branch=int(transformer.num_action_per_block + transformer.num_state_per_block), + ) + + @contextmanager + def bind_ar_diffusion_state(self, session_id, state): + """Bind runner-owned KV only for the duration of one forward.""" + if self._ar_diffusion_kv_state is not None: + raise RuntimeError("DreamZero AR-Diffusion state is already bound") + if state.session_id != session_id: + raise ValueError(f"DreamZero bound session mismatch: {state.session_id!r} != {session_id!r}") + self._ar_diffusion_kv_state = state + try: + yield + finally: + self._ar_diffusion_kv_state = None + + def reset_ar_diffusion_session(self, session_id: str) -> None: + """Reset DreamZero-owned state after the runner releases session KV.""" + self._drop_ar_diffusion_session_state(session_id) + + def close_ar_diffusion_session(self, session_id: str) -> None: + """Drop DreamZero-owned state after close, eviction, or failed forward.""" + self._drop_ar_diffusion_session_state(session_id) + + def _drop_ar_diffusion_session_state(self, session_id: str) -> None: + """Remove model state and clear the compatibility alias when it points there.""" + removed = self._states.pop(str(session_id or "default"), None) + if removed is not None and getattr(self, "state", None) is removed: + self.state = None + + @staticmethod + def _ar_warmup_robot_obs(height: int, width: int, n_frames: int, session_id: str) -> dict: + image = ( + np.zeros((height, width, 3), dtype=np.uint8) + if n_frames == 1 + else np.zeros((n_frames, height, width, 3), dtype=np.uint8) + ) + return { + "observation/exterior_image_0_left": image, + "observation/exterior_image_1_left": image, + "observation/wrist_image_left": image, + "observation/joint_position": np.zeros(7, dtype=np.float32), + "observation/cartesian_position": np.zeros(6, dtype=np.float32), + "observation/gripper_position": np.zeros(1, dtype=np.float32), + "prompt": "warmup", + "session_id": session_id, + } + + def ar_diffusion_warmup_requests(self, session_id: str) -> Iterable[OmniDiffusionRequest]: + """Yield DreamZero-valid requests covering each resident window shape.""" + spec = self.ar_diffusion_kv_cache_spec() + n_forwards = 1 + math.ceil(max(0, spec.window_frames - 1) / spec.frames_per_block) + raw_kv_config = getattr(self.od_config, "ar_diffusion_kv_config", None) + if raw_kv_config is None and isinstance(self.od_config.model_config, dict): + raw_kv_config = self.od_config.model_config.get("ar_diffusion_kv_config") + capture_reset = ( + bool(raw_kv_config.get("warmup_capture_reset", False)) + if isinstance(raw_kv_config, dict) + else bool(getattr(raw_kv_config, "warmup_capture_reset", False)) + ) + if capture_reset: + n_forwards += 1 + policy_config = self.od_config.model_config.get("policy_server_config", {}) + height, width = (int(value) for value in policy_config.get("image_resolution", [180, 320])) + for index in range(n_forwards): + n_frames = 1 if index == 0 else 4 + robot_obs = self._ar_warmup_robot_obs(height, width, n_frames, session_id) + sampling_params = OmniDiffusionSamplingParams( + extra_args={ + "reset": index == 0, + "session_id": session_id, + "robot_obs": robot_obs, + } + ) + yield OmniDiffusionRequest( + prompt="warmup", + sampling_params=sampling_params, + request_id=f"ardiffusion-warmup-{index}", + ) + + def _ar_branch(self, is_negative: bool) -> str: + return self._NEGATIVE_BRANCH if is_negative else self._POSITIVE_BRANCH def _kv_get(self, state, is_negative, seq_len=None, update_kv_cache=False): return self._ar_diffusion_kv_state.get_kv_caches( - is_negative, + self._ar_branch(is_negative), seq_len=seq_len, commit_current=update_kv_cache, ) @@ -115,11 +234,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): return def _kv_commit(self, is_negative: bool): - self._ar_diffusion_kv_state.commit_paged_context(is_negative) + self._ar_diffusion_kv_state.commit_paged_context(self._ar_branch(is_negative)) def _kv_get_cross(self, state, is_negative): """Cross-attn cache from the engine pool (text k/v + I2V image k_img/v_img).""" - return self._ar_diffusion_kv_state.get_cross_kv_caches(is_negative) + kv_branch = self._ar_branch(is_negative) + text_caches = self._ar_diffusion_kv_state.get_cross_attention_kv(kv_branch, "text") + if "image" in self._ar_diffusion_kv_state.kv_cache.cross_attention_lengths: + image_caches = self._ar_diffusion_kv_state.get_cross_attention_kv(kv_branch, "image") + for text_cache, image_cache in zip(text_caches, image_caches, strict=True): + text_cache["k_img"] = image_cache["k"] + text_cache["v_img"] = image_cache["v"] + return text_caches def _kv_populate_cross(self, context: torch.Tensor, clip_feature, is_negative: bool) -> None: """Eagerly project cross-attn K/V for all layers into the AR-Diffusion pool. @@ -133,32 +259,41 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): after the image is encoded so ``clip_feature`` is available. """ s = self._ar_diffusion_kv_state - need_text = not s._cross_text_populated.get(is_negative, False) + kv_branch = self._ar_branch(is_negative) + need_text = not s.is_cross_attention_populated(kv_branch, "text") need_img = ( clip_feature is not None - and getattr(self.transformer, "model_type", "t2v") == "i2v" - and not s._cross_img_populated.get(is_negative, False) + and self.transformer.model_type == "i2v" + and not s.is_cross_attention_populated(kv_branch, "image") ) if not need_text and not need_img: return projected = self.transformer.text_embedding(context) if need_text else None img_ctx = self.transformer.img_emb(clip_feature) if need_img else None - for i, block in enumerate(self.transformer.blocks): - ca = block.cross_attn - n, d = ca.tp_num_heads, ca.head_dim - k = v = None - if projected is not None: - k = ca.norm_k(ca.k(projected)).unflatten(2, (n, d)) - v = ca.v(projected).unflatten(2, (n, d)) - k_img = v_img = None - if img_ctx is not None: - k_img = ca.norm_k_img(ca.k_img(img_ctx)).unflatten(2, (n, d)) - v_img = ca.v_img(img_ctx).unflatten(2, (n, d)) - s.kv_cache.write_cross_kv(i, is_negative, k, v, k_img, v_img) - if need_text: - s._cross_text_populated[is_negative] = True - if need_img: - s._cross_img_populated[is_negative] = True + if projected is not None: + + def text_layer_kv(): + for block in self.transformer.blocks: + ca = block.cross_attn + n, d = ca.tp_num_heads, ca.head_dim + yield ( + ca.norm_k(ca.k(projected)).unflatten(2, (n, d)), + ca.v(projected).unflatten(2, (n, d)), + ) + + s.populate_cross_attention(kv_branch, "text", text_layer_kv()) + if img_ctx is not None: + + def image_layer_kv(): + for block in self.transformer.blocks: + ca = block.cross_attn + n, d = ca.tp_num_heads, ca.head_dim + yield ( + ca.norm_k_img(ca.k_img(img_ctx)).unflatten(2, (n, d)), + ca.v_img(img_ctx).unflatten(2, (n, d)), + ) + + s.populate_cross_attention(kv_branch, "image", image_layer_kv()) logger.info( "AR-Diffusion CROSS POPULATE [%s]: %d layers, text=%s img=%s", "neg" if is_negative else "pos", @@ -179,7 +314,8 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): image half repopulates on the restart forward. """ state.reset(clear_video_latents=clear_video_latents) - self._ar_diffusion_kv_state.reset(keep_cross_text=not clear_video_latents) + keep_cross = ("text",) if not clear_video_latents else () + self._ar_diffusion_kv_state.reset(keep_cross_attention=keep_cross) def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: """Initialize pipeline components. @@ -298,7 +434,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): ) self._states: OrderedDict[str, DreamZeroState] = OrderedDict() - self._max_session_states = MAX_DREAMZERO_SESSIONS self.state = self._get_or_create_state("default") # DiT step cache is configured by StepCacheBackend @@ -373,9 +508,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if state is None: state = DreamZeroState() self._states[session_key] = state - max_states = getattr(self, "_max_session_states", MAX_DREAMZERO_SESSIONS) - while len(self._states) > max_states: - self._states.popitem(last=False) else: self._states.move_to_end(session_key) return state @@ -536,6 +668,10 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if not torch.cuda.is_available(): return + state = self.state + if state is None: + state = self._get_or_create_state("default") + self.state = state device = next(self.text_encoder.parameters()).device with torch.inference_mode(): try: @@ -547,18 +683,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): try: image = torch.zeros(1, 1, 3, 180, 320, dtype=torch.bfloat16, device=device) - self._encode_image(image, self.num_frames, 180, 320, state=self.state) + self._encode_image(image, self.num_frames, 180, 320, state=state) except Exception as exc: logger.warning("DreamZero compile warmup (image_encoder) skipped: %s", exc) try: - self.state.reset_vae_encoder_stream() + state.reset_vae_encoder_stream() dummy_video = torch.zeros(1, 3, 1, 180, 320, dtype=torch.bfloat16, device=device) - self._vae_stream_seed(self.state, dummy_video[:, :, :1]) + self._vae_stream_seed(state, dummy_video[:, :, :1]) for _ in range(4): - self._vae_stream_append_frame(self.state, dummy_video[:, :, :1]) + self._vae_stream_append_frame(state, dummy_video[:, :, :1]) self._vae_stream_get_observation_latents( - self.state, + state, self.num_frame_per_block, dtype=torch.bfloat16, ) @@ -1034,7 +1170,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): state_features = kwargs.get("state_features") embodiment_id = kwargs.get("embodiment_id") - # Shared kwargs for predict_noise (both cond & uncond branches) + # Shared kwargs for predict_noise (both conditional and unconditional branches) common_kwargs = dict( seq_len=seq_len, current_start_frame=state.current_start_frame, @@ -1236,21 +1372,14 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): text_tokens = text_inputs["input_ids"].to(device) attention_mask = text_inputs["attention_mask"].to(device) - # Explicit reset from OpenPI serving is carried by `extra_args["reset"]` - # on the next inference request after websocket reset/session switch. - if extra_args.get("reset", False): + # Explicit request reset is handled by ARDiffusionModelRunner before it + # binds this forward. Model-detected prompt/window resets still happen + # here because they depend on tokenized DreamZero state. + reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size) + if reset_reason == "session": self._kv_reset(state) - else: - # Auto-reset based on model state (before accumulation). Both a "session" - # reset and an "inference" (window-boundary) reset clear the model-local - # KV window, so route both through _kv_reset to drop the AR-Diffusion pool window - # too and stay parity-exact; "inference" keeps the accumulated video - # latents (reset_inference_state == reset(clear_video_latents=False)). - reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size) - if reset_reason == "session": - self._kv_reset(state) - elif reset_reason == "inference": - self._kv_reset(state, clear_video_latents=False) + elif reset_reason == "inference": + self._kv_reset(state, clear_video_latents=False) state.language = text_tokens # Frame accumulation: stitched single frame -> multi-frame video @@ -1280,7 +1409,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin): if state.prompt_embeds is None: state.prompt_embeds = self._encode_text(text_tokens, attention_mask) prompt_embeds = state.prompt_embeds - # Negative prompt for CFG uncond branch (model constant) + # Negative prompt for the unconditional CFG branch (model constant) negative_prompt_embeds = None if self.cfg_scale > 1.0: if self._negative_prompt_embeds_cache is None: diff --git a/vllm_omni/experimental/ar_diffusion/__init__.py b/vllm_omni/experimental/ar_diffusion/__init__.py index b312660ec..b87ac1ccc 100644 --- a/vllm_omni/experimental/ar_diffusion/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/__init__.py @@ -2,11 +2,24 @@ """AR-Diffusion Engine (AR-Diffusion). The AR-Diffusion engine: a ``DiffusionEngine`` subclass that adds engine-level KV -cache management for autoregressive / chunked "world-model" diffusion models -(DreamZero and the AR-DiT family). Selected via +cache management for autoregressive / chunked diffusion models. Selected via ``OmniDiffusionConfig.engine_backend = "ar_diffusion"``. """ +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionCrossAttentionKVSpec, + ARDiffusionKVBranchSpec, + ARDiffusionKVCacheSpec, + SupportsARDiffusionPipeline, + SupportsARDiffusionWarmup, +) from vllm_omni.experimental.ar_diffusion.engine import ARDiffusionEngine -__all__ = ["ARDiffusionEngine"] +__all__ = [ + "ARDiffusionKVBranchSpec", + "ARDiffusionCrossAttentionKVSpec", + "ARDiffusionEngine", + "ARDiffusionKVCacheSpec", + "SupportsARDiffusionPipeline", + "SupportsARDiffusionWarmup", +] diff --git a/vllm_omni/experimental/ar_diffusion/capability.py b/vllm_omni/experimental/ar_diffusion/capability.py new file mode 100644 index 000000000..b30d10c1e --- /dev/null +++ b/vllm_omni/experimental/ar_diffusion/capability.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed pipeline contract for the model-neutral AR-Diffusion runtime.""" + +from __future__ import annotations + +from collections.abc import Iterable +from contextlib import AbstractContextManager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState + + +@dataclass(frozen=True) +class ARDiffusionKVBranchSpec: + """One logical model KV branch and its worker-local storage slot. + + Distinct KV branches may share ``local_index`` when distributed execution + guarantees that at most one of them runs on a worker. For example, two CFG + branches use indices ``0, 1`` without CFG parallelism and may both use index + ``0`` when each worker executes only one branch. + """ + + name: str + local_index: int + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("AR-Diffusion KV branch names must be non-empty") + if self.local_index < 0: + raise ValueError(f"AR-Diffusion KV branch local_index must be non-negative, got {self.local_index}") + + +@dataclass(frozen=True) +class ARDiffusionCrossAttentionKVSpec: + """A fixed-length, per-layer cross-attention KV allocation.""" + + name: str + num_tokens: int + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("AR-Diffusion cross-attention cache names must be non-empty") + if self.num_tokens <= 0: + raise ValueError(f"AR-Diffusion cross-attention num_tokens must be positive, got {self.num_tokens}") + + +@dataclass(frozen=True) +class ARDiffusionKVCacheSpec: + """KV geometry and session policy supplied by an AR-Diffusion pipeline. + + ``tokens_per_frame`` is also the paged KV block size. A model forward may + commit ``frames_per_block`` such blocks at once. ``window_frames`` is the + recent sliding tail and ``sink_frames`` is the separately retained prefix; + both are expressed in the same frame/block unit. + + ``max_scratch_tokens_per_branch`` is the maximum non-video KV (for example, + action/state registers) that must coexist with an uncommitted video block. + ``session_capacity`` is the pipeline's upper bound; a runner may retain + fewer sessions when its current memory and execution policy is stricter. + + The runner owns all pool allocations and session lifetime. The pipeline + owns its non-KV model state and receives reset/close notifications through + :class:`SupportsARDiffusionPipeline`. + """ + + num_layers: int + num_kv_heads: int + head_size: int + tokens_per_frame: int + frames_per_block: int + window_frames: int + kv_branches: tuple[ARDiffusionKVBranchSpec, ...] + session_capacity: int + sink_frames: int = 0 + reset_at_boundary: bool = False + cross_attention: tuple[ARDiffusionCrossAttentionKVSpec, ...] = () + max_model_len: int = 1 << 20 + max_scratch_tokens_per_branch: int = 0 + + def __post_init__(self) -> None: + positive_fields = { + "num_layers": self.num_layers, + "num_kv_heads": self.num_kv_heads, + "head_size": self.head_size, + "tokens_per_frame": self.tokens_per_frame, + "frames_per_block": self.frames_per_block, + "window_frames": self.window_frames, + "session_capacity": self.session_capacity, + "max_model_len": self.max_model_len, + } + for name, value in positive_fields.items(): + if value <= 0: + raise ValueError(f"AR-Diffusion {name} must be positive, got {value}") + if self.sink_frames < 0: + raise ValueError(f"AR-Diffusion sink_frames must be non-negative, got {self.sink_frames}") + if self.max_scratch_tokens_per_branch < 0: + raise ValueError( + "AR-Diffusion max_scratch_tokens_per_branch must be non-negative, " + f"got {self.max_scratch_tokens_per_branch}" + ) + if not self.kv_branches: + raise ValueError("AR-Diffusion requires at least one KV branch") + kv_branch_names = [kv_branch.name for kv_branch in self.kv_branches] + if len(kv_branch_names) != len(set(kv_branch_names)): + raise ValueError(f"AR-Diffusion KV branch names must be unique, got {kv_branch_names}") + local_indices = {kv_branch.local_index for kv_branch in self.kv_branches} + if local_indices != set(range(max(local_indices) + 1)): + raise ValueError( + f"AR-Diffusion KV branch local_index values must be contiguous from zero, got {sorted(local_indices)}" + ) + cross_names = [cache.name for cache in self.cross_attention] + if len(cross_names) != len(set(cross_names)): + raise ValueError(f"AR-Diffusion cross-attention cache names must be unique, got {cross_names}") + + @property + def num_local_kv_branches(self) -> int: + """Number of KV branch storage slots allocated on this worker.""" + return max(kv_branch.local_index for kv_branch in self.kv_branches) + 1 + + @property + def cross_attention_lengths(self) -> dict[str, int]: + return {cache.name: cache.num_tokens for cache in self.cross_attention} + + +@runtime_checkable +class SupportsARDiffusionPipeline(Protocol): + """Required pipeline capability for :class:`ARDiffusionModelRunner`. + + A session begins when the runner first sees its ``session_id`` and persists + across requests until reset, explicit close, LRU eviction, or a failed + forward. ``bind_ar_diffusion_state`` exposes the runner-owned KV state only + for the duration of one request. The pipeline must not retain the state + after the context exits. + """ + + def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec: + """Return immutable KV geometry and session policy for this worker.""" + ... + + def bind_ar_diffusion_state( + self, + session_id: str, + state: ARDiffusionKVState, + ) -> AbstractContextManager[None]: + """Bind ``state`` to model execution for one request.""" + ... + + def reset_ar_diffusion_session(self, session_id: str) -> None: + """Reset model-owned state for ``session_id`` after KV release.""" + ... + + def close_ar_diffusion_session(self, session_id: str) -> None: + """Drop model-owned state after close, eviction, or failed forward.""" + ... + + +@runtime_checkable +class SupportsARDiffusionWarmup(Protocol): + """Optional provider of model-valid warmup requests. + + Pipelines without this capability are loaded without an AR rollout warmup. + """ + + def ar_diffusion_warmup_requests(self, session_id: str) -> Iterable[OmniDiffusionRequest]: + """Yield requests for compiled shapes, each carrying ``session_id``.""" + ... diff --git a/vllm_omni/experimental/ar_diffusion/engine.py b/vllm_omni/experimental/ar_diffusion/engine.py index 6fb6ee7c9..52ef6c9d0 100644 --- a/vllm_omni/experimental/ar_diffusion/engine.py +++ b/vllm_omni/experimental/ar_diffusion/engine.py @@ -20,8 +20,7 @@ class ARDiffusionEngine(DiffusionEngine): (world models, AR-DiT) that materialize persistent attention KV. It reuses vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / ``BlockTables``) as a library, driven from the engine rather than hand-rolled inside each - model. See ``BDE_doc/diffusion_kv_cache_management.md`` for the design and - ``BDE_doc/dreamzero_kv_phase1_plan.md`` for the rollout. + model. It is selected per model via ``OmniDiffusionConfig.engine_backend = "ar_diffusion"`` (resolved by :meth:`DiffusionEngine.make_engine`), so models that do not opt @@ -32,7 +31,7 @@ class ARDiffusionEngine(DiffusionEngine): actual KV *body* is :class:`~vllm_omni.experimental.ar_diffusion.kv_cache.manager.ARDiffusionKVCache`, owned by :class:`~vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner`. ``ARDiffusionEngine`` itself is the thin selection / injection seam; it wires the AR-Diffusion executor → worker → runner - so DreamZero's rollout runs against the runner-owned KV cache. + so capable pipelines run against runner-owned KV session state. """ #: Workers of this engine build the AR-Diffusion runner (unless the config diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py index ab29f6093..b407d24a3 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/__init__.py @@ -3,7 +3,7 @@ Thin glue over vLLM's paged KV stack (``KVCacheManager`` / ``BlockPool`` / ``SlidingWindowManager``), used by the AR-Diffusion Engine to manage KV for -AR-diffusion models. See ``BDE_doc/dreamzero_kv_phase1_plan.md``. +AR-diffusion models. Layout: ``config`` (public knob) · ``paged`` (engine-generic paging mechanics + chunk-window eviction spec) · ``manager`` (the ARDiffusionKVCache orchestrator + its diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py index 4b3c73775..07be2546e 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/config.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/config.py @@ -15,14 +15,13 @@ class ARDiffusionKVConfig: """ enable: bool = False - # Persistent KV tokens materialized per AR chunk. For DreamZero this is - # ``num_frame_per_block * frame_seqlen``. + # Persistent KV tokens materialized per paged cache block. chunk_size: int = 0 # Resident window in chunks. ``None`` means full attention (no eviction). window_chunks: int | None = None # Protected leading chunks (attention sink); never evicted. sink_chunks: int = 0 - # DreamZero-style window reset vs. VGGT-style sliding replace. + # Boundary reset vs. sliding replacement. reset_at_boundary: bool = False # Fraction of free device memory budgeted for the AR-Diffusion KV pool. gpu_memory_fraction: float = 0.1 diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py index 4b3b5d788..8c7d4b2ce 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/manager.py @@ -1,20 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 -"""ARDiffusionKVCache — the engine-level KV cache orchestrator for one AR-Diffusion model. +"""Engine-level KV cache orchestration for AR-Diffusion models. This is the *body* of AR-Diffusion's KV management: it owns a vLLM ``KVCacheManager`` (a single chunk-window group) and the per-request adapter lifecycle, and exposes the per-chunk operations a rollout needs — allocate, slot mapping, commit, window lookup, free. It lives in the model runner (worker / GPU side), co-located with -the model and the KV tensors; the DreamZero pipeline calls these methods during a -rollout. The main-process ``ARDiffusionEngine`` only selects the engine and is otherwise -thin. +the model and the KV tensors. The main-process ``ARDiffusionEngine`` only +selects the engine and is otherwise thin. """ from __future__ import annotations import inspect import os -from collections.abc import Sequence +from collections.abc import Collection, Iterable, Sequence import torch from vllm.logger import init_logger @@ -27,6 +26,7 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.request import RequestStatus +from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.paged import ( ChunkWindowSpec, @@ -48,7 +48,7 @@ class ARDiffusionRequestAdapter: real ``KVCacheManager`` against this adapter so the surface cannot silently drift across vLLM versions. - A AR-Diffusion request advances one *chunk* at a time: ``allocate_slots`` is called + An AR-Diffusion request advances one *chunk* at a time: ``allocate_slots`` is called once per chunk and ``num_computed_tokens`` advances only when a chunk is committed (:meth:`on_chunk_committed`), so the ``T`` denoise steps of a chunk reuse the same slots. @@ -145,7 +145,7 @@ def build_kv_manager( class ARDiffusionKVCache: - """Owns the paged KV pool + per-request lifecycle for a AR-Diffusion model. + """Own the paged KV pool and KV-branch-local storage for one model. Build once per loaded model (dimensions known); then per request: ``begin_request`` → per chunk (``allocate_chunk`` → ``chunk_write_slots`` → @@ -163,11 +163,12 @@ class ARDiffusionKVCache: block_size: int, max_model_len: int, available_bytes: int, - cross_attn_length: int = 0, - cross_attn_img_length: int = 0, + kv_branches: tuple[ARDiffusionKVBranchSpec, ...], + session_capacity: int, + cross_attention_lengths: dict[str, int] | None = None, device: torch.device | None = None, - local_branches: int = 2, - num_frame_per_block: int = 1, + frames_per_block: int = 1, + max_scratch_tokens_per_branch: int = 0, ) -> None: if not config.enable: raise ValueError("ARDiffusionKVCache built with a disabled ARDiffusionKVConfig") @@ -175,77 +176,63 @@ class ARDiffusionKVCache: raise ValueError("Phase 1 requires a bounded window (window_chunks)") if config.chunk_size <= 0: raise ValueError("ARDiffusionKVConfig.chunk_size must be set (> 0)") - if local_branches not in (1, 2): - raise ValueError(f"local_branches must be 1 or 2, got {local_branches}") + if not kv_branches: + raise ValueError("ARDiffusionKVCache requires at least one KV branch") + if session_capacity <= 0: + raise ValueError(f"session_capacity must be positive, got {session_capacity}") + kv_branch_names = [kv_branch.name for kv_branch in kv_branches] + if len(kv_branch_names) != len(set(kv_branch_names)): + raise ValueError(f"ARDiffusionKVCache KV branch names must be unique, got {kv_branch_names}") + local_indices = {kv_branch.local_index for kv_branch in kv_branches} + if local_indices != set(range(max(local_indices) + 1)): + raise ValueError( + "ARDiffusionKVCache KV branch local_index values must be contiguous from zero, " + f"got {sorted(local_indices)}" + ) self.config = config - # How many CFG branches allocate from THIS rank's pool. Under - # CFG-parallel x2 each rank executes exactly one branch (rank0 pos, - # rank1 neg) and the other branch's lazy contexts never allocate, so - # sizing for both would leave ~half the pool as idle capacity. A - # single-process run (TP1 / offline) executes both branches -> 2. - self.local_branches = local_branches - self.num_frame_per_block = max(1, int(num_frame_per_block)) + self.kv_branches = kv_branches + self.session_capacity = session_capacity + self._kv_branch_local_indices = {kv_branch.name: kv_branch.local_index for kv_branch in kv_branches} + self.num_local_kv_branches = max(local_indices) + 1 + if frames_per_block <= 0: + raise ValueError(f"frames_per_block must be positive, got {frames_per_block}") + if max_scratch_tokens_per_branch < 0: + raise ValueError(f"max_scratch_tokens_per_branch must be non-negative, got {max_scratch_tokens_per_branch}") + self.frames_per_block = int(frames_per_block) self.block_size = block_size self.num_layers = num_layers self.num_kv_heads = num_kv_heads self.head_size = head_size self.dtype = dtype - self.cross_attn_length = cross_attn_length + self.cross_attention_lengths = dict(cross_attention_lengths or {}) + invalid_cross = {name: length for name, length in self.cross_attention_lengths.items() if length <= 0} + if invalid_cross: + raise ValueError(f"cross_attention_lengths must be positive, got {invalid_cross}") self.device = device or torch.device("cpu") + self._allocate_tensors = device is not None self._adapters: dict[str, ARDiffusionRequestAdapter] = {} - # -- cross-attention pool (static, one-time-fill) ----------------------- - # Cross-attn KV is computed once from the text encoder and never changes. - # Allocate a per-layer contiguous tensor for each branch: (2, text_len, - # kv_heads, head_dim) where dim-0 = [pos, neg]. Allocated separately - # from the self-attn block pool (static, small — ~400 MiB for - # DreamZero I2V). Both pools draw from the same GPU free memory but - # the cross-attn pool is sized directly rather than via the block pool - # budget (no eviction/chunk lifecycle needed). - self.cross_attn_img_length = cross_attn_img_length - self._cross_k: list[torch.Tensor] = [] - self._cross_v: list[torch.Tensor] = [] - # I2V image-token cross-attn pool. Like the text k/v, the image-token - # k_img/v_img are session-invariant (the conditioning image doesn't change), - # so they are cached once and read every denoise step — see - # WanI2VCrossAttention (#4154 caches these model-side too). Empty for T2V. - self._cross_k_img: list[torch.Tensor] = [] - self._cross_v_img: list[torch.Tensor] = [] + # Named cross-attention K/V is allocated lazily per session and released + # with that session. Reserve its worst-case capacity when sizing the + # paged self-attention pool so the two stores share one memory budget. + self._cross_sessions: dict[ + str, + dict[str, dict[str, tuple[list[torch.Tensor], list[torch.Tensor]]]], + ] = {} - # Bytes the cross-attn pools consume directly (K+V, pos+neg, all layers). - # Deducted from the self-attn paged-pool budget below so the two - # allocations together stay within the GPU free-memory budget. def _cross_pool_bytes(length: int) -> int: - return 2 * 2 * length * num_kv_heads * head_size * dtype.itemsize * num_layers + return int(2 * len(self.kv_branches) * length * num_kv_heads * head_size * dtype.itemsize * num_layers) - cross_total_bytes = 0 - if device is not None and cross_attn_length > 0: - cross_shape = (2, cross_attn_length, num_kv_heads, head_size) - cross_bytes = _cross_pool_bytes(cross_attn_length) - cross_total_bytes += cross_bytes - for _ in range(num_layers): - self._cross_k.append(torch.empty(cross_shape, dtype=dtype, device=device)) - self._cross_v.append(torch.empty(cross_shape, dtype=dtype, device=device)) + cross_bytes_per_session = sum(_cross_pool_bytes(length) for length in self.cross_attention_lengths.values()) + cross_total_bytes = cross_bytes_per_session * session_capacity + if cross_total_bytes: _log.info( - "AR-Diffusion cross-attn pool: %d layers × (%d tok × %d heads × %d) = %.1f MiB", - num_layers, - cross_attn_length, - num_kv_heads, - head_size, - cross_bytes / (1024 * 1024), + "AR-Diffusion cross-attn reservation: %.1f MiB/session × %d sessions = %.1f MiB", + cross_bytes_per_session / (1024 * 1024), + session_capacity, + cross_total_bytes / (1024 * 1024), ) - if cross_attn_img_length > 0: - img_shape = (2, cross_attn_img_length, num_kv_heads, head_size) - cross_total_bytes += _cross_pool_bytes(cross_attn_img_length) - for _ in range(num_layers): - self._cross_k_img.append(torch.empty(img_shape, dtype=dtype, device=device)) - self._cross_v_img.append(torch.empty(img_shape, dtype=dtype, device=device)) - _log.info( - "AR-Diffusion cross-attn IMG pool: %d layers × %d img-tok (I2V)", - num_layers, - cross_attn_img_length, - ) self.spec = ChunkWindowSpec( block_size=block_size, @@ -260,46 +247,51 @@ class ARDiffusionKVCache: ) # Each pool block spans all layers' K/V, so size against the per-layer # page size times the layer count. - # Size the self-attn pool against the memory left after the cross-attn - # pools (allocated above), so cross + self-attn stays within the budget. + # Size the self-attn pool against memory reserved for the maximum number + # of lazily allocated cross-attention sessions. num_blocks = compute_num_blocks( max(0, available_bytes - cross_total_bytes), config.gpu_memory_fraction, self.spec.page_size_bytes * num_layers, ) # Floor: one forward needs the resident window plus the in-flight chunk - # (num_frame_per_block frame-blocks) for every branch THIS rank runs, + # (frames_per_block frame-blocks) for every KV branch THIS rank runs, # with a little eviction-transient headroom. The memory-fraction # heuristic can under-size this once block_size grows — e.g. frame-granular # paging at the true frame_seqlen makes each block larger and the pool # fewer-blocks — so guarantee the minimum the rollout cannot run without, # otherwise allocate_chunk hits an exhausted pool mid-forward. - min_blocks = self.local_branches * (config.window_chunks + self.num_frame_per_block) + 2 + resident_blocks = config.sink_chunks + config.window_chunks + min_blocks = self.num_local_kv_branches * (resident_blocks + self.frames_per_block) + 2 if num_blocks < min_blocks: _log.warning( "AR-Diffusion KV pool: memory-fraction sizing gave %d blocks; raising to the %d-block " - "floor (%d local CFG branch(es) x (window_chunks=%d + num_frame_per_block=%d) + 2 headroom)", + "floor (%d local KV branch(es) x (sink_chunks=%d + window_chunks=%d " + "+ frames_per_block=%d) + 2 headroom)", num_blocks, min_blocks, - self.local_branches, + self.num_local_kv_branches, + config.sink_chunks, config.window_chunks, - self.num_frame_per_block, + self.frames_per_block, ) num_blocks = min_blocks layer_names = [f"ar_diffusion.layer.{i}" for i in range(num_layers)] self.manager = build_kv_manager(self.spec, layer_names, num_blocks, max_model_len) self.managed_num_blocks = num_blocks self.num_blocks = num_blocks - # Scratch blocks are outside KVCacheManager ownership. They hold current - # denoise-step video KV when update_kv_cache=False and action/state KV in - # both modes; they are reused every forward and never committed. - # Default 4: measured high-water is num_frame_per_block video blocks (2) - # + 1 action block per branch per forward, +1 margin. - scratch_per_branch = int(os.environ.get("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH", "4")) - if scratch_per_branch <= 0: - raise ValueError("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH must be positive") - self.scratch_blocks_per_branch = scratch_per_branch - self.scratch_num_blocks = self.local_branches * scratch_per_branch + # Scratch blocks are outside KVCacheManager ownership. A non-committing + # forward needs one block per current frame plus space for any + # model-declared action/state tokens that coexist with video KV. + declared_scratch_blocks = (max_scratch_tokens_per_branch + block_size - 1) // block_size + minimum_scratch_blocks = self.frames_per_block + declared_scratch_blocks + override = os.environ.get("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH") + override_blocks = int(override) if override is not None else 0 + if override_blocks < 0: + raise ValueError("AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH must be non-negative") + scratch_per_kv_branch = max(minimum_scratch_blocks, override_blocks) + self.scratch_blocks_per_kv_branch = scratch_per_kv_branch + self.scratch_num_blocks = self.num_local_kv_branches * scratch_per_kv_branch self.num_blocks_total = self.managed_num_blocks + self.scratch_num_blocks self.null_block_id = self.manager.block_pool.null_block.block_id @@ -322,49 +314,147 @@ class ARDiffusionKVCache: # Cross-attn KV is static once populated — write once (from text encoder), # read many (every denoising step). Not managed through the paged block pool. - def write_cross_kv( + def _kv_branch_index(self, kv_branch: str) -> int: + try: + return self._kv_branch_local_indices[kv_branch] + except KeyError as exc: + expected = tuple(self._kv_branch_local_indices) + raise KeyError(f"Unknown AR-Diffusion KV branch {kv_branch!r}; expected {expected}") from exc + + def _cross_attention_pool( self, - layer_idx: int, - is_negative: bool, - k: torch.Tensor | None, - v: torch.Tensor | None, - k_img: torch.Tensor | None = None, - v_img: torch.Tensor | None = None, + session_id: str, + cache_name: str, + kv_branch: str, + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + if cache_name not in self.cross_attention_lengths: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + if session is None: + raise RuntimeError( + f"AR-Diffusion cross-attention cache {cache_name!r} for session {session_id!r} " + "was read before it was populated" + ) + pool = session.get(cache_name, {}).get(kv_branch) + if pool is None: + raise RuntimeError( + f"AR-Diffusion cross-attention cache {cache_name!r} for session {session_id!r} " + f"and KV branch {kv_branch!r} was read before it was populated" + ) + return pool + + def is_cross_attention_populated(self, session_id: str, cache_name: str, kv_branch: str) -> bool: + """Whether a complete logical-branch cache has been published.""" + if cache_name not in self.cross_attention_lengths: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + return session is not None and kv_branch in session.get(cache_name, {}) + + def populate_cross_attention( + self, + session_id: str, + cache_name: str, + kv_branch: str, + layer_kv: Iterable[tuple[torch.Tensor, torch.Tensor]], ) -> None: - """Write one layer's cross-attn K/V into the pool. + """Atomically populate one logical branch of a named cross-attention cache. - ``k`` / ``v``: ``(B, text_len, tp_num_heads, head_dim)``; only batch-0 - is copied (B=1 for inference). ``None`` skips the text half (window - restarts keep the still-valid text K/V and rewrite only the image half). - The ``is_negative`` flag selects the correct CFG branch slot. ``k_img`` / - ``v_img`` (I2V image tokens, ``(B, 257, ...)``) are written when the - image pool is allocated. + ``layer_kv`` must yield exactly one ``(k, v)`` pair per model layer. + Inputs have shape ``(B, length, local_kv_heads, head_size)``; batch zero + is copied because AR-Diffusion currently supports one sequence per + forward. The new cache is published only after every layer is copied, + so failed projection/copy work cannot expose partially initialized KV. """ - branch = 1 if is_negative else 0 - if k is not None: - self._cross_k[layer_idx][branch].copy_(k[0]) - self._cross_v[layer_idx][branch].copy_(v[0]) - if k_img is not None and self._cross_k_img: - self._cross_k_img[layer_idx][branch].copy_(k_img[0]) - self._cross_v_img[layer_idx][branch].copy_(v_img[0]) + try: + length = self.cross_attention_lengths[cache_name] + except KeyError as exc: + raise KeyError( + f"Unknown AR-Diffusion cross-attention cache {cache_name!r}; " + f"expected {tuple(self.cross_attention_lengths)}" + ) from exc + self._kv_branch_index(kv_branch) + session = self._cross_sessions.get(session_id) + if session is None and len(self._cross_sessions) >= self.session_capacity: + raise RuntimeError( + "AR-Diffusion cross-attention session capacity exhausted; " + "the runner must evict a session before allocating another" + ) + if not self._allocate_tensors: + raise RuntimeError("AR-Diffusion cross-attention tensors require a configured pool device") - def read_cross_kv(self, layer_idx: int, is_negative: bool) -> dict: - """Return a pool-backed cross-attn cache dict for one layer. + shape = (length, self.num_kv_heads, self.head_size) + expected_input_shape = (1, *shape) + k_pool = [torch.empty(shape, dtype=self.dtype, device=self.device) for _ in range(self.num_layers)] + v_pool = [torch.empty(shape, dtype=self.dtype, device=self.device) for _ in range(self.num_layers)] + populated_layers = 0 + for layer_idx, (k, v) in enumerate(layer_kv): + if layer_idx >= self.num_layers: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} expected {self.num_layers} layers, " + f"got more than {self.num_layers}" + ) + if not isinstance(k, torch.Tensor) or not isinstance(v, torch.Tensor): + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} layer {layer_idx} " + f"must yield torch.Tensor k/v, got {type(k).__name__} and {type(v).__name__}" + ) + if tuple(k.shape) != expected_input_shape or tuple(v.shape) != expected_input_shape: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} layer {layer_idx} expected " + f"k/v shape {expected_input_shape}, got {tuple(k.shape)} and {tuple(v.shape)}" + ) + k_pool[layer_idx].copy_(k[0]) + v_pool[layer_idx].copy_(v[0]) + populated_layers += 1 + if populated_layers != self.num_layers: + raise ValueError( + f"AR-Diffusion cross-attention cache {cache_name!r} expected {self.num_layers} layers, " + f"got {populated_layers}" + ) - The dict matches the ``{"is_init": True, "k": Tensor, "v": Tensor}`` - convention the cross-attention module expects — it reads from the pool - slice rather than from the lazy-initialised model-local dict. For I2V, - ``k_img`` / ``v_img`` are added so the image-token cache (added by #4154) - reads from the pool too. - """ - branch = 1 if is_negative else 0 - k = self._cross_k[layer_idx][branch].unsqueeze(0) # (1, L, heads, dim) - v = self._cross_v[layer_idx][branch].unsqueeze(0) - cache = {"is_init": True, "k": k, "v": v} - if self._cross_k_img: - cache["k_img"] = self._cross_k_img[layer_idx][branch].unsqueeze(0) - cache["v_img"] = self._cross_v_img[layer_idx][branch].unsqueeze(0) - return cache + if session is None: + session = {} + self._cross_sessions[session_id] = session + session.setdefault(cache_name, {})[kv_branch] = (k_pool, v_pool) + + def read_cross_attention_kv( + self, + session_id: str, + cache_name: str, + layer_idx: int, + kv_branch: str, + ) -> dict[str, torch.Tensor | bool]: + """Return a model-facing K/V dict for one named cross-attention pool.""" + k_pool, v_pool = self._cross_attention_pool(session_id, cache_name, kv_branch) + return { + "is_init": True, + "k": k_pool[layer_idx].unsqueeze(0), + "v": v_pool[layer_idx].unsqueeze(0), + } + + def retain_cross_attention(self, session_id: str, cache_names: Collection[str]) -> None: + """Release named cross-attention caches not retained by an internal reset.""" + session = self._cross_sessions.get(session_id) + if session is None: + return + keep = set(cache_names) + for cache_name in tuple(session): + if cache_name not in keep: + del session[cache_name] + if not session: + self._cross_sessions.pop(session_id, None) + + def release_cross_attention(self, session_id: str) -> None: + """Release every named cross-attention allocation for one session.""" + self._cross_sessions.pop(session_id, None) # -- request lifecycle --------------------------------------------------- @@ -431,20 +521,19 @@ class ARDiffusionKVCache: self.block_size, ) - def scratch_block_ids(self, is_negative: bool, start: int, count: int) -> list[int]: - """Return branch-local scratch block ids outside manager ownership.""" + def scratch_block_ids(self, kv_branch: str, start: int, count: int) -> list[int]: + """Return KV-branch-local scratch block ids outside manager ownership.""" if count < 0 or start < 0: raise ValueError(f"scratch start/count must be non-negative, got start={start}, count={count}") - if start + count > self.scratch_blocks_per_branch: + if start + count > self.scratch_blocks_per_kv_branch: raise RuntimeError( "AR-Diffusion paged attention scratch blocks exhausted: " - f"need [{start}, {start + count}) of {self.scratch_blocks_per_branch}. " - "Increase AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH." + f"need [{start}, {start + count}) of {self.scratch_blocks_per_kv_branch}. " + "Declare max_scratch_tokens_per_branch in the pipeline capability " + "or increase AR_DIFFUSION_KV_SCRATCH_BLOCKS_PER_BRANCH." ) - # With one local branch (CFG-parallel x2) the rank's only branch maps to - # slot 0 whichever CFG side it is; the other branch never allocates here. - branch_offset = self.scratch_blocks_per_branch if (is_negative and self.local_branches == 2) else 0 - base = self.managed_num_blocks + branch_offset + start + kv_branch_offset = self.scratch_blocks_per_kv_branch * self._kv_branch_index(kv_branch) + base = self.managed_num_blocks + kv_branch_offset + start return list(range(base, base + count)) def key_cache(self, layer_idx: int) -> torch.Tensor: @@ -455,13 +544,13 @@ class ARDiffusionKVCache: def window_block_ids(self, adapter: ARDiffusionRequestAdapter) -> list[int]: """Resident (non-null) managed blocks visible to paged attention.""" - return resident_block_ids(self.block_table(adapter), self.null_block_id) + return [int(block_id) for block_id in resident_block_ids(self.block_table(adapter), self.null_block_id)] def commit_chunk(self, adapter: ARDiffusionRequestAdapter) -> None: """Advance the adapter by one chunk after its K/V is written. - This standalone primitive is used by low-level manager tests. In the - DreamZero paged-attention path, :meth:`ARDiffusionKVState.commit_paged_context` + This standalone primitive is used by low-level manager tests. The + paged-attention path uses :meth:`ARDiffusionKVState.commit_paged_context`, advances the adapter only after the forward succeeds. Call once per committed chunk, not per denoise step. """ diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py index a39a66b91..6b7d7ffe5 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged.py @@ -153,7 +153,7 @@ def chunk_window_skipped_tokens( Pure function so the eviction policy is unit-testable without constructing a manager. Two strategies: - - ``reset_at_boundary`` (DreamZero): at each chunk boundary everything past + - ``reset_at_boundary``: at each chunk boundary everything past the sink is dropped. - otherwise (VGGT-style sliding replace): keep the last ``window`` tokens (plus the sink); the skip count snaps down to a chunk boundary so a chunk @@ -184,6 +184,25 @@ class ChunkWindowManager(SlidingWindowManager): reset_at_boundary=spec.reset_at_boundary, ) + def remove_skipped_blocks( + self, + request_id: str, + total_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + """Free the middle gap while preserving the leading attention sink.""" + del num_prompt_tokens + num_skipped_tokens = self.get_num_skipped_tokens(total_computed_tokens) + if num_skipped_tokens <= 0: + return + sink_blocks = self.kv_cache_spec.sink_chunks + num_skipped_blocks = num_skipped_tokens // self.block_size + self._remove_blocks_in_range( + request_id, + sink_blocks, + sink_blocks + num_skipped_blocks, + ) + # Register so KVCacheManager resolves ChunkWindowSpec to ChunkWindowManager. # Dispatch walks the spec's MRO, so without explicit registration the subclass diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py index 2038e496d..bcd06b63a 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/paged_attention.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Paged self-attention helpers for AR-Diffusion DreamZero KV reuse.""" +"""Paged self-attention helpers for AR-Diffusion KV reuse.""" from __future__ import annotations @@ -10,7 +10,7 @@ import torch from vllm_omni.experimental.ar_diffusion.kv_cache.paged import compute_slot_mapping -# Set by ARDiffusionPagedForwardContext.prepare() before each branch forward and +# Set by ARDiffusionPagedForwardContext.prepare() before each KV branch forward and # read by the fused write+attend custom op below. The pools are process-lifetime # allocations owned by one ARDiffusionKVCache per worker process (CFG-parallel / # TP ranks are separate processes), mirroring vLLM's forward-context pattern for @@ -39,7 +39,7 @@ class ARDiffusionPagedLayerInputs(NamedTuple): A NamedTuple of plain tensors + ints so ``torch.compile`` treats every field as a pytree graph input (no object-attribute guards, no recompiles when only - tensor *values* change). All layers of one branch forward share the same + tensor *values* change). All layers of one KV branch forward share the same metadata tensor objects, built once by ``prepare()``. ``layer_idx`` is a 0-dim CPU tensor, NOT a python int: all 40 DiT blocks @@ -62,11 +62,11 @@ class ARDiffusionPagedLayerInputs(NamedTuple): @dataclass class ARDiffusionPagedForwardContext: - """Mutable branch-level state shared by all layer contexts in one forward.""" + """Mutable KV-branch state shared by all layer contexts in one forward.""" kv_cache: Any adapter: Any - is_negative: bool + kv_branch: str history_block_ids: list[int] seq_len: int commit_current: bool @@ -80,7 +80,7 @@ class ARDiffusionPagedForwardContext: _allocated_video: bool = False _committed: bool = False _action_len: int = 0 - # Set once by prepare(); shared by all layers of the branch forward. + # Set once by prepare(); shared by all layers of the KV branch forward. block_table: torch.Tensor | None = None query_start_loc: torch.Tensor | None = None seq_lens: torch.Tensor | None = None @@ -102,7 +102,7 @@ class ARDiffusionPagedForwardContext: return self.seq_len // self.block_size def ensure_video_slots(self, device: torch.device) -> None: - """Allocate/write targets for the current video tokens, once per branch.""" + """Allocate/write targets for the current video tokens, once per KV branch.""" if self._allocated_video: return @@ -116,7 +116,7 @@ class ARDiffusionPagedForwardContext: positions = torch.arange(start, start + self.seq_len, dtype=torch.long) self.current_video_slot_mapping = compute_slot_mapping(table, positions, self.block_size).to(device=device) else: - self.current_video_block_ids = self.kv_cache.scratch_block_ids(self.is_negative, 0, n_blocks) + self.current_video_block_ids = self.kv_cache.scratch_block_ids(self.kv_branch, 0, n_blocks) positions = torch.arange(self.seq_len, dtype=torch.long) self.current_video_slot_mapping = compute_slot_mapping( self.current_video_block_ids, @@ -140,7 +140,7 @@ class ARDiffusionPagedForwardContext: action_blocks = (action_len + self.block_size - 1) // self.block_size scratch_offset = 0 if self.commit_current else len(self.current_video_block_ids) self.action_scratch_block_ids = self.kv_cache.scratch_block_ids( - self.is_negative, + self.kv_branch, scratch_offset, action_blocks, ) @@ -161,8 +161,15 @@ class ARDiffusionPagedForwardContext: ) all_video_blocks = self.history_block_ids + self.current_video_block_ids max_video_blocks = self.max_video_tokens // self.block_size - visible_video_blocks = all_video_blocks[-max_video_blocks:] if max_video_blocks else [] - video_len = min(len(all_video_blocks) * self.block_size, self.max_video_tokens) + sink_blocks = int(self.kv_cache.spec.sink_chunks) + if len(all_video_blocks) <= max_video_blocks: + visible_video_blocks = all_video_blocks + else: + tail_blocks = max_video_blocks - sink_blocks + visible_video_blocks = all_video_blocks[:sink_blocks] + if tail_blocks: + visible_video_blocks += all_video_blocks[-tail_blocks:] + video_len = len(visible_video_blocks) * self.block_size return visible_video_blocks, video_len def build_block_table( @@ -201,9 +208,9 @@ class ARDiffusionPagedForwardContext: return block_table, query_start_loc, seq_lens, self.query_len, max_seq_len def prepare(self, device: torch.device, action_len: int, query_len: int) -> None: - """Host-side, once-per-branch-forward setup (called OUTSIDE torch.compile). + """Host-side, once-per-KV-branch setup (called OUTSIDE torch.compile). - Allocates the current video/action slots (still lazy: only the branch a + Allocates the current video/action slots (still lazy: only the KV branch a CFG-parallel rank actually runs reaches its ``_forward_blocks``), builds the padded block-table metadata ONCE for all layers, and publishes the pool registry for the fused custom op. The compiled per-layer code then @@ -245,7 +252,7 @@ class ARDiffusionPagedForwardContext: @dataclass class ARDiffusionPagedLayerContext: - """Layer-specific handle passed through DreamZero's existing ``kv_cache`` slot.""" + """Layer-specific handle passed through a model's ``kv_cache`` slot.""" is_ar_diffusion_paged_context: ClassVar[bool] = True layer_idx: int @@ -260,8 +267,8 @@ class ARDiffusionPagedLayerContext: return self.forward_ctx.adapter @property - def is_negative(self) -> bool: - return self.forward_ctx.is_negative + def kv_branch(self) -> str: + return self.forward_ctx.kv_branch @property def history_block_ids(self) -> list[int]: diff --git a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py index 510bbe043..d114212bb 100644 --- a/vllm_omni/experimental/ar_diffusion/kv_cache/state.py +++ b/vllm_omni/experimental/ar_diffusion/kv_cache/state.py @@ -1,8 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -"""DreamZero-facing AR-Diffusion KV bridge backed by paged self-attention.""" +"""Model-facing session state backed by paged AR-Diffusion KV storage.""" from __future__ import annotations +from collections.abc import Collection, Iterable, Mapping +from typing import TYPE_CHECKING + +import torch from vllm.logger import init_logger from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( @@ -10,67 +14,79 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( ARDiffusionPagedLayerContext, ) +if TYPE_CHECKING: + from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ( + ARDiffusionKVCache, + ARDiffusionRequestAdapter, + ) + _log = init_logger(__name__) class ARDiffusionKVState: - """Per-request bridge between a DreamZero rollout and the AR-Diffusion KV pool. + """Runner-owned KV state for one session and any number of KV branches. - The self-attention path is paged-only: ``get_kv_caches`` returns lightweight - per-layer contexts, and each attention layer writes current K/V into the - pool before calling the FlashAttention block-table kernel. No resident - window is gathered into a contiguous tensor. + KV branches are addressed by the names in ``ARDiffusionKVCacheSpec``. Each + branch owns an independent request adapter and therefore independent + resident self-attention blocks. Named cross-attention allocations are also + session-scoped and are released by :meth:`close`. """ - def __init__(self, kv_cache, pos_adapter, neg_adapter, num_layers: int) -> None: - """One state per DreamZero session, holding BOTH CFG branches' adapters. - - ``pos``/``neg`` are the classifier-free-guidance branches: ``pos`` is - the conditional pass (real prompt), ``neg`` the unconditional pass - (negative prompt). Each owns an independent request adapter — and - therefore independent pool blocks — because the two branches produce - different K/V for the same frames. Every ``is_negative`` flag in this - module selects between them (``False`` -> ``pos``, ``True`` -> ``neg``). - """ + def __init__( + self, + kv_cache: ARDiffusionKVCache, + session_id: str, + adapters: Mapping[str, ARDiffusionRequestAdapter], + *, + num_layers: int, + ) -> None: + expected = tuple(kv_branch.name for kv_branch in kv_cache.kv_branches) + if set(adapters) != set(expected): + raise ValueError( + "AR-Diffusion session adapters must match the configured KV branches; " + f"expected {expected}, got {tuple(adapters)}" + ) self.kv_cache = kv_cache - self.pos = pos_adapter - self.neg = neg_adapter + self.session_id = session_id + self.adapters = dict(adapters) self.num_layers = num_layers - self._committed: dict[bool, int] = {False: 0, True: 0} - self._paged_pending: dict[bool, ARDiffusionPagedForwardContext | None] = {False: None, True: None} - # Cross-attn KV is populated once after text encoding (eager), then - # read every denoising step. Tracked per half: the text K/V depend only - # on the session prompt (survive window-boundary resets), while the I2V - # image-token K/V come from the current observation's CLIP features and - # must be re-projected on every window restart. - self._cross_text_populated: dict[bool, bool] = {False: False, True: False} - self._cross_img_populated: dict[bool, bool] = {False: False, True: False} + self._committed: dict[str, int] = dict.fromkeys(expected, 0) + self._paged_pending: dict[str, ARDiffusionPagedForwardContext | None] = dict.fromkeys(expected) + self._closed = False - def _adapter(self, is_negative: bool): - return self.neg if is_negative else self.pos + @property + def kv_branch_names(self) -> tuple[str, ...]: + return tuple(self.adapters) + + def adapter(self, kv_branch: str) -> ARDiffusionRequestAdapter: + """Return the request adapter for one logical KV branch.""" + if self._closed: + raise RuntimeError(f"AR-Diffusion session {self.session_id!r} is closed") + try: + return self.adapters[kv_branch] + except KeyError as exc: + raise KeyError(f"Unknown AR-Diffusion KV branch {kv_branch!r}; expected {self.kv_branch_names}") from exc def get_kv_caches( self, - is_negative: bool, + kv_branch: str, seq_len: int | None = None, commit_current: bool = False, ) -> list[ARDiffusionPagedLayerContext]: if seq_len is None: raise ValueError("AR-Diffusion paged self-attention requires seq_len in get_kv_caches()") - return self.prepare_paged_context(is_negative, seq_len, commit_current) + return self.prepare_paged_context(kv_branch, seq_len, commit_current) def prepare_paged_context( self, - is_negative: bool, + kv_branch: str, seq_len: int, commit_current: bool, ) -> list[ARDiffusionPagedLayerContext]: - """Return per-layer paged attention contexts for one branch forward. + """Return per-layer paged attention contexts for one KV branch forward. - Allocation is lazy: constructing both CFG branches' kwargs must not - allocate manager blocks for a branch that a CFG-parallel rank will not run. - The first self-attn layer that consumes the context allocates the current - video slots, then all layers share that branch-level state. + Allocation is lazy so distributed workers allocate only for KV branches + they actually execute. """ cs = self.kv_cache.spec.chunk_size if int(seq_len) % cs != 0: @@ -78,25 +94,26 @@ class ARDiffusionKVState: f"AR-Diffusion expects frame-aligned seq_len (multiple of chunk_size={cs}), got {seq_len}" ) - pending = self._paged_pending.get(is_negative) + pending = self._paged_pending.get(kv_branch) if pending is not None and pending.commit_current and pending._allocated_video and not pending._committed: raise RuntimeError("AR-Diffusion paged context replaced before its managed current chunk was committed") - adapter = self._adapter(is_negative) - branch = "neg" if is_negative else "pos" + adapter = self.adapter(kv_branch) forward_ctx = ARDiffusionPagedForwardContext( kv_cache=self.kv_cache, adapter=adapter, - is_negative=is_negative, + kv_branch=kv_branch, history_block_ids=self.kv_cache.window_block_ids(adapter), seq_len=int(seq_len), commit_current=bool(commit_current), - max_video_tokens=int(self.kv_cache.spec.sliding_window), + max_video_tokens=int( + self.kv_cache.spec.sliding_window + self.kv_cache.spec.sink_chunks * self.kv_cache.spec.chunk_size + ), ) - self._paged_pending[is_negative] = forward_ctx + self._paged_pending[kv_branch] = forward_ctx _log.debug( - "AR-Diffusion GET [%s] source=paged-attn layers=%d history_blocks=%d seq_len=%d commit_current=%s", - branch, + "AR-Diffusion GET [%s] source=paged-attn layers=%d history_blocks=%d seq_len=%d commit_current=%s", + kv_branch, self.num_layers, len(forward_ctx.history_block_ids), int(seq_len), @@ -104,80 +121,94 @@ class ARDiffusionKVState: ) return [ARDiffusionPagedLayerContext(layer_idx=i, forward_ctx=forward_ctx) for i in range(self.num_layers)] - def commit_paged_context(self, is_negative: bool) -> None: - """Commit the managed current video blocks after a successful forward.""" - ctx = self._paged_pending.get(is_negative) + def commit_paged_context(self, kv_branch: str) -> None: + """Commit managed current blocks after one successful KV branch forward.""" + self.adapter(kv_branch) + ctx = self._paged_pending.get(kv_branch) if ctx is None: return - branch = "neg" if is_negative else "pos" if ctx.commit_current and ctx._allocated_video: n_chunks = ctx.seq_len // self.kv_cache.spec.chunk_size for _ in range(n_chunks): ctx.adapter.on_chunk_committed() - self._committed[is_negative] += ctx.seq_len + self._committed[kv_branch] += ctx.seq_len _log.debug( - "AR-Diffusion COMMIT [%s] paged-attn new_tokens=%d chunks=%d resident=%d/%d", - branch, + "AR-Diffusion COMMIT [%s] new_tokens=%d chunks=%d resident=%d/%d", + kv_branch, ctx.seq_len, n_chunks, len(self.kv_cache.window_block_ids(ctx.adapter)), self.kv_cache.spec.window_chunks, ) ctx.mark_committed() - self._paged_pending[is_negative] = None + self._paged_pending[kv_branch] = None - def get_cross_kv_caches(self, is_negative: bool) -> list[dict]: - """Return pool-backed cross-attn cache dicts. + def is_cross_attention_populated(self, kv_branch: str, cache_name: str) -> bool: + self.adapter(kv_branch) + return self.kv_cache.is_cross_attention_populated(self.session_id, cache_name, kv_branch) - Under AR-Diffusion the cross-attn pool is always populated by ``_kv_populate_cross`` - before the first read, so this must never be reached unpopulated — if it - were, the model would lazily project and write cross KV itself, violating - engine ownership. Fail loud rather than fall back to the model. + def populate_cross_attention( + self, + kv_branch: str, + cache_name: str, + layer_kv: Iterable[tuple[torch.Tensor, torch.Tensor]], + ) -> None: + """Publish one named logical-branch cache after all layers are written. + + The iterable is consumed once in layer order. If projection, validation, + or copying fails, the previous complete cache (if any) remains visible. """ - img_ok = not self.kv_cache._cross_k_img or self._cross_img_populated[is_negative] - if not (self._cross_text_populated[is_negative] and img_ok and self.kv_cache.cross_attn_length > 0): + self.adapter(kv_branch) + self.kv_cache.populate_cross_attention(self.session_id, cache_name, kv_branch, layer_kv) + + def get_cross_attention_kv(self, kv_branch: str, cache_name: str) -> list[dict[str, torch.Tensor | bool]]: + """Return all layers for one populated named cross-attention cache.""" + if not self.is_cross_attention_populated(kv_branch, cache_name): raise RuntimeError( - f"AR-Diffusion cross-attn read before _kv_populate_cross (neg={is_negative}, " - f"cross_attn_length={self.kv_cache.cross_attn_length}, " - f"text={self._cross_text_populated[is_negative]} " - f"img={self._cross_img_populated[is_negative]}) — the engine must own all cross KV" + f"AR-Diffusion cross-attention cache {cache_name!r} for KV branch {kv_branch!r} " + "was read before it was populated" ) - return [self.kv_cache.read_cross_kv(i, is_negative) for i in range(self.num_layers)] + return [ + self.kv_cache.read_cross_attention_kv(self.session_id, cache_name, i, kv_branch) + for i in range(self.num_layers) + ] def close(self) -> None: - """Final teardown: free both branches' resident pool blocks.""" - self.kv_cache.end_request(self.pos) - self.kv_cache.end_request(self.neg) - self._paged_pending = {False: None, True: None} + """Release all self- and cross-attention storage owned by this session.""" + if self._closed: + return + for adapter in self.adapters.values(): + self.kv_cache.end_request(adapter) + self.kv_cache.release_cross_attention(self.session_id) + self._paged_pending = dict.fromkeys(self.adapters) + self._closed = True - def reset(self, *, keep_cross_text: bool = False) -> None: - """Drop this session's KV — mirrors the model-local ``state.reset()``. + def reset(self, *, keep_cross_attention: Collection[str] = ()) -> None: + """Release resident blocks and reopen this session with fresh adapters. - DreamZero resets at the attention-window boundary (``should_reset``); the - model starts a fresh sliding window, so AR-Diffusion must free the resident pool - blocks and start a fresh adapter for each branch. The ``ARDiffusionKVState`` - object (and thus ``pipeline._ar_diffusion_kv_state``) is preserved across the reset - so the runner's session mapping stays valid — only the pool-backed state - is recycled. - - Args: - keep_cross_text: On window ("inference") resets the session prompt is - unchanged, so the text cross-attn K/V in the static pool buffers are - still valid — keep them and only force the image half to repopulate. - Session resets (prompt change / explicit reset) pass ``False``. + ``keep_cross_attention`` names allocations whose conditioning remains + valid across a model-internal window reset. Ordinary session reset + should leave it empty. """ - pos_id, neg_id = self.pos.request_id, self.neg.request_id - self.close() - self.pos = self.kv_cache.begin_request(pos_id) - self.neg = self.kv_cache.begin_request(neg_id) - self._committed = {False: 0, True: 0} - self._paged_pending = {False: None, True: None} - if not keep_cross_text: - self._cross_text_populated = {False: False, True: False} - self._cross_img_populated = {False: False, True: False} + unknown = set(keep_cross_attention) - set(self.kv_cache.cross_attention_lengths) + if unknown: + raise KeyError(f"Unknown AR-Diffusion cross-attention caches to keep: {sorted(unknown)}") + request_ids = {kv_branch: adapter.request_id for kv_branch, adapter in self.adapters.items()} + if keep_cross_attention: + for adapter in self.adapters.values(): + self.kv_cache.end_request(adapter) + self.kv_cache.retain_cross_attention(self.session_id, keep_cross_attention) + else: + self.close() + self.adapters = { + kv_branch: self.kv_cache.begin_request(request_id) for kv_branch, request_id in request_ids.items() + } + self._committed = dict.fromkeys(self.adapters, 0) + self._paged_pending = dict.fromkeys(self.adapters) + self._closed = False _log.info( - "AR-Diffusion RESET [%s/%s] session KV cleared (window boundary; cross text %s)", - pos_id, - neg_id, - "kept" if keep_cross_text else "cleared", + "AR-Diffusion RESET session=%s KV branches=%s kept_cross=%s", + self.session_id, + self.kv_branch_names, + sorted(keep_cross_attention), ) diff --git a/vllm_omni/experimental/ar_diffusion/runner.py b/vllm_omni/experimental/ar_diffusion/runner.py index 8c968f8ce..31a23eaf7 100644 --- a/vllm_omni/experimental/ar_diffusion/runner.py +++ b/vllm_omni/experimental/ar_diffusion/runner.py @@ -1,32 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 -"""ARDiffusionModelRunner — the diffusion model runner for the AR-Diffusion engine. - -Subclasses ``DiffusionModelRunner`` and owns the engine-level KV cache -(:class:`ARDiffusionKVCache`). It brackets a request's rollout with the KV lifecycle -(``begin_request`` / ``end_request``) and exposes the live ``ARDiffusionKVCache`` so the -model pipeline (DreamZero) can drive the per-chunk allocate / slot-mapping / -gather / commit operations during ``pipeline.forward``. - -Selecting the AR-Diffusion engine enables KV management (no env gate); optional KV -parameters come from the deploy yaml's ``model_config["ar_diffusion_kv_config"]``. -""" - from __future__ import annotations import dataclasses -import math import time from collections import OrderedDict -import numpy as np import torch from vllm.logger import init_logger from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import MAX_DREAMZERO_SESSIONS from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.sched.interface import KVPrefetchJob +from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.diffusion.worker.utils import BatchRunnerOutput +from vllm_omni.experimental.ar_diffusion.capability import ( + ARDiffusionKVCacheSpec, + SupportsARDiffusionPipeline, + SupportsARDiffusionWarmup, +) from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState @@ -35,14 +26,7 @@ logger = init_logger(__name__) def resolve_ar_diffusion_kv_config(od_config: OmniDiffusionConfig) -> ARDiffusionKVConfig: - """Resolve the AR-Diffusion KV config from the deploy/command arg. - - Selecting the AR-Diffusion engine implies KV management, so the result is always - enabled. Optional overrides (``window_chunks`` / ``gpu_memory_fraction`` / - ``sink_chunks`` / ``reset_at_boundary``) may be supplied via the deploy yaml as - ``model_config["ar_diffusion_kv_config"]``. ``chunk_size`` / ``window_chunks`` are - finalized from the model geometry at load (see ``_preallocate_kv_cache``). - """ + """Resolve optional deployment overrides and enable AR-Diffusion KV.""" raw = getattr(od_config, "ar_diffusion_kv_config", None) if raw is None: model_config = getattr(od_config, "model_config", None) @@ -56,359 +40,280 @@ def resolve_ar_diffusion_kv_config(od_config: OmniDiffusionConfig) -> ARDiffusio class ARDiffusionModelRunner(DiffusionModelRunner): - def __init__(self, vllm_config, od_config: OmniDiffusionConfig, device) -> None: + """Own AR KV pools and session lifetime for a capable diffusion pipeline. + + The runner never inspects model internals. At load time it consumes an + :class:`ARDiffusionKVCacheSpec`; during execution it binds a runner-owned + :class:`ARDiffusionKVState` through the pipeline's context manager. Sessions + persist across requests and end on explicit close, LRU eviction, or failed + forward. A request with ``extra_args["reset"]`` releases the old session + before executing with a fresh one. + """ + + _WARMUP_SID = "__ardiffusion_warmup__" + _MAX_RESIDENT_SESSIONS = 1 + + def __init__(self, vllm_config: object, od_config: OmniDiffusionConfig, device: torch.device) -> None: super().__init__(vllm_config, od_config, device) self.ar_diffusion_kv_config = resolve_ar_diffusion_kv_config(od_config) - # Built after the model is loaded (dimensions known); stays None while KV - # management is disabled. self.kv_cache: ARDiffusionKVCache | None = None - # DreamZero KV is session-scoped (persists across a session's forwards), - # so AR-Diffusion KV state is keyed by session_id and reused, not created per request. - # Bounded by the same LRU cap as DreamZeroPipeline._states: an evicted - # session's ARDiffusionKVState owns pool blocks (two CFG adapters), so without a - # bound, session-id churn in a long-running server would leak pool ownership - # until the KV pool is exhausted. Evicting frees those blocks (state.close()). - self._ar_diffusion_states: OrderedDict[str, ARDiffusionKVState] = OrderedDict() - self._max_ar_diffusion_states = MAX_DREAMZERO_SESSIONS - # Per-request server-side forward E2E times (seconds), recorded in - # execute_model. This is the worker-side compute time — the analog of the - # upstream server's per-request E2E — and excludes the engine<->worker IPC - # that the client's omni.generate wall time includes. Read (and cleared) - # by the ar_diffusion_perf_stats worker RPC for the perf-compare summary. + self._ar_diffusion_capability: SupportsARDiffusionPipeline | None = None + self._ar_diffusion_kv_cache_spec: ARDiffusionKVCacheSpec | None = None + self._sessions: OrderedDict[str, ARDiffusionKVState] = OrderedDict() + self._session_capacity = 0 self._perf_e2e_times: list[float] = [] - def build_kv_cache( - self, - *, - num_layers: int, - num_kv_heads: int, - head_size: int, - dtype, - block_size: int, - max_model_len: int, - available_bytes: int, - cross_attn_length: int = 0, - cross_attn_img_length: int = 0, - local_branches: int = 2, - num_frame_per_block: int = 1, - ) -> None: - """Construct the ARDiffusionKVCache (preallocating GPU pools on ``self.device``). + @staticmethod + def _require_capability(pipeline: object) -> SupportsARDiffusionPipeline: + if not isinstance(pipeline, SupportsARDiffusionPipeline): + raise TypeError( + "ARDiffusionEngine requires the loaded pipeline to implement " + "SupportsARDiffusionPipeline: ar_diffusion_kv_cache_spec(), " + "bind_ar_diffusion_state(), reset_ar_diffusion_session(), and " + "close_ar_diffusion_session(). Select the default DiffusionEngine " + "or add an explicit AR-Diffusion capability adapter to the model." + ) + return pipeline - No-op when KV management is disabled. - """ - if not self.ar_diffusion_kv_config.enable: - return - self.kv_cache = ARDiffusionKVCache( - self.ar_diffusion_kv_config, - num_layers=num_layers, - num_kv_heads=num_kv_heads, - head_size=head_size, - dtype=dtype, - block_size=block_size, - max_model_len=max_model_len, - available_bytes=available_bytes, - cross_attn_length=cross_attn_length, - cross_attn_img_length=cross_attn_img_length, - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, - device=self.device, - ) - logger.info( - "AR-Diffusion KV cache enabled: %d blocks, chunk_size=%d, window_chunks=%s, " - "layers=%d kv_heads=%d head_dim=%d block_size=%d cross_attn_len=%d device=%s", - self.kv_cache.num_blocks, - self.ar_diffusion_kv_config.chunk_size, - self.ar_diffusion_kv_config.window_chunks, - num_layers, - num_kv_heads, - head_size, - block_size, - cross_attn_length, - self.device, - ) - - # -- preallocation at load ------------------------------------------------- - - def load_model(self, *args, **kwargs): + def load_model(self, *args: object, **kwargs: object) -> None: super().load_model(*args, **kwargs) - if self.ar_diffusion_kv_config.enable and self.pipeline is not None: - self._preallocate_kv_cache() - # Pre-capture the CUDA graphs for every window-fill shape at load time - # (only when compiling/cuda-graph is on) so serving is fast from chunk 0. - if not self.od_config.enforce_eager and self.ar_diffusion_kv_config.warmup_cudagraph: - self._warmup_ar_rollout() + if self.pipeline is None: + return + self._preallocate_kv_cache() + if not self.od_config.enforce_eager and self.ar_diffusion_kv_config.warmup_cudagraph: + self._warmup_ar_rollout() - def _infer_frame_seqlen(self) -> int: - """frame_seqlen = (H//8)*(W//8)//4 from the configured image_resolution.""" - mc = getattr(self.od_config, "model_config", None) or {} - psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} - res = psc.get("image_resolution", [180, 320]) - h, w = int(res[0]), int(res[1]) - return (h // 8) * (w // 8) // 4 + def _available_memory_bytes(self) -> int: + if self.device is None or torch.device(self.device).type != "cuda": + raise RuntimeError("AR-Diffusion KV preallocation currently requires a CUDA device") + return int(torch.cuda.mem_get_info(self.device)[0]) - def _preallocate_kv_cache(self) -> None: - """Build the KV pool once, from the loaded DreamZero transformer geometry.""" - # The paged self-attention path carries one rollout per request: slot - # mappings/adapters are single-sequence and K/V writes address batch - # index 0 only. Reject a multi-sequence config at init instead of - # failing (or silently dropping sequences) mid-forward. + def _preallocate_kv_cache(self, *, available_bytes: int | None = None) -> None: + """Build pools solely from the pipeline capability and runner config.""" + if bool(getattr(self.od_config, "step_execution", False)): + raise ValueError( + "ARDiffusionModelRunner currently supports request-mode execution only; " + "step_execution=True would bypass per-request AR session binding." + ) max_num_seqs = int(getattr(self.od_config, "max_num_seqs", 1) or 1) if max_num_seqs > 1: raise ValueError( "AR-Diffusion paged KV supports max_num_seqs=1 (single-sequence " - f"rollouts); got max_num_seqs={max_num_seqs}. Per-sequence " - "adapters/slot maps are not implemented." + f"rollouts); got max_num_seqs={max_num_seqs}." ) - t = self.pipeline.transformer - num_layers = int(t.num_layers) - num_kv_heads = int(getattr(t.blocks[0].self_attn, "tp_num_heads", t.num_heads)) - head_size = int(t.dim // t.num_heads) - num_frame_per_block = int(t.num_frame_per_block) - local_attn_size = int(t.local_attn_size) - # The loaded transformer is the source of truth for frame_seqlen (it derives - # max_attention_size from it). The deploy-config inference is only a - # cross-check — warn on drift rather than silently sizing chunks wrong. - frame_seqlen = int(getattr(t, "frame_seqlen", 0)) or self._infer_frame_seqlen() - inferred = self._infer_frame_seqlen() - if frame_seqlen != inferred: - logger.warning( - "AR-Diffusion frame_seqlen mismatch: transformer=%d deploy-config=%d; using transformer", - frame_seqlen, - inferred, + capability = self._require_capability(self.pipeline) + if bool(getattr(self.pipeline, "supports_request_batch", False)): + raise ValueError( + "ARDiffusionModelRunner currently supports one request at a time; " + "request-batch execution would bypass per-request AR session binding." + ) + spec = capability.ar_diffusion_kv_cache_spec() + if not isinstance(spec, ARDiffusionKVCacheSpec): + raise TypeError( + f"ar_diffusion_kv_cache_spec() must return ARDiffusionKVCacheSpec, got {type(spec).__name__}" ) - # The model's own attention window, in tokens (= the [-max_attention_size:] - # slice it applies). Read it directly rather than recomputing. - max_attention_size = int(t.blocks[0].self_attn.max_attention_size) - # Cross-attn text length — static pool for the text-encoder output. - # The cached k/v spans the full text sequence (the image tokens - # prepended in _forward_blocks are stripped inside the cross-attn - # forward but are not part of the cached k/v). - cross_attn_length = int(getattr(t, "text_len", 0)) - # I2V image-token cross-attn length: the model splits context[:, :257] as the - # image tokens (the img_emb(clip_feature) output), caching k_img/v_img. T2V: 0. - cross_attn_img_length = 257 if getattr(t, "model_type", "t2v") == "i2v" else 0 - # Frame-granular paging: 1 block = 1 frame = frame_seqlen tokens, so the - # resident window matches max_attention_size exactly (it need not be a whole - # number of num_frame_per_block causal blocks). - chunk_size = frame_seqlen - window_chunks = self.ar_diffusion_kv_config.window_chunks or (max_attention_size // frame_seqlen) - - self.ar_diffusion_kv_config = dataclasses.replace( - self.ar_diffusion_kv_config, chunk_size=chunk_size, window_chunks=window_chunks + config = dataclasses.replace( + self.ar_diffusion_kv_config, + chunk_size=spec.tokens_per_frame, + window_chunks=self.ar_diffusion_kv_config.window_chunks or spec.window_frames, + sink_chunks=self.ar_diffusion_kv_config.sink_chunks or spec.sink_frames, + reset_at_boundary=self.ar_diffusion_kv_config.reset_at_boundary or spec.reset_at_boundary, ) - free_bytes = torch.cuda.mem_get_info(self.device)[0] - # Under CFG-parallel each rank executes exactly ONE branch (rank0 pos, - # rank1 neg; the other branch's lazy contexts never allocate on this - # rank), so its pool only needs one branch's capacity. A single-process - # run (cfg world 1) executes both branches from one pool. - try: - from vllm_omni.diffusion.distributed.parallel_state import ( - get_classifier_free_guidance_world_size, - ) - - cfg_world = int(get_classifier_free_guidance_world_size()) - except Exception: - cfg_world = 1 - local_branches = 1 if cfg_world >= 2 else 2 - logger.info( - "AR-Diffusion preallocating (paged): frame_seqlen=%d num_frame_per_block=%d " - "local_attn_size=%d -> chunk_size=%d window_chunks=%d (window=%d tokens)", - frame_seqlen, - num_frame_per_block, - local_attn_size, - chunk_size, - window_chunks, - window_chunks * chunk_size, - ) - self.build_kv_cache( - num_layers=num_layers, - num_kv_heads=num_kv_heads, - head_size=head_size, + self.ar_diffusion_kv_config = config + self._ar_diffusion_capability = capability + self._ar_diffusion_kv_cache_spec = spec + self._session_capacity = min(spec.session_capacity, self._MAX_RESIDENT_SESSIONS) + self.kv_cache = ARDiffusionKVCache( + config, + num_layers=spec.num_layers, + num_kv_heads=spec.num_kv_heads, + head_size=spec.head_size, dtype=self.od_config.dtype, - block_size=chunk_size, # one pool block per chunk - max_model_len=1 << 20, - available_bytes=free_bytes, - cross_attn_length=cross_attn_length, - cross_attn_img_length=cross_attn_img_length, - local_branches=local_branches, - num_frame_per_block=num_frame_per_block, + block_size=spec.tokens_per_frame, + max_model_len=spec.max_model_len, + available_bytes=self._available_memory_bytes() if available_bytes is None else available_bytes, + kv_branches=spec.kv_branches, + session_capacity=self._session_capacity, + cross_attention_lengths=spec.cross_attention_lengths, + frames_per_block=spec.frames_per_block, + max_scratch_tokens_per_branch=spec.max_scratch_tokens_per_branch, + device=self.device, ) + logger.info( + "AR-Diffusion KV cache: blocks=%d layers=%d local_kv_heads=%d head_size=%d " + "tokens/frame=%d frames/block=%d window=%d sink=%d kv_branches=%s cross=%s " + "resident_capacity=%d requested_capacity=%d", + self.kv_cache.num_blocks, + spec.num_layers, + spec.num_kv_heads, + spec.head_size, + spec.tokens_per_frame, + spec.frames_per_block, + config.window_chunks, + config.sink_chunks, + [(kv_branch.name, kv_branch.local_index) for kv_branch in spec.kv_branches], + spec.cross_attention_lengths, + self._session_capacity, + spec.session_capacity, + ) + + def _new_session_state(self, session_id: str) -> ARDiffusionKVState: + if self.kv_cache is None or self._ar_diffusion_kv_cache_spec is None: + raise RuntimeError("AR-Diffusion session requested before KV cache initialization") + adapters = { + kv_branch.name: self.kv_cache.begin_request(f"ar::{session_id}::{kv_branch.name}") + for kv_branch in self._ar_diffusion_kv_cache_spec.kv_branches + } + return ARDiffusionKVState( + self.kv_cache, + session_id, + adapters, + num_layers=self._ar_diffusion_kv_cache_spec.num_layers, + ) + + def _release_session( + self, + session_id: str, + *, + reset_model: bool, + reason: str, + suppress_errors: bool = False, + ) -> None: + """One release path for reset, close, eviction, and failed forwards.""" + state = self._sessions.pop(session_id, None) + errors: list[Exception] = [] + if state is not None: + try: + state.close() + except Exception as exc: # noqa: BLE001 - notify the pipeline even if pool cleanup fails + errors.append(exc) + capability = self._ar_diffusion_capability + if capability is not None: + try: + if reset_model: + capability.reset_ar_diffusion_session(session_id) + else: + capability.close_ar_diffusion_session(session_id) + except Exception as exc: # noqa: BLE001 - preserve all lifecycle cleanup attempts + errors.append(exc) + logger.debug("AR-Diffusion released session=%s reason=%s", session_id, reason) + if errors: + if suppress_errors: + logger.warning( + "AR-Diffusion session=%s cleanup after %s had %d error(s): %s", + session_id, + reason, + len(errors), + errors, + ) + else: + raise errors[0] + + def reset_session(self, session_id: str) -> None: + """Release KV and notify the pipeline to reset model-owned state.""" + self._release_session(session_id, reset_model=True, reason="reset") + + def close_session(self, session_id: str) -> None: + """Release KV and notify the pipeline to drop model-owned state.""" + self._release_session(session_id, reset_model=False, reason="close") + + def _get_or_create_session(self, session_id: str) -> ARDiffusionKVState: + state = self._sessions.get(session_id) + if state is None: + while len(self._sessions) >= self._session_capacity: + oldest = next(iter(self._sessions)) + self._release_session(oldest, reset_model=False, reason="lru_eviction") + state = self._new_session_state(session_id) + self._sessions[session_id] = state + self._sessions.move_to_end(session_id) + return state + + @staticmethod + def _request_session(req: OmniDiffusionRequest) -> tuple[str, dict]: + extra_args = req.sampling_params.extra_args or {} + return str(extra_args.get("session_id") or "default"), extra_args def execute_model( self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: - # KV disabled -> base behavior, unchanged. if self.kv_cache is None: return super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + capability = self._ar_diffusion_capability + if capability is None: + raise RuntimeError("AR-Diffusion capability missing after KV cache initialization") - kv = self.kv_cache - # DreamZero KV is session-scoped (the model state persists across a - # session's forwards), so the AR-Diffusion KV state is keyed by session_id and - # reused — matching how pipeline.forward resolves the model-local state. - extra_args = req.sampling_params.extra_args or {} - session_id = str(extra_args.get("session_id") or "default") - state = self._ar_diffusion_states.get(session_id) - if state is None: - pos = kv.begin_request(f"bde__{session_id}") - neg = kv.begin_request(f"bde__{session_id}__neg") - state = ARDiffusionKVState(kv, pos, neg, num_layers=kv.num_layers) - self._ar_diffusion_states[session_id] = state - # Evict the least-recently-used session(s) past the cap, freeing their - # pool blocks — mirrors DreamZeroPipeline._states so the AR-Diffusion pool can't - # outlive the model-local state map it shadows. - while len(self._ar_diffusion_states) > self._max_ar_diffusion_states: - old_id, old_state = self._ar_diffusion_states.popitem(last=False) - old_state.close() - logger.debug("AR-Diffusion evicted session=%s (LRU); freed pool blocks", old_id) - # Track recency on every access (hit or miss) so the LRU order is correct. - self._ar_diffusion_states.move_to_end(session_id) - logger.debug( - "AR-Diffusion execute_model: req=%s session=%s chunk_size=%d window_chunks=%s num_blocks=%d", - req.request_id, - session_id, - kv.spec.chunk_size, - kv.config.window_chunks, - kv.num_blocks, - ) - self.pipeline._ar_diffusion_kv_state = state - # Time the worker-side forward (true per-request compute E2E). The CUDA - # sync below makes the stop reflect completed GPU work, not just dispatch. - _e2e_t0 = time.perf_counter() + session_id, extra_args = self._request_session(req) + if extra_args.get("reset", False): + self.reset_session(session_id) + state = self._get_or_create_session(session_id) + started = time.perf_counter() try: - out = super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + with capability.bind_ar_diffusion_state(session_id, state): + output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job) + if self.device is not None and torch.device(self.device).type == "cuda": + torch.accelerator.synchronize(self.device) except Exception: - # Transactional containment: a forward that died partway may have - # written some layers' K/V into allocated-but-uncommitted blocks - # and advanced model-local state. Tear the whole session down - # (engine pool blocks freed via close(); model-local state dropped) - # so the next request with this session id starts clean instead of - # tripping the pending-commit guard or reading half-written KV. - broken = self._ar_diffusion_states.pop(session_id, None) - if broken is not None: - broken.close() - states = getattr(self.pipeline, "_states", None) - if states is not None: - states.pop(session_id, None) + self._release_session( + session_id, + reset_model=False, + reason="forward_exception", + suppress_errors=True, + ) logger.warning( - "AR-Diffusion execute_model failed for session=%s; session state torn down " - "(pool blocks freed) — the next request starts a fresh session", + "AR-Diffusion forward failed for session=%s; KV and model state were released", session_id, ) raise - finally: - self.pipeline._ar_diffusion_kv_state = None - if self.device is not None and torch.cuda.is_available(): - torch.accelerator.synchronize(self.device) - self._perf_e2e_times.append(time.perf_counter() - _e2e_t0) - return out + self._perf_e2e_times.append(time.perf_counter() - started) + if extra_args.get("close_session", False): + self.close_session(session_id) + return output - # -- cuda-graph warm-up ---------------------------------------------------- + def execute_model_batch( + self, + scheduler_output: DiffusionSchedulerOutput, + od_config: OmniDiffusionConfig, + ) -> BatchRunnerOutput: + """Reject request batching until batch-aware AR state binding exists.""" + raise RuntimeError( + "ARDiffusionModelRunner does not support request-batch execution; use request mode with max_num_seqs=1." + ) - #: Dedicated session id for the warm-up rollout; cleaned from both state maps. - _WARMUP_SID = "__ardiffusion_warmup__" - - def _synth_robot_obs(self, h: int, w: int, n_frames: int) -> dict: - """Zero-filled roboarena observation matching the client's obs layout. - - Mirrors ``examples/.../client_schedule.make_obs_from_video`` (2 ext + wrist - cameras, 7/6/1 state vectors) so it survives ``_transform_robot_obs`` exactly - like a real request. One frame -> ``(H,W,3)``; chunk -> ``(n_frames,H,W,3)``. - """ - img = np.zeros((h, w, 3), dtype=np.uint8) if n_frames == 1 else np.zeros((n_frames, h, w, 3), dtype=np.uint8) - return { - "observation/exterior_image_0_left": img, - "observation/exterior_image_1_left": img, - "observation/wrist_image_left": img, - "observation/joint_position": np.zeros(7, dtype=np.float32), - "observation/cartesian_position": np.zeros(6, dtype=np.float32), - "observation/gripper_position": np.zeros(1, dtype=np.float32), - "prompt": "warmup", - "session_id": self._WARMUP_SID, - } + def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput: + """Reject step execution until step-aware AR state binding exists.""" + raise RuntimeError( + "ARDiffusionModelRunner does not support step execution; use request mode with step_execution=False." + ) def _warmup_ar_rollout(self) -> None: - """Drive a synthetic rollout through one window-fill so every DiT graph is - captured at load (off the serving hot path). Frees the synthetic session and - leaves the KV pool exactly as found. Never raises — falls back to lazy capture. - """ - from vllm_omni.inputs.data import OmniDiffusionSamplingParams - - kv = self.kv_cache - if kv is None: + """Run model-provided warmup requests, or safely skip when absent.""" + if self.kv_cache is None or self.pipeline is None: + return + if not isinstance(self.pipeline, SupportsARDiffusionWarmup): + logger.info("AR-Diffusion pipeline provides no warmup requests; skipping rollout warmup") return sid = self._WARMUP_SID - t = self.pipeline.transformer - nfpb = int(t.num_frame_per_block) - window_chunks = int(kv.config.window_chunks or 0) - if window_chunks <= 0: - return - # Distinct DiT window shapes are walked by one fill: prefill + ceil((W-1)/nfpb) - # chunk forwards. +1 optionally to also capture the reset-cycle forward. - n_forwards = 1 + math.ceil(max(0, window_chunks - 1) / nfpb) - if self.ar_diffusion_kv_config.warmup_capture_reset: - n_forwards += 1 - res = self._image_resolution() - h, w = int(res[0]), int(res[1]) - - free_before = kv.manager.block_pool.get_num_free_blocks() - # The step cache is left active: the DiT input shape is per-chunk (the resident - # window length), and the first denoise step of every chunk always computes, so - # every window shape is captured regardless of how many later steps are skipped. - logger.info( - "AR-Diffusion cudagraph warm-up: up to %d synthetic forwards (window_chunks=%d, nfpb=%d)", - n_forwards, - window_chunks, - nfpb, - ) + free_before = self.kv_cache.manager.block_pool.get_num_free_blocks() try: - for i in range(n_forwards): - n_frames = 1 if i == 0 else 4 # client convention: 1-frame prefill, 4-frame chunks - obs = self._synth_robot_obs(h, w, n_frames) - sp = OmniDiffusionSamplingParams(extra_args={"reset": i == 0, "session_id": sid, "robot_obs": obs}) - req = OmniDiffusionRequest(prompts=["warmup"], sampling_params=sp, request_id=f"ardiffusion-warmup-{i}") - self.execute_model(req) - # Stop once the resident window is full — the remaining shapes are - # already captured (the window caps/resets through the same set). - ar_state = self._ar_diffusion_states.get(sid) - if ( - not self.ar_diffusion_kv_config.warmup_capture_reset - and ar_state is not None - and len(kv.window_block_ids(ar_state.pos)) >= window_chunks - ): - break - except Exception as e: # noqa: BLE001 — warm-up must never break model load - logger.warning("AR-Diffusion cudagraph warm-up failed (%s); using lazy capture.", e) + for request in self.pipeline.ar_diffusion_warmup_requests(sid): + self.execute_model(request) + except Exception as exc: # noqa: BLE001 - warmup must not make model load fail + logger.warning("AR-Diffusion rollout warmup failed (%s); using lazy capture", exc) finally: - # Free the synthetic session from BOTH maps (KV pool + pipeline state). - ar_state = self._ar_diffusion_states.pop(sid, None) - if ar_state is not None: - try: - ar_state.close() # return KV blocks to the BlockPool - except Exception as e: # noqa: BLE001 - logger.warning("AR-Diffusion warm-up: state.close() failed (%s)", e) - try: - self.pipeline._states.pop(sid, None) - except Exception: # noqa: BLE001 - pass - # Drop any per-forward timings the warm-up recorded so serving stats start clean. + self._release_session( + sid, + reset_model=False, + reason="warmup_complete", + suppress_errors=True, + ) self._perf_e2e_times.clear() - free_after = kv.manager.block_pool.get_num_free_blocks() + free_after = self.kv_cache.manager.block_pool.get_num_free_blocks() if free_after != free_before: logger.warning( - "AR-Diffusion warm-up: KV pool not restored (free %d -> %d) — possible leak", + "AR-Diffusion warmup did not restore the KV pool (free %d -> %d)", free_before, free_after, ) - else: - logger.info( - "AR-Diffusion cudagraph warm-up complete; KV pool restored (%d free blocks)", - free_after, - ) - - def _image_resolution(self) -> list[int]: - mc = getattr(self.od_config, "model_config", None) or {} - psc = (mc.get("policy_server_config") if isinstance(mc, dict) else None) or {} - return psc.get("image_resolution", [180, 320])