[Core] Session state manager for AR-diffusion world models (RFC #4480 Phase 0) (#4487)

Signed-off-by: Wijnand Suijlen <wijnand.suijlen@huawei.com>
This commit is contained in:
wjsuijlenh authored and GitHub committed 2026-07-27 13:17:38 +08:00
1 parent 9f3a73df17
commit 1f01d05603
16 files changed
+1859 -17

No files matched your search

@@ -0,0 +1,201 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-device byte accounting and the bounded VAE encoder ring (RFC #4480).
Two things are pinned here. First, that a session reports the bytes it actually
holds: both buckets walked, storages rather than logical element counts, and
aliases counted once. Second, that bounding the adapter's VAE encoder history
changes no numbers -- ``quant_conv`` is pointwise in time, so the frames the
reader keeps do not depend on the ones the ring drops.
All CPU, tiny tensors, no model.
"""
from __future__ import annotations
import numpy as np
import pytest
import torch
from vllm_omni.experimental.world_models.adapters.state_dreamzero_adapter import (
DreamZeroStateAdapter,
)
from vllm_omni.experimental.world_models.session_state import (
LatentBuffer,
SessionStateManager,
device_bytes,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _bytes(value: object) -> dict[str, int]:
return device_bytes(value, set())
# -- device_bytes ------------------------------------------------------------
def test_counts_tensor_storage_under_its_device() -> None:
tensor = torch.zeros(64, dtype=torch.float32)
assert _bytes(tensor) == {"cpu": 256}
def test_views_on_one_storage_counted_once() -> None:
tensor = torch.zeros(64, dtype=torch.float32)
seen: set[tuple[str, int]] = set()
first = device_bytes(tensor, seen)
second = device_bytes(tensor[:8], seen)
assert first == {"cpu": 256}
assert second == {}
def test_slice_reports_the_storage_it_pins() -> None:
# The whole point of counting storages: this slice keeps all 256 bytes
# alive, and ``numel() * element_size()`` would claim 32.
sliced = torch.zeros(64, dtype=torch.float32)[:8]
assert _bytes(sliced) == {"cpu": 256}
def test_containers_recurse_and_scalars_contribute_nothing() -> None:
payload = {
"tensors": [torch.zeros(16, dtype=torch.float32), None],
"count": 7,
"flag": True,
"name": "not-a-buffer",
}
assert _bytes(payload) == {"cpu": 64}
def test_numpy_arrays_land_on_the_host() -> None:
assert _bytes(np.zeros(32, dtype=np.float32)) == {"cpu": 128}
# -- SessionState / manager --------------------------------------------------
def test_session_counts_both_buckets_and_dedupes_across_them() -> None:
manager = SessionStateManager()
session = manager.get_or_create_session("s")
buffer: LatentBuffer[torch.Tensor] = LatentBuffer()
buffer.allocate(maxlen=None)
shared = torch.zeros(64, dtype=torch.float32) # 256 bytes
buffer.append(shared)
session.put("latents", buffer)
session.attrs["conditioning"] = torch.zeros(16, dtype=torch.float32) # 64 bytes
assert session.nbytes_by_device() == {"cpu": 320}
# The same storage reachable from both buckets is one allocation, not two.
session.attrs["alias"] = shared
assert session.nbytes_by_device() == {"cpu": 320}
def test_stats_reports_per_device_totals() -> None:
manager = SessionStateManager()
manager.get_or_create_session("s").attrs["x"] = torch.zeros(64, dtype=torch.float32)
stats = manager.stats()
assert stats["nbytes:cpu"] == 256
assert stats["total_nbytes"] == 256
def test_attribute_bytes_were_previously_invisible() -> None:
"""The defect behind tzhouam's review comment on PR #4487.
Every byte here lives in ``attrs``, and the old ``nbytes`` walked only the
typed objects -- so a session holding real memory reported zero.
"""
manager = SessionStateManager()
session = manager.get_or_create_session("s")
session.attrs["conv_cache"] = [torch.zeros(1024, dtype=torch.float32)]
assert session.nbytes_by_device() == {"cpu": 4096}
# -- the largest value DreamZero holds ---------------------------------------
def test_feat_map_bytes_are_counted() -> None:
"""``vae_enc_feat_map`` is the largest thing a DreamZero session holds --
603 MiB at the shipped config -- and it stays an attribute, so this is
exactly the memory the old accounting missed."""
manager = SessionStateManager()
adapter = DreamZeroStateAdapter("s", manager)
adapter.vae_enc_feat_map = [torch.zeros(1024, dtype=torch.float32), None]
assert manager.stats()["nbytes:cpu"] == 4096
# Freeing it piecemeal is not something the manager can do: the cache has no
# recompute source and the stream breaks without it. Ending the stream (or
# the session) is the only release, and that reclaims attributes too.
adapter.reset_vae_encoder_stream()
assert manager.stats().get("nbytes:cpu", 0) == 0
# -- bounded VAE encoder ring ------------------------------------------------
def _quantize(encoder_out: torch.Tensor) -> torch.Tensor:
"""Stand-in for ``_vae_quantize_encoder_out``.
The real one is ``WanCausalConv3d(z_dim*2, z_dim*2, 1)`` followed by a
per-channel affine rescale. Every operation is pointwise in time, which is
the property the ring relies on; a 1x1x1 convolution reproduces it.
"""
weight = torch.arange(1, encoder_out.shape[1] + 1, dtype=encoder_out.dtype).view(1, -1, 1, 1, 1)
return encoder_out * weight + 0.5
def test_ring_is_bounded() -> None:
adapter = DreamZeroStateAdapter("s", SessionStateManager(), vae_encoder_window=2)
adapter.vae_encoder_out = torch.randn(1, 4, 1, 2, 2)
for _ in range(10):
adapter.append_vae_encoder_chunk(torch.randn(1, 4, 1, 2, 2))
assert adapter.vae_encoder_out is not None
assert adapter.vae_encoder_out.shape[2] == 2
def test_ring_matches_unbounded_history_for_the_frames_that_are_read() -> None:
window = 2
adapter = DreamZeroStateAdapter("s", SessionStateManager(), vae_encoder_window=window)
seed = torch.randn(1, 4, 1, 2, 2)
chunks = [torch.randn(1, 4, 1, 2, 2) for _ in range(9)]
adapter.vae_encoder_out = seed
unbounded = seed
for chunk in chunks:
adapter.append_vae_encoder_chunk(chunk)
unbounded = torch.cat([unbounded, chunk], dim=2)
# What the reader actually returns: quantise, then keep the last `window`.
from_ring = _quantize(adapter.vae_encoder_out)[:, :, -window:]
from_history = _quantize(unbounded)[:, :, -window:]
torch.testing.assert_close(from_ring, from_history, rtol=0, atol=0)
def test_ring_holds_a_single_frame_before_the_first_body_chunk() -> None:
# After `_vae_stream_seed` there is exactly one latent frame, which is what
# sends the reader down its padding branch.
adapter = DreamZeroStateAdapter("s", SessionStateManager(), vae_encoder_window=2)
seed = torch.randn(1, 4, 1, 2, 2)
adapter.vae_encoder_out = seed
assert adapter.vae_encoder_out is not None
assert adapter.vae_encoder_out.shape[2] == 1
torch.testing.assert_close(adapter.vae_encoder_out, seed, rtol=0, atol=0)
def test_resetting_the_stream_clears_the_ring() -> None:
adapter = DreamZeroStateAdapter("s", SessionStateManager(), vae_encoder_window=2)
adapter.vae_encoder_out = torch.randn(1, 4, 1, 2, 2)
adapter.reset_vae_encoder_stream()
assert adapter.vae_encoder_out is None
def test_window_must_be_positive() -> None:
with pytest.raises(ValueError, match="vae_encoder_window"):
DreamZeroStateAdapter("s", SessionStateManager(), vae_encoder_window=0)
@@ -0,0 +1,257 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bit-equivalence tests: DreamZeroStateAdapter vs bespoke DreamZeroState.
These drive both state objects directly with tiny CPU tensors -- no model, no
GPU -- and assert the adapter (backed by the SessionStateManager) stores and
returns exactly what the bespoke DreamZeroState does. This gates routing
DreamZero through the shared session state manager (RFC #4480).
DreamZero's attention KV is engine-owned (AR-Diffusion paged pool, PR #4534)
and out of the adapter's scope; the state surface tested here is the model's
non-KV session state.
"""
from __future__ import annotations
import numpy as np
import pytest
import torch
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import (
FRAMES_PER_CHUNK,
DreamZeroState,
)
from vllm_omni.experimental.world_models.adapters.state_dreamzero_adapter import DreamZeroStateAdapter
from vllm_omni.experimental.world_models.session_state import SessionStateManager
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _create_both() -> tuple[DreamZeroState, DreamZeroStateAdapter]:
return DreamZeroState(), DreamZeroStateAdapter("session-0", SessionStateManager())
def test_accumulate_frames_and_call_count_match() -> None:
bespoke, adapter = _create_both()
rng = np.random.default_rng(0)
for _ in range(2 * FRAMES_PER_CHUNK + 1):
frame = rng.integers(0, 255, size=(5, 6, 3), dtype=np.uint8)
b_out = bespoke.accumulate_frames(frame)
a_out = adapter.accumulate_frames(frame)
np.testing.assert_array_equal(b_out, a_out)
assert bespoke.call_count == adapter.call_count
assert len(bespoke.stitched_buffer) == len(adapter.stitched_buffer)
# Deque rollover capped at FRAMES_PER_CHUNK.
assert len(adapter.stitched_buffer) == FRAMES_PER_CHUNK
def test_accumulate_frames_4d_extend_matches() -> None:
bespoke, adapter = _create_both()
clip = np.random.default_rng(1).integers(0, 255, size=(3, 5, 6, 3), dtype=np.uint8)
np.testing.assert_array_equal(bespoke.accumulate_frames(clip), adapter.accumulate_frames(clip))
assert len(bespoke.stitched_buffer) == len(adapter.stitched_buffer)
def test_video_latents_roundtrip_matches() -> None:
bespoke, adapter = _create_both()
assert bespoke.get_concatenated_video_latents() is None
assert adapter.get_concatenated_video_latents() is None
rng = torch.Generator().manual_seed(0)
for _ in range(3):
# (B, T, C, H, W) as returned by the denoise loop.
chunk = torch.randn(1, 2, 3, 4, 4, generator=rng)
bespoke.append_video_latents(chunk)
adapter.append_video_latents(chunk)
torch.testing.assert_close(
bespoke.get_concatenated_video_latents(),
adapter.get_concatenated_video_latents(),
)
bespoke.clear_video_latents()
adapter.clear_video_latents()
assert bespoke.get_concatenated_video_latents() is None
assert adapter.get_concatenated_video_latents() is None
def test_vae_stream_fields_match_bespoke() -> None:
bespoke, adapter = _create_both()
for state in (bespoke, adapter):
assert state.vae_stream_initialized is False
assert state.vae_enc_feat_map is None
assert state.vae_encoder_out is None
assert state.vae_pending_body_frames is None
feat_map: list[torch.Tensor | None] = [torch.ones(1, 2), None]
encoder_out = torch.randn(1, 4, 2, 3, 3)
pending = torch.randn(1, 3, 2, 8, 8)
for state in (bespoke, adapter):
state.vae_enc_feat_map = feat_map
state.vae_encoder_out = encoder_out
state.vae_pending_body_frames = pending
state.vae_stream_initialized = True
assert adapter.vae_stream_initialized is bespoke.vae_stream_initialized is True
torch.testing.assert_close(adapter.vae_encoder_out, bespoke.vae_encoder_out)
torch.testing.assert_close(adapter.vae_pending_body_frames, bespoke.vae_pending_body_frames)
assert adapter.vae_enc_feat_map is feat_map
for state in (bespoke, adapter):
state.reset_vae_encoder_stream()
assert state.vae_stream_initialized is False
assert state.vae_enc_feat_map is None
assert state.vae_encoder_out is None
assert state.vae_pending_body_frames is None
def test_reset_clears_state_and_should_reset_matches() -> None:
bespoke, adapter = _create_both()
tokens = torch.tensor([1, 2, 3])
# language unset -> both want reset.
assert bespoke.should_reset(tokens, 4, -1) is adapter.should_reset(tokens, 4, -1) is True
for state in (bespoke, adapter):
state.language = tokens
state.current_start_frame = 2
# same language -> no reset; both agree.
assert bespoke.should_reset(tokens, 4, -1) == adapter.should_reset(tokens, 4, -1) is False
# language change -> reset.
other = torch.tensor([9, 9])
assert bespoke.should_reset(other, 4, -1) == adapter.should_reset(other, 4, -1) is True
# local_attn_size exceeded -> reset ("inference", not "session").
assert bespoke.reset_reason(tokens, 4, 1) == adapter.reset_reason(tokens, 4, 1) == "inference"
# A session reset clears everything, including the prompt-embed cache and
# the VAE encoder stream (mirrors DreamZeroState.reset()).
for state in (bespoke, adapter):
state.prompt_embeds = torch.ones(1, 2)
state.vae_encoder_out = torch.ones(1, 4, 1, 2, 2)
state.vae_stream_initialized = True
state.reset()
assert state.language is None
assert state.prompt_embeds is None
assert state.current_start_frame == 0
assert state.call_count == 0
assert state.vae_stream_initialized is False
assert state.vae_encoder_out is None
def test_window_reset_preserves_latents_prompt_and_vae_stream() -> None:
# reset(clear_video_latents=False) is the window ("inference") reset: the
# prompt is unchanged and the VAE feat_cache history is independent of the
# DiT attention window, so both survive alongside the video latents.
bespoke, adapter = _create_both()
tokens = torch.tensor([1, 2, 3])
embeds = torch.randn(1, 4)
encoder_out = torch.randn(1, 4, 2, 3, 3)
chunk = torch.randn(1, 2, 3, 4, 4)
for state in (bespoke, adapter):
state.language = tokens
state.prompt_embeds = embeds
state.vae_encoder_out = encoder_out
state.vae_stream_initialized = True
state.current_start_frame = 7
state.append_video_latents(chunk)
state.reset_inference_state()
assert state.current_start_frame == 0
assert state.call_count == 0
assert state.clip_feas is None
torch.testing.assert_close(state.language, tokens)
torch.testing.assert_close(state.prompt_embeds, embeds)
assert state.vae_stream_initialized is True
torch.testing.assert_close(state.vae_encoder_out, encoder_out)
assert state.get_concatenated_video_latents() is not None
torch.testing.assert_close(
bespoke.get_concatenated_video_latents(),
adapter.get_concatenated_video_latents(),
)
def test_metadata_fields_persist_across_adapter_instances() -> None:
# A fresh adapter for the same session must see prior metadata (the manager
# is the single source of truth).
manager = SessionStateManager()
first = DreamZeroStateAdapter("s", manager)
first.current_start_frame = 5
first.clip_feas = torch.ones(2, 2)
first.append_video_latents(torch.ones(1, 2, 3, 4, 4))
second = DreamZeroStateAdapter("s", manager)
assert second.current_start_frame == 5
torch.testing.assert_close(second.clip_feas, torch.ones(2, 2))
# Video latents appended via `first` are visible via `second`.
assert second.get_concatenated_video_latents() is not None
def test_session_lru_caps_retained_sessions() -> None:
manager = SessionStateManager(max_sessions=3)
for i in range(5):
DreamZeroStateAdapter(f"s{i}", manager)
assert len(manager) == 3
# Oldest two evicted; newest three retained.
assert "s0" not in manager
assert "s1" not in manager
for i in (2, 3, 4):
assert f"s{i}" in manager
def test_adapter_keeps_state_when_its_session_is_evicted() -> None:
# Regression: an adapter mid-generation must not lose its data when the
# manager evicts its session to make room for newer ones. The adapter pins
# the session, so its stored latents survive eviction from the lookup table.
manager = SessionStateManager(max_sessions=2)
adapter = DreamZeroStateAdapter("active", manager)
chunk = torch.randn(1, 2, 3, 4, 4)
adapter.append_video_latents(chunk)
adapter.language = torch.tensor([1, 2, 3])
# Push "active" out of the manager's table with newer sessions.
for i in range(3):
DreamZeroStateAdapter(f"other{i}", manager)
assert "active" not in manager
# The in-use adapter still returns its own data.
assert adapter.get_concatenated_video_latents() is not None
torch.testing.assert_close(adapter.language, torch.tensor([1, 2, 3]))
def test_session_reset_clears_metadata() -> None:
# Regression: resetting a session must not leave stale metadata behind, so a
# fresh adapter for the same session sees defaults, not the old values.
manager = SessionStateManager()
adapter = DreamZeroStateAdapter("s", manager)
adapter.language = torch.tensor([1, 2, 3])
adapter.current_start_frame = 5
adapter.clip_feas = torch.ones(2, 2)
adapter.prompt_embeds = torch.ones(1, 2)
adapter.reset()
fresh = DreamZeroStateAdapter("s", manager)
assert fresh.language is None
assert fresh.current_start_frame == 0
assert fresh.clip_feas is None
assert fresh.prompt_embeds is None
def test_bespoke_path_without_memory_manager() -> None:
# Regression: a pipeline built via __new__ (lightweight fixtures) seeds only
# the bespoke fields and never sets _memory_manager; _get_or_create_state
# must fall through to the bespoke DreamZeroState path without AttributeError.
from collections import OrderedDict
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import (
MAX_DREAMZERO_SESSIONS,
DreamZeroPipeline,
)
pipe = DreamZeroPipeline.__new__(DreamZeroPipeline)
pipe._states = OrderedDict()
pipe._max_session_states = MAX_DREAMZERO_SESSIONS
state = DreamZeroPipeline._get_or_create_state(pipe, "x")
assert isinstance(state, DreamZeroState)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Session end-of-life routing (RFC #4480, adapting to #5271).
The AR-Diffusion runner raises an explicit end-of-session signal on reset,
close, eviction, and failed forwards (``reset_ar_diffusion_session`` /
``close_ar_diffusion_session``). On the bespoke path those pop ``_states``; on
the opt-in manager path the session lives in the manager instead, so the hooks
must release it there or every closed session leaks its buffers.
All CPU, tiny tensors, no model.
"""
from __future__ import annotations
from collections import OrderedDict
import pytest
import torch
from vllm_omni.diffusion.models.dreamzero.pipeline_dreamzero import DreamZeroPipeline
from vllm_omni.diffusion.models.dreamzero.state_dreamzero import DreamZeroState
from vllm_omni.experimental.world_models.adapters.state_dreamzero_adapter import (
DreamZeroStateAdapter,
)
from vllm_omni.experimental.world_models.session_state import (
LatentBuffer,
SessionStateManager,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# -- SessionStateManager.drop_session ---------------------------------------
def test_drop_session_absent_returns_false() -> None:
manager = SessionStateManager()
assert manager.drop_session("missing") is False
def test_drop_session_removes_and_frees() -> None:
manager = SessionStateManager()
session = manager.get_or_create_session("s")
buffer: LatentBuffer[torch.Tensor] = LatentBuffer()
buffer.allocate(maxlen=None)
buffer.append(torch.zeros(256, dtype=torch.float32))
session.put("payload", buffer)
assert manager.nbytes_by_device().get("cpu", 0) >= 1024
assert manager.drop_session("s") is True
assert "s" not in manager
assert len(manager) == 0
# The session is gone from the table and its buffers were reset, so the
# manager reports no held bytes.
assert manager.nbytes_by_device() == {}
def test_drop_session_recreates_fresh() -> None:
manager = SessionStateManager()
manager.get_or_create_session("s").attrs["k"] = 1
manager.drop_session("s")
assert manager.get_or_create_session("s").attrs == {}
# -- pipeline hook routing ---------------------------------------------------
def _manager_pipe(manager: SessionStateManager, session_id: str) -> DreamZeroPipeline:
"""A pipeline built via __new__ whose state is a manager-backed adapter."""
pipe = DreamZeroPipeline.__new__(DreamZeroPipeline)
pipe._states = OrderedDict()
pipe._memory_manager = manager
pipe.state = DreamZeroStateAdapter(session_id, manager, vae_encoder_window=2)
return pipe
@pytest.mark.parametrize("hook", ["close_ar_diffusion_session", "reset_ar_diffusion_session"])
def test_hook_releases_manager_session_and_clears_alias(hook: str) -> None:
manager = SessionStateManager()
pipe = _manager_pipe(manager, "sess")
assert "sess" in manager
getattr(DreamZeroPipeline, hook)(pipe, "sess")
assert "sess" not in manager
# The alias viewed the released session, so it is dropped and rebuilt lazily.
assert pipe.state is None
def test_hook_keeps_alias_for_a_different_session() -> None:
manager = SessionStateManager()
pipe = _manager_pipe(manager, "default")
manager.get_or_create_session("other")
DreamZeroPipeline.close_ar_diffusion_session(pipe, "other")
assert "other" not in manager
# The alias views "default", not the closed session, so it survives.
assert isinstance(pipe.state, DreamZeroStateAdapter)
assert pipe.state.session_id == "default"
def test_bespoke_hook_pops_states_and_clears_alias() -> None:
pipe = DreamZeroPipeline.__new__(DreamZeroPipeline)
pipe._states = OrderedDict()
pipe._memory_manager = None
state = DreamZeroState()
pipe._states["sess"] = state
pipe.state = state
DreamZeroPipeline.close_ar_diffusion_session(pipe, "sess")
assert "sess" not in pipe._states
assert pipe.state is None