[Core] Session state manager for AR-diffusion world models (RFC #4480 Phase 0) (#4487)

Signed-off-by: Wijnand Suijlen <wijnand.suijlen@huawei.com>
This commit is contained in:
wjsuijlenh authored and GitHub committed 2026-07-27 13:17:38 +08:00
1 parent 9f3a73df17
commit 1f01d05603
16 files changed
+1859 -17

No files matched your search

@@ -61,11 +61,22 @@ from vllm_omni.experimental.ar_diffusion.capability import (
ARDiffusionKVBranchSpec,
ARDiffusionKVCacheSpec,
)
from vllm_omni.experimental.world_models.adapters.state_dreamzero_adapter import DreamZeroStateAdapter
from vllm_omni.experimental.world_models.session_state import (
SessionStateManager,
resolve_session_state_config,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
logger = logging.getLogger(__name__)
MAX_DREAMZERO_SESSIONS = 64
# The pipeline's per-session state is a bespoke ``DreamZeroState`` by default, or
# a ``DreamZeroStateAdapter`` view when the opt-in session manager is enabled.
# The adapter mirrors ``DreamZeroState``'s surface, so every helper that reads or
# writes session state accepts either.
DreamZeroSessionState = DreamZeroState | DreamZeroStateAdapter
class VideoActionScheduler:
"""Wraps video + action schedulers into single .step() interface."""
@@ -110,7 +121,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
_POSITIVE_BRANCH = "positive"
_NEGATIVE_BRANCH = "negative"
_ar_diffusion_kv_state = None
state: DreamZeroState | None
state: DreamZeroSessionState | None
def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec:
"""Describe DreamZero's local KV geometry to the generic runner."""
@@ -163,7 +174,21 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
def _drop_ar_diffusion_session_state(self, session_id: str) -> None:
"""Remove model state and clear the compatibility alias when it points there."""
removed = self._states.pop(str(session_id or "default"), None)
key = str(session_id or "default")
# Local binding narrows the Optional and guards lightweight test fixtures
# that build the pipeline via __new__ without setting _memory_manager.
manager = getattr(self, "_memory_manager", None)
if manager is not None:
# Manager-backed path: the session lives in the manager, not in
# ``_states``. The runner's explicit close/reset is the session's
# end-of-life signal, so release it there (freeing its buffers) and
# drop the alias if it still views this session.
manager.drop_session(key)
current = getattr(self, "state", None)
if isinstance(current, DreamZeroStateAdapter) and str(current.session_id or "default") == key:
self.state = None
return
removed = self._states.pop(key, None)
if removed is not None and getattr(self, "state", None) is removed:
self.state = None
@@ -433,7 +458,22 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
use_dynamic_shifting=False,
)
# Read before the first `_get_or_create_state` below: the manager-backed
# state bounds its VAE encoder history to this many latent frames.
self.num_frame_per_block: int = ah_config["num_frame_per_block"]
self._states: OrderedDict[str, DreamZeroState] = OrderedDict()
# Opt-in: back per-session state with the shared SessionStateManager
# (RFC #4480). Default off -> the bespoke DreamZeroState path above.
self._use_memory_manager, mm_max_sessions = resolve_session_state_config(
enable=od_config.enable_session_state_manager,
max_sessions=MAX_DREAMZERO_SESSIONS,
)
self._memory_manager: SessionStateManager | None = (
SessionStateManager(max_sessions=mm_max_sessions) if self._use_memory_manager else None
)
if self._use_memory_manager:
logger.info("DreamZero: session state manager enabled (max_sessions=%d)", mm_max_sessions)
self.state = self._get_or_create_state("default")
# DiT step cache is configured by StepCacheBackend
@@ -447,7 +487,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
self.cfg_scale: float = model_config.get("cfg_scale", DEFAULT_CFG_SCALE)
self.sigma_shift: float = model_config.get("sigma_shift", DEFAULT_SIGMA_SHIFT)
self.num_frames: int = ah_config["num_frames"]
self.num_frame_per_block: int = ah_config["num_frame_per_block"]
self.action_horizon: int = ah_config["action_horizon"]
self.decouple_inference_noise: bool = ah_config["decouple_inference_noise"]
@@ -502,7 +541,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
),
]
def _get_or_create_state(self, session_id: str | None) -> DreamZeroState:
def _get_or_create_state(self, session_id: str | None) -> DreamZeroState | DreamZeroStateAdapter:
# getattr guards lightweight test fixtures that build the pipeline via
# __new__ and seed only the bespoke fields, never setting _memory_manager.
if getattr(self, "_memory_manager", None) is not None:
# The manager owns session lifecycle and LRU; the adapter is a thin
# per-call view over it (no second LRU to drift).
return DreamZeroStateAdapter(
session_id,
self._memory_manager,
vae_encoder_window=self.num_frame_per_block,
)
session_key = str(session_id or "default")
state = self._states.get(session_key)
if state is None:
@@ -807,7 +857,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
height: int,
width: int,
*,
state: DreamZeroState | None = None,
state: DreamZeroSessionState | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Encode first frame via CLIP + VAE.
Returns: (clip_feas, ys, image_latent)
@@ -898,7 +948,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
inv_std = self.vae_latents_inv_std.to(device=mu.device, dtype=mu.dtype)
return (mu - mean) * inv_std
def _vae_stream_seed(self, state: DreamZeroState, first_frame: torch.Tensor) -> None:
def _vae_stream_seed(self, state: DreamZeroSessionState, first_frame: torch.Tensor) -> None:
"""Seed incremental VAE encode with the first observation frame."""
state.reset_vae_encoder_stream()
feat_map = self._vae_init_enc_feat_map()
@@ -910,7 +960,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
state.vae_pending_body_frames = None
state.vae_stream_initialized = True
def _vae_stream_append_frame(self, state: DreamZeroState, new_frame: torch.Tensor) -> None:
def _vae_stream_append_frame(self, state: DreamZeroSessionState, new_frame: torch.Tensor) -> None:
"""Append one pixel frame and encode a 4-frame body chunk when ready."""
if not state.vae_stream_initialized or state.vae_enc_feat_map is None or state.vae_encoder_out is None:
raise RuntimeError("VAE encoder stream is not initialized.")
@@ -929,19 +979,27 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
state.vae_pending_body_frames = None
chunk = self._vae_patchify(body)
encoder_chunk, feat_map = self._vae_encode_encoder_chunk(chunk, state.vae_enc_feat_map)
state.vae_encoder_out = torch.cat([state.vae_encoder_out, encoder_chunk], dim=2)
state.append_vae_encoder_chunk(encoder_chunk)
state.vae_enc_feat_map = feat_map
def _vae_stream_get_observation_latents(
self,
state: DreamZeroState,
state: DreamZeroSessionState,
num_latent_frames: int,
*,
dtype: torch.dtype,
) -> torch.Tensor:
if state.vae_encoder_out is None:
# Read once: the manager-backed state rebuilds this from its ring.
encoder_out = state.vae_encoder_out
if encoder_out is None:
raise RuntimeError("VAE encoder stream has no accumulated encoder output.")
latents = self._vae_quantize_encoder_out(state.vae_encoder_out).to(dtype=dtype)
# A state that bounds its encoder history keeps only the most recent
# frames. Asking for more than it retains would silently return a
# shorter (and differently padded) window, so fail instead.
window = state.vae_encoder_window
if window is not None and num_latent_frames > window:
raise ValueError(f"Requested {num_latent_frames} latent frames but the session retains at most {window}.")
latents = self._vae_quantize_encoder_out(encoder_out).to(dtype=dtype)
if latents.shape[2] >= num_latent_frames:
return latents[:, :, -num_latent_frames:]
pad_count = num_latent_frames - latents.shape[2]
@@ -962,7 +1020,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
def _encode_observation_latents(
self,
state: DreamZeroState,
state: DreamZeroSessionState,
videos: torch.Tensor,
*,
latent_dtype: torch.dtype,
@@ -1029,7 +1087,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
frame_seqlen: int,
seq_len: int,
do_true_cfg: bool,
state: DreamZeroState,
state: DreamZeroSessionState,
) -> None:
"""Prefill KV cache with first frame and/or current observation.
@@ -1155,7 +1213,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
negative_prompt_embeds: torch.Tensor | None,
video_action_scheduler: VideoActionScheduler,
do_true_cfg: bool,
state: DreamZeroState,
state: DreamZeroSessionState,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Denoising loop with CFG parallel support.
@@ -1211,10 +1269,13 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
)
csf = state.current_start_frame
if csf + self.num_frame_per_block <= state.ys.shape[2]:
y = state.ys[:, :, csf : csf + self.num_frame_per_block]
ys = state.ys
if ys is None:
raise RuntimeError("diffuse() requires state.ys, populated by the forward pass before denoising")
if csf + self.num_frame_per_block <= ys.shape[2]:
y = ys[:, :, csf : csf + self.num_frame_per_block]
else:
y = state.ys[:, :, -self.num_frame_per_block :]
y = ys[:, :, -self.num_frame_per_block :]
run_dit = _step_cache is None or _step_cache.should_run_step(_prev_predictions)
if run_dit:
@@ -134,6 +134,18 @@ class DreamZeroState:
self.vae_encoder_out: torch.Tensor | None = None
self.vae_pending_body_frames: torch.Tensor | None = None
# How many latent frames of encoder output this state retains. ``None``
# means "everything", which is what this path has always done -- the
# accumulated output is cleared only by a session reset, so it grows for as
# long as the session lives. The manager-backed adapter bounds it instead.
vae_encoder_window: int | None = None
def append_vae_encoder_chunk(self, encoder_chunk: torch.Tensor) -> None:
"""Append one encoded chunk to the accumulated encoder output."""
if self.vae_encoder_out is None:
raise RuntimeError("VAE encoder stream is not seeded; call reset_vae_encoder_stream first.")
self.vae_encoder_out = torch.cat([self.vae_encoder_out, encoder_chunk], dim=2)
def reset_inference_state(self) -> None:
"""Reset KV/frame state after local attention rolls without dropping video latents."""
self.reset(clear_video_latents=False)