Signed-off-by: Wijnand Suijlen <wijnand.suijlen@huawei.com>
This commit is contained in:
1 parent
9f3a73df17
commit
1f01d05603
16 files changed
+1859
-17
No files matched your search
@@ -61,11 +61,22 @@ from vllm_omni.experimental.ar_diffusion.capability import (
|
||||
ARDiffusionKVBranchSpec,
|
||||
ARDiffusionKVCacheSpec,
|
||||
)
|
||||
from vllm_omni.experimental.world_models.adapters.state_dreamzero_adapter import DreamZeroStateAdapter
|
||||
from vllm_omni.experimental.world_models.session_state import (
|
||||
SessionStateManager,
|
||||
resolve_session_state_config,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
MAX_DREAMZERO_SESSIONS = 64
|
||||
|
||||
# The pipeline's per-session state is a bespoke ``DreamZeroState`` by default, or
|
||||
# a ``DreamZeroStateAdapter`` view when the opt-in session manager is enabled.
|
||||
# The adapter mirrors ``DreamZeroState``'s surface, so every helper that reads or
|
||||
# writes session state accepts either.
|
||||
DreamZeroSessionState = DreamZeroState | DreamZeroStateAdapter
|
||||
|
||||
|
||||
class VideoActionScheduler:
|
||||
"""Wraps video + action schedulers into single .step() interface."""
|
||||
@@ -110,7 +121,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
_POSITIVE_BRANCH = "positive"
|
||||
_NEGATIVE_BRANCH = "negative"
|
||||
_ar_diffusion_kv_state = None
|
||||
state: DreamZeroState | None
|
||||
state: DreamZeroSessionState | None
|
||||
|
||||
def ar_diffusion_kv_cache_spec(self) -> ARDiffusionKVCacheSpec:
|
||||
"""Describe DreamZero's local KV geometry to the generic runner."""
|
||||
@@ -163,7 +174,21 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
|
||||
def _drop_ar_diffusion_session_state(self, session_id: str) -> None:
|
||||
"""Remove model state and clear the compatibility alias when it points there."""
|
||||
removed = self._states.pop(str(session_id or "default"), None)
|
||||
key = str(session_id or "default")
|
||||
# Local binding narrows the Optional and guards lightweight test fixtures
|
||||
# that build the pipeline via __new__ without setting _memory_manager.
|
||||
manager = getattr(self, "_memory_manager", None)
|
||||
if manager is not None:
|
||||
# Manager-backed path: the session lives in the manager, not in
|
||||
# ``_states``. The runner's explicit close/reset is the session's
|
||||
# end-of-life signal, so release it there (freeing its buffers) and
|
||||
# drop the alias if it still views this session.
|
||||
manager.drop_session(key)
|
||||
current = getattr(self, "state", None)
|
||||
if isinstance(current, DreamZeroStateAdapter) and str(current.session_id or "default") == key:
|
||||
self.state = None
|
||||
return
|
||||
removed = self._states.pop(key, None)
|
||||
if removed is not None and getattr(self, "state", None) is removed:
|
||||
self.state = None
|
||||
|
||||
@@ -433,7 +458,22 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
use_dynamic_shifting=False,
|
||||
)
|
||||
|
||||
# Read before the first `_get_or_create_state` below: the manager-backed
|
||||
# state bounds its VAE encoder history to this many latent frames.
|
||||
self.num_frame_per_block: int = ah_config["num_frame_per_block"]
|
||||
|
||||
self._states: OrderedDict[str, DreamZeroState] = OrderedDict()
|
||||
# Opt-in: back per-session state with the shared SessionStateManager
|
||||
# (RFC #4480). Default off -> the bespoke DreamZeroState path above.
|
||||
self._use_memory_manager, mm_max_sessions = resolve_session_state_config(
|
||||
enable=od_config.enable_session_state_manager,
|
||||
max_sessions=MAX_DREAMZERO_SESSIONS,
|
||||
)
|
||||
self._memory_manager: SessionStateManager | None = (
|
||||
SessionStateManager(max_sessions=mm_max_sessions) if self._use_memory_manager else None
|
||||
)
|
||||
if self._use_memory_manager:
|
||||
logger.info("DreamZero: session state manager enabled (max_sessions=%d)", mm_max_sessions)
|
||||
self.state = self._get_or_create_state("default")
|
||||
|
||||
# DiT step cache is configured by StepCacheBackend
|
||||
@@ -447,7 +487,6 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
self.cfg_scale: float = model_config.get("cfg_scale", DEFAULT_CFG_SCALE)
|
||||
self.sigma_shift: float = model_config.get("sigma_shift", DEFAULT_SIGMA_SHIFT)
|
||||
self.num_frames: int = ah_config["num_frames"]
|
||||
self.num_frame_per_block: int = ah_config["num_frame_per_block"]
|
||||
self.action_horizon: int = ah_config["action_horizon"]
|
||||
|
||||
self.decouple_inference_noise: bool = ah_config["decouple_inference_noise"]
|
||||
@@ -502,7 +541,18 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
),
|
||||
]
|
||||
|
||||
def _get_or_create_state(self, session_id: str | None) -> DreamZeroState:
|
||||
def _get_or_create_state(self, session_id: str | None) -> DreamZeroState | DreamZeroStateAdapter:
|
||||
# getattr guards lightweight test fixtures that build the pipeline via
|
||||
# __new__ and seed only the bespoke fields, never setting _memory_manager.
|
||||
if getattr(self, "_memory_manager", None) is not None:
|
||||
# The manager owns session lifecycle and LRU; the adapter is a thin
|
||||
# per-call view over it (no second LRU to drift).
|
||||
return DreamZeroStateAdapter(
|
||||
session_id,
|
||||
self._memory_manager,
|
||||
vae_encoder_window=self.num_frame_per_block,
|
||||
)
|
||||
|
||||
session_key = str(session_id or "default")
|
||||
state = self._states.get(session_key)
|
||||
if state is None:
|
||||
@@ -807,7 +857,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
height: int,
|
||||
width: int,
|
||||
*,
|
||||
state: DreamZeroState | None = None,
|
||||
state: DreamZeroSessionState | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Encode first frame via CLIP + VAE.
|
||||
Returns: (clip_feas, ys, image_latent)
|
||||
@@ -898,7 +948,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
inv_std = self.vae_latents_inv_std.to(device=mu.device, dtype=mu.dtype)
|
||||
return (mu - mean) * inv_std
|
||||
|
||||
def _vae_stream_seed(self, state: DreamZeroState, first_frame: torch.Tensor) -> None:
|
||||
def _vae_stream_seed(self, state: DreamZeroSessionState, first_frame: torch.Tensor) -> None:
|
||||
"""Seed incremental VAE encode with the first observation frame."""
|
||||
state.reset_vae_encoder_stream()
|
||||
feat_map = self._vae_init_enc_feat_map()
|
||||
@@ -910,7 +960,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
state.vae_pending_body_frames = None
|
||||
state.vae_stream_initialized = True
|
||||
|
||||
def _vae_stream_append_frame(self, state: DreamZeroState, new_frame: torch.Tensor) -> None:
|
||||
def _vae_stream_append_frame(self, state: DreamZeroSessionState, new_frame: torch.Tensor) -> None:
|
||||
"""Append one pixel frame and encode a 4-frame body chunk when ready."""
|
||||
if not state.vae_stream_initialized or state.vae_enc_feat_map is None or state.vae_encoder_out is None:
|
||||
raise RuntimeError("VAE encoder stream is not initialized.")
|
||||
@@ -929,19 +979,27 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
state.vae_pending_body_frames = None
|
||||
chunk = self._vae_patchify(body)
|
||||
encoder_chunk, feat_map = self._vae_encode_encoder_chunk(chunk, state.vae_enc_feat_map)
|
||||
state.vae_encoder_out = torch.cat([state.vae_encoder_out, encoder_chunk], dim=2)
|
||||
state.append_vae_encoder_chunk(encoder_chunk)
|
||||
state.vae_enc_feat_map = feat_map
|
||||
|
||||
def _vae_stream_get_observation_latents(
|
||||
self,
|
||||
state: DreamZeroState,
|
||||
state: DreamZeroSessionState,
|
||||
num_latent_frames: int,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
if state.vae_encoder_out is None:
|
||||
# Read once: the manager-backed state rebuilds this from its ring.
|
||||
encoder_out = state.vae_encoder_out
|
||||
if encoder_out is None:
|
||||
raise RuntimeError("VAE encoder stream has no accumulated encoder output.")
|
||||
latents = self._vae_quantize_encoder_out(state.vae_encoder_out).to(dtype=dtype)
|
||||
# A state that bounds its encoder history keeps only the most recent
|
||||
# frames. Asking for more than it retains would silently return a
|
||||
# shorter (and differently padded) window, so fail instead.
|
||||
window = state.vae_encoder_window
|
||||
if window is not None and num_latent_frames > window:
|
||||
raise ValueError(f"Requested {num_latent_frames} latent frames but the session retains at most {window}.")
|
||||
latents = self._vae_quantize_encoder_out(encoder_out).to(dtype=dtype)
|
||||
if latents.shape[2] >= num_latent_frames:
|
||||
return latents[:, :, -num_latent_frames:]
|
||||
pad_count = num_latent_frames - latents.shape[2]
|
||||
@@ -962,7 +1020,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
|
||||
def _encode_observation_latents(
|
||||
self,
|
||||
state: DreamZeroState,
|
||||
state: DreamZeroSessionState,
|
||||
videos: torch.Tensor,
|
||||
*,
|
||||
latent_dtype: torch.dtype,
|
||||
@@ -1029,7 +1087,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
frame_seqlen: int,
|
||||
seq_len: int,
|
||||
do_true_cfg: bool,
|
||||
state: DreamZeroState,
|
||||
state: DreamZeroSessionState,
|
||||
) -> None:
|
||||
"""Prefill KV cache with first frame and/or current observation.
|
||||
|
||||
@@ -1155,7 +1213,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
video_action_scheduler: VideoActionScheduler,
|
||||
do_true_cfg: bool,
|
||||
state: DreamZeroState,
|
||||
state: DreamZeroSessionState,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Denoising loop with CFG parallel support.
|
||||
@@ -1211,10 +1269,13 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
)
|
||||
|
||||
csf = state.current_start_frame
|
||||
if csf + self.num_frame_per_block <= state.ys.shape[2]:
|
||||
y = state.ys[:, :, csf : csf + self.num_frame_per_block]
|
||||
ys = state.ys
|
||||
if ys is None:
|
||||
raise RuntimeError("diffuse() requires state.ys, populated by the forward pass before denoising")
|
||||
if csf + self.num_frame_per_block <= ys.shape[2]:
|
||||
y = ys[:, :, csf : csf + self.num_frame_per_block]
|
||||
else:
|
||||
y = state.ys[:, :, -self.num_frame_per_block :]
|
||||
y = ys[:, :, -self.num_frame_per_block :]
|
||||
|
||||
run_dit = _step_cache is None or _step_cache.should_run_step(_prev_predictions)
|
||||
if run_dit:
|
||||
|
||||
@@ -134,6 +134,18 @@ class DreamZeroState:
|
||||
self.vae_encoder_out: torch.Tensor | None = None
|
||||
self.vae_pending_body_frames: torch.Tensor | None = None
|
||||
|
||||
# How many latent frames of encoder output this state retains. ``None``
|
||||
# means "everything", which is what this path has always done -- the
|
||||
# accumulated output is cleared only by a session reset, so it grows for as
|
||||
# long as the session lives. The manager-backed adapter bounds it instead.
|
||||
vae_encoder_window: int | None = None
|
||||
|
||||
def append_vae_encoder_chunk(self, encoder_chunk: torch.Tensor) -> None:
|
||||
"""Append one encoded chunk to the accumulated encoder output."""
|
||||
if self.vae_encoder_out is None:
|
||||
raise RuntimeError("VAE encoder stream is not seeded; call reset_vae_encoder_stream first.")
|
||||
self.vae_encoder_out = torch.cat([self.vae_encoder_out, encoder_chunk], dim=2)
|
||||
|
||||
def reset_inference_state(self) -> None:
|
||||
"""Reset KV/frame state after local attention rolls without dropping video latents."""
|
||||
self.reset(clear_video_latents=False)
|
||||
|
||||
Reference in new issue
Block a user