[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-09-20 16:19:41 +00:00
1 parent 139a47a578
commit e36babd48f
26 files changed
+2071 -200

No files matched your search

+87 -8
View File
@@ -11,8 +11,9 @@ from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
InteractionPayload,
merge_interaction_metadata,
synchronized_monotonic_time,
)
from vllm_omni.diffusion.models.interface import supports_interaction_apply
from vllm_omni.diffusion.models.interface import SupportsInteractionApply, supports_interaction_apply
from vllm_omni.diffusion.worker.utils import StepRequestState
if TYPE_CHECKING:
@@ -52,7 +53,7 @@ class InteractionCoordinator:
@property
def needs_chunk_media(self) -> bool:
"""Whether any registered handler needs chunk ``num_frames``/``fps``."""
"""Whether any registered handler needs chunk media/latent counts + fps."""
return any(handler.needs_chunk_media for handler in self._handlers.values())
def get_handler(self, modality: str) -> InteractionHandler:
@@ -83,31 +84,109 @@ class InteractionCoordinator:
transition_chunks=transition_chunks,
)
def enqueue_parts(
self,
state: StepRequestState,
*,
parts: list[tuple[str, InteractionPayload]],
event_id: str,
received_at: float,
transition_chunks: int | None,
) -> None:
"""Ensure every modality is supported and every payload validates, then enqueue.
Composite events must not partially mutate queues when a later track is
unsupported or carries an invalid payload.
"""
if not parts:
raise ValueError("interaction event requires prompt and/or multi_modal_data")
# Reconcile rank-local samples so USP/SP shards share one arrival time.
received_at = synchronized_monotonic_time(received_at)
handlers = [(self.get_handler(modality), payload) for modality, payload in parts]
for handler, payload in handlers:
handler.validate_payload(
state,
event_id=event_id,
payload=payload,
transition_chunks=transition_chunks,
)
for (modality, payload), (handler, _) in zip(parts, handlers, strict=True):
del handler
self.enqueue(
state,
modality=modality,
event_id=event_id,
received_at=received_at,
payload=payload,
transition_chunks=transition_chunks,
)
def maybe_prepare_initial_session(
self,
state: StepRequestState,
pipeline: SupportsInteractionApply,
) -> InteractionChunkMetadata:
"""Materialize non-lazy modality sessions before chunk 0 denoise.
Handlers with ``lazy_initialize_session=False`` (e.g. camera) must exist
before the first chunk even with no client enqueue.
"""
num_media_frames: int | None = None
fps: float | None = None
num_latent_frames: int | None = None
if any(
(not handler.lazy_initialize_session) and handler.needs_chunk_media for handler in self._handlers.values()
):
media = pipeline.peek_chunk_media(state)
num_media_frames = media.num_media_frames
fps = media.fps
num_latent_frames = media.num_latent_frames
boundary_at = synchronized_monotonic_time()
chunk_index = state.chunk_index
metas: list[InteractionChunkMetadata] = []
for handler in self._handlers_in_apply_order():
if handler.lazy_initialize_session:
continue
meta = handler.apply_at_chunk_boundary(
state,
boundary_at=boundary_at,
chunk_index=chunk_index,
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
)
if meta is not None:
metas.append(meta)
return merge_interaction_metadata(metas)
def apply_at_chunk_boundary(
self,
state: StepRequestState,
*,
boundary_at: float,
chunk_index: int | None = None,
num_frames: int | None = None,
num_media_frames: int | None = None,
fps: float | None = None,
num_latent_frames: int | None = None,
) -> InteractionChunkMetadata:
"""Fan out chunk-boundary apply to handlers in stable order (prompt first)."""
if chunk_index is None:
chunk_index = state.chunk_index
metas: list[InteractionChunkMetadata] = []
for handler in self._handlers_in_apply_order():
if handler.modality not in state.interaction_sessions:
# This function is always called at chunk boundary without knowing if an interaction is really enqueued.
# `state.interaction_sessions[this modality]` is lazy-populated after the first interaction is enqueued.
# So, check here and skip a modality handler if this modality's session is not yet created.
if handler.lazy_initialize_session and handler.modality not in state.interaction_sessions:
# Lazy modalities skip apply until the first enqueue creates the session.
continue
meta = handler.apply_at_chunk_boundary(
state,
boundary_at=boundary_at,
chunk_index=chunk_index,
num_frames=num_frames,
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
)
if meta is not None:
metas.append(meta)
+8 -5
View File
@@ -4,9 +4,9 @@
from __future__ import annotations
import time
from typing import TYPE_CHECKING, cast
from vllm_omni.diffusion.interaction.types import synchronized_monotonic_time
from vllm_omni.diffusion.models.interface import SupportsInteractionApply
from vllm_omni.diffusion.worker.utils import StepRequestState
@@ -35,18 +35,21 @@ class InteractionMixin:
"onto the pipeline before chunked generation"
)
num_frames: int | None = None
num_media_frames: int | None = None
fps: float | None = None
num_latent_frames: int | None = None
if self._interaction_coordinator.needs_chunk_media:
media = cast(SupportsInteractionApply, self).peek_chunk_media(state)
num_frames = media.num_frames
num_media_frames = media.num_media_frames
fps = media.fps
num_latent_frames = media.num_latent_frames
merged = self._interaction_coordinator.apply_at_chunk_boundary(
state,
boundary_at=time.monotonic(),
num_frames=num_frames,
boundary_at=synchronized_monotonic_time(),
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
)
state.interaction_chunk_metadata = merged
@@ -5,7 +5,9 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import ClassVar, Self
from typing import ClassVar
from typing_extensions import Self
from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
@@ -23,9 +25,14 @@ class InteractionHandler(ABC):
"""
modality: ClassVar[str]
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (num_frames/fps)
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (frame count & fps)
# Those information are useful when interaction handler needs interpolation/integration on a frame-by-frame basis
needs_chunk_media: ClassVar[bool] = False
# When True, the coordinator skips ``apply_at_chunk_boundary`` until a session
# exists (normally created on the first enqueue). When False, every chunk
# boundary runs apply even with no prior enqueue so the handler can create
# its session and materialize default chunk data (e.g. identity camera hold).
lazy_initialize_session: ClassVar[bool] = True
@classmethod
def from_pipeline(cls, pipeline: object) -> Self:
@@ -37,6 +44,21 @@ class InteractionHandler(ABC):
del pipeline
return cls()
@abstractmethod
def validate_payload(
self,
state: StepRequestState,
*,
event_id: str,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Validate a modality payload without mutating request state.
Used by ``InteractionCoordinator.enqueue_parts`` so composite events
fail before any track is queued.
"""
@abstractmethod
def enqueue(
self,
@@ -56,7 +78,9 @@ class InteractionHandler(ABC):
*,
boundary_at: float,
chunk_index: int | None = None, # defaults to state.chunk_index when omitted
num_frames: int | None = None, # only set when at least one interaction handler needs it
fps: float | None = None, # only set when at least one interaction handler needs it
# Optional chunk media information. Only present when at least one initialized interaction handler needs it
num_media_frames: int | None = None,
fps: float | None = None,
num_latent_frames: int | None = None,
) -> InteractionChunkMetadata | None:
"""Advance request-local state and materialize this chunk's effects."""
@@ -0,0 +1,490 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Camera modality interaction handlers.
Payload is structural SE(3): ``translation`` (xyz) + unit
quaternion ``rotation`` (x, y, z, w). WASD-style key tokens belong in
clients, not in the engine.
"""
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import ClassVar, cast
import torch
from typing_extensions import Self, override
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
InteractionEvent,
InteractionMode,
InteractionPayload,
InteractionSession,
resolve_event_frame_offset,
)
from vllm_omni.diffusion.worker.utils import StepRequestState
Vec3 = tuple[float, float, float]
# Unit quaternion in ``(x, y, z, w)`` order. Identity is ``(0, 0, 0, 1)``.
Quat = tuple[float, float, float, float]
_IDENTITY_QUAT: Quat = (0.0, 0.0, 0.0, 1.0)
@dataclass
class CameraPose:
"""Canonical absolute camera pose used by the shared timeline.
Coordinates follow Unity semantics: ``+X`` right, ``+Y`` up,
``+Z`` forward. Session start is the identity transform;
``target`` poses are relative to that origin.
"""
translation: Vec3 = (0.0, 0.0, 0.0)
rotation: Quat = _IDENTITY_QUAT
@classmethod
def identity(cls) -> CameraPose:
return cls()
def clone(self) -> CameraPose:
return CameraPose(translation=self.translation, rotation=self.rotation)
def as_matrix(self) -> torch.Tensor:
"""Return a 4x4 rigid transform from translation + unit quaternion."""
mat = torch.eye(4, dtype=torch.float64)
mat[:3, :3] = _quat_to_rotmat(self.rotation)
mat[0, 3] = float(self.translation[0])
mat[1, 3] = float(self.translation[1])
mat[2, 3] = float(self.translation[2])
return mat
@classmethod
def from_matrix(cls, matrix: torch.Tensor) -> CameraPose:
if matrix.shape != (4, 4):
raise ValueError("camera pose matrix must be 4x4")
rotation = _rotmat_to_quat(matrix[:3, :3].detach().cpu().double())
translation = (
float(matrix[0, 3]),
float(matrix[1, 3]),
float(matrix[2, 3]),
)
return cls(translation=translation, rotation=rotation)
@dataclass(kw_only=True)
class QueuedCameraEvent(InteractionEvent):
"""Timestamped camera command waiting for the next chunk boundary."""
# Absolute pose for ``target``; per-frame SE3 delta for ``velocity``.
pose: CameraPose = field(default_factory=CameraPose.identity)
# Integer frame progress for ``target`` lerps (avoids float reciprocal drift).
elapsed_transition_frames: int = 0
@dataclass
class CameraSession(InteractionSession):
"""Request-local camera timeline under ``state.interaction_sessions['camera']``."""
current_pose: CameraPose = field(default_factory=CameraPose.identity)
pending_events: list[QueuedCameraEvent] = field(default_factory=list)
# In-flight command; ``elapsed_transition_chunks`` lives on the event.
active_event: QueuedCameraEvent | None = None
target_source: CameraPose | None = None
# Absolute C2W poses ``[num_latent_frames, 4, 4]`` for the most recent chunk.
last_absolute_poses: torch.Tensor | None = None
# Since a camera session is not lazily initialized, use this field to mark that a client enqueue is ever received.
has_received_input: bool = False
class SE3DeltaCameraHandler(InteractionHandler):
"""Generic camera timeline that samples absolute poses per frame.
* ``mode=target``: lerp toward an absolute pose (relative to session start).
* ``mode=velocity``: hold a structural SE3 delta until replaced.
``received_at`` is resolved on the media grid (``num_media_frames``/``fps``), then mapped onto latent steps.
Later integration and ``last_absolute_poses`` aligns with ``num_latent_frames``.
"""
modality: ClassVar[str] = "camera"
needs_chunk_media: ClassVar[bool] = True
# Camera conditioning must exist every chunk; apply creates the session and
# materializes identity/hold poses before any client enqueue.
lazy_initialize_session: ClassVar[bool] = False
default_transition_chunks: ClassVar[int] = 1
@classmethod
@override
def from_pipeline(cls, pipeline: object) -> Self:
"""It doesn't need anything from the pipeline (e.g., `encode_prompt` method)"""
del pipeline
return cls()
@override
def validate_payload(
self,
state: StepRequestState,
*,
event_id: str,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
if not event_id:
raise ValueError("event_id must be non-empty")
mode = payload.get("mode", "target")
if mode not in ("target", "velocity"):
raise ValueError("camera mode must be 'target' or 'velocity'")
data = payload.get("data")
if not isinstance(data, Mapping):
raise ValueError("camera data must be an object")
if "actions" in data:
raise ValueError(
"camera data.actions (WASD keys) is not accepted by the engine; "
"send structural translation/rotation instead"
)
_parse_pose(data)
if mode == "target":
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
if duration < 0:
raise ValueError("transition_chunks must be >= 0")
del state
@override
def enqueue(
self,
state: StepRequestState,
*,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
self.validate_payload(
state,
event_id=event_id,
payload=payload,
transition_chunks=transition_chunks,
)
mode = payload.get("mode", "target")
camera_mode = cast(InteractionMode, mode)
data = cast(Mapping[str, object], payload.get("data"))
pose = _parse_pose(data)
if camera_mode == "target":
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
else:
duration = 0
session = state.interaction_sessions.setdefault("camera", CameraSession())
assert isinstance(session, CameraSession)
with session.lock:
session.has_received_input = True
session.pending_events.append(
QueuedCameraEvent(
event_id=event_id,
received_at=received_at,
mode=camera_mode,
transition_chunks=duration,
pose=pose,
)
)
@override
def apply_at_chunk_boundary(
self,
state: StepRequestState,
*,
boundary_at: float,
chunk_index: int | None = None,
num_media_frames: int | None = None,
fps: float | None = None,
num_latent_frames: int | None = None,
) -> InteractionChunkMetadata | None:
"""Advance camera state for one chunk and store latent-aligned poses.
.. note::
Hardcoded the following policy of per-frame alignment:
- Resolve ``received_at`` on the media grid, then map each media index onto a latent step.
- Integrate one velocity step per **latent frame**, so holding speed ``0.05`` for a chunk with
``num_latent_frames=3`` yields total travel ``0.15``.
- ``last_absolute_poses.shape[0] == num_latent_frames``: interpolation is latent-aligned.
If future models need different scale/alignment policies, can refactor interaction registry
to be able to mark a policy.
"""
del chunk_index
if num_media_frames is None or fps is None:
raise ValueError("SE3DeltaCameraHandler requires chunk num_media_frames and fps")
if num_latent_frames is None:
num_latent_frames = num_media_frames
if int(num_latent_frames) <= 0:
raise ValueError(f"num_latent_frames must be > 0, got {num_latent_frames}")
session = state.interaction_sessions.setdefault("camera", CameraSession())
assert isinstance(session, CameraSession)
with session.lock:
poses, started, active, completed = self._step_one_chunk(
session,
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
boundary_at=boundary_at,
)
if poses:
session.last_absolute_poses = torch.stack([p.as_matrix() for p in poses], dim=0)
else:
session.last_absolute_poses = torch.zeros((0, 4, 4), dtype=torch.float64)
return InteractionChunkMetadata(
started_event_ids=started,
active_event_ids=active,
completed_event_ids=completed,
)
def _step_one_chunk(
self,
session: CameraSession,
*,
num_media_frames: int,
fps: float,
num_latent_frames: int,
boundary_at: float,
) -> tuple[list[CameraPose], list[str], list[str], list[str]]:
"""Sample absolute poses for this chunk under target/velocity semantics."""
num_media_frames = max(int(num_media_frames), 1)
num_latent_frames = max(int(num_latent_frames), 1)
pending = list(session.pending_events)
session.pending_events.clear()
by_latent: dict[int, list[QueuedCameraEvent]] = {}
for event in pending:
media_frame = resolve_event_frame_offset(
received_at=event.received_at,
previous_boundary_at=session.last_boundary_at,
num_frames=num_media_frames,
fps=fps,
)
latent_idx = _media_frame_to_latent(
media_frame,
num_media_frames=num_media_frames,
num_latent_frames=num_latent_frames,
)
by_latent.setdefault(latent_idx, []).append(event)
for events in by_latent.values():
events.sort(key=lambda item: item.received_at)
started: list[str] = []
completed: list[str] = []
seen_completed: set[str] = set()
poses: list[CameraPose] = []
def _mark_completed(event_id: str) -> None:
if event_id not in seen_completed:
seen_completed.add(event_id)
completed.append(event_id)
for latent_idx in range(num_latent_frames):
for event in by_latent.get(latent_idx, []):
started.append(event.event_id)
cancelled_id = self._activate_event(session, event)
if cancelled_id is not None:
_mark_completed(cancelled_id)
just_completed = self._step_one_frame(session, num_latent_frames)
poses.append(session.current_pose.clone())
if just_completed is not None:
_mark_completed(just_completed)
active: list[str] = []
if session.active_event is not None:
active.append(session.active_event.event_id)
session.last_boundary_at = boundary_at
return poses, started, active, completed
def _activate_event(self, session: CameraSession, event: QueuedCameraEvent) -> str | None:
"""Activate ``event``, returning the event_id cancelled by this replacement if any."""
cancelled_id: str | None = None
if session.active_event is not None and session.active_event.event_id != event.event_id:
cancelled_id = session.active_event.event_id
event.elapsed_transition_chunks = 0.0
event.elapsed_transition_frames = 0
session.active_event = event
if event.mode == "target":
session.target_source = session.current_pose.clone()
else:
session.target_source = None
return cancelled_id
def _clear_active_target(self, session: CameraSession) -> str:
"""Snap to the target pose, clear the active target track, and return its event_id."""
assert session.active_event is not None and session.active_event.mode == "target"
event = session.active_event
event_id = event.event_id
session.current_pose = event.pose.clone()
session.active_event = None
session.target_source = None
return event_id
def _step_one_frame(self, session: CameraSession, total_num_frames_this_chunk: int) -> str | None:
"""Advance the active camera command by one frame."""
event = session.active_event
if event is None:
return None
if event.mode == "target":
duration = event.transition_chunks
source = session.target_source or session.current_pose
target = event.pose
if duration <= 0:
return self._clear_active_target(session)
total_frames = max(1, int(round(duration * float(total_num_frames_this_chunk))))
event.elapsed_transition_frames += 1
event.elapsed_transition_chunks = event.elapsed_transition_frames / float(total_num_frames_this_chunk)
alpha = min(1.0, event.elapsed_transition_frames / float(total_frames))
session.current_pose = _lerp_pose(source, target, alpha)
if event.elapsed_transition_frames >= total_frames:
return self._clear_active_target(session)
return None
# Velocity: apply the held structural SE3 delta once per frame.
session.current_pose = _compose_pose(session.current_pose, event.pose)
return None
def _media_frame_to_latent(
media_frame: int,
*,
num_media_frames: int,
num_latent_frames: int,
) -> int:
"""Map a media-timeline frame index onto the latent control grid."""
if num_latent_frames <= 1 or num_media_frames <= 1:
return 0
return min(int(media_frame) * int(num_latent_frames) // int(num_media_frames), num_latent_frames - 1)
def _as_xyz(value: object, *, name: str) -> Vec3:
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 3:
raise ValueError(f"camera {name} must be a length-3 list/tuple")
if any(math.isnan(v) or math.isinf(v) for v in value):
raise ValueError(f"camera {name} must be non-nan and non-inf")
return (float(value[0]), float(value[1]), float(value[2]))
def _as_quat(value: object, *, name: str) -> Quat:
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 4:
raise ValueError(f"camera {name} must be a length-4 list/tuple (x, y, z, w)")
if any(math.isnan(v) or math.isinf(v) for v in value):
raise ValueError(f"camera {name} must be non-nan and non-inf")
return _quat_normalize((float(value[0]), float(value[1]), float(value[2]), float(value[3])))
def _parse_pose(data: Mapping[str, object]) -> CameraPose:
translation = _as_xyz(data.get("translation", (0.0, 0.0, 0.0)), name="translation")
rotation = _as_quat(data.get("rotation", _IDENTITY_QUAT), name="rotation")
return CameraPose(translation=translation, rotation=rotation)
def _lerp(a: float, b: float, alpha: float) -> float:
return a + (b - a) * alpha
def _lerp_pose(source: CameraPose, target: CameraPose, alpha: float) -> CameraPose:
alpha = min(1.0, max(0.0, alpha))
return CameraPose(
translation=cast(
Vec3,
tuple(_lerp(s, t, alpha) for s, t in zip(source.translation, target.translation)),
),
rotation=_quat_nlerp(source.rotation, target.rotation, alpha),
)
def _compose_pose(base: CameraPose, delta: CameraPose) -> CameraPose:
"""Apply ``delta`` after ``base`` in the session frame (T_new = T_base @ T_delta)."""
return CameraPose.from_matrix(base.as_matrix() @ delta.as_matrix())
def _quat_normalize(q: Quat) -> Quat:
x, y, z, w = q
norm = math.sqrt(x * x + y * y + z * z + w * w)
if norm <= 0.0:
raise ValueError("camera rotation quaternion must be non-zero")
return (x / norm, y / norm, z / norm, w / norm)
def _quat_nlerp(a: Quat, b: Quat, alpha: float) -> Quat:
"""Normalized linear interpolation; flips ``b`` when needed for shortest path."""
dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]
if dot < 0.0:
b = (-b[0], -b[1], -b[2], -b[3])
return _quat_normalize(
(
_lerp(a[0], b[0], alpha),
_lerp(a[1], b[1], alpha),
_lerp(a[2], b[2], alpha),
_lerp(a[3], b[3], alpha),
)
)
def _quat_to_rotmat(q: Quat) -> torch.Tensor:
x, y, z, w = q
xx, yy, zz = x * x, y * y, z * z
xy, xz, yz = x * y, x * z, y * z
wx, wy, wz = w * x, w * y, w * z
return torch.tensor(
[
[1.0 - 2.0 * (yy + zz), 2.0 * (xy - wz), 2.0 * (xz + wy)],
[2.0 * (xy + wz), 1.0 - 2.0 * (xx + zz), 2.0 * (yz - wx)],
[2.0 * (xz - wy), 2.0 * (yz + wx), 1.0 - 2.0 * (xx + yy)],
],
dtype=torch.float64,
)
def _rotmat_to_quat(rotation: torch.Tensor) -> Quat:
"""Convert a 3x3 rotation matrix to a unit quaternion ``(x, y, z, w)``."""
m = rotation
trace = float(m[0, 0] + m[1, 1] + m[2, 2])
if trace > 0.0:
s = math.sqrt(trace + 1.0) * 2.0
w = 0.25 * s
x = (float(m[2, 1]) - float(m[1, 2])) / s
y = (float(m[0, 2]) - float(m[2, 0])) / s
z = (float(m[1, 0]) - float(m[0, 1])) / s
elif float(m[0, 0]) > float(m[1, 1]) and float(m[0, 0]) > float(m[2, 2]):
s = math.sqrt(1.0 + float(m[0, 0]) - float(m[1, 1]) - float(m[2, 2])) * 2.0
w = (float(m[2, 1]) - float(m[1, 2])) / s
x = 0.25 * s
y = (float(m[0, 1]) + float(m[1, 0])) / s
z = (float(m[0, 2]) + float(m[2, 0])) / s
elif float(m[1, 1]) > float(m[2, 2]):
s = math.sqrt(1.0 + float(m[1, 1]) - float(m[0, 0]) - float(m[2, 2])) * 2.0
w = (float(m[0, 2]) - float(m[2, 0])) / s
x = (float(m[0, 1]) + float(m[1, 0])) / s
y = 0.25 * s
z = (float(m[1, 2]) + float(m[2, 1])) / s
else:
s = math.sqrt(1.0 + float(m[2, 2]) - float(m[0, 0]) - float(m[1, 1])) * 2.0
w = (float(m[1, 0]) - float(m[0, 1])) / s
x = (float(m[0, 2]) + float(m[2, 0])) / s
y = (float(m[1, 2]) + float(m[2, 1])) / s
z = 0.25 * s
return _quat_normalize((x, y, z, w))
__all__ = [
"CameraPose",
"CameraSession",
"QueuedCameraEvent",
"SE3DeltaCameraHandler",
]
@@ -90,6 +90,7 @@ class PromptInteractionHandler(InteractionHandler):
"""
modality: ClassVar[str] = "prompt"
lazy_initialize_session: ClassVar[bool] = True
def __init__(
self,
@@ -113,16 +114,14 @@ class PromptInteractionHandler(InteractionHandler):
)
@override
def enqueue(
def validate_payload(
self,
state: StepRequestState,
*,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
prompt = payload.get("prompt")
if not isinstance(prompt, str) or not prompt:
raise ValueError("prompt must be non-empty")
@@ -136,6 +135,27 @@ class PromptInteractionHandler(InteractionHandler):
if duration < 0:
raise ValueError("transition_chunks must be >= 0")
@override
def enqueue(
self,
state: StepRequestState,
*,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
self.validate_payload(
state,
event_id=event_id,
payload=payload,
transition_chunks=transition_chunks,
)
prompt = payload.get("prompt")
assert isinstance(prompt, str)
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
target_prompt_embeds, _ = self._encode_prompt(
prompt=prompt,
negative_prompt=None,
@@ -164,12 +184,13 @@ class PromptInteractionHandler(InteractionHandler):
*,
boundary_at: float,
chunk_index: int | None = None,
num_frames: int | None = None,
num_media_frames: int | None = None,
fps: float | None = None,
num_latent_frames: int | None = None,
) -> InteractionChunkMetadata | None:
"""Advance or start prompt interpolation before the next chunk."""
# Prompt lerp is chunk-LWW; media timeline and caller chunk_index unused.
del chunk_index, num_frames, fps
del chunk_index, num_media_frames, fps, num_latent_frames
session = state.interaction_sessions.get("prompt")
assert isinstance(session, PromptSession)
+5 -1
View File
@@ -5,12 +5,13 @@ A pipeline only declares the modalities that it supports.
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
Outer key: pipeline architecture name (``od_config.model_class_name``).
Inner key: modality name (e.g. ``\"prompt\"``).
Inner key: modality name (e.g. ``\"camera\"``).
"""
from __future__ import annotations
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
from vllm_omni.diffusion.interaction.modality_handlers.camera import SE3DeltaCameraHandler
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
@@ -21,4 +22,7 @@ STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
"HeliosPyramidPipeline": {
"prompt": PromptInteractionHandler,
},
"LingBotWorldCausalDMDPipeline": {
"camera": SE3DeltaCameraHandler,
},
}
+71 -8
View File
@@ -1,38 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared types for midway interaction handlers."""
from __future__ import annotations
import math
import time
from collections.abc import Mapping
from dataclasses import dataclass, field
from threading import Lock
from typing import Literal
import torch
# Wire payload for one modality track. Handlers validate keys they need.
InteractionPayload = Mapping[str, object]
# Shared command semantics across modalities that support them.
# Prompt specialization uses ``"target"`` only.
InteractionMode = Literal["target"]
# Prompt specialization narrows this to ``"target"`` only.
InteractionMode = Literal["target", "velocity"]
def synchronized_monotonic_time(stamp: float | None = None) -> float:
"""Return a monotonic timestamp shared across DiT ranks.
``submit_interaction`` and chunk-boundary apply run as collectives, but
``time.monotonic()`` is rank-local. Under USP/SP, inter-rank skew that
straddles a ``1/fps`` bucket edge would schedule the same event onto
different frames per shard. Rank 0's stamp is broadcast (callers may pass
a local sample; non-source ranks' values are ignored).
"""
if stamp is None:
stamp = time.monotonic()
if not torch.distributed.is_initialized():
return stamp
try:
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
group = get_world_group()
except Exception:
return stamp
if group.world_size <= 1:
return stamp
return group.broadcast_object(stamp, src=0)
@dataclass(frozen=True)
class ChunkMediaSpec:
"""Media extent of one generation chunk, for interaction timeline mapping."""
"""Decoded media extent and latent step count for one generation chunk.
num_frames: int
Attributes:
num_media_frames: Number of frames in this chunk when decoded into playable media.
fps: Frames per second of the media.
num_latent_frames: Number of latent frames in this chunk (internal DiT units).
"""
num_media_frames: int
fps: float
num_latent_frames: int
def __post_init__(self) -> None:
if self.num_frames <= 0:
raise ValueError(f"ChunkMediaSpec.num_frames must be > 0, got {self.num_frames}")
if self.num_media_frames <= 0:
raise ValueError(f"ChunkMediaSpec.num_media_frames must be > 0, got {self.num_media_frames}")
if self.num_latent_frames <= 0:
raise ValueError(f"ChunkMediaSpec.num_latent_frames must be > 0, got {self.num_latent_frames}")
if self.fps <= 0:
raise ValueError(f"ChunkMediaSpec.fps must be > 0, got {self.fps}")
@property
def duration_s(self) -> float:
return float(self.num_frames) / float(self.fps)
return float(self.num_media_frames) / float(self.fps)
@dataclass(kw_only=True)
@@ -41,6 +78,7 @@ class InteractionEvent:
Example:
- ``QueuedPromptEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
- ``QueuedCameraEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
"""
event_id: str
@@ -60,12 +98,37 @@ class InteractionSession:
Example:
- ``PromptSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
- ``CameraSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
"""
lock: Lock = field(default_factory=Lock, repr=False)
last_boundary_at: float | None = None
def resolve_event_frame_offset(
*,
received_at: float,
previous_boundary_at: float | None,
num_frames: int,
fps: float,
) -> int:
"""Map an event arrival time onto ``[0, num_frames - 1]`` for this chunk.
* Uses ``floor((received_at - previous_boundary_at) * fps)``.
* Events without a prior boundary (or with non-positive fps) map to frame 0.
* Offsets past the represented media window are clamped to the final frame.
"""
num_frames = max(int(num_frames), 1)
if previous_boundary_at is None or fps <= 0:
return 0
offset = math.floor((received_at - previous_boundary_at) * float(fps))
if offset < 0:
return 0
if offset >= num_frames:
return num_frames - 1
return int(offset)
@dataclass
class InteractionChunkMetadata:
"""Per-modality (or merged) event-id bookkeeping for one chunk boundary."""
@@ -535,19 +535,23 @@ class HeliosPipeline(
"zero_steps": int(extra.get("zero_steps", 1)),
}
)
self.prepare_next_chunk(state)
return state
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Expose this chunk's decoded media extent for interaction timelines."""
num_frames = int(state.extra["window_num_frames"])
num_media_frames = int(state.extra["window_num_frames"])
num_latent_frames = int(state.extra["num_latent_frames_per_chunk"])
fps = state.sampling.fps
if fps is None or float(fps) <= 0:
raise ValueError(
"sampling.fps is required and must be > 0 for interaction modalities that use the "
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
)
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
return ChunkMediaSpec(
num_media_frames=num_media_frames,
fps=float(fps),
num_latent_frames=num_latent_frames,
)
@override
def prepare_next_chunk(self, state: StepRequestState) -> None:
+2 -2
View File
@@ -151,10 +151,10 @@ class SupportsInteractionApply(Protocol):
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Return the media timeline represented by the upcoming/current chunk.
"""Return the media timeline and latent frame count for the upcoming/current chunk.
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
or for backpressure/pacing.
and/or for backpressure/pacing.
"""
...
@@ -25,6 +25,7 @@ from vllm_omni.experimental.ar_diffusion.tick_protocol import (
LINGBOT_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1"
LINGBOT_CAMERA_TRAJECTORY_SCHEMA = "lingbot.camera_trajectory.v1"
LINGBOT_CONTROLLER_TRANSLATION_UNIT = 0.05
_ACTION_ORDER = ("w", "a", "s", "d", "i", "j", "k", "l")
_VALID_ACTIONS = frozenset(_ACTION_ORDER)
_REFERENCE_HEIGHT = 480
@@ -282,7 +283,7 @@ def integrate_lingbot_camera_actions(
"""Convert latent-frame WASD/IJKL controls to cumulative C2W poses.
Calibration and motion constants intentionally match SGLang's LingBot
adapter: movement 0.05, pitch 4 degrees, yaw 6 degrees, pitch clamp 85
adapter: movement 0.05 (LINGBOT_CONTROLLER_TRANSLATION_UNIT), pitch 4 degrees, yaw 6 degrees, pitch clamp 85
degrees, and target-resolution focal lengths of 500 pixels.
"""
@@ -325,13 +326,31 @@ def integrate_lingbot_camera_actions(
right /= right_norm + 1e-6
movement = np.zeros(3, dtype=np.float64)
movement += forward * 0.05 * (("w" in actions) - ("s" in actions))
movement += right * 0.05 * (("d" in actions) - ("a" in actions))
movement += forward * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("w" in actions) - ("s" in actions))
movement += right * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("d" in actions) - ("a" in actions))
# Keep the cumulative controller pose in FP64 between chunks, matching
# SGLang's NumPy integration and avoiding long-session drift.
current_pose = np.eye(4, dtype=np.float64)
current_pose[:3, :3] = new_rotation
current_pose[:3, 3] = translation + movement
poses.append(current_pose)
trajectory = camera_trajectory_from_absolute_pose(torch.from_numpy(np.stack(poses)), width=width, height=height)
return trajectory, current_pitch
def camera_trajectory_from_absolute_pose(
poses: torch.Tensor,
*,
width: int,
height: int,
) -> CameraTrajectory:
if width <= 0 or height <= 0:
raise ValueError("camera action resolution must be positive.")
if poses.ndim != 3 or poses.shape[-2:] != (4, 4):
raise ValueError(f"absolute camera poses must have shape [frames, 4, 4], got {tuple(poses.shape)}.")
if poses.shape[0] == 0:
raise ValueError("absolute camera poses must not be empty.")
# build_plucker_embedding expects intrinsics in the 832x480 reference
# coordinate system. These values become SGLang's [500, 500, W/2, H/2]
# after that function scales them to the requested resolution.
@@ -342,9 +361,7 @@ def integrate_lingbot_camera_actions(
_REFERENCE_HEIGHT / 2,
)
trajectory = CameraTrajectory(
# Keep the cumulative controller pose in FP64 between chunks, matching
# SGLang's NumPy integration and avoiding long-session drift.
poses=torch.from_numpy(np.stack(poses)),
poses=poses,
intrinsics=torch.tensor(reference_intrinsics, dtype=torch.float32).repeat(len(poses), 1),
)
return trajectory, current_pitch
return trajectory
@@ -389,7 +389,11 @@ def _invert_camera_poses(poses: torch.Tensor) -> torch.Tensor:
return inverse
def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
def _prepare_framewise_poses(
raw_poses: torch.Tensor,
*,
translation_scale: float | None = None,
) -> torch.Tensor:
"""Convert raw C2W poses to normalized first-relative framewise deltas."""
# Remove global placement, then convert the sequence to framewise deltas.
@@ -402,12 +406,19 @@ def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
relative_poses[1:],
)
# Only relative direction/magnitude matters to the checkpoint; normalizing
# by the largest step removes the trajectory's arbitrary translation scale.
# Normalizing the control's scale. Different between scripted and realtime input.
translations = relative_poses[:, :3, 3]
max_translation_norm = torch.linalg.vector_norm(translations, dim=-1).max()
if max_translation_norm > 0:
relative_poses[:, :3, 3] = translations / max_translation_norm
if translation_scale is None:
# Scripted action_path input: divide by the largest step so an arbitrary recording scale becomes 1.
scale = torch.linalg.vector_norm(translations, dim=-1).max()
else:
# Realtime interaction input: use the controller translation unit (one WASD step = full speed).
# It cannot see future motion or normalize globally.
if translation_scale <= 0:
raise ValueError(f"translation_scale must be > 0, got {translation_scale}")
scale = translation_scale
if float(scale) > 0:
relative_poses[:, :3, 3] = translations / scale
return relative_poses
@@ -420,6 +431,7 @@ def build_plucker_embedding(
target_width: int,
device: torch.device,
dtype: torch.dtype,
translation_scale: float | None = None,
) -> torch.Tensor:
"""Build checkpoint-ordered ``(ray origin, ray direction)`` channels."""
@@ -438,7 +450,10 @@ def build_plucker_embedding(
# Compute geometry in at least FP32, then cast at the model boundary.
compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
poses = _prepare_framewise_poses(trajectory.poses.to(device=device, dtype=compute_dtype))
poses = _prepare_framewise_poses(
trajectory.poses.to(device=device, dtype=compute_dtype),
translation_scale=translation_scale,
)
intrinsics = trajectory.intrinsics.to(device=device, dtype=compute_dtype).clone()
intrinsics[:, (0, 2)] *= width / _REFERENCE_WIDTH
intrinsics[:, (1, 3)] *= height / _REFERENCE_HEIGHT
@@ -26,16 +26,21 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery, SupportsStepExecution
from vllm_omni.diffusion.models.lingbot_world.actions import (
LINGBOT_CAMERA_ACTION_SCHEMA,
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
LINGBOT_CONTROLLER_TRANSLATION_UNIT,
LingBotCameraActionFrames,
LingBotCameraActionScript,
as_camera_action_frames,
as_camera_action_script,
camera_trajectory_from_absolute_pose,
integrate_lingbot_camera_actions,
parse_lingbot_camera_action_frames,
parse_lingbot_camera_action_script,
@@ -385,25 +390,36 @@ def get_lingbot_world_pre_process_func(
camera_actions = None
else:
action_path = extra_args.get("action_path")
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
if not configured_action_root:
raise ValueError(
"sampling_params.extra_args.action_path requires a trusted action root configured by "
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
if action_path is None or action_path == "":
# Stepwise / streaming sessions may omit a request-scoped camera
# script and instead drive motion via mid-generation camera
# interaction (idle hold until the first camera event).
if getattr(od_config, "step_execution", False) or getattr(od_config, "streaming_output", False):
trajectory = None
camera_actions = None
camera_action_script = None
else:
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
else:
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
if not configured_action_root:
raise ValueError(
"sampling_params.extra_args.action_path requires a trusted action root configured by "
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
)
action_directory = resolve_trusted_action_directory(
action_path,
configured_action_root,
)
action_directory = resolve_trusted_action_directory(
action_path,
configured_action_root,
)
try:
trajectory = load_camera_trajectory(action_directory)
except OSError:
raise ValueError(
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
) from None
camera_actions = None
camera_action_script = None
try:
trajectory = load_camera_trajectory(action_directory)
except OSError:
raise ValueError(
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
) from None
camera_actions = None
camera_action_script = None
updated_prompt = dict(prompt)
updated_multi_modal_data = dict(multi_modal_data)
@@ -460,6 +476,7 @@ class LingBotWorldCausalDMDPipeline(
SupportImageInput,
SupportsComponentDiscovery,
SupportsStepExecution,
InteractionMixin,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
):
@@ -779,8 +796,10 @@ class LingBotWorldCausalDMDPipeline(
camera_sources = (
int(camera_trajectory is not None) + int(camera_actions is not None) + int(camera_action_script is not None)
)
if camera_sources != 1:
raise ValueError("LingBot pre-processing must materialize exactly one camera input.")
if camera_sources > 1:
raise ValueError("LingBot pre-processing must materialize at most one camera input.")
# Zero camera sources are allowed for stepwise camera-interaction sessions;
# request-mode ``forward`` still requires an explicit trajectory or actions.
request_flow_shift = (
extra_args["flow_shift"] if "flow_shift" in extra_args else getattr(self.scheduler.config, "shift", 5.0)
@@ -1047,7 +1066,7 @@ class LingBotWorldCausalDMDPipeline(
*,
dtype: torch.dtype,
previous: CameraTrajectory | None = None,
latent_aligned: bool = False,
latent_aligned: bool = False, # iff using realtime interaction inputs
) -> tuple[torch.Tensor, CameraTrajectory]:
"""Convert raw camera frames to a latent-aligned ray tensor."""
@@ -1083,10 +1102,10 @@ class LingBotWorldCausalDMDPipeline(
),
)
drop_anchor = True
elif inputs.camera_actions is not None:
# The action integrator returns post-action poses. Keep the first
# action visible to framewise-delta conditioning by prepending its
# known pre-action state (identity for a new realtime session).
elif inputs.camera_actions is not None or latent_aligned:
# Action/interaction integrators return post-action poses. Prepend the
# known pre-action state (identity for a new session) so the first
# pose still contributes a framewise delta.
identity = torch.eye(
4,
device=trajectory.poses.device,
@@ -1100,6 +1119,8 @@ class LingBotWorldCausalDMDPipeline(
),
)
drop_anchor = True
# Realtime interaction cannot see future steps; use the controller translation unit to normalize speed.
# Full action_path keeps max-norm.
camera_embedding = build_plucker_embedding(
embedding_trajectory,
height=inputs.height,
@@ -1108,6 +1129,7 @@ class LingBotWorldCausalDMDPipeline(
target_width=inputs.width,
device=self.device,
dtype=dtype,
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT if latent_aligned else None,
)
if drop_anchor:
camera_embedding = camera_embedding[1:]
@@ -1277,6 +1299,11 @@ class LingBotWorldCausalDMDPipeline(
"request mode takes one three-frame camera_actions control per "
"block, or a camera_trajectory for the request."
)
if inputs.camera_trajectory is None and inputs.camera_actions is None:
raise ValueError(
"LingBot request mode requires camera_trajectory or camera_actions; "
"omit them only for stepwise camera-interaction sessions."
)
tick = ARDiffusionTickRequest.from_extra_args(req.sampling_params.extra_args)
if tick is not None and self._ar_diffusion_kv_state is None:
raise RuntimeError("LingBot typed ticks require ARDiffusionEngine session binding.")
@@ -1645,11 +1672,6 @@ class LingBotWorldCausalDMDPipeline(
}
self._ar_text_caches(prompt_embeds, invalidate=False)
self._ar_sessions[state.request_id] = _LingBotARSessionState()
try:
self._prepare_next_chunk(state)
except Exception:
self._release_condition_encoder_state(state.request_id)
raise
return state
def _prepare_next_chunk(self, state: StepRequestState) -> None:
@@ -1668,7 +1690,14 @@ class LingBotWorldCausalDMDPipeline(
dtype=extra["dtype"],
)
previous = extra.get("camera_tail")
if extra.get("camera_action_script") is not None:
if (
(camera_session := state.interaction_sessions.get("camera"))
and isinstance(camera_session, CameraSession)
and camera_session.has_received_input
):
raise ValueError("Cannot use mid-generation camera interaction together with camera_action_script; ")
chunk_actions = extra["camera_action_script"][state.chunk_index]
action_trajectory, camera_pitch = integrate_lingbot_camera_actions(
chunk_actions,
@@ -1678,11 +1707,12 @@ class LingBotWorldCausalDMDPipeline(
initial_pitch=float(extra.get("camera_pitch", 0.0)),
)
extra["camera_pitch"] = camera_pitch
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
chunk_inputs = replace(
inputs,
camera_trajectory=action_trajectory,
camera_actions=chunk_actions,
num_frames=(block_frames - 1) * self.vae_scale_factor_temporal + 1,
num_frames=media_frames,
num_latent_frames=block_frames,
)
camera, camera_tail = self._prepare_camera(
@@ -1691,14 +1721,53 @@ class LingBotWorldCausalDMDPipeline(
previous=previous,
)
elif extra.get("camera_embedding_cache") is not None:
# Same slice request mode takes from its one full-trajectory
# embedding, so both paths condition a block identically.
if (
(camera_session := state.interaction_sessions.get("camera"))
and isinstance(camera_session, CameraSession)
and camera_session.has_received_input
):
raise ValueError("Cannot use mid-generation camera interaction together with camera_embedding_cache; ")
trajectory = extra["camera_trajectory_cache"]
camera = extra["camera_embedding_cache"][:, :, start_frame:stop_frame]
camera_tail = CameraTrajectory(
poses=trajectory.poses[stop_frame - 1 : stop_frame].clone(),
intrinsics=trajectory.intrinsics[stop_frame - 1 : stop_frame].clone(),
)
elif self._interaction_coordinator is not None and self._interaction_coordinator.has_modality("camera"):
# Poses come from the latest boundary apply: ``prepare_encode`` for
# chunk 0, the diffusion runner for later chunks. This method only
# digests them — it does not apply interactions itself.
camera_session = state.interaction_sessions.get("camera")
if not isinstance(camera_session, CameraSession) or camera_session.last_absolute_poses is None:
raise RuntimeError(
"LingBot camera interaction requires apply_interaction_at_chunk_boundary before prepare_next_chunk."
)
absolute_poses = camera_session.last_absolute_poses
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
num_latent_frames = block_frames
if int(absolute_poses.shape[0]) != num_latent_frames:
raise ValueError(
"camera interaction must produce exactly one pose per latent frame; "
f"got {int(absolute_poses.shape[0])}, expected {num_latent_frames}."
)
action_trajectory = camera_trajectory_from_absolute_pose(
absolute_poses,
width=inputs.width,
height=inputs.height,
)
chunk_inputs = replace(
inputs,
camera_trajectory=action_trajectory,
camera_actions=None,
num_frames=media_frames,
num_latent_frames=num_latent_frames,
)
camera, camera_tail = self._prepare_camera(
chunk_inputs,
dtype=extra["dtype"],
previous=previous,
latent_aligned=True,
)
else:
raise RuntimeError("LingBot step execution is missing a camera trajectory or action script.")
if camera.shape[2:] != condition.shape[2:]:
@@ -1842,19 +1911,48 @@ class LingBotWorldCausalDMDPipeline(
session_state.pending_encoder_cache = None
session_state.next_chunk_index += 1
state.chunk_index += 1
finished = state.request_denoise_completed
if not finished:
self._prepare_next_chunk(state)
else:
self._release_condition_encoder_state(state.request_id)
return DiffusionOutput(
output=output,
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None,
chunk_index=completed_chunk_index,
total_chunks=state.total_chunks,
finished=finished,
finished=state.request_denoise_completed,
)
def _chunk_media_frame_count(self, state: StepRequestState, *, block_frames: int) -> int:
"""Pixel frames this chunk will emit when decoded.
Independent (non-streaming) decode restarts causal expansion every block,
so each chunk is ``(block_frames - 1) * temporal + 1`` frames.
Streaming decode expands only the session's opening latent to one frame and every later latent to the full
temporal factor, so chunk 0 matches the independent count and later chunks are ``block_frames * temporal``.
"""
decode_state = self._streaming_decode_states.get(state.request_id)
if decode_state is not None and decode_state.started:
return block_frames * self.vae_scale_factor_temporal
return (block_frames - 1) * self.vae_scale_factor_temporal + 1
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Expose this chunk's decoded media extent and latent step count."""
num_latent_frames = int(state.extra.get("block_frames") or self.transformer.config.num_frames_per_block)
num_media_frames = self._chunk_media_frame_count(state, block_frames=num_latent_frames)
fps = state.sampling.fps
if fps is None or float(fps) <= 0:
# Media-frame camera controls are not wall-clock paced; a unit fps keeps
# resolve_event_frame_offset well-defined when the client omits sampling.fps.
fps = float(num_media_frames)
return ChunkMediaSpec(
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
)
def prepare_next_chunk(self, state: StepRequestState) -> None:
"""Prepare the next AR block after chunk-boundary interaction apply."""
if state.request_denoise_completed:
return
self._prepare_next_chunk(state)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return cast(set[str], loader.load_weights(weights))
@@ -43,6 +43,7 @@ from vllm_omni.diffusion.diffusion_kv.paged_attention_adapter import (
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_rank
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
from vllm_omni.diffusion.interaction.types import InteractionPayload
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import (
SupportsInteractionApply,
@@ -1080,30 +1081,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
error_outputs: list[RunnerOutput] = []
for state in states:
if state.request_id in new_request_ids:
# Everything that runs before ``_dit_any_rank_failed`` must be
# inside the try: an exception in ``_initialize_generator`` or
# Everything that requires rank-synchronization must be called
# inside a try, record the exception and handle with `_dit_any_rank_failed`.
# Reason (example): An exception in ``_initialize_generator`` or
# ``clear_pipeline_stage_durations`` on one rank would skip the
# all-reduce here while every peer proceeds into it, and the
# peers then hang on the NCCL collective until timeout.
per_req_exc: BaseException | None = None
try:
self._initialize_generator(state.sampling)
clear_pipeline_stage_durations(pipeline)
# encode
pipeline.prepare_encode(state)
merge_stage_durations(
state,
consume_pipeline_stage_durations(pipeline),
)
except Exception as exc:
per_req_exc = exc
# Pipelines that do rank-0-only work (e.g. MiniMax H3
# reference-video prep) must broadcast per-request failures
# internally so downstream collectives stay in step; even so,
# cross-check that every DiT rank agrees so a rank-local error
# (or a future pipeline that omits the guard) does not leave
# the process group half-way through a new request.
if _dit_any_rank_failed(per_req_exc is not None):
def _abort_prep_failure(per_req_exc: BaseException | None) -> None:
self.state_cache.pop(state.request_id, None)
if per_req_exc is None:
per_req_exc = RuntimeError(
@@ -1123,6 +1107,41 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
result=DiffusionOutput.from_exception(per_req_exc),
)
)
per_req_exc: BaseException | None = None
try:
self._initialize_generator(state.sampling)
clear_pipeline_stage_durations(pipeline)
pipeline.prepare_encode(state)
except Exception as exc:
per_req_exc = exc
# Pipelines that do rank-0-only work (e.g. MiniMax H3
# reference-video prep) must broadcast per-request failures
# internally so downstream collectives stay in step; even so,
# cross-check that every DiT rank agrees so a rank-local error
# (or a future pipeline that omits the guard) does not leave
# the process group half-way through a new request.
if _dit_any_rank_failed(per_req_exc is not None):
_abort_prep_failure(per_req_exc)
continue
# If the pipeline supports interaction, the interaction session initialization also needs to call
# synchronized_monotonic_time(). Wrap in another try-block to not block on prepare_encode failures.
try:
if supports_interaction_apply(pipeline) and state.chunk_index == 0:
pipe = cast(SupportsInteractionApply, pipeline)
assert self._interaction_coordinator is not None, "Model not loaded. Call load_model() first."
state.interaction_chunk_metadata = self._interaction_coordinator.maybe_prepare_initial_session(
state, pipe
)
pipe.prepare_next_chunk(state)
merge_stage_durations(
state,
consume_pipeline_stage_durations(pipeline),
)
except Exception as exc:
per_req_exc = exc
if _dit_any_rank_failed(per_req_exc is not None):
_abort_prep_failure(per_req_exc)
continue
prepared_states.append(state)
@@ -1431,47 +1450,33 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
interaction: OmniInteractionPrompt,
) -> None:
"""Route a midway interaction through the pipeline interaction coordinator."""
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
assert self.pipeline is not None and self._interaction_coordinator is not None, (
"Model not loaded. Call load_model() first."
)
if not self.od_config.streaming_output:
raise ValueError("submit_interaction requires streaming_output=True")
if not self._supports_step_mode():
raise ValueError("submit_interaction requires step execution support")
coordinator = self._interaction_coordinator
if coordinator is None:
coordinator = InteractionCoordinator.build(self.pipeline, self.od_config)
self._interaction_coordinator = coordinator
if hasattr(self.pipeline, "_interaction_coordinator"):
self.pipeline._interaction_coordinator = coordinator
event = interaction.get("event")
has_mm = isinstance(event, dict) and "multi_modal_data" in event
has_prompt = isinstance(event, dict) and "prompt" in event and event.get("prompt") is not None
# Prompt-only interactions in this release; multi_modal_data lands with camera support.
if not isinstance(event, dict) or has_mm or not has_prompt:
raise NotImplementedError(
"Only text-only prompt update interactions with 'event.prompt' and optional "
"'transition_chunks' are supported in this release"
)
if not coordinator.has_modality("prompt"):
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
state = self.state_cache.get(request_id)
if state is None:
raise ValueError(f"No active request state for interaction: {request_id!r}")
event_id = interaction.get("event_id")
if not isinstance(event_id, str) or not event_id:
raise ValueError("event_id must be non-empty")
prompt = event["prompt"]
if not isinstance(prompt, str) or not prompt:
raise ValueError("prompt must be non-empty")
coordinator.enqueue(
event = interaction["event"]
parts: list[tuple[str, InteractionPayload]] = []
prompt = event.get("prompt")
if prompt is not None:
parts.append(("prompt", {"prompt": prompt}))
multi_modal_data = event.get("multi_modal_data")
if multi_modal_data:
parts.extend(
(str(modality), cast(InteractionPayload, payload)) for modality, payload in multi_modal_data.items()
)
self._interaction_coordinator.enqueue_parts(
state,
modality="prompt",
event_id=event_id,
parts=parts,
event_id=interaction["event_id"],
received_at=time.monotonic(),
payload={"prompt": prompt},
transition_chunks=interaction.get("transition_chunks"),
)