[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
139a47a578
commit
e36babd48f
26 files changed
+2071
-200
No files matched your search
@@ -11,8 +11,9 @@ from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionPayload,
|
||||
merge_interaction_metadata,
|
||||
synchronized_monotonic_time,
|
||||
)
|
||||
from vllm_omni.diffusion.models.interface import supports_interaction_apply
|
||||
from vllm_omni.diffusion.models.interface import SupportsInteractionApply, supports_interaction_apply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -52,7 +53,7 @@ class InteractionCoordinator:
|
||||
|
||||
@property
|
||||
def needs_chunk_media(self) -> bool:
|
||||
"""Whether any registered handler needs chunk ``num_frames``/``fps``."""
|
||||
"""Whether any registered handler needs chunk media/latent counts + fps."""
|
||||
return any(handler.needs_chunk_media for handler in self._handlers.values())
|
||||
|
||||
def get_handler(self, modality: str) -> InteractionHandler:
|
||||
@@ -83,31 +84,109 @@ class InteractionCoordinator:
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
|
||||
def enqueue_parts(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
parts: list[tuple[str, InteractionPayload]],
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Ensure every modality is supported and every payload validates, then enqueue.
|
||||
|
||||
Composite events must not partially mutate queues when a later track is
|
||||
unsupported or carries an invalid payload.
|
||||
"""
|
||||
if not parts:
|
||||
raise ValueError("interaction event requires prompt and/or multi_modal_data")
|
||||
|
||||
# Reconcile rank-local samples so USP/SP shards share one arrival time.
|
||||
received_at = synchronized_monotonic_time(received_at)
|
||||
|
||||
handlers = [(self.get_handler(modality), payload) for modality, payload in parts]
|
||||
for handler, payload in handlers:
|
||||
handler.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
for (modality, payload), (handler, _) in zip(parts, handlers, strict=True):
|
||||
del handler
|
||||
self.enqueue(
|
||||
state,
|
||||
modality=modality,
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
|
||||
def maybe_prepare_initial_session(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
pipeline: SupportsInteractionApply,
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Materialize non-lazy modality sessions before chunk 0 denoise.
|
||||
|
||||
Handlers with ``lazy_initialize_session=False`` (e.g. camera) must exist
|
||||
before the first chunk even with no client enqueue.
|
||||
"""
|
||||
num_media_frames: int | None = None
|
||||
fps: float | None = None
|
||||
num_latent_frames: int | None = None
|
||||
if any(
|
||||
(not handler.lazy_initialize_session) and handler.needs_chunk_media for handler in self._handlers.values()
|
||||
):
|
||||
media = pipeline.peek_chunk_media(state)
|
||||
num_media_frames = media.num_media_frames
|
||||
fps = media.fps
|
||||
num_latent_frames = media.num_latent_frames
|
||||
|
||||
boundary_at = synchronized_monotonic_time()
|
||||
chunk_index = state.chunk_index
|
||||
metas: list[InteractionChunkMetadata] = []
|
||||
for handler in self._handlers_in_apply_order():
|
||||
if handler.lazy_initialize_session:
|
||||
continue
|
||||
meta = handler.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=boundary_at,
|
||||
chunk_index=chunk_index,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
if meta is not None:
|
||||
metas.append(meta)
|
||||
return merge_interaction_metadata(metas)
|
||||
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Fan out chunk-boundary apply to handlers in stable order (prompt first)."""
|
||||
if chunk_index is None:
|
||||
chunk_index = state.chunk_index
|
||||
metas: list[InteractionChunkMetadata] = []
|
||||
for handler in self._handlers_in_apply_order():
|
||||
if handler.modality not in state.interaction_sessions:
|
||||
# This function is always called at chunk boundary without knowing if an interaction is really enqueued.
|
||||
# `state.interaction_sessions[this modality]` is lazy-populated after the first interaction is enqueued.
|
||||
# So, check here and skip a modality handler if this modality's session is not yet created.
|
||||
if handler.lazy_initialize_session and handler.modality not in state.interaction_sessions:
|
||||
# Lazy modalities skip apply until the first enqueue creates the session.
|
||||
continue
|
||||
meta = handler.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=boundary_at,
|
||||
chunk_index=chunk_index,
|
||||
num_frames=num_frames,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
if meta is not None:
|
||||
metas.append(meta)
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm_omni.diffusion.interaction.types import synchronized_monotonic_time
|
||||
from vllm_omni.diffusion.models.interface import SupportsInteractionApply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
@@ -35,18 +35,21 @@ class InteractionMixin:
|
||||
"onto the pipeline before chunked generation"
|
||||
)
|
||||
|
||||
num_frames: int | None = None
|
||||
num_media_frames: int | None = None
|
||||
fps: float | None = None
|
||||
num_latent_frames: int | None = None
|
||||
if self._interaction_coordinator.needs_chunk_media:
|
||||
media = cast(SupportsInteractionApply, self).peek_chunk_media(state)
|
||||
num_frames = media.num_frames
|
||||
num_media_frames = media.num_media_frames
|
||||
fps = media.fps
|
||||
num_latent_frames = media.num_latent_frames
|
||||
|
||||
merged = self._interaction_coordinator.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=time.monotonic(),
|
||||
num_frames=num_frames,
|
||||
boundary_at=synchronized_monotonic_time(),
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
state.interaction_chunk_metadata = merged
|
||||
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import ClassVar, Self
|
||||
from typing import ClassVar
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
@@ -23,9 +25,14 @@ class InteractionHandler(ABC):
|
||||
"""
|
||||
|
||||
modality: ClassVar[str]
|
||||
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (num_frames/fps)
|
||||
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (frame count & fps)
|
||||
# Those information are useful when interaction handler needs interpolation/integration on a frame-by-frame basis
|
||||
needs_chunk_media: ClassVar[bool] = False
|
||||
# When True, the coordinator skips ``apply_at_chunk_boundary`` until a session
|
||||
# exists (normally created on the first enqueue). When False, every chunk
|
||||
# boundary runs apply even with no prior enqueue so the handler can create
|
||||
# its session and materialize default chunk data (e.g. identity camera hold).
|
||||
lazy_initialize_session: ClassVar[bool] = True
|
||||
|
||||
@classmethod
|
||||
def from_pipeline(cls, pipeline: object) -> Self:
|
||||
@@ -37,6 +44,21 @@ class InteractionHandler(ABC):
|
||||
del pipeline
|
||||
return cls()
|
||||
|
||||
@abstractmethod
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Validate a modality payload without mutating request state.
|
||||
|
||||
Used by ``InteractionCoordinator.enqueue_parts`` so composite events
|
||||
fail before any track is queued.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def enqueue(
|
||||
self,
|
||||
@@ -56,7 +78,9 @@ class InteractionHandler(ABC):
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None, # defaults to state.chunk_index when omitted
|
||||
num_frames: int | None = None, # only set when at least one interaction handler needs it
|
||||
fps: float | None = None, # only set when at least one interaction handler needs it
|
||||
# Optional chunk media information. Only present when at least one initialized interaction handler needs it
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance request-local state and materialize this chunk's effects."""
|
||||
@@ -0,0 +1,490 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Camera modality interaction handlers.
|
||||
|
||||
Payload is structural SE(3): ``translation`` (xyz) + unit
|
||||
quaternion ``rotation`` (x, y, z, w). WASD-style key tokens belong in
|
||||
clients, not in the engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self, override
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionEvent,
|
||||
InteractionMode,
|
||||
InteractionPayload,
|
||||
InteractionSession,
|
||||
resolve_event_frame_offset,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
Vec3 = tuple[float, float, float]
|
||||
# Unit quaternion in ``(x, y, z, w)`` order. Identity is ``(0, 0, 0, 1)``.
|
||||
Quat = tuple[float, float, float, float]
|
||||
|
||||
_IDENTITY_QUAT: Quat = (0.0, 0.0, 0.0, 1.0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraPose:
|
||||
"""Canonical absolute camera pose used by the shared timeline.
|
||||
|
||||
Coordinates follow Unity semantics: ``+X`` right, ``+Y`` up,
|
||||
``+Z`` forward. Session start is the identity transform;
|
||||
``target`` poses are relative to that origin.
|
||||
"""
|
||||
|
||||
translation: Vec3 = (0.0, 0.0, 0.0)
|
||||
rotation: Quat = _IDENTITY_QUAT
|
||||
|
||||
@classmethod
|
||||
def identity(cls) -> CameraPose:
|
||||
return cls()
|
||||
|
||||
def clone(self) -> CameraPose:
|
||||
return CameraPose(translation=self.translation, rotation=self.rotation)
|
||||
|
||||
def as_matrix(self) -> torch.Tensor:
|
||||
"""Return a 4x4 rigid transform from translation + unit quaternion."""
|
||||
mat = torch.eye(4, dtype=torch.float64)
|
||||
mat[:3, :3] = _quat_to_rotmat(self.rotation)
|
||||
mat[0, 3] = float(self.translation[0])
|
||||
mat[1, 3] = float(self.translation[1])
|
||||
mat[2, 3] = float(self.translation[2])
|
||||
return mat
|
||||
|
||||
@classmethod
|
||||
def from_matrix(cls, matrix: torch.Tensor) -> CameraPose:
|
||||
if matrix.shape != (4, 4):
|
||||
raise ValueError("camera pose matrix must be 4x4")
|
||||
rotation = _rotmat_to_quat(matrix[:3, :3].detach().cpu().double())
|
||||
translation = (
|
||||
float(matrix[0, 3]),
|
||||
float(matrix[1, 3]),
|
||||
float(matrix[2, 3]),
|
||||
)
|
||||
return cls(translation=translation, rotation=rotation)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class QueuedCameraEvent(InteractionEvent):
|
||||
"""Timestamped camera command waiting for the next chunk boundary."""
|
||||
|
||||
# Absolute pose for ``target``; per-frame SE3 delta for ``velocity``.
|
||||
pose: CameraPose = field(default_factory=CameraPose.identity)
|
||||
# Integer frame progress for ``target`` lerps (avoids float reciprocal drift).
|
||||
elapsed_transition_frames: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraSession(InteractionSession):
|
||||
"""Request-local camera timeline under ``state.interaction_sessions['camera']``."""
|
||||
|
||||
current_pose: CameraPose = field(default_factory=CameraPose.identity)
|
||||
pending_events: list[QueuedCameraEvent] = field(default_factory=list)
|
||||
# In-flight command; ``elapsed_transition_chunks`` lives on the event.
|
||||
active_event: QueuedCameraEvent | None = None
|
||||
target_source: CameraPose | None = None
|
||||
# Absolute C2W poses ``[num_latent_frames, 4, 4]`` for the most recent chunk.
|
||||
last_absolute_poses: torch.Tensor | None = None
|
||||
# Since a camera session is not lazily initialized, use this field to mark that a client enqueue is ever received.
|
||||
has_received_input: bool = False
|
||||
|
||||
|
||||
class SE3DeltaCameraHandler(InteractionHandler):
|
||||
"""Generic camera timeline that samples absolute poses per frame.
|
||||
|
||||
* ``mode=target``: lerp toward an absolute pose (relative to session start).
|
||||
* ``mode=velocity``: hold a structural SE3 delta until replaced.
|
||||
|
||||
``received_at`` is resolved on the media grid (``num_media_frames``/``fps``), then mapped onto latent steps.
|
||||
Later integration and ``last_absolute_poses`` aligns with ``num_latent_frames``.
|
||||
"""
|
||||
|
||||
modality: ClassVar[str] = "camera"
|
||||
needs_chunk_media: ClassVar[bool] = True
|
||||
# Camera conditioning must exist every chunk; apply creates the session and
|
||||
# materializes identity/hold poses before any client enqueue.
|
||||
lazy_initialize_session: ClassVar[bool] = False
|
||||
default_transition_chunks: ClassVar[int] = 1
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def from_pipeline(cls, pipeline: object) -> Self:
|
||||
"""It doesn't need anything from the pipeline (e.g., `encode_prompt` method)"""
|
||||
del pipeline
|
||||
return cls()
|
||||
|
||||
@override
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
if not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
|
||||
mode = payload.get("mode", "target")
|
||||
if mode not in ("target", "velocity"):
|
||||
raise ValueError("camera mode must be 'target' or 'velocity'")
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, Mapping):
|
||||
raise ValueError("camera data must be an object")
|
||||
if "actions" in data:
|
||||
raise ValueError(
|
||||
"camera data.actions (WASD keys) is not accepted by the engine; "
|
||||
"send structural translation/rotation instead"
|
||||
)
|
||||
_parse_pose(data)
|
||||
if mode == "target":
|
||||
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
del state
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
self.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
mode = payload.get("mode", "target")
|
||||
camera_mode = cast(InteractionMode, mode)
|
||||
data = cast(Mapping[str, object], payload.get("data"))
|
||||
pose = _parse_pose(data)
|
||||
if camera_mode == "target":
|
||||
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
|
||||
else:
|
||||
duration = 0
|
||||
|
||||
session = state.interaction_sessions.setdefault("camera", CameraSession())
|
||||
assert isinstance(session, CameraSession)
|
||||
with session.lock:
|
||||
session.has_received_input = True
|
||||
session.pending_events.append(
|
||||
QueuedCameraEvent(
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
mode=camera_mode,
|
||||
transition_chunks=duration,
|
||||
pose=pose,
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance camera state for one chunk and store latent-aligned poses.
|
||||
|
||||
.. note::
|
||||
Hardcoded the following policy of per-frame alignment:
|
||||
|
||||
- Resolve ``received_at`` on the media grid, then map each media index onto a latent step.
|
||||
- Integrate one velocity step per **latent frame**, so holding speed ``0.05`` for a chunk with
|
||||
``num_latent_frames=3`` yields total travel ``0.15``.
|
||||
- ``last_absolute_poses.shape[0] == num_latent_frames``: interpolation is latent-aligned.
|
||||
|
||||
If future models need different scale/alignment policies, can refactor interaction registry
|
||||
to be able to mark a policy.
|
||||
"""
|
||||
del chunk_index
|
||||
if num_media_frames is None or fps is None:
|
||||
raise ValueError("SE3DeltaCameraHandler requires chunk num_media_frames and fps")
|
||||
if num_latent_frames is None:
|
||||
num_latent_frames = num_media_frames
|
||||
if int(num_latent_frames) <= 0:
|
||||
raise ValueError(f"num_latent_frames must be > 0, got {num_latent_frames}")
|
||||
|
||||
session = state.interaction_sessions.setdefault("camera", CameraSession())
|
||||
assert isinstance(session, CameraSession)
|
||||
with session.lock:
|
||||
poses, started, active, completed = self._step_one_chunk(
|
||||
session,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
boundary_at=boundary_at,
|
||||
)
|
||||
if poses:
|
||||
session.last_absolute_poses = torch.stack([p.as_matrix() for p in poses], dim=0)
|
||||
else:
|
||||
session.last_absolute_poses = torch.zeros((0, 4, 4), dtype=torch.float64)
|
||||
|
||||
return InteractionChunkMetadata(
|
||||
started_event_ids=started,
|
||||
active_event_ids=active,
|
||||
completed_event_ids=completed,
|
||||
)
|
||||
|
||||
def _step_one_chunk(
|
||||
self,
|
||||
session: CameraSession,
|
||||
*,
|
||||
num_media_frames: int,
|
||||
fps: float,
|
||||
num_latent_frames: int,
|
||||
boundary_at: float,
|
||||
) -> tuple[list[CameraPose], list[str], list[str], list[str]]:
|
||||
"""Sample absolute poses for this chunk under target/velocity semantics."""
|
||||
num_media_frames = max(int(num_media_frames), 1)
|
||||
num_latent_frames = max(int(num_latent_frames), 1)
|
||||
pending = list(session.pending_events)
|
||||
session.pending_events.clear()
|
||||
|
||||
by_latent: dict[int, list[QueuedCameraEvent]] = {}
|
||||
for event in pending:
|
||||
media_frame = resolve_event_frame_offset(
|
||||
received_at=event.received_at,
|
||||
previous_boundary_at=session.last_boundary_at,
|
||||
num_frames=num_media_frames,
|
||||
fps=fps,
|
||||
)
|
||||
latent_idx = _media_frame_to_latent(
|
||||
media_frame,
|
||||
num_media_frames=num_media_frames,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
by_latent.setdefault(latent_idx, []).append(event)
|
||||
for events in by_latent.values():
|
||||
events.sort(key=lambda item: item.received_at)
|
||||
|
||||
started: list[str] = []
|
||||
completed: list[str] = []
|
||||
seen_completed: set[str] = set()
|
||||
poses: list[CameraPose] = []
|
||||
|
||||
def _mark_completed(event_id: str) -> None:
|
||||
if event_id not in seen_completed:
|
||||
seen_completed.add(event_id)
|
||||
completed.append(event_id)
|
||||
|
||||
for latent_idx in range(num_latent_frames):
|
||||
for event in by_latent.get(latent_idx, []):
|
||||
started.append(event.event_id)
|
||||
cancelled_id = self._activate_event(session, event)
|
||||
if cancelled_id is not None:
|
||||
_mark_completed(cancelled_id)
|
||||
just_completed = self._step_one_frame(session, num_latent_frames)
|
||||
poses.append(session.current_pose.clone())
|
||||
if just_completed is not None:
|
||||
_mark_completed(just_completed)
|
||||
|
||||
active: list[str] = []
|
||||
if session.active_event is not None:
|
||||
active.append(session.active_event.event_id)
|
||||
|
||||
session.last_boundary_at = boundary_at
|
||||
return poses, started, active, completed
|
||||
|
||||
def _activate_event(self, session: CameraSession, event: QueuedCameraEvent) -> str | None:
|
||||
"""Activate ``event``, returning the event_id cancelled by this replacement if any."""
|
||||
cancelled_id: str | None = None
|
||||
if session.active_event is not None and session.active_event.event_id != event.event_id:
|
||||
cancelled_id = session.active_event.event_id
|
||||
|
||||
event.elapsed_transition_chunks = 0.0
|
||||
event.elapsed_transition_frames = 0
|
||||
session.active_event = event
|
||||
if event.mode == "target":
|
||||
session.target_source = session.current_pose.clone()
|
||||
else:
|
||||
session.target_source = None
|
||||
return cancelled_id
|
||||
|
||||
def _clear_active_target(self, session: CameraSession) -> str:
|
||||
"""Snap to the target pose, clear the active target track, and return its event_id."""
|
||||
assert session.active_event is not None and session.active_event.mode == "target"
|
||||
event = session.active_event
|
||||
event_id = event.event_id
|
||||
session.current_pose = event.pose.clone()
|
||||
session.active_event = None
|
||||
session.target_source = None
|
||||
return event_id
|
||||
|
||||
def _step_one_frame(self, session: CameraSession, total_num_frames_this_chunk: int) -> str | None:
|
||||
"""Advance the active camera command by one frame."""
|
||||
event = session.active_event
|
||||
if event is None:
|
||||
return None
|
||||
|
||||
if event.mode == "target":
|
||||
duration = event.transition_chunks
|
||||
source = session.target_source or session.current_pose
|
||||
target = event.pose
|
||||
if duration <= 0:
|
||||
return self._clear_active_target(session)
|
||||
|
||||
total_frames = max(1, int(round(duration * float(total_num_frames_this_chunk))))
|
||||
event.elapsed_transition_frames += 1
|
||||
event.elapsed_transition_chunks = event.elapsed_transition_frames / float(total_num_frames_this_chunk)
|
||||
alpha = min(1.0, event.elapsed_transition_frames / float(total_frames))
|
||||
session.current_pose = _lerp_pose(source, target, alpha)
|
||||
if event.elapsed_transition_frames >= total_frames:
|
||||
return self._clear_active_target(session)
|
||||
return None
|
||||
|
||||
# Velocity: apply the held structural SE3 delta once per frame.
|
||||
session.current_pose = _compose_pose(session.current_pose, event.pose)
|
||||
return None
|
||||
|
||||
|
||||
def _media_frame_to_latent(
|
||||
media_frame: int,
|
||||
*,
|
||||
num_media_frames: int,
|
||||
num_latent_frames: int,
|
||||
) -> int:
|
||||
"""Map a media-timeline frame index onto the latent control grid."""
|
||||
if num_latent_frames <= 1 or num_media_frames <= 1:
|
||||
return 0
|
||||
return min(int(media_frame) * int(num_latent_frames) // int(num_media_frames), num_latent_frames - 1)
|
||||
|
||||
|
||||
def _as_xyz(value: object, *, name: str) -> Vec3:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 3:
|
||||
raise ValueError(f"camera {name} must be a length-3 list/tuple")
|
||||
if any(math.isnan(v) or math.isinf(v) for v in value):
|
||||
raise ValueError(f"camera {name} must be non-nan and non-inf")
|
||||
return (float(value[0]), float(value[1]), float(value[2]))
|
||||
|
||||
|
||||
def _as_quat(value: object, *, name: str) -> Quat:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 4:
|
||||
raise ValueError(f"camera {name} must be a length-4 list/tuple (x, y, z, w)")
|
||||
if any(math.isnan(v) or math.isinf(v) for v in value):
|
||||
raise ValueError(f"camera {name} must be non-nan and non-inf")
|
||||
return _quat_normalize((float(value[0]), float(value[1]), float(value[2]), float(value[3])))
|
||||
|
||||
|
||||
def _parse_pose(data: Mapping[str, object]) -> CameraPose:
|
||||
translation = _as_xyz(data.get("translation", (0.0, 0.0, 0.0)), name="translation")
|
||||
rotation = _as_quat(data.get("rotation", _IDENTITY_QUAT), name="rotation")
|
||||
return CameraPose(translation=translation, rotation=rotation)
|
||||
|
||||
|
||||
def _lerp(a: float, b: float, alpha: float) -> float:
|
||||
return a + (b - a) * alpha
|
||||
|
||||
|
||||
def _lerp_pose(source: CameraPose, target: CameraPose, alpha: float) -> CameraPose:
|
||||
alpha = min(1.0, max(0.0, alpha))
|
||||
return CameraPose(
|
||||
translation=cast(
|
||||
Vec3,
|
||||
tuple(_lerp(s, t, alpha) for s, t in zip(source.translation, target.translation)),
|
||||
),
|
||||
rotation=_quat_nlerp(source.rotation, target.rotation, alpha),
|
||||
)
|
||||
|
||||
|
||||
def _compose_pose(base: CameraPose, delta: CameraPose) -> CameraPose:
|
||||
"""Apply ``delta`` after ``base`` in the session frame (T_new = T_base @ T_delta)."""
|
||||
return CameraPose.from_matrix(base.as_matrix() @ delta.as_matrix())
|
||||
|
||||
|
||||
def _quat_normalize(q: Quat) -> Quat:
|
||||
x, y, z, w = q
|
||||
norm = math.sqrt(x * x + y * y + z * z + w * w)
|
||||
if norm <= 0.0:
|
||||
raise ValueError("camera rotation quaternion must be non-zero")
|
||||
return (x / norm, y / norm, z / norm, w / norm)
|
||||
|
||||
|
||||
def _quat_nlerp(a: Quat, b: Quat, alpha: float) -> Quat:
|
||||
"""Normalized linear interpolation; flips ``b`` when needed for shortest path."""
|
||||
dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]
|
||||
if dot < 0.0:
|
||||
b = (-b[0], -b[1], -b[2], -b[3])
|
||||
return _quat_normalize(
|
||||
(
|
||||
_lerp(a[0], b[0], alpha),
|
||||
_lerp(a[1], b[1], alpha),
|
||||
_lerp(a[2], b[2], alpha),
|
||||
_lerp(a[3], b[3], alpha),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _quat_to_rotmat(q: Quat) -> torch.Tensor:
|
||||
x, y, z, w = q
|
||||
xx, yy, zz = x * x, y * y, z * z
|
||||
xy, xz, yz = x * y, x * z, y * z
|
||||
wx, wy, wz = w * x, w * y, w * z
|
||||
return torch.tensor(
|
||||
[
|
||||
[1.0 - 2.0 * (yy + zz), 2.0 * (xy - wz), 2.0 * (xz + wy)],
|
||||
[2.0 * (xy + wz), 1.0 - 2.0 * (xx + zz), 2.0 * (yz - wx)],
|
||||
[2.0 * (xz - wy), 2.0 * (yz + wx), 1.0 - 2.0 * (xx + yy)],
|
||||
],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
|
||||
def _rotmat_to_quat(rotation: torch.Tensor) -> Quat:
|
||||
"""Convert a 3x3 rotation matrix to a unit quaternion ``(x, y, z, w)``."""
|
||||
m = rotation
|
||||
trace = float(m[0, 0] + m[1, 1] + m[2, 2])
|
||||
if trace > 0.0:
|
||||
s = math.sqrt(trace + 1.0) * 2.0
|
||||
w = 0.25 * s
|
||||
x = (float(m[2, 1]) - float(m[1, 2])) / s
|
||||
y = (float(m[0, 2]) - float(m[2, 0])) / s
|
||||
z = (float(m[1, 0]) - float(m[0, 1])) / s
|
||||
elif float(m[0, 0]) > float(m[1, 1]) and float(m[0, 0]) > float(m[2, 2]):
|
||||
s = math.sqrt(1.0 + float(m[0, 0]) - float(m[1, 1]) - float(m[2, 2])) * 2.0
|
||||
w = (float(m[2, 1]) - float(m[1, 2])) / s
|
||||
x = 0.25 * s
|
||||
y = (float(m[0, 1]) + float(m[1, 0])) / s
|
||||
z = (float(m[0, 2]) + float(m[2, 0])) / s
|
||||
elif float(m[1, 1]) > float(m[2, 2]):
|
||||
s = math.sqrt(1.0 + float(m[1, 1]) - float(m[0, 0]) - float(m[2, 2])) * 2.0
|
||||
w = (float(m[0, 2]) - float(m[2, 0])) / s
|
||||
x = (float(m[0, 1]) + float(m[1, 0])) / s
|
||||
y = 0.25 * s
|
||||
z = (float(m[1, 2]) + float(m[2, 1])) / s
|
||||
else:
|
||||
s = math.sqrt(1.0 + float(m[2, 2]) - float(m[0, 0]) - float(m[1, 1])) * 2.0
|
||||
w = (float(m[1, 0]) - float(m[0, 1])) / s
|
||||
x = (float(m[0, 2]) + float(m[2, 0])) / s
|
||||
y = (float(m[1, 2]) + float(m[2, 1])) / s
|
||||
z = 0.25 * s
|
||||
return _quat_normalize((x, y, z, w))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CameraPose",
|
||||
"CameraSession",
|
||||
"QueuedCameraEvent",
|
||||
"SE3DeltaCameraHandler",
|
||||
]
|
||||
@@ -90,6 +90,7 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
"""
|
||||
|
||||
modality: ClassVar[str] = "prompt"
|
||||
lazy_initialize_session: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -113,16 +114,14 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
)
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
|
||||
prompt = payload.get("prompt")
|
||||
if not isinstance(prompt, str) or not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
@@ -136,6 +135,27 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
|
||||
self.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
prompt = payload.get("prompt")
|
||||
assert isinstance(prompt, str)
|
||||
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
|
||||
|
||||
target_prompt_embeds, _ = self._encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=None,
|
||||
@@ -164,12 +184,13 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance or start prompt interpolation before the next chunk."""
|
||||
# Prompt lerp is chunk-LWW; media timeline and caller chunk_index unused.
|
||||
del chunk_index, num_frames, fps
|
||||
del chunk_index, num_media_frames, fps, num_latent_frames
|
||||
session = state.interaction_sessions.get("prompt")
|
||||
assert isinstance(session, PromptSession)
|
||||
|
||||
|
||||
@@ -5,12 +5,13 @@ A pipeline only declares the modalities that it supports.
|
||||
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
|
||||
|
||||
Outer key: pipeline architecture name (``od_config.model_class_name``).
|
||||
Inner key: modality name (e.g. ``\"prompt\"``).
|
||||
Inner key: modality name (e.g. ``\"camera\"``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import SE3DeltaCameraHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
|
||||
|
||||
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
@@ -21,4 +22,7 @@ STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
"HeliosPyramidPipeline": {
|
||||
"prompt": PromptInteractionHandler,
|
||||
},
|
||||
"LingBotWorldCausalDMDPipeline": {
|
||||
"camera": SE3DeltaCameraHandler,
|
||||
},
|
||||
}
|
||||
@@ -1,38 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared types for midway interaction handlers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
# Wire payload for one modality track. Handlers validate keys they need.
|
||||
InteractionPayload = Mapping[str, object]
|
||||
|
||||
# Shared command semantics across modalities that support them.
|
||||
# Prompt specialization uses ``"target"`` only.
|
||||
InteractionMode = Literal["target"]
|
||||
# Prompt specialization narrows this to ``"target"`` only.
|
||||
InteractionMode = Literal["target", "velocity"]
|
||||
|
||||
|
||||
def synchronized_monotonic_time(stamp: float | None = None) -> float:
|
||||
"""Return a monotonic timestamp shared across DiT ranks.
|
||||
|
||||
``submit_interaction`` and chunk-boundary apply run as collectives, but
|
||||
``time.monotonic()`` is rank-local. Under USP/SP, inter-rank skew that
|
||||
straddles a ``1/fps`` bucket edge would schedule the same event onto
|
||||
different frames per shard. Rank 0's stamp is broadcast (callers may pass
|
||||
a local sample; non-source ranks' values are ignored).
|
||||
"""
|
||||
if stamp is None:
|
||||
stamp = time.monotonic()
|
||||
if not torch.distributed.is_initialized():
|
||||
return stamp
|
||||
try:
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
|
||||
|
||||
group = get_world_group()
|
||||
except Exception:
|
||||
return stamp
|
||||
if group.world_size <= 1:
|
||||
return stamp
|
||||
return group.broadcast_object(stamp, src=0)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChunkMediaSpec:
|
||||
"""Media extent of one generation chunk, for interaction timeline mapping."""
|
||||
"""Decoded media extent and latent step count for one generation chunk.
|
||||
|
||||
num_frames: int
|
||||
Attributes:
|
||||
num_media_frames: Number of frames in this chunk when decoded into playable media.
|
||||
fps: Frames per second of the media.
|
||||
num_latent_frames: Number of latent frames in this chunk (internal DiT units).
|
||||
"""
|
||||
|
||||
num_media_frames: int
|
||||
fps: float
|
||||
num_latent_frames: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.num_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_frames must be > 0, got {self.num_frames}")
|
||||
if self.num_media_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_media_frames must be > 0, got {self.num_media_frames}")
|
||||
if self.num_latent_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_latent_frames must be > 0, got {self.num_latent_frames}")
|
||||
if self.fps <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.fps must be > 0, got {self.fps}")
|
||||
|
||||
@property
|
||||
def duration_s(self) -> float:
|
||||
return float(self.num_frames) / float(self.fps)
|
||||
return float(self.num_media_frames) / float(self.fps)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
@@ -41,6 +78,7 @@ class InteractionEvent:
|
||||
|
||||
Example:
|
||||
- ``QueuedPromptEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
- ``QueuedCameraEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
|
||||
"""
|
||||
|
||||
event_id: str
|
||||
@@ -60,12 +98,37 @@ class InteractionSession:
|
||||
|
||||
Example:
|
||||
- ``PromptSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
- ``CameraSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
|
||||
"""
|
||||
|
||||
lock: Lock = field(default_factory=Lock, repr=False)
|
||||
last_boundary_at: float | None = None
|
||||
|
||||
|
||||
def resolve_event_frame_offset(
|
||||
*,
|
||||
received_at: float,
|
||||
previous_boundary_at: float | None,
|
||||
num_frames: int,
|
||||
fps: float,
|
||||
) -> int:
|
||||
"""Map an event arrival time onto ``[0, num_frames - 1]`` for this chunk.
|
||||
|
||||
* Uses ``floor((received_at - previous_boundary_at) * fps)``.
|
||||
* Events without a prior boundary (or with non-positive fps) map to frame 0.
|
||||
* Offsets past the represented media window are clamped to the final frame.
|
||||
"""
|
||||
num_frames = max(int(num_frames), 1)
|
||||
if previous_boundary_at is None or fps <= 0:
|
||||
return 0
|
||||
offset = math.floor((received_at - previous_boundary_at) * float(fps))
|
||||
if offset < 0:
|
||||
return 0
|
||||
if offset >= num_frames:
|
||||
return num_frames - 1
|
||||
return int(offset)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InteractionChunkMetadata:
|
||||
"""Per-modality (or merged) event-id bookkeeping for one chunk boundary."""
|
||||
|
||||
@@ -535,19 +535,23 @@ class HeliosPipeline(
|
||||
"zero_steps": int(extra.get("zero_steps", 1)),
|
||||
}
|
||||
)
|
||||
self.prepare_next_chunk(state)
|
||||
return state
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Expose this chunk's decoded media extent for interaction timelines."""
|
||||
num_frames = int(state.extra["window_num_frames"])
|
||||
num_media_frames = int(state.extra["window_num_frames"])
|
||||
num_latent_frames = int(state.extra["num_latent_frames_per_chunk"])
|
||||
fps = state.sampling.fps
|
||||
if fps is None or float(fps) <= 0:
|
||||
raise ValueError(
|
||||
"sampling.fps is required and must be > 0 for interaction modalities that use the "
|
||||
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
|
||||
)
|
||||
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
|
||||
return ChunkMediaSpec(
|
||||
num_media_frames=num_media_frames,
|
||||
fps=float(fps),
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
|
||||
@override
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
|
||||
@@ -151,10 +151,10 @@ class SupportsInteractionApply(Protocol):
|
||||
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Return the media timeline represented by the upcoming/current chunk.
|
||||
"""Return the media timeline and latent frame count for the upcoming/current chunk.
|
||||
|
||||
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
|
||||
or for backpressure/pacing.
|
||||
and/or for backpressure/pacing.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm_omni.experimental.ar_diffusion.tick_protocol import (
|
||||
|
||||
LINGBOT_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1"
|
||||
LINGBOT_CAMERA_TRAJECTORY_SCHEMA = "lingbot.camera_trajectory.v1"
|
||||
LINGBOT_CONTROLLER_TRANSLATION_UNIT = 0.05
|
||||
_ACTION_ORDER = ("w", "a", "s", "d", "i", "j", "k", "l")
|
||||
_VALID_ACTIONS = frozenset(_ACTION_ORDER)
|
||||
_REFERENCE_HEIGHT = 480
|
||||
@@ -282,7 +283,7 @@ def integrate_lingbot_camera_actions(
|
||||
"""Convert latent-frame WASD/IJKL controls to cumulative C2W poses.
|
||||
|
||||
Calibration and motion constants intentionally match SGLang's LingBot
|
||||
adapter: movement 0.05, pitch 4 degrees, yaw 6 degrees, pitch clamp 85
|
||||
adapter: movement 0.05 (LINGBOT_CONTROLLER_TRANSLATION_UNIT), pitch 4 degrees, yaw 6 degrees, pitch clamp 85
|
||||
degrees, and target-resolution focal lengths of 500 pixels.
|
||||
"""
|
||||
|
||||
@@ -325,13 +326,31 @@ def integrate_lingbot_camera_actions(
|
||||
right /= right_norm + 1e-6
|
||||
|
||||
movement = np.zeros(3, dtype=np.float64)
|
||||
movement += forward * 0.05 * (("w" in actions) - ("s" in actions))
|
||||
movement += right * 0.05 * (("d" in actions) - ("a" in actions))
|
||||
movement += forward * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("w" in actions) - ("s" in actions))
|
||||
movement += right * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("d" in actions) - ("a" in actions))
|
||||
# Keep the cumulative controller pose in FP64 between chunks, matching
|
||||
# SGLang's NumPy integration and avoiding long-session drift.
|
||||
current_pose = np.eye(4, dtype=np.float64)
|
||||
current_pose[:3, :3] = new_rotation
|
||||
current_pose[:3, 3] = translation + movement
|
||||
poses.append(current_pose)
|
||||
|
||||
trajectory = camera_trajectory_from_absolute_pose(torch.from_numpy(np.stack(poses)), width=width, height=height)
|
||||
return trajectory, current_pitch
|
||||
|
||||
|
||||
def camera_trajectory_from_absolute_pose(
|
||||
poses: torch.Tensor,
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> CameraTrajectory:
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("camera action resolution must be positive.")
|
||||
if poses.ndim != 3 or poses.shape[-2:] != (4, 4):
|
||||
raise ValueError(f"absolute camera poses must have shape [frames, 4, 4], got {tuple(poses.shape)}.")
|
||||
if poses.shape[0] == 0:
|
||||
raise ValueError("absolute camera poses must not be empty.")
|
||||
# build_plucker_embedding expects intrinsics in the 832x480 reference
|
||||
# coordinate system. These values become SGLang's [500, 500, W/2, H/2]
|
||||
# after that function scales them to the requested resolution.
|
||||
@@ -342,9 +361,7 @@ def integrate_lingbot_camera_actions(
|
||||
_REFERENCE_HEIGHT / 2,
|
||||
)
|
||||
trajectory = CameraTrajectory(
|
||||
# Keep the cumulative controller pose in FP64 between chunks, matching
|
||||
# SGLang's NumPy integration and avoiding long-session drift.
|
||||
poses=torch.from_numpy(np.stack(poses)),
|
||||
poses=poses,
|
||||
intrinsics=torch.tensor(reference_intrinsics, dtype=torch.float32).repeat(len(poses), 1),
|
||||
)
|
||||
return trajectory, current_pitch
|
||||
return trajectory
|
||||
@@ -389,7 +389,11 @@ def _invert_camera_poses(poses: torch.Tensor) -> torch.Tensor:
|
||||
return inverse
|
||||
|
||||
|
||||
def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
|
||||
def _prepare_framewise_poses(
|
||||
raw_poses: torch.Tensor,
|
||||
*,
|
||||
translation_scale: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Convert raw C2W poses to normalized first-relative framewise deltas."""
|
||||
|
||||
# Remove global placement, then convert the sequence to framewise deltas.
|
||||
@@ -402,12 +406,19 @@ def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
|
||||
relative_poses[1:],
|
||||
)
|
||||
|
||||
# Only relative direction/magnitude matters to the checkpoint; normalizing
|
||||
# by the largest step removes the trajectory's arbitrary translation scale.
|
||||
# Normalizing the control's scale. Different between scripted and realtime input.
|
||||
translations = relative_poses[:, :3, 3]
|
||||
max_translation_norm = torch.linalg.vector_norm(translations, dim=-1).max()
|
||||
if max_translation_norm > 0:
|
||||
relative_poses[:, :3, 3] = translations / max_translation_norm
|
||||
if translation_scale is None:
|
||||
# Scripted action_path input: divide by the largest step so an arbitrary recording scale becomes 1.
|
||||
scale = torch.linalg.vector_norm(translations, dim=-1).max()
|
||||
else:
|
||||
# Realtime interaction input: use the controller translation unit (one WASD step = full speed).
|
||||
# It cannot see future motion or normalize globally.
|
||||
if translation_scale <= 0:
|
||||
raise ValueError(f"translation_scale must be > 0, got {translation_scale}")
|
||||
scale = translation_scale
|
||||
if float(scale) > 0:
|
||||
relative_poses[:, :3, 3] = translations / scale
|
||||
return relative_poses
|
||||
|
||||
|
||||
@@ -420,6 +431,7 @@ def build_plucker_embedding(
|
||||
target_width: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
translation_scale: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build checkpoint-ordered ``(ray origin, ray direction)`` channels."""
|
||||
|
||||
@@ -438,7 +450,10 @@ def build_plucker_embedding(
|
||||
|
||||
# Compute geometry in at least FP32, then cast at the model boundary.
|
||||
compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
|
||||
poses = _prepare_framewise_poses(trajectory.poses.to(device=device, dtype=compute_dtype))
|
||||
poses = _prepare_framewise_poses(
|
||||
trajectory.poses.to(device=device, dtype=compute_dtype),
|
||||
translation_scale=translation_scale,
|
||||
)
|
||||
intrinsics = trajectory.intrinsics.to(device=device, dtype=compute_dtype).clone()
|
||||
intrinsics[:, (0, 2)] *= width / _REFERENCE_WIDTH
|
||||
intrinsics[:, (1, 3)] *= height / _REFERENCE_HEIGHT
|
||||
|
||||
@@ -26,16 +26,21 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
|
||||
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery, SupportsStepExecution
|
||||
from vllm_omni.diffusion.models.lingbot_world.actions import (
|
||||
LINGBOT_CAMERA_ACTION_SCHEMA,
|
||||
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
|
||||
LINGBOT_CONTROLLER_TRANSLATION_UNIT,
|
||||
LingBotCameraActionFrames,
|
||||
LingBotCameraActionScript,
|
||||
as_camera_action_frames,
|
||||
as_camera_action_script,
|
||||
camera_trajectory_from_absolute_pose,
|
||||
integrate_lingbot_camera_actions,
|
||||
parse_lingbot_camera_action_frames,
|
||||
parse_lingbot_camera_action_script,
|
||||
@@ -385,25 +390,36 @@ def get_lingbot_world_pre_process_func(
|
||||
camera_actions = None
|
||||
else:
|
||||
action_path = extra_args.get("action_path")
|
||||
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
if not configured_action_root:
|
||||
raise ValueError(
|
||||
"sampling_params.extra_args.action_path requires a trusted action root configured by "
|
||||
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
|
||||
if action_path is None or action_path == "":
|
||||
# Stepwise / streaming sessions may omit a request-scoped camera
|
||||
# script and instead drive motion via mid-generation camera
|
||||
# interaction (idle hold until the first camera event).
|
||||
if getattr(od_config, "step_execution", False) or getattr(od_config, "streaming_output", False):
|
||||
trajectory = None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
else:
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
else:
|
||||
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
if not configured_action_root:
|
||||
raise ValueError(
|
||||
"sampling_params.extra_args.action_path requires a trusted action root configured by "
|
||||
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
|
||||
)
|
||||
action_directory = resolve_trusted_action_directory(
|
||||
action_path,
|
||||
configured_action_root,
|
||||
)
|
||||
action_directory = resolve_trusted_action_directory(
|
||||
action_path,
|
||||
configured_action_root,
|
||||
)
|
||||
try:
|
||||
trajectory = load_camera_trajectory(action_directory)
|
||||
except OSError:
|
||||
raise ValueError(
|
||||
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
|
||||
) from None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
try:
|
||||
trajectory = load_camera_trajectory(action_directory)
|
||||
except OSError:
|
||||
raise ValueError(
|
||||
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
|
||||
) from None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
|
||||
updated_prompt = dict(prompt)
|
||||
updated_multi_modal_data = dict(multi_modal_data)
|
||||
@@ -460,6 +476,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
SupportImageInput,
|
||||
SupportsComponentDiscovery,
|
||||
SupportsStepExecution,
|
||||
InteractionMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
):
|
||||
@@ -779,8 +796,10 @@ class LingBotWorldCausalDMDPipeline(
|
||||
camera_sources = (
|
||||
int(camera_trajectory is not None) + int(camera_actions is not None) + int(camera_action_script is not None)
|
||||
)
|
||||
if camera_sources != 1:
|
||||
raise ValueError("LingBot pre-processing must materialize exactly one camera input.")
|
||||
if camera_sources > 1:
|
||||
raise ValueError("LingBot pre-processing must materialize at most one camera input.")
|
||||
# Zero camera sources are allowed for stepwise camera-interaction sessions;
|
||||
# request-mode ``forward`` still requires an explicit trajectory or actions.
|
||||
|
||||
request_flow_shift = (
|
||||
extra_args["flow_shift"] if "flow_shift" in extra_args else getattr(self.scheduler.config, "shift", 5.0)
|
||||
@@ -1047,7 +1066,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
previous: CameraTrajectory | None = None,
|
||||
latent_aligned: bool = False,
|
||||
latent_aligned: bool = False, # iff using realtime interaction inputs
|
||||
) -> tuple[torch.Tensor, CameraTrajectory]:
|
||||
"""Convert raw camera frames to a latent-aligned ray tensor."""
|
||||
|
||||
@@ -1083,10 +1102,10 @@ class LingBotWorldCausalDMDPipeline(
|
||||
),
|
||||
)
|
||||
drop_anchor = True
|
||||
elif inputs.camera_actions is not None:
|
||||
# The action integrator returns post-action poses. Keep the first
|
||||
# action visible to framewise-delta conditioning by prepending its
|
||||
# known pre-action state (identity for a new realtime session).
|
||||
elif inputs.camera_actions is not None or latent_aligned:
|
||||
# Action/interaction integrators return post-action poses. Prepend the
|
||||
# known pre-action state (identity for a new session) so the first
|
||||
# pose still contributes a framewise delta.
|
||||
identity = torch.eye(
|
||||
4,
|
||||
device=trajectory.poses.device,
|
||||
@@ -1100,6 +1119,8 @@ class LingBotWorldCausalDMDPipeline(
|
||||
),
|
||||
)
|
||||
drop_anchor = True
|
||||
# Realtime interaction cannot see future steps; use the controller translation unit to normalize speed.
|
||||
# Full action_path keeps max-norm.
|
||||
camera_embedding = build_plucker_embedding(
|
||||
embedding_trajectory,
|
||||
height=inputs.height,
|
||||
@@ -1108,6 +1129,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
target_width=inputs.width,
|
||||
device=self.device,
|
||||
dtype=dtype,
|
||||
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT if latent_aligned else None,
|
||||
)
|
||||
if drop_anchor:
|
||||
camera_embedding = camera_embedding[1:]
|
||||
@@ -1277,6 +1299,11 @@ class LingBotWorldCausalDMDPipeline(
|
||||
"request mode takes one three-frame camera_actions control per "
|
||||
"block, or a camera_trajectory for the request."
|
||||
)
|
||||
if inputs.camera_trajectory is None and inputs.camera_actions is None:
|
||||
raise ValueError(
|
||||
"LingBot request mode requires camera_trajectory or camera_actions; "
|
||||
"omit them only for stepwise camera-interaction sessions."
|
||||
)
|
||||
tick = ARDiffusionTickRequest.from_extra_args(req.sampling_params.extra_args)
|
||||
if tick is not None and self._ar_diffusion_kv_state is None:
|
||||
raise RuntimeError("LingBot typed ticks require ARDiffusionEngine session binding.")
|
||||
@@ -1645,11 +1672,6 @@ class LingBotWorldCausalDMDPipeline(
|
||||
}
|
||||
self._ar_text_caches(prompt_embeds, invalidate=False)
|
||||
self._ar_sessions[state.request_id] = _LingBotARSessionState()
|
||||
try:
|
||||
self._prepare_next_chunk(state)
|
||||
except Exception:
|
||||
self._release_condition_encoder_state(state.request_id)
|
||||
raise
|
||||
return state
|
||||
|
||||
def _prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
@@ -1668,7 +1690,14 @@ class LingBotWorldCausalDMDPipeline(
|
||||
dtype=extra["dtype"],
|
||||
)
|
||||
previous = extra.get("camera_tail")
|
||||
|
||||
if extra.get("camera_action_script") is not None:
|
||||
if (
|
||||
(camera_session := state.interaction_sessions.get("camera"))
|
||||
and isinstance(camera_session, CameraSession)
|
||||
and camera_session.has_received_input
|
||||
):
|
||||
raise ValueError("Cannot use mid-generation camera interaction together with camera_action_script; ")
|
||||
chunk_actions = extra["camera_action_script"][state.chunk_index]
|
||||
action_trajectory, camera_pitch = integrate_lingbot_camera_actions(
|
||||
chunk_actions,
|
||||
@@ -1678,11 +1707,12 @@ class LingBotWorldCausalDMDPipeline(
|
||||
initial_pitch=float(extra.get("camera_pitch", 0.0)),
|
||||
)
|
||||
extra["camera_pitch"] = camera_pitch
|
||||
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
|
||||
chunk_inputs = replace(
|
||||
inputs,
|
||||
camera_trajectory=action_trajectory,
|
||||
camera_actions=chunk_actions,
|
||||
num_frames=(block_frames - 1) * self.vae_scale_factor_temporal + 1,
|
||||
num_frames=media_frames,
|
||||
num_latent_frames=block_frames,
|
||||
)
|
||||
camera, camera_tail = self._prepare_camera(
|
||||
@@ -1691,14 +1721,53 @@ class LingBotWorldCausalDMDPipeline(
|
||||
previous=previous,
|
||||
)
|
||||
elif extra.get("camera_embedding_cache") is not None:
|
||||
# Same slice request mode takes from its one full-trajectory
|
||||
# embedding, so both paths condition a block identically.
|
||||
if (
|
||||
(camera_session := state.interaction_sessions.get("camera"))
|
||||
and isinstance(camera_session, CameraSession)
|
||||
and camera_session.has_received_input
|
||||
):
|
||||
raise ValueError("Cannot use mid-generation camera interaction together with camera_embedding_cache; ")
|
||||
trajectory = extra["camera_trajectory_cache"]
|
||||
camera = extra["camera_embedding_cache"][:, :, start_frame:stop_frame]
|
||||
camera_tail = CameraTrajectory(
|
||||
poses=trajectory.poses[stop_frame - 1 : stop_frame].clone(),
|
||||
intrinsics=trajectory.intrinsics[stop_frame - 1 : stop_frame].clone(),
|
||||
)
|
||||
elif self._interaction_coordinator is not None and self._interaction_coordinator.has_modality("camera"):
|
||||
# Poses come from the latest boundary apply: ``prepare_encode`` for
|
||||
# chunk 0, the diffusion runner for later chunks. This method only
|
||||
# digests them — it does not apply interactions itself.
|
||||
camera_session = state.interaction_sessions.get("camera")
|
||||
if not isinstance(camera_session, CameraSession) or camera_session.last_absolute_poses is None:
|
||||
raise RuntimeError(
|
||||
"LingBot camera interaction requires apply_interaction_at_chunk_boundary before prepare_next_chunk."
|
||||
)
|
||||
absolute_poses = camera_session.last_absolute_poses
|
||||
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
|
||||
num_latent_frames = block_frames
|
||||
if int(absolute_poses.shape[0]) != num_latent_frames:
|
||||
raise ValueError(
|
||||
"camera interaction must produce exactly one pose per latent frame; "
|
||||
f"got {int(absolute_poses.shape[0])}, expected {num_latent_frames}."
|
||||
)
|
||||
action_trajectory = camera_trajectory_from_absolute_pose(
|
||||
absolute_poses,
|
||||
width=inputs.width,
|
||||
height=inputs.height,
|
||||
)
|
||||
chunk_inputs = replace(
|
||||
inputs,
|
||||
camera_trajectory=action_trajectory,
|
||||
camera_actions=None,
|
||||
num_frames=media_frames,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
camera, camera_tail = self._prepare_camera(
|
||||
chunk_inputs,
|
||||
dtype=extra["dtype"],
|
||||
previous=previous,
|
||||
latent_aligned=True,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("LingBot step execution is missing a camera trajectory or action script.")
|
||||
if camera.shape[2:] != condition.shape[2:]:
|
||||
@@ -1842,19 +1911,48 @@ class LingBotWorldCausalDMDPipeline(
|
||||
session_state.pending_encoder_cache = None
|
||||
session_state.next_chunk_index += 1
|
||||
state.chunk_index += 1
|
||||
finished = state.request_denoise_completed
|
||||
if not finished:
|
||||
self._prepare_next_chunk(state)
|
||||
else:
|
||||
self._release_condition_encoder_state(state.request_id)
|
||||
return DiffusionOutput(
|
||||
output=output,
|
||||
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None,
|
||||
chunk_index=completed_chunk_index,
|
||||
total_chunks=state.total_chunks,
|
||||
finished=finished,
|
||||
finished=state.request_denoise_completed,
|
||||
)
|
||||
|
||||
def _chunk_media_frame_count(self, state: StepRequestState, *, block_frames: int) -> int:
|
||||
"""Pixel frames this chunk will emit when decoded.
|
||||
|
||||
Independent (non-streaming) decode restarts causal expansion every block,
|
||||
so each chunk is ``(block_frames - 1) * temporal + 1`` frames.
|
||||
Streaming decode expands only the session's opening latent to one frame and every later latent to the full
|
||||
temporal factor, so chunk 0 matches the independent count and later chunks are ``block_frames * temporal``.
|
||||
"""
|
||||
decode_state = self._streaming_decode_states.get(state.request_id)
|
||||
if decode_state is not None and decode_state.started:
|
||||
return block_frames * self.vae_scale_factor_temporal
|
||||
return (block_frames - 1) * self.vae_scale_factor_temporal + 1
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Expose this chunk's decoded media extent and latent step count."""
|
||||
num_latent_frames = int(state.extra.get("block_frames") or self.transformer.config.num_frames_per_block)
|
||||
num_media_frames = self._chunk_media_frame_count(state, block_frames=num_latent_frames)
|
||||
fps = state.sampling.fps
|
||||
if fps is None or float(fps) <= 0:
|
||||
# Media-frame camera controls are not wall-clock paced; a unit fps keeps
|
||||
# resolve_event_frame_offset well-defined when the client omits sampling.fps.
|
||||
fps = float(num_media_frames)
|
||||
return ChunkMediaSpec(
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
"""Prepare the next AR block after chunk-boundary interaction apply."""
|
||||
if state.request_denoise_completed:
|
||||
return
|
||||
self._prepare_next_chunk(state)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return cast(set[str], loader.load_weights(weights))
|
||||
@@ -43,6 +43,7 @@ from vllm_omni.diffusion.diffusion_kv.paged_attention_adapter import (
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_rank
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
|
||||
from vllm_omni.diffusion.interaction.types import InteractionPayload
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import (
|
||||
SupportsInteractionApply,
|
||||
@@ -1080,30 +1081,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
error_outputs: list[RunnerOutput] = []
|
||||
for state in states:
|
||||
if state.request_id in new_request_ids:
|
||||
# Everything that runs before ``_dit_any_rank_failed`` must be
|
||||
# inside the try: an exception in ``_initialize_generator`` or
|
||||
# Everything that requires rank-synchronization must be called
|
||||
# inside a try, record the exception and handle with `_dit_any_rank_failed`.
|
||||
# Reason (example): An exception in ``_initialize_generator`` or
|
||||
# ``clear_pipeline_stage_durations`` on one rank would skip the
|
||||
# all-reduce here while every peer proceeds into it, and the
|
||||
# peers then hang on the NCCL collective until timeout.
|
||||
per_req_exc: BaseException | None = None
|
||||
try:
|
||||
self._initialize_generator(state.sampling)
|
||||
clear_pipeline_stage_durations(pipeline)
|
||||
# encode
|
||||
pipeline.prepare_encode(state)
|
||||
merge_stage_durations(
|
||||
state,
|
||||
consume_pipeline_stage_durations(pipeline),
|
||||
)
|
||||
except Exception as exc:
|
||||
per_req_exc = exc
|
||||
# Pipelines that do rank-0-only work (e.g. MiniMax H3
|
||||
# reference-video prep) must broadcast per-request failures
|
||||
# internally so downstream collectives stay in step; even so,
|
||||
# cross-check that every DiT rank agrees so a rank-local error
|
||||
# (or a future pipeline that omits the guard) does not leave
|
||||
# the process group half-way through a new request.
|
||||
if _dit_any_rank_failed(per_req_exc is not None):
|
||||
def _abort_prep_failure(per_req_exc: BaseException | None) -> None:
|
||||
self.state_cache.pop(state.request_id, None)
|
||||
if per_req_exc is None:
|
||||
per_req_exc = RuntimeError(
|
||||
@@ -1123,6 +1107,41 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
result=DiffusionOutput.from_exception(per_req_exc),
|
||||
)
|
||||
)
|
||||
|
||||
per_req_exc: BaseException | None = None
|
||||
try:
|
||||
self._initialize_generator(state.sampling)
|
||||
clear_pipeline_stage_durations(pipeline)
|
||||
pipeline.prepare_encode(state)
|
||||
except Exception as exc:
|
||||
per_req_exc = exc
|
||||
# Pipelines that do rank-0-only work (e.g. MiniMax H3
|
||||
# reference-video prep) must broadcast per-request failures
|
||||
# internally so downstream collectives stay in step; even so,
|
||||
# cross-check that every DiT rank agrees so a rank-local error
|
||||
# (or a future pipeline that omits the guard) does not leave
|
||||
# the process group half-way through a new request.
|
||||
if _dit_any_rank_failed(per_req_exc is not None):
|
||||
_abort_prep_failure(per_req_exc)
|
||||
continue
|
||||
# If the pipeline supports interaction, the interaction session initialization also needs to call
|
||||
# synchronized_monotonic_time(). Wrap in another try-block to not block on prepare_encode failures.
|
||||
try:
|
||||
if supports_interaction_apply(pipeline) and state.chunk_index == 0:
|
||||
pipe = cast(SupportsInteractionApply, pipeline)
|
||||
assert self._interaction_coordinator is not None, "Model not loaded. Call load_model() first."
|
||||
state.interaction_chunk_metadata = self._interaction_coordinator.maybe_prepare_initial_session(
|
||||
state, pipe
|
||||
)
|
||||
pipe.prepare_next_chunk(state)
|
||||
merge_stage_durations(
|
||||
state,
|
||||
consume_pipeline_stage_durations(pipeline),
|
||||
)
|
||||
except Exception as exc:
|
||||
per_req_exc = exc
|
||||
if _dit_any_rank_failed(per_req_exc is not None):
|
||||
_abort_prep_failure(per_req_exc)
|
||||
continue
|
||||
prepared_states.append(state)
|
||||
|
||||
@@ -1431,47 +1450,33 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Route a midway interaction through the pipeline interaction coordinator."""
|
||||
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
|
||||
assert self.pipeline is not None and self._interaction_coordinator is not None, (
|
||||
"Model not loaded. Call load_model() first."
|
||||
)
|
||||
if not self.od_config.streaming_output:
|
||||
raise ValueError("submit_interaction requires streaming_output=True")
|
||||
if not self._supports_step_mode():
|
||||
raise ValueError("submit_interaction requires step execution support")
|
||||
|
||||
coordinator = self._interaction_coordinator
|
||||
if coordinator is None:
|
||||
coordinator = InteractionCoordinator.build(self.pipeline, self.od_config)
|
||||
self._interaction_coordinator = coordinator
|
||||
if hasattr(self.pipeline, "_interaction_coordinator"):
|
||||
self.pipeline._interaction_coordinator = coordinator
|
||||
|
||||
event = interaction.get("event")
|
||||
has_mm = isinstance(event, dict) and "multi_modal_data" in event
|
||||
has_prompt = isinstance(event, dict) and "prompt" in event and event.get("prompt") is not None
|
||||
|
||||
# Prompt-only interactions in this release; multi_modal_data lands with camera support.
|
||||
if not isinstance(event, dict) or has_mm or not has_prompt:
|
||||
raise NotImplementedError(
|
||||
"Only text-only prompt update interactions with 'event.prompt' and optional "
|
||||
"'transition_chunks' are supported in this release"
|
||||
)
|
||||
if not coordinator.has_modality("prompt"):
|
||||
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
|
||||
|
||||
state = self.state_cache.get(request_id)
|
||||
if state is None:
|
||||
raise ValueError(f"No active request state for interaction: {request_id!r}")
|
||||
|
||||
event_id = interaction.get("event_id")
|
||||
if not isinstance(event_id, str) or not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
prompt = event["prompt"]
|
||||
if not isinstance(prompt, str) or not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
coordinator.enqueue(
|
||||
event = interaction["event"]
|
||||
parts: list[tuple[str, InteractionPayload]] = []
|
||||
prompt = event.get("prompt")
|
||||
if prompt is not None:
|
||||
parts.append(("prompt", {"prompt": prompt}))
|
||||
multi_modal_data = event.get("multi_modal_data")
|
||||
if multi_modal_data:
|
||||
parts.extend(
|
||||
(str(modality), cast(InteractionPayload, payload)) for modality, payload in multi_modal_data.items()
|
||||
)
|
||||
|
||||
self._interaction_coordinator.enqueue_parts(
|
||||
state,
|
||||
modality="prompt",
|
||||
event_id=event_id,
|
||||
parts=parts,
|
||||
event_id=interaction["event_id"],
|
||||
received_at=time.monotonic(),
|
||||
payload={"prompt": prompt},
|
||||
transition_chunks=interaction.get("transition_chunks"),
|
||||
)
|
||||
Reference in new issue
Block a user