[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
139a47a578
commit
e36babd48f
26 files changed
+2071
-200
No files matched your search
@@ -11,8 +11,9 @@ from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionPayload,
|
||||
merge_interaction_metadata,
|
||||
synchronized_monotonic_time,
|
||||
)
|
||||
from vllm_omni.diffusion.models.interface import supports_interaction_apply
|
||||
from vllm_omni.diffusion.models.interface import SupportsInteractionApply, supports_interaction_apply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -52,7 +53,7 @@ class InteractionCoordinator:
|
||||
|
||||
@property
|
||||
def needs_chunk_media(self) -> bool:
|
||||
"""Whether any registered handler needs chunk ``num_frames``/``fps``."""
|
||||
"""Whether any registered handler needs chunk media/latent counts + fps."""
|
||||
return any(handler.needs_chunk_media for handler in self._handlers.values())
|
||||
|
||||
def get_handler(self, modality: str) -> InteractionHandler:
|
||||
@@ -83,31 +84,109 @@ class InteractionCoordinator:
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
|
||||
def enqueue_parts(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
parts: list[tuple[str, InteractionPayload]],
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Ensure every modality is supported and every payload validates, then enqueue.
|
||||
|
||||
Composite events must not partially mutate queues when a later track is
|
||||
unsupported or carries an invalid payload.
|
||||
"""
|
||||
if not parts:
|
||||
raise ValueError("interaction event requires prompt and/or multi_modal_data")
|
||||
|
||||
# Reconcile rank-local samples so USP/SP shards share one arrival time.
|
||||
received_at = synchronized_monotonic_time(received_at)
|
||||
|
||||
handlers = [(self.get_handler(modality), payload) for modality, payload in parts]
|
||||
for handler, payload in handlers:
|
||||
handler.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
for (modality, payload), (handler, _) in zip(parts, handlers, strict=True):
|
||||
del handler
|
||||
self.enqueue(
|
||||
state,
|
||||
modality=modality,
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
|
||||
def maybe_prepare_initial_session(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
pipeline: SupportsInteractionApply,
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Materialize non-lazy modality sessions before chunk 0 denoise.
|
||||
|
||||
Handlers with ``lazy_initialize_session=False`` (e.g. camera) must exist
|
||||
before the first chunk even with no client enqueue.
|
||||
"""
|
||||
num_media_frames: int | None = None
|
||||
fps: float | None = None
|
||||
num_latent_frames: int | None = None
|
||||
if any(
|
||||
(not handler.lazy_initialize_session) and handler.needs_chunk_media for handler in self._handlers.values()
|
||||
):
|
||||
media = pipeline.peek_chunk_media(state)
|
||||
num_media_frames = media.num_media_frames
|
||||
fps = media.fps
|
||||
num_latent_frames = media.num_latent_frames
|
||||
|
||||
boundary_at = synchronized_monotonic_time()
|
||||
chunk_index = state.chunk_index
|
||||
metas: list[InteractionChunkMetadata] = []
|
||||
for handler in self._handlers_in_apply_order():
|
||||
if handler.lazy_initialize_session:
|
||||
continue
|
||||
meta = handler.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=boundary_at,
|
||||
chunk_index=chunk_index,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
if meta is not None:
|
||||
metas.append(meta)
|
||||
return merge_interaction_metadata(metas)
|
||||
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Fan out chunk-boundary apply to handlers in stable order (prompt first)."""
|
||||
if chunk_index is None:
|
||||
chunk_index = state.chunk_index
|
||||
metas: list[InteractionChunkMetadata] = []
|
||||
for handler in self._handlers_in_apply_order():
|
||||
if handler.modality not in state.interaction_sessions:
|
||||
# This function is always called at chunk boundary without knowing if an interaction is really enqueued.
|
||||
# `state.interaction_sessions[this modality]` is lazy-populated after the first interaction is enqueued.
|
||||
# So, check here and skip a modality handler if this modality's session is not yet created.
|
||||
if handler.lazy_initialize_session and handler.modality not in state.interaction_sessions:
|
||||
# Lazy modalities skip apply until the first enqueue creates the session.
|
||||
continue
|
||||
meta = handler.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=boundary_at,
|
||||
chunk_index=chunk_index,
|
||||
num_frames=num_frames,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
if meta is not None:
|
||||
metas.append(meta)
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm_omni.diffusion.interaction.types import synchronized_monotonic_time
|
||||
from vllm_omni.diffusion.models.interface import SupportsInteractionApply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
@@ -35,18 +35,21 @@ class InteractionMixin:
|
||||
"onto the pipeline before chunked generation"
|
||||
)
|
||||
|
||||
num_frames: int | None = None
|
||||
num_media_frames: int | None = None
|
||||
fps: float | None = None
|
||||
num_latent_frames: int | None = None
|
||||
if self._interaction_coordinator.needs_chunk_media:
|
||||
media = cast(SupportsInteractionApply, self).peek_chunk_media(state)
|
||||
num_frames = media.num_frames
|
||||
num_media_frames = media.num_media_frames
|
||||
fps = media.fps
|
||||
num_latent_frames = media.num_latent_frames
|
||||
|
||||
merged = self._interaction_coordinator.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=time.monotonic(),
|
||||
num_frames=num_frames,
|
||||
boundary_at=synchronized_monotonic_time(),
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
state.interaction_chunk_metadata = merged
|
||||
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import ClassVar, Self
|
||||
from typing import ClassVar
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
@@ -23,9 +25,14 @@ class InteractionHandler(ABC):
|
||||
"""
|
||||
|
||||
modality: ClassVar[str]
|
||||
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (num_frames/fps)
|
||||
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (frame count & fps)
|
||||
# Those information are useful when interaction handler needs interpolation/integration on a frame-by-frame basis
|
||||
needs_chunk_media: ClassVar[bool] = False
|
||||
# When True, the coordinator skips ``apply_at_chunk_boundary`` until a session
|
||||
# exists (normally created on the first enqueue). When False, every chunk
|
||||
# boundary runs apply even with no prior enqueue so the handler can create
|
||||
# its session and materialize default chunk data (e.g. identity camera hold).
|
||||
lazy_initialize_session: ClassVar[bool] = True
|
||||
|
||||
@classmethod
|
||||
def from_pipeline(cls, pipeline: object) -> Self:
|
||||
@@ -37,6 +44,21 @@ class InteractionHandler(ABC):
|
||||
del pipeline
|
||||
return cls()
|
||||
|
||||
@abstractmethod
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Validate a modality payload without mutating request state.
|
||||
|
||||
Used by ``InteractionCoordinator.enqueue_parts`` so composite events
|
||||
fail before any track is queued.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def enqueue(
|
||||
self,
|
||||
@@ -56,7 +78,9 @@ class InteractionHandler(ABC):
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None, # defaults to state.chunk_index when omitted
|
||||
num_frames: int | None = None, # only set when at least one interaction handler needs it
|
||||
fps: float | None = None, # only set when at least one interaction handler needs it
|
||||
# Optional chunk media information. Only present when at least one initialized interaction handler needs it
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance request-local state and materialize this chunk's effects."""
|
||||
@@ -0,0 +1,490 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Camera modality interaction handlers.
|
||||
|
||||
Payload is structural SE(3): ``translation`` (xyz) + unit
|
||||
quaternion ``rotation`` (x, y, z, w). WASD-style key tokens belong in
|
||||
clients, not in the engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self, override
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionEvent,
|
||||
InteractionMode,
|
||||
InteractionPayload,
|
||||
InteractionSession,
|
||||
resolve_event_frame_offset,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
Vec3 = tuple[float, float, float]
|
||||
# Unit quaternion in ``(x, y, z, w)`` order. Identity is ``(0, 0, 0, 1)``.
|
||||
Quat = tuple[float, float, float, float]
|
||||
|
||||
_IDENTITY_QUAT: Quat = (0.0, 0.0, 0.0, 1.0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraPose:
|
||||
"""Canonical absolute camera pose used by the shared timeline.
|
||||
|
||||
Coordinates follow Unity semantics: ``+X`` right, ``+Y`` up,
|
||||
``+Z`` forward. Session start is the identity transform;
|
||||
``target`` poses are relative to that origin.
|
||||
"""
|
||||
|
||||
translation: Vec3 = (0.0, 0.0, 0.0)
|
||||
rotation: Quat = _IDENTITY_QUAT
|
||||
|
||||
@classmethod
|
||||
def identity(cls) -> CameraPose:
|
||||
return cls()
|
||||
|
||||
def clone(self) -> CameraPose:
|
||||
return CameraPose(translation=self.translation, rotation=self.rotation)
|
||||
|
||||
def as_matrix(self) -> torch.Tensor:
|
||||
"""Return a 4x4 rigid transform from translation + unit quaternion."""
|
||||
mat = torch.eye(4, dtype=torch.float64)
|
||||
mat[:3, :3] = _quat_to_rotmat(self.rotation)
|
||||
mat[0, 3] = float(self.translation[0])
|
||||
mat[1, 3] = float(self.translation[1])
|
||||
mat[2, 3] = float(self.translation[2])
|
||||
return mat
|
||||
|
||||
@classmethod
|
||||
def from_matrix(cls, matrix: torch.Tensor) -> CameraPose:
|
||||
if matrix.shape != (4, 4):
|
||||
raise ValueError("camera pose matrix must be 4x4")
|
||||
rotation = _rotmat_to_quat(matrix[:3, :3].detach().cpu().double())
|
||||
translation = (
|
||||
float(matrix[0, 3]),
|
||||
float(matrix[1, 3]),
|
||||
float(matrix[2, 3]),
|
||||
)
|
||||
return cls(translation=translation, rotation=rotation)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class QueuedCameraEvent(InteractionEvent):
|
||||
"""Timestamped camera command waiting for the next chunk boundary."""
|
||||
|
||||
# Absolute pose for ``target``; per-frame SE3 delta for ``velocity``.
|
||||
pose: CameraPose = field(default_factory=CameraPose.identity)
|
||||
# Integer frame progress for ``target`` lerps (avoids float reciprocal drift).
|
||||
elapsed_transition_frames: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraSession(InteractionSession):
|
||||
"""Request-local camera timeline under ``state.interaction_sessions['camera']``."""
|
||||
|
||||
current_pose: CameraPose = field(default_factory=CameraPose.identity)
|
||||
pending_events: list[QueuedCameraEvent] = field(default_factory=list)
|
||||
# In-flight command; ``elapsed_transition_chunks`` lives on the event.
|
||||
active_event: QueuedCameraEvent | None = None
|
||||
target_source: CameraPose | None = None
|
||||
# Absolute C2W poses ``[num_latent_frames, 4, 4]`` for the most recent chunk.
|
||||
last_absolute_poses: torch.Tensor | None = None
|
||||
# Since a camera session is not lazily initialized, use this field to mark that a client enqueue is ever received.
|
||||
has_received_input: bool = False
|
||||
|
||||
|
||||
class SE3DeltaCameraHandler(InteractionHandler):
|
||||
"""Generic camera timeline that samples absolute poses per frame.
|
||||
|
||||
* ``mode=target``: lerp toward an absolute pose (relative to session start).
|
||||
* ``mode=velocity``: hold a structural SE3 delta until replaced.
|
||||
|
||||
``received_at`` is resolved on the media grid (``num_media_frames``/``fps``), then mapped onto latent steps.
|
||||
Later integration and ``last_absolute_poses`` aligns with ``num_latent_frames``.
|
||||
"""
|
||||
|
||||
modality: ClassVar[str] = "camera"
|
||||
needs_chunk_media: ClassVar[bool] = True
|
||||
# Camera conditioning must exist every chunk; apply creates the session and
|
||||
# materializes identity/hold poses before any client enqueue.
|
||||
lazy_initialize_session: ClassVar[bool] = False
|
||||
default_transition_chunks: ClassVar[int] = 1
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def from_pipeline(cls, pipeline: object) -> Self:
|
||||
"""It doesn't need anything from the pipeline (e.g., `encode_prompt` method)"""
|
||||
del pipeline
|
||||
return cls()
|
||||
|
||||
@override
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
if not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
|
||||
mode = payload.get("mode", "target")
|
||||
if mode not in ("target", "velocity"):
|
||||
raise ValueError("camera mode must be 'target' or 'velocity'")
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, Mapping):
|
||||
raise ValueError("camera data must be an object")
|
||||
if "actions" in data:
|
||||
raise ValueError(
|
||||
"camera data.actions (WASD keys) is not accepted by the engine; "
|
||||
"send structural translation/rotation instead"
|
||||
)
|
||||
_parse_pose(data)
|
||||
if mode == "target":
|
||||
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
del state
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
self.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
mode = payload.get("mode", "target")
|
||||
camera_mode = cast(InteractionMode, mode)
|
||||
data = cast(Mapping[str, object], payload.get("data"))
|
||||
pose = _parse_pose(data)
|
||||
if camera_mode == "target":
|
||||
duration = self.default_transition_chunks if transition_chunks is None else int(transition_chunks)
|
||||
else:
|
||||
duration = 0
|
||||
|
||||
session = state.interaction_sessions.setdefault("camera", CameraSession())
|
||||
assert isinstance(session, CameraSession)
|
||||
with session.lock:
|
||||
session.has_received_input = True
|
||||
session.pending_events.append(
|
||||
QueuedCameraEvent(
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
mode=camera_mode,
|
||||
transition_chunks=duration,
|
||||
pose=pose,
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance camera state for one chunk and store latent-aligned poses.
|
||||
|
||||
.. note::
|
||||
Hardcoded the following policy of per-frame alignment:
|
||||
|
||||
- Resolve ``received_at`` on the media grid, then map each media index onto a latent step.
|
||||
- Integrate one velocity step per **latent frame**, so holding speed ``0.05`` for a chunk with
|
||||
``num_latent_frames=3`` yields total travel ``0.15``.
|
||||
- ``last_absolute_poses.shape[0] == num_latent_frames``: interpolation is latent-aligned.
|
||||
|
||||
If future models need different scale/alignment policies, can refactor interaction registry
|
||||
to be able to mark a policy.
|
||||
"""
|
||||
del chunk_index
|
||||
if num_media_frames is None or fps is None:
|
||||
raise ValueError("SE3DeltaCameraHandler requires chunk num_media_frames and fps")
|
||||
if num_latent_frames is None:
|
||||
num_latent_frames = num_media_frames
|
||||
if int(num_latent_frames) <= 0:
|
||||
raise ValueError(f"num_latent_frames must be > 0, got {num_latent_frames}")
|
||||
|
||||
session = state.interaction_sessions.setdefault("camera", CameraSession())
|
||||
assert isinstance(session, CameraSession)
|
||||
with session.lock:
|
||||
poses, started, active, completed = self._step_one_chunk(
|
||||
session,
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
boundary_at=boundary_at,
|
||||
)
|
||||
if poses:
|
||||
session.last_absolute_poses = torch.stack([p.as_matrix() for p in poses], dim=0)
|
||||
else:
|
||||
session.last_absolute_poses = torch.zeros((0, 4, 4), dtype=torch.float64)
|
||||
|
||||
return InteractionChunkMetadata(
|
||||
started_event_ids=started,
|
||||
active_event_ids=active,
|
||||
completed_event_ids=completed,
|
||||
)
|
||||
|
||||
def _step_one_chunk(
|
||||
self,
|
||||
session: CameraSession,
|
||||
*,
|
||||
num_media_frames: int,
|
||||
fps: float,
|
||||
num_latent_frames: int,
|
||||
boundary_at: float,
|
||||
) -> tuple[list[CameraPose], list[str], list[str], list[str]]:
|
||||
"""Sample absolute poses for this chunk under target/velocity semantics."""
|
||||
num_media_frames = max(int(num_media_frames), 1)
|
||||
num_latent_frames = max(int(num_latent_frames), 1)
|
||||
pending = list(session.pending_events)
|
||||
session.pending_events.clear()
|
||||
|
||||
by_latent: dict[int, list[QueuedCameraEvent]] = {}
|
||||
for event in pending:
|
||||
media_frame = resolve_event_frame_offset(
|
||||
received_at=event.received_at,
|
||||
previous_boundary_at=session.last_boundary_at,
|
||||
num_frames=num_media_frames,
|
||||
fps=fps,
|
||||
)
|
||||
latent_idx = _media_frame_to_latent(
|
||||
media_frame,
|
||||
num_media_frames=num_media_frames,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
by_latent.setdefault(latent_idx, []).append(event)
|
||||
for events in by_latent.values():
|
||||
events.sort(key=lambda item: item.received_at)
|
||||
|
||||
started: list[str] = []
|
||||
completed: list[str] = []
|
||||
seen_completed: set[str] = set()
|
||||
poses: list[CameraPose] = []
|
||||
|
||||
def _mark_completed(event_id: str) -> None:
|
||||
if event_id not in seen_completed:
|
||||
seen_completed.add(event_id)
|
||||
completed.append(event_id)
|
||||
|
||||
for latent_idx in range(num_latent_frames):
|
||||
for event in by_latent.get(latent_idx, []):
|
||||
started.append(event.event_id)
|
||||
cancelled_id = self._activate_event(session, event)
|
||||
if cancelled_id is not None:
|
||||
_mark_completed(cancelled_id)
|
||||
just_completed = self._step_one_frame(session, num_latent_frames)
|
||||
poses.append(session.current_pose.clone())
|
||||
if just_completed is not None:
|
||||
_mark_completed(just_completed)
|
||||
|
||||
active: list[str] = []
|
||||
if session.active_event is not None:
|
||||
active.append(session.active_event.event_id)
|
||||
|
||||
session.last_boundary_at = boundary_at
|
||||
return poses, started, active, completed
|
||||
|
||||
def _activate_event(self, session: CameraSession, event: QueuedCameraEvent) -> str | None:
|
||||
"""Activate ``event``, returning the event_id cancelled by this replacement if any."""
|
||||
cancelled_id: str | None = None
|
||||
if session.active_event is not None and session.active_event.event_id != event.event_id:
|
||||
cancelled_id = session.active_event.event_id
|
||||
|
||||
event.elapsed_transition_chunks = 0.0
|
||||
event.elapsed_transition_frames = 0
|
||||
session.active_event = event
|
||||
if event.mode == "target":
|
||||
session.target_source = session.current_pose.clone()
|
||||
else:
|
||||
session.target_source = None
|
||||
return cancelled_id
|
||||
|
||||
def _clear_active_target(self, session: CameraSession) -> str:
|
||||
"""Snap to the target pose, clear the active target track, and return its event_id."""
|
||||
assert session.active_event is not None and session.active_event.mode == "target"
|
||||
event = session.active_event
|
||||
event_id = event.event_id
|
||||
session.current_pose = event.pose.clone()
|
||||
session.active_event = None
|
||||
session.target_source = None
|
||||
return event_id
|
||||
|
||||
def _step_one_frame(self, session: CameraSession, total_num_frames_this_chunk: int) -> str | None:
|
||||
"""Advance the active camera command by one frame."""
|
||||
event = session.active_event
|
||||
if event is None:
|
||||
return None
|
||||
|
||||
if event.mode == "target":
|
||||
duration = event.transition_chunks
|
||||
source = session.target_source or session.current_pose
|
||||
target = event.pose
|
||||
if duration <= 0:
|
||||
return self._clear_active_target(session)
|
||||
|
||||
total_frames = max(1, int(round(duration * float(total_num_frames_this_chunk))))
|
||||
event.elapsed_transition_frames += 1
|
||||
event.elapsed_transition_chunks = event.elapsed_transition_frames / float(total_num_frames_this_chunk)
|
||||
alpha = min(1.0, event.elapsed_transition_frames / float(total_frames))
|
||||
session.current_pose = _lerp_pose(source, target, alpha)
|
||||
if event.elapsed_transition_frames >= total_frames:
|
||||
return self._clear_active_target(session)
|
||||
return None
|
||||
|
||||
# Velocity: apply the held structural SE3 delta once per frame.
|
||||
session.current_pose = _compose_pose(session.current_pose, event.pose)
|
||||
return None
|
||||
|
||||
|
||||
def _media_frame_to_latent(
|
||||
media_frame: int,
|
||||
*,
|
||||
num_media_frames: int,
|
||||
num_latent_frames: int,
|
||||
) -> int:
|
||||
"""Map a media-timeline frame index onto the latent control grid."""
|
||||
if num_latent_frames <= 1 or num_media_frames <= 1:
|
||||
return 0
|
||||
return min(int(media_frame) * int(num_latent_frames) // int(num_media_frames), num_latent_frames - 1)
|
||||
|
||||
|
||||
def _as_xyz(value: object, *, name: str) -> Vec3:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 3:
|
||||
raise ValueError(f"camera {name} must be a length-3 list/tuple")
|
||||
if any(math.isnan(v) or math.isinf(v) for v in value):
|
||||
raise ValueError(f"camera {name} must be non-nan and non-inf")
|
||||
return (float(value[0]), float(value[1]), float(value[2]))
|
||||
|
||||
|
||||
def _as_quat(value: object, *, name: str) -> Quat:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)) or len(value) != 4:
|
||||
raise ValueError(f"camera {name} must be a length-4 list/tuple (x, y, z, w)")
|
||||
if any(math.isnan(v) or math.isinf(v) for v in value):
|
||||
raise ValueError(f"camera {name} must be non-nan and non-inf")
|
||||
return _quat_normalize((float(value[0]), float(value[1]), float(value[2]), float(value[3])))
|
||||
|
||||
|
||||
def _parse_pose(data: Mapping[str, object]) -> CameraPose:
|
||||
translation = _as_xyz(data.get("translation", (0.0, 0.0, 0.0)), name="translation")
|
||||
rotation = _as_quat(data.get("rotation", _IDENTITY_QUAT), name="rotation")
|
||||
return CameraPose(translation=translation, rotation=rotation)
|
||||
|
||||
|
||||
def _lerp(a: float, b: float, alpha: float) -> float:
|
||||
return a + (b - a) * alpha
|
||||
|
||||
|
||||
def _lerp_pose(source: CameraPose, target: CameraPose, alpha: float) -> CameraPose:
|
||||
alpha = min(1.0, max(0.0, alpha))
|
||||
return CameraPose(
|
||||
translation=cast(
|
||||
Vec3,
|
||||
tuple(_lerp(s, t, alpha) for s, t in zip(source.translation, target.translation)),
|
||||
),
|
||||
rotation=_quat_nlerp(source.rotation, target.rotation, alpha),
|
||||
)
|
||||
|
||||
|
||||
def _compose_pose(base: CameraPose, delta: CameraPose) -> CameraPose:
|
||||
"""Apply ``delta`` after ``base`` in the session frame (T_new = T_base @ T_delta)."""
|
||||
return CameraPose.from_matrix(base.as_matrix() @ delta.as_matrix())
|
||||
|
||||
|
||||
def _quat_normalize(q: Quat) -> Quat:
|
||||
x, y, z, w = q
|
||||
norm = math.sqrt(x * x + y * y + z * z + w * w)
|
||||
if norm <= 0.0:
|
||||
raise ValueError("camera rotation quaternion must be non-zero")
|
||||
return (x / norm, y / norm, z / norm, w / norm)
|
||||
|
||||
|
||||
def _quat_nlerp(a: Quat, b: Quat, alpha: float) -> Quat:
|
||||
"""Normalized linear interpolation; flips ``b`` when needed for shortest path."""
|
||||
dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]
|
||||
if dot < 0.0:
|
||||
b = (-b[0], -b[1], -b[2], -b[3])
|
||||
return _quat_normalize(
|
||||
(
|
||||
_lerp(a[0], b[0], alpha),
|
||||
_lerp(a[1], b[1], alpha),
|
||||
_lerp(a[2], b[2], alpha),
|
||||
_lerp(a[3], b[3], alpha),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _quat_to_rotmat(q: Quat) -> torch.Tensor:
|
||||
x, y, z, w = q
|
||||
xx, yy, zz = x * x, y * y, z * z
|
||||
xy, xz, yz = x * y, x * z, y * z
|
||||
wx, wy, wz = w * x, w * y, w * z
|
||||
return torch.tensor(
|
||||
[
|
||||
[1.0 - 2.0 * (yy + zz), 2.0 * (xy - wz), 2.0 * (xz + wy)],
|
||||
[2.0 * (xy + wz), 1.0 - 2.0 * (xx + zz), 2.0 * (yz - wx)],
|
||||
[2.0 * (xz - wy), 2.0 * (yz + wx), 1.0 - 2.0 * (xx + yy)],
|
||||
],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
|
||||
def _rotmat_to_quat(rotation: torch.Tensor) -> Quat:
|
||||
"""Convert a 3x3 rotation matrix to a unit quaternion ``(x, y, z, w)``."""
|
||||
m = rotation
|
||||
trace = float(m[0, 0] + m[1, 1] + m[2, 2])
|
||||
if trace > 0.0:
|
||||
s = math.sqrt(trace + 1.0) * 2.0
|
||||
w = 0.25 * s
|
||||
x = (float(m[2, 1]) - float(m[1, 2])) / s
|
||||
y = (float(m[0, 2]) - float(m[2, 0])) / s
|
||||
z = (float(m[1, 0]) - float(m[0, 1])) / s
|
||||
elif float(m[0, 0]) > float(m[1, 1]) and float(m[0, 0]) > float(m[2, 2]):
|
||||
s = math.sqrt(1.0 + float(m[0, 0]) - float(m[1, 1]) - float(m[2, 2])) * 2.0
|
||||
w = (float(m[2, 1]) - float(m[1, 2])) / s
|
||||
x = 0.25 * s
|
||||
y = (float(m[0, 1]) + float(m[1, 0])) / s
|
||||
z = (float(m[0, 2]) + float(m[2, 0])) / s
|
||||
elif float(m[1, 1]) > float(m[2, 2]):
|
||||
s = math.sqrt(1.0 + float(m[1, 1]) - float(m[0, 0]) - float(m[2, 2])) * 2.0
|
||||
w = (float(m[0, 2]) - float(m[2, 0])) / s
|
||||
x = (float(m[0, 1]) + float(m[1, 0])) / s
|
||||
y = 0.25 * s
|
||||
z = (float(m[1, 2]) + float(m[2, 1])) / s
|
||||
else:
|
||||
s = math.sqrt(1.0 + float(m[2, 2]) - float(m[0, 0]) - float(m[1, 1])) * 2.0
|
||||
w = (float(m[1, 0]) - float(m[0, 1])) / s
|
||||
x = (float(m[0, 2]) + float(m[2, 0])) / s
|
||||
y = (float(m[1, 2]) + float(m[2, 1])) / s
|
||||
z = 0.25 * s
|
||||
return _quat_normalize((x, y, z, w))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CameraPose",
|
||||
"CameraSession",
|
||||
"QueuedCameraEvent",
|
||||
"SE3DeltaCameraHandler",
|
||||
]
|
||||
@@ -90,6 +90,7 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
"""
|
||||
|
||||
modality: ClassVar[str] = "prompt"
|
||||
lazy_initialize_session: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -113,16 +114,14 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
)
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
def validate_payload(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
|
||||
prompt = payload.get("prompt")
|
||||
if not isinstance(prompt, str) or not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
@@ -136,6 +135,27 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
|
||||
self.validate_payload(
|
||||
state,
|
||||
event_id=event_id,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
prompt = payload.get("prompt")
|
||||
assert isinstance(prompt, str)
|
||||
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
|
||||
|
||||
target_prompt_embeds, _ = self._encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=None,
|
||||
@@ -164,12 +184,13 @@ class PromptInteractionHandler(InteractionHandler):
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
num_media_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
num_latent_frames: int | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance or start prompt interpolation before the next chunk."""
|
||||
# Prompt lerp is chunk-LWW; media timeline and caller chunk_index unused.
|
||||
del chunk_index, num_frames, fps
|
||||
del chunk_index, num_media_frames, fps, num_latent_frames
|
||||
session = state.interaction_sessions.get("prompt")
|
||||
assert isinstance(session, PromptSession)
|
||||
|
||||
|
||||
@@ -5,12 +5,13 @@ A pipeline only declares the modalities that it supports.
|
||||
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
|
||||
|
||||
Outer key: pipeline architecture name (``od_config.model_class_name``).
|
||||
Inner key: modality name (e.g. ``\"prompt\"``).
|
||||
Inner key: modality name (e.g. ``\"camera\"``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import SE3DeltaCameraHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
|
||||
|
||||
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
@@ -21,4 +22,7 @@ STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
"HeliosPyramidPipeline": {
|
||||
"prompt": PromptInteractionHandler,
|
||||
},
|
||||
"LingBotWorldCausalDMDPipeline": {
|
||||
"camera": SE3DeltaCameraHandler,
|
||||
},
|
||||
}
|
||||
@@ -1,38 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared types for midway interaction handlers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
# Wire payload for one modality track. Handlers validate keys they need.
|
||||
InteractionPayload = Mapping[str, object]
|
||||
|
||||
# Shared command semantics across modalities that support them.
|
||||
# Prompt specialization uses ``"target"`` only.
|
||||
InteractionMode = Literal["target"]
|
||||
# Prompt specialization narrows this to ``"target"`` only.
|
||||
InteractionMode = Literal["target", "velocity"]
|
||||
|
||||
|
||||
def synchronized_monotonic_time(stamp: float | None = None) -> float:
|
||||
"""Return a monotonic timestamp shared across DiT ranks.
|
||||
|
||||
``submit_interaction`` and chunk-boundary apply run as collectives, but
|
||||
``time.monotonic()`` is rank-local. Under USP/SP, inter-rank skew that
|
||||
straddles a ``1/fps`` bucket edge would schedule the same event onto
|
||||
different frames per shard. Rank 0's stamp is broadcast (callers may pass
|
||||
a local sample; non-source ranks' values are ignored).
|
||||
"""
|
||||
if stamp is None:
|
||||
stamp = time.monotonic()
|
||||
if not torch.distributed.is_initialized():
|
||||
return stamp
|
||||
try:
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
|
||||
|
||||
group = get_world_group()
|
||||
except Exception:
|
||||
return stamp
|
||||
if group.world_size <= 1:
|
||||
return stamp
|
||||
return group.broadcast_object(stamp, src=0)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChunkMediaSpec:
|
||||
"""Media extent of one generation chunk, for interaction timeline mapping."""
|
||||
"""Decoded media extent and latent step count for one generation chunk.
|
||||
|
||||
num_frames: int
|
||||
Attributes:
|
||||
num_media_frames: Number of frames in this chunk when decoded into playable media.
|
||||
fps: Frames per second of the media.
|
||||
num_latent_frames: Number of latent frames in this chunk (internal DiT units).
|
||||
"""
|
||||
|
||||
num_media_frames: int
|
||||
fps: float
|
||||
num_latent_frames: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.num_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_frames must be > 0, got {self.num_frames}")
|
||||
if self.num_media_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_media_frames must be > 0, got {self.num_media_frames}")
|
||||
if self.num_latent_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_latent_frames must be > 0, got {self.num_latent_frames}")
|
||||
if self.fps <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.fps must be > 0, got {self.fps}")
|
||||
|
||||
@property
|
||||
def duration_s(self) -> float:
|
||||
return float(self.num_frames) / float(self.fps)
|
||||
return float(self.num_media_frames) / float(self.fps)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
@@ -41,6 +78,7 @@ class InteractionEvent:
|
||||
|
||||
Example:
|
||||
- ``QueuedPromptEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
- ``QueuedCameraEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
|
||||
"""
|
||||
|
||||
event_id: str
|
||||
@@ -60,12 +98,37 @@ class InteractionSession:
|
||||
|
||||
Example:
|
||||
- ``PromptSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
- ``CameraSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.camera``
|
||||
"""
|
||||
|
||||
lock: Lock = field(default_factory=Lock, repr=False)
|
||||
last_boundary_at: float | None = None
|
||||
|
||||
|
||||
def resolve_event_frame_offset(
|
||||
*,
|
||||
received_at: float,
|
||||
previous_boundary_at: float | None,
|
||||
num_frames: int,
|
||||
fps: float,
|
||||
) -> int:
|
||||
"""Map an event arrival time onto ``[0, num_frames - 1]`` for this chunk.
|
||||
|
||||
* Uses ``floor((received_at - previous_boundary_at) * fps)``.
|
||||
* Events without a prior boundary (or with non-positive fps) map to frame 0.
|
||||
* Offsets past the represented media window are clamped to the final frame.
|
||||
"""
|
||||
num_frames = max(int(num_frames), 1)
|
||||
if previous_boundary_at is None or fps <= 0:
|
||||
return 0
|
||||
offset = math.floor((received_at - previous_boundary_at) * float(fps))
|
||||
if offset < 0:
|
||||
return 0
|
||||
if offset >= num_frames:
|
||||
return num_frames - 1
|
||||
return int(offset)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InteractionChunkMetadata:
|
||||
"""Per-modality (or merged) event-id bookkeeping for one chunk boundary."""
|
||||
|
||||
Reference in new issue
Block a user