[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-09-20 16:19:41 +00:00
1 parent 139a47a578
commit e36babd48f
26 files changed
+2071 -200

No files matched your search

@@ -535,19 +535,23 @@ class HeliosPipeline(
"zero_steps": int(extra.get("zero_steps", 1)),
}
)
self.prepare_next_chunk(state)
return state
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Expose this chunk's decoded media extent for interaction timelines."""
num_frames = int(state.extra["window_num_frames"])
num_media_frames = int(state.extra["window_num_frames"])
num_latent_frames = int(state.extra["num_latent_frames_per_chunk"])
fps = state.sampling.fps
if fps is None or float(fps) <= 0:
raise ValueError(
"sampling.fps is required and must be > 0 for interaction modalities that use the "
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
)
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
return ChunkMediaSpec(
num_media_frames=num_media_frames,
fps=float(fps),
num_latent_frames=num_latent_frames,
)
@override
def prepare_next_chunk(self, state: StepRequestState) -> None:
+2 -2
View File
@@ -151,10 +151,10 @@ class SupportsInteractionApply(Protocol):
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Return the media timeline represented by the upcoming/current chunk.
"""Return the media timeline and latent frame count for the upcoming/current chunk.
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
or for backpressure/pacing.
and/or for backpressure/pacing.
"""
...
@@ -25,6 +25,7 @@ from vllm_omni.experimental.ar_diffusion.tick_protocol import (
LINGBOT_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1"
LINGBOT_CAMERA_TRAJECTORY_SCHEMA = "lingbot.camera_trajectory.v1"
LINGBOT_CONTROLLER_TRANSLATION_UNIT = 0.05
_ACTION_ORDER = ("w", "a", "s", "d", "i", "j", "k", "l")
_VALID_ACTIONS = frozenset(_ACTION_ORDER)
_REFERENCE_HEIGHT = 480
@@ -282,7 +283,7 @@ def integrate_lingbot_camera_actions(
"""Convert latent-frame WASD/IJKL controls to cumulative C2W poses.
Calibration and motion constants intentionally match SGLang's LingBot
adapter: movement 0.05, pitch 4 degrees, yaw 6 degrees, pitch clamp 85
adapter: movement 0.05 (LINGBOT_CONTROLLER_TRANSLATION_UNIT), pitch 4 degrees, yaw 6 degrees, pitch clamp 85
degrees, and target-resolution focal lengths of 500 pixels.
"""
@@ -325,13 +326,31 @@ def integrate_lingbot_camera_actions(
right /= right_norm + 1e-6
movement = np.zeros(3, dtype=np.float64)
movement += forward * 0.05 * (("w" in actions) - ("s" in actions))
movement += right * 0.05 * (("d" in actions) - ("a" in actions))
movement += forward * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("w" in actions) - ("s" in actions))
movement += right * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("d" in actions) - ("a" in actions))
# Keep the cumulative controller pose in FP64 between chunks, matching
# SGLang's NumPy integration and avoiding long-session drift.
current_pose = np.eye(4, dtype=np.float64)
current_pose[:3, :3] = new_rotation
current_pose[:3, 3] = translation + movement
poses.append(current_pose)
trajectory = camera_trajectory_from_absolute_pose(torch.from_numpy(np.stack(poses)), width=width, height=height)
return trajectory, current_pitch
def camera_trajectory_from_absolute_pose(
poses: torch.Tensor,
*,
width: int,
height: int,
) -> CameraTrajectory:
if width <= 0 or height <= 0:
raise ValueError("camera action resolution must be positive.")
if poses.ndim != 3 or poses.shape[-2:] != (4, 4):
raise ValueError(f"absolute camera poses must have shape [frames, 4, 4], got {tuple(poses.shape)}.")
if poses.shape[0] == 0:
raise ValueError("absolute camera poses must not be empty.")
# build_plucker_embedding expects intrinsics in the 832x480 reference
# coordinate system. These values become SGLang's [500, 500, W/2, H/2]
# after that function scales them to the requested resolution.
@@ -342,9 +361,7 @@ def integrate_lingbot_camera_actions(
_REFERENCE_HEIGHT / 2,
)
trajectory = CameraTrajectory(
# Keep the cumulative controller pose in FP64 between chunks, matching
# SGLang's NumPy integration and avoiding long-session drift.
poses=torch.from_numpy(np.stack(poses)),
poses=poses,
intrinsics=torch.tensor(reference_intrinsics, dtype=torch.float32).repeat(len(poses), 1),
)
return trajectory, current_pitch
return trajectory
@@ -389,7 +389,11 @@ def _invert_camera_poses(poses: torch.Tensor) -> torch.Tensor:
return inverse
def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
def _prepare_framewise_poses(
raw_poses: torch.Tensor,
*,
translation_scale: float | None = None,
) -> torch.Tensor:
"""Convert raw C2W poses to normalized first-relative framewise deltas."""
# Remove global placement, then convert the sequence to framewise deltas.
@@ -402,12 +406,19 @@ def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
relative_poses[1:],
)
# Only relative direction/magnitude matters to the checkpoint; normalizing
# by the largest step removes the trajectory's arbitrary translation scale.
# Normalizing the control's scale. Different between scripted and realtime input.
translations = relative_poses[:, :3, 3]
max_translation_norm = torch.linalg.vector_norm(translations, dim=-1).max()
if max_translation_norm > 0:
relative_poses[:, :3, 3] = translations / max_translation_norm
if translation_scale is None:
# Scripted action_path input: divide by the largest step so an arbitrary recording scale becomes 1.
scale = torch.linalg.vector_norm(translations, dim=-1).max()
else:
# Realtime interaction input: use the controller translation unit (one WASD step = full speed).
# It cannot see future motion or normalize globally.
if translation_scale <= 0:
raise ValueError(f"translation_scale must be > 0, got {translation_scale}")
scale = translation_scale
if float(scale) > 0:
relative_poses[:, :3, 3] = translations / scale
return relative_poses
@@ -420,6 +431,7 @@ def build_plucker_embedding(
target_width: int,
device: torch.device,
dtype: torch.dtype,
translation_scale: float | None = None,
) -> torch.Tensor:
"""Build checkpoint-ordered ``(ray origin, ray direction)`` channels."""
@@ -438,7 +450,10 @@ def build_plucker_embedding(
# Compute geometry in at least FP32, then cast at the model boundary.
compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
poses = _prepare_framewise_poses(trajectory.poses.to(device=device, dtype=compute_dtype))
poses = _prepare_framewise_poses(
trajectory.poses.to(device=device, dtype=compute_dtype),
translation_scale=translation_scale,
)
intrinsics = trajectory.intrinsics.to(device=device, dtype=compute_dtype).clone()
intrinsics[:, (0, 2)] *= width / _REFERENCE_WIDTH
intrinsics[:, (1, 3)] *= height / _REFERENCE_HEIGHT
@@ -26,16 +26,21 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery, SupportsStepExecution
from vllm_omni.diffusion.models.lingbot_world.actions import (
LINGBOT_CAMERA_ACTION_SCHEMA,
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
LINGBOT_CONTROLLER_TRANSLATION_UNIT,
LingBotCameraActionFrames,
LingBotCameraActionScript,
as_camera_action_frames,
as_camera_action_script,
camera_trajectory_from_absolute_pose,
integrate_lingbot_camera_actions,
parse_lingbot_camera_action_frames,
parse_lingbot_camera_action_script,
@@ -385,25 +390,36 @@ def get_lingbot_world_pre_process_func(
camera_actions = None
else:
action_path = extra_args.get("action_path")
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
if not configured_action_root:
raise ValueError(
"sampling_params.extra_args.action_path requires a trusted action root configured by "
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
if action_path is None or action_path == "":
# Stepwise / streaming sessions may omit a request-scoped camera
# script and instead drive motion via mid-generation camera
# interaction (idle hold until the first camera event).
if getattr(od_config, "step_execution", False) or getattr(od_config, "streaming_output", False):
trajectory = None
camera_actions = None
camera_action_script = None
else:
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
else:
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
if not configured_action_root:
raise ValueError(
"sampling_params.extra_args.action_path requires a trusted action root configured by "
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
)
action_directory = resolve_trusted_action_directory(
action_path,
configured_action_root,
)
action_directory = resolve_trusted_action_directory(
action_path,
configured_action_root,
)
try:
trajectory = load_camera_trajectory(action_directory)
except OSError:
raise ValueError(
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
) from None
camera_actions = None
camera_action_script = None
try:
trajectory = load_camera_trajectory(action_directory)
except OSError:
raise ValueError(
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
) from None
camera_actions = None
camera_action_script = None
updated_prompt = dict(prompt)
updated_multi_modal_data = dict(multi_modal_data)
@@ -460,6 +476,7 @@ class LingBotWorldCausalDMDPipeline(
SupportImageInput,
SupportsComponentDiscovery,
SupportsStepExecution,
InteractionMixin,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
):
@@ -779,8 +796,10 @@ class LingBotWorldCausalDMDPipeline(
camera_sources = (
int(camera_trajectory is not None) + int(camera_actions is not None) + int(camera_action_script is not None)
)
if camera_sources != 1:
raise ValueError("LingBot pre-processing must materialize exactly one camera input.")
if camera_sources > 1:
raise ValueError("LingBot pre-processing must materialize at most one camera input.")
# Zero camera sources are allowed for stepwise camera-interaction sessions;
# request-mode ``forward`` still requires an explicit trajectory or actions.
request_flow_shift = (
extra_args["flow_shift"] if "flow_shift" in extra_args else getattr(self.scheduler.config, "shift", 5.0)
@@ -1047,7 +1066,7 @@ class LingBotWorldCausalDMDPipeline(
*,
dtype: torch.dtype,
previous: CameraTrajectory | None = None,
latent_aligned: bool = False,
latent_aligned: bool = False, # iff using realtime interaction inputs
) -> tuple[torch.Tensor, CameraTrajectory]:
"""Convert raw camera frames to a latent-aligned ray tensor."""
@@ -1083,10 +1102,10 @@ class LingBotWorldCausalDMDPipeline(
),
)
drop_anchor = True
elif inputs.camera_actions is not None:
# The action integrator returns post-action poses. Keep the first
# action visible to framewise-delta conditioning by prepending its
# known pre-action state (identity for a new realtime session).
elif inputs.camera_actions is not None or latent_aligned:
# Action/interaction integrators return post-action poses. Prepend the
# known pre-action state (identity for a new session) so the first
# pose still contributes a framewise delta.
identity = torch.eye(
4,
device=trajectory.poses.device,
@@ -1100,6 +1119,8 @@ class LingBotWorldCausalDMDPipeline(
),
)
drop_anchor = True
# Realtime interaction cannot see future steps; use the controller translation unit to normalize speed.
# Full action_path keeps max-norm.
camera_embedding = build_plucker_embedding(
embedding_trajectory,
height=inputs.height,
@@ -1108,6 +1129,7 @@ class LingBotWorldCausalDMDPipeline(
target_width=inputs.width,
device=self.device,
dtype=dtype,
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT if latent_aligned else None,
)
if drop_anchor:
camera_embedding = camera_embedding[1:]
@@ -1277,6 +1299,11 @@ class LingBotWorldCausalDMDPipeline(
"request mode takes one three-frame camera_actions control per "
"block, or a camera_trajectory for the request."
)
if inputs.camera_trajectory is None and inputs.camera_actions is None:
raise ValueError(
"LingBot request mode requires camera_trajectory or camera_actions; "
"omit them only for stepwise camera-interaction sessions."
)
tick = ARDiffusionTickRequest.from_extra_args(req.sampling_params.extra_args)
if tick is not None and self._ar_diffusion_kv_state is None:
raise RuntimeError("LingBot typed ticks require ARDiffusionEngine session binding.")
@@ -1645,11 +1672,6 @@ class LingBotWorldCausalDMDPipeline(
}
self._ar_text_caches(prompt_embeds, invalidate=False)
self._ar_sessions[state.request_id] = _LingBotARSessionState()
try:
self._prepare_next_chunk(state)
except Exception:
self._release_condition_encoder_state(state.request_id)
raise
return state
def _prepare_next_chunk(self, state: StepRequestState) -> None:
@@ -1668,7 +1690,14 @@ class LingBotWorldCausalDMDPipeline(
dtype=extra["dtype"],
)
previous = extra.get("camera_tail")
if extra.get("camera_action_script") is not None:
if (
(camera_session := state.interaction_sessions.get("camera"))
and isinstance(camera_session, CameraSession)
and camera_session.has_received_input
):
raise ValueError("Cannot use mid-generation camera interaction together with camera_action_script; ")
chunk_actions = extra["camera_action_script"][state.chunk_index]
action_trajectory, camera_pitch = integrate_lingbot_camera_actions(
chunk_actions,
@@ -1678,11 +1707,12 @@ class LingBotWorldCausalDMDPipeline(
initial_pitch=float(extra.get("camera_pitch", 0.0)),
)
extra["camera_pitch"] = camera_pitch
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
chunk_inputs = replace(
inputs,
camera_trajectory=action_trajectory,
camera_actions=chunk_actions,
num_frames=(block_frames - 1) * self.vae_scale_factor_temporal + 1,
num_frames=media_frames,
num_latent_frames=block_frames,
)
camera, camera_tail = self._prepare_camera(
@@ -1691,14 +1721,53 @@ class LingBotWorldCausalDMDPipeline(
previous=previous,
)
elif extra.get("camera_embedding_cache") is not None:
# Same slice request mode takes from its one full-trajectory
# embedding, so both paths condition a block identically.
if (
(camera_session := state.interaction_sessions.get("camera"))
and isinstance(camera_session, CameraSession)
and camera_session.has_received_input
):
raise ValueError("Cannot use mid-generation camera interaction together with camera_embedding_cache; ")
trajectory = extra["camera_trajectory_cache"]
camera = extra["camera_embedding_cache"][:, :, start_frame:stop_frame]
camera_tail = CameraTrajectory(
poses=trajectory.poses[stop_frame - 1 : stop_frame].clone(),
intrinsics=trajectory.intrinsics[stop_frame - 1 : stop_frame].clone(),
)
elif self._interaction_coordinator is not None and self._interaction_coordinator.has_modality("camera"):
# Poses come from the latest boundary apply: ``prepare_encode`` for
# chunk 0, the diffusion runner for later chunks. This method only
# digests them — it does not apply interactions itself.
camera_session = state.interaction_sessions.get("camera")
if not isinstance(camera_session, CameraSession) or camera_session.last_absolute_poses is None:
raise RuntimeError(
"LingBot camera interaction requires apply_interaction_at_chunk_boundary before prepare_next_chunk."
)
absolute_poses = camera_session.last_absolute_poses
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
num_latent_frames = block_frames
if int(absolute_poses.shape[0]) != num_latent_frames:
raise ValueError(
"camera interaction must produce exactly one pose per latent frame; "
f"got {int(absolute_poses.shape[0])}, expected {num_latent_frames}."
)
action_trajectory = camera_trajectory_from_absolute_pose(
absolute_poses,
width=inputs.width,
height=inputs.height,
)
chunk_inputs = replace(
inputs,
camera_trajectory=action_trajectory,
camera_actions=None,
num_frames=media_frames,
num_latent_frames=num_latent_frames,
)
camera, camera_tail = self._prepare_camera(
chunk_inputs,
dtype=extra["dtype"],
previous=previous,
latent_aligned=True,
)
else:
raise RuntimeError("LingBot step execution is missing a camera trajectory or action script.")
if camera.shape[2:] != condition.shape[2:]:
@@ -1842,19 +1911,48 @@ class LingBotWorldCausalDMDPipeline(
session_state.pending_encoder_cache = None
session_state.next_chunk_index += 1
state.chunk_index += 1
finished = state.request_denoise_completed
if not finished:
self._prepare_next_chunk(state)
else:
self._release_condition_encoder_state(state.request_id)
return DiffusionOutput(
output=output,
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None,
chunk_index=completed_chunk_index,
total_chunks=state.total_chunks,
finished=finished,
finished=state.request_denoise_completed,
)
def _chunk_media_frame_count(self, state: StepRequestState, *, block_frames: int) -> int:
"""Pixel frames this chunk will emit when decoded.
Independent (non-streaming) decode restarts causal expansion every block,
so each chunk is ``(block_frames - 1) * temporal + 1`` frames.
Streaming decode expands only the session's opening latent to one frame and every later latent to the full
temporal factor, so chunk 0 matches the independent count and later chunks are ``block_frames * temporal``.
"""
decode_state = self._streaming_decode_states.get(state.request_id)
if decode_state is not None and decode_state.started:
return block_frames * self.vae_scale_factor_temporal
return (block_frames - 1) * self.vae_scale_factor_temporal + 1
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Expose this chunk's decoded media extent and latent step count."""
num_latent_frames = int(state.extra.get("block_frames") or self.transformer.config.num_frames_per_block)
num_media_frames = self._chunk_media_frame_count(state, block_frames=num_latent_frames)
fps = state.sampling.fps
if fps is None or float(fps) <= 0:
# Media-frame camera controls are not wall-clock paced; a unit fps keeps
# resolve_event_frame_offset well-defined when the client omits sampling.fps.
fps = float(num_media_frames)
return ChunkMediaSpec(
num_media_frames=num_media_frames,
fps=fps,
num_latent_frames=num_latent_frames,
)
def prepare_next_chunk(self, state: StepRequestState) -> None:
"""Prepare the next AR block after chunk-boundary interaction apply."""
if state.request_denoise_completed:
return
self._prepare_next_chunk(state)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return cast(set[str], loader.load_weights(weights))