[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>

tests/diffusion/models/lingbot_world/test_lingbot_world_actions.py

@@ -12,6 +12,7 @@ from vllm_omni.diffusion.models.lingbot_world.actions import (
     LINGBOT_CAMERA_ACTION_SCHEMA,
     LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
     LingBotCameraControlReducer,
+    camera_trajectory_from_absolute_pose,
     integrate_lingbot_camera_actions,
     parse_lingbot_camera_action_frames,
     parse_lingbot_camera_action_script,
@@ -212,6 +213,11 @@ def test_action_integrator_matches_lingbot_motion_and_calibration() -> None:
         torch.tensor([500.0, 500.0, 208.0, 120.0]),
     )
 
+    with pytest.raises(ValueError, match="resolution must be positive"):
+        camera_trajectory_from_absolute_pose(trajectory.poses[:1], width=0, height=480)
+    with pytest.raises(ValueError, match=r"\[frames, 4, 4\]"):
+        camera_trajectory_from_absolute_pose(torch.zeros(4, 4), width=832, height=480)
+
 
 @pytest.mark.parametrize(
     ("data", "message"),

tests/diffusion/models/lingbot_world/test_lingbot_world_camera.py

@@ -422,7 +422,17 @@ def test_build_plucker_embedding_uses_framewise_deltas_for_rotations_and_transla
     torch.testing.assert_close(result[2, 3:, 0, 1], expected_direction)
 
 
-def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm() -> None:
+@pytest.mark.parametrize(
+    ("translation_scale", "expected_origins"),
+    [
+        (None, [[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]]),
+        (0.05, [[0.0, 0.0, 0.0], [60.0, 80.0, 0.0], [120.0, 160.0, 0.0]]),
+    ],
+)
+def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm(
+    translation_scale: float | None,
+    expected_origins: list[list[float]],
+) -> None:
     poses = torch.eye(4).repeat(3, 1, 1)
     poses[:, :3, 3] = torch.tensor(
         [
@@ -440,10 +450,37 @@ def test_build_plucker_embedding_normalizes_framewise_translation_by_max_norm()
         target_width=1,
         device=torch.device("cpu"),
         dtype=torch.float32,
+        translation_scale=translation_scale,
     )
 
-    expected_origins = torch.tensor([[0.0, 0.0, 0.0], [0.3, 0.4, 0.0], [0.6, 0.8, 0.0]])
-    torch.testing.assert_close(result[:, :3, 0, 0], expected_origins)
+    torch.testing.assert_close(result[:, :3, 0, 0], torch.tensor(expected_origins))
+
+    from vllm_omni.diffusion.models.lingbot_world.actions import LINGBOT_CONTROLLER_TRANSLATION_UNIT
+
+    if translation_scale != LINGBOT_CONTROLLER_TRANSLATION_UNIT:
+        return
+    # Fixed controller unit keeps relative speed; max-norm would collapse both to 1.
+    slow = torch.eye(4).repeat(2, 1, 1)
+    slow[1, 2, 3] = 0.005
+    fast = torch.eye(4).repeat(2, 1, 1)
+    fast[1, 2, 3] = 0.5
+    kwargs = dict(
+        height=1,
+        width=1,
+        target_height=1,
+        target_width=1,
+        device=torch.device("cpu"),
+        dtype=torch.float32,
+        translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT,
+    )
+    slow_origin = build_plucker_embedding(_trajectory(slow), **kwargs)[1, :3, 0, 0]
+    fast_origin = build_plucker_embedding(_trajectory(fast), **kwargs)[1, :3, 0, 0]
+    torch.testing.assert_close(slow_origin, torch.tensor([0.0, 0.0, 0.1]))
+    torch.testing.assert_close(fast_origin, torch.tensor([0.0, 0.0, 10.0]))
+    wasd = torch.eye(4).repeat(2, 1, 1)
+    wasd[1, 2, 3] = LINGBOT_CONTROLLER_TRANSLATION_UNIT
+    wasd_origin = build_plucker_embedding(_trajectory(wasd), **kwargs)[1, :3, 0, 0]
+    torch.testing.assert_close(wasd_origin, torch.tensor([0.0, 0.0, 1.0]))
 
 
 def test_build_plucker_embedding_interpolates_raw_poses_before_framewise_conversion() -> None:

tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py

@@ -18,6 +18,7 @@ from torch import nn
 import vllm_omni.diffusion.models.lingbot_world.dmd_block as lingbot_dmd_block
 import vllm_omni.diffusion.models.lingbot_world.pipeline as lingbot_pipeline
 from tests.diffusion.models.wan2_2.conftest import noop_progress_bar
+from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
 from vllm_omni.diffusion.models.interface import SupportsStepExecution, supports_step_execution
 from vllm_omni.diffusion.models.lingbot_world.actions import (
     integrate_lingbot_camera_actions,
@@ -411,8 +412,10 @@ def _stub_pipeline_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
             intrinsics=value.intrinsics[:num_frames],
         )
 
-    def build_plucker_embedding(value, *, height, width, target_height, target_width, device, dtype):
-        del target_height, target_width
+    def build_plucker_embedding(
+        value, *, height, width, target_height, target_width, device, dtype, translation_scale=None
+    ):
+        del target_height, target_width, translation_scale
         frames = value.poses.shape[0]
         data = torch.arange(frames * 6 * height * width, device=device, dtype=torch.float32)
         return data.reshape(frames, 6, height, width).to(dtype=dtype)
@@ -1783,6 +1786,7 @@ def test_first_typed_yaw_action_uses_pre_action_identity_anchor(
         target_width=16,
         device=torch.device("cpu"),
         dtype=torch.float32,
+        translation_scale=None,  # None for camera_actions / action_script input (using max-norm)
     )[1:]
 
     assert not torch.equal(action_embedding, neutral_embedding)
@@ -2015,6 +2019,7 @@ def test_stepwise_failed_chunk_releases_pending_encoder_on_close(monkeypatch: py
     transformer.raise_on_call = 1
     with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
         pipeline.prepare_encode(state)
+        pipeline.prepare_next_chunk(state)
         session = pipeline._ar_sessions[state.request_id]
         assert session.encoder_cache is None and session.pending_encoder_cache is not None
         with pytest.raises(RuntimeError, match="forced transformer failure"):
@@ -2037,6 +2042,7 @@ def test_stateful_condition_rejects_tiled_encoder(monkeypatch: pytest.MonkeyPatc
             state = _stepwise_state()
             with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
                 pipeline.prepare_encode(state)
+                pipeline.prepare_next_chunk(state)
     assert pipeline.vae.encoder.inputs == []
     assert all(s.encoder_cache is None and s.pending_encoder_cache is None for s in pipeline._ar_sessions.values())
 
@@ -2168,11 +2174,14 @@ def _stepwise_state(
 def _run_stepwise(pipeline, state):
     outputs = []
     pipeline.prepare_encode(state)
+    pipeline.prepare_next_chunk(state)
     while not state.request_denoise_completed:
         noise = pipeline.denoise_step(None, states=[state])
         pipeline.step_scheduler(state, noise)
         if state.chunk_denoise_completed:
             outputs.append(pipeline.post_decode(state))
+            if not state.request_denoise_completed:
+                pipeline.prepare_next_chunk(state)
     return outputs
 
 
@@ -2186,6 +2195,7 @@ def _stepwise_chunks(pipeline, state, ar_state):
     """
     with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
         pipeline.prepare_encode(state)
+        pipeline.prepare_next_chunk(state)
     while not state.request_denoise_completed:
         output = None
         with pipeline.bind_ar_diffusion_state(state.request_id, ar_state):
@@ -2193,6 +2203,8 @@ def _stepwise_chunks(pipeline, state, ar_state):
             pipeline.step_scheduler(state, noise)
             if state.chunk_denoise_completed:
                 output = pipeline.post_decode(state)
+                if not state.request_denoise_completed:
+                    pipeline.prepare_next_chunk(state)
         if output is not None:
             yield output
 
@@ -2463,6 +2475,64 @@ def test_stepwise_chunks_continue_one_sessions_temporal_decode(monkeypatch) -> N
     assert pipeline.vae._feat_map == ["module-owned"]
 
 
+def test_prepare_next_chunk_rejects_live_camera_with_scripted_paths() -> None:
+    """Scripted/cached camera must not silently ignore mid-generation camera events."""
+    module = _load_pipeline_module()
+    pipeline = _pipeline(module, transformer=_RecordingTransformer())
+    pipeline._ar_height = 16
+    pipeline._ar_width = 16
+
+    # Scenario 1, input pre-scripted camera trajectory via action_script
+    state = _stepwise_state(num_frames=21)
+    with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
+        pipeline.prepare_encode(state)
+        assert state.extra.get("camera_action_script") is not None
+        state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
+        with pytest.raises(ValueError, match="camera_action_script"):
+            pipeline.prepare_next_chunk(state)
+
+    # Scenario 2, input pre-scripted camera trajectory via embedding cache
+    poses = torch.eye(4).repeat(21, 1, 1)
+    trajectory = _CameraTrajectory(
+        poses=poses,
+        intrinsics=torch.tensor([[100.0, 100.0, 8.0, 8.0]]).repeat(21, 1),
+    )
+    state = _stepwise_state(num_frames=21)
+    state.sampling.extra_args["_lingbot_camera_trajectory"] = trajectory
+    state.sampling.extra_args.pop("_lingbot_camera_action_script")
+    with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):  # pyright: ignore[reportArgumentType]
+        pipeline.prepare_encode(state)
+        assert state.extra.get("camera_embedding_cache") is not None
+        state.interaction_sessions["camera"] = CameraSession(has_received_input=True)
+        with pytest.raises(ValueError, match="camera_embedding_cache"):
+            pipeline.prepare_next_chunk(state)
+
+
+def test_peek_chunk_media_matches_streaming_decoder_frame_counts(monkeypatch) -> None:
+    """Camera timelines must track the streaming decoder's 9-then-12 media counts."""
+    module = _load_pipeline_module()
+    _capture_video_processor(monkeypatch)
+    pipeline = _streaming_pipeline(module)
+    state = _stepwise_state(num_frames=21)
+    state.sampling.output_type = "np"
+    state.sampling.fps = 16.0
+    fake = _FakeARState(state.request_id)
+
+    with pipeline.bind_ar_diffusion_state(state.request_id, fake):
+        pipeline.prepare_encode(state)
+        first = pipeline.peek_chunk_media(state)
+        assert (first.num_media_frames, first.fps, first.num_latent_frames) == (9, 16.0, 3)
+
+        pipeline.prepare_next_chunk(state)
+        while not state.chunk_denoise_completed:
+            noise = pipeline.denoise_step(None, states=[state])
+            pipeline.step_scheduler(state, noise)
+        pipeline.post_decode(state)
+
+        second = pipeline.peek_chunk_media(state)
+        assert (second.num_media_frames, second.fps, second.num_latent_frames) == (12, 16.0, 3)
+
+
 def test_streaming_decode_state_is_owned_by_the_session(monkeypatch) -> None:
     """Decoder state is keyed by request id and released with the AR session."""
     module = _load_pipeline_module()
@@ -2599,6 +2669,7 @@ def test_streaming_decode_receives_rescaled_latents(monkeypatch) -> None:
 
     with pipeline.bind_ar_diffusion_state(state.request_id, _FakeARState(state.request_id)):
         pipeline.prepare_encode(state)
+        pipeline.prepare_next_chunk(state)
         while not state.chunk_denoise_completed:
             pipeline.step_scheduler(state, pipeline.denoise_step(None, states=[state]))
         model_space = state.latents.clone()

tests/diffusion/streaming_interactive_generation/test_camera_interaction.py

@@ -0,0 +1,434 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Tests for camera interaction handlers and coordinator registration."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
+from vllm_omni.diffusion.interaction.mixin import InteractionMixin
+from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession, SE3DeltaCameraHandler
+from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptSession
+from vllm_omni.diffusion.interaction.registry import STRUCTURED_HANDLER_REGISTRY
+from vllm_omni.diffusion.interaction.types import ChunkMediaSpec, resolve_event_frame_offset
+from vllm_omni.diffusion.worker.utils import StepRequestState
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+
+_FORWARD_VELOCITY = {
+    "mode": "velocity",
+    "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+}
+
+
+class _FakePromptPipeline(InteractionMixin):
+    """Minimal pipeline stub with prompt-update support."""
+
+    def __init__(self) -> None:
+        self.device = torch.device("cpu")
+        self.transformer = SimpleNamespace(dtype=torch.float32)
+        self.encode_prompt = MagicMock(return_value=(torch.full((1, 4, 2), 2.0), None))
+        self._interaction_coordinator = None
+
+    def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
+        del state
+        return ChunkMediaSpec(num_media_frames=8, fps=16.0, num_latent_frames=8)
+
+
+class _FakeLingBotPipeline(InteractionMixin):
+    """Minimal pipeline stub for LingBot camera registration."""
+
+    def __init__(self) -> None:
+        self._interaction_coordinator = None
+
+    def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
+        del state
+        return ChunkMediaSpec(num_media_frames=3, fps=3.0, num_latent_frames=3)
+
+
+def _make_state(*, request_id: str = "req-1") -> StepRequestState:
+    state = StepRequestState(
+        request_id=request_id,
+        sampling=SimpleNamespace(num_outputs_per_prompt=1, max_sequence_length=226),  # pyright: ignore[reportArgumentType]
+        prompt="hello",
+    )
+    state.prompt_embeds = torch.zeros(1, 4, 2)
+    state.extra = {}
+    return state
+
+
+def _boundary_at(previous_boundary_at: float | None, num_frames: int, fps: float) -> float:
+    if previous_boundary_at is None:
+        return 0.0
+    if fps <= 0:
+        return previous_boundary_at
+    return previous_boundary_at + num_frames / fps
+
+
+class TestCoordinatorResolution:
+    def test_helios_has_prompt_but_no_camera(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        assert coordinator.has_modality("prompt")
+        assert not coordinator.has_modality("camera")
+        assert "camera" not in STRUCTURED_HANDLER_REGISTRY["HeliosPipeline"]
+
+    def test_lingbot_registers_se3_camera_handler(self) -> None:
+        pipeline = _FakeLingBotPipeline()
+        od_config = SimpleNamespace(model_class_name="LingBotWorldCausalDMDPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        assert coordinator.has_modality("camera")
+        handler = coordinator.get_handler("camera")
+        assert isinstance(handler, SE3DeltaCameraHandler)
+        assert handler.lazy_initialize_session is False
+
+    def test_non_lazy_camera_apply_creates_session_without_enqueue(self) -> None:
+        """Camera apply must run before any enqueue to materialize identity poses."""
+        pipeline = _FakeLingBotPipeline()
+        od_config = SimpleNamespace(model_class_name="LingBotWorldCausalDMDPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+
+        assert "camera" not in state.interaction_sessions
+        meta = coordinator.maybe_prepare_initial_session(state, pipeline)
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (3, 4, 4)
+        assert meta.started_event_ids == []
+        assert meta.active_event_ids == []
+
+    def test_lazy_prompt_apply_skips_without_enqueue(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+
+        meta = coordinator.apply_at_chunk_boundary(state, boundary_at=0.0, chunk_index=0)
+        assert "prompt" not in state.interaction_sessions
+        assert meta.started_event_ids == []
+        assert meta.active_event_ids == []
+        assert meta.completed_event_ids == []
+
+        initial = coordinator.maybe_prepare_initial_session(state, pipeline)
+        assert "prompt" not in state.interaction_sessions
+        assert initial.started_event_ids == []
+
+    def test_unsupported_modality_includes_model_context(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        with pytest.raises(ValueError, match="HeliosPipeline"):
+            coordinator.enqueue(
+                _make_state(),
+                modality="camera",
+                event_id="cam-1",
+                received_at=0.0,
+                payload=_FORWARD_VELOCITY,
+                transition_chunks=None,
+            )
+
+    def test_enqueue_parts_rejects_unsupported_modality_atomically(self) -> None:
+        """Unsupported tracks in a composite event must not mutate any queues."""
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+        coordinator.enqueue(
+            state,
+            modality="prompt",
+            event_id="prior",
+            received_at=0.0,
+            payload={"prompt": "already-queued"},
+            transition_chunks=1,
+        )
+        prior_session = state.interaction_sessions["prompt"]
+        assert isinstance(prior_session, PromptSession)
+        prior = prior_session.pending_event
+        assert prior is not None
+        pipeline.encode_prompt.reset_mock()
+
+        with pytest.raises(ValueError, match="camera"):
+            coordinator.enqueue_parts(
+                state,
+                parts=[
+                    ("prompt", {"prompt": "should-not-replace"}),
+                    ("camera", _FORWARD_VELOCITY),
+                ],
+                event_id="composite-bad",
+                received_at=1.0,
+                transition_chunks=2,
+            )
+
+        pipeline.encode_prompt.assert_not_called()
+        pending = prior_session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
+        # Invalid camera payload must also fail before any track is queued.
+        dual = InteractionCoordinator(
+            {
+                "prompt": coordinator.get_handler("prompt"),
+                "camera": SE3DeltaCameraHandler(),
+            },
+            model_class_name="HeliosPipeline",
+        )
+        pipeline.encode_prompt.reset_mock()
+        with pytest.raises(ValueError, match="actions"):
+            dual.enqueue_parts(
+                state,
+                parts=[
+                    ("prompt", {"prompt": "should-not-replace"}),
+                    ("camera", {"mode": "velocity", "data": {"actions": ["w"]}}),
+                ],
+                event_id="composite-bad-payload",
+                received_at=2.0,
+                transition_chunks=2,
+            )
+        pipeline.encode_prompt.assert_not_called()
+        pending = prior_session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
+
+class TestCameraHandlers:
+    def test_rejects_wasd_actions_payload(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        with pytest.raises(ValueError, match="actions"):
+            handler.enqueue(
+                state,
+                event_id="cam-wasd",
+                received_at=0.0,
+                payload={"mode": "velocity", "data": {"actions": ["w"]}},
+                transition_chunks=None,
+            )
+
+    def test_target_transition_progress_and_se3_projection(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="cam-1",
+            received_at=0.0,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 3.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=2,
+        )
+
+        meta = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=4,
+            fps=16.0,
+            boundary_at=0.25,
+        )
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.last_absolute_poses is not None
+        assert meta is not None
+        assert meta.started_event_ids == ["cam-1"]
+        assert meta.active_event_ids == ["cam-1"]
+        assert meta.completed_event_ids == []
+        assert session.last_absolute_poses.shape == (4, 4, 4)
+
+        meta2 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=1,
+            num_media_frames=4,
+            fps=16.0,
+            boundary_at=0.5,
+        )
+        assert meta2 is not None
+        assert meta2.started_event_ids == []
+        assert meta2.active_event_ids == []
+        assert meta2.completed_event_ids == ["cam-1"]
+        assert session.active_event is None
+        assert session.current_pose.translation[2] == pytest.approx(3.0)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (4, 4, 4)
+
+    def test_single_chunk_target_completes_once(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        # Include a non-power-of-two frame count (7) so float reciprocal drift
+        # would previously miss the final snap / completion ack.
+        for num_frames in (8, 7):
+            state = _make_state()
+            handler.enqueue(
+                state,
+                event_id="cam-fast",
+                received_at=0.0,
+                payload={
+                    "mode": "target",
+                    "data": {"translation": [1.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                },
+                transition_chunks=1,
+            )
+            meta = handler.apply_at_chunk_boundary(
+                state,
+                chunk_index=0,
+                num_media_frames=num_frames,
+                fps=16.0,
+                boundary_at=0.5,
+            )
+            assert meta is not None
+            assert meta.started_event_ids == ["cam-fast"]
+            assert meta.active_event_ids == []
+            assert meta.completed_event_ids == ["cam-fast"]
+
+            meta2 = handler.apply_at_chunk_boundary(
+                state,
+                chunk_index=1,
+                num_media_frames=num_frames,
+                fps=16.0,
+                boundary_at=1.0,
+            )
+            assert meta2 is not None
+            assert meta2.started_event_ids == []
+            assert meta2.active_event_ids == []
+            assert meta2.completed_event_ids == []
+            session = state.interaction_sessions["camera"]
+            assert isinstance(session, CameraSession)
+            assert session.current_pose.translation[0] == pytest.approx(1.0)
+
+    def test_velocity_composes_structural_se3_delta(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="vel-1",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+
+        meta0 = handler.apply_at_chunk_boundary(state, chunk_index=0, num_media_frames=3, fps=16.0, boundary_at=0.0)
+        assert meta0 is not None
+        assert meta0.started_event_ids == ["vel-1"]
+        assert meta0.active_event_ids == ["vel-1"]
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.current_pose.translation[2] == pytest.approx(0.15)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (3, 4, 4)
+        torch.testing.assert_close(
+            session.last_absolute_poses[0, :3, 3],
+            torch.tensor([0.0, 0.0, 0.05], dtype=torch.float64),
+            atol=1e-6,
+            rtol=0,
+        )
+        torch.testing.assert_close(
+            session.last_absolute_poses[1, :3, 3],
+            torch.tensor([0.0, 0.0, 0.10], dtype=torch.float64),
+            atol=1e-6,
+            rtol=0,
+        )
+
+        meta1 = handler.apply_at_chunk_boundary(state, chunk_index=1, num_media_frames=2, fps=16.0, boundary_at=0.125)
+        assert meta1 is not None
+        assert meta1.active_event_ids == ["vel-1"]
+        assert session.current_pose.translation[2] == pytest.approx(0.25)
+
+    def test_velocity_completed_when_replaced_by_target(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="vel-1",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+        meta0 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=2,
+            fps=16.0,
+            boundary_at=0.0,
+        )
+        assert meta0 is not None
+        assert meta0.active_event_ids == ["vel-1"]
+
+        handler.enqueue(
+            state,
+            event_id="tgt-1",
+            received_at=0.1,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=0,
+        )
+        meta1 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=1,
+            num_media_frames=2,
+            fps=16.0,
+            boundary_at=0.125,
+        )
+        assert meta1 is not None
+        assert meta1.started_event_ids == ["tgt-1"]
+        assert "vel-1" in meta1.completed_event_ids
+        assert meta1.active_event_ids == []
+
+    def test_same_chunk_velocity_then_target_by_frame_offset(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        fps = 10.0
+        boundary = 0.0
+        handler.enqueue(
+            state,
+            event_id="vel",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+        handler.enqueue(
+            state,
+            event_id="tgt",
+            received_at=0.5,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=0,
+        )
+        state.interaction_sessions["camera"].last_boundary_at = boundary
+        meta = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=10,
+            fps=fps,
+            boundary_at=_boundary_at(boundary, 10, fps),
+        )
+        assert meta is not None
+        assert meta.started_event_ids == ["vel", "tgt"]
+        assert meta.completed_event_ids == ["vel", "tgt"]
+        assert meta.active_event_ids == []
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.active_event is None
+        assert session.current_pose.translation[2] == pytest.approx(0.0)
+
+
+class TestResolveEventFrameOffset:
+    def test_clamps_and_floors(self) -> None:
+        assert resolve_event_frame_offset(received_at=0.0, previous_boundary_at=None, num_frames=8, fps=16.0) == 0
+        assert resolve_event_frame_offset(received_at=0.1, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 1
+        assert resolve_event_frame_offset(received_at=10.0, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 7
+        assert resolve_event_frame_offset(received_at=-1.0, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 0

tests/diffusion/streaming_interactive_generation/test_prompt_interaction.py

@@ -78,7 +78,7 @@ def _make_diffusion_request_state(*, request_id: str = "req-1", fps: int | None
         prompt="hello",
     )
     state.prompt_embeds = torch.zeros(1, 4, 2)
-    state.extra = {"window_num_frames": 8}
+    state.extra = {"window_num_frames": 8, "num_latent_frames_per_chunk": 3}
     return state
 
 
@@ -109,7 +109,12 @@ def _make_diffusion_model_runner(
     # instances; unit tests of submit_interaction stub this. Stepwise path tests
     # rebind to the real method.
     runner._supports_step_mode = lambda: True
-    runner._interaction_coordinator = None
+    # Mirror load_model: coordinator is always present once the runner is "loaded".
+    # Build now so handlers capture the pipeline's current encode_prompt (tests may
+    # rebind the mock before calling this helper).
+    runner._interaction_coordinator = InteractionCoordinator.build(pipeline, runner.od_config)
+    if hasattr(pipeline, "_interaction_coordinator"):
+        pipeline._interaction_coordinator = runner._interaction_coordinator
     return runner
 
 
@@ -155,6 +160,7 @@ class TestPromptUpdateExecution:
             supports_step_execution = True
 
         runner = _make_diffusion_model_runner(pipeline=_UnsupportedPipeline())
+        runner.state_cache["req-1"] = _make_diffusion_request_state()
         with pytest.raises(ValueError, match="not supported"):
             runner.submit_interaction("req-1", _prompt_interaction())
 
@@ -168,25 +174,98 @@ class TestPromptUpdateExecution:
         "interaction",
         [
             {},
-            {"multi_modal_data": {"camera": {"type": "pose"}}},
-            {"event": {"prompt": "updated", "multi_modal_data": {"camera": {"type": "pose"}}}},
-            {"event": {"prompt": "updated", "multi_modal_data": {}}},
-            {"event": {"prompt": "updated", "multi_modal_data": None}},
-            {"event": {"prompt": "updated", "multi_modal_data": "bad"}},
+            {"event_id": "bad", "event": {"multi_modal_data": {}}},
+            {"event_id": "bad", "event": {"multi_modal_data": None}},
+            {"event_id": "bad", "event": {"prompt": "updated", "multi_modal_data": "bad"}},
+            {
+                "event_id": "cam-only",
+                "event": {
+                    "multi_modal_data": {
+                        "camera": {
+                            "mode": "velocity",
+                            "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                        },
+                    }
+                },
+            },
+            {
+                "event_id": "cam-and-prompt",
+                "event": {
+                    "prompt": "updated",
+                    "multi_modal_data": {
+                        "camera": {
+                            "mode": "velocity",
+                            "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                        },
+                    },
+                },
+            },
         ],
     )
-    def test_runner_interaction_rejects_structural_payloads_until_implemented(
+    def test_runner_rejects_unsupported_interactions(
         self,
         pipeline: HeliosPipeline,
         interaction: dict[str, Any],
     ) -> None:
-        """Unsupported interaction dict shapes are preserved to and rejected by the runner."""
+        """Helios has no camera handler; malformed / unsupported payloads are rejected."""
         runner = _make_diffusion_model_runner(pipeline=pipeline)
-        runner.state_cache["req-1"] = _make_diffusion_request_state()
+        state = _make_diffusion_request_state()
+        runner.state_cache["req-1"] = state
 
-        with pytest.raises(NotImplementedError, match="Only text-only prompt update interactions"):
+        with pytest.raises((ValueError, KeyError, AttributeError, TypeError)):
             runner.submit_interaction("req-1", cast(Any, interaction))
 
+        # Rejected composite/malformed events must leave interaction queues unchanged.
+        pipeline.encode_prompt.assert_not_called()  # pyright: ignore[reportAttributeAccessIssue]
+        assert "prompt" not in state.interaction_sessions
+        assert "camera" not in state.interaction_sessions
+
+    def test_rejected_composite_leaves_existing_prompt_queue_unchanged(
+        self,
+        pipeline: HeliosPipeline,
+    ) -> None:
+        """Helios must not apply a composite prompt when camera modality is unsupported."""
+        runner = _make_diffusion_model_runner(pipeline=pipeline)
+        state = _make_diffusion_request_state()
+        runner.state_cache["req-1"] = state
+        runner.submit_interaction("req-1", _prompt_interaction("already-queued", event_id="prior"))
+        session = state.interaction_sessions["prompt"]
+        assert isinstance(session, PromptSession)
+        prior = session.pending_event
+        assert prior is not None
+        assert prior.event_id == "prior"
+        pipeline.encode_prompt.reset_mock()  # pyright: ignore[reportAttributeAccessIssue]
+
+        with pytest.raises(ValueError, match="camera"):
+            runner.submit_interaction(
+                "req-1",
+                cast(
+                    Any,
+                    {
+                        "event_id": "cam-and-prompt",
+                        "event": {
+                            "prompt": "should-not-replace",
+                            "multi_modal_data": {
+                                "camera": {
+                                    "mode": "velocity",
+                                    "data": {
+                                        "translation": [0.0, 0.0, 0.05],
+                                        "rotation": [0.0, 0.0, 0.0, 1.0],
+                                    },
+                                },
+                            },
+                        },
+                    },
+                ),
+            )
+
+        pipeline.encode_prompt.assert_not_called()  # pyright: ignore[reportAttributeAccessIssue]
+        pending = session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
     @pytest.mark.parametrize("model_class_name", ["HeliosPipeline", "HeliosPyramidPipeline"])
     def test_coordinator_registers_prompt_for_helios_aliases(
         self,
@@ -367,7 +446,11 @@ class TestPromptUpdateExecution:
                         f"that use the chunk media timeline, got {fps!r} "
                         f"(request_id={state.request_id!r})"
                     )
-                return ChunkMediaSpec(num_frames=int(state.extra["window_num_frames"]), fps=float(fps))
+                return ChunkMediaSpec(
+                    num_media_frames=int(state.extra["window_num_frames"]),
+                    fps=float(fps),
+                    num_latent_frames=int(state.extra["window_num_frames"]),
+                )
 
             def prepare_encode(self, state: StepRequestState) -> StepRequestState:
                 state.prompt_embeds = torch.zeros(1, 4, 2)

tests/diffusion/test_diffusion_step_pipeline.py

@@ -780,6 +780,65 @@ class TestRunner:
         assert runner.pipeline.denoise_calls == 0
         assert runner.state_cache == {}
 
+    def test_peer_prepare_encode_failure_skips_interaction_session_initialization(self, monkeypatch):
+        """One rank's failure in ``_prepare_batch_inputs -> prepare_encode`` must be handled before all ranks initialize interaction sessions.
+
+        ``maybe_prepare_initial_session`` calls ``synchronized_monotonic_time()``
+        (a broadcast). If this rank's ``prepare_encode`` succeeded while a peer
+        failed, entering that broadcast while the peer enters the failure
+        all-reduce would hang.
+        """
+        from unittest.mock import MagicMock
+
+        from vllm_omni.diffusion.interaction.types import ChunkMediaSpec, InteractionChunkMetadata
+
+        class _InteractionStepPipeline(_StepPipeline):
+            def __init__(self):
+                super().__init__()
+                self.prepare_next_chunk_calls = 0
+
+            def peek_chunk_media(self, state):
+                del state
+                return ChunkMediaSpec(num_media_frames=8, fps=16.0, num_latent_frames=8)
+
+            def apply_interaction_at_chunk_boundary(self, state):
+                del state
+
+            def prepare_next_chunk(self, state):
+                del state
+                self.prepare_next_chunk_calls += 1
+
+        runner = _make_runner()
+        pipeline = _InteractionStepPipeline()
+        runner.pipeline = pipeline
+        coordinator = MagicMock()
+        coordinator.maybe_prepare_initial_session.return_value = InteractionChunkMetadata(
+            started_event_ids=[],
+            active_event_ids=[],
+            completed_event_ids=[],
+        )
+        runner._interaction_coordinator = coordinator
+        monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
+        # Peer failed during prepare_encode; this rank's local encode succeeds.
+        monkeypatch.setattr(
+            model_runner_module,
+            "_dit_any_rank_failed",
+            lambda local_failed: True,
+        )
+
+        req = _make_step_request(num_inference_steps=1)
+        result = DiffusionModelRunner.execute_stepwise(runner, _make_scheduler_output(req))
+
+        output = result.get_request_output("req-1")
+        assert output.finished is True
+        assert output.result is not None
+        assert "another DiT rank" in (output.result.error or "")
+        assert pipeline.prepare_calls == 1
+        coordinator.maybe_prepare_initial_session.assert_not_called()
+        assert pipeline.prepare_next_chunk_calls == 0
+        assert pipeline.denoise_calls == 0
+        assert runner.state_cache == {}
+
     def test_receives_kv_payload_before_prepare_encode(self, monkeypatch):
         runner = _make_runner()
         captured: dict[str, object] = {}