[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>

tests/diffusion/streaming_interactive_generation/test_camera_interaction.py

@@ -0,0 +1,434 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Tests for camera interaction handlers and coordinator registration."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
+from vllm_omni.diffusion.interaction.mixin import InteractionMixin
+from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession, SE3DeltaCameraHandler
+from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptSession
+from vllm_omni.diffusion.interaction.registry import STRUCTURED_HANDLER_REGISTRY
+from vllm_omni.diffusion.interaction.types import ChunkMediaSpec, resolve_event_frame_offset
+from vllm_omni.diffusion.worker.utils import StepRequestState
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+
+_FORWARD_VELOCITY = {
+    "mode": "velocity",
+    "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+}
+
+
+class _FakePromptPipeline(InteractionMixin):
+    """Minimal pipeline stub with prompt-update support."""
+
+    def __init__(self) -> None:
+        self.device = torch.device("cpu")
+        self.transformer = SimpleNamespace(dtype=torch.float32)
+        self.encode_prompt = MagicMock(return_value=(torch.full((1, 4, 2), 2.0), None))
+        self._interaction_coordinator = None
+
+    def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
+        del state
+        return ChunkMediaSpec(num_media_frames=8, fps=16.0, num_latent_frames=8)
+
+
+class _FakeLingBotPipeline(InteractionMixin):
+    """Minimal pipeline stub for LingBot camera registration."""
+
+    def __init__(self) -> None:
+        self._interaction_coordinator = None
+
+    def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
+        del state
+        return ChunkMediaSpec(num_media_frames=3, fps=3.0, num_latent_frames=3)
+
+
+def _make_state(*, request_id: str = "req-1") -> StepRequestState:
+    state = StepRequestState(
+        request_id=request_id,
+        sampling=SimpleNamespace(num_outputs_per_prompt=1, max_sequence_length=226),  # pyright: ignore[reportArgumentType]
+        prompt="hello",
+    )
+    state.prompt_embeds = torch.zeros(1, 4, 2)
+    state.extra = {}
+    return state
+
+
+def _boundary_at(previous_boundary_at: float | None, num_frames: int, fps: float) -> float:
+    if previous_boundary_at is None:
+        return 0.0
+    if fps <= 0:
+        return previous_boundary_at
+    return previous_boundary_at + num_frames / fps
+
+
+class TestCoordinatorResolution:
+    def test_helios_has_prompt_but_no_camera(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        assert coordinator.has_modality("prompt")
+        assert not coordinator.has_modality("camera")
+        assert "camera" not in STRUCTURED_HANDLER_REGISTRY["HeliosPipeline"]
+
+    def test_lingbot_registers_se3_camera_handler(self) -> None:
+        pipeline = _FakeLingBotPipeline()
+        od_config = SimpleNamespace(model_class_name="LingBotWorldCausalDMDPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        assert coordinator.has_modality("camera")
+        handler = coordinator.get_handler("camera")
+        assert isinstance(handler, SE3DeltaCameraHandler)
+        assert handler.lazy_initialize_session is False
+
+    def test_non_lazy_camera_apply_creates_session_without_enqueue(self) -> None:
+        """Camera apply must run before any enqueue to materialize identity poses."""
+        pipeline = _FakeLingBotPipeline()
+        od_config = SimpleNamespace(model_class_name="LingBotWorldCausalDMDPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+
+        assert "camera" not in state.interaction_sessions
+        meta = coordinator.maybe_prepare_initial_session(state, pipeline)
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (3, 4, 4)
+        assert meta.started_event_ids == []
+        assert meta.active_event_ids == []
+
+    def test_lazy_prompt_apply_skips_without_enqueue(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+
+        meta = coordinator.apply_at_chunk_boundary(state, boundary_at=0.0, chunk_index=0)
+        assert "prompt" not in state.interaction_sessions
+        assert meta.started_event_ids == []
+        assert meta.active_event_ids == []
+        assert meta.completed_event_ids == []
+
+        initial = coordinator.maybe_prepare_initial_session(state, pipeline)
+        assert "prompt" not in state.interaction_sessions
+        assert initial.started_event_ids == []
+
+    def test_unsupported_modality_includes_model_context(self) -> None:
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+
+        with pytest.raises(ValueError, match="HeliosPipeline"):
+            coordinator.enqueue(
+                _make_state(),
+                modality="camera",
+                event_id="cam-1",
+                received_at=0.0,
+                payload=_FORWARD_VELOCITY,
+                transition_chunks=None,
+            )
+
+    def test_enqueue_parts_rejects_unsupported_modality_atomically(self) -> None:
+        """Unsupported tracks in a composite event must not mutate any queues."""
+        pipeline = _FakePromptPipeline()
+        od_config = SimpleNamespace(model_class_name="HeliosPipeline")
+        coordinator = InteractionCoordinator.build(pipeline, od_config)
+        state = _make_state()
+        coordinator.enqueue(
+            state,
+            modality="prompt",
+            event_id="prior",
+            received_at=0.0,
+            payload={"prompt": "already-queued"},
+            transition_chunks=1,
+        )
+        prior_session = state.interaction_sessions["prompt"]
+        assert isinstance(prior_session, PromptSession)
+        prior = prior_session.pending_event
+        assert prior is not None
+        pipeline.encode_prompt.reset_mock()
+
+        with pytest.raises(ValueError, match="camera"):
+            coordinator.enqueue_parts(
+                state,
+                parts=[
+                    ("prompt", {"prompt": "should-not-replace"}),
+                    ("camera", _FORWARD_VELOCITY),
+                ],
+                event_id="composite-bad",
+                received_at=1.0,
+                transition_chunks=2,
+            )
+
+        pipeline.encode_prompt.assert_not_called()
+        pending = prior_session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
+        # Invalid camera payload must also fail before any track is queued.
+        dual = InteractionCoordinator(
+            {
+                "prompt": coordinator.get_handler("prompt"),
+                "camera": SE3DeltaCameraHandler(),
+            },
+            model_class_name="HeliosPipeline",
+        )
+        pipeline.encode_prompt.reset_mock()
+        with pytest.raises(ValueError, match="actions"):
+            dual.enqueue_parts(
+                state,
+                parts=[
+                    ("prompt", {"prompt": "should-not-replace"}),
+                    ("camera", {"mode": "velocity", "data": {"actions": ["w"]}}),
+                ],
+                event_id="composite-bad-payload",
+                received_at=2.0,
+                transition_chunks=2,
+            )
+        pipeline.encode_prompt.assert_not_called()
+        pending = prior_session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
+
+class TestCameraHandlers:
+    def test_rejects_wasd_actions_payload(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        with pytest.raises(ValueError, match="actions"):
+            handler.enqueue(
+                state,
+                event_id="cam-wasd",
+                received_at=0.0,
+                payload={"mode": "velocity", "data": {"actions": ["w"]}},
+                transition_chunks=None,
+            )
+
+    def test_target_transition_progress_and_se3_projection(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="cam-1",
+            received_at=0.0,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 3.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=2,
+        )
+
+        meta = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=4,
+            fps=16.0,
+            boundary_at=0.25,
+        )
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.last_absolute_poses is not None
+        assert meta is not None
+        assert meta.started_event_ids == ["cam-1"]
+        assert meta.active_event_ids == ["cam-1"]
+        assert meta.completed_event_ids == []
+        assert session.last_absolute_poses.shape == (4, 4, 4)
+
+        meta2 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=1,
+            num_media_frames=4,
+            fps=16.0,
+            boundary_at=0.5,
+        )
+        assert meta2 is not None
+        assert meta2.started_event_ids == []
+        assert meta2.active_event_ids == []
+        assert meta2.completed_event_ids == ["cam-1"]
+        assert session.active_event is None
+        assert session.current_pose.translation[2] == pytest.approx(3.0)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (4, 4, 4)
+
+    def test_single_chunk_target_completes_once(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        # Include a non-power-of-two frame count (7) so float reciprocal drift
+        # would previously miss the final snap / completion ack.
+        for num_frames in (8, 7):
+            state = _make_state()
+            handler.enqueue(
+                state,
+                event_id="cam-fast",
+                received_at=0.0,
+                payload={
+                    "mode": "target",
+                    "data": {"translation": [1.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                },
+                transition_chunks=1,
+            )
+            meta = handler.apply_at_chunk_boundary(
+                state,
+                chunk_index=0,
+                num_media_frames=num_frames,
+                fps=16.0,
+                boundary_at=0.5,
+            )
+            assert meta is not None
+            assert meta.started_event_ids == ["cam-fast"]
+            assert meta.active_event_ids == []
+            assert meta.completed_event_ids == ["cam-fast"]
+
+            meta2 = handler.apply_at_chunk_boundary(
+                state,
+                chunk_index=1,
+                num_media_frames=num_frames,
+                fps=16.0,
+                boundary_at=1.0,
+            )
+            assert meta2 is not None
+            assert meta2.started_event_ids == []
+            assert meta2.active_event_ids == []
+            assert meta2.completed_event_ids == []
+            session = state.interaction_sessions["camera"]
+            assert isinstance(session, CameraSession)
+            assert session.current_pose.translation[0] == pytest.approx(1.0)
+
+    def test_velocity_composes_structural_se3_delta(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="vel-1",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+
+        meta0 = handler.apply_at_chunk_boundary(state, chunk_index=0, num_media_frames=3, fps=16.0, boundary_at=0.0)
+        assert meta0 is not None
+        assert meta0.started_event_ids == ["vel-1"]
+        assert meta0.active_event_ids == ["vel-1"]
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.current_pose.translation[2] == pytest.approx(0.15)
+        assert session.last_absolute_poses is not None
+        assert session.last_absolute_poses.shape == (3, 4, 4)
+        torch.testing.assert_close(
+            session.last_absolute_poses[0, :3, 3],
+            torch.tensor([0.0, 0.0, 0.05], dtype=torch.float64),
+            atol=1e-6,
+            rtol=0,
+        )
+        torch.testing.assert_close(
+            session.last_absolute_poses[1, :3, 3],
+            torch.tensor([0.0, 0.0, 0.10], dtype=torch.float64),
+            atol=1e-6,
+            rtol=0,
+        )
+
+        meta1 = handler.apply_at_chunk_boundary(state, chunk_index=1, num_media_frames=2, fps=16.0, boundary_at=0.125)
+        assert meta1 is not None
+        assert meta1.active_event_ids == ["vel-1"]
+        assert session.current_pose.translation[2] == pytest.approx(0.25)
+
+    def test_velocity_completed_when_replaced_by_target(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        handler.enqueue(
+            state,
+            event_id="vel-1",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+        meta0 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=2,
+            fps=16.0,
+            boundary_at=0.0,
+        )
+        assert meta0 is not None
+        assert meta0.active_event_ids == ["vel-1"]
+
+        handler.enqueue(
+            state,
+            event_id="tgt-1",
+            received_at=0.1,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=0,
+        )
+        meta1 = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=1,
+            num_media_frames=2,
+            fps=16.0,
+            boundary_at=0.125,
+        )
+        assert meta1 is not None
+        assert meta1.started_event_ids == ["tgt-1"]
+        assert "vel-1" in meta1.completed_event_ids
+        assert meta1.active_event_ids == []
+
+    def test_same_chunk_velocity_then_target_by_frame_offset(self) -> None:
+        handler = SE3DeltaCameraHandler()
+        state = _make_state()
+        fps = 10.0
+        boundary = 0.0
+        handler.enqueue(
+            state,
+            event_id="vel",
+            received_at=0.0,
+            payload=_FORWARD_VELOCITY,
+            transition_chunks=None,
+        )
+        handler.enqueue(
+            state,
+            event_id="tgt",
+            received_at=0.5,
+            payload={
+                "mode": "target",
+                "data": {"translation": [0.0, 0.0, 0.0], "rotation": [0.0, 0.0, 0.0, 1.0]},
+            },
+            transition_chunks=0,
+        )
+        state.interaction_sessions["camera"].last_boundary_at = boundary
+        meta = handler.apply_at_chunk_boundary(
+            state,
+            chunk_index=0,
+            num_media_frames=10,
+            fps=fps,
+            boundary_at=_boundary_at(boundary, 10, fps),
+        )
+        assert meta is not None
+        assert meta.started_event_ids == ["vel", "tgt"]
+        assert meta.completed_event_ids == ["vel", "tgt"]
+        assert meta.active_event_ids == []
+        session = state.interaction_sessions["camera"]
+        assert isinstance(session, CameraSession)
+        assert session.active_event is None
+        assert session.current_pose.translation[2] == pytest.approx(0.0)
+
+
+class TestResolveEventFrameOffset:
+    def test_clamps_and_floors(self) -> None:
+        assert resolve_event_frame_offset(received_at=0.0, previous_boundary_at=None, num_frames=8, fps=16.0) == 0
+        assert resolve_event_frame_offset(received_at=0.1, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 1
+        assert resolve_event_frame_offset(received_at=10.0, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 7
+        assert resolve_event_frame_offset(received_at=-1.0, previous_boundary_at=0.0, num_frames=8, fps=16.0) == 0

tests/diffusion/streaming_interactive_generation/test_prompt_interaction.py

@@ -78,7 +78,7 @@ def _make_diffusion_request_state(*, request_id: str = "req-1", fps: int | None
         prompt="hello",
     )
     state.prompt_embeds = torch.zeros(1, 4, 2)
-    state.extra = {"window_num_frames": 8}
+    state.extra = {"window_num_frames": 8, "num_latent_frames_per_chunk": 3}
     return state
 
 
@@ -109,7 +109,12 @@ def _make_diffusion_model_runner(
     # instances; unit tests of submit_interaction stub this. Stepwise path tests
     # rebind to the real method.
     runner._supports_step_mode = lambda: True
-    runner._interaction_coordinator = None
+    # Mirror load_model: coordinator is always present once the runner is "loaded".
+    # Build now so handlers capture the pipeline's current encode_prompt (tests may
+    # rebind the mock before calling this helper).
+    runner._interaction_coordinator = InteractionCoordinator.build(pipeline, runner.od_config)
+    if hasattr(pipeline, "_interaction_coordinator"):
+        pipeline._interaction_coordinator = runner._interaction_coordinator
     return runner
 
 
@@ -155,6 +160,7 @@ class TestPromptUpdateExecution:
             supports_step_execution = True
 
         runner = _make_diffusion_model_runner(pipeline=_UnsupportedPipeline())
+        runner.state_cache["req-1"] = _make_diffusion_request_state()
         with pytest.raises(ValueError, match="not supported"):
             runner.submit_interaction("req-1", _prompt_interaction())
 
@@ -168,25 +174,98 @@ class TestPromptUpdateExecution:
         "interaction",
         [
             {},
-            {"multi_modal_data": {"camera": {"type": "pose"}}},
-            {"event": {"prompt": "updated", "multi_modal_data": {"camera": {"type": "pose"}}}},
-            {"event": {"prompt": "updated", "multi_modal_data": {}}},
-            {"event": {"prompt": "updated", "multi_modal_data": None}},
-            {"event": {"prompt": "updated", "multi_modal_data": "bad"}},
+            {"event_id": "bad", "event": {"multi_modal_data": {}}},
+            {"event_id": "bad", "event": {"multi_modal_data": None}},
+            {"event_id": "bad", "event": {"prompt": "updated", "multi_modal_data": "bad"}},
+            {
+                "event_id": "cam-only",
+                "event": {
+                    "multi_modal_data": {
+                        "camera": {
+                            "mode": "velocity",
+                            "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                        },
+                    }
+                },
+            },
+            {
+                "event_id": "cam-and-prompt",
+                "event": {
+                    "prompt": "updated",
+                    "multi_modal_data": {
+                        "camera": {
+                            "mode": "velocity",
+                            "data": {"translation": [0.0, 0.0, 0.05], "rotation": [0.0, 0.0, 0.0, 1.0]},
+                        },
+                    },
+                },
+            },
         ],
     )
-    def test_runner_interaction_rejects_structural_payloads_until_implemented(
+    def test_runner_rejects_unsupported_interactions(
         self,
         pipeline: HeliosPipeline,
         interaction: dict[str, Any],
     ) -> None:
-        """Unsupported interaction dict shapes are preserved to and rejected by the runner."""
+        """Helios has no camera handler; malformed / unsupported payloads are rejected."""
         runner = _make_diffusion_model_runner(pipeline=pipeline)
-        runner.state_cache["req-1"] = _make_diffusion_request_state()
+        state = _make_diffusion_request_state()
+        runner.state_cache["req-1"] = state
 
-        with pytest.raises(NotImplementedError, match="Only text-only prompt update interactions"):
+        with pytest.raises((ValueError, KeyError, AttributeError, TypeError)):
             runner.submit_interaction("req-1", cast(Any, interaction))
 
+        # Rejected composite/malformed events must leave interaction queues unchanged.
+        pipeline.encode_prompt.assert_not_called()  # pyright: ignore[reportAttributeAccessIssue]
+        assert "prompt" not in state.interaction_sessions
+        assert "camera" not in state.interaction_sessions
+
+    def test_rejected_composite_leaves_existing_prompt_queue_unchanged(
+        self,
+        pipeline: HeliosPipeline,
+    ) -> None:
+        """Helios must not apply a composite prompt when camera modality is unsupported."""
+        runner = _make_diffusion_model_runner(pipeline=pipeline)
+        state = _make_diffusion_request_state()
+        runner.state_cache["req-1"] = state
+        runner.submit_interaction("req-1", _prompt_interaction("already-queued", event_id="prior"))
+        session = state.interaction_sessions["prompt"]
+        assert isinstance(session, PromptSession)
+        prior = session.pending_event
+        assert prior is not None
+        assert prior.event_id == "prior"
+        pipeline.encode_prompt.reset_mock()  # pyright: ignore[reportAttributeAccessIssue]
+
+        with pytest.raises(ValueError, match="camera"):
+            runner.submit_interaction(
+                "req-1",
+                cast(
+                    Any,
+                    {
+                        "event_id": "cam-and-prompt",
+                        "event": {
+                            "prompt": "should-not-replace",
+                            "multi_modal_data": {
+                                "camera": {
+                                    "mode": "velocity",
+                                    "data": {
+                                        "translation": [0.0, 0.0, 0.05],
+                                        "rotation": [0.0, 0.0, 0.0, 1.0],
+                                    },
+                                },
+                            },
+                        },
+                    },
+                ),
+            )
+
+        pipeline.encode_prompt.assert_not_called()  # pyright: ignore[reportAttributeAccessIssue]
+        pending = session.pending_event
+        assert pending is not None
+        assert pending.event_id == "prior"
+        assert pending.prompt == "already-queued"
+        assert "camera" not in state.interaction_sessions
+
     @pytest.mark.parametrize("model_class_name", ["HeliosPipeline", "HeliosPyramidPipeline"])
     def test_coordinator_registers_prompt_for_helios_aliases(
         self,
@@ -367,7 +446,11 @@ class TestPromptUpdateExecution:
                         f"that use the chunk media timeline, got {fps!r} "
                         f"(request_id={state.request_id!r})"
                     )
-                return ChunkMediaSpec(num_frames=int(state.extra["window_num_frames"]), fps=float(fps))
+                return ChunkMediaSpec(
+                    num_media_frames=int(state.extra["window_num_frames"]),
+                    fps=float(fps),
+                    num_latent_frames=int(state.extra["window_num_frames"]),
+                )
 
             def prepare_encode(self, state: StepRequestState) -> StepRequestState:
                 state.prompt_embeds = torch.zeros(1, 4, 2)