[model]Add experimental Mage-VL full-duplex adapter and offline integration (#6537)

Signed-off-by: Zhanshenzhui <99891002+zyforsure@users.noreply.github.com>
Co-authored-by: Sy03 <1370724210@qq.com>

tests/e2e/features/fullduplex/test_mage_vl_adapter.py

@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from __future__ import annotations
+
+import pytest
+
+from vllm_omni.experimental.fullduplex.core import protocol as ev
+from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig
+from vllm_omni.experimental.fullduplex.mage_vl import (
+    MageVLCodecWindow,
+    MageVLDuplexAdapter,
+    MageVLDuplexRuntime,
+    MageVLGateDecision,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+async def _feed(events):
+    for event in events:
+        yield event
+
+
+def _collector():
+    out: list[dict] = []
+
+    async def emit(event: dict) -> None:
+        out.append(event)
+
+    return out, emit
+
+
+def test_mage_vl_capabilities_match_backend_contract():
+    capabilities = MageVLDuplexAdapter().capabilities()
+
+    assert capabilities.input_modalities == frozenset({"text", "video"})
+    assert capabilities.output_modalities == frozenset({"text"})
+    assert capabilities.proactive
+
+
+@pytest.mark.asyncio
+async def test_mage_vl_proactive_gate_speaks_once_for_event():
+    async def gate(session, windows):
+        del session
+        if windows[-1].segment_id == "goal":
+            return MageVLGateDecision(True, text="Goal scored.", event_id="goal-1", score=0.91)
+        return MageVLGateDecision(False)
+
+    adapter = MageVLDuplexAdapter(gate=gate, window_size=1)
+    session = DuplexSession(
+        "mage",
+        DuplexSessionConfig(
+            input_modalities=("video", "text"),
+            output_modalities=("text",),
+            proactive=True,
+        ),
+    )
+    rt = MageVLDuplexRuntime(session, adapter)
+    out, emit = _collector()
+
+    await rt.run(
+        _feed(
+            [
+                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "warmup", "frames": ["f0"]}},
+                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f1"]}},
+                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f2"]}},
+                {"type": ev.CLOSE},
+            ]
+        ),
+        emit,
+    )
+
+    deltas = [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA]
+    assert deltas == ["Goal scored."]
+
+
+@pytest.mark.asyncio
+async def test_mage_vl_query_bypasses_silent_gate_and_preserves_codec_metadata():
+    seen: dict[str, object] = {}
+
+    async def gate(session, windows):
+        del session, windows
+        return {"should_respond": False}
+
+    async def generate(session, windows, query, gate_decision):
+        del session, gate_decision
+        seen["query"] = query
+        seen["window"] = windows[-1]
+        yield "The player is moving downfield."
+
+    adapter = MageVLDuplexAdapter(gate=gate, generate=generate, window_size=1)
+    session = DuplexSession(
+        "mage",
+        DuplexSessionConfig(
+            input_modalities=("video", "codec_window", "text"),
+            output_modalities=("text",),
+            proactive=True,
+        ),
+    )
+    rt = MageVLDuplexRuntime(session, adapter)
+    out, emit = _collector()
+
+    await rt.run(
+        _feed(
+            [
+                {"type": ev.INPUT_APPEND, "modality": "text", "data": "What changed?"},
+                {
+                    "type": ev.INPUT_APPEND,
+                    "modality": "video",
+                    "data": {
+                        "kind": "h264",
+                        "segment_id": "seg-7",
+                        "pts_ms": 7000,
+                        "duration_ms": 1000,
+                        "codec": {"motion_vectors": "mv", "residual_energy": "res"},
+                        "metadata": {"gop": "P"},
+                    },
+                },
+                {"type": ev.CLOSE},
+            ]
+        ),
+        emit,
+    )
+
+    window = seen["window"]
+    assert seen["query"] == "What changed?"
+    assert isinstance(window, MageVLCodecWindow)
+    assert window.kind == "h264"
+    assert window.segment_id == "seg-7"
+    assert window.pts_ms == 7000
+    assert window.metadata == {"gop": "P"}
+    assert [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA] == ["The player is moving downfield."]
+
+
+@pytest.mark.asyncio
+async def test_mage_vl_session_state_is_isolated():
+    seen: dict[str, tuple[MageVLCodecWindow, ...]] = {}
+
+    async def generate(session, windows, query, gate_decision):
+        del query, gate_decision
+        seen[session.session_id] = tuple(windows)
+        yield session.session_id
+
+    adapter = MageVLDuplexAdapter(generate=generate, window_size=1)
+    session_a = DuplexSession(
+        "mage-a",
+        DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
+    )
+    session_b = DuplexSession(
+        "mage-b",
+        DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
+    )
+
+    await adapter.on_input(session_a, "video", {"segment_id": "a", "frames": ["a0"]})
+    await adapter.on_input(session_b, "video", {"segment_id": "b", "frames": ["b0"]})
+    await adapter.on_input(session_a, "text", "describe a")
+    await adapter.on_input(session_b, "text", "describe b")
+    assert adapter.should_respond(session_a)
+    assert adapter.should_respond(session_b)
+
+    async for _ in adapter.respond(session_a):
+        pass
+    async for _ in adapter.respond(session_b):
+        pass
+    assert [window.segment_id for window in seen["mage-a"]] == ["a"]
+    assert [window.segment_id for window in seen["mage-b"]] == ["b"]

tests/e2e/features/fullduplex/test_mage_vl_serving.py

@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+import pytest
+from fastapi.testclient import TestClient
+from starlette.websockets import WebSocketDisconnect
+
+from vllm_omni.experimental.fullduplex.mage_vl import MageVLDuplexAdapter
+from vllm_omni.experimental.fullduplex.mage_vl.serving import backend as backend_module
+from vllm_omni.experimental.fullduplex.mage_vl.serving.server import (
+    MageVLServingConfig,
+    create_app,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_mage_vl_codec_backend_requires_preprocessor(monkeypatch):
+    monkeypatch.setattr(backend_module.shutil, "which", lambda _name: None)
+
+    with pytest.raises(RuntimeError, match="cv-preinfer"):
+        backend_module.MageVLTransformersBackend(video_backend="codec")
+
+
+def test_mage_vl_frames_backend_does_not_require_preprocessor(monkeypatch):
+    def unexpected_lookup(_name):
+        raise AssertionError("frames backend must not inspect codec dependencies")
+
+    monkeypatch.setattr(backend_module.shutil, "which", unexpected_lookup)
+    backend = backend_module.MageVLTransformersBackend(video_backend="frames")
+
+    assert backend.video_backend == "frames"
+
+
+def _factory():
+    async def gate(_session, windows):
+        return {
+            "should_respond": windows[-1].segment_id == "goal",
+            "event_id": "goal-1",
+            "score": 0.9,
+        }
+
+    async def generate(_session, windows, query, _decision):
+        yield f"{query or 'proactive'}:{windows[-1].segment_id}"
+
+    return MageVLDuplexAdapter(gate=gate, generate=generate, window_size=1)
+
+
+def test_mage_vl_health_ready_and_models():
+    with TestClient(create_app(_factory)) as client:
+        assert client.get("/health").json() == {"status": "ok"}
+        assert client.get("/ready").json() == {"status": "ready", "active_sessions": 0}
+        assert client.get("/v1/models").json()["data"][0]["id"] == "microsoft/Mage-VL"
+
+
+def test_mage_vl_websocket_runs_gate_and_generation():
+    with TestClient(create_app(_factory)) as client:
+        with client.websocket_connect("/v1/mage-vl/duplex?session_id=test-session") as websocket:
+            created = websocket.receive_json()
+            assert created["type"] == "session.created"
+            assert created["session_id"] == "test-session"
+            websocket.send_json(
+                {
+                    "type": "input.append",
+                    "modality": "video",
+                    "data": {"segment_id": "goal", "frames": ["frame-1"]},
+                }
+            )
+            assert websocket.receive_json()["type"] == "response.created"
+            delta = websocket.receive_json()
+            assert delta == {
+                "type": "response.delta",
+                "response_index": 1,
+                "modality": "text",
+                "data": "proactive:goal",
+            }
+            assert websocket.receive_json()["type"] == "response.done"
+            websocket.send_json({"type": "close"})
+
+
+def test_mage_vl_websocket_requires_bearer_token():
+    app = create_app(_factory, MageVLServingConfig(auth_token="secret"))
+    with TestClient(app) as client:
+        with pytest.raises(WebSocketDisconnect) as denied:
+            with client.websocket_connect("/v1/mage-vl/duplex"):
+                pass
+        assert denied.value.code == 4401
+
+        with client.websocket_connect(
+            "/v1/mage-vl/duplex",
+            headers={"Authorization": "Bearer secret"},
+        ) as websocket:
+            assert websocket.receive_json()["type"] == "session.created"
+            websocket.send_json({"type": "close"})
+
+
+def test_mage_vl_rejects_duplicate_session_and_releases_disconnect():
+    app = create_app(_factory, MageVLServingConfig(max_sessions=1))
+    with TestClient(app) as client:
+        with client.websocket_connect("/v1/mage-vl/duplex?session_id=one") as first:
+            assert first.receive_json()["session_id"] == "one"
+            with pytest.raises(WebSocketDisconnect) as denied:
+                with client.websocket_connect("/v1/mage-vl/duplex?session_id=two") as second:
+                    second.receive_json()
+            assert denied.value.code == 4429
+            first.send_json({"type": "close"})
+
+        with client.websocket_connect("/v1/mage-vl/duplex?session_id=two") as second:
+            assert second.receive_json()["session_id"] == "two"
+            second.send_json({"type": "close"})
+
+
+def test_mage_vl_rejects_malformed_event():
+    with TestClient(create_app(_factory)) as client:
+        with client.websocket_connect("/v1/mage-vl/duplex") as websocket:
+            websocket.receive_json()
+            websocket.send_text("[]")
+            event = websocket.receive_json()
+            assert event["type"] == "error"
+            assert "JSON object" in event["message"]
+
+
+def test_mage_vl_cancel_is_not_blocked_by_slow_gate():
+    async def slow_gate(_session, _windows):
+        await asyncio.sleep(2)
+        return {"should_respond": True, "event_id": "late"}
+
+    def factory():
+        return MageVLDuplexAdapter(gate=slow_gate, window_size=1)
+
+    with TestClient(create_app(factory)) as client:
+        with client.websocket_connect("/v1/mage-vl/duplex") as websocket:
+            websocket.receive_json()
+            websocket.send_json(
+                {
+                    "type": "input.append",
+                    "modality": "video",
+                    "data": {"segment_id": "slow", "frames": ["frame"]},
+                }
+            )
+            started = time.monotonic()
+            websocket.send_json({"type": "response.cancel"})
+            event = websocket.receive_json()
+            assert time.monotonic() - started < 0.5
+            assert event["type"] == "response.cancelled"
+            websocket.send_json({"type": "close"})