vllm-omni/tests/e2e/features/fullduplex/test_mage_vl_adapter.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

from __future__ import annotations

import pytest

from vllm_omni.experimental.fullduplex.core import protocol as ev
from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig
from vllm_omni.experimental.fullduplex.mage_vl import (
    MageVLCodecWindow,
    MageVLDuplexAdapter,
    MageVLDuplexRuntime,
    MageVLGateDecision,
)

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


async def _feed(events):
    for event in events:
        yield event


def _collector():
    out: list[dict] = []

    async def emit(event: dict) -> None:
        out.append(event)

    return out, emit


def test_mage_vl_capabilities_match_backend_contract():
    capabilities = MageVLDuplexAdapter().capabilities()

    assert capabilities.input_modalities == frozenset({"text", "video"})
    assert capabilities.output_modalities == frozenset({"text"})
    assert capabilities.proactive


@pytest.mark.asyncio
async def test_mage_vl_proactive_gate_speaks_once_for_event():
    async def gate(session, windows):
        del session
        if windows[-1].segment_id == "goal":
            return MageVLGateDecision(True, text="Goal scored.", event_id="goal-1", score=0.91)
        return MageVLGateDecision(False)

    adapter = MageVLDuplexAdapter(gate=gate, window_size=1)
    session = DuplexSession(
        "mage",
        DuplexSessionConfig(
            input_modalities=("video", "text"),
            output_modalities=("text",),
            proactive=True,
        ),
    )
    rt = MageVLDuplexRuntime(session, adapter)
    out, emit = _collector()

    await rt.run(
        _feed(
            [
                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "warmup", "frames": ["f0"]}},
                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f1"]}},
                {"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f2"]}},
                {"type": ev.CLOSE},
            ]
        ),
        emit,
    )

    deltas = [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA]
    assert deltas == ["Goal scored."]


@pytest.mark.asyncio
async def test_mage_vl_query_bypasses_silent_gate_and_preserves_codec_metadata():
    seen: dict[str, object] = {}

    async def gate(session, windows):
        del session, windows
        return {"should_respond": False}

    async def generate(session, windows, query, gate_decision):
        del session, gate_decision
        seen["query"] = query
        seen["window"] = windows[-1]
        yield "The player is moving downfield."

    adapter = MageVLDuplexAdapter(gate=gate, generate=generate, window_size=1)
    session = DuplexSession(
        "mage",
        DuplexSessionConfig(
            input_modalities=("video", "codec_window", "text"),
            output_modalities=("text",),
            proactive=True,
        ),
    )
    rt = MageVLDuplexRuntime(session, adapter)
    out, emit = _collector()

    await rt.run(
        _feed(
            [
                {"type": ev.INPUT_APPEND, "modality": "text", "data": "What changed?"},
                {
                    "type": ev.INPUT_APPEND,
                    "modality": "video",
                    "data": {
                        "kind": "h264",
                        "segment_id": "seg-7",
                        "pts_ms": 7000,
                        "duration_ms": 1000,
                        "codec": {"motion_vectors": "mv", "residual_energy": "res"},
                        "metadata": {"gop": "P"},
                    },
                },
                {"type": ev.CLOSE},
            ]
        ),
        emit,
    )

    window = seen["window"]
    assert seen["query"] == "What changed?"
    assert isinstance(window, MageVLCodecWindow)
    assert window.kind == "h264"
    assert window.segment_id == "seg-7"
    assert window.pts_ms == 7000
    assert window.metadata == {"gop": "P"}
    assert [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA] == ["The player is moving downfield."]


@pytest.mark.asyncio
async def test_mage_vl_session_state_is_isolated():
    seen: dict[str, tuple[MageVLCodecWindow, ...]] = {}

    async def generate(session, windows, query, gate_decision):
        del query, gate_decision
        seen[session.session_id] = tuple(windows)
        yield session.session_id

    adapter = MageVLDuplexAdapter(generate=generate, window_size=1)
    session_a = DuplexSession(
        "mage-a",
        DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
    )
    session_b = DuplexSession(
        "mage-b",
        DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
    )

    await adapter.on_input(session_a, "video", {"segment_id": "a", "frames": ["a0"]})
    await adapter.on_input(session_b, "video", {"segment_id": "b", "frames": ["b0"]})
    await adapter.on_input(session_a, "text", "describe a")
    await adapter.on_input(session_b, "text", "describe b")
    assert adapter.should_respond(session_a)
    assert adapter.should_respond(session_b)

    async for _ in adapter.respond(session_a):
        pass
    async for _ in adapter.respond(session_b):
        pass
    assert [window.segment_id for window in seen["mage-a"]] == ["a"]
    assert [window.segment_id for window in seen["mage-b"]] == ["b"]