[model]Add experimental Mage-VL full-duplex adapter and offline integration (#6537)

Signed-off-by: Zhanshenzhui <99891002+zyforsure@users.noreply.github.com>
Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
ZhanshenzhuiandSy03 authored and GitHub committed 2026-09-02 20:25:32 -07:00
1 parent b5e898f295
commit 48ae412a2f
14 files changed
+1805 -33

No files matched your search

@@ -0,0 +1,167 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import pytest
from vllm_omni.experimental.fullduplex.core import protocol as ev
from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig
from vllm_omni.experimental.fullduplex.mage_vl import (
MageVLCodecWindow,
MageVLDuplexAdapter,
MageVLDuplexRuntime,
MageVLGateDecision,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
async def _feed(events):
for event in events:
yield event
def _collector():
out: list[dict] = []
async def emit(event: dict) -> None:
out.append(event)
return out, emit
def test_mage_vl_capabilities_match_backend_contract():
capabilities = MageVLDuplexAdapter().capabilities()
assert capabilities.input_modalities == frozenset({"text", "video"})
assert capabilities.output_modalities == frozenset({"text"})
assert capabilities.proactive
@pytest.mark.asyncio
async def test_mage_vl_proactive_gate_speaks_once_for_event():
async def gate(session, windows):
del session
if windows[-1].segment_id == "goal":
return MageVLGateDecision(True, text="Goal scored.", event_id="goal-1", score=0.91)
return MageVLGateDecision(False)
adapter = MageVLDuplexAdapter(gate=gate, window_size=1)
session = DuplexSession(
"mage",
DuplexSessionConfig(
input_modalities=("video", "text"),
output_modalities=("text",),
proactive=True,
),
)
rt = MageVLDuplexRuntime(session, adapter)
out, emit = _collector()
await rt.run(
_feed(
[
{"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "warmup", "frames": ["f0"]}},
{"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f1"]}},
{"type": ev.INPUT_APPEND, "modality": "video", "data": {"segment_id": "goal", "frames": ["f2"]}},
{"type": ev.CLOSE},
]
),
emit,
)
deltas = [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA]
assert deltas == ["Goal scored."]
@pytest.mark.asyncio
async def test_mage_vl_query_bypasses_silent_gate_and_preserves_codec_metadata():
seen: dict[str, object] = {}
async def gate(session, windows):
del session, windows
return {"should_respond": False}
async def generate(session, windows, query, gate_decision):
del session, gate_decision
seen["query"] = query
seen["window"] = windows[-1]
yield "The player is moving downfield."
adapter = MageVLDuplexAdapter(gate=gate, generate=generate, window_size=1)
session = DuplexSession(
"mage",
DuplexSessionConfig(
input_modalities=("video", "codec_window", "text"),
output_modalities=("text",),
proactive=True,
),
)
rt = MageVLDuplexRuntime(session, adapter)
out, emit = _collector()
await rt.run(
_feed(
[
{"type": ev.INPUT_APPEND, "modality": "text", "data": "What changed?"},
{
"type": ev.INPUT_APPEND,
"modality": "video",
"data": {
"kind": "h264",
"segment_id": "seg-7",
"pts_ms": 7000,
"duration_ms": 1000,
"codec": {"motion_vectors": "mv", "residual_energy": "res"},
"metadata": {"gop": "P"},
},
},
{"type": ev.CLOSE},
]
),
emit,
)
window = seen["window"]
assert seen["query"] == "What changed?"
assert isinstance(window, MageVLCodecWindow)
assert window.kind == "h264"
assert window.segment_id == "seg-7"
assert window.pts_ms == 7000
assert window.metadata == {"gop": "P"}
assert [event["data"] for event in out if event["type"] == ev.RESPONSE_DELTA] == ["The player is moving downfield."]
@pytest.mark.asyncio
async def test_mage_vl_session_state_is_isolated():
seen: dict[str, tuple[MageVLCodecWindow, ...]] = {}
async def generate(session, windows, query, gate_decision):
del query, gate_decision
seen[session.session_id] = tuple(windows)
yield session.session_id
adapter = MageVLDuplexAdapter(generate=generate, window_size=1)
session_a = DuplexSession(
"mage-a",
DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
)
session_b = DuplexSession(
"mage-b",
DuplexSessionConfig(input_modalities=("video", "text"), output_modalities=("text",), proactive=True),
)
await adapter.on_input(session_a, "video", {"segment_id": "a", "frames": ["a0"]})
await adapter.on_input(session_b, "video", {"segment_id": "b", "frames": ["b0"]})
await adapter.on_input(session_a, "text", "describe a")
await adapter.on_input(session_b, "text", "describe b")
assert adapter.should_respond(session_a)
assert adapter.should_respond(session_b)
async for _ in adapter.respond(session_a):
pass
async for _ in adapter.respond(session_b):
pass
assert [window.segment_id for window in seen["mage-a"]] == ["a"]
assert [window.segment_id for window in seen["mage-b"]] == ["b"]
@@ -0,0 +1,151 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
import time
import pytest
from fastapi.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
from vllm_omni.experimental.fullduplex.mage_vl import MageVLDuplexAdapter
from vllm_omni.experimental.fullduplex.mage_vl.serving import backend as backend_module
from vllm_omni.experimental.fullduplex.mage_vl.serving.server import (
MageVLServingConfig,
create_app,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_mage_vl_codec_backend_requires_preprocessor(monkeypatch):
monkeypatch.setattr(backend_module.shutil, "which", lambda _name: None)
with pytest.raises(RuntimeError, match="cv-preinfer"):
backend_module.MageVLTransformersBackend(video_backend="codec")
def test_mage_vl_frames_backend_does_not_require_preprocessor(monkeypatch):
def unexpected_lookup(_name):
raise AssertionError("frames backend must not inspect codec dependencies")
monkeypatch.setattr(backend_module.shutil, "which", unexpected_lookup)
backend = backend_module.MageVLTransformersBackend(video_backend="frames")
assert backend.video_backend == "frames"
def _factory():
async def gate(_session, windows):
return {
"should_respond": windows[-1].segment_id == "goal",
"event_id": "goal-1",
"score": 0.9,
}
async def generate(_session, windows, query, _decision):
yield f"{query or 'proactive'}:{windows[-1].segment_id}"
return MageVLDuplexAdapter(gate=gate, generate=generate, window_size=1)
def test_mage_vl_health_ready_and_models():
with TestClient(create_app(_factory)) as client:
assert client.get("/health").json() == {"status": "ok"}
assert client.get("/ready").json() == {"status": "ready", "active_sessions": 0}
assert client.get("/v1/models").json()["data"][0]["id"] == "microsoft/Mage-VL"
def test_mage_vl_websocket_runs_gate_and_generation():
with TestClient(create_app(_factory)) as client:
with client.websocket_connect("/v1/mage-vl/duplex?session_id=test-session") as websocket:
created = websocket.receive_json()
assert created["type"] == "session.created"
assert created["session_id"] == "test-session"
websocket.send_json(
{
"type": "input.append",
"modality": "video",
"data": {"segment_id": "goal", "frames": ["frame-1"]},
}
)
assert websocket.receive_json()["type"] == "response.created"
delta = websocket.receive_json()
assert delta == {
"type": "response.delta",
"response_index": 1,
"modality": "text",
"data": "proactive:goal",
}
assert websocket.receive_json()["type"] == "response.done"
websocket.send_json({"type": "close"})
def test_mage_vl_websocket_requires_bearer_token():
app = create_app(_factory, MageVLServingConfig(auth_token="secret"))
with TestClient(app) as client:
with pytest.raises(WebSocketDisconnect) as denied:
with client.websocket_connect("/v1/mage-vl/duplex"):
pass
assert denied.value.code == 4401
with client.websocket_connect(
"/v1/mage-vl/duplex",
headers={"Authorization": "Bearer secret"},
) as websocket:
assert websocket.receive_json()["type"] == "session.created"
websocket.send_json({"type": "close"})
def test_mage_vl_rejects_duplicate_session_and_releases_disconnect():
app = create_app(_factory, MageVLServingConfig(max_sessions=1))
with TestClient(app) as client:
with client.websocket_connect("/v1/mage-vl/duplex?session_id=one") as first:
assert first.receive_json()["session_id"] == "one"
with pytest.raises(WebSocketDisconnect) as denied:
with client.websocket_connect("/v1/mage-vl/duplex?session_id=two") as second:
second.receive_json()
assert denied.value.code == 4429
first.send_json({"type": "close"})
with client.websocket_connect("/v1/mage-vl/duplex?session_id=two") as second:
assert second.receive_json()["session_id"] == "two"
second.send_json({"type": "close"})
def test_mage_vl_rejects_malformed_event():
with TestClient(create_app(_factory)) as client:
with client.websocket_connect("/v1/mage-vl/duplex") as websocket:
websocket.receive_json()
websocket.send_text("[]")
event = websocket.receive_json()
assert event["type"] == "error"
assert "JSON object" in event["message"]
def test_mage_vl_cancel_is_not_blocked_by_slow_gate():
async def slow_gate(_session, _windows):
await asyncio.sleep(2)
return {"should_respond": True, "event_id": "late"}
def factory():
return MageVLDuplexAdapter(gate=slow_gate, window_size=1)
with TestClient(create_app(factory)) as client:
with client.websocket_connect("/v1/mage-vl/duplex") as websocket:
websocket.receive_json()
websocket.send_json(
{
"type": "input.append",
"modality": "video",
"data": {"segment_id": "slow", "frames": ["frame"]},
}
)
started = time.monotonic()
websocket.send_json({"type": "response.cancel"})
event = websocket.receive_json()
assert time.monotonic() - started < 0.5
assert event["type"] == "response.cancelled"
websocket.send_json({"type": "close"})
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.example]
REPO_ROOT = Path(__file__).resolve().parents[3]
EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "mage_vl" / "end2end.py"
def _fake_mage_repo(tmp_path: Path) -> Path:
repo = tmp_path / "Mage"
mage_vl = repo / "mage_vl"
mage_vl.mkdir(parents=True)
(mage_vl / "inference_base.py").write_text("print('base')\n", encoding="utf-8")
(mage_vl / "inference_streaming.py").write_text("print('streaming')\n", encoding="utf-8")
return repo
def test_mage_vl_offline_example_builds_codec_command(tmp_path: Path) -> None:
mage_repo = _fake_mage_repo(tmp_path)
video_path = tmp_path / "input.mp4"
video_path.write_bytes(b"fake")
output_dir = tmp_path / "outputs"
subprocess.run(
[
sys.executable,
str(EXAMPLE_SCRIPT),
"--mage-repo",
str(mage_repo),
"--task",
"video",
"--video-path",
str(video_path),
"--video-backend",
"codec",
"--codec-engine",
"traditional",
"--output-dir",
str(output_dir),
"--dry-run",
],
check=True,
)
with open(output_dir / "summary.json", encoding="utf-8") as f:
summary = json.load(f)
assert summary["task"] == "video"
assert summary["dry_run"] is True
assert "--video-backend" in summary["command"]
assert "codec" in summary["command"]
assert "--codec-engine" in summary["command"]
assert "traditional" in summary["command"]
def test_mage_vl_offline_example_builds_streaming_command(tmp_path: Path) -> None:
mage_repo = _fake_mage_repo(tmp_path)
video_path = tmp_path / "input.mp4"
video_path.write_bytes(b"fake")
output_dir = tmp_path / "outputs"
subprocess.run(
[
sys.executable,
str(EXAMPLE_SCRIPT),
"--mage-repo",
str(mage_repo),
"--task",
"streaming",
"--video-path",
str(video_path),
"--max-segments",
"1",
"--output-dir",
str(output_dir),
"--dry-run",
],
check=True,
)
with open(output_dir / "summary.json", encoding="utf-8") as f:
summary = json.load(f)
command = summary["command"]
assert summary["task"] == "streaming"
assert str(mage_repo / "mage_vl" / "inference_streaming.py") in command
assert "--checkpoint" in command
assert "--max_segments" in command