# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """CI coverage for the MiniCPM-o 4.5 native-duplex Realtime API.""" from __future__ import annotations import asyncio import base64 import json from pathlib import Path import pytest import websockets from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import ( SERVER_PARAMS, demo_args, duplex_camera_frames, multi_session_args, realtime_url, resolve_ref_audio, validated_input_wav, ) from tests.e2e.online_serving.helpers.minicpmo_realtime_duplex_scenarios import ( _ref_audio_data_url, run_demo, ) from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import ( run_multi_session, ) from tests.helpers.mark import hardware_test from vllm_omni.clients.duplex import build_realtime_url, metric_mean from vllm_omni.experimental.fullduplex.video_stacking import concat_frames_b64 pytestmark = pytest.mark.omni def _assert_positive_int(value: object) -> None: assert isinstance(value, int) assert value > 0 def _assert_request_metrics(metrics: object, *, expected_count: int) -> None: assert isinstance(metrics, list) assert len(metrics) == expected_count for request_index, request in enumerate(metrics): assert isinstance(request["session_id"], str) assert request["request_index"] == request_index assert isinstance(request["response_id"], str) assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0 assert request["ttfp_ms"] >= 0 assert request["rtf"] is not None and request["rtf"] >= 0 assert request["audio_generation_ms"] >= 0 assert request["audio_duration_ms"] > 0 def _assert_session_metrics(metrics: object, *, expected_count: int) -> None: assert isinstance(metrics, dict) assert isinstance(metrics["session_id"], str) assert metrics["audio_turn_count"] == expected_count ttft_ms = metric_mean(metrics["ttft_ms"]) ttfp_ms = metric_mean(metrics["ttfp_ms"]) rtf = metric_mean(metrics["rtf"]) assert ttft_ms is not None and ttft_ms >= 0 assert ttfp_ms is not None and ttfp_ms >= 0 assert rtf is not None and rtf >= 0 async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]: async def receive() -> list[dict[str, object]]: events: list[dict[str, object]] = [] seen: set[str] = set() while not required_types.issubset(seen): raw = await ws.recv() if not isinstance(raw, str): continue event = json.loads(raw) if not isinstance(event, dict): continue events.append(event) event_type = event.get("type") if event_type == "error": raise AssertionError(f"WebSocket protocol smoke received an error: {event}") if isinstance(event_type, str): seen.add(event_type) return events return await asyncio.wait_for(receive(), timeout=timeout_s) async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]: websocket_url = build_realtime_url(url, model, autostart=False) async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws: await ws.send( json.dumps( { "type": "session.update", "session": { "model": model, "modalities": ["audio", "text"], "ref_audio": _ref_audio_data_url(str(ref_audio)), }, } ) ) events = await _receive_protocol_events( ws, {"session.created", "session.updated"}, timeout_s=60, ) await ws.send(json.dumps({"type": "session.close"})) events.extend(await _receive_protocol_events(ws, {"session.closed"}, timeout_s=60)) return events async def _run_text_only_response_create( *, url: str, model: str, ref_audio: Path, text: str, timeout_s: float = 120.0, ) -> dict[str, object]: """Drive a text prompt with no audio and return the server's answer. A Realtime client asks for speech from text with a user message item followed by ``response.create``. A model-native duplex session generates from audio units, so this returns whatever the server answers rather than assuming a response. """ websocket_url = build_realtime_url(url, model, autostart=False) async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws: await ws.send( json.dumps( { "type": "session.update", "session": { "model": model, "modalities": ["audio", "text"], "output_audio_format": "pcm16", "turn_detection": None, "temperature": 0.0, "ref_audio": _ref_audio_data_url(str(ref_audio)), "extra_body": {"auto_response": False}, }, } ) ) await _receive_protocol_events(ws, {"session.created", "session.updated"}, timeout_s=60) await ws.send( json.dumps( { "type": "conversation.item.create", "item": { "type": "message", "role": "user", "content": [{"type": "input_text", "text": text}], }, } ) ) await ws.send(json.dumps({"type": "response.create"})) async def receive_outcome() -> dict[str, object]: while True: raw = await ws.recv() if not isinstance(raw, str): continue event = json.loads(raw) if isinstance(event, dict) and event.get("type") in {"error", "response.done"}: return event return await asyncio.wait_for(receive_outcome(), timeout=timeout_s) async def _run_seeded_text_to_audio( *, url: str, model: str, ref_audio: Path | None, text: str, modalities: tuple[str, ...] = ("audio", "text"), silence_seconds: float = 12.0, timeout_s: float = 180.0, ) -> dict[str, object]: """Speak a seeded text: the duplex route's text-to-speech shape. A model-native session takes its text once, in the session context (``duplex_initial_user_text``), and then generates per audio unit. Silence carries no content of its own, so it only advances the clock and lets the model answer the seeded turn. """ websocket_url = build_realtime_url(url, model, autostart=False) audio_bytes = 0 transcript: list[str] = [] output_text: list[str] = [] seen: list[str] = [] session_payload: dict[str, object] = { "model": model, "modalities": list(modalities), "input_audio_format": "pcm16", "output_audio_format": "pcm16", "turn_detection": None, "temperature": 0.0, "extra_body": { "auto_response": True, "force_listen_count": 0, "duplex_initial_user_text": text, }, } if ref_audio is not None: session_payload["ref_audio"] = _ref_audio_data_url(str(ref_audio)) async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws: await ws.send(json.dumps({"type": "session.update", "session": session_payload})) async def reader() -> None: nonlocal audio_bytes while True: raw = await ws.recv() if not isinstance(raw, str): continue event = json.loads(raw) if not isinstance(event, dict): continue event_type = event.get("type") if isinstance(event_type, str): seen.append(event_type) delta = event.get("delta") if event_type == "response.output_audio.delta" and isinstance(delta, str): audio_bytes += len(base64.b64decode(delta)) elif event_type == "response.output_audio_transcript.delta" and isinstance(delta, str): transcript.append(delta) elif event_type == "response.output_text.delta" and isinstance(delta, str): output_text.append(delta) elif event_type == "error": raise AssertionError(f"seeded text turn received an error: {event}") reader_task = asyncio.create_task(reader()) silence = bytes(2 * 16_000 * 200 // 1000) sent_ms = 0 try: while sent_ms < silence_seconds * 1000 and "response.done" not in seen: sent_ms += 200 await ws.send( json.dumps( { "type": "input_audio_buffer.append", "audio": base64.b64encode(silence).decode("ascii"), "input_audio_format": "pcm16", "sample_rate_hz": 16_000, "duration_ms": 200, "audio_end_ms": sent_ms, } ) ) await asyncio.sleep(0.2) loop = asyncio.get_running_loop() deadline = loop.time() + timeout_s while loop.time() < deadline and "response.done" not in seen: await asyncio.sleep(0.5) if reader_task.done(): await reader_task # Close explicitly. Dropping the socket parks the session in its # disconnect grace, where it keeps holding an admission slot and # starves the tests that follow. await ws.send(json.dumps({"type": "session.close"})) deadline = loop.time() + 30 while loop.time() < deadline and "session.closed" not in seen: await asyncio.sleep(0.25) finally: reader_task.cancel() return { "audio_bytes": audio_bytes, "transcript": "".join(transcript), "output_text": "".join(output_text), "event_types": seen, } @pytest.mark.core_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_websocket_protocol_smoke(omni_server) -> None: ref_audio = resolve_ref_audio() events = asyncio.run( _run_protocol_smoke( url=realtime_url(omni_server), model=omni_server.model, ref_audio=ref_audio, ) ) event_types = [event.get("type") for event in events] assert "session.created" in event_types assert "session.updated" in event_types assert event_types[-1] == "session.closed" @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_single_session_response_required(omni_server, tmp_path: Path) -> None: args = demo_args( omni_server=omni_server, input_wav=validated_input_wav(), ref_audio=resolve_ref_audio(), output_dir=tmp_path / "single_session", ) args.turns = 2 # Every turn replays the same active-speech window as the first one. The default # shorter follow-up window is a different mid-utterance slice, which the native # duplex model may legitimately answer with "listen" instead of a response. args.turn_duration_ms = [args.first_turn_ms] * args.turns result = asyncio.run(run_demo(args)) assert result["ok"] is True _assert_positive_int(result["audio_delta_count"]) assert result["done_count"] == 2 assert result["error_count"] == 0 assert result["all_audio_responses_have_transcript"] is True assert result["transcript_delta_done_ok"] is True _assert_request_metrics(result["request_metrics"], expected_count=2) _assert_session_metrics(result["session_metrics"], expected_count=2) @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_single_session_video_input(omni_server, tmp_path: Path) -> None: """Audio plus a 1 fps camera track, the omni-duplex video contract. Frames ride the audio appends, so the response contract is unchanged: what this covers is that interleaved vision input keeps the turn intact instead of stalling or erroring out mid-segment. """ args = demo_args( omni_server=omni_server, input_wav=validated_input_wav(), ref_audio=resolve_ref_audio(), output_dir=tmp_path / "video_input", ) args.turns = 2 args.turn_duration_ms = [args.first_turn_ms] * args.turns frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") args.video_frames_b64 = frames # Each unit also carries a composite of its interior sub-frames, so the # append exercises the official two-image frame_list that carries motion. args.video_stacked_frames_b64 = [concat_frames_b64([frames[index]] * 2) for index in range(len(frames))] result = asyncio.run(run_demo(args)) assert result["ok"] is True assert result["video_frame_count"] == 4 assert result["video_stacked_frame_count"] == 4 _assert_positive_int(result["audio_delta_count"]) assert result["done_count"] == 2 assert result["error_count"] == 0 assert result["all_audio_responses_have_transcript"] is True assert result["transcript_delta_done_ok"] is True _assert_request_metrics(result["request_metrics"], expected_count=2) _assert_session_metrics(result["session_metrics"], expected_count=2) @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) @pytest.mark.parametrize( ("locale", "text"), [ ("en", "Please say exactly: the quick brown fox jumps over the lazy dog."), ("zh", "请朗读：今天天气很好，我们一起去公园散步。"), ], ) def test_duplex_seeded_text_to_audio(omni_server, locale: str, text: str) -> None: """text -> audio over the duplex route, in both locales. This is the case the deleted turn-based ``text -> audio`` tests covered. It is also the shape the Seed-TTS Realtime backend uses, so it guards the benchmark path as well as the modality. """ result = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text=text, ) ) assert "response.done" in result["event_types"], result["event_types"] assert int(result["audio_bytes"]) > 0, f"{locale} seeded text produced no audio" assert str(result["transcript"]).strip(), f"{locale} seeded text produced audio with no transcript" @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_seeded_text_to_text_needs_no_reference_voice(omni_server) -> None: """text -> text, the one duplex session that opens without a reference voice. ``ref_audio`` is required only when the session asks for audio output, so a ``modalities: ["text"]`` session is the duplex equivalent of the deleted turn-based ``text -> text`` case. The model is model-native and still speaks its answer, so this asserts the text side and the absence of the ``ref_audio_required`` rejection, not the absence of audio. """ result = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=None, text="What is the capital of France? Answer in one short sentence.", modalities=("text",), ) ) assert "response.done" in result["event_types"], result["event_types"] produced_text = str(result["output_text"]) or str(result["transcript"]) assert produced_text.strip(), f"text-only session produced nothing: {result['event_types']}" @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_seeded_text_to_long_audio_output(omni_server) -> None: """A longer answer, so Code2Wav runs across many frames rather than one. Replaces the deleted ``text_to_audio_long_output`` case. The assertion is on the amount of audio rather than its content: the point is that a long answer streams to completion, not that the model says any given word. """ result = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text="What is the capital of China? Answer in about 40 words.", silence_seconds=30.0, timeout_s=240.0, ) ) assert "response.done" in result["event_types"], result["event_types"] # 24 kHz mono pcm16: 2 s of speech is 96000 bytes, comfortably more than a # single Code2Wav frame and well under a 40-word answer. assert int(result["audio_bytes"]) > 96_000, f"expected a long answer, got {result['audio_bytes']} bytes" assert str(result["transcript"]).strip() @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_seeded_long_form_generation(omni_server) -> None: """Long-form Chinese generation, replacing the deleted long-form case.""" result = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text="帮我讲一个100字的故事。", silence_seconds=30.0, timeout_s=240.0, ) ) assert "response.done" in result["event_types"], result["event_types"] assert int(result["audio_bytes"]) > 96_000, f"expected long-form audio, got {result['audio_bytes']} bytes" assert str(result["transcript"]).strip() @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_sequential_sessions_are_independent(omni_server) -> None: """A second session must answer its own prompt, not replay the first one. Replaces the deleted ``sequential_requests_independent`` case. Sessions are engine-resident now, so this is the check that one session's context does not leak into the next one on the same stages. """ first = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text="What is the capital of France? Answer in one short sentence.", ) ) second = asyncio.run( _run_seeded_text_to_audio( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text="What is the capital of China? Answer in one short sentence.", ) ) for result in (first, second): assert "response.done" in result["event_types"], result["event_types"] assert int(result["audio_bytes"]) > 0 assert str(result["transcript"]).strip() # Different prompts must not produce the same answer: that would mean the # second session inherited the first one's context. assert str(first["transcript"]).strip() != str(second["transcript"]).strip() @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_text_only_response_create_is_rejected(omni_server) -> None: """A duplex session generates from audio: a text-only prompt is refused, not ignored. Every session on this route is model-native, and such a model decides per audio unit whether to speak: conversation items are context, not a turn. Refusing is the point -- opening a response the model never fills costs the caller the session's whole idle timeout to discover. A text prompt reaches the model through the session's seeded opening turn (``initial_user_text``) instead, which is what ``/v1/chat/completions`` uses and what the Seed-TTS Realtime backend has to be ported to. """ outcome = asyncio.run( _run_text_only_response_create( url=realtime_url(omni_server), model=omni_server.model, ref_audio=resolve_ref_audio(), text="Please say: the quick brown fox jumps over the lazy dog.", ) ) assert outcome.get("type") == "error", outcome error = outcome.get("error") assert isinstance(error, dict), outcome assert error.get("code") == "text_only_turn_unsupported", outcome assert error.get("type") == "invalid_request_error", outcome @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_single_session_still_image_input(omni_server, tmp_path: Path) -> None: """A spoken turn with one still image attached. The still-image case is a one-element frame list, which the client repeats on every unit of the turn: unlike the camera track it never advances, so this covers the image path the turn-based ``image -> text + audio`` request used to cover. """ args = demo_args( omni_server=omni_server, input_wav=validated_input_wav(), ref_audio=resolve_ref_audio(), output_dir=tmp_path / "still_image", ) args.turns = 2 args.turn_duration_ms = [args.first_turn_ms] * args.turns args.video_frames_b64 = duplex_camera_frames(seconds=1, cache_dir=tmp_path / "still")[:1] result = asyncio.run(run_demo(args)) assert result["ok"] is True assert result["video_frame_count"] == 1 assert result["video_stacked_frame_count"] == 0 _assert_positive_int(result["audio_delta_count"]) assert result["done_count"] == 2 assert result["error_count"] == 0 assert result["all_audio_responses_have_transcript"] is True assert result["transcript_delta_done_ok"] is True _assert_request_metrics(result["request_metrics"], expected_count=2) _assert_session_metrics(result["session_metrics"], expected_count=2) @pytest.mark.advanced_model @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_two_sessions_resume_and_takeover(omni_server, tmp_path: Path) -> None: result = asyncio.run( run_multi_session( multi_session_args( omni_server=omni_server, input_wav=validated_input_wav(), ref_audio=resolve_ref_audio(), output_dir=tmp_path / "multi_session", response_required=True, ) ) ) assert result["ok"] is True assert result["session_count"] == 2 assert result["resume"]["ok"] is True assert result["takeover"]["ok"] is True assert not result["failures"] assert all(session["audio_delta_count"] > 0 for session in result["sessions"]) assert all(session["done_count"] == 1 for session in result["sessions"]) assert all(session["error_count"] == 0 for session in result["sessions"]) for session in result["sessions"]: _assert_request_metrics(session["request_metrics"], expected_count=1) _assert_session_metrics(session["session_metrics"], expected_count=1)