[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/benchmarks/patch/test_patch.py

@@ -6,13 +6,21 @@ Unit tests for patch.py
 """
 
 import asyncio
+import base64
 import json
+import time
+from types import SimpleNamespace
 
 import pytest
 from pytest_mock import MockerFixture
 from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
 
-from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
+from vllm_omni.benchmarks.patch.patch import (
+    MixRequestFuncOutput,
+    async_request_openai_chat_omni_completions,
+    async_request_openai_realtime_duplex,
+)
+from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
 
 pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
 
@@ -40,6 +48,156 @@ class MockResponse:
         pass
 
 
+@pytest.mark.asyncio
+async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
+    class FakeRealtimeClient:
+        last_instance = None
+
+        def __init__(self, url):
+            assert url == "ws://localhost:8000/v1/realtime?duplex=1"
+            self.events = RealtimeEventCollector()
+            self.configure_kwargs = None
+            self.sent = []
+            self.response_count = 0
+            self.ack_count = 0
+            type(self).last_instance = self
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return None
+
+        async def configure(self, model, **kwargs):
+            assert model == "openbmb/MiniCPM-o-4_5"
+            self.configure_kwargs = kwargs
+
+        async def send(self, event):
+            self.sent.append(event)
+            if event["type"] != "response.create":
+                return
+            self.response_count += 1
+            response_id = f"resp-{self.response_count}"
+            now = time.monotonic()
+            self.events.add(
+                {"type": "response.created", "response": {"id": response_id}},
+                received_at_s=now + 0.01,
+            )
+            self.events.add(
+                {
+                    "type": "response.audio_transcript.delta",
+                    "response_id": response_id,
+                    "delta": f"turn {self.response_count}",
+                },
+                received_at_s=now + 0.02,
+            )
+            self.events.add(
+                {
+                    "type": "response.audio.delta",
+                    "response_id": response_id,
+                    "delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
+                    "sample_rate_hz": 24_000,
+                    "metadata": {"audio_duration_ms": 100},
+                },
+                received_at_s=now + 0.03,
+            )
+            self.events.add(
+                {"type": "response.done", "response": {"id": response_id}},
+                received_at_s=now + 0.04,
+            )
+
+        async def acknowledge_playback(self):
+            self.ack_count += 1
+
+        async def close_session(self, **_kwargs):
+            return None
+
+    monkeypatch.setattr(
+        "vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
+        FakeRealtimeClient,
+    )
+    request_input = RequestFuncInput(
+        model="openbmb/MiniCPM-o-4_5",
+        model_name="openbmb/MiniCPM-o-4_5",
+        prompt="hello",
+        api_url="http://localhost:8000/v1/realtime",
+        prompt_len=1,
+        output_len=20,
+        logprobs=None,
+        multi_modal_content=None,
+        ignore_eos=False,
+        extra_body={"save_duplex_request_metrics": True},
+    )
+    request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
+    request_input.seed_tts_system_prompt = "Speak exactly."
+    request_input.seed_tts_turns = tuple(
+        SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
+    )
+
+    output = await async_request_openai_realtime_duplex(
+        request_input,
+        session=None,
+    )
+
+    client = FakeRealtimeClient.last_instance
+    assert client.configure_kwargs["native_duplex"] is False
+    assert client.configure_kwargs["extra_body"] == {
+        "ref_audio": "data:audio/wav;base64,AAAA",
+    }
+    assert client.sent == [
+        event
+        for index in range(4)
+        for event in (
+            {
+                "type": "conversation.item.create",
+                "item": {
+                    "type": "message",
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": f"text {index}"}],
+                },
+            },
+            {"type": "response.create"},
+        )
+    ]
+    assert client.ack_count == 4
+    assert output.success is True
+    assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
+    assert output.audio_duration == pytest.approx(0.4)
+    assert output.ttft > 0
+    assert output.audio_ttfp > output.ttft
+    assert output.audio_rtf > 0
+    assert output.latency > 0
+    assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
+    assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
+    session_id = output.duplex_request_metrics[0]["session_id"]
+    assert len(output.duplex_request_metrics) == 4
+    for index, metrics in enumerate(output.duplex_request_metrics):
+        assert metrics == {
+            "session_id": session_id,
+            "request_index": index,
+            "utterance_id": f"utt-{index}",
+            "response_id": f"resp-{index + 1}",
+            "source": "client_monotonic_receive",
+            "measurement_origin": {
+                "ttft": "conversation.item.create client send to first non-empty text delta",
+                "ttfp": "conversation.item.create client send to first audio packet",
+                "rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
+            },
+            "ttft_ms": pytest.approx(20.0, abs=2.0),
+            "ttfp_ms": pytest.approx(30.0, abs=2.0),
+            "rtf": pytest.approx(0.3, abs=0.03),
+            "audio_generation_ms": pytest.approx(30.0, abs=2.0),
+            "audio_duration_ms": 100.0,
+        }
+    assert output.duplex_session_metrics == {
+        "session_id": session_id,
+        "audio_turn_count": 4,
+        "mean_ttft_ms": pytest.approx(20.0, abs=2.0),
+        "mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
+        "mean_rtf": pytest.approx(0.3, abs=0.03),
+    }
+
+
 def create_sse_chunk(data_dict):
     """Helper to create SSE formatted chunk"""
     return f"data: {json.dumps(data_dict)}\n\n".encode()

tests/benchmarks/test_accuracy_bench_utils.py

@@ -42,7 +42,10 @@ from benchmarks.accuracy.text_to_image.gbench import (
     summarize_generated_records as summarize_gebench_generated_records,
     summarize_gebench_results,
 )
-from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
+from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
+    build_serve_common_argv,
+    seed_tts_bench_argv,
+)
 from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
 from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
 
@@ -57,6 +60,25 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
     assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
 
 
+def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
+    argv = build_serve_common_argv(
+        host="127.0.0.1",
+        port=8000,
+        model="openbmb/MiniCPM-o-4_5",
+        num_prompts=4,
+        max_concurrency=1,
+        num_warmups=0,
+        percentile_metrics="ttft,audio_ttfp,audio_rtf",
+        result_dir=tmp_path,
+        result_filename="result.json",
+        backend="openai-realtime-tts",
+        endpoint="/v1/realtime",
+    )
+
+    assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
+    assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
+
+
 def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
     ns = argparse.Namespace(
         daily_omni_repo=None,

tests/benchmarks/test_seed_tts_dataset_variants.py

@@ -28,10 +28,13 @@ if _MODULE_NAME not in sys.modules:
     _spec.loader.exec_module(_mod)
 
 from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (  # noqa: E402
+    SeedTTSDataset,
     SeedTTSDesignDataset,
     SeedTTSDesignSampleRequest,
+    SeedTTSSampleRequest,
     SeedTTSTextDataset,
     SeedTTSTextSampleRequest,
+    SeedTTSTurn,
 )
 
 # ---------------------------------------------------------------------------
@@ -70,6 +73,94 @@ def mock_tokenizer(mocker):
 # ---------------------------------------------------------------------------
 
 
+def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
+    ds = SeedTTSDataset(
+        dataset_path=str(seed_tts_root),
+        random_seed=0,
+        locale="en",
+        disable_shuffle=True,
+    )
+
+    requests = ds.sample(
+        mock_tokenizer,
+        num_requests=1,
+        turns_per_session=4,
+        no_oversample=True,
+    )
+
+    assert len(requests) == 1
+    request = requests[0]
+    assert [turn.utterance_id for turn in request.seed_tts_turns] == [
+        "utt000",
+        "utt001",
+        "utt002",
+        "utt003",
+    ]
+    assert [turn.target_text for turn in request.seed_tts_turns] == [
+        "target text 0",
+        "target text 1",
+        "target text 2",
+        "target text 3",
+    ]
+    assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
+    assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
+
+
+def test_seed_tts_eval_expands_grouped_pcm_by_turn():
+    from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
+        _expand_seed_tts_turn_outputs,
+    )
+
+    request = SeedTTSSampleRequest(
+        prompt="target 0",
+        prompt_len=4,
+        expected_output_len=100,
+        multi_modal_data=None,
+        request_id="session-0",
+        seed_tts_turns=tuple(
+            SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
+        ),
+    )
+    output = types.SimpleNamespace(
+        success=True,
+        tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
+    )
+
+    requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
+
+    assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
+    assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
+    assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
+
+
+def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
+    """Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
+    from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
+        _expand_seed_tts_turn_outputs,
+    )
+
+    request = SeedTTSSampleRequest(
+        prompt="target 0",
+        prompt_len=4,
+        expected_output_len=100,
+        multi_modal_data=None,
+        request_id="session-0",
+        seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
+    )
+    pcm = b"\x01\x02\x03\x04"
+    output = types.SimpleNamespace(
+        success=True,
+        tts_output_pcm_bytes=pcm,
+        tts_turn_pcm_bytes=None,
+    )
+
+    requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
+
+    assert len(requests) == 1
+    assert requests[0].prompt == "target 0"
+    assert outputs[0].tts_output_pcm_bytes == pcm
+
+
 def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
     ds = SeedTTSTextDataset(
         dataset_path=str(seed_tts_root),