[CI/Build][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance nightly coverage (#5710)

Signed-off-by: psv666 <2693925048@qq.com>
Signed-off-by: psv666 <150513104+psv666@users.noreply.github.com>
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/benchmarks/test_daily_omni_local_source.py

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Offline Daily-Omni sources: local mirrors must not fall back to the Hub id."""
+
+import json
+import os
+import tarfile
+from pathlib import Path
+
+import pytest
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
+    daily_omni_local_qa_json,
+    daily_omni_local_videos_dir,
+    resolve_daily_omni_local_root,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
+
+
+def _write_qa_json(root: Path) -> Path:
+    qa = root / "qa.json"
+    qa.write_text(json.dumps([{"Question": "q", "Answer": "A", "video_id": "vid0"}]), encoding="utf-8")
+    return qa
+
+
+def test_resolve_local_root_returns_none_for_hub_id() -> None:
+    assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None
+    assert resolve_daily_omni_local_root("") is None
+    assert resolve_daily_omni_local_root(None) is None
+
+
+def test_plain_local_mirror(tmp_path: Path) -> None:
+    _write_qa_json(tmp_path)
+    (tmp_path / "Videos" / "vid0").mkdir(parents=True)
+
+    root = resolve_daily_omni_local_root(str(tmp_path))
+    assert root == tmp_path.resolve()
+    assert daily_omni_local_qa_json(root) == (tmp_path / "qa.json").resolve()
+    assert daily_omni_local_videos_dir(root) == root / "Videos"
+
+
+def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
+    cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
+    snapshot = cache_dir / "snapshots" / "deadbeef"
+    snapshot.mkdir(parents=True)
+    _write_qa_json(snapshot)
+    (cache_dir / "refs").mkdir()
+    (cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
+
+    root = resolve_daily_omni_local_root(str(cache_dir))
+    assert root == snapshot.resolve()
+    assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
+
+
+def test_hf_cache_dir_without_refs_picks_newest_snapshot(tmp_path: Path) -> None:
+    cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
+    older = cache_dir / "snapshots" / "aaa"
+    newer = cache_dir / "snapshots" / "bbb"
+    older.mkdir(parents=True)
+    newer.mkdir(parents=True)
+    os.utime(older, (1, 1))
+    os.utime(newer, (10, 10))
+
+    assert resolve_daily_omni_local_root(str(cache_dir)) == newer.resolve()
+
+
+def test_videos_tar_is_extracted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("HF_HOME", str(tmp_path / "hf_home"))
+    root = tmp_path / "mirror"
+    (root / "Videos" / "vid0").mkdir(parents=True)
+    (root / "Videos" / "vid0" / "vid0_video.mp4").write_bytes(b"fake")
+    with tarfile.open(root / "Videos.tar", "w") as tf:
+        tf.add(root / "Videos", arcname="Videos")
+    # Only the tarball is left in the mirror.
+    for child in (root / "Videos").rglob("*"):
+        if child.is_file():
+            child.unlink()
+    (root / "Videos" / "vid0").rmdir()
+    (root / "Videos").rmdir()
+
+    videos_dir = daily_omni_local_videos_dir(root)
+    assert videos_dir is not None
+    assert (videos_dir / "vid0" / "vid0_video.mp4").is_file()
+
+    # Second call reuses the extraction instead of redoing it.
+    assert daily_omni_local_videos_dir(root) == videos_dir
+
+
+def test_missing_qa_json_reports_none(tmp_path: Path) -> None:
+    root = resolve_daily_omni_local_root(str(tmp_path))
+    assert daily_omni_local_qa_json(root) is None
+    assert daily_omni_local_videos_dir(root) is None

tests/dfx/perf/tests/test_minicpmo_4_5.json

@@ -5,7 +5,8 @@
       {
         "hardware_marks": {
           "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
           },
           "num_cards": 1
         }
@@ -30,14 +31,12 @@
         "num_prompts": [
           32,
           64,
-          128,
-          256
+          128
         ],
         "max_concurrency": [
           1,
           4,
-          8,
-          16
+          8
         ],
         "no_oversample": true,
         "trust_remote_code": true,
@@ -55,34 +54,29 @@
         "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
         "baseline": {
           "request_throughput": [
-            0.7803,
-            1.1008,
-            1.2027,
-            1.3254
+            0.5383,
+            0.6042,
+            0.7547
           ],
           "mean_ttft_ms": [
-            166.9764,
-            307.7133,
-            323.584,
-            356.6077
+            333.2633,
+            533.8656,
+            547.3388
           ],
           "mean_e2el_ms": [
-            1361.1682,
-            3800.6332,
-            6961.8636,
-            12609.4295
+            1857.2154,
+            6600.5101,
+            10450.6811
           ],
           "mean_audio_ttfp_ms": [
-            635.8099,
-            1615.6591,
-            2888.9831,
-            8320.5422
+            986.4666,
+            3411.0754,
+            3352.7515
           ],
           "mean_audio_rtf": [
-            0.3151,
-            0.8568,
-            1.5425,
-            2.8224
+            0.4423,
+            1.5734,
+            2.3024
           ]
         }
       }

tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json

@@ -5,7 +5,8 @@
       {
         "hardware_marks": {
           "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
           },
           "num_cards": 1
         }

tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py

@@ -39,10 +39,7 @@ _MAX_SEED_TTS_MEAN_WER = 0.05
 # Match the validated Daily-Omni client body from daily_omni_bench.sh.
 _DAILY_EXTRA_BODY = {
     "modalities": ["text"],
-    "chat_template_kwargs": {
-        "enable_thinking": False,
-        "use_tts_template": True,
-    },
+    "chat_template_kwargs": {"enable_thinking": False},
 }
 _SEED_EXTRA_BODY = {
     "modalities": ["text", "audio"],
@@ -151,6 +148,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
         omni_server,
         skip_seed=False,
         skip_daily=True,
+        max_concurrency=4,
     )
     argv.extend(
         [

tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py

@@ -963,12 +963,23 @@ async def _send_clean_turn(
         if outcome == "listen":
             return None, "listen"
     else:
-        await _wait_for(
-            state,
-            lambda: state.count("response.created") > before_created,
-            timeout_s=timeout_s,
-            label=f"{transcript} response.created",
-        )
+        try:
+            await _wait_for(
+                state,
+                lambda: state.count("response.created") > before_created,
+                timeout_s=timeout_s,
+                label=f"{transcript} response.created",
+            )
+        except TimeoutError:
+            # ``auto_response`` sessions only get ``response.created`` once the model
+            # decides to speak, so report the listen decisions that took its place.
+            raise TimeoutError(
+                f"Timed out waiting for {transcript} response.created after "
+                f"{state.count('input_audio_buffer.committed')} committed input turns; "
+                f"model_listen={state.model_listen_count - before_model_listen} "
+                f"buffering_listen={state.buffering_listen_count} "
+                f"errors={state.count('error')}"
+            ) from None
         await _wait_for(
             state,
             lambda: eligible_response_id() is not None,

tests/e2e/online_serving/test_minicpmo_4_5_duplex.py

@@ -133,6 +133,10 @@ def test_duplex_single_session_response_required(omni_server, model_prefix: str,
         output_dir=tmp_path / "single_session",
     )
     args.turns = 2
+    # Every turn replays the same active-speech window as the first one. The default
+    # shorter follow-up window is a different mid-utterance slice, which the native
+    # duplex model may legitimately answer with "listen" instead of a response.
+    args.turn_duration_ms = [args.first_turn_ms] * args.turns
     result = asyncio.run(run_demo(args))
     assert result["ok"] is True
     assert result["audio_delta_count"] > 0

tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py

@@ -53,7 +53,11 @@ def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_
     assert result["admission"]["overflow_error_code"] == "resource_exhausted"
 
 
-@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+# CUDA-only for now: this contract asserts the model speaks while the user is
+# still talking, which only holds when the duplex pipeline sustains real-time
+# throughput. The current NPU stack runs several times slower than real time,
+# so it never reaches a mid-stream decision point.
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
 def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
     input_wav = validated_soft_interrupt_wav()

tests/helpers/minicpmo_4_5_duplex.py

@@ -74,9 +74,11 @@ def validated_soft_interrupt_wav() -> Path:
 
 
 def resolve_ref_audio(model_prefix: str) -> Path:
-    if model_prefix:
-        model_root = Path(model_prefix) / MODEL
-    else:
+    # ``MODEL`` may be a local checkpoint directory instead of a Hub repo id
+    # (``Path("/prefix") / "/abs/path"`` collapses to the absolute path), so only
+    # fall back to the Hub cache when no such directory exists on disk.
+    model_root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
+    if not model_root.is_dir():
         model_root = Path(snapshot_download(MODEL, local_files_only=True))
     ref_audio = model_root / REF_AUDIO_RELATIVE_PATH
     if not ref_audio.is_file():