[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/e2e/accuracy/minicpmo_4_5/__init__.py

@@ -0,0 +1 @@
+# SPDX-License-Identifier: Apache-2.0

tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py

@@ -0,0 +1,211 @@
+# SPDX-License-Identifier: Apache-2.0
+"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
+
+Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
+~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
+text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
++ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
+from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
+    build_acc_benchmark_cli_argv,
+    find_vllm_cli,
+)
+from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
+_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
+_RESULT_DIR = Path(
+    os.environ.get(
+        "ACC_BENCH_RESULT_DIR",
+        str(Path(__file__).resolve().parent / "results"),
+    )
+)
+
+_MIN_DAILY_OMNI_ACCURACY = 0.78
+_MAX_SEED_TTS_MEAN_WER = 0.05
+# Match the validated Daily-Omni client body from daily_omni_bench.sh.
+_DAILY_EXTRA_BODY = {
+    "modalities": ["text"],
+    "chat_template_kwargs": {
+        "enable_thinking": False,
+        "use_tts_template": True,
+    },
+}
+_SEED_EXTRA_BODY = {
+    "modalities": ["text", "audio"],
+    "chat_template_kwargs": {
+        "enable_thinking": False,
+        "use_tts_template": True,
+    },
+}
+# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
+# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
+# msgspec ValidationError and kill the orchestrator mid-suite.
+# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
+_DAILY_OMNI_SERVER_ARGS = [
+    "--trust-remote-code",
+    "--interleave-mm-strings",
+    "--media-io-kwargs",
+    '{"video":{"fps":1,"num_frames":128}}',
+]
+_SEED_TTS_SERVER_ARGS = [
+    "--trust-remote-code",
+]
+
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
+# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An
+# ``importorskip`` inside the test body would skip only after paying several minutes
+# of startup, and the resulting skip is easy to miss in a nightly log.
+_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None]
+requires_daily_omni_deps = pytest.mark.skipif(
+    bool(_missing_daily_omni_deps),
+    reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}",
+)
+
+daily_test_params = [
+    OmniServerParams(
+        model=_MODEL,
+        stage_config_path=_DEPLOY_CONFIG,
+        server_args=list(_DAILY_OMNI_SERVER_ARGS),
+    )
+]
+
+seed_test_params = [
+    OmniServerParams(
+        model=_MODEL,
+        stage_config_path=_DEPLOY_CONFIG,
+        server_args=list(_SEED_TTS_SERVER_ARGS),
+    )
+]
+
+
+def _require_vllm_cli() -> None:
+    try:
+        find_vllm_cli()
+    except FileNotFoundError as exc:
+        pytest.skip(str(exc))
+
+
+@pytest.fixture(autouse=True)
+def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Inline cached Hub/local videos because the test server has no local-media allowlist."""
+    original = _acc_bench.daily_omni_bench_argv
+
+    def _wrapped() -> list[str]:
+        argv = list(original())
+        if "--daily-omni-inline-local-video" not in argv:
+            argv.append("--daily-omni-inline-local-video")
+        return argv
+
+    monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
+    monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+
+@requires_daily_omni_deps
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
+def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
+    _require_vllm_cli()
+
+    argv = build_acc_benchmark_cli_argv(
+        omni_server,
+        skip_seed=True,
+        skip_daily=False,
+    )
+    argv.extend(
+        [
+            "--result-dir",
+            str(_RESULT_DIR),
+            "--min-daily-omni-accuracy",
+            str(_MIN_DAILY_OMNI_ACCURACY),
+            "--temperature",
+            "0",
+            "--output-len",
+            "512",
+            "--daily-omni-input-mode",
+            "all",
+            "--daily-omni-pack-mode",
+            "minicpm-interleave",
+            "--daily-extra-body-json",
+            json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
+            "--trust-remote-code",
+        ]
+    )
+
+    assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
+
+
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
+def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
+    _require_vllm_cli()
+    pytest.importorskip("huggingface_hub")
+
+    argv = build_acc_benchmark_cli_argv(
+        omni_server,
+        skip_seed=False,
+        skip_daily=True,
+    )
+    argv.extend(
+        [
+            "--result-dir",
+            str(_RESULT_DIR),
+            "--max-seed-tts-mean-wer",
+            str(_MAX_SEED_TTS_MEAN_WER),
+            "--seed-extra-body-json",
+            json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
+            "--trust-remote-code",
+        ]
+    )
+
+    assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
+
+
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
+def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
+    """Gate Seed-TTS WER through the explicit Realtime TTS contract."""
+    _require_vllm_cli()
+    pytest.importorskip("huggingface_hub")
+
+    argv = build_acc_benchmark_cli_argv(
+        omni_server,
+        skip_seed=False,
+        skip_daily=True,
+        num_prompts=50,
+        max_concurrency=1,
+    )
+    argv.extend(
+        [
+            "--result-dir",
+            str(_RESULT_DIR),
+            "--max-seed-tts-mean-wer",
+            str(_MAX_SEED_TTS_MEAN_WER),
+            "--seed-extra-body-json",
+            json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
+            "--seed-backend",
+            "openai-realtime-tts",
+            "--seed-endpoint",
+            "/v1/realtime",
+            "--seed-tts-turns-per-session",
+            "4",
+            "--trust-remote-code",
+        ]
+    )
+
+    assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0

tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py

@@ -19,6 +19,7 @@ import json
 import os
 import shutil
 import subprocess
+import sys
 from pathlib import Path
 from typing import Any, Protocol
 
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
 
 def find_vllm_cli() -> str:
     exe = shutil.which("vllm")
-    if not exe:
-        raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
-    return exe
+    if exe:
+        return exe
+    # Fallback when pytest is invoked via absolute venv path without activating PATH.
+    sibling = Path(sys.executable).resolve().parent / "vllm"
+    if sibling.is_file() and os.access(sibling, os.X_OK):
+        return str(sibling)
+    raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
 
 
 def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
@@ -150,6 +155,11 @@ def build_serve_common_argv(
     result_dir: Path,
     result_filename: str,
     ready_check_timeout_sec: int | None = None,
+    trust_remote_code: bool = False,
+    backend: str = "openai-chat-omni",
+    endpoint: str = "/v1/chat/completions",
+    temperature: float | None = None,
+    output_len: int | None = None,
 ) -> list[str]:
     out = [
         "bench",
@@ -162,9 +172,9 @@ def build_serve_common_argv(
         "--model",
         model,
         "--endpoint",
-        "/v1/chat/completions",
+        endpoint,
         "--backend",
-        "openai-chat-omni",
+        backend,
         "--request-rate",
         "inf",
         "--num-prompts",
@@ -184,6 +194,12 @@ def build_serve_common_argv(
     ]
     if ready_check_timeout_sec is not None:
         out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
+    if trust_remote_code:
+        out.append("--trust-remote-code")
+    if temperature is not None:
+        out.extend(["--temperature", str(temperature)])
+    if output_len is not None:
+        out.extend(["--output-len", str(int(output_len))])
     return out
 
 

tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py

@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
                 os.environ[key] = value
 
 
-def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
+def _build_common_args(
+    ns: argparse.Namespace,
+    *,
+    result_filename: str,
+    backend: str = "openai-chat-omni",
+    endpoint: str = "/v1/chat/completions",
+) -> list[str]:
     return build_serve_common_argv(
         host=ns.host,
         port=ns.port,
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
         result_dir=ns.result_dir,
         result_filename=result_filename,
         ready_check_timeout_sec=ns.ready_check_timeout_sec,
+        trust_remote_code=ns.trust_remote_code,
+        backend=backend,
+        endpoint=endpoint,
+        temperature=getattr(ns, "temperature", None),
+        output_len=getattr(ns, "output_len", None),
     )
 
 
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
             json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
         ]
     )
+    pack_mode = getattr(ns, "daily_omni_pack_mode", None)
+    if pack_mode:
+        argv.extend(["--daily-omni-pack-mode", pack_mode])
     if ns.daily_omni_save_eval_items:
         argv.append("--daily-omni-save-eval-items")
     print("\n$", vllm, *argv, "\n", flush=True)
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
     result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
     extra = json.loads(ns.seed_extra_body_json)
     argv = (
-        _build_common_args(ns, result_filename=result_filename)
+        _build_common_args(
+            ns,
+            result_filename=result_filename,
+            backend=ns.seed_backend,
+            endpoint=ns.seed_endpoint,
+        )
         + seed_tts_bench_argv(locale=ns.seed_tts_locale)
         + [
             "--seed-tts-wer-eval",
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
             json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
         ]
     )
+    if ns.seed_tts_turns_per_session > 1:
+        argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
     if ns.seed_tts_wer_save_items:
         argv.append("--seed-tts-wer-save-items")
     if ns.seed_tts_file_ref_audio:
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
         help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
         "Omit to use upstream default (typically skip).",
     )
+    p.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
+    )
+    p.add_argument(
+        "--temperature",
+        type=float,
+        default=None,
+        help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
+    )
+    p.add_argument(
+        "--output-len",
+        type=int,
+        default=None,
+        help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
+    )
     p.add_argument(
         "--result-dir",
         type=Path,
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
         help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
     )
     p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
+    p.add_argument(
+        "--daily-omni-pack-mode",
+        choices=("qwen", "minicpm-interleave"),
+        default=None,
+        help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
+        "(1fps interleaved image/audio; see PR #5606).",
+    )
     p.add_argument(
         "--daily-extra-body-json",
         default='{"modalities":["text"]}',
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
         default='{"modalities":["text","audio"]}',
         help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
     )
+    p.add_argument(
+        "--seed-backend",
+        default="openai-chat-omni",
+        help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
+    )
+    p.add_argument(
+        "--seed-endpoint",
+        default="/v1/chat/completions",
+        help="Seed-TTS serving endpoint corresponding to --seed-backend.",
+    )
+    p.add_argument(
+        "--seed-tts-turns-per-session",
+        type=int,
+        default=1,
+        help="Group this many Seed-TTS target texts into each Realtime session.",
+    )
     p.add_argument("--seed-tts-wer-save-items", action="store_true")
     p.add_argument(
         "--seed-tts-file-ref-audio",
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
                 min_mean_sim=ns.min_seed_tts_mean_sim,
                 min_mean_utmos=ns.min_seed_tts_mean_utmos,
             )
+            if ns.seed_tts_turns_per_session > 1:
+                expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
+                evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
+                if evaluated_turns != expected_turns:
+                    errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
             if errs:
                 failed.extend([f"[Seed-TTS] {e}" for e in errs])
             else: