diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index a984c2f32..edc509a81 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -41,7 +41,7 @@ "no_oversample": true, "disable_shuffle": true, "trust_remote_code": true, - "seed_tts_locale": "en", + "seed_tts_locale": "zh", "extra_body": { "modalities": [ "text", @@ -126,7 +126,7 @@ "no_oversample": true, "disable_shuffle": true, "trust_remote_code": true, - "seed_tts_locale": "en", + "seed_tts_locale": "zh", "extra_body": { "modalities": [ "text", diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index 986d0764d..8476628bf 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -37,7 +37,7 @@ "no_oversample": true, "disable_shuffle": true, "trust_remote_code": true, - "seed_tts_locale": "en", + "seed_tts_locale": "zh", "seed_tts_turns_per_session": 4, "expected_duplex_audio_turns_per_session": 4, "extra_body": { diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json index 15d52e6ad..cd919e02f 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -39,7 +39,7 @@ "no_oversample": true, "disable_shuffle": true, "trust_remote_code": true, - "seed_tts_locale": "en", + "seed_tts_locale": "zh", "extra_body": { "modalities": [ "text", diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index ce14f19fe..1940f77ac 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 """MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage. -Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches -~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``, -text modalities with ``use_tts_template``, and server ``--interleave-mm-strings`` -+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``). +Daily-Omni settings follow the MiniCPM interleaved AV recipe (``minicpm-interleave``, +``temperature=0``, text modalities with ``use_tts_template``, and server +``--interleave-mm-strings`` + 1fps / 128-frame media-io kwargs, also pinned in +``minicpmo_4_5.yaml``). Baseline **79.5**; gate **77.5** (drop ≤ 2pp). Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs): ``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``. -Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin). +Baseline **69.0**; gate **67.0** (drop ≤ 2pp). + +Seed-TTS (zh, seed-tts-eval protocol): WER baseline **1.414%** / gate **≤ 1.56%** +(+10%). """ from __future__ import annotations @@ -26,7 +29,6 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import ( find_vllm_cli, ) from tests.helpers.mark import hardware_test -from tests.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path @@ -39,10 +41,11 @@ _RESULT_DIR = Path( ) ) -_MIN_DAILY_OMNI_ACCURACY = 0.78 -# MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni. -_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68")) -_MAX_SEED_TTS_MEAN_WER = 0.02 +# Accuracy gates: baseline minus allowed drop (≤2pp / ≤10% WER). +_MIN_DAILY_OMNI_ACCURACY = 0.775 # 79.5 - 2pp +_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.67")) +# jiwer reports a 0-1 ratio; table 1.414 / 1.56 is percent → 0.0156. +_MAX_SEED_TTS_MEAN_WER = 0.0156 # Full Seed-TTS zh split (seed-tts-eval/zh/meta.lst) is 2020 rows. _SEED_TTS_LOCALE = "zh" _SEED_TTS_NUM_PROMPTS = 2020 @@ -291,41 +294,3 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: ) assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 - - -@hardware_test(res={"cuda": "H100"}, num_cards=2) -@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True) -def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None: - """Gate Seed-TTS WER through the explicit Realtime TTS contract.""" - _require_vllm_cli() - pytest.importorskip("huggingface_hub") - - argv = build_acc_benchmark_cli_argv( - omni_server, - skip_seed=False, - skip_daily=True, - skip_videomme=True, - num_prompts=50, - max_concurrency=1, - ) - argv.extend( - [ - "--result-dir", - str(_RESULT_DIR), - "--max-seed-tts-mean-wer", - str(_MAX_SEED_TTS_MEAN_WER), - "--seed-tts-locale", - _SEED_TTS_LOCALE, - "--seed-extra-body-json", - json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")), - "--seed-backend", - "openai-realtime-tts", - "--seed-endpoint", - "/v1/realtime", - "--seed-tts-turns-per-session", - "4", - "--trust-remote-code", - ] - ) - - assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0