diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index ab9f0e060..49a0aca2f 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -41,6 +41,7 @@ steps: # timeout_in_minutes: 180 # commands: # - export SEED_TTS_WER_EVAL=1 + # - export SEED_TTS_SIM_EVAL=1 # - export SEED_TTS_EVAL_DEVICE=cuda:1 # - | # set +e @@ -54,6 +55,7 @@ steps: timeout_in_minutes: 180 commands: - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_SIM_EVAL=1 - export SEED_TTS_EVAL_DEVICE=cuda:1 - | set +e diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index f66631184..a602aba59 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -84,6 +84,7 @@ steps: HF_TOKEN: "${HF_TOKEN}" commands: - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_SIM_EVAL=1 - export SEED_TTS_EVAL_DEVICE=npu:1 # Video-MME needs no repo override here: the NPU runners pre-stage the dataset under # the lmms-eval mirror, which is now VIDEOMME_DEFAULT_HF_REPO, so snapshot_download diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index 40cac6ee3..010763f35 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "jiwer>=4.0.0" \ "zhon>=2.1.1" \ "zhconv>=1.4.3" \ + "funasr>=1.3.0" \ "datasets>=3.0.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 4682643c1..29ad35d9a 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "jiwer>=4.0.0" \ "zhon>=2.1.1" \ "zhconv>=1.4.3" \ + "funasr>=1.3.0" \ "datasets>=3.0.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index 0c942e9c3..ce14f19fe 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -42,7 +42,10 @@ _RESULT_DIR = Path( _MIN_DAILY_OMNI_ACCURACY = 0.78 # MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni. _MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68")) -_MAX_SEED_TTS_MEAN_WER = 0.05 +_MAX_SEED_TTS_MEAN_WER = 0.02 +# Full Seed-TTS zh split (seed-tts-eval/zh/meta.lst) is 2020 rows. +_SEED_TTS_LOCALE = "zh" +_SEED_TTS_NUM_PROMPTS = 2020 # Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts. _DAILY_EXTRA_BODY = { "modalities": ["text"], @@ -270,6 +273,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: skip_seed=False, skip_daily=True, skip_videomme=True, + num_prompts=_SEED_TTS_NUM_PROMPTS, max_concurrency=4, ) argv.extend( @@ -278,6 +282,8 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: str(_RESULT_DIR), "--max-seed-tts-mean-wer", str(_MAX_SEED_TTS_MEAN_WER), + "--seed-tts-locale", + _SEED_TTS_LOCALE, "--seed-extra-body-json", json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")), "--trust-remote-code", @@ -308,6 +314,8 @@ def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None: str(_RESULT_DIR), "--max-seed-tts-mean-wer", str(_MAX_SEED_TTS_MEAN_WER), + "--seed-tts-locale", + _SEED_TTS_LOCALE, "--seed-extra-body-json", json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")), "--seed-backend",