[CI/Build][MiniCPM-o] Gate Seed-TTS WER over chat completions instead of the native duplex path (#7963)
Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Fable 5.1 authored and GitHub committed
1 parent
205aa01c7b
commit
21f167eacd
1 file changed
+24
-49
@@ -6,6 +6,13 @@ Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
|
||||
~77% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
|
||||
text modalities, and server ``--interleave-mm-strings`` + 1fps / 128-frame
|
||||
media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
|
||||
|
||||
Seed-TTS WER is gated over ``/v1/chat/completions`` (whole-utterance TTS) on
|
||||
the duplex deploy config, once with the stock Code2Wav and once with its
|
||||
bfloat16 attention cache. The native per-unit full-duplex path is not WER
|
||||
gated here: the checkpoint's own per-unit boundaries split words across
|
||||
Talker conditions, so that path scores ~0.075 on this benchmark (the HF
|
||||
``MiniCPMODuplex`` reference scores ~0.14) and cannot meet the 0.05 gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -22,20 +29,15 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_acc_benchmark_cli_argv,
|
||||
find_vllm_cli,
|
||||
)
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
DEPLOY_CONFIG as _DUPLEX_DEPLOY_CONFIG,
|
||||
)
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS as DUPLEX_TEST_PARAMS,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
|
||||
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_DUPLEX_BF16_DEPLOY_CONFIG = modify_stage_config(
|
||||
_DUPLEX_DEPLOY_CONFIG,
|
||||
# CUDA only: the NPU Code2Wav rejects ``code2wav_bfloat16_attention_cache``.
|
||||
_BF16_ATTENTION_CACHE_DEPLOY_CONFIG = modify_stage_config(
|
||||
_DEPLOY_CONFIG,
|
||||
updates={
|
||||
"base_config": _DEPLOY_CONFIG,
|
||||
"connectors.connector_of_shared_memory.extra.code2wav_bfloat16_attention_cache": True,
|
||||
@@ -108,17 +110,14 @@ seed_test_params = [
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
duplex_accuracy_test_params = [
|
||||
*DUPLEX_TEST_PARAMS,
|
||||
seed_bf16_attention_cache_test_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DUPLEX_BF16_DEPLOY_CONFIG,
|
||||
use_stage_cli=False,
|
||||
stage_config_path=_BF16_ATTENTION_CACHE_DEPLOY_CONFIG,
|
||||
server_args=list(_SEED_TTS_SERVER_ARGS),
|
||||
),
|
||||
id="three-stage-single-gpu-bf16-attention-cache",
|
||||
id="bf16-attention-cache",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -179,9 +178,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
def _run_seed_tts_wer_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
@@ -205,36 +202,14 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
_run_seed_tts_wer_bench(omni_server)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", duplex_accuracy_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
|
||||
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
num_prompts=50,
|
||||
max_concurrency=1,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-extra-body-json",
|
||||
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
|
||||
"--seed-backend",
|
||||
"openai-realtime-tts",
|
||||
"--seed-endpoint",
|
||||
"/v1/realtime",
|
||||
"--seed-tts-turns-per-session",
|
||||
"4",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
@pytest.mark.parametrize("omni_server", seed_bf16_attention_cache_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench_bf16_attention_cache(omni_server) -> None:
|
||||
"""Same Seed-TTS gate with the Code2Wav bfloat16 attention cache enabled."""
|
||||
_run_seed_tts_wer_bench(omni_server)
|
||||
Reference in new issue
Block a user