[CI/Build][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance nightly coverage (#5710)
Signed-off-by: psv666 <2693925048@qq.com> Signed-off-by: psv666 <150513104+psv666@users.noreply.github.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
a5eb2a9a87
commit
4ce5fcb16c
15 files changed
+428
-108
No files matched your search
@@ -39,10 +39,7 @@ _MAX_SEED_TTS_MEAN_WER = 0.05
|
||||
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||
_DAILY_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
_SEED_EXTRA_BODY = {
|
||||
"modalities": ["text", "audio"],
|
||||
@@ -151,6 +148,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
max_concurrency=4,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
|
||||
@@ -963,12 +963,23 @@ async def _send_clean_turn(
|
||||
if outcome == "listen":
|
||||
return None, "listen"
|
||||
else:
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("response.created") > before_created,
|
||||
timeout_s=timeout_s,
|
||||
label=f"{transcript} response.created",
|
||||
)
|
||||
try:
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("response.created") > before_created,
|
||||
timeout_s=timeout_s,
|
||||
label=f"{transcript} response.created",
|
||||
)
|
||||
except TimeoutError:
|
||||
# ``auto_response`` sessions only get ``response.created`` once the model
|
||||
# decides to speak, so report the listen decisions that took its place.
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for {transcript} response.created after "
|
||||
f"{state.count('input_audio_buffer.committed')} committed input turns; "
|
||||
f"model_listen={state.model_listen_count - before_model_listen} "
|
||||
f"buffering_listen={state.buffering_listen_count} "
|
||||
f"errors={state.count('error')}"
|
||||
) from None
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: eligible_response_id() is not None,
|
||||
|
||||
@@ -133,6 +133,10 @@ def test_duplex_single_session_response_required(omni_server, model_prefix: str,
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
args.turns = 2
|
||||
# Every turn replays the same active-speech window as the first one. The default
|
||||
# shorter follow-up window is a different mid-utterance slice, which the native
|
||||
# duplex model may legitimately answer with "listen" instead of a response.
|
||||
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||
result = asyncio.run(run_demo(args))
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
|
||||
@@ -53,7 +53,11 @@ def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_
|
||||
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
# CUDA-only for now: this contract asserts the model speaks while the user is
|
||||
# still talking, which only holds when the duplex pipeline sustains real-time
|
||||
# throughput. The current NPU stack runs several times slower than real time,
|
||||
# so it never reaches a mid-stream decision point.
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
input_wav = validated_soft_interrupt_wav()
|
||||
|
||||
Reference in new issue
Block a user