anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Browse Source 
Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
Ruirui Yang | Reinpsv666Cursoramy-why-3459
 authored and GitHub committed 2026-08-11 14:53:49 +08:00
1 parent c4a14091c9
commit 322600bfb2
59 files changed
+1908 -155
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/dfx/perf/tests/test_minicpmo_4_5.json
+87
View File
@@ -0,0 +1,87 @@
[
{
"test_name": "test_minicpmo_4_5",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100",
"npu": "A3"
},
"num_cards": 1
}
},
"full_model",
"omni"
],
"server_params": {
"model": "openbmb/MiniCPM-o-4_5",
"extra_cli_args": [
"--deploy-config",
"vllm_omni/deploy/minicpmo_4_5.yaml",
"--trust-remote-code"
]
},
"benchmark_params": [
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
128
],
"max_concurrency": [
1,
4,
8
],
"no_oversample": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.7803,
1.1008,
1.2027
],
"mean_ttft_ms": [
166.9764,
307.7133,
323.584
],
"mean_e2el_ms": [
1361.1682,
3800.6332,
6961.8636
],
"mean_audio_ttfp_ms": [
635.8099,
1615.6591,
2888.9831
],
"mean_audio_rtf": [
0.3151,
0.8568,
1.5425
]
}
}
}
]
}
]
tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+68
View File
@@ -0,0 +1,68 @@
[
{
"test_name": "test_minicpmo_4_5_duplex_seed_tts",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100",
"npu": "A3"
},
"num_cards": 1
}
},
"full_model",
"omni"
],
"server_params": {
"model": "openbmb/MiniCPM-o-4_5",
"extra_cli_args": [
"--deploy-config",
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
"--trust-remote-code"
]
},
"benchmark_params": [
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"num_prompts": [
1
],
"max_concurrency": [
1
],
"no_oversample": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"seed_tts_turns_per_session": 4,
"expected_duplex_audio_turns_per_session": 4,
"extra_body": {
"save_duplex_request_metrics": true
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.1594
],
"mean_ttft_ms": [
274.164
],
"mean_e2el_ms": [
6250.7696
],
"mean_audio_ttfp_ms": [
994.422
],
"mean_audio_rtf": [
0.46163
]
}
}
}
]
}
]
tests/dfx/perf/tests/test_runner_metadata.py
+27
View File
@@ -485,3 +485,30 @@ def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkey
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
assert "A3" in sweeps[2]["params"]["baseline"]
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
def test_omni_duplex_expected_audio_turns_accepts_complete_session():
from tests.dfx.perf.scripts.run_benchmark import assert_result
assert_result(
{
"completed": 1,
"duplex_session_metrics": [{"audio_turn_count": 4}],
},
{"expected_duplex_audio_turns_per_session": 4},
1,
)
def test_omni_duplex_expected_audio_turns_rejects_incomplete_session():
from tests.dfx.perf.scripts.run_benchmark import assert_result
with pytest.raises(AssertionError, match="emitted 4 audio turns"):
assert_result(
{
"completed": 1,
"duplex_session_metrics": [{"audio_turn_count": 3}],
},
{"expected_duplex_audio_turns_per_session": 4},
1,
)