[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: psv666 <2693925048@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
59 files changed
+1908
-155
No files matched your search
@@ -752,3 +752,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
default=None,
|
||||
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
||||
)
|
||||
parser.addoption(
|
||||
"--assert-baseline",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
|
||||
"(default: off)."
|
||||
),
|
||||
)
|
||||
@@ -157,8 +157,19 @@ def benchmark_params(request):
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, num_prompt) -> None:
|
||||
def assert_result(result, params, num_prompt) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -212,6 +223,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
@@ -251,7 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, num_prompt)
|
||||
assert_result(result, params, num_prompt)
|
||||
|
||||
# concurrency test
|
||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
@@ -268,4 +280,4 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, num_prompt)
|
||||
assert_result(result, params, num_prompt)
|
||||
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.7803,
|
||||
1.1008,
|
||||
1.2027
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
166.9764,
|
||||
307.7133,
|
||||
323.584
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1361.1682,
|
||||
3800.6332,
|
||||
6961.8636
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
635.8099,
|
||||
1615.6591,
|
||||
2888.9831
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.3151,
|
||||
0.8568,
|
||||
1.5425
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,68 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_duplex_seed_tts",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
1
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"no_oversample": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_turns_per_session": 4,
|
||||
"expected_duplex_audio_turns_per_session": 4,
|
||||
"extra_body": {
|
||||
"save_duplex_request_metrics": true
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.1594
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
274.164
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6250.7696
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
994.422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -485,3 +485,30 @@ def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkey
|
||||
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
|
||||
assert "A3" in sweeps[2]["params"]["baseline"]
|
||||
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
|
||||
|
||||
|
||||
def test_omni_duplex_expected_audio_turns_accepts_complete_session():
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
assert_result(
|
||||
{
|
||||
"completed": 1,
|
||||
"duplex_session_metrics": [{"audio_turn_count": 4}],
|
||||
},
|
||||
{"expected_duplex_audio_turns_per_session": 4},
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
def test_omni_duplex_expected_audio_turns_rejects_incomplete_session():
|
||||
from tests.dfx.perf.scripts.run_benchmark import assert_result
|
||||
|
||||
with pytest.raises(AssertionError, match="emitted 4 audio turns"):
|
||||
assert_result(
|
||||
{
|
||||
"completed": 1,
|
||||
"duplex_session_metrics": [{"audio_turn_count": 3}],
|
||||
},
|
||||
{"expected_duplex_audio_turns_per_session": 4},
|
||||
1,
|
||||
)
|
||||
@@ -50,12 +50,12 @@ def _prompt() -> str:
|
||||
return "What is the capital of China? Answer in 20 words."
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
|
||||
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
|
||||
"""
|
||||
Test that invalid audio format is properly rejected.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + invalid audio format
|
||||
Output Modal: error
|
||||
"""
|
||||
|
||||
Reference in new issue
Block a user