[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: psv666 <2693925048@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
59 files changed
+1908
-155
No files matched your search
@@ -157,8 +157,19 @@ def benchmark_params(request):
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, num_prompt) -> None:
|
||||
def assert_result(result, params, num_prompt) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -212,6 +223,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
@@ -251,7 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, num_prompt)
|
||||
assert_result(result, params, num_prompt)
|
||||
|
||||
# concurrency test
|
||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
@@ -268,4 +280,4 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, num_prompt)
|
||||
assert_result(result, params, num_prompt)
|
||||
Reference in new issue
Block a user