[CI][MiniCPM-o]MiniCPM-o ACC PERF CI (#5575)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
434c220da1
commit
a5eb2a9a87
50 files changed
+1756
-102
No files matched your search
@@ -128,6 +128,17 @@ def assert_result(
|
||||
request_rate: Any | None = None,
|
||||
) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
if not assert_baseline:
|
||||
return
|
||||
baseline_data = params.get("baseline", {})
|
||||
@@ -140,13 +151,13 @@ def assert_result(
|
||||
request_rate=request_rate,
|
||||
)
|
||||
if "throughput" in metric_name:
|
||||
if current_value <= baseline_value:
|
||||
print(
|
||||
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
assert current_value >= baseline_value, (
|
||||
f"Throughput test result was below baseline: {metric_name}: {current_value} < {baseline_value}"
|
||||
)
|
||||
else:
|
||||
if current_value >= baseline_value:
|
||||
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
|
||||
assert current_value <= baseline_value, (
|
||||
f"Test result exceeded baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -201,6 +212,8 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
"baseline_hardware",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
|
||||
Reference in new issue
Block a user