[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-08-11 14:53:49 +08:00
1 parent c4a14091c9
commit 322600bfb2
59 files changed
+1908 -155

No files matched your search

+105 -5
View File
@@ -14,6 +14,8 @@ from pathlib import Path
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from vllm_omni.metrics.definitions import compute_audio_rtf
try:
import websockets
from websockets.exceptions import ConnectionClosed
@@ -46,6 +48,30 @@ def _interval_summary(values: list[float]) -> dict[str, float | int]:
}
def summarize_session_request_metrics(
request_metrics: list[dict[str, object]],
*,
session_id: str | None,
) -> dict[str, object]:
"""Average client-observed metrics across turns that emitted audio."""
def mean(metric: str, *, digits: int = 3) -> float | None:
values = [
float(request[metric])
for request in request_metrics
if isinstance(request.get(metric), int | float) and math.isfinite(float(request[metric]))
]
return round(sum(values) / len(values), digits) if values else None
return {
"session_id": session_id,
"audio_turn_count": len(request_metrics),
"mean_ttft_ms": mean("ttft_ms"),
"mean_ttfp_ms": mean("ttfp_ms"),
"mean_rtf": mean("rtf", digits=6),
}
def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None:
candidates: list[object] = [event.get("vllm_omni")]
metadata = event.get("metadata")
@@ -205,10 +231,12 @@ class RealtimeEventCollector:
after_s: float,
input_committed_at_s: float | None = None,
response_id: str | None = None,
measurement_origin: dict[str, str] | None = None,
) -> dict[str, object]:
"""Summarize engine token metrics and client-observed audio cadence."""
stage0_metrics: dict[str, object] | None = None
response_created_at_s: float | None = None
first_text_received_at_s: float | None = None
audio_received_at_s: list[float] = []
cumulative_audio_ms: list[float] = []
for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True):
@@ -219,6 +247,18 @@ class RealtimeEventCollector:
continue
if event.get("type") == "response.created" and response_created_at_s is None:
response_created_at_s = received_at_s
if (
event.get("type")
in {
"response.audio_transcript.delta",
"response.output_text.delta",
"response.text.delta",
}
and isinstance(event.get("delta"), str)
and bool(event["delta"])
and first_text_received_at_s is None
):
first_text_received_at_s = received_at_s
stage_metrics = _event_stage_metrics(event)
stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None
@@ -283,6 +323,45 @@ class RealtimeEventCollector:
"chunk_duration_ms": _interval_summary(chunk_durations_ms),
"max_chunk_gap_ms": interval_summary["max"],
}
request_started_at_s = input_committed_at_s if input_committed_at_s is not None else response_created_at_s
if request_started_at_s is not None:
audio_duration_ms = (
max(cumulative_audio_ms)
if cumulative_audio_ms
else len(self.audio_bytes(response_id))
* 1000.0
/ (self.output_sample_rate_hz * PCM16_BYTES_PER_SAMPLE)
)
audio_generation_ms = max(
0.0,
(audio_received_at_s[-1] - request_started_at_s) * 1000.0,
)
result["request_metrics"] = {
"source": "client_monotonic_receive",
"measurement_origin": measurement_origin
or {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
"ttft_ms": (
_rounded_ms((first_text_received_at_s - request_started_at_s) * 1000.0)
if first_text_received_at_s is not None
else None
),
"ttfp_ms": _rounded_ms((audio_received_at_s[0] - request_started_at_s) * 1000.0),
"rtf": round(
compute_audio_rtf(
audio_generation_ms / 1000.0,
audio_duration_ms / 1000.0,
),
6,
)
if audio_duration_ms > 0
else None,
"audio_generation_ms": _rounded_ms(audio_generation_ms),
"audio_duration_ms": _rounded_ms(audio_duration_ms),
}
return result
@@ -332,9 +411,26 @@ class RealtimeDuplexClient:
*,
output_audio_format: str = "pcm16",
ref_audio: str | None = None,
instructions: str | None = None,
initial_user_text: str | None = None,
native_duplex: bool = True,
auto_response: bool = True,
temperature: float | None = None,
extra_body: dict[str, object] | None = None,
session_id: str | None = None,
timeout_s: float = 20.0,
) -> None:
session_extra_body = dict(extra_body or {})
if native_duplex:
session_extra_body.update(
{
"auto_response": auto_response,
"minicpmo45_native_duplex": True,
"force_listen_count": 0,
}
)
else:
session_extra_body["minicpmo45_native_duplex"] = False
session: dict[str, object] = {
"model": model,
"modalities": ["audio", "text"],
@@ -343,14 +439,18 @@ class RealtimeDuplexClient:
"turn_detection": None,
"overlap_policy": "listen_only",
"playback_commit_policy": "ack_only",
"extra_body": {
"auto_response": True,
"minicpmo45_native_duplex": True,
"force_listen_count": 0,
},
"extra_body": session_extra_body,
}
if temperature is not None:
session["temperature"] = float(temperature)
if ref_audio is not None:
session["ref_audio"] = ref_audio
if instructions is not None:
session["instructions"] = instructions
if initial_user_text is not None:
extra_body = session["extra_body"]
assert isinstance(extra_body, dict)
extra_body["duplex_initial_user_text"] = initial_user_text
if session_id:
session["session_id"] = session_id
await self.send({"type": "session.update", "session": session})