[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: psv666 <2693925048@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
59 files changed
+1908
-155
No files matched your search
@@ -14,6 +14,8 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from vllm_omni.metrics.definitions import compute_audio_rtf
|
||||
|
||||
try:
|
||||
import websockets
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
@@ -46,6 +48,30 @@ def _interval_summary(values: list[float]) -> dict[str, float | int]:
|
||||
}
|
||||
|
||||
|
||||
def summarize_session_request_metrics(
|
||||
request_metrics: list[dict[str, object]],
|
||||
*,
|
||||
session_id: str | None,
|
||||
) -> dict[str, object]:
|
||||
"""Average client-observed metrics across turns that emitted audio."""
|
||||
|
||||
def mean(metric: str, *, digits: int = 3) -> float | None:
|
||||
values = [
|
||||
float(request[metric])
|
||||
for request in request_metrics
|
||||
if isinstance(request.get(metric), int | float) and math.isfinite(float(request[metric]))
|
||||
]
|
||||
return round(sum(values) / len(values), digits) if values else None
|
||||
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"audio_turn_count": len(request_metrics),
|
||||
"mean_ttft_ms": mean("ttft_ms"),
|
||||
"mean_ttfp_ms": mean("ttfp_ms"),
|
||||
"mean_rtf": mean("rtf", digits=6),
|
||||
}
|
||||
|
||||
|
||||
def _event_stage_metrics(event: dict[str, object]) -> dict[str, object] | None:
|
||||
candidates: list[object] = [event.get("vllm_omni")]
|
||||
metadata = event.get("metadata")
|
||||
@@ -205,10 +231,12 @@ class RealtimeEventCollector:
|
||||
after_s: float,
|
||||
input_committed_at_s: float | None = None,
|
||||
response_id: str | None = None,
|
||||
measurement_origin: dict[str, str] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Summarize engine token metrics and client-observed audio cadence."""
|
||||
stage0_metrics: dict[str, object] | None = None
|
||||
response_created_at_s: float | None = None
|
||||
first_text_received_at_s: float | None = None
|
||||
audio_received_at_s: list[float] = []
|
||||
cumulative_audio_ms: list[float] = []
|
||||
for event, received_at_s in zip(self.events, self.event_received_at_s, strict=True):
|
||||
@@ -219,6 +247,18 @@ class RealtimeEventCollector:
|
||||
continue
|
||||
if event.get("type") == "response.created" and response_created_at_s is None:
|
||||
response_created_at_s = received_at_s
|
||||
if (
|
||||
event.get("type")
|
||||
in {
|
||||
"response.audio_transcript.delta",
|
||||
"response.output_text.delta",
|
||||
"response.text.delta",
|
||||
}
|
||||
and isinstance(event.get("delta"), str)
|
||||
and bool(event["delta"])
|
||||
and first_text_received_at_s is None
|
||||
):
|
||||
first_text_received_at_s = received_at_s
|
||||
|
||||
stage_metrics = _event_stage_metrics(event)
|
||||
stage0 = stage_metrics.get("0") if isinstance(stage_metrics, dict) else None
|
||||
@@ -283,6 +323,45 @@ class RealtimeEventCollector:
|
||||
"chunk_duration_ms": _interval_summary(chunk_durations_ms),
|
||||
"max_chunk_gap_ms": interval_summary["max"],
|
||||
}
|
||||
request_started_at_s = input_committed_at_s if input_committed_at_s is not None else response_created_at_s
|
||||
if request_started_at_s is not None:
|
||||
audio_duration_ms = (
|
||||
max(cumulative_audio_ms)
|
||||
if cumulative_audio_ms
|
||||
else len(self.audio_bytes(response_id))
|
||||
* 1000.0
|
||||
/ (self.output_sample_rate_hz * PCM16_BYTES_PER_SAMPLE)
|
||||
)
|
||||
audio_generation_ms = max(
|
||||
0.0,
|
||||
(audio_received_at_s[-1] - request_started_at_s) * 1000.0,
|
||||
)
|
||||
result["request_metrics"] = {
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": measurement_origin
|
||||
or {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
"ttft_ms": (
|
||||
_rounded_ms((first_text_received_at_s - request_started_at_s) * 1000.0)
|
||||
if first_text_received_at_s is not None
|
||||
else None
|
||||
),
|
||||
"ttfp_ms": _rounded_ms((audio_received_at_s[0] - request_started_at_s) * 1000.0),
|
||||
"rtf": round(
|
||||
compute_audio_rtf(
|
||||
audio_generation_ms / 1000.0,
|
||||
audio_duration_ms / 1000.0,
|
||||
),
|
||||
6,
|
||||
)
|
||||
if audio_duration_ms > 0
|
||||
else None,
|
||||
"audio_generation_ms": _rounded_ms(audio_generation_ms),
|
||||
"audio_duration_ms": _rounded_ms(audio_duration_ms),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@@ -332,9 +411,26 @@ class RealtimeDuplexClient:
|
||||
*,
|
||||
output_audio_format: str = "pcm16",
|
||||
ref_audio: str | None = None,
|
||||
instructions: str | None = None,
|
||||
initial_user_text: str | None = None,
|
||||
native_duplex: bool = True,
|
||||
auto_response: bool = True,
|
||||
temperature: float | None = None,
|
||||
extra_body: dict[str, object] | None = None,
|
||||
session_id: str | None = None,
|
||||
timeout_s: float = 20.0,
|
||||
) -> None:
|
||||
session_extra_body = dict(extra_body or {})
|
||||
if native_duplex:
|
||||
session_extra_body.update(
|
||||
{
|
||||
"auto_response": auto_response,
|
||||
"minicpmo45_native_duplex": True,
|
||||
"force_listen_count": 0,
|
||||
}
|
||||
)
|
||||
else:
|
||||
session_extra_body["minicpmo45_native_duplex"] = False
|
||||
session: dict[str, object] = {
|
||||
"model": model,
|
||||
"modalities": ["audio", "text"],
|
||||
@@ -343,14 +439,18 @@ class RealtimeDuplexClient:
|
||||
"turn_detection": None,
|
||||
"overlap_policy": "listen_only",
|
||||
"playback_commit_policy": "ack_only",
|
||||
"extra_body": {
|
||||
"auto_response": True,
|
||||
"minicpmo45_native_duplex": True,
|
||||
"force_listen_count": 0,
|
||||
},
|
||||
"extra_body": session_extra_body,
|
||||
}
|
||||
if temperature is not None:
|
||||
session["temperature"] = float(temperature)
|
||||
if ref_audio is not None:
|
||||
session["ref_audio"] = ref_audio
|
||||
if instructions is not None:
|
||||
session["instructions"] = instructions
|
||||
if initial_user_text is not None:
|
||||
extra_body = session["extra_body"]
|
||||
assert isinstance(extra_body, dict)
|
||||
extra_body["duplex_initial_user_text"] = initial_user_text
|
||||
if session_id:
|
||||
session["session_id"] = session_id
|
||||
await self.send({"type": "session.update", "session": session})
|
||||
|
||||
Reference in new issue
Block a user