anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Browse Source 
Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
Ruirui Yang | Reinpsv666Cursoramy-why-3459
 authored and GitHub committed 2026-08-11 14:53:49 +08:00
1 parent c4a14091c9
commit 322600bfb2
59 files changed
+1908 -155
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vllm_omni/benchmarks/patch/patch.py
+227 -3
View File
@@ -9,11 +9,13 @@ import ssl
import sys
import time
import traceback
import uuid
import wave
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Literal
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import aiohttp
import numpy as np
@@ -47,6 +49,11 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
)
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
from vllm_omni.experimental.fullduplex.client import (
RealtimeDuplexClient,
summarize_session_request_metrics,
wait_for,
)
from vllm_omni.metrics import definitions as defs
from vllm_omni.metrics.utils import coerce_positive_int_scalar
@@ -188,6 +195,9 @@ def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFu
# Mark for PCM capture (WER / UTMOS eval) regardless of extra body presence.
setattr(rfi, "seed_tts_row", True)
sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
setattr(rfi, "seed_tts_system_prompt", sys_prompt)
setattr(rfi, "seed_tts_speech_extra", sample.seed_tts_speech_extra)
setattr(rfi, "seed_tts_turns", sample.seed_tts_turns)
setattr(
rfi,
"omni_chat_messages",
@@ -235,7 +245,12 @@ def get_samples(args, tokenizer):
)
# Check if we need to handle omni-related backends/datasets
is_omni_backend = args.backend in ["openai-chat-omni", "openai-audio-speech", "daily-omni"]
is_omni_backend = args.backend in [
"openai-chat-omni",
"openai-audio-speech",
"openai-realtime-duplex",
"daily-omni",
]
is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
if not is_omni_backend and not is_omni_dataset:
@@ -334,10 +349,17 @@ def get_samples(args, tokenizer):
return input_requests
if is_seed_tts:
if args.backend not in ("openai-audio-speech", "openai-chat-omni"):
if args.backend not in (
"openai-audio-speech",
"openai-chat-omni",
"openai-realtime-duplex",
"openai-realtime-tts",
):
raise ValueError(
"Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text). "
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text), or "
"--backend openai-realtime-duplex or openai-realtime-tts "
"(WebSocket /v1/realtime). "
f"Got backend={args.backend!r}."
)
repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
@@ -346,6 +368,14 @@ def get_samples(args, tokenizer):
"Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
"--hf-name for the Hub dataset id."
)
turns_per_session = int(getattr(args, "seed_tts_turns_per_session", 1))
if turns_per_session > 1 and args.backend not in {
"openai-realtime-duplex",
"openai-realtime-tts",
}:
raise ValueError(
f"--seed-tts-turns-per-session > 1 requires a Realtime Seed-TTS backend. Got backend={args.backend!r}."
)
_cls_map = {
"seed-tts": SeedTTSDataset,
@@ -375,6 +405,7 @@ def get_samples(args, tokenizer):
output_len=out_len,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
turns_per_session=turns_per_session,
)
# Handle random-mm dataset (Omni's synthetic multimodal dataset)
@@ -430,10 +461,14 @@ class MixRequestFuncOutput(RequestFuncOutput):
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
#: resampled export after ``openai-chat-omni`` audio deltas.
tts_output_pcm_bytes: bytes | None = None
#: Per-turn 24 kHz mono PCM for grouped Realtime Seed-TTS WER.
tts_turn_pcm_bytes: list[bytes] | None = None
#: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks).
stage_metrics: dict[str, dict] | None = None
stage_id: int | None = None
final_output_type: str | None = None
duplex_request_metrics: list[dict[str, object]] | None = None
duplex_session_metrics: dict[str, object] | None = None
_IMAGE_EDITS_EXTRA_BODY_FORM_FIELDS = (
@@ -1249,6 +1284,180 @@ async def async_request_openai_audio_speech(
return output
def _realtime_websocket_url(api_url: str) -> str:
parts = urlsplit(api_url)
scheme = "wss" if parts.scheme == "https" else "ws"
query = dict(parse_qsl(parts.query, keep_blank_values=True))
query.setdefault("duplex", "1")
return urlunsplit((scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
async def async_request_openai_realtime_duplex(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
) -> MixRequestFuncOutput:
"""Run one or more Seed-TTS turns through one explicit Realtime TTS session."""
del session
output = MixRequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output.start_time = time.perf_counter()
speech_extra = getattr(request_func_input, "seed_tts_speech_extra", None)
if not isinstance(speech_extra, dict):
speech_extra = {}
configured_turns = getattr(request_func_input, "seed_tts_turns", ())
turn_prompts = [
(
str(getattr(turn, "utterance_id", "") or ""),
str(getattr(turn, "target_text", "") or ""),
)
for turn in configured_turns
if str(getattr(turn, "target_text", "") or "").strip()
]
if not turn_prompts:
turn_prompts = [("", request_func_input.prompt)]
session_id = f"seed-tts-{request_func_input.request_id or uuid.uuid4().hex}"
try:
async with RealtimeDuplexClient(_realtime_websocket_url(request_func_input.api_url)) as client:
await client.configure(
request_func_input.model_name or request_func_input.model,
output_audio_format="pcm16",
instructions=getattr(
request_func_input,
"seed_tts_system_prompt",
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
),
native_duplex=False,
auto_response=False,
extra_body=speech_extra,
session_id=session_id,
timeout_s=120.0,
)
turn_metrics: list[dict[str, object]] = []
turn_timings: list[dict[str, object]] = []
turn_pcm_bytes: list[bytes] = []
turn_transcripts: list[str] = []
measurement_origin = {
"ttft": "conversation.item.create client send to first non-empty text delta",
"ttfp": "conversation.item.create client send to first audio packet",
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
}
for request_index, (utterance_id, target_text) in enumerate(turn_prompts):
response_offset = len(client.events.response_ids)
done_before = client.events.count("response.done")
errors_before = len(client.events.errors())
turn_started_at_s = time.monotonic()
await client.send(
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": target_text}],
},
}
)
await client.send({"type": "response.create"})
await wait_for(
lambda: client.events.count("response.done") > done_before
or len(client.events.errors()) > errors_before,
timeout_s=180.0,
label=f"Seed-TTS Realtime TTS turn {request_index} response.done",
)
errors = client.events.errors()
if len(errors) > errors_before:
raise RuntimeError(f"Seed-TTS Realtime TTS server error: {errors[-1]}")
new_audio_response_ids = [
response_id
for response_id in client.events.response_ids[response_offset:]
if client.events.audio_bytes(response_id)
]
if len(new_audio_response_ids) != 1:
raise RuntimeError(
f"Seed-TTS Realtime TTS turn {request_index} expected one audio response, "
f"got {len(new_audio_response_ids)}"
)
response_id = new_audio_response_ids[0]
timing = client.events.timing_summary(
after_s=turn_started_at_s,
input_committed_at_s=turn_started_at_s,
response_id=response_id,
measurement_origin=measurement_origin,
)
request_metrics = timing.get("request_metrics")
if not isinstance(request_metrics, dict):
raise RuntimeError(f"Seed-TTS duplex audio turn {response_id} omitted per-request metrics")
turn_timings.append(timing)
turn_metrics.append(
{
"session_id": session_id,
"request_index": request_index,
"utterance_id": utterance_id or None,
"response_id": response_id,
**request_metrics,
}
)
response_audio = client.events.audio_bytes(response_id)
turn_pcm_bytes.append(
_pcm_s16le_to_seed_tts_wer_bytes(
response_audio,
sample_rate=client.events.output_sample_rate_hz,
channels=1,
)
)
turn_transcripts.append(
"".join(
str(event.get("delta") or "")
for event in client.events.events
if client.events.response_id(event) == response_id
and event.get("type")
in {
"response.audio_transcript.delta",
"response.output_text.delta",
"response.text.delta",
}
)
)
await client.acknowledge_playback()
request_finished_at = time.perf_counter()
session_metrics = summarize_session_request_metrics(
turn_metrics,
session_id=session_id,
)
await client.close_session(timeout_s=30.0)
output.generated_text = " ".join(filter(None, turn_transcripts))
output.ttft = float(session_metrics.get("mean_ttft_ms") or 0.0) / 1000.0
output.audio_ttfp = float(session_metrics.get("mean_ttfp_ms") or 0.0) / 1000.0
output.audio_rtf = float(session_metrics.get("mean_rtf") or 0.0)
output.audio_duration = (
sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
)
output.audio_frames = int(output.audio_duration * client.events.output_sample_rate_hz)
output.latency = request_finished_at - output.start_time
output.tts_turn_pcm_bytes = turn_pcm_bytes
output.tts_output_pcm_bytes = b"".join(turn_pcm_bytes)
if bool((request_func_input.extra_body or {}).get("save_duplex_request_metrics")):
output.duplex_request_metrics = turn_metrics
output.duplex_session_metrics = session_metrics
output.output_tokens = sum(
int(stage0.get("output_token_count") or 0)
for timing in turn_timings
if isinstance((stage0 := timing.get("stage0_tokens")), dict)
)
output.success = True
except Exception:
output.success = False
output.error = traceback.format_exc()
logger.error(
"Seed-TTS Realtime TTS request failed: %s",
output.error,
)
if pbar:
pbar.update(1)
return output
ASYNC_REQUEST_FUNCS["openai-chat-omni"] = async_request_openai_chat_omni_completions
if "openai-chat-omni" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-chat-omni")
@@ -1257,6 +1466,13 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
ASYNC_REQUEST_FUNCS["openai-realtime-duplex"] = async_request_openai_realtime_duplex
if "openai-realtime-duplex" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-realtime-duplex")
ASYNC_REQUEST_FUNCS["openai-realtime-tts"] = async_request_openai_realtime_duplex
if "openai-realtime-tts" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-realtime-tts")
ASYNC_REQUEST_FUNCS["openai-image-edits-omni"] = async_request_openai_image_edits_omni
if "openai-image-edits-omni" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-image-edits-omni")
@@ -1603,6 +1819,14 @@ async def benchmark(
"input_lens": [output.prompt_len for output in outputs],
"errors": [output.error for output in outputs],
}
duplex_request_metrics = [metric for output in outputs for metric in (output.duplex_request_metrics or [])]
if duplex_request_metrics:
result["duplex_request_metrics"] = duplex_request_metrics
duplex_session_metrics = [
output.duplex_session_metrics for output in outputs if output.duplex_session_metrics is not None
]
if duplex_session_metrics:
result["duplex_session_metrics"] = duplex_session_metrics
from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
compute_daily_omni_accuracy_metrics,