[CI][MiniCPM-o]MiniCPM-o ACC PERF CI (#5575)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
ruirui(rein) yangandamy-why-3459 authored and GitHub committed 2026-08-04 18:46:20 +08:00
1 parent 434c220da1
commit a5eb2a9a87
50 files changed
+1756 -102

No files matched your search

+159 -1
View File
@@ -6,13 +6,21 @@ Unit tests for patch.py
"""
import asyncio
import base64
import json
import time
from types import SimpleNamespace
import pytest
from pytest_mock import MockerFixture
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
from vllm_omni.benchmarks.patch.patch import (
MixRequestFuncOutput,
async_request_openai_chat_omni_completions,
async_request_openai_realtime_duplex,
)
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
@@ -40,6 +48,156 @@ class MockResponse:
pass
@pytest.mark.asyncio
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
class FakeRealtimeClient:
last_instance = None
def __init__(self, url):
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
self.events = RealtimeEventCollector()
self.configure_kwargs = None
self.sent = []
self.response_count = 0
self.ack_count = 0
type(self).last_instance = self
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return None
async def configure(self, model, **kwargs):
assert model == "openbmb/MiniCPM-o-4_5"
self.configure_kwargs = kwargs
async def send(self, event):
self.sent.append(event)
if event["type"] != "response.create":
return
self.response_count += 1
response_id = f"resp-{self.response_count}"
now = time.monotonic()
self.events.add(
{"type": "response.created", "response": {"id": response_id}},
received_at_s=now + 0.01,
)
self.events.add(
{
"type": "response.audio_transcript.delta",
"response_id": response_id,
"delta": f"turn {self.response_count}",
},
received_at_s=now + 0.02,
)
self.events.add(
{
"type": "response.audio.delta",
"response_id": response_id,
"delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
"sample_rate_hz": 24_000,
"metadata": {"audio_duration_ms": 100},
},
received_at_s=now + 0.03,
)
self.events.add(
{"type": "response.done", "response": {"id": response_id}},
received_at_s=now + 0.04,
)
async def acknowledge_playback(self):
self.ack_count += 1
async def close_session(self, **_kwargs):
return None
monkeypatch.setattr(
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
FakeRealtimeClient,
)
request_input = RequestFuncInput(
model="openbmb/MiniCPM-o-4_5",
model_name="openbmb/MiniCPM-o-4_5",
prompt="hello",
api_url="http://localhost:8000/v1/realtime",
prompt_len=1,
output_len=20,
logprobs=None,
multi_modal_content=None,
ignore_eos=False,
extra_body={"save_duplex_request_metrics": True},
)
request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
request_input.seed_tts_system_prompt = "Speak exactly."
request_input.seed_tts_turns = tuple(
SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
)
output = await async_request_openai_realtime_duplex(
request_input,
session=None,
)
client = FakeRealtimeClient.last_instance
assert client.configure_kwargs["native_duplex"] is False
assert client.configure_kwargs["extra_body"] == {
"ref_audio": "data:audio/wav;base64,AAAA",
}
assert client.sent == [
event
for index in range(4)
for event in (
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": f"text {index}"}],
},
},
{"type": "response.create"},
)
]
assert client.ack_count == 4
assert output.success is True
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
assert output.audio_duration == pytest.approx(0.4)
assert output.ttft > 0
assert output.audio_ttfp > output.ttft
assert output.audio_rtf > 0
assert output.latency > 0
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
session_id = output.duplex_request_metrics[0]["session_id"]
assert len(output.duplex_request_metrics) == 4
for index, metrics in enumerate(output.duplex_request_metrics):
assert metrics == {
"session_id": session_id,
"request_index": index,
"utterance_id": f"utt-{index}",
"response_id": f"resp-{index + 1}",
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "conversation.item.create client send to first non-empty text delta",
"ttfp": "conversation.item.create client send to first audio packet",
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
},
"ttft_ms": pytest.approx(20.0, abs=2.0),
"ttfp_ms": pytest.approx(30.0, abs=2.0),
"rtf": pytest.approx(0.3, abs=0.03),
"audio_generation_ms": pytest.approx(30.0, abs=2.0),
"audio_duration_ms": 100.0,
}
assert output.duplex_session_metrics == {
"session_id": session_id,
"audio_turn_count": 4,
"mean_ttft_ms": pytest.approx(20.0, abs=2.0),
"mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
"mean_rtf": pytest.approx(0.3, abs=0.03),
}
def create_sse_chunk(data_dict):
"""Helper to create SSE formatted chunk"""
return f"data: {json.dumps(data_dict)}\n\n".encode()
+23 -1
View File
@@ -42,7 +42,10 @@ from benchmarks.accuracy.text_to_image.gbench import (
summarize_generated_records as summarize_gebench_generated_records,
summarize_gebench_results,
)
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
build_serve_common_argv,
seed_tts_bench_argv,
)
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
@@ -57,6 +60,25 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
argv = build_serve_common_argv(
host="127.0.0.1",
port=8000,
model="openbmb/MiniCPM-o-4_5",
num_prompts=4,
max_concurrency=1,
num_warmups=0,
percentile_metrics="ttft,audio_ttfp,audio_rtf",
result_dir=tmp_path,
result_filename="result.json",
backend="openai-realtime-tts",
endpoint="/v1/realtime",
)
assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
ns = argparse.Namespace(
daily_omni_repo=None,
@@ -28,10 +28,13 @@ if _MODULE_NAME not in sys.modules:
_spec.loader.exec_module(_mod)
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
SeedTTSDataset,
SeedTTSDesignDataset,
SeedTTSDesignSampleRequest,
SeedTTSSampleRequest,
SeedTTSTextDataset,
SeedTTSTextSampleRequest,
SeedTTSTurn,
)
# ---------------------------------------------------------------------------
@@ -70,6 +73,94 @@ def mock_tokenizer(mocker):
# ---------------------------------------------------------------------------
def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
ds = SeedTTSDataset(
dataset_path=str(seed_tts_root),
random_seed=0,
locale="en",
disable_shuffle=True,
)
requests = ds.sample(
mock_tokenizer,
num_requests=1,
turns_per_session=4,
no_oversample=True,
)
assert len(requests) == 1
request = requests[0]
assert [turn.utterance_id for turn in request.seed_tts_turns] == [
"utt000",
"utt001",
"utt002",
"utt003",
]
assert [turn.target_text for turn in request.seed_tts_turns] == [
"target text 0",
"target text 1",
"target text 2",
"target text 3",
]
assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
def test_seed_tts_eval_expands_grouped_pcm_by_turn():
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
_expand_seed_tts_turn_outputs,
)
request = SeedTTSSampleRequest(
prompt="target 0",
prompt_len=4,
expected_output_len=100,
multi_modal_data=None,
request_id="session-0",
seed_tts_turns=tuple(
SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
),
)
output = types.SimpleNamespace(
success=True,
tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
)
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
"""Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
_expand_seed_tts_turn_outputs,
)
request = SeedTTSSampleRequest(
prompt="target 0",
prompt_len=4,
expected_output_len=100,
multi_modal_data=None,
request_id="session-0",
seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
)
pcm = b"\x01\x02\x03\x04"
output = types.SimpleNamespace(
success=True,
tts_output_pcm_bytes=pcm,
tts_turn_pcm_bytes=None,
)
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
assert len(requests) == 1
assert requests[0].prompt == "target 0"
assert outputs[0].tts_output_pcm_bytes == pcm
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
ds = SeedTTSTextDataset(
dataset_path=str(seed_tts_root),
+19 -6
View File
@@ -128,6 +128,17 @@ def assert_result(
request_rate: Any | None = None,
) -> None:
assert result["completed"] == num_prompt, "Request failures exist"
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
if expected_audio_turns is not None:
session_metrics = result.get("duplex_session_metrics")
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
assert len(session_metrics) == num_prompt, (
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
)
assert all(
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
for metric in session_metrics
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
if not assert_baseline:
return
baseline_data = params.get("baseline", {})
@@ -140,13 +151,13 @@ def assert_result(
request_rate=request_rate,
)
if "throughput" in metric_name:
if current_value <= baseline_value:
print(
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
)
assert current_value >= baseline_value, (
f"Throughput test result was below baseline: {metric_name}: {current_value} < {baseline_value}"
)
else:
if current_value >= baseline_value:
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
assert current_value <= baseline_value, (
f"Test result exceeded baseline: {metric_name}: {current_value} > {baseline_value}"
)
@pytest.mark.benchmark
@@ -201,6 +212,8 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
"enabled",
"eval_phase",
"trust_remote_code",
"expected_duplex_audio_turns_per_session",
"baseline_hardware",
}
for key, value in params.items():
@@ -0,0 +1,91 @@
[
{
"test_name": "test_minicpmo_4_5",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 1
}
},
"full_model",
"omni"
],
"server_params": {
"model": "openbmb/MiniCPM-o-4_5",
"extra_cli_args": [
"--deploy-config",
"vllm_omni/deploy/minicpmo_4_5.yaml",
"--trust-remote-code"
]
},
"benchmark_params": [
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
128,
256
],
"max_concurrency": [
1,
4,
8,
16
],
"no_oversample": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"request_throughput": [
0.7803,
1.1008,
1.2027,
1.3254
],
"mean_ttft_ms": [
166.9764,
307.7133,
323.584,
356.6077
],
"mean_e2el_ms": [
1361.1682,
3800.6332,
6961.8636,
12609.4295
],
"mean_audio_ttfp_ms": [
635.8099,
1615.6591,
2888.9831,
8320.5422
],
"mean_audio_rtf": [
0.3151,
0.8568,
1.5425,
2.8224
]
}
}
]
}
]
@@ -0,0 +1,65 @@
[
{
"test_name": "test_minicpmo_4_5_duplex_seed_tts",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 1
}
},
"full_model",
"omni"
],
"server_params": {
"model": "openbmb/MiniCPM-o-4_5",
"extra_cli_args": [
"--deploy-config",
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
"--trust-remote-code"
]
},
"benchmark_params": [
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"num_prompts": [
1
],
"max_concurrency": [
1
],
"no_oversample": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"seed_tts_turns_per_session": 4,
"expected_duplex_audio_turns_per_session": 4,
"extra_body": {
"save_duplex_request_metrics": true
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"request_throughput": [
0.2415
],
"mean_ttft_ms": [
122.6731
],
"mean_e2el_ms": [
4073.804
],
"mean_audio_ttfp_ms": [
523.4768
],
"mean_audio_rtf": [
0.28295
]
}
}
]
}
]
@@ -285,3 +285,69 @@ def test_create_paired_omni_benchmark_pytest_params(tmp_path):
assert bench_row == ("test_tts", 0)
assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)
@pytest.mark.parametrize(
("metric", "current", "baseline"),
[
("request_throughput", 1.0, 1.0),
("mean_ttft_ms", 100.0, 100.0),
],
)
def test_omni_baseline_accepts_values_at_threshold(metric, current, baseline):
from tests.dfx.perf.scripts.run_benchmark import assert_result
assert_result(
{"completed": 1, metric: current},
{"baseline": {metric: baseline}},
1,
assert_baseline=True,
)
@pytest.mark.parametrize(
("metric", "current", "baseline", "message"),
[
("request_throughput", 0.9, 1.0, "below baseline"),
("mean_ttft_ms", 101.0, 100.0, "exceeded baseline"),
],
)
def test_omni_baseline_rejects_regressions(metric, current, baseline, message):
from tests.dfx.perf.scripts.run_benchmark import assert_result
with pytest.raises(AssertionError, match=message):
assert_result(
{"completed": 1, metric: current},
{"baseline": {metric: baseline}},
1,
assert_baseline=True,
)
def test_omni_duplex_expected_audio_turns_accepts_complete_session():
from tests.dfx.perf.scripts.run_benchmark import assert_result
assert_result(
{
"completed": 1,
"duplex_session_metrics": [{"audio_turn_count": 4}],
},
{"expected_duplex_audio_turns_per_session": 4},
1,
assert_baseline=False,
)
def test_omni_duplex_expected_audio_turns_rejects_incomplete_session():
from tests.dfx.perf.scripts.run_benchmark import assert_result
with pytest.raises(AssertionError, match="emitted 4 audio turns"):
assert_result(
{
"completed": 1,
"duplex_session_metrics": [{"audio_turn_count": 3}],
},
{"expected_duplex_audio_turns_per_session": 4},
1,
assert_baseline=False,
)
@@ -48,12 +48,12 @@ def _prompt() -> str:
return "What is the capital of China? Answer in 20 words."
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
"""
Test that invalid audio format is properly rejected.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + invalid audio format
Output Modal: error
"""
@@ -0,0 +1 @@
# SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
build_acc_benchmark_cli_argv,
find_vllm_cli,
)
from tests.helpers.mark import hardware_test
from tests.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
_RESULT_DIR = Path(
os.environ.get(
"ACC_BENCH_RESULT_DIR",
str(Path(__file__).resolve().parent / "results"),
)
)
_MIN_DAILY_OMNI_ACCURACY = 0.78
_MAX_SEED_TTS_MEAN_WER = 0.05
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
_DAILY_EXTRA_BODY = {
"modalities": ["text"],
"chat_template_kwargs": {
"enable_thinking": False,
"use_tts_template": True,
},
}
_SEED_EXTRA_BODY = {
"modalities": ["text", "audio"],
"chat_template_kwargs": {
"enable_thinking": False,
"use_tts_template": True,
},
}
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
# msgspec ValidationError and kill the orchestrator mid-suite.
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
_DAILY_OMNI_SERVER_ARGS = [
"--trust-remote-code",
"--interleave-mm-strings",
"--media-io-kwargs",
'{"video":{"fps":1,"num_frames":128}}',
]
_SEED_TTS_SERVER_ARGS = [
"--trust-remote-code",
]
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
daily_test_params = [
OmniServerParams(
model=_MODEL,
stage_config_path=_DEPLOY_CONFIG,
server_args=list(_DAILY_OMNI_SERVER_ARGS),
)
]
seed_test_params = [
OmniServerParams(
model=_MODEL,
stage_config_path=_DEPLOY_CONFIG,
server_args=list(_SEED_TTS_SERVER_ARGS),
)
]
def _require_vllm_cli() -> None:
try:
find_vllm_cli()
except FileNotFoundError as exc:
pytest.skip(str(exc))
@pytest.fixture(autouse=True)
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
original = _acc_bench.daily_omni_bench_argv
def _wrapped() -> list[str]:
argv = list(original())
if "--daily-omni-inline-local-video" not in argv:
argv.append("--daily-omni-inline-local-video")
return argv
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
_require_vllm_cli()
pytest.importorskip("datasets")
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=True,
skip_daily=False,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--min-daily-omni-accuracy",
str(_MIN_DAILY_OMNI_ACCURACY),
"--temperature",
"0",
"--output-len",
"512",
"--daily-omni-input-mode",
"all",
"--daily-omni-pack-mode",
"minicpm-interleave",
"--daily-extra-body-json",
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
_require_vllm_cli()
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=False,
skip_daily=True,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--max-seed-tts-mean-wer",
str(_MAX_SEED_TTS_MEAN_WER),
"--seed-extra-body-json",
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
_require_vllm_cli()
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=False,
skip_daily=True,
num_prompts=50,
max_concurrency=1,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--max-seed-tts-mean-wer",
str(_MAX_SEED_TTS_MEAN_WER),
"--seed-extra-body-json",
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
"--seed-backend",
"openai-realtime-tts",
"--seed-endpoint",
"/v1/realtime",
"--seed-tts-turns-per-session",
"4",
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@@ -19,6 +19,7 @@ import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any, Protocol
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
def find_vllm_cli() -> str:
exe = shutil.which("vllm")
if not exe:
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
return exe
if exe:
return exe
# Fallback when pytest is invoked via absolute venv path without activating PATH.
sibling = Path(sys.executable).resolve().parent / "vllm"
if sibling.is_file() and os.access(sibling, os.X_OK):
return str(sibling)
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
@@ -150,6 +155,11 @@ def build_serve_common_argv(
result_dir: Path,
result_filename: str,
ready_check_timeout_sec: int | None = None,
trust_remote_code: bool = False,
backend: str = "openai-chat-omni",
endpoint: str = "/v1/chat/completions",
temperature: float | None = None,
output_len: int | None = None,
) -> list[str]:
out = [
"bench",
@@ -162,9 +172,9 @@ def build_serve_common_argv(
"--model",
model,
"--endpoint",
"/v1/chat/completions",
endpoint,
"--backend",
"openai-chat-omni",
backend,
"--request-rate",
"inf",
"--num-prompts",
@@ -184,6 +194,12 @@ def build_serve_common_argv(
]
if ready_check_timeout_sec is not None:
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
if trust_remote_code:
out.append("--trust-remote-code")
if temperature is not None:
out.extend(["--temperature", str(temperature)])
if output_len is not None:
out.extend(["--output-len", str(int(output_len))])
return out
@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
os.environ[key] = value
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
def _build_common_args(
ns: argparse.Namespace,
*,
result_filename: str,
backend: str = "openai-chat-omni",
endpoint: str = "/v1/chat/completions",
) -> list[str]:
return build_serve_common_argv(
host=ns.host,
port=ns.port,
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
result_dir=ns.result_dir,
result_filename=result_filename,
ready_check_timeout_sec=ns.ready_check_timeout_sec,
trust_remote_code=ns.trust_remote_code,
backend=backend,
endpoint=endpoint,
temperature=getattr(ns, "temperature", None),
output_len=getattr(ns, "output_len", None),
)
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
if pack_mode:
argv.extend(["--daily-omni-pack-mode", pack_mode])
if ns.daily_omni_save_eval_items:
argv.append("--daily-omni-save-eval-items")
print("\n$", vllm, *argv, "\n", flush=True)
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
extra = json.loads(ns.seed_extra_body_json)
argv = (
_build_common_args(ns, result_filename=result_filename)
_build_common_args(
ns,
result_filename=result_filename,
backend=ns.seed_backend,
endpoint=ns.seed_endpoint,
)
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
+ [
"--seed-tts-wer-eval",
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
if ns.seed_tts_turns_per_session > 1:
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
if ns.seed_tts_wer_save_items:
argv.append("--seed-tts-wer-save-items")
if ns.seed_tts_file_ref_audio:
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
"Omit to use upstream default (typically skip).",
)
p.add_argument(
"--trust-remote-code",
action="store_true",
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
)
p.add_argument(
"--temperature",
type=float,
default=None,
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
)
p.add_argument(
"--output-len",
type=int,
default=None,
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
)
p.add_argument(
"--result-dir",
type=Path,
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
)
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
p.add_argument(
"--daily-omni-pack-mode",
choices=("qwen", "minicpm-interleave"),
default=None,
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
"(1fps interleaved image/audio; see PR #5606).",
)
p.add_argument(
"--daily-extra-body-json",
default='{"modalities":["text"]}',
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
default='{"modalities":["text","audio"]}',
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
)
p.add_argument(
"--seed-backend",
default="openai-chat-omni",
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
)
p.add_argument(
"--seed-endpoint",
default="/v1/chat/completions",
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
)
p.add_argument(
"--seed-tts-turns-per-session",
type=int,
default=1,
help="Group this many Seed-TTS target texts into each Realtime session.",
)
p.add_argument("--seed-tts-wer-save-items", action="store_true")
p.add_argument(
"--seed-tts-file-ref-audio",
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
min_mean_sim=ns.min_seed_tts_mean_sim,
min_mean_utmos=ns.min_seed_tts_mean_utmos,
)
if ns.seed_tts_turns_per_session > 1:
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
if evaluated_turns != expected_turns:
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
if errs:
failed.extend([f"[Seed-TTS] {e}" for e in errs])
else:
@@ -53,6 +53,7 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
RealtimeEventCollector,
build_realtime_url,
read_pcm16_wav,
summarize_session_request_metrics,
)
_url_with_model = build_realtime_url
@@ -194,6 +195,38 @@ class DemoState:
summaries[response_id] = timing
return summaries
def session_request_metrics(
self,
*,
session_id: str | None = None,
) -> list[dict[str, object]]:
"""Return benchmark-style metrics for every response in this session."""
requests: list[dict[str, object]] = []
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
metrics = timing.get("request_metrics")
if not isinstance(metrics, dict):
continue
requests.append(
{
"session_id": session_id,
"request_index": request_index,
"response_id": response_id,
**metrics,
}
)
return requests
def session_metric_summary(
self,
*,
session_id: str | None = None,
) -> dict[str, object]:
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
return summarize_session_request_metrics(
self.session_request_metrics(session_id=session_id),
session_id=session_id,
)
def first_index(self, event_type: str, predicate=None) -> int | None:
for index, event in enumerate(self.events):
if event.get("type") != event_type:
@@ -1523,6 +1556,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
"input_transcription_ok": input_transcription_ok,
"completed_response_ids": completed_response_ids,
"response_timings": state.response_timing_summaries(),
"request_metrics": state.session_request_metrics(session_id=args.session_id),
"session_metrics": state.session_metric_summary(session_id=args.session_id),
"lifecycle_counts_ok": lifecycle_counts_ok,
"validation_mode": validation_mode,
"scenario": scenario,
@@ -8,6 +8,7 @@ import hashlib
import json
import sys
import uuid
from functools import lru_cache
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -19,11 +20,25 @@ SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
_ref_audio_data_url,
_url_with_model,
run_demo,
)
@lru_cache(maxsize=1)
def _scenario_module():
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
import minicpmo_realtime_duplex_scenarios
return minicpmo_realtime_duplex_scenarios
def _ref_audio_data_url(path: str) -> str:
return _scenario_module()._ref_audio_data_url(path)
def _url_with_model(*args, **kwargs) -> str:
return _scenario_module()._url_with_model(*args, **kwargs)
async def run_demo(args):
return await _scenario_module().run_demo(args)
def _with_resume_mode(url: str) -> str:
@@ -73,7 +73,7 @@ def get_max_batch_size(size_type="few"):
def test_text_to_text_001(omni_server, openai_client) -> None:
"""
Test text-only input generating text output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text
Output Modal: text
Input Setting: stream=False
@@ -99,7 +99,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
"""
Test text-only input generating text + audio output via OpenAI API.
This exercises Talker TTS region detection and the Code2Wav stage.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text
Output Modal: text + audio
Input Setting: stream=True
@@ -123,7 +123,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test audio input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + audio
Output Modal: text + audio
Input Setting: stream=True
@@ -159,7 +159,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test image input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + image
Output Modal: text + audio
Input Setting: stream=True
@@ -187,7 +187,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test video input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + video
Output Modal: text + audio
Input Setting: stream=True
@@ -216,7 +216,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test multi-modal input (text + audio + video + image) generating text + audio output.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + audio + video + image
Output Modal: text + audio
Input Setting: stream=True
@@ -31,6 +31,29 @@ from vllm_omni.experimental.fullduplex.client import build_realtime_url
pytestmark = pytest.mark.omni
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
assert isinstance(metrics, list)
assert len(metrics) == expected_count
for request_index, request in enumerate(metrics):
assert isinstance(request["session_id"], str)
assert request["request_index"] == request_index
assert isinstance(request["response_id"], str)
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
assert request["ttfp_ms"] >= 0
assert request["rtf"] is not None and request["rtf"] >= 0
assert request["audio_generation_ms"] >= 0
assert request["audio_duration_ms"] > 0
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
assert isinstance(metrics, dict)
assert isinstance(metrics["session_id"], str)
assert metrics["audio_turn_count"] == expected_count
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
async def receive() -> list[dict[str, object]]:
events: list[dict[str, object]] = []
@@ -103,22 +126,22 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
result = asyncio.run(
run_demo(
demo_args(
omni_server=omni_server,
input_wav=validated_input_wav(),
ref_audio=resolve_ref_audio(model_prefix),
output_dir=tmp_path / "single_session",
)
)
args = demo_args(
omni_server=omni_server,
input_wav=validated_input_wav(),
ref_audio=resolve_ref_audio(model_prefix),
output_dir=tmp_path / "single_session",
)
args.turns = 2
result = asyncio.run(run_demo(args))
assert result["ok"] is True
assert result["audio_delta_count"] > 0
assert result["done_count"] == 1
assert result["done_count"] == 2
assert result["error_count"] == 0
assert result["all_audio_responses_have_transcript"] is True
assert result["transcript_delta_done_ok"] is True
_assert_request_metrics(result["request_metrics"], expected_count=2)
_assert_session_metrics(result["session_metrics"], expected_count=2)
@pytest.mark.advanced_model
@@ -144,3 +167,6 @@ def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str,
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
assert all(session["done_count"] == 1 for session in result["sessions"])
assert all(session["error_count"] == 0 for session in result["sessions"])
for session in result["sessions"]:
_assert_request_metrics(session["request_metrics"], expected_count=1)
_assert_session_metrics(session["session_metrics"], expected_count=1)
@@ -80,7 +80,7 @@ def get_max_batch_size(size_type="few"):
def test_text_video_to_text_001(omni_server, openai_client) -> None:
"""
Test text + video input generating text output.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + video
Output Modal: text
Input Setting: stream=False
@@ -108,7 +108,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
"""
Verify that sequential requests produce independent results and that the
second request does not receive the first request's audio or token state.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text (two different prompts)
Output Modal: text + audio (both)
"""
@@ -149,7 +149,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
"""
Test text input generating a longer audio output to exercise the
Code2Wav stage across multiple frames.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text (longer prompt)
Output Modal: text + audio
Input Setting: stream=True
@@ -799,6 +799,7 @@ def test_realtime_duplex_demo_reads_response_playback_cursor():
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
demo = _load_demo_module()
state = demo.DemoState()
state.input_commit_sent_at_s.extend((9.9, 19.9))
for response_id, created_at_s, audio_at_s, token_count in (
("resp-1", 10.0, 10.1, 3),
("resp-2", 20.0, 20.2, 5),
@@ -826,15 +827,66 @@ def test_realtime_duplex_demo_partitions_timing_by_response_identity():
},
received_at_s=audio_at_s,
)
state.add(
{
"type": "response.audio_transcript.delta",
"response_id": response_id,
"delta": "audio",
},
received_at_s=created_at_s + 0.05,
)
timings = state.response_timing_summaries()
requests = state.session_request_metrics(session_id="seed-tts-session")
session_metrics = state.session_metric_summary(session_id="seed-tts-session")
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] == 200.0
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] == 300.0
assert requests == [
{
"session_id": "seed-tts-session",
"request_index": 0,
"response_id": "resp-1",
"ttft_ms": 150.0,
"ttfp_ms": 200.0,
"rtf": 2.5,
"audio_generation_ms": 200.0,
"audio_duration_ms": 80.0,
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
},
{
"session_id": "seed-tts-session",
"request_index": 1,
"response_id": "resp-2",
"ttft_ms": 150.0,
"ttfp_ms": 300.0,
"rtf": 3.75,
"audio_generation_ms": 300.0,
"audio_duration_ms": 80.0,
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
},
]
assert session_metrics == {
"session_id": "seed-tts-session",
"audio_turn_count": 2,
"mean_ttft_ms": 150.0,
"mean_ttfp_ms": 250.0,
"mean_rtf": 3.125,
}
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
+50
View File
@@ -300,6 +300,56 @@ def test_strip_single_engine_args_model_does_not_trigger_warning(mocker):
assert "model" not in warned_args
def test_new_format_deploy_yaml_does_not_strip_interleave_cli_overrides(tmp_path):
"""New-format deploy YAMLs must not strip interleave_mm_strings / media_io_kwargs.
Pytest historically passed ``--stage-configs-path`` for deploy YAMLs; stripping
those CLI flags dropped MiniCPM interleaved AV packing and cut Daily-Omni
accuracy from ~78% to ~70%.
"""
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
yaml_path = tmp_path / "minicpm_deploy.yaml"
yaml_path.write_text(
"""
pipeline: minicpmo_4_5
async_chunk: true
stages:
- stage_id: 0
devices: "0"
""".strip(),
encoding="utf-8",
)
path = str(yaml_path)
assert is_new_format_deploy_config(path)
kwargs = {
"stage_configs_path": path,
"trust_remote_code": True,
"interleave_mm_strings": True,
"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}},
"gpu_memory_utilization": 0.55,
}
# Mirror AsyncOmniEngine._resolve_stage_configs strip decision.
if kwargs.get("stage_configs_path") is not None and not is_new_format_deploy_config(kwargs["stage_configs_path"]):
base = AsyncOmniEngine._strip_single_engine_args(dict(kwargs))
else:
base = dict(kwargs)
assert base.get("interleave_mm_strings") is True
assert base.get("media_io_kwargs") == {"video": {"fps": 1, "num_frames": 128}}
# Parent EngineArgs fields also remain available for deploy merge.
assert base.get("gpu_memory_utilization") == 0.55
legacy = tmp_path / "legacy.yaml"
legacy.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8")
assert not is_new_format_deploy_config(str(legacy))
legacy_kwargs = dict(kwargs, stage_configs_path=str(legacy))
stripped = AsyncOmniEngine._strip_single_engine_args(legacy_kwargs)
assert "interleave_mm_strings" not in stripped
assert "media_io_kwargs" not in stripped
assert "gpu_memory_utilization" not in stripped
# For https://github.com/vllm-project/vllm-omni/issues/3293
def test_tensor_parallel_size_none_is_handled():
"""Ensure the tensor parallel size of None isn't forwarded."""
@@ -51,6 +51,19 @@ from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_native_input_append_supports_explicit_session_opt_out():
session = DuplexSession(
"sid-explicit-tts",
DuplexSessionConfig(model="test-model"),
capabilities=DuplexCapabilities.minicpmo45_native(),
)
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
session.config.extra_body["minicpmo45_native_duplex"] = False
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
class _ModelConfig:
model = "test-model"
+102
View File
@@ -11,6 +11,9 @@ from vllm_omni.experimental.fullduplex.client import (
read_pcm16_wav,
write_pcm16_wav,
)
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
MiniCPMO45DuplexPolicy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -32,6 +35,19 @@ def test_realtime_client_builds_explicit_native_duplex_url():
}
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
"Speak exactly.",
True,
"The quick brown fox.",
)
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
assert suffix == (
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
)
def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?duplex=1",
@@ -86,6 +102,59 @@ async def test_realtime_client_configure_sends_explicit_ref_audio():
assert session["ref_audio"] == "data:audio/wav;base64,AAAA"
@pytest.mark.asyncio
async def test_realtime_client_configure_sends_seed_tts_text_condition():
class Client(RealtimeDuplexClient):
def __init__(self):
super().__init__("ws://unused")
self.sent = []
async def send(self, event):
self.sent.append(event)
self.events.add({"type": "session.created"})
client = Client()
await client.configure(
"openbmb/MiniCPM-o-4_5",
instructions="Speak the requested text exactly.",
initial_user_text="The quick brown fox.",
timeout_s=1,
)
session = client.sent[0]["session"]
assert session["instructions"] == "Speak the requested text exactly."
assert session["extra_body"]["duplex_initial_user_text"] == "The quick brown fox."
@pytest.mark.asyncio
async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex():
class Client(RealtimeDuplexClient):
def __init__(self):
super().__init__("ws://unused")
self.sent = []
async def send(self, event):
self.sent.append(event)
self.events.add({"type": "session.created"})
client = Client()
await client.configure(
"openbmb/MiniCPM-o-4_5",
native_duplex=False,
auto_response=False,
extra_body={"ref_audio": "data:audio/wav;base64,AAAA"},
timeout_s=1,
)
session_extra_body = client.sent[0]["session"]["extra_body"]
assert session_extra_body == {
"ref_audio": "data:audio/wav;base64,AAAA",
"minicpmo45_native_duplex": False,
}
def test_realtime_event_collector_partitions_audio_by_response():
collector = RealtimeEventCollector()
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
@@ -134,6 +203,26 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
},
received_at_s=received_at_s,
)
collector.add(
{
"type": "response.audio_transcript.delta",
"response_id": "resp-a",
"delta": "",
},
received_at_s=10.1,
)
collector.add(
{
"type": "response.audio_transcript.delta",
"response_id": "resp-a",
"delta": "hello",
},
received_at_s=10.15,
)
collector.add(
{"type": "response.done", "response": {"id": "resp-a"}},
received_at_s=10.4,
)
timing = collector.timing_summary(
after_s=10.0,
@@ -175,6 +264,19 @@ def test_realtime_event_collector_reports_engine_token_and_audio_intervals():
},
"max_chunk_gap_ms": 110.0,
}
assert timing["request_metrics"] == {
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
"ttft_ms": 250.0,
"ttfp_ms": 300.0,
"rtf": pytest.approx(1.916667),
"audio_generation_ms": 460.0,
"audio_duration_ms": 240.0,
}
def test_response_timing_ignores_unowned_session_level_metrics():
+5
View File
@@ -19,6 +19,11 @@ DEPLOY_CONFIG = modify_stage_config(
get_deploy_config_path("minicpmo_4_5_duplex.yaml"),
updates={
"base_config": get_deploy_config_path("minicpmo_4_5.yaml"),
# Talker context is 4096 (tts_config.max_position_embeddings); KV sizing
# is left automatic so duplex matches the simplex deploy profiles.
"stages": {
1: {"max_model_len": 4096},
},
},
)
ASSET_DIR = Path(__file__).resolve().parents[1] / "assets" / "minicpmo_4_5"
+6 -1
View File
@@ -2881,7 +2881,12 @@ def iter_omni_server(
print("OmniServer stopping...")
else:
if stage_config_path is not None:
server_args += ["--stage-configs-path", stage_config_path]
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
if is_new_format_deploy_config(stage_config_path):
server_args += ["--deploy-config", stage_config_path]
else:
server_args += ["--stage-configs-path", stage_config_path]
with (
OmniServer(
+27
View File
@@ -2164,6 +2164,33 @@ class TestBaseConfigInheritance:
class TestPlatformOverrides:
"""Test platform-specific deploy config overrides."""
def test_minicpmo_4_5_cuda_caps_talker_kv_cache(self):
pipeline = resolve_pipeline_config("minicpmo_4_5")
assert isinstance(pipeline, PipelineConfig)
deploy_path = Path(get_deploy_config_path("minicpmo_4_5.yaml"))
cuda = _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda")
cuda_stages = merge_pipeline_deploy(pipeline, cuda)
# Talker KV is left to automatic sizing (no hard kv_cache_memory_bytes cap).
assert "kv_cache_memory_bytes" not in cuda_stages[1].yaml_engine_args
# The CUDA memory budget must not leak into the existing NPU profile.
npu = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
npu_stages = merge_pipeline_deploy(pipeline, npu)
assert "kv_cache_memory_bytes" not in npu_stages[1].yaml_engine_args
# Multi-GPU replica profiles place Talker on dedicated devices and
# also leave KV sizing automatic.
for filename in (
"minicpmo_4_5_3gpu_stage1_replicas.yaml",
"minicpmo_4_5_4gpu_stage1_replicas.yaml",
):
replica = _apply_platform_overrides(
load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
)
replica_stages = merge_pipeline_deploy(pipeline, replica)
assert "kv_cache_memory_bytes" not in replica_stages[1].yaml_engine_args
def test_npu_overrides(self):
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():