[CI][MiniCPM-o]MiniCPM-o ACC PERF CI (#5575)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
434c220da1
commit
a5eb2a9a87
50 files changed
+1756
-102
No files matched your search
@@ -0,0 +1 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
|
||||
|
||||
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
|
||||
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
|
||||
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
|
||||
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_acc_benchmark_cli_argv,
|
||||
find_vllm_cli,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
|
||||
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_RESULT_DIR = Path(
|
||||
os.environ.get(
|
||||
"ACC_BENCH_RESULT_DIR",
|
||||
str(Path(__file__).resolve().parent / "results"),
|
||||
)
|
||||
)
|
||||
|
||||
_MIN_DAILY_OMNI_ACCURACY = 0.78
|
||||
_MAX_SEED_TTS_MEAN_WER = 0.05
|
||||
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||
_DAILY_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
}
|
||||
_SEED_EXTRA_BODY = {
|
||||
"modalities": ["text", "audio"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
}
|
||||
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
|
||||
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
|
||||
# msgspec ValidationError and kill the orchestrator mid-suite.
|
||||
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
|
||||
_DAILY_OMNI_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--interleave-mm-strings",
|
||||
"--media-io-kwargs",
|
||||
'{"video":{"fps":1,"num_frames":128}}',
|
||||
]
|
||||
_SEED_TTS_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
daily_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_DAILY_OMNI_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
seed_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_SEED_TTS_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _require_vllm_cli() -> None:
|
||||
try:
|
||||
find_vllm_cli()
|
||||
except FileNotFoundError as exc:
|
||||
pytest.skip(str(exc))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
|
||||
original = _acc_bench.daily_omni_bench_argv
|
||||
|
||||
def _wrapped() -> list[str]:
|
||||
argv = list(original())
|
||||
if "--daily-omni-inline-local-video" not in argv:
|
||||
argv.append("--daily-omni-inline-local-video")
|
||||
return argv
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("datasets")
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=False,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--min-daily-omni-accuracy",
|
||||
str(_MIN_DAILY_OMNI_ACCURACY),
|
||||
"--temperature",
|
||||
"0",
|
||||
"--output-len",
|
||||
"512",
|
||||
"--daily-omni-input-mode",
|
||||
"all",
|
||||
"--daily-omni-pack-mode",
|
||||
"minicpm-interleave",
|
||||
"--daily-extra-body-json",
|
||||
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-extra-body-json",
|
||||
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
|
||||
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
|
||||
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
num_prompts=50,
|
||||
max_concurrency=1,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-extra-body-json",
|
||||
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
|
||||
"--seed-backend",
|
||||
"openai-realtime-tts",
|
||||
"--seed-endpoint",
|
||||
"/v1/realtime",
|
||||
"--seed-tts-turns-per-session",
|
||||
"4",
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
@@ -19,6 +19,7 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
|
||||
def find_vllm_cli() -> str:
|
||||
exe = shutil.which("vllm")
|
||||
if not exe:
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
return exe
|
||||
if exe:
|
||||
return exe
|
||||
# Fallback when pytest is invoked via absolute venv path without activating PATH.
|
||||
sibling = Path(sys.executable).resolve().parent / "vllm"
|
||||
if sibling.is_file() and os.access(sibling, os.X_OK):
|
||||
return str(sibling)
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
|
||||
|
||||
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
|
||||
@@ -150,6 +155,11 @@ def build_serve_common_argv(
|
||||
result_dir: Path,
|
||||
result_filename: str,
|
||||
ready_check_timeout_sec: int | None = None,
|
||||
trust_remote_code: bool = False,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
temperature: float | None = None,
|
||||
output_len: int | None = None,
|
||||
) -> list[str]:
|
||||
out = [
|
||||
"bench",
|
||||
@@ -162,9 +172,9 @@ def build_serve_common_argv(
|
||||
"--model",
|
||||
model,
|
||||
"--endpoint",
|
||||
"/v1/chat/completions",
|
||||
endpoint,
|
||||
"--backend",
|
||||
"openai-chat-omni",
|
||||
backend,
|
||||
"--request-rate",
|
||||
"inf",
|
||||
"--num-prompts",
|
||||
@@ -184,6 +194,12 @@ def build_serve_common_argv(
|
||||
]
|
||||
if ready_check_timeout_sec is not None:
|
||||
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
|
||||
if trust_remote_code:
|
||||
out.append("--trust-remote-code")
|
||||
if temperature is not None:
|
||||
out.extend(["--temperature", str(temperature)])
|
||||
if output_len is not None:
|
||||
out.extend(["--output-len", str(int(output_len))])
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
|
||||
def _build_common_args(
|
||||
ns: argparse.Namespace,
|
||||
*,
|
||||
result_filename: str,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
) -> list[str]:
|
||||
return build_serve_common_argv(
|
||||
host=ns.host,
|
||||
port=ns.port,
|
||||
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
|
||||
result_dir=ns.result_dir,
|
||||
result_filename=result_filename,
|
||||
ready_check_timeout_sec=ns.ready_check_timeout_sec,
|
||||
trust_remote_code=ns.trust_remote_code,
|
||||
backend=backend,
|
||||
endpoint=endpoint,
|
||||
temperature=getattr(ns, "temperature", None),
|
||||
output_len=getattr(ns, "output_len", None),
|
||||
)
|
||||
|
||||
|
||||
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
|
||||
if pack_mode:
|
||||
argv.extend(["--daily-omni-pack-mode", pack_mode])
|
||||
if ns.daily_omni_save_eval_items:
|
||||
argv.append("--daily-omni-save-eval-items")
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
|
||||
extra = json.loads(ns.seed_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
_build_common_args(
|
||||
ns,
|
||||
result_filename=result_filename,
|
||||
backend=ns.seed_backend,
|
||||
endpoint=ns.seed_endpoint,
|
||||
)
|
||||
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
|
||||
+ [
|
||||
"--seed-tts-wer-eval",
|
||||
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
|
||||
if ns.seed_tts_wer_save_items:
|
||||
argv.append("--seed-tts-wer-save-items")
|
||||
if ns.seed_tts_file_ref_audio:
|
||||
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
|
||||
"Omit to use upstream default (typically skip).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--result-dir",
|
||||
type=Path,
|
||||
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
|
||||
)
|
||||
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
|
||||
p.add_argument(
|
||||
"--daily-omni-pack-mode",
|
||||
choices=("qwen", "minicpm-interleave"),
|
||||
default=None,
|
||||
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
|
||||
"(1fps interleaved image/audio; see PR #5606).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--daily-extra-body-json",
|
||||
default='{"modalities":["text"]}',
|
||||
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
default='{"modalities":["text","audio"]}',
|
||||
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-backend",
|
||||
default="openai-chat-omni",
|
||||
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-endpoint",
|
||||
default="/v1/chat/completions",
|
||||
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-tts-turns-per-session",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Group this many Seed-TTS target texts into each Realtime session.",
|
||||
)
|
||||
p.add_argument("--seed-tts-wer-save-items", action="store_true")
|
||||
p.add_argument(
|
||||
"--seed-tts-file-ref-audio",
|
||||
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
min_mean_sim=ns.min_seed_tts_mean_sim,
|
||||
min_mean_utmos=ns.min_seed_tts_mean_utmos,
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
|
||||
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
|
||||
if evaluated_turns != expected_turns:
|
||||
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
|
||||
if errs:
|
||||
failed.extend([f"[Seed-TTS] {e}" for e in errs])
|
||||
else:
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
RealtimeEventCollector,
|
||||
build_realtime_url,
|
||||
read_pcm16_wav,
|
||||
summarize_session_request_metrics,
|
||||
)
|
||||
|
||||
_url_with_model = build_realtime_url
|
||||
@@ -194,6 +195,38 @@ class DemoState:
|
||||
summaries[response_id] = timing
|
||||
return summaries
|
||||
|
||||
def session_request_metrics(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return benchmark-style metrics for every response in this session."""
|
||||
requests: list[dict[str, object]] = []
|
||||
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
|
||||
metrics = timing.get("request_metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
requests.append(
|
||||
{
|
||||
"session_id": session_id,
|
||||
"request_index": request_index,
|
||||
"response_id": response_id,
|
||||
**metrics,
|
||||
}
|
||||
)
|
||||
return requests
|
||||
|
||||
def session_metric_summary(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
|
||||
return summarize_session_request_metrics(
|
||||
self.session_request_metrics(session_id=session_id),
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
def first_index(self, event_type: str, predicate=None) -> int | None:
|
||||
for index, event in enumerate(self.events):
|
||||
if event.get("type") != event_type:
|
||||
@@ -1523,6 +1556,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
"input_transcription_ok": input_transcription_ok,
|
||||
"completed_response_ids": completed_response_ids,
|
||||
"response_timings": state.response_timing_summaries(),
|
||||
"request_metrics": state.session_request_metrics(session_id=args.session_id),
|
||||
"session_metrics": state.session_metric_summary(session_id=args.session_id),
|
||||
"lifecycle_counts_ok": lifecycle_counts_ok,
|
||||
"validation_mode": validation_mode,
|
||||
"scenario": scenario,
|
||||
|
||||
@@ -8,6 +8,7 @@ import hashlib
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
@@ -19,11 +20,25 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
|
||||
_ref_audio_data_url,
|
||||
_url_with_model,
|
||||
run_demo,
|
||||
)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _scenario_module():
|
||||
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
|
||||
import minicpmo_realtime_duplex_scenarios
|
||||
|
||||
return minicpmo_realtime_duplex_scenarios
|
||||
|
||||
|
||||
def _ref_audio_data_url(path: str) -> str:
|
||||
return _scenario_module()._ref_audio_data_url(path)
|
||||
|
||||
|
||||
def _url_with_model(*args, **kwargs) -> str:
|
||||
return _scenario_module()._url_with_model(*args, **kwargs)
|
||||
|
||||
|
||||
async def run_demo(args):
|
||||
return await _scenario_module().run_demo(args)
|
||||
|
||||
|
||||
def _with_resume_mode(url: str) -> str:
|
||||
|
||||
@@ -73,7 +73,7 @@ def get_max_batch_size(size_type="few"):
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -99,7 +99,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text + audio output via OpenAI API.
|
||||
This exercises Talker TTS region detection and the Code2Wav stage.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -123,7 +123,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test audio input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + audio
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -159,7 +159,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test image input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -187,7 +187,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test video input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -216,7 +216,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test multi-modal input (text + audio + video + image) generating text + audio output.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + audio + video + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
|
||||
@@ -31,6 +31,29 @@ from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
pytestmark = pytest.mark.omni
|
||||
|
||||
|
||||
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, list)
|
||||
assert len(metrics) == expected_count
|
||||
for request_index, request in enumerate(metrics):
|
||||
assert isinstance(request["session_id"], str)
|
||||
assert request["request_index"] == request_index
|
||||
assert isinstance(request["response_id"], str)
|
||||
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
|
||||
assert request["ttfp_ms"] >= 0
|
||||
assert request["rtf"] is not None and request["rtf"] >= 0
|
||||
assert request["audio_generation_ms"] >= 0
|
||||
assert request["audio_duration_ms"] > 0
|
||||
|
||||
|
||||
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, dict)
|
||||
assert isinstance(metrics["session_id"], str)
|
||||
assert metrics["audio_turn_count"] == expected_count
|
||||
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
|
||||
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
|
||||
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
|
||||
|
||||
|
||||
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
||||
async def receive() -> list[dict[str, object]]:
|
||||
events: list[dict[str, object]] = []
|
||||
@@ -103,22 +126,22 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
result = asyncio.run(
|
||||
run_demo(
|
||||
demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
)
|
||||
args = demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
args.turns = 2
|
||||
result = asyncio.run(run_demo(args))
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
assert result["done_count"] == 1
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -144,3 +167,6 @@ def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str,
|
||||
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
||||
assert all(session["done_count"] == 1 for session in result["sessions"])
|
||||
assert all(session["error_count"] == 0 for session in result["sessions"])
|
||||
for session in result["sessions"]:
|
||||
_assert_request_metrics(session["request_metrics"], expected_count=1)
|
||||
_assert_session_metrics(session["session_metrics"], expected_count=1)
|
||||
@@ -80,7 +80,7 @@ def get_max_batch_size(size_type="few"):
|
||||
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text + video input generating text output.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -108,7 +108,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Verify that sequential requests produce independent results and that the
|
||||
second request does not receive the first request's audio or token state.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text (two different prompts)
|
||||
Output Modal: text + audio (both)
|
||||
"""
|
||||
@@ -149,7 +149,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input generating a longer audio output to exercise the
|
||||
Code2Wav stage across multiple frames.
|
||||
Deploy Setting: default 2GPU
|
||||
Deploy Setting: default single GPU
|
||||
Input Modal: text (longer prompt)
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
|
||||
@@ -799,6 +799,7 @@ def test_realtime_duplex_demo_reads_response_playback_cursor():
|
||||
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||||
demo = _load_demo_module()
|
||||
state = demo.DemoState()
|
||||
state.input_commit_sent_at_s.extend((9.9, 19.9))
|
||||
for response_id, created_at_s, audio_at_s, token_count in (
|
||||
("resp-1", 10.0, 10.1, 3),
|
||||
("resp-2", 20.0, 20.2, 5),
|
||||
@@ -826,15 +827,66 @@ def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||||
},
|
||||
received_at_s=audio_at_s,
|
||||
)
|
||||
state.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": response_id,
|
||||
"delta": "audio",
|
||||
},
|
||||
received_at_s=created_at_s + 0.05,
|
||||
)
|
||||
|
||||
timings = state.response_timing_summaries()
|
||||
requests = state.session_request_metrics(session_id="seed-tts-session")
|
||||
session_metrics = state.session_metric_summary(session_id="seed-tts-session")
|
||||
|
||||
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
|
||||
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
|
||||
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||||
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] == 200.0
|
||||
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
|
||||
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
|
||||
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||||
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] == 300.0
|
||||
assert requests == [
|
||||
{
|
||||
"session_id": "seed-tts-session",
|
||||
"request_index": 0,
|
||||
"response_id": "resp-1",
|
||||
"ttft_ms": 150.0,
|
||||
"ttfp_ms": 200.0,
|
||||
"rtf": 2.5,
|
||||
"audio_generation_ms": 200.0,
|
||||
"audio_duration_ms": 80.0,
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
},
|
||||
{
|
||||
"session_id": "seed-tts-session",
|
||||
"request_index": 1,
|
||||
"response_id": "resp-2",
|
||||
"ttft_ms": 150.0,
|
||||
"ttfp_ms": 300.0,
|
||||
"rtf": 3.75,
|
||||
"audio_generation_ms": 300.0,
|
||||
"audio_duration_ms": 80.0,
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
|
||||
"ttfp": "input_audio_buffer.commit client send to first audio packet",
|
||||
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
},
|
||||
]
|
||||
assert session_metrics == {
|
||||
"session_id": "seed-tts-session",
|
||||
"audio_turn_count": 2,
|
||||
"mean_ttft_ms": 150.0,
|
||||
"mean_ttfp_ms": 250.0,
|
||||
"mean_rtf": 3.125,
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
|
||||
|
||||
Reference in new issue
Block a user