[CI][MiniCPM-o]MiniCPM-o ACC PERF CI (#5575)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
ruirui(rein) yangandamy-why-3459 authored and GitHub committed 2026-08-04 18:46:20 +08:00
1 parent 434c220da1
commit a5eb2a9a87
50 files changed
+1756 -102

No files matched your search

@@ -0,0 +1 @@
# SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
build_acc_benchmark_cli_argv,
find_vllm_cli,
)
from tests.helpers.mark import hardware_test
from tests.helpers.minicpmo_4_5_duplex import SERVER_PARAMS as DUPLEX_TEST_PARAMS
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
_RESULT_DIR = Path(
os.environ.get(
"ACC_BENCH_RESULT_DIR",
str(Path(__file__).resolve().parent / "results"),
)
)
_MIN_DAILY_OMNI_ACCURACY = 0.78
_MAX_SEED_TTS_MEAN_WER = 0.05
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
_DAILY_EXTRA_BODY = {
"modalities": ["text"],
"chat_template_kwargs": {
"enable_thinking": False,
"use_tts_template": True,
},
}
_SEED_EXTRA_BODY = {
"modalities": ["text", "audio"],
"chat_template_kwargs": {
"enable_thinking": False,
"use_tts_template": True,
},
}
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
# msgspec ValidationError and kill the orchestrator mid-suite.
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
_DAILY_OMNI_SERVER_ARGS = [
"--trust-remote-code",
"--interleave-mm-strings",
"--media-io-kwargs",
'{"video":{"fps":1,"num_frames":128}}',
]
_SEED_TTS_SERVER_ARGS = [
"--trust-remote-code",
]
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
daily_test_params = [
OmniServerParams(
model=_MODEL,
stage_config_path=_DEPLOY_CONFIG,
server_args=list(_DAILY_OMNI_SERVER_ARGS),
)
]
seed_test_params = [
OmniServerParams(
model=_MODEL,
stage_config_path=_DEPLOY_CONFIG,
server_args=list(_SEED_TTS_SERVER_ARGS),
)
]
def _require_vllm_cli() -> None:
try:
find_vllm_cli()
except FileNotFoundError as exc:
pytest.skip(str(exc))
@pytest.fixture(autouse=True)
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
original = _acc_bench.daily_omni_bench_argv
def _wrapped() -> list[str]:
argv = list(original())
if "--daily-omni-inline-local-video" not in argv:
argv.append("--daily-omni-inline-local-video")
return argv
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
_require_vllm_cli()
pytest.importorskip("datasets")
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=True,
skip_daily=False,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--min-daily-omni-accuracy",
str(_MIN_DAILY_OMNI_ACCURACY),
"--temperature",
"0",
"--output-len",
"512",
"--daily-omni-input-mode",
"all",
"--daily-omni-pack-mode",
"minicpm-interleave",
"--daily-extra-body-json",
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
_require_vllm_cli()
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=False,
skip_daily=True,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--max-seed-tts-mean-wer",
str(_MAX_SEED_TTS_MEAN_WER),
"--seed-extra-body-json",
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_server", DUPLEX_TEST_PARAMS, indirect=True)
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
_require_vllm_cli()
pytest.importorskip("huggingface_hub")
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=False,
skip_daily=True,
num_prompts=50,
max_concurrency=1,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--max-seed-tts-mean-wer",
str(_MAX_SEED_TTS_MEAN_WER),
"--seed-extra-body-json",
json.dumps({"save_duplex_request_metrics": True}, separators=(",", ":")),
"--seed-backend",
"openai-realtime-tts",
"--seed-endpoint",
"/v1/realtime",
"--seed-tts-turns-per-session",
"4",
"--trust-remote-code",
]
)
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@@ -19,6 +19,7 @@ import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any, Protocol
@@ -121,9 +122,13 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
def find_vllm_cli() -> str:
exe = shutil.which("vllm")
if not exe:
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
return exe
if exe:
return exe
# Fallback when pytest is invoked via absolute venv path without activating PATH.
sibling = Path(sys.executable).resolve().parent / "vllm"
if sibling.is_file() and os.access(sibling, os.X_OK):
return str(sibling)
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
@@ -150,6 +155,11 @@ def build_serve_common_argv(
result_dir: Path,
result_filename: str,
ready_check_timeout_sec: int | None = None,
trust_remote_code: bool = False,
backend: str = "openai-chat-omni",
endpoint: str = "/v1/chat/completions",
temperature: float | None = None,
output_len: int | None = None,
) -> list[str]:
out = [
"bench",
@@ -162,9 +172,9 @@ def build_serve_common_argv(
"--model",
model,
"--endpoint",
"/v1/chat/completions",
endpoint,
"--backend",
"openai-chat-omni",
backend,
"--request-rate",
"inf",
"--num-prompts",
@@ -184,6 +194,12 @@ def build_serve_common_argv(
]
if ready_check_timeout_sec is not None:
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
if trust_remote_code:
out.append("--trust-remote-code")
if temperature is not None:
out.extend(["--temperature", str(temperature)])
if output_len is not None:
out.extend(["--output-len", str(int(output_len))])
return out
@@ -171,7 +171,13 @@ def _preserve_benchmark_dataset_env() -> Any:
os.environ[key] = value
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
def _build_common_args(
ns: argparse.Namespace,
*,
result_filename: str,
backend: str = "openai-chat-omni",
endpoint: str = "/v1/chat/completions",
) -> list[str]:
return build_serve_common_argv(
host=ns.host,
port=ns.port,
@@ -183,6 +189,11 @@ def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[
result_dir=ns.result_dir,
result_filename=result_filename,
ready_check_timeout_sec=ns.ready_check_timeout_sec,
trust_remote_code=ns.trust_remote_code,
backend=backend,
endpoint=endpoint,
temperature=getattr(ns, "temperature", None),
output_len=getattr(ns, "output_len", None),
)
@@ -201,6 +212,9 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
if pack_mode:
argv.extend(["--daily-omni-pack-mode", pack_mode])
if ns.daily_omni_save_eval_items:
argv.append("--daily-omni-save-eval-items")
print("\n$", vllm, *argv, "\n", flush=True)
@@ -217,7 +231,12 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
extra = json.loads(ns.seed_extra_body_json)
argv = (
_build_common_args(ns, result_filename=result_filename)
_build_common_args(
ns,
result_filename=result_filename,
backend=ns.seed_backend,
endpoint=ns.seed_endpoint,
)
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
+ [
"--seed-tts-wer-eval",
@@ -225,6 +244,8 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
if ns.seed_tts_turns_per_session > 1:
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
if ns.seed_tts_wer_save_items:
argv.append("--seed-tts-wer-save-items")
if ns.seed_tts_file_ref_audio:
@@ -266,6 +287,23 @@ def build_arg_parser() -> argparse.ArgumentParser:
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
"Omit to use upstream default (typically skip).",
)
p.add_argument(
"--trust-remote-code",
action="store_true",
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
)
p.add_argument(
"--temperature",
type=float,
default=None,
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
)
p.add_argument(
"--output-len",
type=int,
default=None,
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
)
p.add_argument(
"--result-dir",
type=Path,
@@ -295,6 +333,13 @@ def build_arg_parser() -> argparse.ArgumentParser:
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
)
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
p.add_argument(
"--daily-omni-pack-mode",
choices=("qwen", "minicpm-interleave"),
default=None,
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
"(1fps interleaved image/audio; see PR #5606).",
)
p.add_argument(
"--daily-extra-body-json",
default='{"modalities":["text"]}',
@@ -326,6 +371,22 @@ def build_arg_parser() -> argparse.ArgumentParser:
default='{"modalities":["text","audio"]}',
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
)
p.add_argument(
"--seed-backend",
default="openai-chat-omni",
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
)
p.add_argument(
"--seed-endpoint",
default="/v1/chat/completions",
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
)
p.add_argument(
"--seed-tts-turns-per-session",
type=int,
default=1,
help="Group this many Seed-TTS target texts into each Realtime session.",
)
p.add_argument("--seed-tts-wer-save-items", action="store_true")
p.add_argument(
"--seed-tts-file-ref-audio",
@@ -400,6 +461,11 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
min_mean_sim=ns.min_seed_tts_mean_sim,
min_mean_utmos=ns.min_seed_tts_mean_utmos,
)
if ns.seed_tts_turns_per_session > 1:
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
if evaluated_turns != expected_turns:
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
if errs:
failed.extend([f"[Seed-TTS] {e}" for e in errs])
else:
@@ -53,6 +53,7 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
RealtimeEventCollector,
build_realtime_url,
read_pcm16_wav,
summarize_session_request_metrics,
)
_url_with_model = build_realtime_url
@@ -194,6 +195,38 @@ class DemoState:
summaries[response_id] = timing
return summaries
def session_request_metrics(
self,
*,
session_id: str | None = None,
) -> list[dict[str, object]]:
"""Return benchmark-style metrics for every response in this session."""
requests: list[dict[str, object]] = []
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
metrics = timing.get("request_metrics")
if not isinstance(metrics, dict):
continue
requests.append(
{
"session_id": session_id,
"request_index": request_index,
"response_id": response_id,
**metrics,
}
)
return requests
def session_metric_summary(
self,
*,
session_id: str | None = None,
) -> dict[str, object]:
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
return summarize_session_request_metrics(
self.session_request_metrics(session_id=session_id),
session_id=session_id,
)
def first_index(self, event_type: str, predicate=None) -> int | None:
for index, event in enumerate(self.events):
if event.get("type") != event_type:
@@ -1523,6 +1556,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
"input_transcription_ok": input_transcription_ok,
"completed_response_ids": completed_response_ids,
"response_timings": state.response_timing_summaries(),
"request_metrics": state.session_request_metrics(session_id=args.session_id),
"session_metrics": state.session_metric_summary(session_id=args.session_id),
"lifecycle_counts_ok": lifecycle_counts_ok,
"validation_mode": validation_mode,
"scenario": scenario,
@@ -8,6 +8,7 @@ import hashlib
import json
import sys
import uuid
from functools import lru_cache
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -19,11 +20,25 @@ SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
_ref_audio_data_url,
_url_with_model,
run_demo,
)
@lru_cache(maxsize=1)
def _scenario_module():
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
import minicpmo_realtime_duplex_scenarios
return minicpmo_realtime_duplex_scenarios
def _ref_audio_data_url(path: str) -> str:
return _scenario_module()._ref_audio_data_url(path)
def _url_with_model(*args, **kwargs) -> str:
return _scenario_module()._url_with_model(*args, **kwargs)
async def run_demo(args):
return await _scenario_module().run_demo(args)
def _with_resume_mode(url: str) -> str:
@@ -73,7 +73,7 @@ def get_max_batch_size(size_type="few"):
def test_text_to_text_001(omni_server, openai_client) -> None:
"""
Test text-only input generating text output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text
Output Modal: text
Input Setting: stream=False
@@ -99,7 +99,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
"""
Test text-only input generating text + audio output via OpenAI API.
This exercises Talker TTS region detection and the Code2Wav stage.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text
Output Modal: text + audio
Input Setting: stream=True
@@ -123,7 +123,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test audio input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + audio
Output Modal: text + audio
Input Setting: stream=True
@@ -159,7 +159,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test image input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + image
Output Modal: text + audio
Input Setting: stream=True
@@ -187,7 +187,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test video input generating text + audio output via OpenAI API.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + video
Output Modal: text + audio
Input Setting: stream=True
@@ -216,7 +216,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test multi-modal input (text + audio + video + image) generating text + audio output.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + audio + video + image
Output Modal: text + audio
Input Setting: stream=True
@@ -31,6 +31,29 @@ from vllm_omni.experimental.fullduplex.client import build_realtime_url
pytestmark = pytest.mark.omni
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
assert isinstance(metrics, list)
assert len(metrics) == expected_count
for request_index, request in enumerate(metrics):
assert isinstance(request["session_id"], str)
assert request["request_index"] == request_index
assert isinstance(request["response_id"], str)
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
assert request["ttfp_ms"] >= 0
assert request["rtf"] is not None and request["rtf"] >= 0
assert request["audio_generation_ms"] >= 0
assert request["audio_duration_ms"] > 0
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
assert isinstance(metrics, dict)
assert isinstance(metrics["session_id"], str)
assert metrics["audio_turn_count"] == expected_count
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
async def receive() -> list[dict[str, object]]:
events: list[dict[str, object]] = []
@@ -103,22 +126,22 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
result = asyncio.run(
run_demo(
demo_args(
omni_server=omni_server,
input_wav=validated_input_wav(),
ref_audio=resolve_ref_audio(model_prefix),
output_dir=tmp_path / "single_session",
)
)
args = demo_args(
omni_server=omni_server,
input_wav=validated_input_wav(),
ref_audio=resolve_ref_audio(model_prefix),
output_dir=tmp_path / "single_session",
)
args.turns = 2
result = asyncio.run(run_demo(args))
assert result["ok"] is True
assert result["audio_delta_count"] > 0
assert result["done_count"] == 1
assert result["done_count"] == 2
assert result["error_count"] == 0
assert result["all_audio_responses_have_transcript"] is True
assert result["transcript_delta_done_ok"] is True
_assert_request_metrics(result["request_metrics"], expected_count=2)
_assert_session_metrics(result["session_metrics"], expected_count=2)
@pytest.mark.advanced_model
@@ -144,3 +167,6 @@ def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str,
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
assert all(session["done_count"] == 1 for session in result["sessions"])
assert all(session["error_count"] == 0 for session in result["sessions"])
for session in result["sessions"]:
_assert_request_metrics(session["request_metrics"], expected_count=1)
_assert_session_metrics(session["session_metrics"], expected_count=1)
@@ -80,7 +80,7 @@ def get_max_batch_size(size_type="few"):
def test_text_video_to_text_001(omni_server, openai_client) -> None:
"""
Test text + video input generating text output.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text + video
Output Modal: text
Input Setting: stream=False
@@ -108,7 +108,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
"""
Verify that sequential requests produce independent results and that the
second request does not receive the first request's audio or token state.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text (two different prompts)
Output Modal: text + audio (both)
"""
@@ -149,7 +149,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
"""
Test text input generating a longer audio output to exercise the
Code2Wav stage across multiple frames.
Deploy Setting: default 2GPU
Deploy Setting: default single GPU
Input Modal: text (longer prompt)
Output Modal: text + audio
Input Setting: stream=True
@@ -799,6 +799,7 @@ def test_realtime_duplex_demo_reads_response_playback_cursor():
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
demo = _load_demo_module()
state = demo.DemoState()
state.input_commit_sent_at_s.extend((9.9, 19.9))
for response_id, created_at_s, audio_at_s, token_count in (
("resp-1", 10.0, 10.1, 3),
("resp-2", 20.0, 20.2, 5),
@@ -826,15 +827,66 @@ def test_realtime_duplex_demo_partitions_timing_by_response_identity():
},
received_at_s=audio_at_s,
)
state.add(
{
"type": "response.audio_transcript.delta",
"response_id": response_id,
"delta": "audio",
},
received_at_s=created_at_s + 0.05,
)
timings = state.response_timing_summaries()
requests = state.session_request_metrics(session_id="seed-tts-session")
session_metrics = state.session_metric_summary(session_id="seed-tts-session")
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] == 200.0
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] == 300.0
assert requests == [
{
"session_id": "seed-tts-session",
"request_index": 0,
"response_id": "resp-1",
"ttft_ms": 150.0,
"ttfp_ms": 200.0,
"rtf": 2.5,
"audio_generation_ms": 200.0,
"audio_duration_ms": 80.0,
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
},
{
"session_id": "seed-tts-session",
"request_index": 1,
"response_id": "resp-2",
"ttft_ms": 150.0,
"ttfp_ms": 300.0,
"rtf": 3.75,
"audio_generation_ms": 300.0,
"audio_duration_ms": 80.0,
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
},
]
assert session_metrics == {
"session_id": "seed-tts-session",
"audio_turn_count": 2,
"mean_ttft_ms": 150.0,
"mean_ttfp_ms": 250.0,
"mean_rtf": 3.125,
}
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():