# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import argparse import asyncio import base64 import io import json import subprocess import tarfile import threading import time import wave from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput from vllm.benchmarks.serve import TaskType from vllm_omni.benchmarks import omniinteract as oi from vllm_omni.benchmarks import serve as benchmark_serve from vllm_omni.benchmarks.data_modules import omniinteract_dataset as data from vllm_omni.benchmarks.metrics.metrics import calculate_metrics from vllm_omni.benchmarks.patch import patch as benchmark_patch from vllm_omni.clients.duplex import EventCollector from vllm_omni.entrypoints.cli.benchmark.cli_args import preprocess_serve_args from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand from vllm_omni.utils.tracking_parser import TrackingArgumentParser pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] MODEL = "openbmb/MiniCPM-o-4_5" def _case(tmp_path: Path, *, subset: str = "1q1a", name: str = "video.mp4") -> data.OmniInteractCase: video, annotation = tmp_path / name, tmp_path / f"{Path(name).stem}.json" video.parent.mkdir(parents=True, exist_ok=True) video.touch() annotation.write_text("{}") return data.OmniInteractCase(subset, name, video, annotation, "multi_turn") def _collector(*events: tuple[dict[str, object], float]) -> EventCollector: collector = EventCollector() for event, received_at in events: collector.add(event, received_at_s=received_at) return collector def _created(response_id: str = "r1") -> dict[str, object]: return {"type": "response.created", "response": {"id": response_id}} def _done(response_id: str = "r1", status: str = "completed") -> dict[str, object]: return {"type": "response.done", "response": {"id": response_id, "status": status}} def _audio( response_id: str = "r1", *, samples: int = 2400, rate: int | None = 24_000, value: int = 1, ) -> dict[str, object]: event: dict[str, object] = { "type": "response.output_audio.delta", "response_id": response_id, "format": "pcm16", "delta": base64.b64encode(bytes((value, 0)) * samples).decode(), } if rate is not None: event["sample_rate_hz"] = rate return event def _text(response_id: str = "r1", value: str = "hello") -> dict[str, object]: return {"type": "response.output_audio_transcript.delta", "response_id": response_id, "delta": value} def _listen(*, buffering: bool = False) -> dict[str, object]: return { "type": "response.listen", "response": {"metadata": {"model_listen": True, "buffering": buffering}}, } def _session_options(tmp_path: Path) -> data.OmniInteractSessionOptions: return data.OmniInteractSessionOptions(tmp_path / "artifacts", 10.0, 5.0, "reference.wav", True) def _sample(case: data.OmniInteractCase, options: data.OmniInteractSessionOptions): return data.OmniInteractSampleRequest( prompt="", prompt_len=0, expected_output_len=0, multi_modal_data=None, request_id="measured-0", omniinteract_case=case, omniinteract_options=options, ) def _write_dataset(root: Path) -> Path: data_root = root / "data" for subset in ("1q1a", "1q1a_math"): subset_root = data_root / subset (subset_root / "videos").mkdir(parents=True) (subset_root / "annotations").mkdir() (subset_root / "videos" / f"{subset}.mp4").touch() (subset_root / "annotations" / f"{subset}.json").write_text("{}") (subset_root / "video_json_map.json").write_text( json.dumps( { "entries": [ { "video": f"videos/{subset}.mp4", "annotation": f"annotations/{subset}.json", "scene_type": "multi_turn", } ] } ) ) one_to_many = data_root / "1qna" (one_to_many / "videos_bench" / "nested").mkdir(parents=True) (one_to_many / "annotations" / "nested").mkdir(parents=True) (one_to_many / "videos_bench" / "nested" / "guide.mp4").touch() (one_to_many / "annotations" / "nested" / "guide.json").write_text("{}") return data_root def _successful_output( tmp_path: Path, *, audio_time: float = 10.0, status: str = "completed", ) -> tuple[data.OmniInteractCase, EventCollector, oi.OmniInteractCaseResult]: case = _case(tmp_path) collector = _collector( (_created(), 9.9), (_audio(), audio_time), (_text(), audio_time + 0.01), (_done(status=status), audio_time + 0.1), ) result = oi.OmniInteractCaseResult(case.subset, str(case.video_path), "") return case, collector, result def test_response_metrics_include_engine_tpot_and_stream_window(): first_audio = _audio("r1") first_audio["metadata"] = { "audio_duration_ms": 100, "vllm_omni": { "stage_metrics": { "0": { "num_tokens_out": 3, "vllm_tpot_ms": 10.0, "vllm_itls_ms": [10.0, 10.0], } } }, } second_audio = _audio("r2") second_audio["metadata"] = { "audio_duration_ms": 200, "vllm_omni": { "stage_metrics": { "0": { "num_tokens_out": 2, "vllm_tpot_ms": 20.0, "vllm_itls_ms": [20.0], } } }, } collector = _collector( (_created("r1"), 10.1), (_text("r1", "first"), 10.2), (first_audio, 10.3), (_done("r1"), 10.4), (_created("r2"), 12.1), (_text("r2", "second"), 12.2), (second_audio, 12.5), (_done("r2"), 12.6), ) result = oi.OmniInteractCaseResult("1q1a", "video.mp4", "", session_id="session") oi._populate_response_metrics(result, collector, stream_start=10.0) assert result.output_tokens == 5 assert [metric["tpot_ms"] for metric in result.duplex_request_metrics] == [10.0, 20.0] assert result.duplex_session_metrics == { "session_id": "session", "audio_turn_count": 2, "ttft_ms": {"count": 2, "mean": 100.0, "p50": 100.0, "p99": 100.0}, "tpot_ms": {"count": 2, "mean": 15.0, "p50": 10.0, "p99": 20.0}, "ttfp_ms": {"count": 2, "mean": 300.0, "p50": 200.0, "p99": 400.0}, "rtf": {"count": 2, "mean": 2.0, "p50": 2.0, "p99": 2.0}, "stages": { "0": { "ttft_ms": {"count": 2, "mean": 0.0, "p50": 0.0, "p99": 0.0}, "tpot_ms": {"count": 2, "mean": 15.0, "p50": 10.0, "p99": 20.0}, } }, "stream_ttft_ms": 200.0, "stream_ttfp_ms": 300.0, "stream_rtf": 8.333333, "stream_audio_generation_ms": 2500.0, "stream_audio_duration_ms": 300.0, "stream_measurement_origin": { "ttft": "input stream start to first non-empty text delta", "ttfp": "input stream start to first audio packet", "rtf": ( "input stream start-to-last-audio receive time divided by total emitted audio duration; " "includes concurrent realtime input" ), }, } def _write_success( root: Path, case: data.OmniInteractCase, collector: EventCollector, result: oi.OmniInteractCaseResult, **kwargs, ): require_response = kwargs.pop("require_response", True) return oi.prepare_success_artifacts( root, case, collector, stream_start=10.0, video_duration_s=1.0, require_response=require_response, result=result, **kwargs, ) def test_dataset_discovers_official_layouts_and_total_selection(tmp_path: Path): root = _write_dataset(tmp_path) all_cases = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=0, disable_shuffle=True) selected = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=2, disable_shuffle=True) assert [case.subset for case in all_cases] == ["1q1a", "1q1a_math", "1qna"] assert all_cases[-1].video_rel == "videos_bench/nested/guide.mp4" assert [case.subset for case in selected] == ["1q1a", "1q1a_math"] def _write_scenario_dataset(root: Path) -> Path: data_root = root / "data" one_q1a = data_root / "1q1a" (one_q1a / "videos").mkdir(parents=True) (one_q1a / "annotations").mkdir() entries = [] specs = [ ("realtime", "multi_turn", [{"question_time": 0.0, "answer_time": 1.0, "question_type": "realtime"}]), ("proactive", "multi_turn", [{"question_time": 0.0, "answer_time": 1.0, "question_type": "proactive"}]), ( "nested", "nested", [ { "question_time": 0.0, "answer_time": 4.0, "question_type": "proactive", "question_text": "outer", "answer_text": "outer", }, { "question_time": 1.0, "answer_time": 2.0, "question_type": "realtime", "question_text": "inner", "answer_text": "inner", }, ], ), ( "interrupted", "multi_turn", [ { "question_time": 0.0, "answer_time": 1.0, "question_type": "realtime", "is_interrupted": True, } ], ), ] for name, scene, annotation in specs: video = one_q1a / "videos" / f"{name}.mp4" ann = one_q1a / "annotations" / f"{name}.json" video.touch() ann.write_text(json.dumps(annotation)) entries.append( { "video": f"videos/{name}.mp4", "annotation": f"annotations/{name}.json", "scene_type": scene, } ) (one_q1a / "video_json_map.json").write_text(json.dumps({"entries": entries})) one_qna = data_root / "1qna" (one_qna / "videos_bench").mkdir(parents=True) (one_qna / "annotations").mkdir(parents=True) (one_qna / "videos_bench" / "guide.mp4").touch() (one_qna / "annotations" / "guide.json").write_text( json.dumps( { "question_time": 0.0, "question_text": "q", "answers": [{"answer_time": 1.0, "answer_text": "a", "label": "step"}], } ) ) math_root = data_root / "1q1a_math" (math_root / "videos").mkdir(parents=True) (math_root / "annotations").mkdir() (math_root / "videos" / "math.mp4").touch() (math_root / "annotations" / "math.json").write_text( json.dumps([{"question_time": 0.0, "answer_time": 1.0, "question_type": "realtime"}]) ) (math_root / "video_json_map.json").write_text( json.dumps( { "entries": [ { "video": "videos/math.mp4", "annotation": "annotations/math.json", "scene_type": "multi_turn", } ] } ) ) return data_root def test_scenario_cover_and_focus_sampling(tmp_path: Path) -> None: root = _write_scenario_dataset(tmp_path) covered = data.discover_omniinteract_cases( root, ("1q1a", "1q1a_math", "1qna"), num_prompts=5, disable_shuffle=True, scenario_tags=("realtime", "proactive", "nested", "interrupted", "1qna"), ) assert len(covered) == 5 covered_tags = set().union(*(data.case_scenario_tags(case) for case in covered)) assert {"realtime", "proactive", "nested", "interrupted", "1qna"} <= covered_tags focused = data.discover_omniinteract_cases( root, ("1q1a", "1q1a_math", "1qna"), num_prompts=0, disable_shuffle=True, scenario_tags=("nested", "1qna"), scenario_focus=True, ) assert {case.scene_type for case in focused} == {"nested", "1qna"} def test_load_cases_from_video_list_preserves_order_and_caps(tmp_path: Path) -> None: root = _write_dataset(tmp_path) all_cases = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=0, disable_shuffle=True) ordered = list(reversed(all_cases)) video_list = tmp_path / "sampled_cases.jsonl" video_list.write_text("".join(json.dumps(data.sampled_case_row(case)) + "\n" for case in ordered)) capped = data.load_omniinteract_cases_from_video_list(video_list, num_prompts=2) assert [case.video_path for case in capped] == [case.video_path for case in ordered[:2]] full = data.load_omniinteract_cases_from_video_list(video_list, num_prompts=0) assert [case.video_path for case in full] == [case.video_path for case in ordered] oversized = data.load_omniinteract_cases_from_video_list(video_list, num_prompts=99) assert [case.video_path for case in oversized] == [case.video_path for case in ordered] def test_load_cases_from_video_list_requires_existing_video(tmp_path: Path) -> None: root = _write_dataset(tmp_path) case = data.discover_omniinteract_cases(root, ("1q1a",), num_prompts=1, disable_shuffle=True)[0] row = data.sampled_case_row(case) row["video_path"] = str(tmp_path / "missing.mp4") video_list = tmp_path / "bad.jsonl" video_list.write_text(json.dumps(row) + "\n") with pytest.raises(FileNotFoundError, match="video missing"): data.load_omniinteract_cases_from_video_list(video_list, num_prompts=0) def test_get_samples_from_video_list_without_dataset_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: root = _write_dataset(tmp_path) cases = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=0, disable_shuffle=True) video_list = tmp_path / "sampled_cases.jsonl" video_list.write_text("".join(json.dumps(data.sampled_case_row(case)) + "\n" for case in cases)) ref = tmp_path / "ref.wav" ref.touch() parser = TrackingArgumentParser() OmniBenchmarkServingSubcommand.add_cli_args(parser) args = parser.parse_args( [ "--backend", "openai-realtime-duplex", "--dataset-name", "omniinteract", "--model", MODEL, "--endpoint", "/v1/realtime", "--num-prompts", "2", "--omniinteract-ref-audio", str(ref), "--omniinteract-video-list", str(video_list), ] ) preprocess_serve_args(args) monkeypatch.setattr( "vllm_omni.clients.duplex.reference_audio_data_url", lambda _: "data:audio/wav;base64,ref", ) monkeypatch.setattr(benchmark_patch, "prepare_media", lambda *a, **k: (1.0, b"pcm", ["frame"])) samples = benchmark_patch.get_samples(args, None) assert len(samples) == 2 assert [sample.omniinteract_case.video_path for sample in samples] == [case.video_path for case in cases[:2]] def _write_overlapping_cover_dataset(root: Path) -> Path: data_root = root / "data" one_q1a = data_root / "1q1a" (one_q1a / "videos").mkdir(parents=True) (one_q1a / "annotations").mkdir() entries = [] specs = [ ( "pair_a", [ { "question_time": 0.0, "answer_time": 1.0, "question_type": "realtime", "is_interrupted": True, } ], ), ( "pair_b", [ { "question_time": 0.0, "answer_time": 1.0, "question_type": "realtime", "is_interrupted": True, } ], ), ( "proactive_only", [{"question_time": 0.0, "answer_time": 1.0, "question_type": "proactive"}], ), ] for name, annotation in specs: video = one_q1a / "videos" / f"{name}.mp4" ann = one_q1a / "annotations" / f"{name}.json" video.touch() ann.write_text(json.dumps(annotation)) entries.append( { "video": f"videos/{name}.mp4", "annotation": f"annotations/{name}.json", "scene_type": "multi_turn", } ) (one_q1a / "video_json_map.json").write_text(json.dumps({"entries": entries})) return data_root def test_cover_sampling_skips_tags_already_covered(tmp_path: Path) -> None: root = _write_overlapping_cover_dataset(tmp_path) selected = data.discover_omniinteract_cases( root, ("1q1a",), num_prompts=2, disable_shuffle=True, scenario_tags=("realtime", "interrupted", "proactive"), ) names = [case.video_path.stem for case in selected] assert names == ["pair_a", "proactive_only"] selected_tags = set().union(*(data.case_scenario_tags(case) for case in selected)) assert {"realtime", "interrupted", "proactive"} <= selected_tags def _archive(path: Path, member: str) -> None: with tarfile.open(path, "w") as archive: content = b"{}" item = tarfile.TarInfo(member) item.size = len(content) archive.addfile(item, io.BytesIO(content)) def test_archive_is_safe_and_atomically_shared(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): bad = tmp_path / "bad.tar" _archive(bad, "../escape") with pytest.raises(ValueError, match="Unsafe path"): data._extract_archive(bad, tmp_path / "bad-cache") good = tmp_path / "good.tar" _archive(good, "data/1q1a/video_json_map.json") target = tmp_path / "cache" barrier, safe_extract = threading.Barrier(2), data._safe_extract monkeypatch.setattr(data, "_safe_extract", lambda *args: (barrier.wait(), safe_extract(*args))) with ThreadPoolExecutor(2) as executor: roots = list(executor.map(lambda _: data._extract_archive(good, target), range(2))) assert roots[0] == roots[1] assert (roots[0] / "1q1a" / "video_json_map.json").is_file() assert not list(target.glob(".tmp-*")) @pytest.mark.parametrize("dataset_repo", ["org/repo", "org/repo@revision"]) def test_hub_archive_uses_vllm_filesystem(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, dataset_repo: str): source = tmp_path / "source.tar" _archive(source, "data/1q1a/video_json_map.json") class FS: def get_file(self, remote: str, local: str) -> None: assert remote == f"datasets/{dataset_repo}/data.tar.gz" Path(local).write_bytes(source.read_bytes()) monkeypatch.setenv("HF_HOME", str(tmp_path / "hf")) monkeypatch.setattr(data, "hf_fs", lambda: FS()) assert (data.resolve_omniinteract_root(None, dataset_repo) / "1q1a").is_dir() def test_media_commands_are_bounded(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): calls: list[float] = [] commands: list[list[str]] = [] bounded_run = oi._run_media_command def run(command, *, timeout_s, text=False): calls.append(timeout_s) commands.append(command) if command[0] == "ffprobe": return subprocess.CompletedProcess(command, 0, "1.0", "") return subprocess.CompletedProcess(command, 0, b"", b"") monkeypatch.setattr(oi, "_run_media_command", run) duration, pcm, frames = oi.prepare_media(tmp_path / "video.mp4", 1.0, timeout_s=3.0, max_duration_s=1.0) assert (duration, len(pcm), frames, calls) == (1.0, 32_000, [None], [3.0, 3.0, 3.0]) assert commands[1][commands[1].index("-t") + 1] == "1.0" assert commands[1][commands[1].index("-fs") + 1] == "32000" monkeypatch.setattr(subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(subprocess.TimeoutExpired("x", 3))) with pytest.raises(TimeoutError, match="timed out"): bounded_run(["ffmpeg"], timeout_s=3.0) def test_media_duration_is_bounded_before_decode(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): calls = 0 def run(command, *, timeout_s, text=False): nonlocal calls calls += 1 return subprocess.CompletedProcess(command, 0, "601.0", "") monkeypatch.setattr(oi, "_run_media_command", run) with pytest.raises(ValueError, match="Invalid video duration"): oi.prepare_media(tmp_path / "video.mp4", 1.0, timeout_s=3.0, max_duration_s=600.0) assert calls == 1 def test_final_commit_keeps_a_partial_model_unit(): """Exact units retain a sample for the server's correlated final flush.""" events = [{"session": {"capabilities": {"chunk_period_ms": 1000}}}] exact_unit = b"\0\0" * 16_000 assert oi._ensure_final_commit_tail(exact_unit, events) == exact_unit[:-2] assert oi._ensure_final_commit_tail(exact_unit + b"\0\0", events) == exact_unit + b"\0\0" @pytest.mark.parametrize( ("events", "match"), [ (((_done(), 0.0),), "without response.created"), (((_created(), 0.0), (_created(), 0.1)), "duplicate response.created"), (((_created(), 0.0), (_done(), 0.1), (_done(), 0.2)), "duplicate response.done"), ( ( (_created(), 0.0), ({"type": "response.done", "response": {"id": "r1", "status_details": {"type": "failed"}}}, 0.1), ), "reports failure", ), ], ) def test_response_ledger_rejects_identity_errors(events, match: str): with pytest.raises(ValueError, match=match): oi.response_ledger(_collector(*events)) class _CompletionClient: def __init__(self, collector: EventCollector): self.events, self.acks = collector, [] def raise_if_reader_stopped(self) -> None: return None async def send_playback_ack(self, response_id: str, played_ms: int) -> None: self.acks.append((response_id, played_ms)) @pytest.mark.asyncio @pytest.mark.parametrize( ("tail", "succeeds"), [ (((_created(), 1.1), (_done(), 1.2)), True), (((_listen(), 1.1),), True), (((_listen(buffering=True), 1.1),), False), ((({"type": "session.closed", "reason": "timeout"}, 1.1),), False), ], ) async def test_completion_requires_a_final_model_decision(tail, succeeds: bool): events = _collector(({"type": "input_audio_buffer.committed"}, 1.0), *tail) coroutine = oi.wait_for_session_completion( _CompletionClient(events), oi._Playback(), commit_from=0, timeout_s=0.08, settle_s=0.0 ) if succeeds: assert await coroutine == 0 else: with pytest.raises((TimeoutError, RuntimeError)): await coroutine @pytest.mark.asyncio async def test_deferred_commit_ignores_the_prior_response_terminal(): events = _collector( (_created("old"), 0.0), ({"type": "input_audio_buffer.committed", "event": {"overlap_deferred": True}}, 1.0), (_done("old"), 1.1), (_listen(), 1.2), ) assert ( await oi.wait_for_session_completion( _CompletionClient(events), oi._Playback(), commit_from=1, timeout_s=0.1, settle_s=0.0 ) == 1 ) @pytest.mark.asyncio async def test_playback_acks_incrementally_along_the_serial_clock(): # 1 s of audio playing from t=10.0: the first ack checkpoints the # response as soon as audio arrives, later acks report cumulative # progress at least every _ACK_STEP_MS of new audio, and the completion # ack lands exactly at the total once the serial clock drains it. collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01)) client, playback = _CompletionClient(collector), oi._Playback() await playback.acknowledge(client, now=10.05) assert client.acks == [("r1", 50)] # checkpoint ack, reserves the slot await playback.acknowledge(client, now=10.1) assert client.acks == [("r1", 50)] # step not reached, no spam await playback.acknowledge(client, now=10.3) assert client.acks == [("r1", 50), ("r1", 300)] await playback.acknowledge(client, now=11.0) assert client.acks == [("r1", 50), ("r1", 300), ("r1", 1000)] assert playback.completion_acked == {"r1"} await playback.acknowledge(client, now=11.5) assert client.acks[-1] == ("r1", 1000) # completion acked once @pytest.mark.asyncio async def test_playback_checkpoint_ack_fires_before_audio_plays(): # Audio queued but none played yet: a 0 ms checkpoint ack still goes out # so a later input commit cannot displace the response's history slot. collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01)) client, playback = _CompletionClient(collector), oi._Playback() await playback.acknowledge(client, now=10.0) assert client.acks == [("r1", 0)] @pytest.mark.asyncio async def test_acks_continue_after_committed_input(): # A later committed user input must not stop the acks: the checkpointed # response's history slot is updated in place, so the drain ack for the # tail is still delivered afterwards. collector = _collector( (_audio(samples=24_000), 10.0), (_done(), 10.01), ({"type": "input_audio_buffer.committed"}, 10.4), ) client, playback = _CompletionClient(collector), oi._Playback() await playback.acknowledge(client, now=10.05) assert client.acks == [("r1", 50)] await playback.acknowledge(client, now=11.5) assert client.acks == [("r1", 50), ("r1", 1000)] assert playback.completion_acked == {"r1"} def test_tolerated_playback_ack_rejection_is_a_warning_not_a_failure(): collector = _collector( ( { "type": "error", "error": {"type": "invalid_request_error", "code": "playback_ack_too_late"}, }, 1.0, ), ) warnings: list[str] = [] oi._raise_if_session_terminated(collector, 0, warnings=warnings) assert warnings == ["tolerated in-flight server rejection: playback_ack_too_late"] collector.add({"type": "error", "error": {"code": "bad_event", "message": "boom"}}, received_at_s=1.1) with pytest.raises(RuntimeError, match="bad_event"): oi._raise_if_session_terminated(collector, 0, warnings=warnings) def test_our_own_close_is_expected_even_though_the_server_stamps_a_reason(): """``session.close`` is answered with ``session.closed`` carrying ``client_close``. The guard exists to catch a session that ended for a reason we did not ask for. A close we requested is not that, whether or not the server names it. """ collector = _collector(({"type": "session.closed", "reason": "client_close"}, 1.0)) oi._raise_if_session_terminated(collector, 0, explicit_close_from=0) collector = _collector(({"type": "session.closed", "event": {"reason": "client_close"}}, 1.0)) oi._raise_if_session_terminated(collector, 0, explicit_close_from=0) collector = _collector(({"type": "session.closed", "reason": "disconnect"}, 1.0)) with pytest.raises(RuntimeError, match="Unexpected session.closed: disconnect"): oi._raise_if_session_terminated(collector, 0, explicit_close_from=0) collector = _collector(({"type": "session.expired", "reason": "timeout"}, 1.0)) with pytest.raises(RuntimeError, match="session.expired: timeout"): oi._raise_if_session_terminated(collector, 0, explicit_close_from=0) @pytest.mark.parametrize( ("event", "match"), [ (_audio(rate=16_000), "24000 Hz"), ({**_audio(), "format": "opus"}, "pcm16"), ({**_audio(), "delta": "!"}, "base64"), ], ) def test_playback_rejects_non_official_audio(event: dict[str, object], match: str): playback = oi._Playback() with pytest.raises(ValueError, match=match): playback.ingest(_collector((event, 1.0))) def test_artifacts_publish_official_bundle_and_sparse_deferred_state(tmp_path: Path): case, collector, result = _successful_output(tmp_path) summary = _write_success( tmp_path / "out", case, collector, result, ) context = result._artifact_context assert result.success and result.eligible_for_official_eval assert summary["scene_type"] == "multi_turn" assert context and context.spans assert all("delta" not in event for event in context.events if event["type"] == "response.output_audio.delta") oi.publish_deferred_case_artifacts(tmp_path / "out", case, result) directory = oi._output_dir(tmp_path / "out", case) assert all((directory / name).is_file() for name in oi.SUCCESS_ARTIFACTS) with wave.open(str(directory / "output.wav")) as output: assert (output.getframerate(), output.getnframes()) == (24_000, 24_000) transcript = json.loads((directory / "wav_transcript.json").read_text()) assert transcript["text"] == "hello" and transcript["chunks"][0]["timestamp"] == [0.0, 0.1] assert "serialized playback queue" in transcript["timestamp_semantics"] assert result._artifact_context is None @pytest.mark.parametrize( ("audio_time", "status", "reason"), [(10.95, "completed", "audio_clipped"), (10.0, "cancelled", "cancelled_response")], ) def test_ineligible_outputs_are_excluded_from_manifest(tmp_path: Path, audio_time: float, status: str, reason: str): case, collector, result = _successful_output(tmp_path, audio_time=audio_time, status=status) _write_success(tmp_path / "out", case, collector, result, require_response=False) assert reason in result.official_eval_ineligible_reasons oi.write_batch_artifacts(tmp_path / "out", [case], [result]) assert (tmp_path / "out" / "official_eval_manifest.jsonl").read_text() == "" sampled = [ json.loads(line) for line in (tmp_path / "out" / "sampled_cases.jsonl").read_text().splitlines() if line.strip() ] assert sampled == [ { "video_path": str(case.video_path.resolve()), "output_name": data.official_output_name(case), "subset": case.subset, } ] def test_official_output_name_matches_minicpmo_batch_layout(tmp_path: Path): one_q1a = _case(tmp_path, subset="1q1a", name="videos/clip_a.mp4") one_qna = _case(tmp_path, subset="1qna", name="videos_bench/nested/guide.mp4") assert data.official_output_name(one_q1a) == "1q1a/videos__clip_a" assert data.official_output_name(one_qna) == "1qna/nested__guide" row = data.sampled_case_row(one_q1a) assert row["video_path"] == str(one_q1a.video_path.resolve()) assert row["output_name"] == "1q1a/videos__clip_a" assert row["subset"] == "1q1a" def test_sampled_cases_jsonl_lists_every_case_including_failures(tmp_path: Path): ok_case = _case(tmp_path, name="videos/ok.mp4") bad_case = _case(tmp_path, name="videos/bad.mp4") ok_result = oi.OmniInteractCaseResult( ok_case.subset, str(ok_case.video_path), str(oi._output_dir(tmp_path / "out", ok_case)), success=True ) ok_result.eligible_for_official_eval = True bad_result = oi.OmniInteractCaseResult( bad_case.subset, str(bad_case.video_path), str(oi._output_dir(tmp_path / "out", bad_case)), success=False ) oi.write_batch_artifacts(tmp_path / "out", [ok_case, bad_case], [ok_result, bad_result]) sampled = [ json.loads(line) for line in (tmp_path / "out" / "sampled_cases.jsonl").read_text().splitlines() if line.strip() ] assert [row["output_name"] for row in sampled] == ["1q1a/videos__ok", "1q1a/videos__bad"] manifest = [ json.loads(line) for line in (tmp_path / "out" / "official_eval_manifest.jsonl").read_text().splitlines() if line.strip() ] assert len(manifest) == 1 assert manifest[0]["sample_id"].startswith("1q1a__") def test_artifacts_require_complete_audio_and_transcript_response(tmp_path: Path): case, _, result = _successful_output(tmp_path) collector = _collector((_created(), 10.0), (_audio(), 10.0), (_done(), 10.1)) with pytest.raises(ValueError, match="audio and transcript"): _write_success(tmp_path, case, collector, result) def test_publication_failure_removes_partial_success_bundle(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case, collector, result = _successful_output(tmp_path) _write_success(tmp_path, case, collector, result) original = oi._atomic_write_json def fail(path: Path, value: object) -> None: if path.name == "events.json": raise OSError("disk full") original(path, value) monkeypatch.setattr(oi, "_atomic_write_json", fail) with pytest.raises(OSError, match="disk full"): oi.publish_deferred_case_artifacts(tmp_path, case, result) directory = oi._output_dir(tmp_path, case) assert not any((directory / name).exists() for name in oi.SUCCESS_ARTIFACTS) assert not result.success and "artifact_write_failed" in result.official_eval_ineligible_reasons def test_atomic_write_preserves_destination_on_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): destination = tmp_path / "output.wav" destination.write_bytes(b"old") monkeypatch.setattr( "vllm_omni.clients.duplex.write_pcm16_wav", lambda *a, **k: (_ for _ in ()).throw(OSError("write")), ) with pytest.raises(OSError): oi._atomic_write_wav(destination, b"\0\0", 24_000) assert destination.read_bytes() == b"old" assert not list(tmp_path.glob(".*.tmp")) class _RealtimeClient(oi._RealtimeSession): """Fake transport over the real session wrapper: URL and session-config construction stay real, only the network-facing calls are stubbed.""" instances: list[_RealtimeClient] = [] def __init__(self, config: oi.OmniInteractBenchmarkConfig, reference_audio: str): super().__init__(config, reference_audio) self.acks: list[tuple[str, int]] = [] self.instances.append(self) async def __aenter__(self): self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}}) return self async def __aexit__(self, *args): return None async def send(self, event: dict[str, object]) -> None: return None async def commit(self) -> None: now = time.monotonic() for event, offset in ( ({"type": "input_audio_buffer.committed"}, 0.0), (_created(), 0.001), (_audio(samples=24), 0.002), (_text(value="answer"), 0.003), (_done(), 0.004), ): self.events.add(event, received_at_s=now + offset) async def send_playback_ack(self, response_id: str, played_ms: int) -> None: self.acks.append((response_id, played_ms)) async def close_session(self, **kwargs) -> None: self.events.add({"type": "session.closed"}) def raise_if_reader_stopped(self) -> None: return None @pytest.mark.asyncio async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case = _case(tmp_path) config = oi.OmniInteractBenchmarkConfig( model=MODEL, output_root=tmp_path / "out", ref_audio="ref.wav", require_response=True, extra_body={"custom": "value"}, ) prepared = data.OmniInteractPreparedInput(1.0, b"\0\0\0\0", ("frame",), "data:audio/wav;base64,ref") monkeypatch.setattr(oi, "_RealtimeSession", _RealtimeClient) monkeypatch.setattr(oi, "_COMPLETION_SETTLE_S", 0.0) result = await oi.run_omniinteract_case( case, config, request_index=0, prepared_input=prepared, ) assert result.success and result.transcript == "answer" and result._artifact_context is not None assert not config.output_root.exists() assert "autostart=0" in _RealtimeClient.instances[-1].url session_config = _RealtimeClient.instances[-1].session_config assert session_config.extra_body["custom"] == "value" assert "native_duplex" not in session_config.extra_body assert session_config.ref_audio == "data:audio/wav;base64,ref" acks = _RealtimeClient.instances[-1].acks # Cumulative acks for the one response: an optional 0 ms checkpoint the # moment its audio arrived, then the exact total once drained. assert acks[-1] == ("r1", 1) assert acks in ([("r1", 1)], [("r1", 0), ("r1", 1)]) def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): _write_dataset(tmp_path) ref = tmp_path / "ref.wav" ref.touch() parser = TrackingArgumentParser() OmniBenchmarkServingSubcommand.add_cli_args(parser) args = parser.parse_args( [ "--backend", "openai-realtime-duplex", "--dataset-name", "omniinteract", "--dataset-path", str(tmp_path), "--model", MODEL, "--endpoint", "/v1/realtime", "--num-prompts", "0", "--omniinteract-ref-audio", str(ref), ] ) preprocess_serve_args(args) assert args.endpoint == "/v1/realtime" # The benchmark patch module imports the duplex client library lazily # (it must stay out of the CLI import graph), so patch the provider. monkeypatch.setattr( "vllm_omni.clients.duplex.reference_audio_data_url", lambda _: "data:audio/wav;base64,ref", ) monkeypatch.setattr(benchmark_patch, "prepare_media", lambda *a, **k: (1.0, b"pcm", ["frame"])) samples = benchmark_patch.get_samples(args, None) assert len(samples) == args.num_prompts == 3 assert all(sample.omniinteract_prepared_input.video_frames == ("frame",) for sample in samples) def _request(case: data.OmniInteractCase, options: data.OmniInteractSessionOptions) -> RequestFuncInput: request = RequestFuncInput( model=MODEL, model_name=MODEL, prompt="", api_url="http://server:8000/v1/realtime", prompt_len=0, output_len=0, logprobs=None, multi_modal_content=None, ignore_eos=False, request_id="measured-0", ) request.omniinteract_case, request.omniinteract_options = case, options return request @pytest.mark.asyncio async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case, options = _case(tmp_path), _session_options(tmp_path) request = _request(case, options) output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None) assert not output.success and "prepared before benchmark request timing" in output.error prepared = data.OmniInteractPreparedInput(1.0, b"pcm", ("frame",), "data:audio/wav;base64,ref") request.omniinteract_prepared_input = prepared request.extra_headers = {"X-Proxy": "test"} request.extra_body = {"custom": "value"} monkeypatch.setenv("OPENAI_API_KEY", "secret") seen = {} async def run(_case_arg, config, **kwargs): seen["headers"] = config.extra_headers seen["extra_body"] = config.extra_body seen.update(kwargs) return oi.OmniInteractCaseResult( case.subset, str(case.video_path), str(options.output_root), success=True, transcript="timing metadata missing", output_tokens=0, duplex_request_metrics=[{"request_metrics": {"ttft_ms": 1.0}}], duplex_session_metrics={"ttft_ms": 10.0}, ) monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run) output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None) assert output.success and seen["prepared_input"] is prepared assert seen["headers"] == {"Authorization": "Bearer secret", "X-Proxy": "test", "x-request-id": "measured-0"} assert seen["extra_body"] == {"custom": "value"} assert output.itl == [] assert output.tpot_measured is False assert seen["capture_artifacts"] is True @pytest.mark.asyncio @pytest.mark.parametrize( ("request_metrics", "expected_itl", "expected_text_latency", "expected_tpot_measured"), [ ( [ {"stage0_tokens": {"output_token_count": 3, "itls_ms": [10.0, 20.0]}}, {"stage0_tokens": {"output_token_count": 2, "itls_ms": [30.0]}}, ], [0.01, 0.02, 0.03], 0.16, True, ), ( [ {"stage0_tokens": {"output_token_count": 3, "tpot_ms": 10.0}}, {"stage0_tokens": {"output_token_count": 2, "tpot_ms": 40.0}}, ], [], 0.18, True, ), ( [ {"stage0_tokens": {"output_token_count": 3, "tpot_ms": 0.0}}, {"stage0_tokens": {"output_token_count": 2, "tpot_ms": 0.0}}, ], [], 0.1, False, ), ], ) async def test_adapter_reports_exact_or_weighted_token_timing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, request_metrics: list[dict[str, object]], expected_itl: list[float], expected_text_latency: float, expected_tpot_measured: bool, ): case, options = _case(tmp_path), _session_options(tmp_path) request = _request(case, options) request.omniinteract_prepared_input = data.OmniInteractPreparedInput( 1.0, b"pcm", ("frame",), "data:audio/wav;base64,ref" ) async def run(*args, **kwargs): return oi.OmniInteractCaseResult( case.subset, str(case.video_path), str(options.output_root), success=True, output_tokens=5, duplex_request_metrics=request_metrics, duplex_session_metrics={"ttft_ms": 100.0}, ) monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run) output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None) assert output.success assert output.itl == expected_itl assert output.text_latency == pytest.approx(expected_text_latency) assert output.tpot_measured is expected_tpot_measured def test_batch_finalization_publishes_only_measured_results(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case, options = _case(tmp_path), _session_options(tmp_path) options.output_root.mkdir() sample = _sample(case, options) result = oi.OmniInteractCaseResult( case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True ) output = benchmark_patch.MixRequestFuncOutput(success=True) output.omniinteract_case_result = result published = [] monkeypatch.setattr(benchmark_patch, "publish_deferred_case_artifacts", lambda *args: published.append(args)) summary = benchmark_patch._finalize_omniinteract_batch([sample], [output]) assert summary and summary["total"] == summary["success"] == summary["eligible_for_official_eval"] == 1 assert summary["failed"] == summary["successful_but_ineligible"] == summary["audio_clipped_bytes"] == 0 assert published == [(options.output_root, case, result)] def test_case_artifact_failures_do_not_discard_serving_metrics(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case, options = _case(tmp_path), _session_options(tmp_path) sample = _sample(case, options) result = oi.OmniInteractCaseResult( case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True ) output = benchmark_patch.MixRequestFuncOutput(success=True, error="inference warning") output.omniinteract_case_result = result monkeypatch.setattr( benchmark_patch, "publish_deferred_case_artifacts", lambda *args: (_ for _ in ()).throw(OSError("case disk full")), ) monkeypatch.setattr( benchmark_patch, "write_failure_artifacts", lambda *args: (_ for _ in ()).throw(OSError("failure disk full")), ) monkeypatch.setattr(benchmark_patch, "write_omniinteract_batch_artifacts", lambda *args: None) summary = benchmark_patch._finalize_omniinteract_batch([sample], [output]) metrics, _ = calculate_metrics( [], [output], 1.0, None, [50.0], {}, TaskType.GENERATION, [], None, float("inf"), 1.0, ) assert summary and summary["failed"] == 1 assert summary["artifacts_complete"] is False assert len(summary["artifact_errors"]) == 2 assert output.success and metrics.completed == 1 assert output.error.startswith("inference warning\n") assert "Artifact publication failed: case disk full" in output.error def test_batch_artifact_failure_removes_partial_batch_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): case, options = _case(tmp_path), _session_options(tmp_path) sample = _sample(case, options) result = oi.OmniInteractCaseResult( case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True ) output = benchmark_patch.MixRequestFuncOutput(success=True) output.omniinteract_case_result = result monkeypatch.setattr(benchmark_patch, "publish_deferred_case_artifacts", lambda *args: None) def fail_batch(root, *args): root.mkdir(parents=True, exist_ok=True) (root / "batch_summary.json").write_text("partial") raise OSError("batch disk full") monkeypatch.setattr(benchmark_patch, "write_omniinteract_batch_artifacts", fail_batch) summary = benchmark_patch._finalize_omniinteract_batch([sample], [output]) assert summary and summary["success"] == 1 and summary["artifacts_complete"] is False assert summary["artifact_errors"] == ["Batch artifact publication failed: batch disk full"] assert output.success assert not (options.output_root / "batch_summary.json").exists() @pytest.mark.parametrize( ("field", "value", "match"), [ ("omniinteract_ref_audio", None, "ref-audio"), ("endpoint", "/v1/completions", "v1/realtime"), ("max_concurrency", 0, "positive"), ("skip_tokenizer_init", True, "skip-tokenizer-init"), ], ) def test_cli_rejects_incompatible_omniinteract_options(field: str, value, match: str): args = argparse.Namespace( dataset_name="omniinteract", backend="openai-realtime-duplex", endpoint="/v1/realtime", omniinteract_ref_audio="ref.wav", max_concurrency=1, skip_tokenizer_init=False, ) setattr(args, field, value) with pytest.raises(ValueError, match=match): preprocess_serve_args(args) def test_whole_benchmark_lock_serializes_a_shared_output_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): entered, release = threading.Event(), threading.Event() calls: list[int] = [] async def fake_main_async(args): calls.append(len(calls)) if len(calls) == 1: entered.set() await asyncio.to_thread(release.wait) return {"total": 1} monkeypatch.setattr(benchmark_serve, "main_async", fake_main_async) args = argparse.Namespace(dataset_name="omniinteract", omniinteract_output_dir=tmp_path, extra_body=None) with ThreadPoolExecutor(2) as executor: first = executor.submit(benchmark_serve.main, args) assert entered.wait(1) second = executor.submit(benchmark_serve.main, args) time.sleep(0.05) assert calls == [0] release.set() assert first.result() == second.result() == {"total": 1} assert calls == [0, 1]