[Benchmark] Add local OmniInteract realtime benchmark (#6522)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
c608910d20
commit
497c537c6f
16 files changed
+3025
-149
No files matched your search
@@ -0,0 +1,778 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
import wave
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
from vllm.benchmarks.serve import TaskType
|
||||
|
||||
from vllm_omni.benchmarks import omniinteract as oi
|
||||
from vllm_omni.benchmarks import serve as benchmark_serve
|
||||
from vllm_omni.benchmarks.data_modules import omniinteract_dataset as data
|
||||
from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
|
||||
from vllm_omni.benchmarks.patch import patch as benchmark_patch
|
||||
from vllm_omni.entrypoints.cli.benchmark.cli_args import preprocess_serve_args
|
||||
from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||
MODEL = "openbmb/MiniCPM-o-4_5"
|
||||
|
||||
|
||||
def _case(tmp_path: Path, *, subset: str = "1q1a", name: str = "video.mp4") -> data.OmniInteractCase:
|
||||
video, annotation = tmp_path / name, tmp_path / f"{Path(name).stem}.json"
|
||||
video.parent.mkdir(parents=True, exist_ok=True)
|
||||
video.touch()
|
||||
annotation.write_text("{}")
|
||||
return data.OmniInteractCase(subset, name, video, annotation, "multi_turn")
|
||||
|
||||
|
||||
def _collector(*events: tuple[dict[str, object], float]) -> RealtimeEventCollector:
|
||||
collector = RealtimeEventCollector()
|
||||
for event, received_at in events:
|
||||
collector.add(event, received_at_s=received_at)
|
||||
return collector
|
||||
|
||||
|
||||
def _created(response_id: str = "r1") -> dict[str, object]:
|
||||
return {"type": "response.created", "response": {"id": response_id}}
|
||||
|
||||
|
||||
def _done(response_id: str = "r1", status: str = "completed") -> dict[str, object]:
|
||||
return {"type": "response.done", "response": {"id": response_id, "status": status}}
|
||||
|
||||
|
||||
def _audio(
|
||||
response_id: str = "r1",
|
||||
*,
|
||||
samples: int = 2400,
|
||||
rate: int | None = 24_000,
|
||||
value: int = 1,
|
||||
) -> dict[str, object]:
|
||||
event: dict[str, object] = {
|
||||
"type": "response.audio.delta",
|
||||
"response_id": response_id,
|
||||
"format": "pcm16",
|
||||
"delta": base64.b64encode(bytes((value, 0)) * samples).decode(),
|
||||
}
|
||||
if rate is not None:
|
||||
event["sample_rate_hz"] = rate
|
||||
return event
|
||||
|
||||
|
||||
def _text(response_id: str = "r1", value: str = "hello") -> dict[str, object]:
|
||||
return {"type": "response.audio_transcript.delta", "response_id": response_id, "delta": value}
|
||||
|
||||
|
||||
def _listen(*, buffering: bool = False) -> dict[str, object]:
|
||||
return {
|
||||
"type": "response.listen",
|
||||
"response": {"metadata": {"model_listen": True, "buffering": buffering}},
|
||||
}
|
||||
|
||||
|
||||
def _session_options(tmp_path: Path) -> data.OmniInteractSessionOptions:
|
||||
return data.OmniInteractSessionOptions(tmp_path / "artifacts", 10.0, 5.0, "reference.wav", True)
|
||||
|
||||
|
||||
def _sample(case: data.OmniInteractCase, options: data.OmniInteractSessionOptions):
|
||||
return data.OmniInteractSampleRequest(
|
||||
prompt="",
|
||||
prompt_len=0,
|
||||
expected_output_len=0,
|
||||
multi_modal_data=None,
|
||||
request_id="measured-0",
|
||||
omniinteract_case=case,
|
||||
omniinteract_options=options,
|
||||
)
|
||||
|
||||
|
||||
def _write_dataset(root: Path) -> Path:
|
||||
data_root = root / "data"
|
||||
for subset in ("1q1a", "1q1a_math"):
|
||||
subset_root = data_root / subset
|
||||
(subset_root / "videos").mkdir(parents=True)
|
||||
(subset_root / "annotations").mkdir()
|
||||
(subset_root / "videos" / f"{subset}.mp4").touch()
|
||||
(subset_root / "annotations" / f"{subset}.json").write_text("{}")
|
||||
(subset_root / "video_json_map.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"video": f"videos/{subset}.mp4",
|
||||
"annotation": f"annotations/{subset}.json",
|
||||
"scene_type": "multi_turn",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
one_to_many = data_root / "1qna"
|
||||
(one_to_many / "videos_bench" / "nested").mkdir(parents=True)
|
||||
(one_to_many / "annotations" / "nested").mkdir(parents=True)
|
||||
(one_to_many / "videos_bench" / "nested" / "guide.mp4").touch()
|
||||
(one_to_many / "annotations" / "nested" / "guide.json").write_text("{}")
|
||||
return data_root
|
||||
|
||||
|
||||
def _successful_output(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
audio_time: float = 10.0,
|
||||
status: str = "completed",
|
||||
) -> tuple[data.OmniInteractCase, RealtimeEventCollector, oi.OmniInteractCaseResult]:
|
||||
case = _case(tmp_path)
|
||||
collector = _collector(
|
||||
(_created(), 9.9),
|
||||
(_audio(), audio_time),
|
||||
(_text(), audio_time + 0.01),
|
||||
(_done(status=status), audio_time + 0.1),
|
||||
)
|
||||
result = oi.OmniInteractCaseResult(case.subset, str(case.video_path), "")
|
||||
return case, collector, result
|
||||
|
||||
|
||||
def _write_success(
|
||||
root: Path,
|
||||
case: data.OmniInteractCase,
|
||||
collector: RealtimeEventCollector,
|
||||
result: oi.OmniInteractCaseResult,
|
||||
**kwargs,
|
||||
):
|
||||
require_response = kwargs.pop("require_response", True)
|
||||
return oi.prepare_success_artifacts(
|
||||
root,
|
||||
case,
|
||||
collector,
|
||||
stream_start=10.0,
|
||||
video_duration_s=1.0,
|
||||
require_response=require_response,
|
||||
result=result,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_dataset_discovers_official_layouts_and_total_selection(tmp_path: Path):
|
||||
root = _write_dataset(tmp_path)
|
||||
all_cases = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=0, disable_shuffle=True)
|
||||
selected = data.discover_omniinteract_cases(root, data.OMNIINTERACT_SUBSETS, num_prompts=2, disable_shuffle=True)
|
||||
assert [case.subset for case in all_cases] == ["1q1a", "1q1a_math", "1qna"]
|
||||
assert all_cases[-1].video_rel == "videos_bench/nested/guide.mp4"
|
||||
assert [case.subset for case in selected] == ["1q1a", "1q1a_math"]
|
||||
|
||||
|
||||
def _archive(path: Path, member: str) -> None:
|
||||
with tarfile.open(path, "w") as archive:
|
||||
content = b"{}"
|
||||
item = tarfile.TarInfo(member)
|
||||
item.size = len(content)
|
||||
archive.addfile(item, io.BytesIO(content))
|
||||
|
||||
|
||||
def test_archive_is_safe_and_atomically_shared(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
bad = tmp_path / "bad.tar"
|
||||
_archive(bad, "../escape")
|
||||
with pytest.raises(ValueError, match="Unsafe path"):
|
||||
data._extract_archive(bad, tmp_path / "bad-cache")
|
||||
|
||||
good = tmp_path / "good.tar"
|
||||
_archive(good, "data/1q1a/video_json_map.json")
|
||||
target = tmp_path / "cache"
|
||||
barrier, safe_extract = threading.Barrier(2), data._safe_extract
|
||||
monkeypatch.setattr(data, "_safe_extract", lambda *args: (barrier.wait(), safe_extract(*args)))
|
||||
with ThreadPoolExecutor(2) as executor:
|
||||
roots = list(executor.map(lambda _: data._extract_archive(good, target), range(2)))
|
||||
assert roots[0] == roots[1]
|
||||
assert (roots[0] / "1q1a" / "video_json_map.json").is_file()
|
||||
assert not list(target.glob(".tmp-*"))
|
||||
|
||||
|
||||
def test_hub_archive_uses_vllm_filesystem(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
source = tmp_path / "source.tar"
|
||||
_archive(source, "data/1q1a/video_json_map.json")
|
||||
|
||||
class FS:
|
||||
def get_file(self, remote: str, local: str) -> None:
|
||||
assert remote == "datasets/org/repo/data.tar.gz"
|
||||
Path(local).write_bytes(source.read_bytes())
|
||||
|
||||
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf"))
|
||||
monkeypatch.setattr(data, "hf_fs", lambda: FS())
|
||||
assert (data.resolve_omniinteract_root(None, "org/repo") / "1q1a").is_dir()
|
||||
|
||||
|
||||
def test_media_commands_are_bounded(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
|
||||
calls: list[float] = []
|
||||
commands: list[list[str]] = []
|
||||
bounded_run = oi._run_media_command
|
||||
|
||||
def run(command, *, timeout_s, text=False):
|
||||
calls.append(timeout_s)
|
||||
commands.append(command)
|
||||
if command[0] == "ffprobe":
|
||||
return subprocess.CompletedProcess(command, 0, "1.0", "")
|
||||
return subprocess.CompletedProcess(command, 0, b"", b"")
|
||||
|
||||
monkeypatch.setattr(oi, "_run_media_command", run)
|
||||
duration, pcm, frames = oi.prepare_media(tmp_path / "video.mp4", 1.0, timeout_s=3.0, max_duration_s=1.0)
|
||||
assert (duration, len(pcm), frames, calls) == (1.0, 32_000, [None], [3.0, 3.0, 3.0])
|
||||
assert commands[1][commands[1].index("-t") + 1] == "1.0"
|
||||
assert commands[1][commands[1].index("-fs") + 1] == "32000"
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(subprocess.TimeoutExpired("x", 3)))
|
||||
with pytest.raises(TimeoutError, match="timed out"):
|
||||
bounded_run(["ffmpeg"], timeout_s=3.0)
|
||||
|
||||
|
||||
def test_media_duration_is_bounded_before_decode(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
|
||||
calls = 0
|
||||
|
||||
def run(command, *, timeout_s, text=False):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return subprocess.CompletedProcess(command, 0, "601.0", "")
|
||||
|
||||
monkeypatch.setattr(oi, "_run_media_command", run)
|
||||
with pytest.raises(ValueError, match="Invalid video duration"):
|
||||
oi.prepare_media(tmp_path / "video.mp4", 1.0, timeout_s=3.0, max_duration_s=600.0)
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_final_commit_keeps_a_partial_model_unit():
|
||||
"""Exact units retain a sample for the server's correlated final flush."""
|
||||
|
||||
events = [{"session": {"capabilities": {"chunk_period_ms": 1000}}}]
|
||||
exact_unit = b"\0\0" * 16_000
|
||||
assert oi._ensure_final_commit_tail(exact_unit, events) == exact_unit[:-2]
|
||||
assert oi._ensure_final_commit_tail(exact_unit + b"\0\0", events) == exact_unit + b"\0\0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("events", "match"),
|
||||
[
|
||||
(((_done(), 0.0),), "without response.created"),
|
||||
(((_created(), 0.0), (_created(), 0.1)), "duplicate response.created"),
|
||||
(((_created(), 0.0), (_done(), 0.1), (_done(), 0.2)), "duplicate response.done"),
|
||||
(
|
||||
(
|
||||
(_created(), 0.0),
|
||||
({"type": "response.done", "response": {"id": "r1", "status_details": {"type": "failed"}}}, 0.1),
|
||||
),
|
||||
"reports failure",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_response_ledger_rejects_identity_errors(events, match: str):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
oi.response_ledger(_collector(*events))
|
||||
|
||||
|
||||
class _CompletionClient:
|
||||
def __init__(self, collector: RealtimeEventCollector):
|
||||
self.events, self.acks = collector, []
|
||||
|
||||
def raise_if_reader_stopped(self) -> None:
|
||||
return None
|
||||
|
||||
async def send_playback_ack(self, response_id: str, played_ms: int) -> None:
|
||||
self.acks.append((response_id, played_ms))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tail", "succeeds"),
|
||||
[
|
||||
(((_created(), 1.1), (_done(), 1.2)), True),
|
||||
(((_listen(), 1.1),), True),
|
||||
(((_listen(buffering=True), 1.1),), False),
|
||||
((({"type": "session.closed", "reason": "timeout"}, 1.1),), False),
|
||||
],
|
||||
)
|
||||
async def test_completion_requires_a_final_model_decision(tail, succeeds: bool):
|
||||
events = _collector(({"type": "input_audio_buffer.committed"}, 1.0), *tail)
|
||||
coroutine = oi.wait_for_session_completion(
|
||||
_CompletionClient(events), oi._Playback(), commit_from=0, timeout_s=0.08, settle_s=0.0
|
||||
)
|
||||
if succeeds:
|
||||
assert await coroutine == 0
|
||||
else:
|
||||
with pytest.raises((TimeoutError, RuntimeError)):
|
||||
await coroutine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_commit_ignores_the_prior_response_terminal():
|
||||
events = _collector(
|
||||
(_created("old"), 0.0),
|
||||
({"type": "input_audio_buffer.committed", "event": {"overlap_deferred": True}}, 1.0),
|
||||
(_done("old"), 1.1),
|
||||
(_listen(), 1.2),
|
||||
)
|
||||
assert (
|
||||
await oi.wait_for_session_completion(
|
||||
_CompletionClient(events), oi._Playback(), commit_from=1, timeout_s=0.1, settle_s=0.0
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_playback_acks_only_after_serial_audio_drain():
|
||||
collector = _collector((_audio(samples=2400), 10.0), (_done(), 10.01))
|
||||
client, playback = _CompletionClient(collector), oi._Playback()
|
||||
await playback.acknowledge(client, now=10.05)
|
||||
assert client.acks == []
|
||||
await playback.acknowledge(client, now=10.1)
|
||||
assert client.acks == [("r1", 100)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "match"),
|
||||
[
|
||||
(_audio(rate=16_000), "24000 Hz"),
|
||||
({**_audio(), "format": "opus"}, "pcm16"),
|
||||
({**_audio(), "delta": "!"}, "base64"),
|
||||
],
|
||||
)
|
||||
def test_playback_rejects_non_official_audio(event: dict[str, object], match: str):
|
||||
playback = oi._Playback()
|
||||
with pytest.raises(ValueError, match=match):
|
||||
playback.ingest(_collector((event, 1.0)))
|
||||
|
||||
|
||||
def test_artifacts_publish_official_bundle_and_sparse_deferred_state(tmp_path: Path):
|
||||
case, collector, result = _successful_output(tmp_path)
|
||||
summary = _write_success(
|
||||
tmp_path / "out",
|
||||
case,
|
||||
collector,
|
||||
result,
|
||||
)
|
||||
context = result._artifact_context
|
||||
assert result.success and result.eligible_for_official_eval
|
||||
assert summary["scene_type"] == "multi_turn"
|
||||
assert context and context.spans
|
||||
assert all("delta" not in event for event in context.events if event["type"] == "response.audio.delta")
|
||||
oi.publish_deferred_case_artifacts(tmp_path / "out", case, result)
|
||||
directory = oi._output_dir(tmp_path / "out", case)
|
||||
assert all((directory / name).is_file() for name in oi.SUCCESS_ARTIFACTS)
|
||||
with wave.open(str(directory / "output.wav")) as output:
|
||||
assert (output.getframerate(), output.getnframes()) == (24_000, 24_000)
|
||||
transcript = json.loads((directory / "wav_transcript.json").read_text())
|
||||
assert transcript["text"] == "hello" and transcript["chunks"][0]["timestamp"] == [0.0, 0.1]
|
||||
assert "serialized playback queue" in transcript["timestamp_semantics"]
|
||||
assert result._artifact_context is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("audio_time", "status", "reason"),
|
||||
[(10.95, "completed", "audio_clipped"), (10.0, "cancelled", "cancelled_response")],
|
||||
)
|
||||
def test_ineligible_outputs_are_excluded_from_manifest(tmp_path: Path, audio_time: float, status: str, reason: str):
|
||||
case, collector, result = _successful_output(tmp_path, audio_time=audio_time, status=status)
|
||||
_write_success(tmp_path / "out", case, collector, result, require_response=False)
|
||||
assert reason in result.official_eval_ineligible_reasons
|
||||
oi.write_batch_artifacts(tmp_path / "out", [case], [result])
|
||||
assert (tmp_path / "out" / "official_eval_manifest.jsonl").read_text() == ""
|
||||
|
||||
|
||||
def test_artifacts_require_complete_audio_and_transcript_response(tmp_path: Path):
|
||||
case, _, result = _successful_output(tmp_path)
|
||||
collector = _collector((_created(), 10.0), (_audio(), 10.0), (_done(), 10.1))
|
||||
with pytest.raises(ValueError, match="audio and transcript"):
|
||||
_write_success(tmp_path, case, collector, result)
|
||||
|
||||
|
||||
def test_publication_failure_removes_partial_success_bundle(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case, collector, result = _successful_output(tmp_path)
|
||||
_write_success(tmp_path, case, collector, result)
|
||||
original = oi._atomic_write_json
|
||||
|
||||
def fail(path: Path, value: object) -> None:
|
||||
if path.name == "events.json":
|
||||
raise OSError("disk full")
|
||||
original(path, value)
|
||||
|
||||
monkeypatch.setattr(oi, "_atomic_write_json", fail)
|
||||
with pytest.raises(OSError, match="disk full"):
|
||||
oi.publish_deferred_case_artifacts(tmp_path, case, result)
|
||||
directory = oi._output_dir(tmp_path, case)
|
||||
assert not any((directory / name).exists() for name in oi.SUCCESS_ARTIFACTS)
|
||||
assert not result.success and "artifact_write_failed" in result.official_eval_ineligible_reasons
|
||||
|
||||
|
||||
def test_atomic_write_preserves_destination_on_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
destination = tmp_path / "output.wav"
|
||||
destination.write_bytes(b"old")
|
||||
monkeypatch.setattr(oi, "write_pcm16_wav", lambda *a, **k: (_ for _ in ()).throw(OSError("write")))
|
||||
with pytest.raises(OSError):
|
||||
oi._atomic_write_wav(destination, b"\0\0", 24_000)
|
||||
assert destination.read_bytes() == b"old"
|
||||
assert not list(tmp_path.glob(".*.tmp"))
|
||||
|
||||
|
||||
class _RealtimeClient:
|
||||
instances: list[_RealtimeClient] = []
|
||||
|
||||
def __init__(self, url: str, **kwargs):
|
||||
self.url, self.events, self.acks, self.configure_kwargs = url, RealtimeEventCollector(), [], {}
|
||||
self.instances.append(self)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
return None
|
||||
|
||||
async def configure(self, model: str, **kwargs) -> None:
|
||||
self.configure_kwargs = kwargs
|
||||
self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}})
|
||||
|
||||
async def send(self, event: dict[str, object]) -> None:
|
||||
return None
|
||||
|
||||
async def commit(self) -> None:
|
||||
now = time.monotonic()
|
||||
for event, offset in (
|
||||
({"type": "input_audio_buffer.committed"}, 0.0),
|
||||
(_created(), 0.001),
|
||||
(_audio(samples=24), 0.002),
|
||||
(_text(value="answer"), 0.003),
|
||||
(_done(), 0.004),
|
||||
):
|
||||
self.events.add(event, received_at_s=now + offset)
|
||||
|
||||
async def send_playback_ack(self, response_id: str, played_ms: int) -> None:
|
||||
self.acks.append((response_id, played_ms))
|
||||
|
||||
async def close_session(self, **kwargs) -> None:
|
||||
self.events.add({"type": "session.closed"})
|
||||
|
||||
def raise_if_reader_stopped(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case = _case(tmp_path)
|
||||
config = oi.OmniInteractBenchmarkConfig(
|
||||
model=MODEL,
|
||||
output_root=tmp_path / "out",
|
||||
ref_audio="ref.wav",
|
||||
require_response=True,
|
||||
extra_body={"custom": "value"},
|
||||
)
|
||||
prepared = data.OmniInteractPreparedInput(1.0, b"\0\0\0\0", ("frame",), "data:audio/wav;base64,ref")
|
||||
monkeypatch.setattr(oi, "RealtimeDuplexClient", _RealtimeClient)
|
||||
monkeypatch.setattr(oi, "_COMPLETION_SETTLE_S", 0.0)
|
||||
result = await oi.run_omniinteract_case(
|
||||
case,
|
||||
config,
|
||||
request_index=0,
|
||||
prepared_input=prepared,
|
||||
)
|
||||
assert result.success and result.transcript == "answer" and result._artifact_context is not None
|
||||
assert not config.output_root.exists()
|
||||
assert "autostart=0" in _RealtimeClient.instances[-1].url
|
||||
assert _RealtimeClient.instances[-1].configure_kwargs["extra_body"] == {"custom": "value"}
|
||||
assert _RealtimeClient.instances[-1].acks == [("r1", 1)]
|
||||
|
||||
|
||||
def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
_write_dataset(tmp_path)
|
||||
ref = tmp_path / "ref.wav"
|
||||
ref.touch()
|
||||
parser = TrackingArgumentParser()
|
||||
OmniBenchmarkServingSubcommand.add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--backend",
|
||||
"openai-realtime-duplex",
|
||||
"--dataset-name",
|
||||
"omniinteract",
|
||||
"--dataset-path",
|
||||
str(tmp_path),
|
||||
"--model",
|
||||
MODEL,
|
||||
"--endpoint",
|
||||
"/v1/realtime",
|
||||
"--num-prompts",
|
||||
"0",
|
||||
"--omniinteract-ref-audio",
|
||||
str(ref),
|
||||
]
|
||||
)
|
||||
preprocess_serve_args(args)
|
||||
assert args.endpoint == "/v1/realtime"
|
||||
monkeypatch.setattr(benchmark_patch, "reference_audio_data_url", lambda _: "data:audio/wav;base64,ref")
|
||||
monkeypatch.setattr(benchmark_patch, "prepare_media", lambda *a, **k: (1.0, b"pcm", ["frame"]))
|
||||
samples = benchmark_patch.get_samples(args, None)
|
||||
assert len(samples) == args.num_prompts == 3
|
||||
assert all(sample.omniinteract_prepared_input.video_frames == ("frame",) for sample in samples)
|
||||
|
||||
|
||||
def _request(case: data.OmniInteractCase, options: data.OmniInteractSessionOptions) -> RequestFuncInput:
|
||||
request = RequestFuncInput(
|
||||
model=MODEL,
|
||||
model_name=MODEL,
|
||||
prompt="",
|
||||
api_url="http://server:8000/v1/realtime",
|
||||
prompt_len=0,
|
||||
output_len=0,
|
||||
logprobs=None,
|
||||
multi_modal_content=None,
|
||||
ignore_eos=False,
|
||||
request_id="measured-0",
|
||||
)
|
||||
request.omniinteract_case, request.omniinteract_options = case, options
|
||||
return request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case, options = _case(tmp_path), _session_options(tmp_path)
|
||||
request = _request(case, options)
|
||||
output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None)
|
||||
assert not output.success and "prepared before benchmark request timing" in output.error
|
||||
|
||||
prepared = data.OmniInteractPreparedInput(1.0, b"pcm", ("frame",), "data:audio/wav;base64,ref")
|
||||
request.omniinteract_prepared_input = prepared
|
||||
request.extra_headers = {"X-Proxy": "test"}
|
||||
request.extra_body = {"custom": "value"}
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "secret")
|
||||
seen = {}
|
||||
|
||||
async def run(_case_arg, config, **kwargs):
|
||||
seen["headers"] = config.extra_headers
|
||||
seen["extra_body"] = config.extra_body
|
||||
seen.update(kwargs)
|
||||
return oi.OmniInteractCaseResult(
|
||||
case.subset,
|
||||
str(case.video_path),
|
||||
str(options.output_root),
|
||||
success=True,
|
||||
transcript="timing metadata missing",
|
||||
output_tokens=0,
|
||||
duplex_request_metrics=[{"request_metrics": {"ttft_ms": 1.0}}],
|
||||
duplex_session_metrics={"mean_ttft_ms": 10.0},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run)
|
||||
output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None)
|
||||
assert output.success and seen["prepared_input"] is prepared
|
||||
assert seen["headers"] == {"Authorization": "Bearer secret", "X-Proxy": "test", "x-request-id": "measured-0"}
|
||||
assert seen["extra_body"] == {"custom": "value"}
|
||||
assert output.itl == []
|
||||
assert output.tpot_measured is False
|
||||
assert seen["capture_artifacts"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_metrics", "expected_itl", "expected_text_latency"),
|
||||
[
|
||||
(
|
||||
[
|
||||
{"stage0_tokens": {"output_token_count": 3, "itls_ms": [10.0, 20.0]}},
|
||||
{"stage0_tokens": {"output_token_count": 2, "itls_ms": [30.0]}},
|
||||
],
|
||||
[0.01, 0.02, 0.03],
|
||||
0.16,
|
||||
),
|
||||
(
|
||||
[
|
||||
{"stage0_tokens": {"output_token_count": 3, "tpot_ms": 10.0}},
|
||||
{"stage0_tokens": {"output_token_count": 2, "tpot_ms": 40.0}},
|
||||
],
|
||||
[],
|
||||
0.18,
|
||||
),
|
||||
(
|
||||
[
|
||||
{"stage0_tokens": {"output_token_count": 3, "tpot_ms": 0.0}},
|
||||
{"stage0_tokens": {"output_token_count": 2, "tpot_ms": 0.0}},
|
||||
],
|
||||
[],
|
||||
0.1,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_adapter_reports_exact_or_weighted_token_timing(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
request_metrics: list[dict[str, object]],
|
||||
expected_itl: list[float],
|
||||
expected_text_latency: float,
|
||||
):
|
||||
case, options = _case(tmp_path), _session_options(tmp_path)
|
||||
request = _request(case, options)
|
||||
request.omniinteract_prepared_input = data.OmniInteractPreparedInput(
|
||||
1.0, b"pcm", ("frame",), "data:audio/wav;base64,ref"
|
||||
)
|
||||
|
||||
async def run(*args, **kwargs):
|
||||
return oi.OmniInteractCaseResult(
|
||||
case.subset,
|
||||
str(case.video_path),
|
||||
str(options.output_root),
|
||||
success=True,
|
||||
output_tokens=5,
|
||||
duplex_request_metrics=request_metrics,
|
||||
duplex_session_metrics={"mean_ttft_ms": 100.0},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run)
|
||||
output = await benchmark_patch.async_request_openai_realtime_duplex(request, session=None)
|
||||
|
||||
assert output.success
|
||||
assert output.itl == expected_itl
|
||||
assert output.text_latency == pytest.approx(expected_text_latency)
|
||||
assert output.tpot_measured is True
|
||||
|
||||
|
||||
def test_batch_finalization_publishes_only_measured_results(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case, options = _case(tmp_path), _session_options(tmp_path)
|
||||
options.output_root.mkdir()
|
||||
sample = _sample(case, options)
|
||||
result = oi.OmniInteractCaseResult(
|
||||
case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True
|
||||
)
|
||||
output = benchmark_patch.MixRequestFuncOutput(success=True)
|
||||
output.omniinteract_case_result = result
|
||||
published = []
|
||||
monkeypatch.setattr(benchmark_patch, "publish_deferred_case_artifacts", lambda *args: published.append(args))
|
||||
summary = benchmark_patch._finalize_omniinteract_batch([sample], [output])
|
||||
assert summary and summary["total"] == summary["success"] == summary["eligible_for_official_eval"] == 1
|
||||
assert summary["failed"] == summary["successful_but_ineligible"] == summary["audio_clipped_bytes"] == 0
|
||||
assert published == [(options.output_root, case, result)]
|
||||
|
||||
|
||||
def test_case_artifact_failures_do_not_discard_serving_metrics(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case, options = _case(tmp_path), _session_options(tmp_path)
|
||||
sample = _sample(case, options)
|
||||
result = oi.OmniInteractCaseResult(
|
||||
case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True
|
||||
)
|
||||
output = benchmark_patch.MixRequestFuncOutput(success=True, error="inference warning")
|
||||
output.omniinteract_case_result = result
|
||||
monkeypatch.setattr(
|
||||
benchmark_patch,
|
||||
"publish_deferred_case_artifacts",
|
||||
lambda *args: (_ for _ in ()).throw(OSError("case disk full")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
benchmark_patch,
|
||||
"write_failure_artifacts",
|
||||
lambda *args: (_ for _ in ()).throw(OSError("failure disk full")),
|
||||
)
|
||||
monkeypatch.setattr(benchmark_patch, "write_omniinteract_batch_artifacts", lambda *args: None)
|
||||
|
||||
summary = benchmark_patch._finalize_omniinteract_batch([sample], [output])
|
||||
metrics, _ = calculate_metrics(
|
||||
[],
|
||||
[output],
|
||||
1.0,
|
||||
None,
|
||||
[50.0],
|
||||
{},
|
||||
TaskType.GENERATION,
|
||||
[],
|
||||
None,
|
||||
float("inf"),
|
||||
1.0,
|
||||
)
|
||||
|
||||
assert summary and summary["failed"] == 1
|
||||
assert summary["artifacts_complete"] is False
|
||||
assert len(summary["artifact_errors"]) == 2
|
||||
assert output.success and metrics.completed == 1
|
||||
assert output.error.startswith("inference warning\n")
|
||||
assert "Artifact publication failed: case disk full" in output.error
|
||||
|
||||
|
||||
def test_batch_artifact_failure_removes_partial_batch_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
case, options = _case(tmp_path), _session_options(tmp_path)
|
||||
sample = _sample(case, options)
|
||||
result = oi.OmniInteractCaseResult(
|
||||
case.subset, str(case.video_path), str(options.output_root), success=True, eligible_for_official_eval=True
|
||||
)
|
||||
output = benchmark_patch.MixRequestFuncOutput(success=True)
|
||||
output.omniinteract_case_result = result
|
||||
monkeypatch.setattr(benchmark_patch, "publish_deferred_case_artifacts", lambda *args: None)
|
||||
|
||||
def fail_batch(root, *args):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
(root / "batch_summary.json").write_text("partial")
|
||||
raise OSError("batch disk full")
|
||||
|
||||
monkeypatch.setattr(benchmark_patch, "write_omniinteract_batch_artifacts", fail_batch)
|
||||
|
||||
summary = benchmark_patch._finalize_omniinteract_batch([sample], [output])
|
||||
|
||||
assert summary and summary["success"] == 1 and summary["artifacts_complete"] is False
|
||||
assert summary["artifact_errors"] == ["Batch artifact publication failed: batch disk full"]
|
||||
assert output.success
|
||||
assert not (options.output_root / "batch_summary.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "match"),
|
||||
[
|
||||
("omniinteract_ref_audio", None, "ref-audio"),
|
||||
("endpoint", "/v1/completions", "v1/realtime"),
|
||||
("max_concurrency", 0, "positive"),
|
||||
("skip_tokenizer_init", True, "skip-tokenizer-init"),
|
||||
],
|
||||
)
|
||||
def test_cli_rejects_incompatible_omniinteract_options(field: str, value, match: str):
|
||||
args = argparse.Namespace(
|
||||
dataset_name="omniinteract",
|
||||
backend="openai-realtime-duplex",
|
||||
endpoint="/v1/realtime",
|
||||
omniinteract_ref_audio="ref.wav",
|
||||
max_concurrency=1,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
setattr(args, field, value)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
preprocess_serve_args(args)
|
||||
|
||||
|
||||
def test_whole_benchmark_lock_serializes_a_shared_output_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
calls: list[int] = []
|
||||
|
||||
async def fake_main_async(args):
|
||||
calls.append(len(calls))
|
||||
if len(calls) == 1:
|
||||
entered.set()
|
||||
await asyncio.to_thread(release.wait)
|
||||
return {"total": 1}
|
||||
|
||||
monkeypatch.setattr(benchmark_serve, "main_async", fake_main_async)
|
||||
args = argparse.Namespace(dataset_name="omniinteract", omniinteract_output_dir=tmp_path, extra_body=None)
|
||||
with ThreadPoolExecutor(2) as executor:
|
||||
first = executor.submit(benchmark_serve.main, args)
|
||||
assert entered.wait(1)
|
||||
second = executor.submit(benchmark_serve.main, args)
|
||||
time.sleep(0.05)
|
||||
assert calls == [0]
|
||||
release.set()
|
||||
assert first.result() == second.result() == {"total": 1}
|
||||
assert calls == [0, 1]
|
||||
Reference in new issue
Block a user