[Frontend][Benchmark] Add duplex performance metrics for OmniInteract and Omni-DuplexEval benchmark (#7242)
This commit is contained in:
1 parent
27f0e8c1ea
commit
f062124ba4
21 files changed
+1484
-161
No files matched your search
@@ -15,6 +15,15 @@ import pytest
|
||||
from vllm_omni.benchmarks.duplex import omni_duplex_eval_runner as runner
|
||||
from vllm_omni.benchmarks.duplex.omni_duplex_eval_dataset import DuplexSample
|
||||
from vllm_omni.benchmarks.duplex.omni_duplex_eval_judge import DuplexJudge
|
||||
from vllm_omni.benchmarks.duplex.omni_duplex_eval_runner import GenerateSampleResult
|
||||
from vllm_omni.benchmarks.duplex_session_metrics import (
|
||||
DUPLEX_METRICS_FILENAME,
|
||||
build_duplex_metrics_report,
|
||||
collect_duplex_session_metrics,
|
||||
merge_duplex_metrics_report,
|
||||
read_duplex_metrics_report,
|
||||
)
|
||||
from vllm_omni.clients.duplex import EventCollector
|
||||
from vllm_omni.entrypoints.cli.benchmark import omni_duplex_eval as cli
|
||||
from vllm_omni.entrypoints.cli.benchmark.omni_duplex_eval import OmniDuplexEvalSubcommand
|
||||
|
||||
@@ -44,7 +53,25 @@ def test_cli_generate_evaluate_summarize_flow(tmp_path: Path, monkeypatch: pytes
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps([{"sentence": "It moved.", "start": 0, "end": 1}]), encoding="utf-8")
|
||||
output.with_name(output.stem + ".meta.json").write_text(json.dumps({"clock": "media"}), encoding="utf-8")
|
||||
return output
|
||||
return GenerateSampleResult(
|
||||
output=output,
|
||||
request_metrics=[
|
||||
{
|
||||
"sample_id": sample.id,
|
||||
"split": sample.split,
|
||||
"ttft_ms": 10.0,
|
||||
"ttfp_ms": 20.0,
|
||||
"rtf": 0.5,
|
||||
}
|
||||
],
|
||||
session_metrics={
|
||||
"sample_id": sample.id,
|
||||
"split": sample.split,
|
||||
"stream_ttft_ms": 10.0,
|
||||
"stream_ttfp_ms": 20.0,
|
||||
"stream_rtf": 0.5,
|
||||
},
|
||||
)
|
||||
|
||||
class FakeJudge:
|
||||
def __init__(self, *args, **kwargs):
|
||||
@@ -75,6 +102,19 @@ def test_cli_generate_evaluate_summarize_flow(tmp_path: Path, monkeypatch: pytes
|
||||
]
|
||||
)
|
||||
OmniDuplexEvalSubcommand.cmd(generate)
|
||||
duplex_metrics = json.loads((response_root / DUPLEX_METRICS_FILENAME).read_text(encoding="utf-8"))
|
||||
assert duplex_metrics["duplex_request_metrics"] == [
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"split": "PR_correction",
|
||||
"ttft_ms": 10.0,
|
||||
"ttfp_ms": 20.0,
|
||||
"rtf": 0.5,
|
||||
}
|
||||
]
|
||||
assert duplex_metrics["duplex_stream_ttft_ms"]["mean"] == 10.0
|
||||
assert duplex_metrics["duplex_stream_ttfp_ms"]["mean"] == 20.0
|
||||
assert duplex_metrics["duplex_stream_rtf"]["mean"] == 0.5
|
||||
evaluate = parser.parse_args(
|
||||
[
|
||||
"evaluate",
|
||||
@@ -144,7 +184,7 @@ async def test_generate_exercises_realtime_socket_and_media_clock(tmp_path: Path
|
||||
|
||||
async with websockets.serve(handler, "127.0.0.1", 0) as server:
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
output = await runner.generate_sample(
|
||||
result = await runner.generate_sample(
|
||||
sample,
|
||||
url=f"ws://127.0.0.1:{port}/v1/realtime?duplex=1",
|
||||
model="mock",
|
||||
@@ -152,13 +192,231 @@ async def test_generate_exercises_realtime_socket_and_media_clock(tmp_path: Path
|
||||
output_root=tmp_path / "responses",
|
||||
)
|
||||
|
||||
assert json.loads(output.read_text(encoding="utf-8")) == [{"sentence": "Done.", "start": 0.8, "end": 0.8}]
|
||||
meta = json.loads(output.with_name("sample.meta.json").read_text(encoding="utf-8"))
|
||||
assert json.loads(result.output.read_text(encoding="utf-8")) == [{"sentence": "Done.", "start": 0.8, "end": 0.8}]
|
||||
meta = json.loads(result.output.with_name("sample.meta.json").read_text(encoding="utf-8"))
|
||||
assert meta["response_done"] is True
|
||||
assert meta["drain_timeout"] is None
|
||||
assert "duplex_request_metrics" not in meta
|
||||
assert result.request_metrics
|
||||
assert result.request_metrics[0]["sample_id"] == "sample"
|
||||
assert result.session_metrics["stream_ttfp_ms"] is not None
|
||||
assert any(event["type"] == "playback.ack" for event in received)
|
||||
|
||||
|
||||
def _pr_manifest(tmp_path: Path, sample_ids: tuple[str, ...]) -> Path:
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"id": sample_id,
|
||||
"split": "PR_correction",
|
||||
"question_text": "What changed?",
|
||||
"answer1": "The object moved.",
|
||||
}
|
||||
for sample_id in sample_ids
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def _run_generate_cli(manifest: Path, response_root: Path) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
OmniDuplexEvalSubcommand.add_cli_args(parser)
|
||||
OmniDuplexEvalSubcommand.cmd(
|
||||
parser.parse_args(
|
||||
[
|
||||
"generate",
|
||||
"--dataset",
|
||||
str(manifest),
|
||||
"--family",
|
||||
"pr",
|
||||
"--model",
|
||||
"mock",
|
||||
"--ref-audio",
|
||||
str(manifest),
|
||||
"--response-root",
|
||||
str(response_root),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _metrics_for(sample_id: str, *, ttft_ms: float, ttfp_ms: float, rtf: float) -> GenerateSampleResult:
|
||||
split = "PR_correction"
|
||||
return GenerateSampleResult(
|
||||
output=Path(split) / f"{sample_id}.json",
|
||||
request_metrics=[{"sample_id": sample_id, "split": split, "ttft_ms": ttft_ms, "ttfp_ms": ttfp_ms, "rtf": rtf}],
|
||||
session_metrics={
|
||||
"sample_id": sample_id,
|
||||
"split": split,
|
||||
"stream_ttft_ms": ttft_ms,
|
||||
"stream_ttfp_ms": ttfp_ms,
|
||||
"stream_rtf": rtf,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fake_generate(results: dict[str, GenerateSampleResult | None]):
|
||||
async def generate(
|
||||
sample: DuplexSample,
|
||||
*,
|
||||
output_root: str | Path,
|
||||
url: str,
|
||||
model: str,
|
||||
ref_audio: str | Path,
|
||||
fps: float = 1.0,
|
||||
mix: str = "question",
|
||||
pace: str = "realtime",
|
||||
clock: str = "media",
|
||||
overwrite: bool = False,
|
||||
unit_ms: int = 1000,
|
||||
) -> GenerateSampleResult:
|
||||
_ = (url, model, ref_audio, fps, mix, pace, clock, overwrite, unit_ms)
|
||||
result = results[sample.id]
|
||||
if result is None:
|
||||
return GenerateSampleResult(output=Path(output_root) / sample.split / f"{sample.id}.json")
|
||||
return result
|
||||
|
||||
return generate
|
||||
|
||||
|
||||
def test_merge_duplex_metrics_report_replaces_only_incoming_sample_keys():
|
||||
existing = build_duplex_metrics_report(
|
||||
request_metrics=[
|
||||
{"sample_id": "kept", "split": "PR_correction", "ttft_ms": 11.0},
|
||||
{"sample_id": "replaced", "split": "PR_correction", "ttft_ms": 12.0},
|
||||
],
|
||||
session_metrics=[
|
||||
{"sample_id": "kept", "split": "PR_correction", "stream_ttft_ms": 11.0},
|
||||
{"sample_id": "replaced", "split": "PR_correction", "stream_ttft_ms": 12.0},
|
||||
],
|
||||
)
|
||||
merged = merge_duplex_metrics_report(
|
||||
existing,
|
||||
request_metrics=[{"sample_id": "replaced", "split": "PR_correction", "ttft_ms": 99.0}],
|
||||
session_metrics=[{"sample_id": "replaced", "split": "PR_correction", "stream_ttft_ms": 99.0}],
|
||||
)
|
||||
assert [row["sample_id"] for row in merged["duplex_request_metrics"]] == ["kept", "replaced"]
|
||||
assert merged["duplex_request_metrics"][1]["ttft_ms"] == 99.0
|
||||
assert merged["duplex_stream_ttft_ms"]["mean"] == 55.0
|
||||
|
||||
|
||||
def test_read_duplex_metrics_report_returns_none_when_missing(tmp_path: Path):
|
||||
assert read_duplex_metrics_report(tmp_path / DUPLEX_METRICS_FILENAME) is None
|
||||
|
||||
|
||||
def test_read_duplex_metrics_report_rejects_invalid_json(tmp_path: Path):
|
||||
path = tmp_path / DUPLEX_METRICS_FILENAME
|
||||
path.write_text("{not-json", encoding="utf-8")
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
read_duplex_metrics_report(path)
|
||||
|
||||
|
||||
def test_merge_duplex_metrics_report_without_existing_is_passthrough():
|
||||
merged = merge_duplex_metrics_report(
|
||||
None,
|
||||
request_metrics=[{"sample_id": "a", "split": "PR_correction", "ttft_ms": 1.0}],
|
||||
session_metrics=[{"sample_id": "a", "split": "PR_correction", "stream_ttft_ms": 1.0}],
|
||||
)
|
||||
assert merged["duplex_request_metrics"][0]["ttft_ms"] == 1.0
|
||||
assert merged["duplex_stream_ttft_ms"]["mean"] == 1.0
|
||||
|
||||
|
||||
def test_merge_duplex_metrics_report_rejects_non_object_rows():
|
||||
with pytest.raises(ValueError, match="duplex_request_metrics entries must be JSON objects"):
|
||||
merge_duplex_metrics_report(
|
||||
{"duplex_request_metrics": ["bad"]},
|
||||
request_metrics=[],
|
||||
session_metrics=[],
|
||||
)
|
||||
|
||||
|
||||
def test_cli_generate_all_skipped_keeps_existing_metrics(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
manifest = _pr_manifest(tmp_path, ("sample-1",))
|
||||
response_root = tmp_path / "responses"
|
||||
monkeypatch.setattr(
|
||||
cli,
|
||||
"generate_sample",
|
||||
_fake_generate({"sample-1": _metrics_for("sample-1", ttft_ms=10.0, ttfp_ms=20.0, rtf=0.5)}),
|
||||
)
|
||||
_run_generate_cli(manifest, response_root)
|
||||
monkeypatch.setattr(cli, "generate_sample", _fake_generate({"sample-1": None}))
|
||||
_run_generate_cli(manifest, response_root)
|
||||
duplex_metrics = json.loads((response_root / DUPLEX_METRICS_FILENAME).read_text(encoding="utf-8"))
|
||||
assert duplex_metrics["duplex_request_metrics"][0]["ttft_ms"] == 10.0
|
||||
assert duplex_metrics["duplex_stream_ttft_ms"]["mean"] == 10.0
|
||||
assert duplex_metrics["duplex_stream_ttfp_ms"]["mean"] == 20.0
|
||||
assert duplex_metrics["duplex_stream_rtf"]["mean"] == 0.5
|
||||
|
||||
|
||||
def test_cli_generate_partial_resume_merges_metrics(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
manifest = _pr_manifest(tmp_path, ("sample-1", "sample-2"))
|
||||
response_root = tmp_path / "responses"
|
||||
monkeypatch.setattr(
|
||||
cli,
|
||||
"generate_sample",
|
||||
_fake_generate(
|
||||
{
|
||||
"sample-1": _metrics_for("sample-1", ttft_ms=10.0, ttfp_ms=20.0, rtf=0.5),
|
||||
"sample-2": _metrics_for("sample-2", ttft_ms=30.0, ttfp_ms=40.0, rtf=0.25),
|
||||
}
|
||||
),
|
||||
)
|
||||
_run_generate_cli(manifest, response_root)
|
||||
monkeypatch.setattr(
|
||||
cli,
|
||||
"generate_sample",
|
||||
_fake_generate(
|
||||
{
|
||||
"sample-1": None,
|
||||
"sample-2": _metrics_for("sample-2", ttft_ms=90.0, ttfp_ms=80.0, rtf=1.5),
|
||||
}
|
||||
),
|
||||
)
|
||||
_run_generate_cli(manifest, response_root)
|
||||
duplex_metrics = json.loads((response_root / DUPLEX_METRICS_FILENAME).read_text(encoding="utf-8"))
|
||||
by_id = {row["sample_id"]: row for row in duplex_metrics["duplex_request_metrics"]}
|
||||
assert by_id["sample-1"]["ttft_ms"] == 10.0
|
||||
assert by_id["sample-2"]["ttft_ms"] == 90.0
|
||||
assert duplex_metrics["duplex_stream_ttft_ms"]["mean"] == 50.0
|
||||
assert duplex_metrics["duplex_stream_ttfp_ms"]["mean"] == 50.0
|
||||
assert duplex_metrics["duplex_stream_rtf"]["mean"] == 1.0
|
||||
|
||||
|
||||
def test_collect_duplex_session_metrics_matches_omniinteract_window():
|
||||
collector = EventCollector()
|
||||
audio = {
|
||||
"type": "response.audio.delta",
|
||||
"response_id": "r1",
|
||||
"format": "pcm16",
|
||||
"delta": base64.b64encode(bytes((1, 0)) * 2400).decode(),
|
||||
"sample_rate_hz": 24_000,
|
||||
"metadata": {"audio_duration_ms": 100},
|
||||
}
|
||||
collector.add({"type": "response.created", "response": {"id": "r1"}}, received_at_s=10.1)
|
||||
collector.add(
|
||||
{"type": "response.output_text.delta", "response_id": "r1", "delta": "Done."},
|
||||
received_at_s=10.2,
|
||||
)
|
||||
collector.add(audio, received_at_s=10.3)
|
||||
collector.add({"type": "response.done", "response": {"id": "r1"}}, received_at_s=10.4)
|
||||
bundle = collect_duplex_session_metrics(collector, stream_start=10.0, session_id="sample")
|
||||
assert bundle.request_metrics[0]["ttft_ms"] == 100.0
|
||||
assert bundle.request_metrics[0]["ttfp_ms"] == 200.0
|
||||
assert bundle.request_metrics[0]["rtf"] == 2.0
|
||||
assert bundle.session_metrics["stream_ttft_ms"] == 200.0
|
||||
assert bundle.session_metrics["stream_ttfp_ms"] == 300.0
|
||||
report = build_duplex_metrics_report(
|
||||
request_metrics=bundle.request_metrics,
|
||||
session_metrics=[bundle.session_metrics],
|
||||
)
|
||||
assert report["duplex_stream_ttft_ms"]["mean"] == 200.0
|
||||
assert report["duplex_stream_ttfp_ms"]["mean"] == 300.0
|
||||
|
||||
|
||||
def test_judge_exercises_openai_http_schema():
|
||||
requests = []
|
||||
|
||||
|
||||
Reference in new issue
Block a user