From 8d98cc0cb7be1b7f75056572093174a97f42962d Mon Sep 17 00:00:00 2001 From: NATURE Date: Fri, 4 Sep 2026 10:36:02 +0800 Subject: [PATCH] [Benchmark] Add local OmniInteract performance cases (#6817) Signed-off-by: natureofnature Co-authored-by: Ruirui Yang | Rein <73573651+R2-Y@users.noreply.github.com> Co-authored-by: amy-why-3459 --- docs/cli/bench/serve.md | 20 +++++ tests/benchmarks/test_omniinteract.py | 7 +- tests/dfx/perf/scripts/run_benchmark.py | 22 ++++- .../tests/test_minicpmo_4_5_omniinteract.json | 87 +++++++++++++++++++ tests/dfx/perf/tests/test_runner_metadata.py | 34 ++++++++ tools/nightly/run_nightly_jobs.sh | 2 +- 6 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 tests/dfx/perf/tests/test_minicpmo_4_5_omniinteract.json diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index 3e16ae554..efb23df00 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -307,6 +307,26 @@ omitted from the official manifest; `audio_clipped_bytes` records output beyond TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every token interval is present. +The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos +total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its +measured cases. Run it from the repository root with: + +```bash +export HF_HOME=/path/to/persistent/huggingface-cache +export BENCHMARK_DIR=tests/dfx/perf/results +bash tools/nightly/run_nightly_jobs.sh \ + --test-type local \ + --model-type omni \ + --label-substr minicpmo_4_5_omniinteract +``` + +The first run downloads the pinned OmniInteract archive into `HF_HOME`; later runs reuse that cache. + +It requires every case to commit its input, complete any emitted response lifecycles, and publish the expected WAV, +transcript, event, and result artifacts without errors. A valid LISTEN-only case may have no response audio or transcript +chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality +signal, not a transport failure. This local performance test does not score answer accuracy. + ### Multi-Modal Benchmark
diff --git a/tests/benchmarks/test_omniinteract.py b/tests/benchmarks/test_omniinteract.py index c38e001bf..c51724e0a 100644 --- a/tests/benchmarks/test_omniinteract.py +++ b/tests/benchmarks/test_omniinteract.py @@ -203,18 +203,19 @@ def test_archive_is_safe_and_atomically_shared(tmp_path: Path, monkeypatch: pyte assert not list(target.glob(".tmp-*")) -def test_hub_archive_uses_vllm_filesystem(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize("dataset_repo", ["org/repo", "org/repo@revision"]) +def test_hub_archive_uses_vllm_filesystem(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, dataset_repo: str): source = tmp_path / "source.tar" _archive(source, "data/1q1a/video_json_map.json") class FS: def get_file(self, remote: str, local: str) -> None: - assert remote == "datasets/org/repo/data.tar.gz" + assert remote == f"datasets/{dataset_repo}/data.tar.gz" Path(local).write_bytes(source.read_bytes()) monkeypatch.setenv("HF_HOME", str(tmp_path / "hf")) monkeypatch.setattr(data, "hf_fs", lambda: FS()) - assert (data.resolve_omniinteract_root(None, "org/repo") / "1q1a").is_dir() + assert (data.resolve_omniinteract_root(None, dataset_repo) / "1q1a").is_dir() def test_media_commands_are_bounded(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 64285dca3..94df2edc5 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -161,8 +161,26 @@ def benchmark_params(request): } +def _resolve_num_warmups(params: dict[str, Any], *, default: int) -> int: + value = params.get("num_warmups") + if value is None: + return default + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError("num_warmups must be a non-negative integer") + return value + + def assert_result(result, params, num_prompt) -> None: assert result["completed"] == num_prompt, "Request failures exist" + if params.get("dataset_name") == "omniinteract": + summary = result.get("omniinteract") + assert isinstance(summary, dict), "OmniInteract summary is missing" + assert (summary.get("total"), summary.get("success"), summary.get("failed")) == ( + num_prompt, + num_prompt, + 0, + ), "OmniInteract requests did not all succeed" + assert summary.get("artifacts_complete") is True, "OmniInteract artifacts are incomplete" baseline = params.get("baseline") hardware = result.get("Hardware") hardware_baseline = baseline.get(hardware) if isinstance(baseline, dict) and isinstance(hardware, str) else None @@ -235,6 +253,7 @@ def test_performance_benchmark(omni_server, benchmark_params): "baseline", "num_prompts", "max_concurrency", + "num_warmups", "task", "enabled", "eval_phase", @@ -278,6 +297,7 @@ def test_performance_benchmark(omni_server, benchmark_params): random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, + num_warmups=_resolve_num_warmups(params, default=2), ) assert_result(result, params, num_prompt) @@ -295,6 +315,6 @@ def test_performance_benchmark(omni_server, benchmark_params): random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, - num_warmups=max(2, int(concurrency)), + num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))), ) assert_result(result, params, num_prompt) diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_omniinteract.json b/tests/dfx/perf/tests/test_minicpmo_4_5_omniinteract.json new file mode 100644 index 000000000..f101fd515 --- /dev/null +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_omniinteract.json @@ -0,0 +1,87 @@ +[ + { + "test_name": "test_minicpmo_4_5_omniinteract", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "local_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "omniinteract", + "dataset_path": "lucky-lance/OmniInteract@e195f75fe2666fcc5fe74f537ae49ca143a79969", + "backend": "openai-realtime-duplex", + "endpoint": "/v1/realtime", + "num_prompts": [ + 4 + ], + "max_concurrency": [ + 2 + ], + "num_warmups": 0, + "disable_shuffle": true, + "trust_remote_code": true, + "omniinteract_subsets": "1q1a", + "omniinteract_ref_audio": "tests/assets/minicpmo_4_5/response_required_16k.wav", + "omniinteract_output_dir": "tests/dfx/perf/results/omniinteract/1q1a", + "omniinteract_max_video_duration_s": 600, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration" + }, + { + "dataset_name": "omniinteract", + "dataset_path": "lucky-lance/OmniInteract@e195f75fe2666fcc5fe74f537ae49ca143a79969", + "backend": "openai-realtime-duplex", + "endpoint": "/v1/realtime", + "num_prompts": [ + 4 + ], + "max_concurrency": [ + 2 + ], + "num_warmups": 0, + "disable_shuffle": true, + "trust_remote_code": true, + "omniinteract_subsets": "1q1a_math", + "omniinteract_ref_audio": "tests/assets/minicpmo_4_5/response_required_16k.wav", + "omniinteract_output_dir": "tests/dfx/perf/results/omniinteract/1q1a_math", + "omniinteract_max_video_duration_s": 600, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration" + }, + { + "dataset_name": "omniinteract", + "dataset_path": "lucky-lance/OmniInteract@e195f75fe2666fcc5fe74f537ae49ca143a79969", + "backend": "openai-realtime-duplex", + "endpoint": "/v1/realtime", + "num_prompts": [ + 4 + ], + "max_concurrency": [ + 2 + ], + "num_warmups": 0, + "disable_shuffle": true, + "trust_remote_code": true, + "omniinteract_subsets": "1qna", + "omniinteract_ref_audio": "tests/assets/minicpmo_4_5/response_required_16k.wav", + "omniinteract_output_dir": "tests/dfx/perf/results/omniinteract/1qna", + "omniinteract_max_video_duration_s": 600, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index d25f66709..df99f090d 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -518,6 +518,40 @@ def test_omni_duplex_expected_audio_turns_rejects_incomplete_session(): ) +def test_num_warmups_preserves_explicit_zero(): + from tests.dfx.perf.scripts.run_benchmark import _resolve_num_warmups + + assert _resolve_num_warmups({}, default=4) == 4 + assert _resolve_num_warmups({"num_warmups": 0}, default=4) == 0 + + +def test_omniinteract_result_accepts_complete_artifacts(): + from tests.dfx.perf.scripts.run_benchmark import assert_result + + assert_result( + { + "completed": 4, + "omniinteract": {"total": 4, "success": 4, "failed": 0, "artifacts_complete": True}, + }, + {"dataset_name": "omniinteract"}, + 4, + ) + + +def test_omniinteract_result_rejects_incomplete_artifacts(): + from tests.dfx.perf.scripts.run_benchmark import assert_result + + with pytest.raises(AssertionError, match="artifacts are incomplete"): + assert_result( + { + "completed": 4, + "omniinteract": {"total": 4, "success": 4, "failed": 0, "artifacts_complete": False}, + }, + {"dataset_name": "omniinteract"}, + 4, + ) + + def test_omni_tpot_baseline_accepts_measured_finite_sample(): from tests.dfx.perf.scripts.run_benchmark import assert_result diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index 48bc030ca..c1cca73e2 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -678,7 +678,7 @@ TTS_PERF_JSON_HINTS = ("test_tts", "voxcpm", "higgs_audio") def perf_json_model_family(json_basename: str) -> str: """Classify a perf JSON config as omni, tts, or diffusion (mirrors nightly YAML runners).""" name = json_basename.lower() - if name.startswith("test_qwen3_omni"): + if name.startswith(("test_qwen3_omni", "test_minicpmo_")): return "omni" if any(hint in name for hint in TTS_PERF_JSON_HINTS): return "tts"