[Benchmark][MiniCPM-o] Add Video-MME dataset support for Omni bench s… (#5868)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-08-07 01:10:29 +08:00
1 parent 271d2f51e2
commit d7a22612db
12 files changed
+1809 -17

No files matched your search

@@ -81,5 +81,60 @@
}
}
]
},
{
"test_name": "test_minicpmo_4_5_a2",
"mark": [
{
"hardware_marks": {
"res": {
"npu": "A2"
},
"num_cards": 1
}
},
"full_model",
"omni"
],
"server_params": {
"model": "openbmb/MiniCPM-o-4_5",
"extra_cli_args": [
"--deploy-config",
"vllm_omni/deploy/minicpmo_4_5.yaml",
"--trust-remote-code"
]
},
"benchmark_params": [
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
128
],
"max_concurrency": [
1,
4,
8
],
"no_oversample": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
}
]
}
]
@@ -1,10 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage.
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs):
``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``.
Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin).
"""
from __future__ import annotations
@@ -35,12 +39,18 @@ _RESULT_DIR = Path(
)
_MIN_DAILY_OMNI_ACCURACY = 0.78
# MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni.
_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68"))
_MAX_SEED_TTS_MEAN_WER = 0.05
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
# Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts.
_DAILY_EXTRA_BODY = {
"modalities": ["text"],
"chat_template_kwargs": {"enable_thinking": False},
}
_VIDEOMME_EXTRA_BODY = {
"modalities": ["text"],
"chat_template_kwargs": {"enable_thinking": False},
}
_SEED_EXTRA_BODY = {
"modalities": ["text", "audio"],
"chat_template_kwargs": {
@@ -58,6 +68,15 @@ _DAILY_OMNI_SERVER_ARGS = [
"--media-io-kwargs",
'{"video":{"fps":1,"num_frames":128}}',
]
# Video-MME ``minicpm-frames`` sends sampled frames as image_url (no AV interleave).
# Allow local ``file://`` frame-cache URLs so CI does not need megabyte-scale base64
# payloads; the autouse fixture still appends ``--videomme-inline-local-video`` when
# the allowlist is unavailable.
_VIDEOMME_SERVER_ARGS = [
"--trust-remote-code",
"--allowed-local-media-path",
os.environ.get("VLLM_TEST_ALLOWED_LOCAL_MEDIA_PATH", "/"),
]
_SEED_TTS_SERVER_ARGS = [
"--trust-remote-code",
]
@@ -72,6 +91,14 @@ daily_test_params = [
)
]
videomme_test_params = [
OmniServerParams(
model=_MODEL,
stage_config_path=_DEPLOY_CONFIG,
server_args=list(_VIDEOMME_SERVER_ARGS),
)
]
seed_test_params = [
OmniServerParams(
model=_MODEL,
@@ -88,18 +115,53 @@ def _require_vllm_cli() -> None:
pytest.skip(str(exc))
@pytest.fixture(autouse=True)
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
original = _acc_bench.daily_omni_bench_argv
def _optional_local_videomme_dataset_path() -> str | None:
"""Return an explicit local Video-MME root, or ``None`` to use the Hub default.
def _wrapped() -> list[str]:
argv = list(original())
Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as
local. Hard-coded workspace paths are intentionally not probed so CI defaults to
``lmms-lab/Video-MME`` via ``--videomme-repo``.
"""
for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"):
raw = os.environ.get(key, "").strip()
if not raw:
continue
path = Path(raw).expanduser()
if path.is_absolute() and path.is_dir():
return str(path.resolve())
return None
@pytest.fixture(autouse=True)
def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None:
"""Inline Hub/local media when the server has no usable local-media allowlist.
Daily-Omni always inlines (matches the historical Qwen/MiniCPM accuracy fixture).
Video-MME prefers ``file://`` via ``--allowed-local-media-path`` (see
``_VIDEOMME_SERVER_ARGS``); force inline only when ``VLLM_VIDEOMME_FORCE_INLINE=1``.
"""
original_daily = _acc_bench.daily_omni_bench_argv
original_videomme = _acc_bench.videomme_bench_argv
def _wrap_daily() -> list[str]:
argv = list(original_daily())
if "--daily-omni-inline-local-video" not in argv:
argv.append("--daily-omni-inline-local-video")
return argv
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
def _wrap_videomme() -> list[str]:
argv = list(original_videomme())
force_inline = os.environ.get("VLLM_VIDEOMME_FORCE_INLINE", "").strip().lower() in {
"1",
"true",
"yes",
}
if force_inline and "--videomme-inline-local-video" not in argv:
argv.append("--videomme-inline-local-video")
return argv
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrap_daily)
monkeypatch.setattr(_acc_bench, "videomme_bench_argv", _wrap_videomme)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
@@ -114,6 +176,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
omni_server,
skip_seed=True,
skip_daily=False,
skip_videomme=True,
)
argv.extend(
[
@@ -138,6 +201,55 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True)
def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None:
"""Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe."""
_require_vllm_cli()
pytest.importorskip("datasets")
pytest.importorskip("huggingface_hub")
pytest.importorskip("av")
dataset_path = _optional_local_videomme_dataset_path()
num_prompts = int(os.environ.get("ACC_BENCH_VIDEOMME_NUM_PROMPTS", "2700"))
max_concurrency = int(os.environ.get("ACC_BENCH_VIDEOMME_MAX_CONCURRENCY", "4"))
argv = build_acc_benchmark_cli_argv(
omni_server,
skip_seed=True,
skip_daily=True,
skip_videomme=False,
num_prompts=num_prompts,
max_concurrency=max_concurrency,
)
argv.extend(
[
"--result-dir",
str(_RESULT_DIR),
"--min-videomme-accuracy",
str(_MIN_VIDEOMME_ACCURACY),
"--temperature",
"0",
"--output-len",
"128",
"--videomme-pack-mode",
"minicpm-frames",
"--videomme-max-frames",
"96",
"--videomme-duration",
os.environ.get("ACC_BENCH_VIDEOMME_DURATION", "all"),
"--videomme-extra-body-json",
json.dumps(_VIDEOMME_EXTRA_BODY, separators=(",", ":")),
"--trust-remote-code",
]
)
# Absolute local root only; otherwise --videomme-repo (Hub) from build_acc_benchmark_cli_argv.
if dataset_path is not None:
argv.extend(["--videomme-dataset-path", dataset_path])
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
@@ -148,6 +260,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
omni_server,
skip_seed=False,
skip_daily=True,
skip_videomme=True,
max_concurrency=4,
)
argv.extend(
@@ -176,6 +289,7 @@ def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
omni_server,
skip_seed=False,
skip_daily=True,
skip_videomme=True,
num_prompts=50,
max_concurrency=1,
)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
point to existing files, those are used with inline video. Otherwise the benchmark falls back to
@@ -9,6 +9,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
``lmms-lab/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
"""
@@ -25,6 +29,7 @@ from typing import Any, Protocol
DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
DEFAULT_VIDEOMME_HF_REPO = "lmms-lab/Video-MME"
class OmniBenchServerEndpoint(Protocol):
@@ -40,6 +45,7 @@ def build_acc_benchmark_cli_argv(
*,
skip_seed: bool,
skip_daily: bool,
skip_videomme: bool = True,
num_prompts: int | None = None,
max_concurrency: int | None = None,
) -> list[str]:
@@ -49,6 +55,9 @@ def build_acc_benchmark_cli_argv(
``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
"""
n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
@@ -71,6 +80,12 @@ def build_acc_benchmark_cli_argv(
argv.append("--skip-seed-tts")
if skip_daily:
argv.append("--skip-daily-omni")
if skip_videomme:
argv.append("--skip-videomme")
else:
argv.append("--run-videomme")
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
argv.extend(["--videomme-repo", repo])
return argv
@@ -100,6 +115,43 @@ def daily_omni_bench_argv() -> list[str]:
]
def videomme_bench_argv() -> list[str]:
"""CLI args for Video-MME (after ``vllm bench serve --omni``).
Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
* If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
use that local mirror as ``--dataset-path``.
* Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-lab/Video-MME``); the child
bench downloads via ``huggingface_hub.snapshot_download``.
"""
root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip()
if root:
p = Path(root).expanduser()
# Preserve Hub ids verbatim. Only canonicalize when the value is a real directory.
if p.exists() and p.is_dir():
return [
"--dataset-name",
"videomme",
"--dataset-path",
str(p.resolve()),
]
# Non-directory values (e.g. accidental Hub id in DATASET_PATH) fall through as path.
return [
"--dataset-name",
"videomme",
"--dataset-path",
root,
]
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
return [
"--dataset-name",
"videomme",
"--dataset-path",
repo,
]
def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
"""CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
@@ -209,6 +261,12 @@ def assert_daily_omni_scored(result: dict[str, Any]) -> None:
assert int(result.get("daily_omni_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (daily_omni_evaluated_ok==0)"
def assert_videomme_scored(result: dict[str, Any]) -> None:
acc = result.get("videomme_accuracy")
assert acc is not None, "videomme_accuracy missing — wrong dataset or benchmark wiring"
assert int(result.get("videomme_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (videomme_evaluated_ok==0)"
def assert_seed_tts_scored(result: dict[str, Any]) -> None:
err = result.get("seed_tts_eval_setup_error")
assert not err, f"Seed-TTS eval deps/setup failed: {err}"
@@ -11,6 +11,9 @@ H100 / MI325) because they launch the live Omni server inside the test.
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
**0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
Prerequisites
-------------
@@ -21,7 +24,8 @@ Prerequisites
On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
only drives the **client** benchmark.
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
``videomme``).
* **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
@@ -31,6 +35,13 @@ Prerequisites
the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo
with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed.
* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-lab/Video-MME`` (override with
``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an
existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` /
``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without ``--allowed-local-media-path``,
pass ``--videomme-inline-local-video`` (or rely on the pytest fixture that appends it).
* **Seed-TTS** optional extras for WER/SIM/UTMOS::
pip install 'vllm-omni[dev]'
@@ -55,6 +66,18 @@ Skip one suite, tighten gates::
--skip-daily-omni \\
--max-seed-tts-mean-wer 0.35 \\
--min-seed-tts-mean-sim 0.75
MiniCPM-o Video-MME (w/o subs)::
python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
--skip-daily-omni --skip-seed-tts --run-videomme \\
--videomme-pack-mode minicpm-frames \\
--videomme-max-frames 96 \\
--temperature 0 --output-len 128 \\
--min-videomme-accuracy 0.68
# Optional local mirror (absolute path) instead of Hub download:
# --videomme-dataset-path /path/to/Video-MME
"""
from __future__ import annotations
@@ -75,6 +98,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
load_benchmark_result,
run_vllm_bench_subprocess,
seed_tts_bench_argv,
videomme_bench_argv,
)
_REPO_ROOT = Path(__file__).resolve().parents[4]
@@ -102,6 +126,20 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
return errs
def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
errs: list[str] = []
acc = result.get("videomme_accuracy")
if acc is None:
errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
return errs
ev = int(result.get("videomme_evaluated_ok", 0) or 0)
if ev <= 0:
errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
return errs
def _validate_seed_tts(
result: dict[str, Any],
*,
@@ -130,7 +168,7 @@ def _validate_seed_tts(
def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
"""Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
"""Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
repo = getattr(ns, "daily_omni_repo", None)
if repo is not None and str(repo).strip():
os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
@@ -138,6 +176,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
if ns.daily_omni_video_dir is not None:
os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
vm_repo = getattr(ns, "videomme_repo", None)
if vm_repo is not None and str(vm_repo).strip():
os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
vm_path = getattr(ns, "videomme_dataset_path", None)
if vm_path is not None and str(vm_path).strip():
raw = str(vm_path).strip()
p = Path(raw).expanduser()
os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
if ns.seed_tts_dataset_path is not None:
# ``--seed-tts-dataset-path`` accepts either a local directory or a
# Hugging Face repo id. Only resolve to an absolute filesystem path
@@ -157,6 +203,9 @@ def _preserve_benchmark_dataset_env() -> Any:
"VLLM_DAILY_OMNI_REPO",
"VLLM_DAILY_OMNI_QA_JSON",
"VLLM_DAILY_OMNI_VIDEO_DIR",
"VLLM_VIDEOMME_REPO",
"VLLM_VIDEOMME_DATASET_PATH",
"VIDEOMME_ROOT",
"VLLM_SEED_TTS_DATASET_PATH",
"SEED_TTS_ROOT",
)
@@ -225,6 +274,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
return out
def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
ns.result_dir.mkdir(parents=True, exist_ok=True)
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
result_filename = f"omni_acc_videomme_{tag}.json"
extra = json.loads(ns.videomme_extra_body_json)
argv = (
_build_common_args(ns, result_filename=result_filename)
+ videomme_bench_argv()
+ [
"--disable-shuffle",
"--videomme-pack-mode",
ns.videomme_pack_mode,
"--videomme-duration",
ns.videomme_duration,
"--extra-body",
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
if ns.videomme_max_frames is not None:
argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
if ns.videomme_use_subtitle:
argv.append("--videomme-use-subtitle")
if ns.videomme_inline_local_video:
argv.append("--videomme-inline-local-video")
if ns.videomme_save_eval_items:
argv.append("--videomme-save-eval-items")
if ns.videomme_video_dir is not None:
argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
if ns.videomme_parquet is not None:
argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
if ns.videomme_subtitle_dir is not None:
argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
print("\n$", vllm, *argv, "\n", flush=True)
run_vllm_bench_subprocess(vllm, argv)
out = Path(ns.result_dir) / result_filename
if not out.is_file():
raise FileNotFoundError(f"Expected result JSON at {out}")
return out
def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
ns.result_dir.mkdir(parents=True, exist_ok=True)
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
@@ -312,6 +401,20 @@ def build_arg_parser() -> argparse.ArgumentParser:
p.add_argument("--skip-daily-omni", action="store_true")
p.add_argument("--skip-seed-tts", action="store_true")
# Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
p.set_defaults(skip_videomme=True)
p.add_argument(
"--skip-videomme",
action="store_true",
dest="skip_videomme",
help="Skip Video-MME (default).",
)
p.add_argument(
"--run-videomme",
action="store_false",
dest="skip_videomme",
help="Enable the Video-MME accuracy suite.",
)
p.add_argument(
"--daily-omni-repo",
@@ -358,6 +461,87 @@ def build_arg_parser() -> argparse.ArgumentParser:
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
)
p.add_argument(
"--videomme-repo",
type=str,
default=None,
help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
"Default when no local path is set: lmms-lab/Video-MME. "
"Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
)
p.add_argument(
"--videomme-dataset-path",
type=str,
default=None,
help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
"download; Hub ids are also accepted and passed through as --dataset-path. "
"Sets VLLM_VIDEOMME_DATASET_PATH.",
)
p.add_argument(
"--videomme-parquet",
type=Path,
default=None,
help="Optional explicit path to videomme/test-*.parquet.",
)
p.add_argument(
"--videomme-video-dir",
type=Path,
default=None,
help="Optional explicit directory of extracted Video-MME videos.",
)
p.add_argument(
"--videomme-subtitle-dir",
type=Path,
default=None,
help="Optional subtitle directory (used with --videomme-use-subtitle).",
)
p.add_argument(
"--videomme-pack-mode",
choices=("minicpm-frames", "minicpm-interleave", "video_url"),
default="minicpm-frames",
help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
)
p.add_argument(
"--videomme-max-frames",
type=int,
default=None,
help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
)
p.add_argument(
"--videomme-duration",
choices=("all", "short", "medium", "long"),
default="all",
help="Filter Video-MME by duration bucket (default: all).",
)
p.add_argument(
"--videomme-use-subtitle",
action="store_true",
help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
)
p.add_argument(
"--videomme-inline-local-video",
action="store_true",
help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
)
p.add_argument(
"--videomme-extra-body-json",
default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
help="JSON merged into each chat request for Video-MME.",
)
p.add_argument(
"--videomme-save-eval-items",
action="store_true",
help="Include per-request Video-MME accuracy rows in the saved JSON.",
)
p.add_argument(
"--min-videomme-accuracy",
type=float,
default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
"Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
"override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
)
p.add_argument(
"--seed-tts-dataset-path",
type=str,
@@ -427,7 +611,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
def run_acc_benchmark(ns: argparse.Namespace) -> int:
"""Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
"""Run configured client benches against a running server; return 0 on success."""
failed: list[str] = []
with _preserve_benchmark_dataset_env():
@@ -451,6 +635,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
flush=True,
)
if not ns.skip_videomme:
path = run_videomme(ns, vllm)
print(f"\n[Video-MME] result JSON: {path}", flush=True)
data = load_benchmark_result(path)
errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
if errs:
failed.extend([f"[Video-MME] {e}" for e in errs])
else:
print(
f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
f"evaluated_ok={data.get('videomme_evaluated_ok')} "
f"per_duration={data.get('videomme_per_duration_accuracy')}",
flush=True,
)
if not ns.skip_seed_tts:
path = run_seed_tts(ns, vllm)
print(f"\n[Seed-TTS] result JSON: {path}", flush=True)