[Benchmark][MiniCPM-o] Add Video-MME dataset support for Omni bench s… (#5868)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
271d2f51e2
commit
d7a22612db
12 files changed
+1809
-17
No files matched your search
@@ -81,5 +81,60 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_a2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"npu": "A2"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,10 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
|
||||
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage.
|
||||
|
||||
Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
|
||||
~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
|
||||
text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
|
||||
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
|
||||
|
||||
Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs):
|
||||
``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``.
|
||||
Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,12 +39,18 @@ _RESULT_DIR = Path(
|
||||
)
|
||||
|
||||
_MIN_DAILY_OMNI_ACCURACY = 0.78
|
||||
# MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni.
|
||||
_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68"))
|
||||
_MAX_SEED_TTS_MEAN_WER = 0.05
|
||||
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||
# Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts.
|
||||
_DAILY_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
_VIDEOMME_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
_SEED_EXTRA_BODY = {
|
||||
"modalities": ["text", "audio"],
|
||||
"chat_template_kwargs": {
|
||||
@@ -58,6 +68,15 @@ _DAILY_OMNI_SERVER_ARGS = [
|
||||
"--media-io-kwargs",
|
||||
'{"video":{"fps":1,"num_frames":128}}',
|
||||
]
|
||||
# Video-MME ``minicpm-frames`` sends sampled frames as image_url (no AV interleave).
|
||||
# Allow local ``file://`` frame-cache URLs so CI does not need megabyte-scale base64
|
||||
# payloads; the autouse fixture still appends ``--videomme-inline-local-video`` when
|
||||
# the allowlist is unavailable.
|
||||
_VIDEOMME_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--allowed-local-media-path",
|
||||
os.environ.get("VLLM_TEST_ALLOWED_LOCAL_MEDIA_PATH", "/"),
|
||||
]
|
||||
_SEED_TTS_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
]
|
||||
@@ -72,6 +91,14 @@ daily_test_params = [
|
||||
)
|
||||
]
|
||||
|
||||
videomme_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_VIDEOMME_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
seed_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
@@ -88,18 +115,53 @@ def _require_vllm_cli() -> None:
|
||||
pytest.skip(str(exc))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Inline cached Hub/local videos because the test server has no local-media allowlist."""
|
||||
original = _acc_bench.daily_omni_bench_argv
|
||||
def _optional_local_videomme_dataset_path() -> str | None:
|
||||
"""Return an explicit local Video-MME root, or ``None`` to use the Hub default.
|
||||
|
||||
def _wrapped() -> list[str]:
|
||||
argv = list(original())
|
||||
Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as
|
||||
local. Hard-coded workspace paths are intentionally not probed so CI defaults to
|
||||
``lmms-lab/Video-MME`` via ``--videomme-repo``.
|
||||
"""
|
||||
for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"):
|
||||
raw = os.environ.get(key, "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
path = Path(raw).expanduser()
|
||||
if path.is_absolute() and path.is_dir():
|
||||
return str(path.resolve())
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Inline Hub/local media when the server has no usable local-media allowlist.
|
||||
|
||||
Daily-Omni always inlines (matches the historical Qwen/MiniCPM accuracy fixture).
|
||||
Video-MME prefers ``file://`` via ``--allowed-local-media-path`` (see
|
||||
``_VIDEOMME_SERVER_ARGS``); force inline only when ``VLLM_VIDEOMME_FORCE_INLINE=1``.
|
||||
"""
|
||||
original_daily = _acc_bench.daily_omni_bench_argv
|
||||
original_videomme = _acc_bench.videomme_bench_argv
|
||||
|
||||
def _wrap_daily() -> list[str]:
|
||||
argv = list(original_daily())
|
||||
if "--daily-omni-inline-local-video" not in argv:
|
||||
argv.append("--daily-omni-inline-local-video")
|
||||
return argv
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
|
||||
def _wrap_videomme() -> list[str]:
|
||||
argv = list(original_videomme())
|
||||
force_inline = os.environ.get("VLLM_VIDEOMME_FORCE_INLINE", "").strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
}
|
||||
if force_inline and "--videomme-inline-local-video" not in argv:
|
||||
argv.append("--videomme-inline-local-video")
|
||||
return argv
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrap_daily)
|
||||
monkeypatch.setattr(_acc_bench, "videomme_bench_argv", _wrap_videomme)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@@ -114,6 +176,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=False,
|
||||
skip_videomme=True,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
@@ -138,6 +201,55 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None:
|
||||
"""Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe."""
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("datasets")
|
||||
pytest.importorskip("huggingface_hub")
|
||||
pytest.importorskip("av")
|
||||
|
||||
dataset_path = _optional_local_videomme_dataset_path()
|
||||
num_prompts = int(os.environ.get("ACC_BENCH_VIDEOMME_NUM_PROMPTS", "2700"))
|
||||
max_concurrency = int(os.environ.get("ACC_BENCH_VIDEOMME_MAX_CONCURRENCY", "4"))
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=True,
|
||||
skip_videomme=False,
|
||||
num_prompts=num_prompts,
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--min-videomme-accuracy",
|
||||
str(_MIN_VIDEOMME_ACCURACY),
|
||||
"--temperature",
|
||||
"0",
|
||||
"--output-len",
|
||||
"128",
|
||||
"--videomme-pack-mode",
|
||||
"minicpm-frames",
|
||||
"--videomme-max-frames",
|
||||
"96",
|
||||
"--videomme-duration",
|
||||
os.environ.get("ACC_BENCH_VIDEOMME_DURATION", "all"),
|
||||
"--videomme-extra-body-json",
|
||||
json.dumps(_VIDEOMME_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
# Absolute local root only; otherwise --videomme-repo (Hub) from build_acc_benchmark_cli_argv.
|
||||
if dataset_path is not None:
|
||||
argv.extend(["--videomme-dataset-path", dataset_path])
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
@@ -148,6 +260,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
skip_videomme=True,
|
||||
max_concurrency=4,
|
||||
)
|
||||
argv.extend(
|
||||
@@ -176,6 +289,7 @@ def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
skip_videomme=True,
|
||||
num_prompts=50,
|
||||
max_concurrency=1,
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
|
||||
"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
|
||||
|
||||
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
|
||||
point to existing files, those are used with inline video. Otherwise the benchmark falls back to
|
||||
@@ -9,6 +9,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
|
||||
Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
|
||||
and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
|
||||
|
||||
Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
|
||||
``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
|
||||
``lmms-lab/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
|
||||
|
||||
Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
|
||||
and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
|
||||
"""
|
||||
@@ -25,6 +29,7 @@ from typing import Any, Protocol
|
||||
|
||||
DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
|
||||
DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
|
||||
DEFAULT_VIDEOMME_HF_REPO = "lmms-lab/Video-MME"
|
||||
|
||||
|
||||
class OmniBenchServerEndpoint(Protocol):
|
||||
@@ -40,6 +45,7 @@ def build_acc_benchmark_cli_argv(
|
||||
*,
|
||||
skip_seed: bool,
|
||||
skip_daily: bool,
|
||||
skip_videomme: bool = True,
|
||||
num_prompts: int | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> list[str]:
|
||||
@@ -49,6 +55,9 @@ def build_acc_benchmark_cli_argv(
|
||||
``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
|
||||
``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
|
||||
matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
|
||||
|
||||
Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
|
||||
callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
|
||||
"""
|
||||
n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
|
||||
n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
|
||||
@@ -71,6 +80,12 @@ def build_acc_benchmark_cli_argv(
|
||||
argv.append("--skip-seed-tts")
|
||||
if skip_daily:
|
||||
argv.append("--skip-daily-omni")
|
||||
if skip_videomme:
|
||||
argv.append("--skip-videomme")
|
||||
else:
|
||||
argv.append("--run-videomme")
|
||||
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
|
||||
argv.extend(["--videomme-repo", repo])
|
||||
return argv
|
||||
|
||||
|
||||
@@ -100,6 +115,43 @@ def daily_omni_bench_argv() -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def videomme_bench_argv() -> list[str]:
|
||||
"""CLI args for Video-MME (after ``vllm bench serve --omni``).
|
||||
|
||||
Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
|
||||
|
||||
* If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
|
||||
use that local mirror as ``--dataset-path``.
|
||||
* Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-lab/Video-MME``); the child
|
||||
bench downloads via ``huggingface_hub.snapshot_download``.
|
||||
"""
|
||||
root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip()
|
||||
if root:
|
||||
p = Path(root).expanduser()
|
||||
# Preserve Hub ids verbatim. Only canonicalize when the value is a real directory.
|
||||
if p.exists() and p.is_dir():
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
str(p.resolve()),
|
||||
]
|
||||
# Non-directory values (e.g. accidental Hub id in DATASET_PATH) fall through as path.
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
root,
|
||||
]
|
||||
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
repo,
|
||||
]
|
||||
|
||||
|
||||
def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
"""CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
|
||||
dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
|
||||
@@ -209,6 +261,12 @@ def assert_daily_omni_scored(result: dict[str, Any]) -> None:
|
||||
assert int(result.get("daily_omni_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (daily_omni_evaluated_ok==0)"
|
||||
|
||||
|
||||
def assert_videomme_scored(result: dict[str, Any]) -> None:
|
||||
acc = result.get("videomme_accuracy")
|
||||
assert acc is not None, "videomme_accuracy missing — wrong dataset or benchmark wiring"
|
||||
assert int(result.get("videomme_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (videomme_evaluated_ok==0)"
|
||||
|
||||
|
||||
def assert_seed_tts_scored(result: dict[str, Any]) -> None:
|
||||
err = result.get("seed_tts_eval_setup_error")
|
||||
assert not err, f"Seed-TTS eval deps/setup failed: {err}"
|
||||
|
||||
@@ -11,6 +11,9 @@ H100 / MI325) because they launch the live Omni server inside the test.
|
||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||
3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
|
||||
stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
|
||||
**0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
@@ -21,7 +24,8 @@ Prerequisites
|
||||
On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
|
||||
only drives the **client** benchmark.
|
||||
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
|
||||
``videomme``).
|
||||
|
||||
* **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
|
||||
the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
|
||||
@@ -31,6 +35,13 @@ Prerequisites
|
||||
the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo
|
||||
with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed.
|
||||
|
||||
* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-lab/Video-MME`` (override with
|
||||
``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
|
||||
``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an
|
||||
existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` /
|
||||
``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without ``--allowed-local-media-path``,
|
||||
pass ``--videomme-inline-local-video`` (or rely on the pytest fixture that appends it).
|
||||
|
||||
* **Seed-TTS** optional extras for WER/SIM/UTMOS::
|
||||
|
||||
pip install 'vllm-omni[dev]'
|
||||
@@ -55,6 +66,18 @@ Skip one suite, tighten gates::
|
||||
--skip-daily-omni \\
|
||||
--max-seed-tts-mean-wer 0.35 \\
|
||||
--min-seed-tts-mean-sim 0.75
|
||||
|
||||
MiniCPM-o Video-MME (w/o subs)::
|
||||
|
||||
python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
|
||||
--skip-daily-omni --skip-seed-tts --run-videomme \\
|
||||
--videomme-pack-mode minicpm-frames \\
|
||||
--videomme-max-frames 96 \\
|
||||
--temperature 0 --output-len 128 \\
|
||||
--min-videomme-accuracy 0.68
|
||||
|
||||
# Optional local mirror (absolute path) instead of Hub download:
|
||||
# --videomme-dataset-path /path/to/Video-MME
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -75,6 +98,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
load_benchmark_result,
|
||||
run_vllm_bench_subprocess,
|
||||
seed_tts_bench_argv,
|
||||
videomme_bench_argv,
|
||||
)
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
@@ -102,6 +126,20 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
|
||||
errs: list[str] = []
|
||||
acc = result.get("videomme_accuracy")
|
||||
if acc is None:
|
||||
errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
|
||||
return errs
|
||||
ev = int(result.get("videomme_evaluated_ok", 0) or 0)
|
||||
if ev <= 0:
|
||||
errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
|
||||
if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
|
||||
errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_seed_tts(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
@@ -130,7 +168,7 @@ def _validate_seed_tts(
|
||||
|
||||
|
||||
def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
"""Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
|
||||
"""Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
|
||||
repo = getattr(ns, "daily_omni_repo", None)
|
||||
if repo is not None and str(repo).strip():
|
||||
os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
|
||||
@@ -138,6 +176,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
|
||||
if ns.daily_omni_video_dir is not None:
|
||||
os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
|
||||
vm_repo = getattr(ns, "videomme_repo", None)
|
||||
if vm_repo is not None and str(vm_repo).strip():
|
||||
os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
|
||||
vm_path = getattr(ns, "videomme_dataset_path", None)
|
||||
if vm_path is not None and str(vm_path).strip():
|
||||
raw = str(vm_path).strip()
|
||||
p = Path(raw).expanduser()
|
||||
os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
|
||||
if ns.seed_tts_dataset_path is not None:
|
||||
# ``--seed-tts-dataset-path`` accepts either a local directory or a
|
||||
# Hugging Face repo id. Only resolve to an absolute filesystem path
|
||||
@@ -157,6 +203,9 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
"VLLM_DAILY_OMNI_REPO",
|
||||
"VLLM_DAILY_OMNI_QA_JSON",
|
||||
"VLLM_DAILY_OMNI_VIDEO_DIR",
|
||||
"VLLM_VIDEOMME_REPO",
|
||||
"VLLM_VIDEOMME_DATASET_PATH",
|
||||
"VIDEOMME_ROOT",
|
||||
"VLLM_SEED_TTS_DATASET_PATH",
|
||||
"SEED_TTS_ROOT",
|
||||
)
|
||||
@@ -225,6 +274,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
return out
|
||||
|
||||
|
||||
def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"omni_acc_videomme_{tag}.json"
|
||||
extra = json.loads(ns.videomme_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
+ videomme_bench_argv()
|
||||
+ [
|
||||
"--disable-shuffle",
|
||||
"--videomme-pack-mode",
|
||||
ns.videomme_pack_mode,
|
||||
"--videomme-duration",
|
||||
ns.videomme_duration,
|
||||
"--extra-body",
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.videomme_max_frames is not None:
|
||||
argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
|
||||
if ns.videomme_use_subtitle:
|
||||
argv.append("--videomme-use-subtitle")
|
||||
if ns.videomme_inline_local_video:
|
||||
argv.append("--videomme-inline-local-video")
|
||||
if ns.videomme_save_eval_items:
|
||||
argv.append("--videomme-save-eval-items")
|
||||
if ns.videomme_video_dir is not None:
|
||||
argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
|
||||
if ns.videomme_parquet is not None:
|
||||
argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
|
||||
if ns.videomme_subtitle_dir is not None:
|
||||
argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
run_vllm_bench_subprocess(vllm, argv)
|
||||
out = Path(ns.result_dir) / result_filename
|
||||
if not out.is_file():
|
||||
raise FileNotFoundError(f"Expected result JSON at {out}")
|
||||
return out
|
||||
|
||||
|
||||
def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
@@ -312,6 +401,20 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
|
||||
p.add_argument("--skip-daily-omni", action="store_true")
|
||||
p.add_argument("--skip-seed-tts", action="store_true")
|
||||
# Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
|
||||
p.set_defaults(skip_videomme=True)
|
||||
p.add_argument(
|
||||
"--skip-videomme",
|
||||
action="store_true",
|
||||
dest="skip_videomme",
|
||||
help="Skip Video-MME (default).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--run-videomme",
|
||||
action="store_false",
|
||||
dest="skip_videomme",
|
||||
help="Enable the Video-MME accuracy suite.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--daily-omni-repo",
|
||||
@@ -358,6 +461,87 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--videomme-repo",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
|
||||
"Default when no local path is set: lmms-lab/Video-MME. "
|
||||
"Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-dataset-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
|
||||
"download; Hub ids are also accepted and passed through as --dataset-path. "
|
||||
"Sets VLLM_VIDEOMME_DATASET_PATH.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-parquet",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit path to videomme/test-*.parquet.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-video-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit directory of extracted Video-MME videos.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-subtitle-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional subtitle directory (used with --videomme-use-subtitle).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-pack-mode",
|
||||
choices=("minicpm-frames", "minicpm-interleave", "video_url"),
|
||||
default="minicpm-frames",
|
||||
help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-max-frames",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-duration",
|
||||
choices=("all", "short", "medium", "long"),
|
||||
default="all",
|
||||
help="Filter Video-MME by duration bucket (default: all).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-use-subtitle",
|
||||
action="store_true",
|
||||
help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-inline-local-video",
|
||||
action="store_true",
|
||||
help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-extra-body-json",
|
||||
default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
|
||||
help="JSON merged into each chat request for Video-MME.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-save-eval-items",
|
||||
action="store_true",
|
||||
help="Include per-request Video-MME accuracy rows in the saved JSON.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--min-videomme-accuracy",
|
||||
type=float,
|
||||
default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
|
||||
help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
|
||||
"Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
|
||||
"override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--seed-tts-dataset-path",
|
||||
type=str,
|
||||
@@ -427,7 +611,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
|
||||
|
||||
|
||||
def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
"""Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
|
||||
"""Run configured client benches against a running server; return 0 on success."""
|
||||
failed: list[str] = []
|
||||
|
||||
with _preserve_benchmark_dataset_env():
|
||||
@@ -451,6 +635,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_videomme:
|
||||
path = run_videomme(ns, vllm)
|
||||
print(f"\n[Video-MME] result JSON: {path}", flush=True)
|
||||
data = load_benchmark_result(path)
|
||||
errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
|
||||
if errs:
|
||||
failed.extend([f"[Video-MME] {e}" for e in errs])
|
||||
else:
|
||||
print(
|
||||
f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
|
||||
f"evaluated_ok={data.get('videomme_evaluated_ok')} "
|
||||
f"per_duration={data.get('videomme_per_duration_accuracy')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_seed_tts:
|
||||
path = run_seed_tts(ns, vllm)
|
||||
print(f"\n[Seed-TTS] result JSON: {path}", flush=True)
|
||||
|
||||
Reference in new issue
Block a user