diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 8b641a0f4..ab9f0e060 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -122,7 +122,7 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "cuda and H100" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 4925f84c6..b06201146 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -98,6 +98,22 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)" + key: nightly-omni-performance-minicpmo-4-5-npu-a2 + timeout_in_minutes: 180 + mirror_hardwares: a2b3_npu_1 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test" key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu timeout_in_minutes: 180 diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index 19b8540d3..3feaf2e4c 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -81,5 +81,60 @@ } } ] + }, + { + "test_name": "test_minicpmo_4_5_a2", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A2" + }, + "num_cards": 1 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8 + ], + "no_oversample": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] } ] diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index 941a87c5b..ef939b9be 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -1,10 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 -"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage. +"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage. Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches ~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``, text modalities with ``use_tts_template``, and server ``--interleave-mm-strings`` + 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``). + +Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs): +``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``. +Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin). """ from __future__ import annotations @@ -35,12 +39,18 @@ _RESULT_DIR = Path( ) _MIN_DAILY_OMNI_ACCURACY = 0.78 +# MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni. +_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68")) _MAX_SEED_TTS_MEAN_WER = 0.05 -# Match the validated Daily-Omni client body from daily_omni_bench.sh. +# Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts. _DAILY_EXTRA_BODY = { "modalities": ["text"], "chat_template_kwargs": {"enable_thinking": False}, } +_VIDEOMME_EXTRA_BODY = { + "modalities": ["text"], + "chat_template_kwargs": {"enable_thinking": False}, +} _SEED_EXTRA_BODY = { "modalities": ["text", "audio"], "chat_template_kwargs": { @@ -58,6 +68,15 @@ _DAILY_OMNI_SERVER_ARGS = [ "--media-io-kwargs", '{"video":{"fps":1,"num_frames":128}}', ] +# Video-MME ``minicpm-frames`` sends sampled frames as image_url (no AV interleave). +# Allow local ``file://`` frame-cache URLs so CI does not need megabyte-scale base64 +# payloads; the autouse fixture still appends ``--videomme-inline-local-video`` when +# the allowlist is unavailable. +_VIDEOMME_SERVER_ARGS = [ + "--trust-remote-code", + "--allowed-local-media-path", + os.environ.get("VLLM_TEST_ALLOWED_LOCAL_MEDIA_PATH", "/"), +] _SEED_TTS_SERVER_ARGS = [ "--trust-remote-code", ] @@ -72,6 +91,14 @@ daily_test_params = [ ) ] +videomme_test_params = [ + OmniServerParams( + model=_MODEL, + stage_config_path=_DEPLOY_CONFIG, + server_args=list(_VIDEOMME_SERVER_ARGS), + ) +] + seed_test_params = [ OmniServerParams( model=_MODEL, @@ -88,18 +115,53 @@ def _require_vllm_cli() -> None: pytest.skip(str(exc)) -@pytest.fixture(autouse=True) -def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None: - """Inline cached Hub/local videos because the test server has no local-media allowlist.""" - original = _acc_bench.daily_omni_bench_argv +def _optional_local_videomme_dataset_path() -> str | None: + """Return an explicit local Video-MME root, or ``None`` to use the Hub default. - def _wrapped() -> list[str]: - argv = list(original()) + Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as + local. Hard-coded workspace paths are intentionally not probed so CI defaults to + ``lmms-lab/Video-MME`` via ``--videomme-repo``. + """ + for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"): + raw = os.environ.get(key, "").strip() + if not raw: + continue + path = Path(raw).expanduser() + if path.is_absolute() and path.is_dir(): + return str(path.resolve()) + return None + + +@pytest.fixture(autouse=True) +def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None: + """Inline Hub/local media when the server has no usable local-media allowlist. + + Daily-Omni always inlines (matches the historical Qwen/MiniCPM accuracy fixture). + Video-MME prefers ``file://`` via ``--allowed-local-media-path`` (see + ``_VIDEOMME_SERVER_ARGS``); force inline only when ``VLLM_VIDEOMME_FORCE_INLINE=1``. + """ + original_daily = _acc_bench.daily_omni_bench_argv + original_videomme = _acc_bench.videomme_bench_argv + + def _wrap_daily() -> list[str]: + argv = list(original_daily()) if "--daily-omni-inline-local-video" not in argv: argv.append("--daily-omni-inline-local-video") return argv - monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped) + def _wrap_videomme() -> list[str]: + argv = list(original_videomme()) + force_inline = os.environ.get("VLLM_VIDEOMME_FORCE_INLINE", "").strip().lower() in { + "1", + "true", + "yes", + } + if force_inline and "--videomme-inline-local-video" not in argv: + argv.append("--videomme-inline-local-video") + return argv + + monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrap_daily) + monkeypatch.setattr(_acc_bench, "videomme_bench_argv", _wrap_videomme) monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") @@ -114,6 +176,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: omni_server, skip_seed=True, skip_daily=False, + skip_videomme=True, ) argv.extend( [ @@ -138,6 +201,55 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True) +def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None: + """Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe.""" + _require_vllm_cli() + pytest.importorskip("datasets") + pytest.importorskip("huggingface_hub") + pytest.importorskip("av") + + dataset_path = _optional_local_videomme_dataset_path() + num_prompts = int(os.environ.get("ACC_BENCH_VIDEOMME_NUM_PROMPTS", "2700")) + max_concurrency = int(os.environ.get("ACC_BENCH_VIDEOMME_MAX_CONCURRENCY", "4")) + + argv = build_acc_benchmark_cli_argv( + omni_server, + skip_seed=True, + skip_daily=True, + skip_videomme=False, + num_prompts=num_prompts, + max_concurrency=max_concurrency, + ) + argv.extend( + [ + "--result-dir", + str(_RESULT_DIR), + "--min-videomme-accuracy", + str(_MIN_VIDEOMME_ACCURACY), + "--temperature", + "0", + "--output-len", + "128", + "--videomme-pack-mode", + "minicpm-frames", + "--videomme-max-frames", + "96", + "--videomme-duration", + os.environ.get("ACC_BENCH_VIDEOMME_DURATION", "all"), + "--videomme-extra-body-json", + json.dumps(_VIDEOMME_EXTRA_BODY, separators=(",", ":")), + "--trust-remote-code", + ] + ) + # Absolute local root only; otherwise --videomme-repo (Hub) from build_acc_benchmark_cli_argv. + if dataset_path is not None: + argv.extend(["--videomme-dataset-path", dataset_path]) + + assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 + + @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", seed_test_params, indirect=True) def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: @@ -148,6 +260,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: omni_server, skip_seed=False, skip_daily=True, + skip_videomme=True, max_concurrency=4, ) argv.extend( @@ -176,6 +289,7 @@ def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None: omni_server, skip_seed=False, skip_daily=True, + skip_videomme=True, num_prompts=50, max_concurrency=1, ) diff --git a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py index dfc53a06e..d0b9f067e 100644 --- a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py +++ b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs. +"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs. Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR`` point to existing files, those are used with inline video. Otherwise the benchmark falls back to @@ -9,6 +9,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand. +Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` / +``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id +``lmms-lab/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives. + Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver. """ @@ -25,6 +29,7 @@ from typing import Any, Protocol DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni" DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval" +DEFAULT_VIDEOMME_HF_REPO = "lmms-lab/Video-MME" class OmniBenchServerEndpoint(Protocol): @@ -40,6 +45,7 @@ def build_acc_benchmark_cli_argv( *, skip_seed: bool, skip_daily: bool, + skip_videomme: bool = True, num_prompts: int | None = None, max_concurrency: int | None = None, ) -> list[str]: @@ -49,6 +55,9 @@ def build_acc_benchmark_cli_argv( ``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` / ``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env. + + Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS + callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it. """ n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts) n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency) @@ -71,6 +80,12 @@ def build_acc_benchmark_cli_argv( argv.append("--skip-seed-tts") if skip_daily: argv.append("--skip-daily-omni") + if skip_videomme: + argv.append("--skip-videomme") + else: + argv.append("--run-videomme") + repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO + argv.extend(["--videomme-repo", repo]) return argv @@ -100,6 +115,43 @@ def daily_omni_bench_argv() -> list[str]: ] +def videomme_bench_argv() -> list[str]: + """CLI args for Video-MME (after ``vllm bench serve --omni``). + + Same local-vs-Hub rule as :func:`seed_tts_bench_argv`: + + * If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory, + use that local mirror as ``--dataset-path``. + * Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-lab/Video-MME``); the child + bench downloads via ``huggingface_hub.snapshot_download``. + """ + root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip() + if root: + p = Path(root).expanduser() + # Preserve Hub ids verbatim. Only canonicalize when the value is a real directory. + if p.exists() and p.is_dir(): + return [ + "--dataset-name", + "videomme", + "--dataset-path", + str(p.resolve()), + ] + # Non-directory values (e.g. accidental Hub id in DATASET_PATH) fall through as path. + return [ + "--dataset-name", + "videomme", + "--dataset-path", + root, + ] + repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO + return [ + "--dataset-name", + "videomme", + "--dataset-path", + repo, + ] + + def seed_tts_bench_argv(*, locale: str = "en") -> list[str]: """CLI args for Seed-TTS (after ``vllm bench serve --omni``).""" dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip() @@ -209,6 +261,12 @@ def assert_daily_omni_scored(result: dict[str, Any]) -> None: assert int(result.get("daily_omni_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (daily_omni_evaluated_ok==0)" +def assert_videomme_scored(result: dict[str, Any]) -> None: + acc = result.get("videomme_accuracy") + assert acc is not None, "videomme_accuracy missing — wrong dataset or benchmark wiring" + assert int(result.get("videomme_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (videomme_evaluated_ok==0)" + + def assert_seed_tts_scored(result: dict[str, Any]) -> None: err = result.get("seed_tts_eval_setup_error") assert not err, f"Seed-TTS eval deps/setup failed: {err}" diff --git a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py index 2bacfa798..27a07b9a9 100644 --- a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py +++ b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py @@ -11,6 +11,9 @@ H100 / MI325) because they launch the live Omni server inside the test. run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``). 2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used (WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`). +3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers + stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is + **0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``). Prerequisites ------------- @@ -21,7 +24,8 @@ Prerequisites On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script only drives the **client** benchmark. -* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``). +* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` / + ``videomme``). * **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env), the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child** @@ -31,6 +35,13 @@ Prerequisites the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed. +* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-lab/Video-MME`` (override with + ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via + ``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an + existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` / + ``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without ``--allowed-local-media-path``, + pass ``--videomme-inline-local-video`` (or rely on the pytest fixture that appends it). + * **Seed-TTS** optional extras for WER/SIM/UTMOS:: pip install 'vllm-omni[dev]' @@ -55,6 +66,18 @@ Skip one suite, tighten gates:: --skip-daily-omni \\ --max-seed-tts-mean-wer 0.35 \\ --min-seed-tts-mean-sim 0.75 + +MiniCPM-o Video-MME (w/o subs):: + + python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\ + --skip-daily-omni --skip-seed-tts --run-videomme \\ + --videomme-pack-mode minicpm-frames \\ + --videomme-max-frames 96 \\ + --temperature 0 --output-len 128 \\ + --min-videomme-accuracy 0.68 + + # Optional local mirror (absolute path) instead of Hub download: + # --videomme-dataset-path /path/to/Video-MME """ from __future__ import annotations @@ -75,6 +98,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import ( load_benchmark_result, run_vllm_bench_subprocess, seed_tts_bench_argv, + videomme_bench_argv, ) _REPO_ROOT = Path(__file__).resolve().parents[4] @@ -102,6 +126,20 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None) return errs +def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]: + errs: list[str] = [] + acc = result.get("videomme_accuracy") + if acc is None: + errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).") + return errs + ev = int(result.get("videomme_evaluated_ok", 0) or 0) + if ev <= 0: + errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.") + if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy): + errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}") + return errs + + def _validate_seed_tts( result: dict[str, Any], *, @@ -130,7 +168,7 @@ def _validate_seed_tts( def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None: - """Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``.""" + """Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers.""" repo = getattr(ns, "daily_omni_repo", None) if repo is not None and str(repo).strip(): os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip() @@ -138,6 +176,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None: os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve()) if ns.daily_omni_video_dir is not None: os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve()) + vm_repo = getattr(ns, "videomme_repo", None) + if vm_repo is not None and str(vm_repo).strip(): + os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip() + vm_path = getattr(ns, "videomme_dataset_path", None) + if vm_path is not None and str(vm_path).strip(): + raw = str(vm_path).strip() + p = Path(raw).expanduser() + os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw if ns.seed_tts_dataset_path is not None: # ``--seed-tts-dataset-path`` accepts either a local directory or a # Hugging Face repo id. Only resolve to an absolute filesystem path @@ -157,6 +203,9 @@ def _preserve_benchmark_dataset_env() -> Any: "VLLM_DAILY_OMNI_REPO", "VLLM_DAILY_OMNI_QA_JSON", "VLLM_DAILY_OMNI_VIDEO_DIR", + "VLLM_VIDEOMME_REPO", + "VLLM_VIDEOMME_DATASET_PATH", + "VIDEOMME_ROOT", "VLLM_SEED_TTS_DATASET_PATH", "SEED_TTS_ROOT", ) @@ -225,6 +274,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path: return out +def run_videomme(ns: argparse.Namespace, vllm: str) -> Path: + ns.result_dir.mkdir(parents=True, exist_ok=True) + tag = datetime.now().strftime("%Y%m%d-%H%M%S") + result_filename = f"omni_acc_videomme_{tag}.json" + extra = json.loads(ns.videomme_extra_body_json) + argv = ( + _build_common_args(ns, result_filename=result_filename) + + videomme_bench_argv() + + [ + "--disable-shuffle", + "--videomme-pack-mode", + ns.videomme_pack_mode, + "--videomme-duration", + ns.videomme_duration, + "--extra-body", + json.dumps(extra, ensure_ascii=False, separators=(",", ":")), + ] + ) + if ns.videomme_max_frames is not None: + argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))]) + if ns.videomme_use_subtitle: + argv.append("--videomme-use-subtitle") + if ns.videomme_inline_local_video: + argv.append("--videomme-inline-local-video") + if ns.videomme_save_eval_items: + argv.append("--videomme-save-eval-items") + if ns.videomme_video_dir is not None: + argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())]) + if ns.videomme_parquet is not None: + argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())]) + if ns.videomme_subtitle_dir is not None: + argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())]) + print("\n$", vllm, *argv, "\n", flush=True) + run_vllm_bench_subprocess(vllm, argv) + out = Path(ns.result_dir) / result_filename + if not out.is_file(): + raise FileNotFoundError(f"Expected result JSON at {out}") + return out + + def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path: ns.result_dir.mkdir(parents=True, exist_ok=True) tag = datetime.now().strftime("%Y%m%d-%H%M%S") @@ -312,6 +401,20 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--skip-daily-omni", action="store_true") p.add_argument("--skip-seed-tts", action="store_true") + # Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged. + p.set_defaults(skip_videomme=True) + p.add_argument( + "--skip-videomme", + action="store_true", + dest="skip_videomme", + help="Skip Video-MME (default).", + ) + p.add_argument( + "--run-videomme", + action="store_false", + dest="skip_videomme", + help="Enable the Video-MME accuracy suite.", + ) p.add_argument( "--daily-omni-repo", @@ -358,6 +461,87 @@ def build_arg_parser() -> argparse.ArgumentParser: "Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.", ) + p.add_argument( + "--videomme-repo", + type=str, + default=None, + help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). " + "Default when no local path is set: lmms-lab/Video-MME. " + "Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.", + ) + p.add_argument( + "--videomme-dataset-path", + type=str, + default=None, + help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub " + "download; Hub ids are also accepted and passed through as --dataset-path. " + "Sets VLLM_VIDEOMME_DATASET_PATH.", + ) + p.add_argument( + "--videomme-parquet", + type=Path, + default=None, + help="Optional explicit path to videomme/test-*.parquet.", + ) + p.add_argument( + "--videomme-video-dir", + type=Path, + default=None, + help="Optional explicit directory of extracted Video-MME videos.", + ) + p.add_argument( + "--videomme-subtitle-dir", + type=Path, + default=None, + help="Optional subtitle directory (used with --videomme-use-subtitle).", + ) + p.add_argument( + "--videomme-pack-mode", + choices=("minicpm-frames", "minicpm-interleave", "video_url"), + default="minicpm-frames", + help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).", + ) + p.add_argument( + "--videomme-max-frames", + type=int, + default=None, + help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).", + ) + p.add_argument( + "--videomme-duration", + choices=("all", "short", "medium", "long"), + default="all", + help="Filter Video-MME by duration bucket (default: all).", + ) + p.add_argument( + "--videomme-use-subtitle", + action="store_true", + help="Prepend subtitle text to the user prompt (Video-MME w/ subs).", + ) + p.add_argument( + "--videomme-inline-local-video", + action="store_true", + help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).", + ) + p.add_argument( + "--videomme-extra-body-json", + default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}', + help="JSON merged into each chat request for Video-MME.", + ) + p.add_argument( + "--videomme-save-eval-items", + action="store_true", + help="Include per-request Video-MME accuracy rows in the saved JSON.", + ) + p.add_argument( + "--min-videomme-accuracy", + type=float, + default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"), + help="Fail when videomme_accuracy is strictly below this threshold (0–1). " + "Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); " + "override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.", + ) + p.add_argument( "--seed-tts-dataset-path", type=str, @@ -427,7 +611,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac def run_acc_benchmark(ns: argparse.Namespace) -> int: - """Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success.""" + """Run configured client benches against a running server; return 0 on success.""" failed: list[str] = [] with _preserve_benchmark_dataset_env(): @@ -451,6 +635,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int: flush=True, ) + if not ns.skip_videomme: + path = run_videomme(ns, vllm) + print(f"\n[Video-MME] result JSON: {path}", flush=True) + data = load_benchmark_result(path) + errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy) + if errs: + failed.extend([f"[Video-MME] {e}" for e in errs]) + else: + print( + f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} " + f"evaluated_ok={data.get('videomme_evaluated_ok')} " + f"per_duration={data.get('videomme_per_duration_accuracy')}", + flush=True, + ) + if not ns.skip_seed_tts: path = run_seed_tts(ns, vllm) print(f"\n[Seed-TTS] result JSON: {path}", flush=True) diff --git a/vllm_omni/benchmarks/data_modules/videomme_dataset.py b/vllm_omni/benchmarks/data_modules/videomme_dataset.py new file mode 100644 index 000000000..68a5263e6 --- /dev/null +++ b/vllm_omni/benchmarks/data_modules/videomme_dataset.py @@ -0,0 +1,894 @@ +"""Video-MME dataset loader for vLLM-Omni bench serve. + +Video-MME (lmms-lab/Video-MME) is a full-spectrum video MLLM benchmark with +900 videos / 2,700 MCQ pairs across short / medium / long durations. + +This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe +(``o_e_Kit/configs/generation_configs.json``): + +- ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false`` +- ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave`` + +Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame +timestamps match OmniEvalKit ``_sample_video_frame_indices``. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import os +import re +import time +import zipfile +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +try: + from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest +except ImportError: + from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset + from vllm.benchmarks.datasets import SampleRequest +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.hf import get_cached_tokenizer + +try: + from datasets import load_dataset +except ImportError: + load_dataset = None + +from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( + MINICPM_OMNI_SYSTEM_TEXT, + _ListDatasetIterator, + _numpy_to_wav_bytes, + _pil_to_jpeg_bytes, + _uniform_sample_indices, +) + +logger = logging.getLogger(__name__) + +_MINICPM_AUDIO_SR = 16000 + +# OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults. +VIDEOMME_DEFAULT_MAX_FRAMES = 96 +VIDEOMME_SHORT_MAX_FRAMES = 64 +VIDEOMME_DEFAULT_HF_REPO = "lmms-lab/Video-MME" + +#: Decode forward to reach a target this close, seek to the preceding keyframe beyond it. +_SEEK_THRESHOLD_S = 3.0 + +#: How often ``sample`` reports progress while warming a cold frame cache. +_SAMPLE_PROGRESS_INTERVAL_S = 30.0 + +VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"] +VideoMMEDurationFilter = Literal["all", "short", "medium", "long"] + +# OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped; +# media parts are prepended as OpenAI content parts, matching ``build_content``). +VIDEOMME_USER_PROMPT_TEMPLATE = ( + "Carefully read the following question and select the letter corresponding to the " + "correct answer.Highlight the applicable choices without giving explanations.\n" + "{question}\n" + "Options:\n" + "{options}\n" + "Please select the correct answer from the options above. Only respond with the letter." +) + +_OPTION_LETTERS = "ABCDEFGHIJKL" +_OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$") + +_VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi") + + +def _iter_video_files(root: Path) -> Iterator[Path]: + """Yield video files under ``root``, skipping dot-dirs such as the frame cache.""" + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + for name in filenames: + if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES: + yield Path(dirpath) / name + + +def _resolve_hf_cache_snapshot(path: Path) -> Path: + """Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir.""" + snapshots = path / "snapshots" + if not snapshots.is_dir(): + return path + ref = path / "refs" / "main" + if ref.is_file(): + try: + target = snapshots / ref.read_text(encoding="utf-8").strip() + except OSError: + target = snapshots + if target.is_dir(): + return target + revisions = [p for p in snapshots.iterdir() if p.is_dir()] + if revisions: + return max(revisions, key=lambda p: p.stat().st_mtime) + return path + + +def resolve_videomme_local_root(dataset_path: str | None) -> Path | None: + """Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror. + + Hub ids such as ``lmms-lab/Video-MME`` return ``None`` (they are not directories). + """ + raw = (dataset_path or "").strip() + if not raw: + return None + path = Path(raw).expanduser() + if not path.is_dir(): + return None + return _resolve_hf_cache_snapshot(path.resolve()) + + +def ensure_videomme_hub_root(repo_id: str) -> Path: + """Download the Video-MME dataset snapshot from Hugging Face and return its root. + + Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`: + Hub ids trigger ``snapshot_download``; callers then extract videos via + :func:`ensure_videomme_videos_extracted`. + + Raises: + ImportError: if ``huggingface_hub`` is not installed. + ValueError: if ``repo_id`` is empty. + """ + rid = (repo_id or "").strip() + if not rid: + raise ValueError("repo_id is required to download Video-MME from Hugging Face") + + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise ImportError( + "Install huggingface_hub to download Video-MME from the Hub, or pass a local " + "--dataset-path / --videomme-video-dir with parquet + extracted videos." + ) from e + + # Pull QA parquet + video/subtitle archives; skip unrelated repo files. + cache = snapshot_download( + repo_id=rid, + repo_type="dataset", + allow_patterns=[ + "videomme/**", + "*.parquet", + "videos_chunked_*.zip", + "subtitle.zip", + "subtitle/**", + "video/**", + "videos/**", + ], + ) + root = _resolve_hf_cache_snapshot(Path(cache).resolve()) + logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid) + return root + + +def resolve_videomme_root(dataset_path: str | None) -> Path: + """Return a Video-MME root containing parquet and (or zip) video assets. + + * Existing local directories (absolute or relative) are used as-is. + * Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and + downloaded via :func:`ensure_videomme_hub_root` (default + ``lmms-lab/Video-MME``). + """ + local = resolve_videomme_local_root(dataset_path) + if local is not None: + return local + rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO + return ensure_videomme_hub_root(rid) + + +def videomme_local_parquet(root: Path) -> Path | None: + for candidate in ( + root / "videomme" / "test-00000-of-00001.parquet", + root / "test-00000-of-00001.parquet", + ): + if candidate.is_file(): + return candidate + return None + + +def videomme_local_video_dir(root: Path) -> Path | None: + """Return a directory already holding extracted videos, flat or nested. + + Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``, + so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of + the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable. + """ + for name in ("video", "videos", "data"): + candidate = root / name + if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None: + return candidate + return None + + +def videomme_local_subtitle_dir(root: Path) -> Path | None: + candidate = root / "subtitle" + return candidate if candidate.is_dir() else None + + +def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None: + """Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout).""" + name = os.path.basename(member) + if not name: + return + dest = dest_dir / name + if dest.is_file() and dest.stat().st_size > 0: + return + dest_dir.mkdir(parents=True, exist_ok=True) + with zf.open(member) as src, open(dest, "wb") as out: + out.write(src.read()) + + +def ensure_videomme_videos_extracted(root: Path) -> Path: + """Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``.""" + video_dir = root / "video" + marker = root / ".videomme_videos_extracted" + if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None: + return video_dir + + # Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it. + existing = videomme_local_video_dir(root) + if existing is not None and existing != video_dir: + return existing + + zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()] + if not zips and not video_dir.is_dir(): + raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip") + + video_dir.mkdir(parents=True, exist_ok=True) + for zp in zips: + logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir) + with zipfile.ZipFile(zp, "r") as zf: + for member in zf.namelist(): + if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")): + _unzip_member_flat(zf, member, video_dir) + + marker.write_text("ok", encoding="utf-8") + return video_dir + + +def ensure_videomme_subtitles_extracted(root: Path) -> Path | None: + """Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present.""" + subtitle_dir = root / "subtitle" + if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None: + return subtitle_dir + + zip_path = root / "subtitle.zip" + if not zip_path.is_file(): + return subtitle_dir if subtitle_dir.is_dir() else None + + subtitle_dir.mkdir(parents=True, exist_ok=True) + logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir) + with zipfile.ZipFile(zip_path, "r") as zf: + for member in zf.namelist(): + if member.lower().endswith(".srt"): + _unzip_member_flat(zf, member, subtitle_dir) + return subtitle_dir + + +def _probe_video_duration(video_path: Path) -> float: + """Container duration in seconds, without decoding the stream. + + Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to + an hour, and ``stream.frames == 0`` containers would otherwise force a full decode + just to learn the duration. + """ + import av + + with av.open(str(video_path)) as container: + stream = container.streams.video[0] + duration: float | None = None + if stream.duration is not None and stream.time_base is not None: + duration = float(stream.duration * stream.time_base) + elif container.duration is not None: + duration = container.duration / av.time_base + elif stream.frames and stream.average_rate: + duration = stream.frames / float(stream.average_rate) + + if not duration or duration <= 0: + raise ValueError(f"Could not determine duration of {video_path}") + return duration + + +def _decode_frames_at_timestamps( + video_path: Path, + timestamps: list[float], + *, + seek_threshold_s: float = _SEEK_THRESHOLD_S, +) -> list[Any]: + """Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp. + + A single forward pass over the stream — the Daily-Omni approach — is not affordable + here: Video-MME samples points spread across the whole video, so the last target sits + near the end and every frame of an hour-long file would be decoded. Targets within + ``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper + than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding + keyframe first. + """ + import av + + if not timestamps: + return [] + + images: list[Any] = [] + with av.open(str(video_path)) as container: + stream = container.streams.video[0] + stream.thread_type = "AUTO" + time_base = float(stream.time_base) if stream.time_base else 0.0 + start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0 + + decoder = None + frame = None + frame_time = -1.0 + image = None + + for ts in timestamps: + if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s: + if time_base > 0: + container.seek(int((ts + start_s) / time_base), stream=stream, backward=True) + else: + container.seek(0, stream=stream, backward=True) + decoder = container.decode(stream) + frame, image, frame_time = None, None, -1.0 + + while frame is None or frame_time < ts: + nxt = next(decoder, None) + if nxt is None: + break + frame, image = nxt, None + if nxt.pts is not None and time_base > 0: + frame_time = float(nxt.pts) * time_base - start_s + + if frame is None: + break + if image is None: + image = frame.to_image() + images.append(image) + + if not images: + raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}") + # Container duration can overshoot the real stream; repeat the last frame (decord clamping). + images.extend([images[-1]] * (len(timestamps) - len(images))) + return images + + +@dataclass +class VideoMMESampleRequest(SampleRequest): + """``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring.""" + + videomme_gold_answer: str = "" + videomme_video_id: str = "" + videomme_question_id: str = "" + videomme_duration: str = "" + videomme_domain: str = "" + videomme_sub_category: str = "" + videomme_task_type: str = "" + omni_extra_body: dict[str, Any] | None = None + omni_chat_messages: list[dict[str, Any]] | None = None + omni_chat_mm_position: Literal["first", "last"] = "first" + + +class VideoMMEDataset(BenchmarkDataset): + """Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing).""" + + SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO} + DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO + IS_MULTIMODAL = True + DEFAULT_OUTPUT_LEN = 128 + + def __init__( + self, + dataset_path: str | None = None, + dataset_split: str = "test", + random_seed: int = 0, + video_dir: str | None = None, + subtitle_dir: str | None = None, + parquet_path: str | None = None, + pack_mode: VideoMMEPackMode = "minicpm-frames", + max_frames: int | None = None, + duration_filter: VideoMMEDurationFilter = "all", + use_subtitle: bool = False, + inline_local_video: bool = False, + trust_remote_code: bool = False, + dataset_subset: str | None = None, + no_stream: bool = False, + **kwargs, + ) -> None: + if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"): + raise ValueError( + f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}" + ) + if duration_filter not in ("all", "short", "medium", "long"): + raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}") + if parquet_path is None and dataset_path is None: + raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.") + + self.parquet_path = Path(parquet_path) if parquet_path else None + self.dataset_path = dataset_path + self.dataset_split = dataset_split + self.dataset_subset = dataset_subset + self._hf_streaming = not no_stream + self.video_dir = Path(video_dir) if video_dir else None + self.subtitle_dir = Path(subtitle_dir) if subtitle_dir else None + self.pack_mode: VideoMMEPackMode = pack_mode + self.duration_filter: VideoMMEDurationFilter = duration_filter + self.use_subtitle = use_subtitle + self.inline_local_video = inline_local_video + self.trust_remote_code = trust_remote_code + self.max_frames = int( + max_frames + if max_frames is not None + else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES) + ) + #: In-process memo of content parts; the on-disk frame cache survives across runs. + self._frame_cache: dict[str, list[dict[str, Any]]] = {} + self._video_index: dict[str, Path] | None = None + + super().__init__( + dataset_path=dataset_path if self.parquet_path is None else None, + random_seed=random_seed, + **kwargs, + ) + self.load_data() + logger.info( + "Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, " + "use_subtitle=%s, video_dir=%s", + str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}", + pack_mode, + self.max_frames, + duration_filter, + use_subtitle, + self.video_dir, + ) + + # ------------------------------------------------------------------ loading + + def load_data(self) -> None: + if self.parquet_path is not None: + self._load_from_parquet(self.parquet_path) + return + + local_root = resolve_videomme_local_root(self.dataset_path) + if local_root is not None: + local_pq = videomme_local_parquet(local_root) + if local_pq is not None: + if self.video_dir is None: + try: + self.video_dir = ensure_videomme_videos_extracted(local_root) + except FileNotFoundError: + self.video_dir = videomme_local_video_dir(local_root) + if self.subtitle_dir is None: + self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root) + self._load_from_parquet(local_pq) + return + + self._load_from_huggingface() + + def _load_from_parquet(self, path: Path) -> None: + try: + import pandas as pd + except ImportError as e: + raise ImportError("pandas is required to load Video-MME parquet") from e + if not path.is_file(): + raise FileNotFoundError(f"Video-MME parquet not found: {path}") + df = pd.read_parquet(path) + self._set_rows([row.to_dict() for _, row in df.iterrows()]) + + def _load_from_huggingface(self) -> None: + if load_dataset is None: + raise ImportError( + "datasets library is required for HuggingFace Video-MME loading. " + "Install with: pip install datasets, or pass --videomme-parquet." + ) + load_kw: dict[str, Any] = { + "split": self.dataset_split, + "streaming": self._hf_streaming, + "trust_remote_code": self.trust_remote_code, + } + if self.dataset_subset is not None: + load_kw["name"] = self.dataset_subset + ds = load_dataset(self.dataset_path, **load_kw) + self._set_rows([self._coerce_row(item) for item in ds]) + + def _set_rows(self, rows: list[dict[str, Any]]) -> None: + """Apply the duration filter and shuffle, then expose rows as ``self.data``.""" + if self.duration_filter != "all": + want = self.duration_filter.lower() + rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want] + if not getattr(self, "disable_shuffle", False) and self.random_seed is not None: + import random + + rows = rows[:] + random.Random(self.random_seed).shuffle(rows) + self.data = _ListDatasetIterator(rows) + + @staticmethod + def _coerce_row(item: Any) -> dict[str, Any]: + if isinstance(item, dict): + return item + if hasattr(item, "as_py"): + return dict(item.as_py()) + try: + return dict(item) + except (TypeError, ValueError): + return {k: item[k] for k in item} # type: ignore[misc] + + @staticmethod + def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]: + # The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain + # over the alias keys is not usable here. + options: Any = [] + for key in ("options", "candidates", "choices"): + value = row.get(key) + if hasattr(value, "tolist"): + value = value.tolist() + if value is not None and len(value) > 0: + options = value + break + return { + "video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(), + "question_id": str(row.get("question_id") or "").strip(), + "question": str(row.get("question") or "").strip(), + "options": [str(x) for x in list(options)], + "answer": str(row.get("answer") or "").strip(), + "duration": str(row.get("duration") or "").strip(), + "domain": str(row.get("domain") or "").strip(), + "sub_category": str(row.get("sub_category") or "").strip(), + "task_type": str(row.get("task_type") or "").strip(), + } + + # ------------------------------------------------------------------ sampling + + def sample( + self, + tokenizer: TokenizerLike, + num_requests: int, + output_len: int | None = None, + request_id_prefix: str = "", + no_oversample: bool = False, + **kwargs, + ) -> list[SampleRequest]: + if output_len is None: + output_len = self.DEFAULT_OUTPUT_LEN + + sampled: list[SampleRequest] = [] + cached_tokenizer = get_cached_tokenizer(tokenizer) + # A cold frame cache costs a few seconds per video, so the whole set takes tens of + # minutes before the first request goes out; report progress to keep that legible. + started = time.monotonic() + last_report = started + try: + total = min(num_requests, len(self.data)) + except TypeError: # HF streaming datasets are not sized + total = num_requests + for seen, item in enumerate(self.data, start=1): + if len(sampled) >= num_requests: + break + req = self._create_sample_request( + self._coerce_row(item), cached_tokenizer, output_len, request_id_prefix, len(sampled) + ) + if req: + sampled.append(req) + now = time.monotonic() + if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S: + logger.info( + "Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed", + seen, + total, + len(sampled), + now - started, + ) + last_report = now + + logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started) + self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample) + return sampled + + def _create_sample_request( + self, + qa_item: dict[str, Any], + tokenizer: TokenizerLike, + output_len: int, + request_id_prefix: str, + index: int, + ) -> SampleRequest | None: + fields = self._normalize_fields(qa_item) + if not fields["video_id"] or not fields["question"]: + logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"]) + return None + + mm_payload, omni_extra = self._compose_multimodal(fields["video_id"]) + if not mm_payload: + return None + + user_text = self._build_user_prompt(fields) + return VideoMMESampleRequest( + prompt=user_text, + prompt_len=len(tokenizer.encode(user_text)), + expected_output_len=output_len, + multi_modal_data=None, + request_id=f"{request_id_prefix}{index}", + videomme_gold_answer=fields["answer"], + videomme_video_id=fields["video_id"], + videomme_question_id=fields["question_id"], + videomme_duration=fields["duration"], + videomme_domain=fields["domain"], + videomme_sub_category=fields["sub_category"], + videomme_task_type=fields["task_type"], + omni_extra_body=omni_extra, + omni_chat_messages=self._build_openai_messages(mm_payload, user_text), + omni_chat_mm_position="first", + ) + + # ------------------------------------------------------------------ prompt + + @staticmethod + def _format_options(options: list[str]) -> str: + """Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line). + + The Hub parquet already ships options as ``"A. content"``. Existing labels are + dropped before relabelling so the prompt never reads ``A. A. content``, but only + when *every* option carries one — otherwise contents such as ``A.M. shift`` would + be truncated. + """ + if not options: + return "" + matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options] + relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS)) + lines = [] + for key, raw, m in zip(_OPTION_LETTERS, options, matches): + content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip() + lines.append(f"{key}. {content}") + return "\n".join(lines) + + def _build_user_prompt(self, fields: dict[str, Any]) -> str: + prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format( + question=fields["question"], + options=self._format_options(fields["options"]), + ) + if self.use_subtitle: + subs = self._load_subtitle_text(fields["video_id"]) + if subs: + prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}" + return prompt + + def _load_subtitle_text(self, video_id: str) -> str: + if not self.subtitle_dir: + return "" + path = self.subtitle_dir / f"{video_id}.srt" + if not path.is_file(): + return "" + try: + raw = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + lines = [ + s.replace("\\N", " ") + for s in (line.strip() for line in raw.splitlines()) + if s and not s.isdigit() and "-->" not in s + ] + return "\n".join(lines) + + def _build_openai_messages( + self, + mm_payload: dict[str, Any] | list[dict[str, Any]], + user_text: str, + ) -> list[dict[str, Any]]: + mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload] + messages: list[dict[str, Any]] = [] + if MINICPM_OMNI_SYSTEM_TEXT: + messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]}) + messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]}) + return messages + + # ------------------------------------------------------------------ media + + def _video_path_index(self) -> dict[str, Path]: + """Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses.""" + if self._video_index is None: + index: dict[str, Path] = {} + if self.video_dir is not None: + for path in _iter_video_files(self.video_dir): + index.setdefault(path.stem, path) + logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir) + self._video_index = index + return self._video_index + + def _resolve_local_video_path(self, video_id: str) -> Path | None: + if not self.video_dir or not video_id: + return None + for p in ( + self.video_dir / f"{video_id}.mp4", + self.video_dir / f"{video_id}.MP4", + self.video_dir / f"{video_id}.mkv", + self.video_dir / video_id / f"{video_id}.mp4", + ): + if p.is_file(): + return p + return self._video_path_index().get(video_id) + + def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]: + path = path.expanduser().resolve() + if self.inline_local_video: + b64 = base64.b64encode(path.read_bytes()).decode("ascii") + return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} + return {"type": typ, typ: {"url": path.as_uri()}} + + def _compose_multimodal( + self, + video_id: str, + ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]: + # OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the + # sample carries a video path: 9 slices per frame would blow past max_model_len and + # N would label each frame as a separate picture. + extra: dict[str, Any] = { + "mm_processor_kwargs": { + "use_audio_in_video": False, + "max_slice_nums": 1, + "use_image_id": False, + } + } + + if self.pack_mode == "video_url": + video_path = self._resolve_local_video_path(video_id) + if video_path is None: + logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir) + return None, None + return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra + + parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave") + if not parts: + return None, None + return parts, extra + + def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path: + assert self.video_dir is not None + tag = f"f{self.max_frames}{'_av' if include_audio else ''}" + return self.video_dir / ".minicpm_videomme_frames" / video_id / tag + + def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None: + """Rebuild content parts from a previous run's frame dump, or ``None`` on a miss.""" + manifest = cache_dir / "manifest.json" + if not manifest.is_file(): + return None + try: + count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"]) + except (OSError, ValueError, KeyError, TypeError): + return None + + parts: list[dict[str, Any]] = [] + for i in range(count): + frame_path = cache_dir / f"frame_{i:04d}.jpg" + if not frame_path.is_file(): + return None + parts.append({"type": "image_url", "image_url": {"url": frame_path.as_uri()}}) + if include_audio: + audio_path = cache_dir / f"audio_{i:04d}.wav" + if not audio_path.is_file(): + return None + parts.append({"type": "audio_url", "audio_url": {"url": audio_path.as_uri()}}) + return parts or None + + def _get_minicpm_frame_parts( + self, + video_id: str, + *, + include_audio: bool, + ) -> list[dict[str, Any]] | None: + cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}" + cached = self._frame_cache.get(cache_key) + if cached is not None: + return cached + + # Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS. + cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio) + if cache_dir is not None: + disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio) + if disk_parts is not None: + self._frame_cache[cache_key] = disk_parts + return disk_parts + + video_path = self._resolve_local_video_path(video_id) + if video_path is None: + logger.warning( + "Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r", + self.video_dir, + video_id, + ) + return None + + try: + frames, audio_segments = self._extract_frames_and_audio( + video_path, + include_audio=include_audio, + max_num_frames=self.max_frames, + ) + except Exception: + logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path) + return None + + if not frames: + logger.warning("No frames extracted for video_id=%r", video_id) + return None + if include_audio and len(audio_segments) != len(frames): + n = min(len(frames), len(audio_segments)) + frames, audio_segments = frames[:n], audio_segments[:n] + + if cache_dir is not None: + cache_dir.mkdir(parents=True, exist_ok=True) + + parts: list[dict[str, Any]] = [] + for i, frame in enumerate(frames): + parts.append( + self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg") + ) + if include_audio: + parts.append( + self._emit_media( + cache_dir, + f"audio_{i:04d}.wav", + _numpy_to_wav_bytes(audio_segments[i]), + "audio_url", + "audio/wav", + ) + ) + + if cache_dir is not None: + (cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8") + self._frame_cache[cache_key] = parts + logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts)) + return parts + + @staticmethod + def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]: + """Write ``payload`` to the frame cache and return a file URL part, or inline base64.""" + if cache_dir is None: + b64 = base64.b64encode(payload).decode("ascii") + return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} + path = cache_dir / name + if not path.is_file() or path.stat().st_size == 0: + path.write_bytes(payload) + return {"type": typ, typ: {"url": path.as_uri()}} + + @staticmethod + def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]: + """Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps.""" + if duration > max_num_frames: + grid = [round(i * 0.1, 1) for i in range(int(duration / 0.1))] + return [grid[i] for i in _uniform_sample_indices(len(grid), max_num_frames)] + # OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame. + return [float(i) for i in range(max(1, int(duration)))] + + @classmethod + def _extract_frames_and_audio( + cls, + video_path: Path, + *, + include_audio: bool, + max_num_frames: int, + ) -> tuple[list[Any], list[Any]]: + import numpy as np + from vllm.multimodal.media.audio import load_audio + + duration = _probe_video_duration(video_path) + timestamps = cls._sample_timestamps(duration, max_num_frames) + frames = _decode_frames_at_timestamps(video_path, timestamps) + + audio_segments: list[Any] = [] + if include_audio: + audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True) + for i, start_time in enumerate(timestamps): + end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration + segment = audio_np[int(start_time * sr) : int(end_time * sr)] + if i == len(timestamps) - 1 and len(segment) < 1600: + segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)]) + if len(segment) == 0: + segment = np.zeros(1600, dtype=np.float32) + audio_segments.append(segment) + + return frames, audio_segments diff --git a/vllm_omni/benchmarks/data_modules/videomme_eval.py b/vllm_omni/benchmarks/data_modules/videomme_eval.py new file mode 100644 index 000000000..a3e98c07f --- /dev/null +++ b/vllm_omni/benchmarks/data_modules/videomme_eval.py @@ -0,0 +1,214 @@ +"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve. + +Answer extraction ports ``extract_characters_regex`` from the official Video-MME +script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval). + +**Denominator:** responses where no A–D can be extracted are counted as wrong, which +matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The +official standalone script instead drops them from ``answered``; ``videomme_parse_failed`` +is reported so that view can be recovered. +""" + +from __future__ import annotations + +import re +from typing import Any + +from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput + +from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest + +_VALID = frozenset("ABCD") + +VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long") + +#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns). +_DIMENSIONS: dict[str, str] = { + "duration": "videomme_duration", + "domain": "videomme_domain", + "sub_category": "videomme_sub_category", + "task": "videomme_task_type", +} + +# Official prefix list, with the two accidental string concatenations in the upstream +# tuple (missing commas after "The best option is" / "Best answer:") split apart. +_ANSWER_PREFIXES = ( + "The best answer is", + "The correct answer is", + "The answer is", + "The answer", + "The best option is", + "The correct option is", + "Best answer:", + "Best option:", + "Answer:", + "Option:", + "The correct answer", + "The correct option", +) + + +def extract_characters_regex(text: str | None) -> str | None: + """Port of official Video-MME / lmms-eval ``extract_characters_regex``.""" + if not text: + return None + s = str(text).strip() + for prefix in _ANSWER_PREFIXES: + s = s.replace(prefix, "") + if len(s.split()) > 10 and not re.search(r"[ABCD]", s): + return None + match = re.search(r"[ABCD]", s) + return match.group(0) if match else None + + +def normalize_gold_answer(gold: str) -> str | None: + g = (gold or "").strip().upper() + if len(g) == 1 and g in _VALID: + return g + m = re.search(r"([ABCD])\b", g) + return m.group(1).upper() if m else None + + +def _bucket(req: VideoMMESampleRequest, dimension: str) -> str: + value = (getattr(req, _DIMENSIONS[dimension]) or "").strip() + if dimension == "duration": + return value.lower() + return value or "unknown" + + +def compute_videomme_accuracy_metrics( + input_requests: list[Any], + outputs: list[RequestFuncOutput], + *, + include_per_item: bool = False, +) -> dict[str, Any] | None: + """If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats. + + Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are + excluded from ``videomme_accuracy`` and counted in + ``videomme_accuracy_incl_http_fail`` instead. + """ + if not input_requests or len(input_requests) != len(outputs): + return None + if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests): + return None + + stats: dict[str, dict[str, dict[str, int]]] = { + dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {}) + for dim in _DIMENSIONS + } + items: list[dict[str, Any]] = [] + correct = evaluated = no_gold = request_failed = parse_failed = 0 + + for req, out in zip(input_requests, outputs, strict=True): + assert isinstance(req, VideoMMESampleRequest) + gold_raw = (req.videomme_gold_answer or "").strip() + buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS} + row: dict[str, Any] = { + "request_id": req.request_id, + "video_id": req.videomme_video_id, + "question_id": req.videomme_question_id, + **buckets, + } + + if not gold_raw: + no_gold += 1 + items.append({**row, "skipped": True, "reason": "no_gold"}) + continue + + evaluated += 1 + gold_norm = normalize_gold_answer(gold_raw) + pred = extract_characters_regex(out.generated_text) if out.success else None + is_correct = bool(pred) and pred.upper() == (gold_norm or gold_raw).upper() + + if not out.success: + request_failed += 1 + else: + if pred is None: + parse_failed += 1 + # Only successful responses enter the per-dimension denominators, mirroring + # the overall videomme_accuracy definition. + for dim, key in buckets.items(): + bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0}) + bucket["total"] += 1 + bucket["correct"] += int(is_correct) + + if is_correct: + correct += 1 + + items.append( + { + **row, + "gold": gold_raw, + "gold_normalized": gold_norm, + "predicted": pred, + "correct": is_correct, + "parse_failed": out.success and pred is None, + **({"error": (out.error or "")[:500]} if not out.success else {}), + } + ) + + evaluated_ok = evaluated - request_failed + result: dict[str, Any] = { + "videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None, + "videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None, + "videomme_correct": correct, + "videomme_evaluated": evaluated, + "videomme_evaluated_ok": evaluated_ok, + "videomme_no_gold": no_gold, + "videomme_request_failed": request_failed, + "videomme_parse_failed": parse_failed, + } + for dim, per_bucket in stats.items(): + result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()} + result[f"videomme_per_{dim}_accuracy"] = { + k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items() + } + if include_per_item: + result["videomme_eval_items"] = items + return result + + +def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None: + """Pretty-print the Video-MME accuracy block (stdout).""" + acc = metrics.get("videomme_accuracy") + if acc is None and not metrics.get("videomme_evaluated", 0): + return + + print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="=")) + ok = int(metrics.get("videomme_evaluated_ok", 0) or 0) + if ok and acc is not None: + print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}") + else: + print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)") + for label, key in ( + ("Submitted (gold present):", "videomme_evaluated"), + ("Successful HTTP (denominator):", "videomme_evaluated_ok"), + ("Correct:", "videomme_correct"), + ("Skipped (no gold):", "videomme_no_gold"), + ("HTTP failed:", "videomme_request_failed"), + ("Parsed OK but no A-D found:", "videomme_parse_failed"), + ): + print(f"{label:<40} {metrics.get(key, 0):<10}") + + for dim, title in ( + ("duration", "Duration"), + ("domain", "Domain"), + ("sub_category", "Sub Category"), + ("task", "Task Type"), + ): + per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {} + if not per_acc: + continue + counts = metrics.get(f"videomme_per_{dim}") or {} + names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc) + print(f"\n--- Accuracy by {title} ---") + for name in names: + st = counts.get(name) or {} + total = int(st.get("total", 0) or 0) + value = per_acc.get(name) + if total and value is not None: + print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}") + else: + print(f"{name}: 0/0 = N/A") + print("=" * 50) diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index ccedfb75e..6b18f1b48 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -56,6 +56,18 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( ) from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset +from vllm_omni.benchmarks.data_modules.videomme_dataset import ( + VIDEOMME_DEFAULT_HF_REPO, + VideoMMEDataset, + VideoMMESampleRequest, + ensure_videomme_subtitles_extracted, + ensure_videomme_videos_extracted, + resolve_videomme_local_root, + resolve_videomme_root, + videomme_local_parquet, + videomme_local_subtitle_dir, + videomme_local_video_dir, +) from vllm_omni.experimental.fullduplex.client import ( RealtimeDuplexClient, summarize_session_request_metrics, @@ -151,6 +163,7 @@ def _pcm_s16le_to_seed_tts_wer_bytes( get_samples_old = datasets.get_samples _DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni" +_DEFAULT_VIDEOMME_REPO = VIDEOMME_DEFAULT_HF_REPO def _seed_tts_capture_pcm_for_wer() -> bool: @@ -189,6 +202,17 @@ def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: Request setattr(rfi, "mm_position", sample.omni_chat_mm_position) +def _attach_videomme_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None: + """Apply per-request OpenAI fields for Video-MME (same message path as Daily-Omni).""" + if not isinstance(sample, VideoMMESampleRequest): + return + rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body) + if sample.omni_chat_messages is not None: + setattr(rfi, "omni_chat_messages", sample.omni_chat_messages) + else: + setattr(rfi, "mm_position", sample.omni_chat_mm_position) + + def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None: """Merge Seed-TTS per-row TTS fields into ``extra_body`` and mark for PCM capture. @@ -238,11 +262,27 @@ def _daily_omni_repo_from_args(args) -> str | None: return None +def _videomme_repo_from_args(args) -> str | None: + """Resolve HuggingFace repo id for Video-MME from CLI args.""" + dp = getattr(args, "dataset_path", None) + hn = getattr(args, "hf_name", None) + supported = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS} + supported.add(_DEFAULT_VIDEOMME_REPO.lower()) + if isinstance(dp, str) and dp.strip().lower() in supported: + return dp.strip() + if isinstance(hn, str) and hn.strip().lower() in supported: + return hn.strip() + return None + + def get_samples(args, tokenizer): # Daily-Omni: explicit dataset name, or hf + matching path/hf-name is_daily_omni = args.dataset_name == "daily-omni" or ( args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None ) + is_videomme = args.dataset_name == "videomme" or ( + args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None + ) is_seed_tts = args.dataset_name in ( "seed-tts", "seed-tts-text", @@ -258,7 +298,7 @@ def get_samples(args, tokenizer): "openai-realtime-duplex", "daily-omni", ] - is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm" + is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or args.dataset_name == "random-mm" if not is_omni_backend and not is_omni_dataset: # Not an omni-related request, delegate to original implementation @@ -376,6 +416,113 @@ def get_samples(args, tokenizer): ) return input_requests + if is_videomme: + if args.backend not in ["openai-chat-omni", "daily-omni"]: + raise ValueError( + f"Video-MME dataset requires a multimodal backend that supports video. " + f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'" + ) + + video_dir = getattr(args, "videomme_video_dir", None) + subtitle_dir = getattr(args, "videomme_subtitle_dir", None) + parquet_path = getattr(args, "videomme_parquet", None) + if isinstance(parquet_path, str): + parquet_path = parquet_path.strip() or None + dataset_split = getattr(args, "hf_split", None) or "test" + + # Local directory wins (absolute/relative existing path). Hub ids fall through to + # snapshot_download — same pattern as Seed-TTS / Daily-Omni. + local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or ( + resolve_videomme_local_root(getattr(args, "hf_name", None)) + ) + repo_id = _videomme_repo_from_args(args) + if local_root is None and video_dir is None and parquet_path is None: + # Default Hub mode: download parquet + video zips on demand. + if args.dataset_name == "videomme": + hub_id = repo_id or _DEFAULT_VIDEOMME_REPO + elif repo_id is not None: + hub_id = repo_id + else: + raise ValueError( + "Video-MME requires --videomme-parquet, a local --dataset-path mirror, or " + f"--dataset-path / --hf-name {_DEFAULT_VIDEOMME_REPO}." + ) + local_root = resolve_videomme_root(hub_id) + repo_id = hub_id + logger.info("Using Hugging Face Video-MME snapshot: root=%s repo=%s", local_root, hub_id) + elif local_root is not None: + logger.info("Using local Video-MME mirror: root=%s", local_root) + + if local_root is not None: + if parquet_path is None: + local_pq = videomme_local_parquet(local_root) + if local_pq is not None: + parquet_path = str(local_pq) + if video_dir is None: + try: + video_dir = str(ensure_videomme_videos_extracted(local_root)) + except FileNotFoundError: + found = videomme_local_video_dir(local_root) + video_dir = str(found) if found is not None else None + if subtitle_dir is None: + found_sub = ensure_videomme_subtitles_extracted(local_root) or videomme_local_subtitle_dir(local_root) + subtitle_dir = str(found_sub) if found_sub is not None else None + + if parquet_path is None: + if args.dataset_name == "videomme": + repo_id = repo_id or (str(local_root) if local_root is not None else _DEFAULT_VIDEOMME_REPO) + elif repo_id is None and local_root is None: + raise ValueError( + "Video-MME requires --videomme-parquet, a local --dataset-path mirror, or " + f"--dataset-path / --hf-name {_DEFAULT_VIDEOMME_REPO}." + ) + + if video_dir is None: + raise ValueError( + "Video-MME requires --videomme-video-dir pointing at extracted videos " + "(directory of {videoID}.mp4), or a local/Hub dataset root containing video/ " + "or videos_chunked_*.zip." + ) + + logger.info( + "Loading Video-MME: parquet=%s, hf_repo=%s, video_dir=%s, pack_mode=%s", + parquet_path, + repo_id, + video_dir, + getattr(args, "videomme_pack_mode", "minicpm-frames"), + ) + dataset = VideoMMEDataset( + parquet_path=parquet_path, + dataset_path=None if parquet_path is not None else (repo_id or str(local_root)), + dataset_split=dataset_split, + dataset_subset=getattr(args, "hf_subset", None), + random_seed=args.seed, + video_dir=video_dir, + subtitle_dir=subtitle_dir, + pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"), + max_frames=getattr(args, "videomme_max_frames", None), + duration_filter=getattr(args, "videomme_duration", "all"), + use_subtitle=getattr(args, "videomme_use_subtitle", False), + inline_local_video=getattr(args, "videomme_inline_local_video", False), + trust_remote_code=getattr(args, "trust_remote_code", False), + no_stream=getattr(args, "no_stream", False), + disable_shuffle=getattr(args, "disable_shuffle", False), + ) + + out_len = getattr(args, "output_len", None) + if out_len is None: + out_len = getattr(args, "hf_output_len", None) + if out_len is None: + out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN + + return dataset.sample( + tokenizer=tokenizer, + num_requests=args.num_prompts, + output_len=out_len, + request_id_prefix=args.request_id_prefix, + no_oversample=args.no_oversample, + ) + if is_seed_tts: if args.backend not in ( "openai-audio-speech", @@ -1645,6 +1792,7 @@ async def benchmark( chat_messages=test_chat_messages, ) _attach_daily_omni_to_request_func_input(input_requests[0], test_input) + _attach_videomme_to_request_func_input(input_requests[0], test_input) _attach_seed_tts_to_request_func_input(input_requests[0], test_input) if ready_check_timeout_sec > 0: @@ -1710,6 +1858,7 @@ async def benchmark( chat_messages=test_chat_messages, ) _attach_daily_omni_to_request_func_input(input_requests[0], profile_input) + _attach_videomme_to_request_func_input(input_requests[0], profile_input) _attach_seed_tts_to_request_func_input(input_requests[0], profile_input) profile_output = await request_func(request_func_input=profile_input, session=session) if profile_output.success: @@ -1795,6 +1944,7 @@ async def benchmark( chat_messages=request.chat_messages, ) _attach_daily_omni_to_request_func_input(request, request_func_input) + _attach_videomme_to_request_func_input(request, request_func_input) _attach_seed_tts_to_request_func_input(request, request_func_input) tasks.append( asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar)) @@ -1892,6 +2042,21 @@ async def benchmark( result.update(_daily_acc) print_daily_omni_accuracy_summary(_daily_acc) + from vllm_omni.benchmarks.data_modules.videomme_eval import ( + compute_videomme_accuracy_metrics, + print_videomme_accuracy_summary, + ) + + _save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in ( + "1", + "true", + "yes", + ) + _vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm) + if _vm_acc is not None: + result.update(_vm_acc) + print_videomme_accuracy_summary(_vm_acc) + if _seed_tts_capture_pcm_for_wer(): from vllm_omni.benchmarks.data_modules.seed_tts_eval import ( compute_seed_tts_wer_metrics, diff --git a/vllm_omni/benchmarks/serve.py b/vllm_omni/benchmarks/serve.py index 8e3b19ea9..5ed6fddbe 100644 --- a/vllm_omni/benchmarks/serve.py +++ b/vllm_omni/benchmarks/serve.py @@ -23,6 +23,8 @@ def main(args: argparse.Namespace) -> dict[str, Any]: os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1" if getattr(args, "daily_omni_save_eval_items", False): os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1" + if getattr(args, "videomme_save_eval_items", False): + os.environ["VIDEOMME_SAVE_EVAL_ITEMS"] = "1" set_print_stage(getattr(args, "print_stage", False)) args.extra_body = maybe_enable_stage_metrics( getattr(args, "extra_body", None), diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index fbc2853df..fb3f1b829 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -35,8 +35,9 @@ stages: fps: 1 num_frames: 128 limit_mm_per_prompt: - # Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps). - image: 64 + # Daily-Omni: up to 64 interleaved image/audio pairs (1fps). + # Video-MME minicpm-frames: OmniEvalKit samples up to 96 frames as image_url. + image: 96 audio: 64 video: 1 default_sampling_params: diff --git a/vllm_omni/entrypoints/cli/benchmark/cli_args.py b/vllm_omni/entrypoints/cli/benchmark/cli_args.py index 80d0e685d..08fceba8e 100644 --- a/vllm_omni/entrypoints/cli/benchmark/cli_args.py +++ b/vllm_omni/entrypoints/cli/benchmark/cli_args.py @@ -117,6 +117,78 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None: ) +def add_videomme_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments specific to the Video-MME dataset.""" + group = parser.add_argument_group("Video-MME Dataset Options") + group.add_argument( + "--videomme-parquet", + type=str, + default=None, + help="Path to local Video-MME parquet " + "(e.g. videomme/test-00000-of-00001.parquet). When set, Hub QA loading is skipped.", + ) + group.add_argument( + "--videomme-video-dir", + type=str, + default=None, + help="Directory containing extracted Video-MME videos (videoID.mp4). " + "Typical layout after unzipping videos_chunked_*.zip: /video/. " + "When using file:// URLs, start the server with --allowed-local-media-path " + "covering this directory.", + ) + group.add_argument( + "--videomme-subtitle-dir", + type=str, + default=None, + help="Directory containing Video-MME .srt subtitles (videoID.srt). Used only with --videomme-use-subtitle.", + ) + group.add_argument( + "--videomme-pack-mode", + type=str, + choices=["minicpm-frames", "minicpm-interleave", "video_url"], + default="minicpm-frames", + help="Multimodal packing. " + "'minicpm-frames' (default): OmniEvalKit MiniCPM videomme recipe — sampled frames " + "as image_url only (max_frames=96). " + "'minicpm-interleave': OmniEvalKit videomme_short recipe — 1fps frame/audio pairs " + "(max_frames=64). " + "'video_url': single video_url part (models with native video input).", + ) + group.add_argument( + "--videomme-max-frames", + type=int, + default=None, + help="Override max sampled frames (OmniEvalKit defaults: 96 for minicpm-frames, 64 for minicpm-interleave).", + ) + group.add_argument( + "--videomme-duration", + type=str, + choices=["all", "short", "medium", "long"], + default="all", + help="Filter by Video-MME duration bucket (default: all).", + ) + group.add_argument( + "--videomme-use-subtitle", + action="store_true", + default=False, + help="Prepend subtitle text to the user prompt (Video-MME w/ subs setting).", + ) + group.add_argument( + "--videomme-inline-local-video", + action="store_true", + default=False, + help="Embed local frames/audio as base64 data URLs so the server does not need " + "--allowed-local-media-path. Increases request size; use for small --num-prompts.", + ) + group.add_argument( + "--videomme-save-eval-items", + action="store_true", + default=False, + help="Include per-request Video-MME accuracy rows in the saved JSON under " + "videomme_eval_items. Or set env VIDEOMME_SAVE_EVAL_ITEMS=1.", + ) + + def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None: """Add CLI arguments for Seed-TTS benchmarks.""" group = parser.add_argument_group("Seed-TTS Dataset Options") @@ -183,6 +255,7 @@ def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None: _OMNI_BENCH_DATASET_CHOICES = ( "daily-omni", + "videomme", "seed-tts", "seed-tts-text", "seed-tts-design", @@ -249,6 +322,7 @@ def update_omni_help(parser: argparse.ArgumentParser) -> None: def add_omni_args(parser: argparse.ArgumentParser) -> None: """Register all vLLM-Omni serving benchmark arguments.""" add_daily_omni_cli_args(parser) + add_videomme_cli_args(parser) add_seed_tts_cli_args(parser) add_multi_stage_cli_args(parser) add_diffusion_cli_args(parser)