[Benchmark][MiniCPM-o] Add Video-MME dataset support for Omni bench s… (#5868)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>

.buildkite/cuda/test-nightly.yml

@@ -122,7 +122,7 @@ steps:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "cuda and H100" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT

.buildkite/npu/test-npu-nightly.yml

@@ -98,6 +98,22 @@ steps:
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
+        key: nightly-omni-performance-minicpmo-4-5-npu-a2
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_1
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
         timeout_in_minutes: 180

tests/dfx/perf/tests/test_minicpmo_4_5.json

@@ -81,5 +81,60 @@
         }
       }
     ]
+  },
+  {
+    "test_name": "test_minicpmo_4_5_a2",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "npu": "A2"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "omni"
+    ],
+    "server_params": {
+      "model": "openbmb/MiniCPM-o-4_5",
+      "extra_cli_args": [
+        "--deploy-config",
+        "vllm_omni/deploy/minicpmo_4_5.yaml",
+        "--trust-remote-code"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "dataset_name": "seed-tts",
+        "dataset_path": "zhaochenyang20/seed-tts-eval",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          32,
+          64,
+          128
+        ],
+        "max_concurrency": [
+          1,
+          4,
+          8
+        ],
+        "no_oversample": true,
+        "trust_remote_code": true,
+        "seed_tts_locale": "en",
+        "extra_body": {
+          "modalities": [
+            "text",
+            "audio"
+          ],
+          "chat_template_kwargs": {
+            "enable_thinking": false,
+            "use_tts_template": true
+          }
+        },
+        "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
+      }
+    ]
   }
 ]

tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py

@@ -1,10 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
-"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage.
+"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage.
 
 Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
 ~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
 text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
 + 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
+
+Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs):
+``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``.
+Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin).
 """
 
 from __future__ import annotations
@@ -35,12 +39,18 @@ _RESULT_DIR = Path(
 )
 
 _MIN_DAILY_OMNI_ACCURACY = 0.78
+# MiniCPM-o 4.5 reports 70.4 on Video-MME (w/o subs); ~2pp margin like Daily-Omni.
+_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.68"))
 _MAX_SEED_TTS_MEAN_WER = 0.05
-# Match the validated Daily-Omni client body from daily_omni_bench.sh.
+# Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts.
 _DAILY_EXTRA_BODY = {
     "modalities": ["text"],
     "chat_template_kwargs": {"enable_thinking": False},
 }
+_VIDEOMME_EXTRA_BODY = {
+    "modalities": ["text"],
+    "chat_template_kwargs": {"enable_thinking": False},
+}
 _SEED_EXTRA_BODY = {
     "modalities": ["text", "audio"],
     "chat_template_kwargs": {
@@ -58,6 +68,15 @@ _DAILY_OMNI_SERVER_ARGS = [
     "--media-io-kwargs",
     '{"video":{"fps":1,"num_frames":128}}',
 ]
+# Video-MME ``minicpm-frames`` sends sampled frames as image_url (no AV interleave).
+# Allow local ``file://`` frame-cache URLs so CI does not need megabyte-scale base64
+# payloads; the autouse fixture still appends ``--videomme-inline-local-video`` when
+# the allowlist is unavailable.
+_VIDEOMME_SERVER_ARGS = [
+    "--trust-remote-code",
+    "--allowed-local-media-path",
+    os.environ.get("VLLM_TEST_ALLOWED_LOCAL_MEDIA_PATH", "/"),
+]
 _SEED_TTS_SERVER_ARGS = [
     "--trust-remote-code",
 ]
@@ -72,6 +91,14 @@ daily_test_params = [
     )
 ]
 
+videomme_test_params = [
+    OmniServerParams(
+        model=_MODEL,
+        stage_config_path=_DEPLOY_CONFIG,
+        server_args=list(_VIDEOMME_SERVER_ARGS),
+    )
+]
+
 seed_test_params = [
     OmniServerParams(
         model=_MODEL,
@@ -88,18 +115,53 @@ def _require_vllm_cli() -> None:
         pytest.skip(str(exc))
 
 
-@pytest.fixture(autouse=True)
-def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Inline cached Hub/local videos because the test server has no local-media allowlist."""
-    original = _acc_bench.daily_omni_bench_argv
+def _optional_local_videomme_dataset_path() -> str | None:
+    """Return an explicit local Video-MME root, or ``None`` to use the Hub default.
 
-    def _wrapped() -> list[str]:
-        argv = list(original())
+    Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as
+    local. Hard-coded workspace paths are intentionally not probed so CI defaults to
+    ``lmms-lab/Video-MME`` via ``--videomme-repo``.
+    """
+    for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"):
+        raw = os.environ.get(key, "").strip()
+        if not raw:
+            continue
+        path = Path(raw).expanduser()
+        if path.is_absolute() and path.is_dir():
+            return str(path.resolve())
+    return None
+
+
+@pytest.fixture(autouse=True)
+def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Inline Hub/local media when the server has no usable local-media allowlist.
+
+    Daily-Omni always inlines (matches the historical Qwen/MiniCPM accuracy fixture).
+    Video-MME prefers ``file://`` via ``--allowed-local-media-path`` (see
+    ``_VIDEOMME_SERVER_ARGS``); force inline only when ``VLLM_VIDEOMME_FORCE_INLINE=1``.
+    """
+    original_daily = _acc_bench.daily_omni_bench_argv
+    original_videomme = _acc_bench.videomme_bench_argv
+
+    def _wrap_daily() -> list[str]:
+        argv = list(original_daily())
         if "--daily-omni-inline-local-video" not in argv:
             argv.append("--daily-omni-inline-local-video")
         return argv
 
-    monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
+    def _wrap_videomme() -> list[str]:
+        argv = list(original_videomme())
+        force_inline = os.environ.get("VLLM_VIDEOMME_FORCE_INLINE", "").strip().lower() in {
+            "1",
+            "true",
+            "yes",
+        }
+        if force_inline and "--videomme-inline-local-video" not in argv:
+            argv.append("--videomme-inline-local-video")
+        return argv
+
+    monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrap_daily)
+    monkeypatch.setattr(_acc_bench, "videomme_bench_argv", _wrap_videomme)
     monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
 
 
@@ -114,6 +176,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
         omni_server,
         skip_seed=True,
         skip_daily=False,
+        skip_videomme=True,
     )
     argv.extend(
         [
@@ -138,6 +201,55 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
     assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
 
 
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True)
+def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None:
+    """Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe."""
+    _require_vllm_cli()
+    pytest.importorskip("datasets")
+    pytest.importorskip("huggingface_hub")
+    pytest.importorskip("av")
+
+    dataset_path = _optional_local_videomme_dataset_path()
+    num_prompts = int(os.environ.get("ACC_BENCH_VIDEOMME_NUM_PROMPTS", "2700"))
+    max_concurrency = int(os.environ.get("ACC_BENCH_VIDEOMME_MAX_CONCURRENCY", "4"))
+
+    argv = build_acc_benchmark_cli_argv(
+        omni_server,
+        skip_seed=True,
+        skip_daily=True,
+        skip_videomme=False,
+        num_prompts=num_prompts,
+        max_concurrency=max_concurrency,
+    )
+    argv.extend(
+        [
+            "--result-dir",
+            str(_RESULT_DIR),
+            "--min-videomme-accuracy",
+            str(_MIN_VIDEOMME_ACCURACY),
+            "--temperature",
+            "0",
+            "--output-len",
+            "128",
+            "--videomme-pack-mode",
+            "minicpm-frames",
+            "--videomme-max-frames",
+            "96",
+            "--videomme-duration",
+            os.environ.get("ACC_BENCH_VIDEOMME_DURATION", "all"),
+            "--videomme-extra-body-json",
+            json.dumps(_VIDEOMME_EXTRA_BODY, separators=(",", ":")),
+            "--trust-remote-code",
+        ]
+    )
+    # Absolute local root only; otherwise --videomme-repo (Hub) from build_acc_benchmark_cli_argv.
+    if dataset_path is not None:
+        argv.extend(["--videomme-dataset-path", dataset_path])
+
+    assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
+
+
 @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
 def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
@@ -148,6 +260,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
         omni_server,
         skip_seed=False,
         skip_daily=True,
+        skip_videomme=True,
         max_concurrency=4,
     )
     argv.extend(
@@ -176,6 +289,7 @@ def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
         omni_server,
         skip_seed=False,
         skip_daily=True,
+        skip_videomme=True,
         num_prompts=50,
         max_concurrency=1,
     )

tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
+"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
 
 Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
 point to existing files, those are used with inline video. Otherwise the benchmark falls back to
@@ -9,6 +9,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
 Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
 and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
 
+Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
+``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
+``lmms-lab/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
+
 Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
 and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
 """
@@ -25,6 +29,7 @@ from typing import Any, Protocol
 
 DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
 DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
+DEFAULT_VIDEOMME_HF_REPO = "lmms-lab/Video-MME"
 
 
 class OmniBenchServerEndpoint(Protocol):
@@ -40,6 +45,7 @@ def build_acc_benchmark_cli_argv(
     *,
     skip_seed: bool,
     skip_daily: bool,
+    skip_videomme: bool = True,
     num_prompts: int | None = None,
     max_concurrency: int | None = None,
 ) -> list[str]:
@@ -49,6 +55,9 @@ def build_acc_benchmark_cli_argv(
     ``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
     ``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
     matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
+
+    Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
+    callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
     """
     n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
     n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
@@ -71,6 +80,12 @@ def build_acc_benchmark_cli_argv(
         argv.append("--skip-seed-tts")
     if skip_daily:
         argv.append("--skip-daily-omni")
+    if skip_videomme:
+        argv.append("--skip-videomme")
+    else:
+        argv.append("--run-videomme")
+        repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
+        argv.extend(["--videomme-repo", repo])
     return argv
 
 
@@ -100,6 +115,43 @@ def daily_omni_bench_argv() -> list[str]:
     ]
 
 
+def videomme_bench_argv() -> list[str]:
+    """CLI args for Video-MME (after ``vllm bench serve --omni``).
+
+    Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
+
+    * If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
+      use that local mirror as ``--dataset-path``.
+    * Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-lab/Video-MME``); the child
+      bench downloads via ``huggingface_hub.snapshot_download``.
+    """
+    root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip()
+    if root:
+        p = Path(root).expanduser()
+        # Preserve Hub ids verbatim. Only canonicalize when the value is a real directory.
+        if p.exists() and p.is_dir():
+            return [
+                "--dataset-name",
+                "videomme",
+                "--dataset-path",
+                str(p.resolve()),
+            ]
+        # Non-directory values (e.g. accidental Hub id in DATASET_PATH) fall through as path.
+        return [
+            "--dataset-name",
+            "videomme",
+            "--dataset-path",
+            root,
+        ]
+    repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
+    return [
+        "--dataset-name",
+        "videomme",
+        "--dataset-path",
+        repo,
+    ]
+
+
 def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
     """CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
     dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
@@ -209,6 +261,12 @@ def assert_daily_omni_scored(result: dict[str, Any]) -> None:
     assert int(result.get("daily_omni_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (daily_omni_evaluated_ok==0)"
 
 
+def assert_videomme_scored(result: dict[str, Any]) -> None:
+    acc = result.get("videomme_accuracy")
+    assert acc is not None, "videomme_accuracy missing — wrong dataset or benchmark wiring"
+    assert int(result.get("videomme_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (videomme_evaluated_ok==0)"
+
+
 def assert_seed_tts_scored(result: dict[str, Any]) -> None:
     err = result.get("seed_tts_eval_setup_error")
     assert not err, f"Seed-TTS eval deps/setup failed: {err}"

tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py

@@ -11,6 +11,9 @@ H100 / MI325) because they launch the live Omni server inside the test.
    run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
 2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
    (WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
+3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
+   stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
+   **0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
 
 Prerequisites
 -------------
@@ -21,7 +24,8 @@ Prerequisites
   On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
   only drives the **client** benchmark.
 
-* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
+* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
+  ``videomme``).
 
 * **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
   the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
@@ -31,6 +35,13 @@ Prerequisites
   the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo
   with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed.
 
+* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-lab/Video-MME`` (override with
+  ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
+  ``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an
+  existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` /
+  ``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without ``--allowed-local-media-path``,
+  pass ``--videomme-inline-local-video`` (or rely on the pytest fixture that appends it).
+
 * **Seed-TTS** optional extras for WER/SIM/UTMOS::
 
     pip install 'vllm-omni[dev]'
@@ -55,6 +66,18 @@ Skip one suite, tighten gates::
         --skip-daily-omni \\
         --max-seed-tts-mean-wer 0.35 \\
         --min-seed-tts-mean-sim 0.75
+
+MiniCPM-o Video-MME (w/o subs)::
+
+    python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
+        --skip-daily-omni --skip-seed-tts --run-videomme \\
+        --videomme-pack-mode minicpm-frames \\
+        --videomme-max-frames 96 \\
+        --temperature 0 --output-len 128 \\
+        --min-videomme-accuracy 0.68
+
+    # Optional local mirror (absolute path) instead of Hub download:
+    #   --videomme-dataset-path /path/to/Video-MME
 """
 
 from __future__ import annotations
@@ -75,6 +98,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
     load_benchmark_result,
     run_vllm_bench_subprocess,
     seed_tts_bench_argv,
+    videomme_bench_argv,
 )
 
 _REPO_ROOT = Path(__file__).resolve().parents[4]
@@ -102,6 +126,20 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
     return errs
 
 
+def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
+    errs: list[str] = []
+    acc = result.get("videomme_accuracy")
+    if acc is None:
+        errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
+        return errs
+    ev = int(result.get("videomme_evaluated_ok", 0) or 0)
+    if ev <= 0:
+        errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
+    if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
+        errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
+    return errs
+
+
 def _validate_seed_tts(
     result: dict[str, Any],
     *,
@@ -130,7 +168,7 @@ def _validate_seed_tts(
 
 
 def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
-    """Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
+    """Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
     repo = getattr(ns, "daily_omni_repo", None)
     if repo is not None and str(repo).strip():
         os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
@@ -138,6 +176,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
         os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
     if ns.daily_omni_video_dir is not None:
         os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
+    vm_repo = getattr(ns, "videomme_repo", None)
+    if vm_repo is not None and str(vm_repo).strip():
+        os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
+    vm_path = getattr(ns, "videomme_dataset_path", None)
+    if vm_path is not None and str(vm_path).strip():
+        raw = str(vm_path).strip()
+        p = Path(raw).expanduser()
+        os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
     if ns.seed_tts_dataset_path is not None:
         # ``--seed-tts-dataset-path`` accepts either a local directory or a
         # Hugging Face repo id. Only resolve to an absolute filesystem path
@@ -157,6 +203,9 @@ def _preserve_benchmark_dataset_env() -> Any:
         "VLLM_DAILY_OMNI_REPO",
         "VLLM_DAILY_OMNI_QA_JSON",
         "VLLM_DAILY_OMNI_VIDEO_DIR",
+        "VLLM_VIDEOMME_REPO",
+        "VLLM_VIDEOMME_DATASET_PATH",
+        "VIDEOMME_ROOT",
         "VLLM_SEED_TTS_DATASET_PATH",
         "SEED_TTS_ROOT",
     )
@@ -225,6 +274,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
     return out
 
 
+def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
+    ns.result_dir.mkdir(parents=True, exist_ok=True)
+    tag = datetime.now().strftime("%Y%m%d-%H%M%S")
+    result_filename = f"omni_acc_videomme_{tag}.json"
+    extra = json.loads(ns.videomme_extra_body_json)
+    argv = (
+        _build_common_args(ns, result_filename=result_filename)
+        + videomme_bench_argv()
+        + [
+            "--disable-shuffle",
+            "--videomme-pack-mode",
+            ns.videomme_pack_mode,
+            "--videomme-duration",
+            ns.videomme_duration,
+            "--extra-body",
+            json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
+        ]
+    )
+    if ns.videomme_max_frames is not None:
+        argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
+    if ns.videomme_use_subtitle:
+        argv.append("--videomme-use-subtitle")
+    if ns.videomme_inline_local_video:
+        argv.append("--videomme-inline-local-video")
+    if ns.videomme_save_eval_items:
+        argv.append("--videomme-save-eval-items")
+    if ns.videomme_video_dir is not None:
+        argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
+    if ns.videomme_parquet is not None:
+        argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
+    if ns.videomme_subtitle_dir is not None:
+        argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
+    print("\n$", vllm, *argv, "\n", flush=True)
+    run_vllm_bench_subprocess(vllm, argv)
+    out = Path(ns.result_dir) / result_filename
+    if not out.is_file():
+        raise FileNotFoundError(f"Expected result JSON at {out}")
+    return out
+
+
 def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
     ns.result_dir.mkdir(parents=True, exist_ok=True)
     tag = datetime.now().strftime("%Y%m%d-%H%M%S")
@@ -312,6 +401,20 @@ def build_arg_parser() -> argparse.ArgumentParser:
 
     p.add_argument("--skip-daily-omni", action="store_true")
     p.add_argument("--skip-seed-tts", action="store_true")
+    # Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
+    p.set_defaults(skip_videomme=True)
+    p.add_argument(
+        "--skip-videomme",
+        action="store_true",
+        dest="skip_videomme",
+        help="Skip Video-MME (default).",
+    )
+    p.add_argument(
+        "--run-videomme",
+        action="store_false",
+        dest="skip_videomme",
+        help="Enable the Video-MME accuracy suite.",
+    )
 
     p.add_argument(
         "--daily-omni-repo",
@@ -358,6 +461,87 @@ def build_arg_parser() -> argparse.ArgumentParser:
         "Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
     )
 
+    p.add_argument(
+        "--videomme-repo",
+        type=str,
+        default=None,
+        help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
+        "Default when no local path is set: lmms-lab/Video-MME. "
+        "Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
+    )
+    p.add_argument(
+        "--videomme-dataset-path",
+        type=str,
+        default=None,
+        help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
+        "download; Hub ids are also accepted and passed through as --dataset-path. "
+        "Sets VLLM_VIDEOMME_DATASET_PATH.",
+    )
+    p.add_argument(
+        "--videomme-parquet",
+        type=Path,
+        default=None,
+        help="Optional explicit path to videomme/test-*.parquet.",
+    )
+    p.add_argument(
+        "--videomme-video-dir",
+        type=Path,
+        default=None,
+        help="Optional explicit directory of extracted Video-MME videos.",
+    )
+    p.add_argument(
+        "--videomme-subtitle-dir",
+        type=Path,
+        default=None,
+        help="Optional subtitle directory (used with --videomme-use-subtitle).",
+    )
+    p.add_argument(
+        "--videomme-pack-mode",
+        choices=("minicpm-frames", "minicpm-interleave", "video_url"),
+        default="minicpm-frames",
+        help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
+    )
+    p.add_argument(
+        "--videomme-max-frames",
+        type=int,
+        default=None,
+        help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
+    )
+    p.add_argument(
+        "--videomme-duration",
+        choices=("all", "short", "medium", "long"),
+        default="all",
+        help="Filter Video-MME by duration bucket (default: all).",
+    )
+    p.add_argument(
+        "--videomme-use-subtitle",
+        action="store_true",
+        help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
+    )
+    p.add_argument(
+        "--videomme-inline-local-video",
+        action="store_true",
+        help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
+    )
+    p.add_argument(
+        "--videomme-extra-body-json",
+        default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
+        help="JSON merged into each chat request for Video-MME.",
+    )
+    p.add_argument(
+        "--videomme-save-eval-items",
+        action="store_true",
+        help="Include per-request Video-MME accuracy rows in the saved JSON.",
+    )
+    p.add_argument(
+        "--min-videomme-accuracy",
+        type=float,
+        default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
+        help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
+        "Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
+        "override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
+    )
+
     p.add_argument(
         "--seed-tts-dataset-path",
         type=str,
@@ -427,7 +611,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
 
 
 def run_acc_benchmark(ns: argparse.Namespace) -> int:
-    """Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
+    """Run configured client benches against a running server; return 0 on success."""
     failed: list[str] = []
 
     with _preserve_benchmark_dataset_env():
@@ -451,6 +635,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
                     flush=True,
                 )
 
+        if not ns.skip_videomme:
+            path = run_videomme(ns, vllm)
+            print(f"\n[Video-MME] result JSON: {path}", flush=True)
+            data = load_benchmark_result(path)
+            errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
+            if errs:
+                failed.extend([f"[Video-MME] {e}" for e in errs])
+            else:
+                print(
+                    f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
+                    f"evaluated_ok={data.get('videomme_evaluated_ok')} "
+                    f"per_duration={data.get('videomme_per_duration_accuracy')}",
+                    flush=True,
+                )
+
         if not ns.skip_seed_tts:
             path = run_seed_tts(ns, vllm)
             print(f"\n[Seed-TTS] result JSON: {path}", flush=True)

vllm_omni/benchmarks/data_modules/videomme_dataset.py

@@ -0,0 +1,894 @@
+"""Video-MME dataset loader for vLLM-Omni bench serve.
+
+Video-MME (lmms-lab/Video-MME) is a full-spectrum video MLLM benchmark with
+900 videos / 2,700 MCQ pairs across short / medium / long durations.
+
+This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe
+(``o_e_Kit/configs/generation_configs.json``):
+
+- ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false``
+- ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave``
+
+Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame
+timestamps match OmniEvalKit ``_sample_video_frame_indices``.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import re
+import time
+import zipfile
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+try:
+    from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
+except ImportError:
+    from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset
+    from vllm.benchmarks.datasets import SampleRequest
+from vllm.tokenizers import TokenizerLike
+from vllm.tokenizers.hf import get_cached_tokenizer
+
+try:
+    from datasets import load_dataset
+except ImportError:
+    load_dataset = None
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
+    MINICPM_OMNI_SYSTEM_TEXT,
+    _ListDatasetIterator,
+    _numpy_to_wav_bytes,
+    _pil_to_jpeg_bytes,
+    _uniform_sample_indices,
+)
+
+logger = logging.getLogger(__name__)
+
+_MINICPM_AUDIO_SR = 16000
+
+# OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults.
+VIDEOMME_DEFAULT_MAX_FRAMES = 96
+VIDEOMME_SHORT_MAX_FRAMES = 64
+VIDEOMME_DEFAULT_HF_REPO = "lmms-lab/Video-MME"
+
+#: Decode forward to reach a target this close, seek to the preceding keyframe beyond it.
+_SEEK_THRESHOLD_S = 3.0
+
+#: How often ``sample`` reports progress while warming a cold frame cache.
+_SAMPLE_PROGRESS_INTERVAL_S = 30.0
+
+VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"]
+VideoMMEDurationFilter = Literal["all", "short", "medium", "long"]
+
+# OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped;
+# media parts are prepended as OpenAI content parts, matching ``build_content``).
+VIDEOMME_USER_PROMPT_TEMPLATE = (
+    "Carefully read the following question and select the letter corresponding to the "
+    "correct answer.Highlight the applicable choices without giving explanations.\n"
+    "{question}\n"
+    "Options:\n"
+    "{options}\n"
+    "Please select the correct answer from the options above. Only respond with the letter."
+)
+
+_OPTION_LETTERS = "ABCDEFGHIJKL"
+_OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$")
+
+_VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi")
+
+
+def _iter_video_files(root: Path) -> Iterator[Path]:
+    """Yield video files under ``root``, skipping dot-dirs such as the frame cache."""
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+        for name in filenames:
+            if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES:
+                yield Path(dirpath) / name
+
+
+def _resolve_hf_cache_snapshot(path: Path) -> Path:
+    """Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir."""
+    snapshots = path / "snapshots"
+    if not snapshots.is_dir():
+        return path
+    ref = path / "refs" / "main"
+    if ref.is_file():
+        try:
+            target = snapshots / ref.read_text(encoding="utf-8").strip()
+        except OSError:
+            target = snapshots
+        if target.is_dir():
+            return target
+    revisions = [p for p in snapshots.iterdir() if p.is_dir()]
+    if revisions:
+        return max(revisions, key=lambda p: p.stat().st_mtime)
+    return path
+
+
+def resolve_videomme_local_root(dataset_path: str | None) -> Path | None:
+    """Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror.
+
+    Hub ids such as ``lmms-lab/Video-MME`` return ``None`` (they are not directories).
+    """
+    raw = (dataset_path or "").strip()
+    if not raw:
+        return None
+    path = Path(raw).expanduser()
+    if not path.is_dir():
+        return None
+    return _resolve_hf_cache_snapshot(path.resolve())
+
+
+def ensure_videomme_hub_root(repo_id: str) -> Path:
+    """Download the Video-MME dataset snapshot from Hugging Face and return its root.
+
+    Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`:
+    Hub ids trigger ``snapshot_download``; callers then extract videos via
+    :func:`ensure_videomme_videos_extracted`.
+
+    Raises:
+        ImportError: if ``huggingface_hub`` is not installed.
+        ValueError: if ``repo_id`` is empty.
+    """
+    rid = (repo_id or "").strip()
+    if not rid:
+        raise ValueError("repo_id is required to download Video-MME from Hugging Face")
+
+    try:
+        from huggingface_hub import snapshot_download
+    except ImportError as e:
+        raise ImportError(
+            "Install huggingface_hub to download Video-MME from the Hub, or pass a local "
+            "--dataset-path / --videomme-video-dir with parquet + extracted videos."
+        ) from e
+
+    # Pull QA parquet + video/subtitle archives; skip unrelated repo files.
+    cache = snapshot_download(
+        repo_id=rid,
+        repo_type="dataset",
+        allow_patterns=[
+            "videomme/**",
+            "*.parquet",
+            "videos_chunked_*.zip",
+            "subtitle.zip",
+            "subtitle/**",
+            "video/**",
+            "videos/**",
+        ],
+    )
+    root = _resolve_hf_cache_snapshot(Path(cache).resolve())
+    logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid)
+    return root
+
+
+def resolve_videomme_root(dataset_path: str | None) -> Path:
+    """Return a Video-MME root containing parquet and (or zip) video assets.
+
+    * Existing local directories (absolute or relative) are used as-is.
+    * Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and
+      downloaded via :func:`ensure_videomme_hub_root` (default
+      ``lmms-lab/Video-MME``).
+    """
+    local = resolve_videomme_local_root(dataset_path)
+    if local is not None:
+        return local
+    rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO
+    return ensure_videomme_hub_root(rid)
+
+
+def videomme_local_parquet(root: Path) -> Path | None:
+    for candidate in (
+        root / "videomme" / "test-00000-of-00001.parquet",
+        root / "test-00000-of-00001.parquet",
+    ):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def videomme_local_video_dir(root: Path) -> Path | None:
+    """Return a directory already holding extracted videos, flat or nested.
+
+    Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``,
+    so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of
+    the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable.
+    """
+    for name in ("video", "videos", "data"):
+        candidate = root / name
+        if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None:
+            return candidate
+    return None
+
+
+def videomme_local_subtitle_dir(root: Path) -> Path | None:
+    candidate = root / "subtitle"
+    return candidate if candidate.is_dir() else None
+
+
+def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None:
+    """Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout)."""
+    name = os.path.basename(member)
+    if not name:
+        return
+    dest = dest_dir / name
+    if dest.is_file() and dest.stat().st_size > 0:
+        return
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    with zf.open(member) as src, open(dest, "wb") as out:
+        out.write(src.read())
+
+
+def ensure_videomme_videos_extracted(root: Path) -> Path:
+    """Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``."""
+    video_dir = root / "video"
+    marker = root / ".videomme_videos_extracted"
+    if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None:
+        return video_dir
+
+    # Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it.
+    existing = videomme_local_video_dir(root)
+    if existing is not None and existing != video_dir:
+        return existing
+
+    zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()]
+    if not zips and not video_dir.is_dir():
+        raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip")
+
+    video_dir.mkdir(parents=True, exist_ok=True)
+    for zp in zips:
+        logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir)
+        with zipfile.ZipFile(zp, "r") as zf:
+            for member in zf.namelist():
+                if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")):
+                    _unzip_member_flat(zf, member, video_dir)
+
+    marker.write_text("ok", encoding="utf-8")
+    return video_dir
+
+
+def ensure_videomme_subtitles_extracted(root: Path) -> Path | None:
+    """Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present."""
+    subtitle_dir = root / "subtitle"
+    if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None:
+        return subtitle_dir
+
+    zip_path = root / "subtitle.zip"
+    if not zip_path.is_file():
+        return subtitle_dir if subtitle_dir.is_dir() else None
+
+    subtitle_dir.mkdir(parents=True, exist_ok=True)
+    logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir)
+    with zipfile.ZipFile(zip_path, "r") as zf:
+        for member in zf.namelist():
+            if member.lower().endswith(".srt"):
+                _unzip_member_flat(zf, member, subtitle_dir)
+    return subtitle_dir
+
+
+def _probe_video_duration(video_path: Path) -> float:
+    """Container duration in seconds, without decoding the stream.
+
+    Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to
+    an hour, and ``stream.frames == 0`` containers would otherwise force a full decode
+    just to learn the duration.
+    """
+    import av
+
+    with av.open(str(video_path)) as container:
+        stream = container.streams.video[0]
+        duration: float | None = None
+        if stream.duration is not None and stream.time_base is not None:
+            duration = float(stream.duration * stream.time_base)
+        elif container.duration is not None:
+            duration = container.duration / av.time_base
+        elif stream.frames and stream.average_rate:
+            duration = stream.frames / float(stream.average_rate)
+
+    if not duration or duration <= 0:
+        raise ValueError(f"Could not determine duration of {video_path}")
+    return duration
+
+
+def _decode_frames_at_timestamps(
+    video_path: Path,
+    timestamps: list[float],
+    *,
+    seek_threshold_s: float = _SEEK_THRESHOLD_S,
+) -> list[Any]:
+    """Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp.
+
+    A single forward pass over the stream — the Daily-Omni approach — is not affordable
+    here: Video-MME samples points spread across the whole video, so the last target sits
+    near the end and every frame of an hour-long file would be decoded. Targets within
+    ``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper
+    than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding
+    keyframe first.
+    """
+    import av
+
+    if not timestamps:
+        return []
+
+    images: list[Any] = []
+    with av.open(str(video_path)) as container:
+        stream = container.streams.video[0]
+        stream.thread_type = "AUTO"
+        time_base = float(stream.time_base) if stream.time_base else 0.0
+        start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0
+
+        decoder = None
+        frame = None
+        frame_time = -1.0
+        image = None
+
+        for ts in timestamps:
+            if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s:
+                if time_base > 0:
+                    container.seek(int((ts + start_s) / time_base), stream=stream, backward=True)
+                else:
+                    container.seek(0, stream=stream, backward=True)
+                decoder = container.decode(stream)
+                frame, image, frame_time = None, None, -1.0
+
+            while frame is None or frame_time < ts:
+                nxt = next(decoder, None)
+                if nxt is None:
+                    break
+                frame, image = nxt, None
+                if nxt.pts is not None and time_base > 0:
+                    frame_time = float(nxt.pts) * time_base - start_s
+
+            if frame is None:
+                break
+            if image is None:
+                image = frame.to_image()
+            images.append(image)
+
+    if not images:
+        raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}")
+    # Container duration can overshoot the real stream; repeat the last frame (decord clamping).
+    images.extend([images[-1]] * (len(timestamps) - len(images)))
+    return images
+
+
+@dataclass
+class VideoMMESampleRequest(SampleRequest):
+    """``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring."""
+
+    videomme_gold_answer: str = ""
+    videomme_video_id: str = ""
+    videomme_question_id: str = ""
+    videomme_duration: str = ""
+    videomme_domain: str = ""
+    videomme_sub_category: str = ""
+    videomme_task_type: str = ""
+    omni_extra_body: dict[str, Any] | None = None
+    omni_chat_messages: list[dict[str, Any]] | None = None
+    omni_chat_mm_position: Literal["first", "last"] = "first"
+
+
+class VideoMMEDataset(BenchmarkDataset):
+    """Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing)."""
+
+    SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO}
+    DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO
+    IS_MULTIMODAL = True
+    DEFAULT_OUTPUT_LEN = 128
+
+    def __init__(
+        self,
+        dataset_path: str | None = None,
+        dataset_split: str = "test",
+        random_seed: int = 0,
+        video_dir: str | None = None,
+        subtitle_dir: str | None = None,
+        parquet_path: str | None = None,
+        pack_mode: VideoMMEPackMode = "minicpm-frames",
+        max_frames: int | None = None,
+        duration_filter: VideoMMEDurationFilter = "all",
+        use_subtitle: bool = False,
+        inline_local_video: bool = False,
+        trust_remote_code: bool = False,
+        dataset_subset: str | None = None,
+        no_stream: bool = False,
+        **kwargs,
+    ) -> None:
+        if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"):
+            raise ValueError(
+                f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}"
+            )
+        if duration_filter not in ("all", "short", "medium", "long"):
+            raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}")
+        if parquet_path is None and dataset_path is None:
+            raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.")
+
+        self.parquet_path = Path(parquet_path) if parquet_path else None
+        self.dataset_path = dataset_path
+        self.dataset_split = dataset_split
+        self.dataset_subset = dataset_subset
+        self._hf_streaming = not no_stream
+        self.video_dir = Path(video_dir) if video_dir else None
+        self.subtitle_dir = Path(subtitle_dir) if subtitle_dir else None
+        self.pack_mode: VideoMMEPackMode = pack_mode
+        self.duration_filter: VideoMMEDurationFilter = duration_filter
+        self.use_subtitle = use_subtitle
+        self.inline_local_video = inline_local_video
+        self.trust_remote_code = trust_remote_code
+        self.max_frames = int(
+            max_frames
+            if max_frames is not None
+            else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES)
+        )
+        #: In-process memo of content parts; the on-disk frame cache survives across runs.
+        self._frame_cache: dict[str, list[dict[str, Any]]] = {}
+        self._video_index: dict[str, Path] | None = None
+
+        super().__init__(
+            dataset_path=dataset_path if self.parquet_path is None else None,
+            random_seed=random_seed,
+            **kwargs,
+        )
+        self.load_data()
+        logger.info(
+            "Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, "
+            "use_subtitle=%s, video_dir=%s",
+            str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}",
+            pack_mode,
+            self.max_frames,
+            duration_filter,
+            use_subtitle,
+            self.video_dir,
+        )
+
+    # ------------------------------------------------------------------ loading
+
+    def load_data(self) -> None:
+        if self.parquet_path is not None:
+            self._load_from_parquet(self.parquet_path)
+            return
+
+        local_root = resolve_videomme_local_root(self.dataset_path)
+        if local_root is not None:
+            local_pq = videomme_local_parquet(local_root)
+            if local_pq is not None:
+                if self.video_dir is None:
+                    try:
+                        self.video_dir = ensure_videomme_videos_extracted(local_root)
+                    except FileNotFoundError:
+                        self.video_dir = videomme_local_video_dir(local_root)
+                if self.subtitle_dir is None:
+                    self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root)
+                self._load_from_parquet(local_pq)
+                return
+
+        self._load_from_huggingface()
+
+    def _load_from_parquet(self, path: Path) -> None:
+        try:
+            import pandas as pd
+        except ImportError as e:
+            raise ImportError("pandas is required to load Video-MME parquet") from e
+        if not path.is_file():
+            raise FileNotFoundError(f"Video-MME parquet not found: {path}")
+        df = pd.read_parquet(path)
+        self._set_rows([row.to_dict() for _, row in df.iterrows()])
+
+    def _load_from_huggingface(self) -> None:
+        if load_dataset is None:
+            raise ImportError(
+                "datasets library is required for HuggingFace Video-MME loading. "
+                "Install with: pip install datasets, or pass --videomme-parquet."
+            )
+        load_kw: dict[str, Any] = {
+            "split": self.dataset_split,
+            "streaming": self._hf_streaming,
+            "trust_remote_code": self.trust_remote_code,
+        }
+        if self.dataset_subset is not None:
+            load_kw["name"] = self.dataset_subset
+        ds = load_dataset(self.dataset_path, **load_kw)
+        self._set_rows([self._coerce_row(item) for item in ds])
+
+    def _set_rows(self, rows: list[dict[str, Any]]) -> None:
+        """Apply the duration filter and shuffle, then expose rows as ``self.data``."""
+        if self.duration_filter != "all":
+            want = self.duration_filter.lower()
+            rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want]
+        if not getattr(self, "disable_shuffle", False) and self.random_seed is not None:
+            import random
+
+            rows = rows[:]
+            random.Random(self.random_seed).shuffle(rows)
+        self.data = _ListDatasetIterator(rows)
+
+    @staticmethod
+    def _coerce_row(item: Any) -> dict[str, Any]:
+        if isinstance(item, dict):
+            return item
+        if hasattr(item, "as_py"):
+            return dict(item.as_py())
+        try:
+            return dict(item)
+        except (TypeError, ValueError):
+            return {k: item[k] for k in item}  # type: ignore[misc]
+
+    @staticmethod
+    def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]:
+        # The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain
+        # over the alias keys is not usable here.
+        options: Any = []
+        for key in ("options", "candidates", "choices"):
+            value = row.get(key)
+            if hasattr(value, "tolist"):
+                value = value.tolist()
+            if value is not None and len(value) > 0:
+                options = value
+                break
+        return {
+            "video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(),
+            "question_id": str(row.get("question_id") or "").strip(),
+            "question": str(row.get("question") or "").strip(),
+            "options": [str(x) for x in list(options)],
+            "answer": str(row.get("answer") or "").strip(),
+            "duration": str(row.get("duration") or "").strip(),
+            "domain": str(row.get("domain") or "").strip(),
+            "sub_category": str(row.get("sub_category") or "").strip(),
+            "task_type": str(row.get("task_type") or "").strip(),
+        }
+
+    # ------------------------------------------------------------------ sampling
+
+    def sample(
+        self,
+        tokenizer: TokenizerLike,
+        num_requests: int,
+        output_len: int | None = None,
+        request_id_prefix: str = "",
+        no_oversample: bool = False,
+        **kwargs,
+    ) -> list[SampleRequest]:
+        if output_len is None:
+            output_len = self.DEFAULT_OUTPUT_LEN
+
+        sampled: list[SampleRequest] = []
+        cached_tokenizer = get_cached_tokenizer(tokenizer)
+        # A cold frame cache costs a few seconds per video, so the whole set takes tens of
+        # minutes before the first request goes out; report progress to keep that legible.
+        started = time.monotonic()
+        last_report = started
+        try:
+            total = min(num_requests, len(self.data))
+        except TypeError:  # HF streaming datasets are not sized
+            total = num_requests
+        for seen, item in enumerate(self.data, start=1):
+            if len(sampled) >= num_requests:
+                break
+            req = self._create_sample_request(
+                self._coerce_row(item), cached_tokenizer, output_len, request_id_prefix, len(sampled)
+            )
+            if req:
+                sampled.append(req)
+            now = time.monotonic()
+            if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S:
+                logger.info(
+                    "Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed",
+                    seen,
+                    total,
+                    len(sampled),
+                    now - started,
+                )
+                last_report = now
+
+        logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started)
+        self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample)
+        return sampled
+
+    def _create_sample_request(
+        self,
+        qa_item: dict[str, Any],
+        tokenizer: TokenizerLike,
+        output_len: int,
+        request_id_prefix: str,
+        index: int,
+    ) -> SampleRequest | None:
+        fields = self._normalize_fields(qa_item)
+        if not fields["video_id"] or not fields["question"]:
+            logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"])
+            return None
+
+        mm_payload, omni_extra = self._compose_multimodal(fields["video_id"])
+        if not mm_payload:
+            return None
+
+        user_text = self._build_user_prompt(fields)
+        return VideoMMESampleRequest(
+            prompt=user_text,
+            prompt_len=len(tokenizer.encode(user_text)),
+            expected_output_len=output_len,
+            multi_modal_data=None,
+            request_id=f"{request_id_prefix}{index}",
+            videomme_gold_answer=fields["answer"],
+            videomme_video_id=fields["video_id"],
+            videomme_question_id=fields["question_id"],
+            videomme_duration=fields["duration"],
+            videomme_domain=fields["domain"],
+            videomme_sub_category=fields["sub_category"],
+            videomme_task_type=fields["task_type"],
+            omni_extra_body=omni_extra,
+            omni_chat_messages=self._build_openai_messages(mm_payload, user_text),
+            omni_chat_mm_position="first",
+        )
+
+    # ------------------------------------------------------------------ prompt
+
+    @staticmethod
+    def _format_options(options: list[str]) -> str:
+        """Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line).
+
+        The Hub parquet already ships options as ``"A. content"``. Existing labels are
+        dropped before relabelling so the prompt never reads ``A. A. content``, but only
+        when *every* option carries one — otherwise contents such as ``A.M. shift`` would
+        be truncated.
+        """
+        if not options:
+            return ""
+        matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options]
+        relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS))
+        lines = []
+        for key, raw, m in zip(_OPTION_LETTERS, options, matches):
+            content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip()
+            lines.append(f"{key}. {content}")
+        return "\n".join(lines)
+
+    def _build_user_prompt(self, fields: dict[str, Any]) -> str:
+        prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format(
+            question=fields["question"],
+            options=self._format_options(fields["options"]),
+        )
+        if self.use_subtitle:
+            subs = self._load_subtitle_text(fields["video_id"])
+            if subs:
+                prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}"
+        return prompt
+
+    def _load_subtitle_text(self, video_id: str) -> str:
+        if not self.subtitle_dir:
+            return ""
+        path = self.subtitle_dir / f"{video_id}.srt"
+        if not path.is_file():
+            return ""
+        try:
+            raw = path.read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            return ""
+        lines = [
+            s.replace("\\N", " ")
+            for s in (line.strip() for line in raw.splitlines())
+            if s and not s.isdigit() and "-->" not in s
+        ]
+        return "\n".join(lines)
+
+    def _build_openai_messages(
+        self,
+        mm_payload: dict[str, Any] | list[dict[str, Any]],
+        user_text: str,
+    ) -> list[dict[str, Any]]:
+        mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload]
+        messages: list[dict[str, Any]] = []
+        if MINICPM_OMNI_SYSTEM_TEXT:
+            messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]})
+        messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]})
+        return messages
+
+    # ------------------------------------------------------------------ media
+
+    def _video_path_index(self) -> dict[str, Path]:
+        """Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses."""
+        if self._video_index is None:
+            index: dict[str, Path] = {}
+            if self.video_dir is not None:
+                for path in _iter_video_files(self.video_dir):
+                    index.setdefault(path.stem, path)
+            logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir)
+            self._video_index = index
+        return self._video_index
+
+    def _resolve_local_video_path(self, video_id: str) -> Path | None:
+        if not self.video_dir or not video_id:
+            return None
+        for p in (
+            self.video_dir / f"{video_id}.mp4",
+            self.video_dir / f"{video_id}.MP4",
+            self.video_dir / f"{video_id}.mkv",
+            self.video_dir / video_id / f"{video_id}.mp4",
+        ):
+            if p.is_file():
+                return p
+        return self._video_path_index().get(video_id)
+
+    def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]:
+        path = path.expanduser().resolve()
+        if self.inline_local_video:
+            b64 = base64.b64encode(path.read_bytes()).decode("ascii")
+            return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
+        return {"type": typ, typ: {"url": path.as_uri()}}
+
+    def _compose_multimodal(
+        self,
+        video_id: str,
+    ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]:
+        # OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the
+        # sample carries a video path: 9 slices per frame would blow past max_model_len and
+        # <image_id>N</image_id> would label each frame as a separate picture.
+        extra: dict[str, Any] = {
+            "mm_processor_kwargs": {
+                "use_audio_in_video": False,
+                "max_slice_nums": 1,
+                "use_image_id": False,
+            }
+        }
+
+        if self.pack_mode == "video_url":
+            video_path = self._resolve_local_video_path(video_id)
+            if video_path is None:
+                logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir)
+                return None, None
+            return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra
+
+        parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave")
+        if not parts:
+            return None, None
+        return parts, extra
+
+    def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path:
+        assert self.video_dir is not None
+        tag = f"f{self.max_frames}{'_av' if include_audio else ''}"
+        return self.video_dir / ".minicpm_videomme_frames" / video_id / tag
+
+    def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None:
+        """Rebuild content parts from a previous run's frame dump, or ``None`` on a miss."""
+        manifest = cache_dir / "manifest.json"
+        if not manifest.is_file():
+            return None
+        try:
+            count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"])
+        except (OSError, ValueError, KeyError, TypeError):
+            return None
+
+        parts: list[dict[str, Any]] = []
+        for i in range(count):
+            frame_path = cache_dir / f"frame_{i:04d}.jpg"
+            if not frame_path.is_file():
+                return None
+            parts.append({"type": "image_url", "image_url": {"url": frame_path.as_uri()}})
+            if include_audio:
+                audio_path = cache_dir / f"audio_{i:04d}.wav"
+                if not audio_path.is_file():
+                    return None
+                parts.append({"type": "audio_url", "audio_url": {"url": audio_path.as_uri()}})
+        return parts or None
+
+    def _get_minicpm_frame_parts(
+        self,
+        video_id: str,
+        *,
+        include_audio: bool,
+    ) -> list[dict[str, Any]] | None:
+        cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}"
+        cached = self._frame_cache.get(cache_key)
+        if cached is not None:
+            return cached
+
+        # Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS.
+        cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio)
+        if cache_dir is not None:
+            disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio)
+            if disk_parts is not None:
+                self._frame_cache[cache_key] = disk_parts
+                return disk_parts
+
+        video_path = self._resolve_local_video_path(video_id)
+        if video_path is None:
+            logger.warning(
+                "Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r",
+                self.video_dir,
+                video_id,
+            )
+            return None
+
+        try:
+            frames, audio_segments = self._extract_frames_and_audio(
+                video_path,
+                include_audio=include_audio,
+                max_num_frames=self.max_frames,
+            )
+        except Exception:
+            logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path)
+            return None
+
+        if not frames:
+            logger.warning("No frames extracted for video_id=%r", video_id)
+            return None
+        if include_audio and len(audio_segments) != len(frames):
+            n = min(len(frames), len(audio_segments))
+            frames, audio_segments = frames[:n], audio_segments[:n]
+
+        if cache_dir is not None:
+            cache_dir.mkdir(parents=True, exist_ok=True)
+
+        parts: list[dict[str, Any]] = []
+        for i, frame in enumerate(frames):
+            parts.append(
+                self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg")
+            )
+            if include_audio:
+                parts.append(
+                    self._emit_media(
+                        cache_dir,
+                        f"audio_{i:04d}.wav",
+                        _numpy_to_wav_bytes(audio_segments[i]),
+                        "audio_url",
+                        "audio/wav",
+                    )
+                )
+
+        if cache_dir is not None:
+            (cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8")
+            self._frame_cache[cache_key] = parts
+        logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts))
+        return parts
+
+    @staticmethod
+    def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]:
+        """Write ``payload`` to the frame cache and return a file URL part, or inline base64."""
+        if cache_dir is None:
+            b64 = base64.b64encode(payload).decode("ascii")
+            return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
+        path = cache_dir / name
+        if not path.is_file() or path.stat().st_size == 0:
+            path.write_bytes(payload)
+        return {"type": typ, typ: {"url": path.as_uri()}}
+
+    @staticmethod
+    def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]:
+        """Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps."""
+        if duration > max_num_frames:
+            grid = [round(i * 0.1, 1) for i in range(int(duration / 0.1))]
+            return [grid[i] for i in _uniform_sample_indices(len(grid), max_num_frames)]
+        # OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame.
+        return [float(i) for i in range(max(1, int(duration)))]
+
+    @classmethod
+    def _extract_frames_and_audio(
+        cls,
+        video_path: Path,
+        *,
+        include_audio: bool,
+        max_num_frames: int,
+    ) -> tuple[list[Any], list[Any]]:
+        import numpy as np
+        from vllm.multimodal.media.audio import load_audio
+
+        duration = _probe_video_duration(video_path)
+        timestamps = cls._sample_timestamps(duration, max_num_frames)
+        frames = _decode_frames_at_timestamps(video_path, timestamps)
+
+        audio_segments: list[Any] = []
+        if include_audio:
+            audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True)
+            for i, start_time in enumerate(timestamps):
+                end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration
+                segment = audio_np[int(start_time * sr) : int(end_time * sr)]
+                if i == len(timestamps) - 1 and len(segment) < 1600:
+                    segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)])
+                if len(segment) == 0:
+                    segment = np.zeros(1600, dtype=np.float32)
+                audio_segments.append(segment)
+
+        return frames, audio_segments

vllm_omni/benchmarks/data_modules/videomme_eval.py

@@ -0,0 +1,214 @@
+"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve.
+
+Answer extraction ports ``extract_characters_regex`` from the official Video-MME
+script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval).
+
+**Denominator:** responses where no A–D can be extracted are counted as wrong, which
+matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The
+official standalone script instead drops them from ``answered``; ``videomme_parse_failed``
+is reported so that view can be recovered.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
+
+from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest
+
+_VALID = frozenset("ABCD")
+
+VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long")
+
+#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns).
+_DIMENSIONS: dict[str, str] = {
+    "duration": "videomme_duration",
+    "domain": "videomme_domain",
+    "sub_category": "videomme_sub_category",
+    "task": "videomme_task_type",
+}
+
+# Official prefix list, with the two accidental string concatenations in the upstream
+# tuple (missing commas after "The best option is" / "Best answer:") split apart.
+_ANSWER_PREFIXES = (
+    "The best answer is",
+    "The correct answer is",
+    "The answer is",
+    "The answer",
+    "The best option is",
+    "The correct option is",
+    "Best answer:",
+    "Best option:",
+    "Answer:",
+    "Option:",
+    "The correct answer",
+    "The correct option",
+)
+
+
+def extract_characters_regex(text: str | None) -> str | None:
+    """Port of official Video-MME / lmms-eval ``extract_characters_regex``."""
+    if not text:
+        return None
+    s = str(text).strip()
+    for prefix in _ANSWER_PREFIXES:
+        s = s.replace(prefix, "")
+    if len(s.split()) > 10 and not re.search(r"[ABCD]", s):
+        return None
+    match = re.search(r"[ABCD]", s)
+    return match.group(0) if match else None
+
+
+def normalize_gold_answer(gold: str) -> str | None:
+    g = (gold or "").strip().upper()
+    if len(g) == 1 and g in _VALID:
+        return g
+    m = re.search(r"([ABCD])\b", g)
+    return m.group(1).upper() if m else None
+
+
+def _bucket(req: VideoMMESampleRequest, dimension: str) -> str:
+    value = (getattr(req, _DIMENSIONS[dimension]) or "").strip()
+    if dimension == "duration":
+        return value.lower()
+    return value or "unknown"
+
+
+def compute_videomme_accuracy_metrics(
+    input_requests: list[Any],
+    outputs: list[RequestFuncOutput],
+    *,
+    include_per_item: bool = False,
+) -> dict[str, Any] | None:
+    """If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats.
+
+    Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are
+    excluded from ``videomme_accuracy`` and counted in
+    ``videomme_accuracy_incl_http_fail`` instead.
+    """
+    if not input_requests or len(input_requests) != len(outputs):
+        return None
+    if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests):
+        return None
+
+    stats: dict[str, dict[str, dict[str, int]]] = {
+        dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {})
+        for dim in _DIMENSIONS
+    }
+    items: list[dict[str, Any]] = []
+    correct = evaluated = no_gold = request_failed = parse_failed = 0
+
+    for req, out in zip(input_requests, outputs, strict=True):
+        assert isinstance(req, VideoMMESampleRequest)
+        gold_raw = (req.videomme_gold_answer or "").strip()
+        buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS}
+        row: dict[str, Any] = {
+            "request_id": req.request_id,
+            "video_id": req.videomme_video_id,
+            "question_id": req.videomme_question_id,
+            **buckets,
+        }
+
+        if not gold_raw:
+            no_gold += 1
+            items.append({**row, "skipped": True, "reason": "no_gold"})
+            continue
+
+        evaluated += 1
+        gold_norm = normalize_gold_answer(gold_raw)
+        pred = extract_characters_regex(out.generated_text) if out.success else None
+        is_correct = bool(pred) and pred.upper() == (gold_norm or gold_raw).upper()
+
+        if not out.success:
+            request_failed += 1
+        else:
+            if pred is None:
+                parse_failed += 1
+            # Only successful responses enter the per-dimension denominators, mirroring
+            # the overall videomme_accuracy definition.
+            for dim, key in buckets.items():
+                bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0})
+                bucket["total"] += 1
+                bucket["correct"] += int(is_correct)
+
+        if is_correct:
+            correct += 1
+
+        items.append(
+            {
+                **row,
+                "gold": gold_raw,
+                "gold_normalized": gold_norm,
+                "predicted": pred,
+                "correct": is_correct,
+                "parse_failed": out.success and pred is None,
+                **({"error": (out.error or "")[:500]} if not out.success else {}),
+            }
+        )
+
+    evaluated_ok = evaluated - request_failed
+    result: dict[str, Any] = {
+        "videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None,
+        "videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None,
+        "videomme_correct": correct,
+        "videomme_evaluated": evaluated,
+        "videomme_evaluated_ok": evaluated_ok,
+        "videomme_no_gold": no_gold,
+        "videomme_request_failed": request_failed,
+        "videomme_parse_failed": parse_failed,
+    }
+    for dim, per_bucket in stats.items():
+        result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()}
+        result[f"videomme_per_{dim}_accuracy"] = {
+            k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items()
+        }
+    if include_per_item:
+        result["videomme_eval_items"] = items
+    return result
+
+
+def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None:
+    """Pretty-print the Video-MME accuracy block (stdout)."""
+    acc = metrics.get("videomme_accuracy")
+    if acc is None and not metrics.get("videomme_evaluated", 0):
+        return
+
+    print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="="))
+    ok = int(metrics.get("videomme_evaluated_ok", 0) or 0)
+    if ok and acc is not None:
+        print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}")
+    else:
+        print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)")
+    for label, key in (
+        ("Submitted (gold present):", "videomme_evaluated"),
+        ("Successful HTTP (denominator):", "videomme_evaluated_ok"),
+        ("Correct:", "videomme_correct"),
+        ("Skipped (no gold):", "videomme_no_gold"),
+        ("HTTP failed:", "videomme_request_failed"),
+        ("Parsed OK but no A-D found:", "videomme_parse_failed"),
+    ):
+        print(f"{label:<40} {metrics.get(key, 0):<10}")
+
+    for dim, title in (
+        ("duration", "Duration"),
+        ("domain", "Domain"),
+        ("sub_category", "Sub Category"),
+        ("task", "Task Type"),
+    ):
+        per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {}
+        if not per_acc:
+            continue
+        counts = metrics.get(f"videomme_per_{dim}") or {}
+        names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc)
+        print(f"\n--- Accuracy by {title} ---")
+        for name in names:
+            st = counts.get(name) or {}
+            total = int(st.get("total", 0) or 0)
+            value = per_acc.get(name)
+            if total and value is not None:
+                print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}")
+            else:
+                print(f"{name}: 0/0 = N/A")
+    print("=" * 50)

vllm_omni/benchmarks/patch/patch.py

@@ -56,6 +56,18 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
 )
 from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
 from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
+from vllm_omni.benchmarks.data_modules.videomme_dataset import (
+    VIDEOMME_DEFAULT_HF_REPO,
+    VideoMMEDataset,
+    VideoMMESampleRequest,
+    ensure_videomme_subtitles_extracted,
+    ensure_videomme_videos_extracted,
+    resolve_videomme_local_root,
+    resolve_videomme_root,
+    videomme_local_parquet,
+    videomme_local_subtitle_dir,
+    videomme_local_video_dir,
+)
 from vllm_omni.experimental.fullduplex.client import (
     RealtimeDuplexClient,
     summarize_session_request_metrics,
@@ -151,6 +163,7 @@ def _pcm_s16le_to_seed_tts_wer_bytes(
 get_samples_old = datasets.get_samples
 
 _DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni"
+_DEFAULT_VIDEOMME_REPO = VIDEOMME_DEFAULT_HF_REPO
 
 
 def _seed_tts_capture_pcm_for_wer() -> bool:
@@ -189,6 +202,17 @@ def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: Request
         setattr(rfi, "mm_position", sample.omni_chat_mm_position)
 
 
+def _attach_videomme_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
+    """Apply per-request OpenAI fields for Video-MME (same message path as Daily-Omni)."""
+    if not isinstance(sample, VideoMMESampleRequest):
+        return
+    rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
+    if sample.omni_chat_messages is not None:
+        setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
+    else:
+        setattr(rfi, "mm_position", sample.omni_chat_mm_position)
+
+
 def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
     """Merge Seed-TTS per-row TTS fields into ``extra_body`` and mark for PCM capture.
 
@@ -238,11 +262,27 @@ def _daily_omni_repo_from_args(args) -> str | None:
     return None
 
 
+def _videomme_repo_from_args(args) -> str | None:
+    """Resolve HuggingFace repo id for Video-MME from CLI args."""
+    dp = getattr(args, "dataset_path", None)
+    hn = getattr(args, "hf_name", None)
+    supported = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS}
+    supported.add(_DEFAULT_VIDEOMME_REPO.lower())
+    if isinstance(dp, str) and dp.strip().lower() in supported:
+        return dp.strip()
+    if isinstance(hn, str) and hn.strip().lower() in supported:
+        return hn.strip()
+    return None
+
+
 def get_samples(args, tokenizer):
     # Daily-Omni: explicit dataset name, or hf + matching path/hf-name
     is_daily_omni = args.dataset_name == "daily-omni" or (
         args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
     )
+    is_videomme = args.dataset_name == "videomme" or (
+        args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None
+    )
     is_seed_tts = args.dataset_name in (
         "seed-tts",
         "seed-tts-text",
@@ -258,7 +298,7 @@ def get_samples(args, tokenizer):
         "openai-realtime-duplex",
         "daily-omni",
     ]
-    is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
+    is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or args.dataset_name == "random-mm"
 
     if not is_omni_backend and not is_omni_dataset:
         # Not an omni-related request, delegate to original implementation
@@ -376,6 +416,113 @@ def get_samples(args, tokenizer):
         )
         return input_requests
 
+    if is_videomme:
+        if args.backend not in ["openai-chat-omni", "daily-omni"]:
+            raise ValueError(
+                f"Video-MME dataset requires a multimodal backend that supports video. "
+                f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
+            )
+
+        video_dir = getattr(args, "videomme_video_dir", None)
+        subtitle_dir = getattr(args, "videomme_subtitle_dir", None)
+        parquet_path = getattr(args, "videomme_parquet", None)
+        if isinstance(parquet_path, str):
+            parquet_path = parquet_path.strip() or None
+        dataset_split = getattr(args, "hf_split", None) or "test"
+
+        # Local directory wins (absolute/relative existing path). Hub ids fall through to
+        # snapshot_download — same pattern as Seed-TTS / Daily-Omni.
+        local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or (
+            resolve_videomme_local_root(getattr(args, "hf_name", None))
+        )
+        repo_id = _videomme_repo_from_args(args)
+        if local_root is None and video_dir is None and parquet_path is None:
+            # Default Hub mode: download parquet + video zips on demand.
+            if args.dataset_name == "videomme":
+                hub_id = repo_id or _DEFAULT_VIDEOMME_REPO
+            elif repo_id is not None:
+                hub_id = repo_id
+            else:
+                raise ValueError(
+                    "Video-MME requires --videomme-parquet, a local --dataset-path mirror, or "
+                    f"--dataset-path / --hf-name {_DEFAULT_VIDEOMME_REPO}."
+                )
+            local_root = resolve_videomme_root(hub_id)
+            repo_id = hub_id
+            logger.info("Using Hugging Face Video-MME snapshot: root=%s repo=%s", local_root, hub_id)
+        elif local_root is not None:
+            logger.info("Using local Video-MME mirror: root=%s", local_root)
+
+        if local_root is not None:
+            if parquet_path is None:
+                local_pq = videomme_local_parquet(local_root)
+                if local_pq is not None:
+                    parquet_path = str(local_pq)
+            if video_dir is None:
+                try:
+                    video_dir = str(ensure_videomme_videos_extracted(local_root))
+                except FileNotFoundError:
+                    found = videomme_local_video_dir(local_root)
+                    video_dir = str(found) if found is not None else None
+            if subtitle_dir is None:
+                found_sub = ensure_videomme_subtitles_extracted(local_root) or videomme_local_subtitle_dir(local_root)
+                subtitle_dir = str(found_sub) if found_sub is not None else None
+
+        if parquet_path is None:
+            if args.dataset_name == "videomme":
+                repo_id = repo_id or (str(local_root) if local_root is not None else _DEFAULT_VIDEOMME_REPO)
+            elif repo_id is None and local_root is None:
+                raise ValueError(
+                    "Video-MME requires --videomme-parquet, a local --dataset-path mirror, or "
+                    f"--dataset-path / --hf-name {_DEFAULT_VIDEOMME_REPO}."
+                )
+
+        if video_dir is None:
+            raise ValueError(
+                "Video-MME requires --videomme-video-dir pointing at extracted videos "
+                "(directory of {videoID}.mp4), or a local/Hub dataset root containing video/ "
+                "or videos_chunked_*.zip."
+            )
+
+        logger.info(
+            "Loading Video-MME: parquet=%s, hf_repo=%s, video_dir=%s, pack_mode=%s",
+            parquet_path,
+            repo_id,
+            video_dir,
+            getattr(args, "videomme_pack_mode", "minicpm-frames"),
+        )
+        dataset = VideoMMEDataset(
+            parquet_path=parquet_path,
+            dataset_path=None if parquet_path is not None else (repo_id or str(local_root)),
+            dataset_split=dataset_split,
+            dataset_subset=getattr(args, "hf_subset", None),
+            random_seed=args.seed,
+            video_dir=video_dir,
+            subtitle_dir=subtitle_dir,
+            pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"),
+            max_frames=getattr(args, "videomme_max_frames", None),
+            duration_filter=getattr(args, "videomme_duration", "all"),
+            use_subtitle=getattr(args, "videomme_use_subtitle", False),
+            inline_local_video=getattr(args, "videomme_inline_local_video", False),
+            trust_remote_code=getattr(args, "trust_remote_code", False),
+            no_stream=getattr(args, "no_stream", False),
+            disable_shuffle=getattr(args, "disable_shuffle", False),
+        )
+
+        out_len = getattr(args, "output_len", None)
+        if out_len is None:
+            out_len = getattr(args, "hf_output_len", None)
+        if out_len is None:
+            out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN
+
+        return dataset.sample(
+            tokenizer=tokenizer,
+            num_requests=args.num_prompts,
+            output_len=out_len,
+            request_id_prefix=args.request_id_prefix,
+            no_oversample=args.no_oversample,
+        )
+
     if is_seed_tts:
         if args.backend not in (
             "openai-audio-speech",
@@ -1645,6 +1792,7 @@ async def benchmark(
         chat_messages=test_chat_messages,
     )
     _attach_daily_omni_to_request_func_input(input_requests[0], test_input)
+    _attach_videomme_to_request_func_input(input_requests[0], test_input)
     _attach_seed_tts_to_request_func_input(input_requests[0], test_input)
 
     if ready_check_timeout_sec > 0:
@@ -1710,6 +1858,7 @@ async def benchmark(
             chat_messages=test_chat_messages,
         )
         _attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
+        _attach_videomme_to_request_func_input(input_requests[0], profile_input)
         _attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
         profile_output = await request_func(request_func_input=profile_input, session=session)
         if profile_output.success:
@@ -1795,6 +1944,7 @@ async def benchmark(
             chat_messages=request.chat_messages,
         )
         _attach_daily_omni_to_request_func_input(request, request_func_input)
+        _attach_videomme_to_request_func_input(request, request_func_input)
         _attach_seed_tts_to_request_func_input(request, request_func_input)
         tasks.append(
             asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar))
@@ -1892,6 +2042,21 @@ async def benchmark(
         result.update(_daily_acc)
         print_daily_omni_accuracy_summary(_daily_acc)
 
+    from vllm_omni.benchmarks.data_modules.videomme_eval import (
+        compute_videomme_accuracy_metrics,
+        print_videomme_accuracy_summary,
+    )
+
+    _save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    _vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm)
+    if _vm_acc is not None:
+        result.update(_vm_acc)
+        print_videomme_accuracy_summary(_vm_acc)
+
     if _seed_tts_capture_pcm_for_wer():
         from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
             compute_seed_tts_wer_metrics,

vllm_omni/benchmarks/serve.py

@@ -23,6 +23,8 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
         os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
     if getattr(args, "daily_omni_save_eval_items", False):
         os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
+    if getattr(args, "videomme_save_eval_items", False):
+        os.environ["VIDEOMME_SAVE_EVAL_ITEMS"] = "1"
     set_print_stage(getattr(args, "print_stage", False))
     args.extra_body = maybe_enable_stage_metrics(
         getattr(args, "extra_body", None),

vllm_omni/deploy/minicpmo_4_5.yaml

@@ -35,8 +35,9 @@ stages:
         fps: 1
         num_frames: 128
     limit_mm_per_prompt:
-      # Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
-      image: 64
+      # Daily-Omni: up to 64 interleaved image/audio pairs (1fps).
+      # Video-MME minicpm-frames: OmniEvalKit samples up to 96 frames as image_url.
+      image: 96
       audio: 64
       video: 1
     default_sampling_params:

vllm_omni/entrypoints/cli/benchmark/cli_args.py

@@ -117,6 +117,78 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
     )
 
 
+def add_videomme_cli_args(parser: argparse.ArgumentParser) -> None:
+    """Add CLI arguments specific to the Video-MME dataset."""
+    group = parser.add_argument_group("Video-MME Dataset Options")
+    group.add_argument(
+        "--videomme-parquet",
+        type=str,
+        default=None,
+        help="Path to local Video-MME parquet "
+        "(e.g. videomme/test-00000-of-00001.parquet). When set, Hub QA loading is skipped.",
+    )
+    group.add_argument(
+        "--videomme-video-dir",
+        type=str,
+        default=None,
+        help="Directory containing extracted Video-MME videos (videoID.mp4). "
+        "Typical layout after unzipping videos_chunked_*.zip: <root>/video/. "
+        "When using file:// URLs, start the server with --allowed-local-media-path "
+        "covering this directory.",
+    )
+    group.add_argument(
+        "--videomme-subtitle-dir",
+        type=str,
+        default=None,
+        help="Directory containing Video-MME .srt subtitles (videoID.srt). Used only with --videomme-use-subtitle.",
+    )
+    group.add_argument(
+        "--videomme-pack-mode",
+        type=str,
+        choices=["minicpm-frames", "minicpm-interleave", "video_url"],
+        default="minicpm-frames",
+        help="Multimodal packing. "
+        "'minicpm-frames' (default): OmniEvalKit MiniCPM videomme recipe — sampled frames "
+        "as image_url only (max_frames=96). "
+        "'minicpm-interleave': OmniEvalKit videomme_short recipe — 1fps frame/audio pairs "
+        "(max_frames=64). "
+        "'video_url': single video_url part (models with native video input).",
+    )
+    group.add_argument(
+        "--videomme-max-frames",
+        type=int,
+        default=None,
+        help="Override max sampled frames (OmniEvalKit defaults: 96 for minicpm-frames, 64 for minicpm-interleave).",
+    )
+    group.add_argument(
+        "--videomme-duration",
+        type=str,
+        choices=["all", "short", "medium", "long"],
+        default="all",
+        help="Filter by Video-MME duration bucket (default: all).",
+    )
+    group.add_argument(
+        "--videomme-use-subtitle",
+        action="store_true",
+        default=False,
+        help="Prepend subtitle text to the user prompt (Video-MME w/ subs setting).",
+    )
+    group.add_argument(
+        "--videomme-inline-local-video",
+        action="store_true",
+        default=False,
+        help="Embed local frames/audio as base64 data URLs so the server does not need "
+        "--allowed-local-media-path. Increases request size; use for small --num-prompts.",
+    )
+    group.add_argument(
+        "--videomme-save-eval-items",
+        action="store_true",
+        default=False,
+        help="Include per-request Video-MME accuracy rows in the saved JSON under "
+        "videomme_eval_items. Or set env VIDEOMME_SAVE_EVAL_ITEMS=1.",
+    )
+
+
 def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
     """Add CLI arguments for Seed-TTS benchmarks."""
     group = parser.add_argument_group("Seed-TTS Dataset Options")
@@ -183,6 +255,7 @@ def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
 
 _OMNI_BENCH_DATASET_CHOICES = (
     "daily-omni",
+    "videomme",
     "seed-tts",
     "seed-tts-text",
     "seed-tts-design",
@@ -249,6 +322,7 @@ def update_omni_help(parser: argparse.ArgumentParser) -> None:
 def add_omni_args(parser: argparse.ArgumentParser) -> None:
     """Register all vLLM-Omni serving benchmark arguments."""
     add_daily_omni_cli_args(parser)
+    add_videomme_cli_args(parser)
     add_seed_tts_cli_args(parser)
     add_multi_stage_cli_args(parser)
     add_diffusion_cli_args(parser)