From 9f5521351e4471d6538e74ca66cb63cde23b9140 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Sun, 20 Sep 2026 17:15:53 +0800 Subject: [PATCH] =?UTF-8?q?[Benchmark][MiniCPM-o]=20Port=20Video-MME=20dat?= =?UTF-8?q?aset=20support=20to=20Omni=20bench=20s=E2=80=A6=20(#6987)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: amy-why-3459 --- docs/cli/bench/serve.md | 55 +- tests/benchmarks/test_accuracy_bench_utils.py | 39 + .../benchmarks/test_videomme_local_source.py | 546 +++++++++ tests/config/test_environment_variables.py | 2 +- .../qwen3_omni/qwen3_omni_acc_bench_core.py | 39 +- .../qwen3_omni/run_qwen_omni_acc_benchmark.py | 233 +++- .../models/minicpmo_4_5/test_pipeline.py | 2 +- .../data_modules/videomme_dataset.py | 1008 +++++++++++++++++ .../benchmarks/data_modules/videomme_eval.py | 237 ++++ vllm_omni/benchmarks/patch/patch.py | 247 +++- vllm_omni/benchmarks/serve.py | 4 +- .../config/environment_variable_inventory.py | 1 + vllm_omni/config/omni_config.py | 2 +- vllm_omni/deploy/minicpmo_4_5.yaml | 2 +- .../entrypoints/cli/benchmark/cli_args.py | 74 ++ 15 files changed, 2432 insertions(+), 59 deletions(-) create mode 100644 tests/benchmarks/test_videomme_local_source.py create mode 100644 vllm_omni/benchmarks/data_modules/videomme_dataset.py create mode 100644 vllm_omni/benchmarks/data_modules/videomme_eval.py diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index 8dc5e1ed8..450a01e5a 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para The API endpoint exposed externally, to which clients send their requests. - `--dataset-name` - The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos. + The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio), + `omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos + (`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing). - `--num-prompts` The total number of requests to send, an integer. @@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality signal, not a transport failure. This local performance test does not score answer accuracy. +### Video-MME Benchmark + +Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is +OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local +mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging +Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when +`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved +to absolute `file://` URLs. For those URLs, start the server with +`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`. + +For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the +96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME` +with the local video root shared by the benchmark client and server): + +```bash +vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --allowed-local-media-path /path/to/Video-MME \ + --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}' +``` + +The override applies to this server invocation only. Alternatively, use +`--videomme-max-frames 64` in the benchmark to stay within the default image limit; +this evaluates a different frame-sampling setting from the 96-frame recipe. + +Run the benchmark against that server: + +```bash +vllm bench serve --omni \ + --backend openai-chat-omni \ + --dataset-name videomme \ + --dataset-path /path/to/Video-MME \ + --videomme-pack-mode minicpm-frames \ + --videomme-max-frames 96 \ + --model openbmb/MiniCPM-o-4_5 \ + --endpoint /v1/chat/completions \ + --num-prompts 8 \ + --max-concurrency 1 \ + --save-result +``` + +Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into +the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail` +counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when +oversampling made the request count differ from unique-question coverage. +`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable +media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP +failures, duplicate question IDs, and missing gold answers. This checks the sampled subset; +it does not establish that a supplied local mirror contains the entire official dataset. +Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request +`videomme_eval_items` rows. + ### Multi-Modal Benchmark
diff --git a/tests/benchmarks/test_accuracy_bench_utils.py b/tests/benchmarks/test_accuracy_bench_utils.py index 8d78da2b2..8f9f74785 100644 --- a/tests/benchmarks/test_accuracy_bench_utils.py +++ b/tests/benchmarks/test_accuracy_bench_utils.py @@ -47,11 +47,50 @@ from benchmarks.accuracy.text_to_image.gbench import ( from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import ( build_serve_common_argv, seed_tts_bench_argv, + videomme_bench_argv, ) from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root +def test_videomme_bench_argv_preserves_hf_repo_id_from_env(monkeypatch): + monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", "lmms-eval/Video-MME") + monkeypatch.delenv("VLLM_VIDEOMME_REPO", raising=False) + monkeypatch.delenv("VIDEOMME_ROOT", raising=False) + + argv = videomme_bench_argv() + + dataset_idx = argv.index("--dataset-path") + assert argv[argv.index("--dataset-name") + 1] == "videomme" + assert argv[dataset_idx + 1] == "lmms-eval/Video-MME" + + +def test_videomme_bench_argv_uses_local_directory(monkeypatch, tmp_path: Path): + monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", str(tmp_path)) + monkeypatch.delenv("VIDEOMME_ROOT", raising=False) + + argv = videomme_bench_argv() + + assert argv[argv.index("--dataset-path") + 1] == str(tmp_path.resolve()) + + +def test_sync_dataset_env_preserves_videomme_hf_repo_id(monkeypatch): + ns = argparse.Namespace( + daily_omni_repo=None, + daily_omni_qa_json=None, + daily_omni_video_dir=None, + videomme_repo=None, + videomme_dataset_path="lmms-eval/Video-MME", + seed_tts_dataset_path=None, + seed_tts_root=None, + ) + + monkeypatch.delenv("VLLM_VIDEOMME_DATASET_PATH", raising=False) + sync_dataset_env_from_ns(ns) + + assert os.environ["VLLM_VIDEOMME_DATASET_PATH"] == "lmms-eval/Video-MME" + + def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch): monkeypatch.setenv("VLLM_SEED_TTS_DATASET_PATH", "zhaochenyang20/seed-tts-eval") monkeypatch.delenv("VLLM_SEED_TTS_REPO", raising=False) diff --git a/tests/benchmarks/test_videomme_local_source.py b/tests/benchmarks/test_videomme_local_source.py new file mode 100644 index 000000000..d1029b1e2 --- /dev/null +++ b/tests/benchmarks/test_videomme_local_source.py @@ -0,0 +1,546 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Offline Video-MME sources: local mirrors must not fall back to a Hub id.""" + +from __future__ import annotations + +import os +import zipfile +from argparse import Namespace +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse + +import pytest +from PIL import Image + +from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import _validate_videomme +from vllm_omni.benchmarks.data_modules.videomme_dataset import ( + VideoMMEDataset, + VideoMMESampleRequest, + cached_videomme_snapshots, + ensure_videomme_hub_root, + ensure_videomme_videos_extracted, + resolve_videomme_local_root, + videomme_local_parquet, + videomme_local_subtitle_dir, + videomme_local_video_dir, +) +from vllm_omni.benchmarks.data_modules.videomme_eval import ( + compute_videomme_accuracy_metrics, + extract_characters_regex, + normalize_gold_answer, +) +from vllm_omni.benchmarks.patch.patch import _looks_like_hf_dataset_id, _videomme_repo_from_args, get_samples + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] + + +def _write_parquet_placeholder(root: Path) -> Path: + path = root / "videomme" / "test-00000-of-00001.parquet" + path.parent.mkdir(parents=True) + path.write_bytes(b"parquet") + return path + + +def test_resolve_local_root_returns_none_for_hub_id() -> None: + assert resolve_videomme_local_root("lmms-eval/Video-MME") is None + assert resolve_videomme_local_root("") is None + assert resolve_videomme_local_root(None) is None + + +def test_plain_local_mirror(tmp_path: Path) -> None: + pq = _write_parquet_placeholder(tmp_path) + video = tmp_path / "video" / "abc.mp4" + video.parent.mkdir() + video.write_bytes(b"fake") + (tmp_path / "subtitle").mkdir() + (tmp_path / "subtitle" / "abc.srt").write_text("1\n", encoding="utf-8") + + root = resolve_videomme_local_root(str(tmp_path)) + assert root == tmp_path.resolve() + assert videomme_local_parquet(root) == pq.resolve() + assert videomme_local_video_dir(root) == tmp_path.resolve() / "video" + assert videomme_local_subtitle_dir(root) == tmp_path.resolve() / "subtitle" + + +def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None: + cache_dir = tmp_path / "datasets--lmms-eval--Video-MME" + snapshot = cache_dir / "snapshots" / "deadbeef" + snapshot.mkdir(parents=True) + _write_parquet_placeholder(snapshot) + (cache_dir / "refs").mkdir() + (cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8") + + root = resolve_videomme_local_root(str(cache_dir)) + assert root == snapshot.resolve() + assert videomme_local_parquet(root) is not None + + +def test_hf_cache_dir_with_unusable_ref_falls_back_to_revision(tmp_path: Path) -> None: + cache_dir = tmp_path / "datasets--lmms-eval--Video-MME" + snapshot = cache_dir / "snapshots" / "deadbeef" + snapshot.mkdir(parents=True) + _write_parquet_placeholder(snapshot) + (cache_dir / "refs").mkdir() + # An empty ref must not resolve to snapshots/, which holds no dataset files. + (cache_dir / "refs" / "main").write_text("", encoding="utf-8") + + assert resolve_videomme_local_root(str(cache_dir)) == snapshot.resolve() + + +def _stub_failing_hub(monkeypatch: pytest.MonkeyPatch, hub_cache: Path) -> dict[str, object]: + """Point the Hub cache at ``hub_cache`` and make every snapshot_download miss.""" + from huggingface_hub import constants as hf_constants + from huggingface_hub.errors import LocalEntryNotFoundError + + from vllm_omni.transformers_utils import repo_utils + + seen: dict[str, object] = {} + + class _Api: + def snapshot_download(self, **kwargs: object) -> str: + seen.update(kwargs) + raise LocalEntryNotFoundError("no cached snapshot") + + monkeypatch.setattr(hf_constants, "HF_HUB_OFFLINE", True) + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(hub_cache)) + monkeypatch.setattr(repo_utils, "hf_api", lambda: _Api()) + return seen + + +def test_hub_root_offline_without_cache_points_at_local_options( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen = _stub_failing_hub(monkeypatch, tmp_path) + + with pytest.raises(FileNotFoundError, match="--videomme-video-dir") as excinfo: + ensure_videomme_hub_root("lmms-lab/Video-MME") + + # Offline runs must consult the cache directly instead of resolving over the network. + assert seen["local_files_only"] is True + assert "already cached under another repo id" not in str(excinfo.value) + + +def test_hub_root_offline_names_snapshot_cached_under_another_repo_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + sibling = tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef" + sibling.mkdir(parents=True) + _write_parquet_placeholder(sibling) + _stub_failing_hub(monkeypatch, tmp_path) + + with pytest.raises(FileNotFoundError) as excinfo: + ensure_videomme_hub_root("lmms-lab/Video-MME") + + assert str(sibling.resolve()) in str(excinfo.value) + + +def test_cached_videomme_snapshots_skips_partial_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from huggingface_hub import constants as hf_constants + + (tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef").mkdir(parents=True) + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + + assert cached_videomme_snapshots() == [] + + +def test_extract_videos_rejects_empty_tree(tmp_path: Path) -> None: + (tmp_path / "video").mkdir() + + with pytest.raises(FileNotFoundError): + ensure_videomme_videos_extracted(tmp_path) + assert not (tmp_path / ".videomme_videos_extracted").exists() + + +def test_nested_unzipped_videos_are_discovered(tmp_path: Path) -> None: + nested = tmp_path / "videos" / "videos_chunked_01" / "data" / "xyz.mp4" + nested.parent.mkdir(parents=True) + nested.write_bytes(b"fake") + + assert videomme_local_video_dir(tmp_path) == tmp_path / "videos" + + +def test_extract_characters_regex_matches_official_prefixes() -> None: + assert extract_characters_regex("The best answer is B.") == "B" + assert extract_characters_regex("Answer: C") == "C" + assert extract_characters_regex("I think D is correct") == "D" + assert extract_characters_regex("no letter here at all in this long sentence") is None + + +def test_normalize_gold_answer() -> None: + assert normalize_gold_answer("a") == "A" + assert normalize_gold_answer("B. content") == "B" + assert normalize_gold_answer("") is None + + +class _Out: + def __init__(self, text: str, *, success: bool = True) -> None: + self.generated_text = text + self.success = success + self.error = "" if success else "http fail" + + +def test_compute_videomme_accuracy_metrics() -> None: + reqs = [ + VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_gold_answer="A", + videomme_video_id="v1", + videomme_question_id="q1", + videomme_duration="short", + videomme_domain="Knowledge", + videomme_sub_category="Science", + videomme_task_type="QA", + ), + VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_gold_answer="B", + videomme_video_id="v2", + videomme_question_id="q2", + videomme_duration="long", + videomme_domain="Knowledge", + videomme_sub_category="Science", + videomme_task_type="QA", + ), + ] + metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("The answer is C")]) + assert metrics is not None + assert metrics["videomme_correct"] == 1 + assert metrics["videomme_evaluated_ok"] == 2 + assert metrics["videomme_accuracy"] == 0.5 + assert metrics["videomme_submitted"] == 2 + assert metrics["videomme_unique_question_ids"] == 2 + assert _validate_videomme(metrics, min_accuracy=0.4) == [] + assert "videomme_eval_items" not in metrics + assert metrics["videomme_per_duration_accuracy"]["short"] == 1.0 + assert metrics["videomme_per_duration_accuracy"]["long"] == 0.0 + + +def test_compute_videomme_accuracy_metrics_http_fail_excluded_from_default() -> None: + reqs = [ + VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_gold_answer="A", + videomme_question_id="q1", + ), + VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_gold_answer="B", + videomme_question_id="q2", + ), + ] + metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("", success=False)]) + assert metrics is not None + assert metrics["videomme_accuracy"] == 1.0 + assert metrics["videomme_accuracy_incl_http_fail"] == 0.5 + assert metrics["videomme_request_failed"] == 1 + errs = _validate_videomme(metrics, min_accuracy=0.68) + assert any("videomme_request_failed" in err for err in errs) + + +def test_compute_videomme_accuracy_metrics_saves_eval_items() -> None: + req = VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_gold_answer="A", + videomme_question_id="q1", + ) + metrics = compute_videomme_accuracy_metrics([req], [_Out("A")], include_per_item=True) + assert metrics is not None + assert metrics["videomme_eval_items"][0]["question_id"] == "q1" + + +def test_videomme_save_eval_items_cli_sets_env(monkeypatch: pytest.MonkeyPatch) -> None: + from vllm_omni.benchmarks import serve + + seen: dict[str, str | None] = {} + + async def fake_main_async(args: Namespace) -> dict[str, str]: + seen["env"] = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS") + return {"ok": "1"} + + monkeypatch.delenv("VIDEOMME_SAVE_EVAL_ITEMS", raising=False) + monkeypatch.setattr(serve, "main_async", fake_main_async) + + args = Namespace( + videomme_save_eval_items=True, + seed_tts_wer_eval=False, + seed_tts_wer_save_items=False, + daily_omni_save_eval_items=False, + omni_request_timeout_s=None, + endpoint=None, + backend="openai-chat-omni", + explicit_keys=frozenset(), + extra_body=None, + print_stage=False, + dataset_name="videomme", + ) + assert serve.main(args) == {"ok": "1"} + assert seen["env"] == "1" + + +def _file_uri_path(url: str) -> Path: + parsed = urlparse(url) + assert parsed.scheme == "file" + return Path(unquote(parsed.path)) + + +def test_relative_video_dir_emits_absolute_file_uris_cold_and_warm( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + videos = tmp_path / "videos" + videos.mkdir() + (videos / "vid1.mp4").write_bytes(b"fake") + + def fake_extract(cls, video_path, *, include_audio, max_num_frames): # noqa: ANN001 + return [Image.new("RGB", (4, 4), color="red")], [] + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None) + monkeypatch.setattr(VideoMMEDataset, "_extract_frames_and_audio", classmethod(fake_extract)) + + ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1) + assert ds.video_dir == videos.resolve() + + cold = ds._get_minicpm_frame_parts("vid1", include_audio=False) + assert cold is not None + cold_path = _file_uri_path(cold[0]["image_url"]["url"]) + assert cold_path.is_absolute() + assert cold_path.is_file() + + warm_ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1) + warm = warm_ds._get_minicpm_frame_parts("vid1", include_audio=False) + assert warm is not None + warm_path = _file_uri_path(warm[0]["image_url"]["url"]) + assert warm_path.is_absolute() + assert warm_path == cold_path + + +def test_videomme_repo_from_args_honors_custom_hub_id() -> None: + args = Namespace(dataset_name="videomme", dataset_path="my-org/custom-videomme", hf_name=None) + assert _videomme_repo_from_args(args, explicit=True) == "my-org/custom-videomme" + + +def test_videomme_repo_from_args_defaults_when_path_omitted() -> None: + args = Namespace(dataset_name="videomme", dataset_path=None, hf_name=None) + assert _videomme_repo_from_args(args, explicit=True) is None + + +def test_videomme_repo_from_args_rejects_unsupported_path() -> None: + args = Namespace(dataset_name="videomme", dataset_path="not-a-local-dir-or-hub-id", hf_name=None) + with pytest.raises(ValueError, match="Unsupported Video-MME"): + _videomme_repo_from_args(args, explicit=True) + + +def test_get_samples_rejects_unsupported_videomme_dataset_path() -> None: + args = Namespace( + dataset_name="videomme", + dataset_path="not-a-local-dir-or-hub-id", + hf_name=None, + backend="openai-chat-omni", + seed=0, + ) + with pytest.raises(ValueError, match="Unsupported Video-MME"): + get_samples(args, None) + + +def test_hf_dataset_name_does_not_auto_detect_custom_repo() -> None: + args = Namespace(dataset_name="hf", dataset_path="my-org/custom-videomme", hf_name=None) + assert _videomme_repo_from_args(args) is None + assert _looks_like_hf_dataset_id("my-org/custom-videomme") + assert not _looks_like_hf_dataset_id("./videos") + official = Namespace(dataset_name="hf", dataset_path="lmms-eval/Video-MME", hf_name=None) + assert _videomme_repo_from_args(official) == "lmms-eval/Video-MME" + + +@pytest.mark.parametrize("max_frames", [0, -1]) +def test_rejects_nonpositive_max_frames(max_frames: int, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None) + with pytest.raises(ValueError, match="max_frames must be positive"): + VideoMMEDataset(parquet_path="unused.parquet", max_frames=max_frames) + + +def test_frame_cache_rebuild_overwrites_incomplete_files(tmp_path: Path) -> None: + frame = tmp_path / "frame_0000.jpg" + frame.write_bytes(b"interrupted JPEG") + VideoMMEDataset._emit_media(tmp_path, frame.name, b"complete JPEG", "image_url", "image/jpeg") + assert frame.read_bytes() == b"complete JPEG" + + +def test_frame_cache_rejects_empty_media(tmp_path: Path) -> None: + (tmp_path / "manifest.json").write_text('{"count": 1}', encoding="utf-8") + (tmp_path / "frame_0000.jpg").touch() + dataset = object.__new__(VideoMMEDataset) + assert dataset._cached_parts_from_disk(tmp_path, include_audio=False) is None + + +@pytest.mark.parametrize("case", ["duplicates", "missing_gold"]) +def test_accuracy_gate_rejects_incomplete_question_coverage(case: str) -> None: + def request(question_id: str, gold: str) -> VideoMMESampleRequest: + return VideoMMESampleRequest( + prompt="q", + prompt_len=1, + expected_output_len=8, + videomme_question_id=question_id, + videomme_gold_answer=gold, + ) + + requests = [ + request("q1", "A"), + request("q1" if case == "duplicates" else "q2", "" if case == "missing_gold" else "A"), + ] + metrics = compute_videomme_accuracy_metrics(requests, [_Out("A"), _Out("A")]) + assert metrics is not None + assert metrics["videomme_accuracy"] == 1.0 + errors = _validate_videomme(metrics, min_accuracy=0.68) + assert any(("unique" if case == "duplicates" else "no_gold") in error for error in errors) + + +@pytest.fixture +def videomme_mirror(tmp_path: Path) -> Path: + import pandas as pd + + parquet = tmp_path / "videomme" / "test-00000-of-00001.parquet" + parquet.parent.mkdir() + pd.DataFrame( + [ + { + "videoID": f"v{i}", + "question_id": f"q{i}", + "question": "Which option is correct?", + "options": ["A. first", "B. second", "C. third", "D. fourth"], + "answer": "A", + "duration": duration, + } + for i, duration in enumerate(("short", "long"), start=1) + ] + ).to_parquet(parquet) + videos = tmp_path / "video" + videos.mkdir() + for i in (1, 2): + (videos / f"v{i}.mp4").write_bytes(b"video") + return tmp_path + + +def _sample_video_urls(dataset: VideoMMEDataset, monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> list: + monkeypatch.setattr( + "vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer + ) + return dataset.sample(Namespace(encode=lambda text: [1]), **kwargs) + + +@pytest.mark.parametrize("no_oversample", [True, False]) +def test_missing_video_cannot_pass_accuracy_gate( + videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch, no_oversample: bool +) -> None: + (videomme_mirror / "video" / "v2.mp4").unlink() + dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", disable_shuffle=True) + requests = _sample_video_urls(dataset, monkeypatch, num_requests=2, no_oversample=no_oversample) + metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests]) + assert metrics is not None + assert metrics["videomme_accuracy"] == 1.0 + assert metrics["videomme_submitted"] == (1 if no_oversample else 2) + assert metrics["videomme_skipped_rows"] == 1 + assert any("videomme_skipped_rows" in error for error in _validate_videomme(metrics, min_accuracy=0.68)) + + +def test_complete_duration_subset_can_pass_accuracy_gate( + videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", duration_filter="short") + requests = _sample_video_urls(dataset, monkeypatch, num_requests=2700, no_oversample=True) + metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests]) + assert metrics is not None + assert metrics["videomme_submitted"] == 1 + assert metrics["videomme_skipped_rows"] == 0 + assert _validate_videomme(metrics, min_accuracy=0.68) == [] + + +def test_dataset_direct_hub_loading_resolves_video_assets( + videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + downloaded = [] + + def download(repo_id: str) -> Path: + downloaded.append(repo_id) + return videomme_mirror + + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.ensure_videomme_hub_root", download) + dataset = VideoMMEDataset(dataset_path="my-org/custom-videomme", pack_mode="video_url") + assert downloaded == ["my-org/custom-videomme"] + assert dataset.video_dir == videomme_mirror / "video" + assert len(dataset.data) == 2 + + +def test_local_source_with_explicit_parquet_keeps_video_root(videomme_mirror: Path) -> None: + dataset = VideoMMEDataset( + dataset_path=str(videomme_mirror), + parquet_path=str(videomme_local_parquet(videomme_mirror)), + pack_mode="video_url", + ) + assert dataset.video_dir == videomme_mirror / "video" + assert len(dataset.data) == 2 + + +def test_local_source_without_parquet_fails_locally(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def unexpected_hub(*args, **kwargs): + pytest.fail("An explicitly selected local mirror must not fall back to Hub loading") + + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.load_dataset", unexpected_hub) + with pytest.raises(FileNotFoundError, match="No Video-MME parquet"): + VideoMMEDataset(dataset_path=str(tmp_path)) + + +def test_get_samples_prefers_local_root_over_hf_name(videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch) -> None: + args = Namespace( + dataset_name="videomme", + dataset_path=str(videomme_mirror), + hf_name="unused-hf-name", + backend="openai-chat-omni", + seed=0, + videomme_pack_mode="video_url", + num_prompts=2, + request_id_prefix="", + no_oversample=True, + ) + monkeypatch.setattr( + "vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer + ) + requests = get_samples(args, Namespace(encode=lambda text: [1])) + assert len(requests) == 2 + assert {request.videomme_question_id for request in requests} == {"q1", "q2"} + + +def test_extraction_replaces_truncated_video(tmp_path: Path) -> None: + video = tmp_path / "video" / "v1.mp4" + video.parent.mkdir() + video.write_bytes(b"partial") + with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive: + archive.writestr("nested/v1.mp4", b"complete video") + assert ensure_videomme_videos_extracted(tmp_path) == video.parent + assert video.read_bytes() == b"complete video" + + +def test_failed_extraction_does_not_publish_partial_video(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive: + archive.writestr("v1.mp4", b"complete video") + + def interrupted_copy(src, dst): + dst.write(b"partial") + raise OSError("interrupted") + + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.shutil.copyfileobj", interrupted_copy) + with pytest.raises(OSError, match="interrupted"): + ensure_videomme_videos_extracted(tmp_path) + assert list((tmp_path / "video").iterdir()) == [] + assert not (tmp_path / ".videomme_videos_extracted").exists() diff --git a/tests/config/test_environment_variables.py b/tests/config/test_environment_variables.py index 039073303..e6e61dff5 100644 --- a/tests/config/test_environment_variables.py +++ b/tests/config/test_environment_variables.py @@ -168,7 +168,7 @@ def test_inventory_matches_reviewed_snapshot_counts(): EnvironmentVariableCategory.INHERITED_VLLM: 20, EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27, EnvironmentVariableCategory.MODEL_SPECIFIC: 57, - EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20, + EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21, EnvironmentVariableCategory.INTERNAL: 2, } diff --git a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py index d612b96a5..a900e43d5 100644 --- a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py +++ b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs. +"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs. Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR`` point to existing files, those are used with inline video. Otherwise the benchmark falls back to @@ -10,6 +10,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand. +Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` / +``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id +``lmms-eval/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives. + Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver. """ @@ -26,6 +30,7 @@ from typing import Any, Protocol DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni" DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval" +DEFAULT_VIDEOMME_HF_REPO = "lmms-eval/Video-MME" class OmniBenchServerEndpoint(Protocol): @@ -41,6 +46,7 @@ def build_acc_benchmark_cli_argv( *, skip_seed: bool, skip_daily: bool, + skip_videomme: bool = True, num_prompts: int | None = None, max_concurrency: int | None = None, ) -> list[str]: @@ -50,6 +56,9 @@ def build_acc_benchmark_cli_argv( ``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` / ``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env. + + Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS + callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it. """ n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts) n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency) @@ -72,6 +81,12 @@ def build_acc_benchmark_cli_argv( argv.append("--skip-seed-tts") if skip_daily: argv.append("--skip-daily-omni") + if skip_videomme: + argv.append("--skip-videomme") + else: + argv.append("--run-videomme") + repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO + argv.extend(["--videomme-repo", repo]) return argv @@ -101,6 +116,28 @@ def daily_omni_bench_argv() -> list[str]: ] +def videomme_bench_argv() -> list[str]: + """CLI args for Video-MME (after ``vllm bench serve --omni``). + + Same local-vs-Hub rule as :func:`seed_tts_bench_argv`: + + * If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory, + use that local mirror as ``--dataset-path``. + * Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-eval/Video-MME``); the child + bench downloads via ``huggingface_hub.snapshot_download``. + """ + source = ( + os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() + or os.environ.get("VIDEOMME_ROOT", "").strip() + or os.environ.get("VLLM_VIDEOMME_REPO", "").strip() + or DEFAULT_VIDEOMME_HF_REPO + ) + path = Path(source).expanduser() + if path.is_dir(): + source = str(path.resolve()) + return ["--dataset-name", "videomme", "--dataset-path", source] + + def seed_tts_bench_argv(*, locale: str = "en") -> list[str]: """CLI args for Seed-TTS (after ``vllm bench serve --omni``).""" dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip() diff --git a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py index 2bacfa798..fab9ea590 100644 --- a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py +++ b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Accuracy (and light perf) checks for Qwen3-Omni via ``vllm bench serve --omni``. The standalone CLI uses small ``--num-prompts`` / ``--max-concurrency`` defaults suitable for @@ -11,6 +12,9 @@ H100 / MI325) because they launch the live Omni server inside the test. run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``). 2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used (WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`). +3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers + stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is + **0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``). Prerequisites ------------- @@ -21,7 +25,8 @@ Prerequisites On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script only drives the **client** benchmark. -* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``). +* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` / + ``videomme``). * **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env), the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child** @@ -35,6 +40,12 @@ Prerequisites pip install 'vllm-omni[dev]' +* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-eval/Video-MME`` (override with + ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via + ``huggingface_hub.snapshot_download``. Pass an existing local root with ``--videomme-dataset-path`` / + ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without + ``--allowed-local-media-path``, pass ``--videomme-inline-local-video``. + Examples -------- Pytest (same checks; needs a running server):: @@ -55,6 +66,24 @@ Skip one suite, tighten gates:: --skip-daily-omni \\ --max-seed-tts-mean-wer 0.35 \\ --min-seed-tts-mean-sim 0.75 + +MiniCPM-o Video-MME (w/o subs). Start the server with the 96-image stage 0 +limit first; replace /path/to/Video-MME with the local dataset root:: + + vllm serve openbmb/MiniCPM-o-4_5 --omni \\ + --allowed-local-media-path /path/to/Video-MME \\ + --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}' + +Then run the opt-in evaluation in another terminal:: + + python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\ + --model openbmb/MiniCPM-o-4_5 \\ + --videomme-dataset-path /path/to/Video-MME \\ + --skip-daily-omni --skip-seed-tts --run-videomme \\ + --videomme-pack-mode minicpm-frames \\ + --videomme-max-frames 96 \\ + --temperature 0 --output-len 128 \\ + --min-videomme-accuracy 0.68 """ from __future__ import annotations @@ -75,6 +104,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import ( load_benchmark_result, run_vllm_bench_subprocess, seed_tts_bench_argv, + videomme_bench_argv, ) _REPO_ROOT = Path(__file__).resolve().parents[4] @@ -102,6 +132,42 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None) return errs +def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]: + errs: list[str] = [] + acc = result.get("videomme_accuracy") + if acc is None: + errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).") + return errs + ev = int(result.get("videomme_evaluated_ok", 0) or 0) + if ev <= 0: + errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.") + failed = int(result.get("videomme_request_failed", 0) or 0) + if failed > 0: + errs.append( + f"videomme_request_failed={failed}; the accuracy gate requires every " + "gold-labeled request to complete HTTP successfully " + "(videomme_accuracy excludes HTTP failures)." + ) + submitted = int(result.get("videomme_submitted", 0) or 0) + unique = int(result.get("videomme_unique_question_ids", 0) or 0) + if submitted <= 0 or unique != submitted: + errs.append( + f"Video-MME coverage is incomplete: submitted={submitted}, unique={unique}; " + "the accuracy gate requires distinct questions, not oversampled replacements." + ) + skipped = int(result.get("videomme_skipped_rows", 0) or 0) + if skipped: + errs.append( + f"videomme_skipped_rows={skipped}; fix missing or unreadable source rows before accuracy evaluation." + ) + no_gold = int(result.get("videomme_no_gold", 0) or 0) + if no_gold: + errs.append(f"videomme_no_gold={no_gold}; every submitted question must have a gold answer.") + if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy): + errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}") + return errs + + def _validate_seed_tts( result: dict[str, Any], *, @@ -130,7 +196,7 @@ def _validate_seed_tts( def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None: - """Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``.""" + """Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers.""" repo = getattr(ns, "daily_omni_repo", None) if repo is not None and str(repo).strip(): os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip() @@ -138,6 +204,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None: os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve()) if ns.daily_omni_video_dir is not None: os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve()) + vm_repo = getattr(ns, "videomme_repo", None) + if vm_repo is not None and str(vm_repo).strip(): + os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip() + vm_path = getattr(ns, "videomme_dataset_path", None) + if vm_path is not None and str(vm_path).strip(): + raw = str(vm_path).strip() + p = Path(raw).expanduser() + os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw if ns.seed_tts_dataset_path is not None: # ``--seed-tts-dataset-path`` accepts either a local directory or a # Hugging Face repo id. Only resolve to an absolute filesystem path @@ -157,6 +231,9 @@ def _preserve_benchmark_dataset_env() -> Any: "VLLM_DAILY_OMNI_REPO", "VLLM_DAILY_OMNI_QA_JSON", "VLLM_DAILY_OMNI_VIDEO_DIR", + "VLLM_VIDEOMME_REPO", + "VLLM_VIDEOMME_DATASET_PATH", + "VIDEOMME_ROOT", "VLLM_SEED_TTS_DATASET_PATH", "SEED_TTS_ROOT", ) @@ -225,6 +302,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path: return out +def run_videomme(ns: argparse.Namespace, vllm: str) -> Path: + ns.result_dir.mkdir(parents=True, exist_ok=True) + tag = datetime.now().strftime("%Y%m%d-%H%M%S") + result_filename = f"omni_acc_videomme_{tag}.json" + extra = json.loads(ns.videomme_extra_body_json) + argv = ( + _build_common_args(ns, result_filename=result_filename) + + videomme_bench_argv() + + [ + "--disable-shuffle", + "--videomme-pack-mode", + ns.videomme_pack_mode, + "--videomme-duration", + ns.videomme_duration, + "--extra-body", + json.dumps(extra, ensure_ascii=False, separators=(",", ":")), + ] + ) + if ns.videomme_max_frames is not None: + argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))]) + if ns.videomme_use_subtitle: + argv.append("--videomme-use-subtitle") + if ns.videomme_inline_local_video: + argv.append("--videomme-inline-local-video") + if ns.videomme_save_eval_items: + argv.append("--videomme-save-eval-items") + if ns.videomme_video_dir is not None: + argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())]) + if ns.videomme_parquet is not None: + argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())]) + if ns.videomme_subtitle_dir is not None: + argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())]) + print("\n$", vllm, *argv, "\n", flush=True) + run_vllm_bench_subprocess(vllm, argv) + out = Path(ns.result_dir) / result_filename + if not out.is_file(): + raise FileNotFoundError(f"Expected result JSON at {out}") + return out + + def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path: ns.result_dir.mkdir(parents=True, exist_ok=True) tag = datetime.now().strftime("%Y%m%d-%H%M%S") @@ -312,6 +429,20 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument("--skip-daily-omni", action="store_true") p.add_argument("--skip-seed-tts", action="store_true") + # Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged. + p.set_defaults(skip_videomme=True) + p.add_argument( + "--skip-videomme", + action="store_true", + dest="skip_videomme", + help="Skip Video-MME (default).", + ) + p.add_argument( + "--run-videomme", + action="store_false", + dest="skip_videomme", + help="Enable the Video-MME accuracy suite.", + ) p.add_argument( "--daily-omni-repo", @@ -358,6 +489,87 @@ def build_arg_parser() -> argparse.ArgumentParser: "Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.", ) + p.add_argument( + "--videomme-repo", + type=str, + default=None, + help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). " + "Default when no local path is set: lmms-eval/Video-MME. " + "Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.", + ) + p.add_argument( + "--videomme-dataset-path", + type=str, + default=None, + help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub " + "download; Hub ids are also accepted and passed through as --dataset-path. " + "Sets VLLM_VIDEOMME_DATASET_PATH.", + ) + p.add_argument( + "--videomme-parquet", + type=Path, + default=None, + help="Optional explicit path to videomme/test-*.parquet.", + ) + p.add_argument( + "--videomme-video-dir", + type=Path, + default=None, + help="Optional explicit directory of extracted Video-MME videos.", + ) + p.add_argument( + "--videomme-subtitle-dir", + type=Path, + default=None, + help="Optional subtitle directory (used with --videomme-use-subtitle).", + ) + p.add_argument( + "--videomme-pack-mode", + choices=("minicpm-frames", "minicpm-interleave", "video_url"), + default="minicpm-frames", + help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).", + ) + p.add_argument( + "--videomme-max-frames", + type=int, + default=None, + help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).", + ) + p.add_argument( + "--videomme-duration", + choices=("all", "short", "medium", "long"), + default="all", + help="Filter Video-MME by duration bucket (default: all).", + ) + p.add_argument( + "--videomme-use-subtitle", + action="store_true", + help="Prepend subtitle text to the user prompt (Video-MME w/ subs).", + ) + p.add_argument( + "--videomme-inline-local-video", + action="store_true", + help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).", + ) + p.add_argument( + "--videomme-extra-body-json", + default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}', + help="JSON merged into each chat request for Video-MME.", + ) + p.add_argument( + "--videomme-save-eval-items", + action="store_true", + help="Include per-request Video-MME accuracy rows in the saved JSON.", + ) + p.add_argument( + "--min-videomme-accuracy", + type=float, + default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"), + help="Fail when videomme_accuracy is strictly below this threshold (0–1). " + "Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); " + "override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.", + ) + p.add_argument( "--seed-tts-dataset-path", type=str, @@ -427,7 +639,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac def run_acc_benchmark(ns: argparse.Namespace) -> int: - """Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success.""" + """Run Daily-Omni / Video-MME / Seed-TTS client benches against a running server; return 0 on success.""" failed: list[str] = [] with _preserve_benchmark_dataset_env(): @@ -451,6 +663,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int: flush=True, ) + if not ns.skip_videomme: + path = run_videomme(ns, vllm) + print(f"\n[Video-MME] result JSON: {path}", flush=True) + data = load_benchmark_result(path) + errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy) + if errs: + failed.extend([f"[Video-MME] {e}" for e in errs]) + else: + print( + f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} " + f"evaluated_ok={data.get('videomme_evaluated_ok')} " + f"per_duration={data.get('videomme_per_duration_accuracy')}", + flush=True, + ) + if not ns.skip_seed_tts: path = run_seed_tts(ns, vllm) print(f"\n[Seed-TTS] result JSON: {path}", flush=True) diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 6c22b818b..92e96b25a 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -178,7 +178,7 @@ class TestDeployTopology: 0.18, ] assert sum(memory_utilizations) <= 0.9 + 1e-6 - # Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video). + # Keep the default Daily-Omni budget; Video-MME overrides stage 0 explicitly. assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == { "image": 64, "audio": 64, diff --git a/vllm_omni/benchmarks/data_modules/videomme_dataset.py b/vllm_omni/benchmarks/data_modules/videomme_dataset.py new file mode 100644 index 000000000..e2f34e425 --- /dev/null +++ b/vllm_omni/benchmarks/data_modules/videomme_dataset.py @@ -0,0 +1,1008 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Video-MME dataset loader for vLLM-Omni bench serve. + +Video-MME (lmms-eval/Video-MME) is a full-spectrum video MLLM benchmark with +900 videos / 2,700 MCQ pairs across short / medium / long durations. + +This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe +(``o_e_Kit/configs/generation_configs.json``): + +- ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false`` +- ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave`` + +Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame +timestamps match OmniEvalKit ``_sample_video_frame_indices``. +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil +import tempfile +import time +import zipfile +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +import pybase64 as base64 +import regex as re + +try: + from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest +except ImportError: + from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset + from vllm.benchmarks.datasets import SampleRequest +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.hf import get_cached_tokenizer + +try: + from datasets import load_dataset +except ImportError: + load_dataset = None + +from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( + MINICPM_OMNI_SYSTEM_TEXT, + _ListDatasetIterator, + _numpy_to_wav_bytes, + _pil_to_jpeg_bytes, + _uniform_sample_indices, +) + +logger = logging.getLogger(__name__) + +_MINICPM_AUDIO_SR = 16000 + +# OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults. +VIDEOMME_DEFAULT_MAX_FRAMES = 96 +VIDEOMME_SHORT_MAX_FRAMES = 64 +VIDEOMME_DEFAULT_HF_REPO = "lmms-eval/Video-MME" + +#: Decode forward to reach a target this close, seek to the preceding keyframe beyond it. +_SEEK_THRESHOLD_S = 3.0 + +#: How often ``sample`` reports progress while warming a cold frame cache. +_SAMPLE_PROGRESS_INTERVAL_S = 30.0 + +VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"] +VideoMMEDurationFilter = Literal["all", "short", "medium", "long"] + +# OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped; +# media parts are prepended as OpenAI content parts, matching ``build_content``). +VIDEOMME_USER_PROMPT_TEMPLATE = ( + "Carefully read the following question and select the letter corresponding to the " + "correct answer.Highlight the applicable choices without giving explanations.\n" + "{question}\n" + "Options:\n" + "{options}\n" + "Please select the correct answer from the options above. Only respond with the letter." +) + +_OPTION_LETTERS = "ABCDEFGHIJKL" +_OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$") + +_VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi") + + +def _iter_video_files(root: Path) -> Iterator[Path]: + """Yield video files under ``root``, skipping dot-dirs such as the frame cache.""" + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + for name in filenames: + if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES: + yield Path(dirpath) / name + + +def _resolve_hf_cache_snapshot(path: Path) -> Path: + """Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir.""" + snapshots = path / "snapshots" + if not snapshots.is_dir(): + return path + ref = path / "refs" / "main" + if ref.is_file(): + try: + revision = ref.read_text(encoding="utf-8").strip() + except OSError: + revision = "" + # An empty or unreadable ref must not resolve to ``snapshots`` itself, which holds + # no dataset files; fall through to the newest revision below. + if revision and (snapshots / revision).is_dir(): + return snapshots / revision + revisions = [p for p in snapshots.iterdir() if p.is_dir()] + if revisions: + return max(revisions, key=lambda p: p.stat().st_mtime) + return path + + +def absolute_path(value: str | Path | None) -> Path | None: + """Return an absolute path, or ``None`` when ``value`` is empty.""" + if value is None: + return None + raw = str(value).strip() + if not raw: + return None + return Path(raw).expanduser().resolve() + + +def resolve_videomme_local_root(dataset_path: str | None) -> Path | None: + """Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror. + + Hub ids such as ``lmms-eval/Video-MME`` return ``None`` (they are not directories). + """ + raw = (dataset_path or "").strip() + if not raw: + return None + path = Path(raw).expanduser() + if not path.is_dir(): + return None + return _resolve_hf_cache_snapshot(path.resolve()) + + +def cached_videomme_snapshots() -> list[Path]: + """Usable Video-MME snapshots already in the Hub cache, whatever org published them. + + The dataset is mirrored under several ids (``lmms-lab``/``lmms-eval``), so an offline + run routinely requests one while a sibling is what actually sits on disk. + """ + from huggingface_hub import constants as hf_constants + + cache = Path(hf_constants.HF_HUB_CACHE) + if not cache.is_dir(): + return [] + + found: list[Path] = [] + for entry in sorted(cache.iterdir()): + name = entry.name.lower() + if not entry.is_dir() or not name.startswith("datasets--") or not name.endswith("--video-mme"): + continue + snapshot = _resolve_hf_cache_snapshot(entry) + # A repo dir with no parquet is a partial download, not a usable root. + if videomme_local_parquet(snapshot) is not None: + found.append(snapshot) + return found + + +def ensure_videomme_hub_root(repo_id: str) -> Path: + """Download the Video-MME dataset snapshot from Hugging Face and return its root. + + Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`: + Hub ids trigger ``hf_api().snapshot_download``; callers then extract videos via + :func:`ensure_videomme_videos_extracted`. + + Raises: + ImportError: if Hugging Face Hub helpers are not installed. + ValueError: if ``repo_id`` is empty. + FileNotFoundError: if the snapshot is neither cached nor downloadable. + """ + rid = (repo_id or "").strip() + if not rid: + raise ValueError("repo_id is required to download Video-MME from Hugging Face") + + try: + from huggingface_hub import constants as hf_constants + from huggingface_hub.errors import ( + HfHubHTTPError, + HFValidationError, + LocalEntryNotFoundError, + OfflineModeIsEnabled, + ) + + from vllm_omni.transformers_utils.repo_utils import hf_api + except ImportError as e: + raise ImportError( + "Install huggingface_hub to download Video-MME from the Hub, or pass a local " + "--dataset-path / --videomme-video-dir with parquet + extracted videos." + ) from e + + offline = bool(hf_constants.HF_HUB_OFFLINE) + try: + # Pull QA parquet + video/subtitle archives; skip unrelated repo files. + cache = hf_api().snapshot_download( + repo_id=rid, + repo_type="dataset", + # Offline runs otherwise resolve repo metadata over the network first and + # surface a connection failure instead of the missing cache entry. + local_files_only=offline, + allow_patterns=[ + "videomme/**", + "*.parquet", + "videos_chunked_*.zip", + "subtitle.zip", + "subtitle/**", + "video/**", + "videos/**", + ], + ) + # ``HfHubHTTPError`` also covers gated / missing repos; Video-MME requires accepting + # the dataset terms, so a plain token-less run lands here rather than downloading. + except (HFValidationError, HfHubHTTPError, LocalEntryNotFoundError, OfflineModeIsEnabled) as e: + if offline: + reason = f"HF_HUB_OFFLINE is set and it is not cached under {hf_constants.HF_HUB_CACHE}" + else: + reason = f"snapshot_download failed with {type(e).__name__}" + + hints = [ + "Point --dataset-path at a local Video-MME root (a directory holding " + "videomme/test-*.parquet plus video/ or videos_chunked_*.zip), or pass " + "--videomme-parquet / --videomme-video-dir." + ] + cached = cached_videomme_snapshots() + if cached: + hints.append( + "Video-MME is already cached under another repo id: " + ", ".join(str(p) for p in cached) + "." + ) + if offline: + hints.append("Unset HF_HUB_OFFLINE to download it instead.") + + raise FileNotFoundError( + f"Video-MME is unavailable from Hugging Face repo '{rid}': {reason}. " + " ".join(hints) + ) from e + + root = _resolve_hf_cache_snapshot(Path(cache).resolve()) + logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid) + return root + + +def resolve_videomme_root(dataset_path: str | None) -> Path: + """Return a Video-MME root containing parquet and (or zip) video assets. + + * Existing local directories (absolute or relative) are used as-is. + * Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and + downloaded via :func:`ensure_videomme_hub_root` (default + ``lmms-eval/Video-MME``). + """ + local = resolve_videomme_local_root(dataset_path) + if local is not None: + return local + rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO + return ensure_videomme_hub_root(rid) + + +def videomme_local_parquet(root: Path) -> Path | None: + for candidate in ( + root / "videomme" / "test-00000-of-00001.parquet", + root / "test-00000-of-00001.parquet", + ): + if candidate.is_file(): + return candidate + return None + + +def videomme_local_video_dir(root: Path) -> Path | None: + """Return a directory already holding extracted videos, flat or nested. + + Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``, + so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of + the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable. + """ + for name in ("video", "videos", "data"): + candidate = root / name + if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None: + return candidate + return None + + +def videomme_local_subtitle_dir(root: Path) -> Path | None: + candidate = root / "subtitle" + return candidate if candidate.is_dir() else None + + +def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None: + """Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout).""" + name = os.path.basename(member) + if not name: + return + dest = dest_dir / name + if dest.is_file() and dest.stat().st_size == zf.getinfo(member).file_size: + return + dest_dir.mkdir(parents=True, exist_ok=True) + # Publish only after ZipFile has read and checked the complete member. + with tempfile.TemporaryDirectory(prefix=".videomme-", dir=dest_dir) as temporary: + temporary_path = Path(temporary) / name + with zf.open(member) as src, temporary_path.open("wb") as out: + shutil.copyfileobj(src, out) + temporary_path.replace(dest) + + +def ensure_videomme_videos_extracted(root: Path) -> Path: + """Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``.""" + video_dir = root / "video" + marker = root / ".videomme_videos_extracted" + if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None: + return video_dir + + # Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it. + existing = videomme_local_video_dir(root) + if existing is not None and existing != video_dir: + return existing + + zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()] + if not zips and not video_dir.is_dir(): + raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip") + + video_dir.mkdir(parents=True, exist_ok=True) + for zp in zips: + logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir) + with zipfile.ZipFile(zp, "r") as zf: + for member in zf.namelist(): + if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")): + _unzip_member_flat(zf, member, video_dir) + + if next(_iter_video_files(video_dir), None) is None: + # Marking an empty tree as extracted would make every later request log + # "video not found" and yield a run with zero samples. + raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip") + marker.write_text("ok", encoding="utf-8") + return video_dir + + +def ensure_videomme_subtitles_extracted(root: Path) -> Path | None: + """Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present.""" + subtitle_dir = root / "subtitle" + if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None: + return subtitle_dir + + zip_path = root / "subtitle.zip" + if not zip_path.is_file(): + return subtitle_dir if subtitle_dir.is_dir() else None + + subtitle_dir.mkdir(parents=True, exist_ok=True) + logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir) + with zipfile.ZipFile(zip_path, "r") as zf: + for member in zf.namelist(): + if member.lower().endswith(".srt"): + _unzip_member_flat(zf, member, subtitle_dir) + return subtitle_dir + + +def _probe_video_duration(video_path: Path) -> float: + """Container duration in seconds, without decoding the stream. + + Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to + an hour, and ``stream.frames == 0`` containers would otherwise force a full decode + just to learn the duration. + """ + import av + + with av.open(str(video_path)) as container: + stream = container.streams.video[0] + duration: float | None = None + if stream.duration is not None and stream.time_base is not None: + duration = float(stream.duration * stream.time_base) + elif container.duration is not None: + duration = container.duration / av.time_base + elif stream.frames and stream.average_rate: + duration = stream.frames / float(stream.average_rate) + + if not duration or duration <= 0: + raise ValueError(f"Could not determine duration of {video_path}") + return duration + + +def _decode_frames_at_timestamps( + video_path: Path, + timestamps: list[float], + *, + seek_threshold_s: float = _SEEK_THRESHOLD_S, +) -> list[Any]: + """Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp. + + A single forward pass over the stream — the Daily-Omni approach — is not affordable + here: Video-MME samples points spread across the whole video, so the last target sits + near the end and every frame of an hour-long file would be decoded. Targets within + ``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper + than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding + keyframe first. + """ + import av + + if not timestamps: + return [] + + images: list[Any] = [] + with av.open(str(video_path)) as container: + stream = container.streams.video[0] + stream.thread_type = "AUTO" + time_base = float(stream.time_base) if stream.time_base else 0.0 + start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0 + + decoder = None + frame = None + frame_time = -1.0 + image = None + + for ts in timestamps: + if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s: + if time_base > 0: + container.seek(int((ts + start_s) / time_base), stream=stream, backward=True) + else: + container.seek(0, stream=stream, backward=True) + decoder = container.decode(stream) + frame, image, frame_time = None, None, -1.0 + + while frame is None or frame_time < ts: + nxt = next(decoder, None) + if nxt is None: + break + frame, image = nxt, None + if nxt.pts is not None and time_base > 0: + frame_time = float(nxt.pts) * time_base - start_s + + if frame is None: + break + if image is None: + image = frame.to_image() + images.append(image) + + if not images: + raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}") + # Container duration can overshoot the real stream; repeat the last frame (decord clamping). + images.extend([images[-1]] * (len(timestamps) - len(images))) + return images + + +@dataclass +class VideoMMESampleRequest(SampleRequest): + """``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring.""" + + videomme_gold_answer: str = "" + videomme_video_id: str = "" + videomme_question_id: str = "" + videomme_duration: str = "" + videomme_domain: str = "" + videomme_sub_category: str = "" + videomme_task_type: str = "" + videomme_skipped_rows: int = 0 + omni_extra_body: dict[str, Any] | None = None + omni_chat_messages: list[dict[str, Any]] | None = None + + +class VideoMMEDataset(BenchmarkDataset): + """Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing).""" + + SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO} + DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO + IS_MULTIMODAL = True + DEFAULT_OUTPUT_LEN = 128 + + def __init__( + self, + dataset_path: str | None = None, + dataset_split: str = "test", + random_seed: int = 0, + video_dir: str | None = None, + subtitle_dir: str | None = None, + parquet_path: str | None = None, + pack_mode: VideoMMEPackMode = "minicpm-frames", + max_frames: int | None = None, + duration_filter: VideoMMEDurationFilter = "all", + use_subtitle: bool = False, + inline_local_video: bool = False, + trust_remote_code: bool = False, + dataset_subset: str | None = None, + no_stream: bool = False, + **kwargs, + ) -> None: + if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"): + raise ValueError( + f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}" + ) + if duration_filter not in ("all", "short", "medium", "long"): + raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}") + if parquet_path is None and dataset_path is None: + raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.") + if pack_mode == "minicpm-interleave" and duration_filter != "short": + logger.warning( + "Video-MME 'minicpm-interleave' ports the OmniEvalKit videomme_short recipe. Beyond the " + "short bucket, frames are spread across the whole clip and each paired audio segment covers " + "the gap between them (tens of seconds on medium/long videos), which inflates the audio " + "token count far past the 1s pairing the recipe assumes. Pass --videomme-duration short." + ) + + self.parquet_path = absolute_path(parquet_path) + self.dataset_path = dataset_path + self.dataset_split = dataset_split + self.dataset_subset = dataset_subset + self._hf_streaming = not no_stream + self.video_dir = absolute_path(video_dir) + self.subtitle_dir = absolute_path(subtitle_dir) + self.pack_mode: VideoMMEPackMode = pack_mode + self.duration_filter: VideoMMEDurationFilter = duration_filter + self.use_subtitle = use_subtitle + self.inline_local_video = inline_local_video + self.trust_remote_code = trust_remote_code + self.max_frames = int( + max_frames + if max_frames is not None + else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES) + ) + if self.max_frames <= 0: + raise ValueError("Video-MME max_frames must be positive") + #: In-process memo of content parts; the on-disk frame cache survives across runs. + self._frame_cache: dict[str, list[dict[str, Any]]] = {} + self._video_index: dict[str, Path] | None = None + + super().__init__( + dataset_path=dataset_path, + random_seed=random_seed, + **kwargs, + ) + self.load_data() + logger.info( + "Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, " + "use_subtitle=%s, video_dir=%s", + str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}", + pack_mode, + self.max_frames, + duration_filter, + use_subtitle, + self.video_dir, + ) + + # ------------------------------------------------------------------ loading + + def load_data(self) -> None: + """Resolve local/Hub assets once, shared by the CLI and direct callers.""" + local_root = resolve_videomme_local_root(self.dataset_path) + if local_root is None and self.parquet_path is None and self.video_dir is None: + local_root = resolve_videomme_root(self.dataset_path) + + if local_root is not None: + if self.parquet_path is None: + self.parquet_path = videomme_local_parquet(local_root) + if self.parquet_path is None: + raise FileNotFoundError(f"No Video-MME parquet under {local_root}; pass --videomme-parquet.") + if self.video_dir is None: + self.video_dir = ensure_videomme_videos_extracted(local_root) + if self.use_subtitle and self.subtitle_dir is None: + self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root) + + if self.video_dir is None or not self.video_dir.is_dir(): + raise ValueError( + "Video-MME requires --videomme-video-dir pointing at extracted videos, " + "or a local/Hub dataset root containing video/ or videos_chunked_*.zip." + ) + if self.parquet_path is not None: + self._load_from_parquet(self.parquet_path) + else: + self._load_from_huggingface() + + def _load_from_parquet(self, path: Path) -> None: + try: + import pandas as pd + except ImportError as e: + raise ImportError("pandas is required to load Video-MME parquet") from e + if not path.is_file(): + raise FileNotFoundError(f"Video-MME parquet not found: {path}") + df = pd.read_parquet(path) + self._set_rows(df.to_dict(orient="records")) + + def _load_from_huggingface(self) -> None: + if load_dataset is None: + raise ImportError( + "datasets library is required for HuggingFace Video-MME loading. " + "Install with: pip install datasets, or pass --videomme-parquet." + ) + load_kw: dict[str, Any] = { + "split": self.dataset_split, + "streaming": self._hf_streaming, + "trust_remote_code": self.trust_remote_code, + } + if self.dataset_subset is not None: + load_kw["name"] = self.dataset_subset + ds = load_dataset(self.dataset_path, **load_kw) + self._set_rows(list(ds)) + + def _set_rows(self, rows: list[dict[str, Any]]) -> None: + """Apply the duration filter and shuffle, then expose rows as ``self.data``.""" + if self.duration_filter != "all": + want = self.duration_filter.lower() + rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want] + if not getattr(self, "disable_shuffle", False) and self.random_seed is not None: + import random + + rows = rows[:] + random.Random(self.random_seed).shuffle(rows) + self.data = _ListDatasetIterator(rows) + + @staticmethod + def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]: + # The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain + # over the alias keys is not usable here. + options: Any = [] + for key in ("options", "candidates", "choices"): + value = row.get(key) + if value is not None and hasattr(value, "tolist"): + value = value.tolist() + if value is not None and len(value) > 0: + options = value + break + return { + "video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(), + "question_id": str(row.get("question_id") or "").strip(), + "question": str(row.get("question") or "").strip(), + "options": [str(x) for x in list(options)], + "answer": str(row.get("answer") or "").strip(), + "duration": str(row.get("duration") or "").strip(), + "domain": str(row.get("domain") or "").strip(), + "sub_category": str(row.get("sub_category") or "").strip(), + "task_type": str(row.get("task_type") or "").strip(), + } + + # ------------------------------------------------------------------ sampling + + def sample( + self, + tokenizer: TokenizerLike, + num_requests: int, + output_len: int | None = None, + request_id_prefix: str = "", + no_oversample: bool = False, + **kwargs, + ) -> list[SampleRequest]: + if output_len is None: + output_len = self.DEFAULT_OUTPUT_LEN + + sampled: list[SampleRequest] = [] + cached_tokenizer = get_cached_tokenizer(tokenizer) + # A cold frame cache costs a few seconds per video, so the whole set takes tens of + # minutes before the first request goes out; report progress to keep that legible. + started = time.monotonic() + last_report = started + total = min(num_requests, len(self.data)) + skipped_rows = 0 + seen = 0 + for seen, item in enumerate(self.data, start=1): # noqa: B007 + if len(sampled) >= num_requests: + break + req = self._create_sample_request(item, cached_tokenizer, output_len, request_id_prefix, len(sampled)) + if req is not None: + sampled.append(req) + else: + skipped_rows += 1 + now = time.monotonic() + if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S: + logger.info( + "Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed", + seen, + total, + len(sampled), + now - started, + ) + last_report = now + + logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started) + if not sampled and num_requests > 0: + # Oversampling an empty list raises a bare IndexError; the usual cause is a + # video_dir that holds none of the videoIDs referenced by the QA rows. + raise ValueError( + f"Video-MME produced no requests from {seen} QA rows " + f"(duration_filter={self.duration_filter!r}, video_dir={self.video_dir}). " + "Check --videomme-duration and that --videomme-video-dir holds the videoIDs the rows reference." + ) + for request in sampled: + assert isinstance(request, VideoMMESampleRequest) + request.videomme_skipped_rows = skipped_rows + self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample) + unique_questions = {getattr(req, "videomme_question_id", "") or req.request_id for req in sampled} + if len(sampled) > len(unique_questions): + logger.warning( + "Video-MME submitted %d requests covering %d unique questions " + "(missing media is skipped, then the remainder may be reused). " + "Submitted request count is not unique-question coverage.", + len(sampled), + len(unique_questions), + ) + return sampled + + def _create_sample_request( + self, + qa_item: dict[str, Any], + tokenizer: TokenizerLike, + output_len: int, + request_id_prefix: str, + index: int, + ) -> SampleRequest | None: + fields = self._normalize_fields(qa_item) + if not fields["video_id"] or not fields["question"]: + logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"]) + return None + + mm_payload, omni_extra = self._compose_multimodal(fields["video_id"]) + if not mm_payload: + return None + + user_text = self._build_user_prompt(fields) + return VideoMMESampleRequest( + prompt=user_text, + prompt_len=len(tokenizer.encode(user_text)), + expected_output_len=output_len, + multi_modal_data=None, + request_id=f"{request_id_prefix}{index}", + videomme_gold_answer=fields["answer"], + videomme_video_id=fields["video_id"], + videomme_question_id=fields["question_id"], + videomme_duration=fields["duration"], + videomme_domain=fields["domain"], + videomme_sub_category=fields["sub_category"], + videomme_task_type=fields["task_type"], + omni_extra_body=omni_extra, + omni_chat_messages=self._build_openai_messages(mm_payload, user_text), + ) + + # ------------------------------------------------------------------ prompt + + @staticmethod + def _format_options(options: list[str]) -> str: + """Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line). + + The Hub parquet already ships options as ``"A. content"``. Existing labels are + dropped before relabelling so the prompt never reads ``A. A. content``, but only + when *every* option carries one — otherwise contents such as ``A.M. shift`` would + be truncated. + """ + if not options: + return "" + matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options] + relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS)) + lines = [] + for key, raw, m in zip(_OPTION_LETTERS, options, matches): + content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip() + lines.append(f"{key}. {content}") + return "\n".join(lines) + + def _build_user_prompt(self, fields: dict[str, Any]) -> str: + prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format( + question=fields["question"], + options=self._format_options(fields["options"]), + ) + if self.use_subtitle: + subs = self._load_subtitle_text(fields["video_id"]) + if subs: + prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}" + return prompt + + def _load_subtitle_text(self, video_id: str) -> str: + if not self.subtitle_dir: + return "" + path = self.subtitle_dir / f"{video_id}.srt" + if not path.is_file(): + return "" + try: + raw = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + lines = [ + s.replace("\\N", " ") + for s in (line.strip() for line in raw.splitlines()) + if s and not s.isdigit() and "-->" not in s + ] + return "\n".join(lines) + + def _build_openai_messages( + self, + mm_payload: dict[str, Any] | list[dict[str, Any]], + user_text: str, + ) -> list[dict[str, Any]]: + mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload] + messages: list[dict[str, Any]] = [] + if MINICPM_OMNI_SYSTEM_TEXT: + messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]}) + messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]}) + return messages + + # ------------------------------------------------------------------ media + + def _video_path_index(self) -> dict[str, Path]: + """Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses.""" + if self._video_index is None: + index: dict[str, Path] = {} + if self.video_dir is not None: + for path in _iter_video_files(self.video_dir): + index.setdefault(path.stem, path) + logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir) + self._video_index = index + return self._video_index + + def _resolve_local_video_path(self, video_id: str) -> Path | None: + if not self.video_dir or not video_id: + return None + for p in ( + self.video_dir / f"{video_id}.mp4", + self.video_dir / f"{video_id}.MP4", + self.video_dir / f"{video_id}.mkv", + self.video_dir / video_id / f"{video_id}.mp4", + ): + if p.is_file(): + return p + return self._video_path_index().get(video_id) + + def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]: + path = path.expanduser().resolve() + if self.inline_local_video: + b64 = base64.b64encode(path.read_bytes()).decode("ascii") + return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} + return {"type": typ, typ: {"url": path.as_uri()}} + + def _compose_multimodal( + self, + video_id: str, + ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]: + # OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the + # sample carries a video path: 9 slices per frame would blow past max_model_len and + # N would label each frame as a separate picture. + extra: dict[str, Any] = { + "mm_processor_kwargs": { + "use_audio_in_video": False, + "max_slice_nums": 1, + "use_image_id": False, + } + } + + if self.pack_mode == "video_url": + video_path = self._resolve_local_video_path(video_id) + if video_path is None: + logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir) + return None, None + return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra + + parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave") + if not parts: + return None, None + return parts, extra + + def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path | None: + """Where the on-disk frame dump lives, or ``None`` when there is no ``video_dir`` to host it.""" + if self.video_dir is None: + return None + tag = f"f{self.max_frames}{'_av' if include_audio else ''}" + return self.video_dir / ".minicpm_videomme_frames" / video_id / tag + + def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None: + """Rebuild content parts from a previous run's frame dump, or ``None`` on a miss.""" + manifest = cache_dir / "manifest.json" + if not manifest.is_file(): + return None + try: + count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"]) + except (OSError, ValueError, KeyError, TypeError): + return None + + parts: list[dict[str, Any]] = [] + for i in range(count): + frame_path = cache_dir / f"frame_{i:04d}.jpg" + if not frame_path.is_file() or frame_path.stat().st_size == 0: + return None + parts.append({"type": "image_url", "image_url": {"url": frame_path.expanduser().resolve().as_uri()}}) + if include_audio: + audio_path = cache_dir / f"audio_{i:04d}.wav" + if not audio_path.is_file() or audio_path.stat().st_size == 0: + return None + parts.append({"type": "audio_url", "audio_url": {"url": audio_path.expanduser().resolve().as_uri()}}) + return parts or None + + def _get_minicpm_frame_parts( + self, + video_id: str, + *, + include_audio: bool, + ) -> list[dict[str, Any]] | None: + cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}" + cached = self._frame_cache.get(cache_key) + if cached is not None: + return cached + + # Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS. + cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio) + if cache_dir is not None: + disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio) + if disk_parts is not None: + self._frame_cache[cache_key] = disk_parts + return disk_parts + + video_path = self._resolve_local_video_path(video_id) + if video_path is None: + logger.warning( + "Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r", + self.video_dir, + video_id, + ) + return None + + try: + frames, audio_segments = self._extract_frames_and_audio( + video_path, + include_audio=include_audio, + max_num_frames=self.max_frames, + ) + except Exception: + logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path) + return None + + if not frames: + logger.warning("No frames extracted for video_id=%r", video_id) + return None + if include_audio and len(audio_segments) != len(frames): + n = min(len(frames), len(audio_segments)) + frames, audio_segments = frames[:n], audio_segments[:n] + + if cache_dir is not None: + cache_dir.mkdir(parents=True, exist_ok=True) + # A failed rebuild must not leave a manifest advertising a complete cache. + (cache_dir / "manifest.json").unlink(missing_ok=True) + + parts: list[dict[str, Any]] = [] + for i, frame in enumerate(frames): + parts.append( + self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg") + ) + if include_audio: + parts.append( + self._emit_media( + cache_dir, + f"audio_{i:04d}.wav", + _numpy_to_wav_bytes(audio_segments[i]), + "audio_url", + "audio/wav", + ) + ) + + if cache_dir is not None: + (cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8") + self._frame_cache[cache_key] = parts + logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts)) + return parts + + @staticmethod + def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]: + """Write ``payload`` to the frame cache and return a file URL part, or inline base64.""" + if cache_dir is None: + b64 = base64.b64encode(payload).decode("ascii") + return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} + path = (cache_dir / name).expanduser().resolve() + # Reaching this path means the cache missed validation. Replace residual + # files as well: a nonempty file may be from an interrupted write. + path.write_bytes(payload) + return {"type": typ, typ: {"url": path.as_uri()}} + + @staticmethod + def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]: + """Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps.""" + if duration > max_num_frames: + return [round(i * 0.1, 1) for i in _uniform_sample_indices(int(duration / 0.1), max_num_frames)] + # OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame. + return [float(i) for i in range(max(1, int(duration)))] + + @classmethod + def _extract_frames_and_audio( + cls, + video_path: Path, + *, + include_audio: bool, + max_num_frames: int, + ) -> tuple[list[Any], list[Any]]: + import numpy as np + from vllm.multimodal.media.audio import load_audio + + duration = _probe_video_duration(video_path) + timestamps = cls._sample_timestamps(duration, max_num_frames) + frames = _decode_frames_at_timestamps(video_path, timestamps) + + audio_segments: list[Any] = [] + if include_audio: + audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True) + for i, start_time in enumerate(timestamps): + end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration + segment = audio_np[int(start_time * sr) : int(end_time * sr)] + if i == len(timestamps) - 1 and len(segment) < 1600: + segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)]) + if len(segment) == 0: + segment = np.zeros(1600, dtype=np.float32) + audio_segments.append(segment) + + return frames, audio_segments diff --git a/vllm_omni/benchmarks/data_modules/videomme_eval.py b/vllm_omni/benchmarks/data_modules/videomme_eval.py new file mode 100644 index 000000000..cb8107e7a --- /dev/null +++ b/vllm_omni/benchmarks/data_modules/videomme_eval.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve. + +Answer extraction ports ``extract_characters_regex`` from the official Video-MME +script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval). + +**Denominator:** responses where no A–D can be extracted are counted as wrong, which +matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The +official standalone script instead drops them from ``answered``; ``videomme_parse_failed`` +is reported so that view can be recovered. +""" + +from __future__ import annotations + +from typing import Any + +import regex as re +from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput + +from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest + +_VALID = frozenset("ABCD") + +VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long") + +#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns). +_DIMENSIONS: dict[str, str] = { + "duration": "videomme_duration", + "domain": "videomme_domain", + "sub_category": "videomme_sub_category", + "task": "videomme_task_type", +} + +# Official prefix list, with the two accidental string concatenations in the upstream +# tuple (missing commas after "The best option is" / "Best answer:") split apart. +_ANSWER_PREFIXES = ( + "The best answer is", + "The correct answer is", + "The answer is", + "The answer", + "The best option is", + "The correct option is", + "Best answer:", + "Best option:", + "Answer:", + "Option:", + "The correct answer", + "The correct option", +) + + +def extract_characters_regex(text: str | None) -> str | None: + """Port of official Video-MME / lmms-eval ``extract_characters_regex``.""" + if not text: + return None + s = str(text).strip() + for prefix in _ANSWER_PREFIXES: + s = s.replace(prefix, "") + if len(s.split()) > 10 and not re.search(r"[ABCD]", s): + return None + match = re.search(r"[ABCD]", s) + return match.group(0) if match else None + + +def normalize_gold_answer(gold: str) -> str | None: + g = (gold or "").strip().upper() + if len(g) == 1 and g in _VALID: + return g + m = re.search(r"([ABCD])\b", g) + return m.group(1).upper() if m else None + + +def _bucket(req: VideoMMESampleRequest, dimension: str) -> str: + value = (getattr(req, _DIMENSIONS[dimension]) or "").strip() + if dimension == "duration": + return value.lower() + return value or "unknown" + + +def compute_videomme_accuracy_metrics( + input_requests: list[Any], + outputs: list[RequestFuncOutput], + *, + include_per_item: bool = False, +) -> dict[str, Any] | None: + """If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats. + + Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are + excluded from ``videomme_accuracy`` and counted in + ``videomme_accuracy_incl_http_fail`` instead. The accuracy wrapper gate + (``_validate_videomme``) additionally rejects HTTP failures, duplicate question + IDs, skipped source rows and missing gold answers. Direct + ``vllm bench serve`` still reports both metrics without that completeness + check. + + ``videomme_submitted`` / ``videomme_unique_question_ids`` show when + oversampling made the request count differ from unique question coverage. + ``videomme_skipped_rows`` counts source rows skipped during sampling (for + example, missing or undecodable media), before any oversampling. + """ + if not input_requests or len(input_requests) != len(outputs): + return None + if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests): + return None + + stats: dict[str, dict[str, dict[str, int]]] = { + dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {}) + for dim in _DIMENSIONS + } + items: list[dict[str, Any]] = [] + correct = evaluated = no_gold = request_failed = parse_failed = 0 + + for req, out in zip(input_requests, outputs, strict=True): + assert isinstance(req, VideoMMESampleRequest) + gold_raw = (req.videomme_gold_answer or "").strip() + buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS} + row: dict[str, Any] = { + "request_id": req.request_id, + "video_id": req.videomme_video_id, + "question_id": req.videomme_question_id, + **buckets, + } + + if not gold_raw: + no_gold += 1 + if include_per_item: + items.append({**row, "skipped": True, "reason": "no_gold"}) + continue + + evaluated += 1 + gold_norm = normalize_gold_answer(gold_raw) + pred = extract_characters_regex(out.generated_text) if out.success else None + is_correct = pred is not None and pred.upper() == (gold_norm or gold_raw).upper() + + if not out.success: + request_failed += 1 + else: + if pred is None: + parse_failed += 1 + # Only successful responses enter the per-dimension denominators, mirroring + # the overall videomme_accuracy definition. + for dim, key in buckets.items(): + bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0}) + bucket["total"] += 1 + bucket["correct"] += int(is_correct) + + if is_correct: + correct += 1 + + if include_per_item: + items.append( + { + **row, + "gold": gold_raw, + "gold_normalized": gold_norm, + "predicted": pred, + "correct": is_correct, + "parse_failed": out.success and pred is None, + **({"error": (out.error or "")[:500]} if not out.success else {}), + } + ) + + evaluated_ok = evaluated - request_failed + unique_question_ids = { + (req.videomme_question_id or req.request_id) for req in input_requests if isinstance(req, VideoMMESampleRequest) + } + result: dict[str, Any] = { + "videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None, + "videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None, + "videomme_correct": correct, + "videomme_evaluated": evaluated, + "videomme_evaluated_ok": evaluated_ok, + "videomme_no_gold": no_gold, + "videomme_request_failed": request_failed, + "videomme_parse_failed": parse_failed, + "videomme_submitted": len(input_requests), + "videomme_unique_question_ids": len(unique_question_ids), + "videomme_skipped_rows": max(req.videomme_skipped_rows for req in input_requests), + } + for dim, per_bucket in stats.items(): + result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()} + result[f"videomme_per_{dim}_accuracy"] = { + k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items() + } + if include_per_item: + result["videomme_eval_items"] = items + return result + + +def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None: + """Pretty-print the Video-MME accuracy block (stdout).""" + acc = metrics.get("videomme_accuracy") + if acc is None and not metrics.get("videomme_evaluated", 0): + return + + print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="=")) + ok = int(metrics.get("videomme_evaluated_ok", 0) or 0) + if ok and acc is not None: + print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}") + else: + print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)") + for label, key in ( + ("Submitted (gold present):", "videomme_evaluated"), + ("Successful HTTP (denominator):", "videomme_evaluated_ok"), + ("Correct:", "videomme_correct"), + ("Skipped (no gold):", "videomme_no_gold"), + ("HTTP failed:", "videomme_request_failed"), + ("Parsed OK but no A-D found:", "videomme_parse_failed"), + ("Submitted requests:", "videomme_submitted"), + ("Unique questions:", "videomme_unique_question_ids"), + ("Skipped source rows:", "videomme_skipped_rows"), + ): + print(f"{label:<40} {metrics.get(key, 0):<10}") + + for dim, title in ( + ("duration", "Duration"), + ("domain", "Domain"), + ("sub_category", "Sub Category"), + ("task", "Task Type"), + ): + per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {} + if not per_acc: + continue + counts = metrics.get(f"videomme_per_{dim}") or {} + names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc) + print(f"\n--- Accuracy by {title} ---") + for name in names: + st = counts.get(name) or {} + total = int(st.get("total", 0) or 0) + value = per_acc.get(name) + if total and value is not None: + print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}") + else: + print(f"{name}: 0/0 = N/A") + print("=" * 50) diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 5a5e74b0d..87fdb2daa 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -14,7 +14,7 @@ import time import traceback import uuid import wave -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass, replace from datetime import datetime from pathlib import Path @@ -67,6 +67,12 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( ) from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset +from vllm_omni.benchmarks.data_modules.videomme_dataset import ( + VIDEOMME_DEFAULT_HF_REPO, + VideoMMEDataset, + VideoMMESampleRequest, + resolve_videomme_local_root, +) from vllm_omni.benchmarks.omniinteract import ( VIDEO_FPS, OmniInteractBenchmarkConfig, @@ -283,14 +289,14 @@ def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict return out -def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None: - """Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni.""" - if not isinstance(sample, DailyOmniSampleRequest): +def _attach_omni_chat_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None: + """Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni / Video-MME.""" + if not isinstance(sample, (DailyOmniSampleRequest, VideoMMESampleRequest)): return rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body) if sample.omni_chat_messages is not None: setattr(rfi, "omni_chat_messages", sample.omni_chat_messages) - else: + elif isinstance(sample, DailyOmniSampleRequest): setattr(rfi, "mm_position", sample.omni_chat_mm_position) @@ -334,6 +340,29 @@ def _attach_omniinteract_to_request_func_input(sample: SampleRequest, rfi: Reque setattr(rfi, "omniinteract_prepared_input", sample.omniinteract_prepared_input) +def _async_limiter(max_concurrency: int | None) -> contextlib.AbstractAsyncContextManager[object]: + if max_concurrency: + return asyncio.Semaphore(max_concurrency) + return contextlib.nullcontext() + + +def _as_float(value: object, default: float = 0.0) -> float: + """Coerce loosely-typed metric dict values to ``float`` for mypy. + + Strings are parsed rather than dropped: JSON-sourced session metrics such as + ``audio_duration_ms`` may arrive quoted, and silently reporting ``0.0`` would + corrupt the derived TTFT / RTF numbers. + """ + if isinstance(value, bool) or value is None: + return default + if isinstance(value, (int, float, str)): + try: + return float(value) + except ValueError: + return default + return default + + def _append_error(existing: str, message: str) -> str: return f"{existing}\n{message}" if existing else message @@ -404,9 +433,13 @@ def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None: for sample in input_requests: if not isinstance(sample, OmniInteractSampleRequest): continue - root = sample.omniinteract_options.output_root.resolve() + options = sample.omniinteract_options + case = sample.omniinteract_case + if options is None or case is None: + raise RuntimeError("OmniInteract benchmark input lost its dataset identity") + root = options.output_root.resolve() roots.add(root) - clear_case_artifacts(sample.omniinteract_options.output_root, sample.omniinteract_case) + clear_case_artifacts(options.output_root, case) for root in roots: clear_batch_artifacts(root) @@ -428,11 +461,55 @@ def _daily_omni_repo_from_args(args) -> str | None: return None +def _looks_like_hf_dataset_id(value: str) -> bool: + """True for Hub ids such as ``org/name``; false for local paths.""" + raw = value.strip() + if not raw or raw.startswith((".", "~", "/")) or "\\" in raw: + return False + parts = raw.split("/") + return len(parts) == 2 and all(part.strip() and part.strip() not in (".", "..") for part in parts) + + +def _videomme_repo_from_args(args, *, explicit: bool = False) -> str | None: + """Resolve a Hugging Face repo id for Video-MME from CLI args. + + ``--dataset-name hf`` auto-detect only recognizes the official + ``lmms-eval/Video-MME`` id so a custom Hub dataset is not silently treated + as Video-MME. Explicit ``--dataset-name videomme`` accepts any ``org/name`` + Hub id (for ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO`` overrides) and + raises when ``--dataset-path`` is neither a local directory nor a Hub id. + """ + official = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS} + official.add(VIDEOMME_DEFAULT_HF_REPO.lower()) + candidates: list[str] = [] + for attr in ("dataset_path", "hf_name"): + val = getattr(args, attr, None) + if isinstance(val, str) and val.strip(): + candidates.append(val.strip()) + for raw in candidates: + if resolve_videomme_local_root(raw) is not None: + continue + if raw.lower() in official: + return raw + if explicit and _looks_like_hf_dataset_id(raw): + return raw + if explicit: + raise ValueError( + f"Unsupported Video-MME --dataset-path={raw!r}. Pass an existing local " + "directory, a Hugging Face dataset id (org/name), or omit --dataset-path " + f"to use {VIDEOMME_DEFAULT_HF_REPO}." + ) + return None + + def get_samples(args, tokenizer): # Daily-Omni: explicit dataset name, or hf + matching path/hf-name is_daily_omni = args.dataset_name == "daily-omni" or ( args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None ) + is_videomme = args.dataset_name == "videomme" or ( + args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None + ) is_seed_tts = args.dataset_name in ( "seed-tts", "seed-tts-text", @@ -449,7 +526,7 @@ def get_samples(args, tokenizer): "openai-realtime-duplex", "daily-omni", ] - is_omni_dataset = is_daily_omni or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm" + is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm" if not is_omni_backend and not is_omni_dataset: # Not an omni-related request, delegate to original implementation @@ -625,6 +702,54 @@ def get_samples(args, tokenizer): ) return input_requests + if is_videomme: + if args.backend not in ["openai-chat-omni", "daily-omni"]: + raise ValueError( + f"Video-MME dataset requires a multimodal backend that supports video. " + f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'" + ) + + # Resolve the source identity here; the dataset owns loading and extraction. + local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or ( + resolve_videomme_local_root(getattr(args, "hf_name", None)) + ) + source = ( + str(local_root) + if local_root is not None + else (_videomme_repo_from_args(args, explicit=args.dataset_name == "videomme") or VIDEOMME_DEFAULT_HF_REPO) + ) + dataset = VideoMMEDataset( + parquet_path=getattr(args, "videomme_parquet", None), + dataset_path=source, + dataset_split=getattr(args, "hf_split", None) or "test", + dataset_subset=getattr(args, "hf_subset", None), + random_seed=args.seed, + video_dir=getattr(args, "videomme_video_dir", None), + subtitle_dir=getattr(args, "videomme_subtitle_dir", None), + pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"), + max_frames=getattr(args, "videomme_max_frames", None), + duration_filter=getattr(args, "videomme_duration", "all"), + use_subtitle=getattr(args, "videomme_use_subtitle", False), + inline_local_video=getattr(args, "videomme_inline_local_video", False), + trust_remote_code=getattr(args, "trust_remote_code", False), + no_stream=getattr(args, "no_stream", False), + disable_shuffle=getattr(args, "disable_shuffle", False), + ) + + out_len = getattr(args, "output_len", None) + if out_len is None: + out_len = getattr(args, "hf_output_len", None) + if out_len is None: + out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN + + return dataset.sample( + tokenizer=tokenizer, + num_requests=args.num_prompts, + output_len=out_len, + request_id_prefix=args.request_id_prefix, + no_oversample=args.no_oversample, + ) + if is_seed_tts: if args.backend not in ( "openai-audio-speech", @@ -711,7 +836,7 @@ datasets.get_samples = get_samples _serve_mod = sys.modules.get("vllm.benchmarks.serve") if _serve_mod is not None: - _serve_mod.get_samples = get_samples + setattr(_serve_mod, "get_samples", get_samples) @dataclass @@ -2637,7 +2762,9 @@ async def async_request_openai_realtime_duplex( output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0 output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0 output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0 - output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0 + output.audio_duration = ( + sum((_as_float(metric.get("audio_duration_ms")) for metric in turn_metrics), start=0.0) / 1000.0 + ) output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ) output.latency = request_finished_at - output.start_time output.tts_turn_pcm_bytes = turn_pcm_bytes @@ -2811,7 +2938,7 @@ async def benchmark( extra_body=test_extra_body, chat_messages=test_chat_messages, ) - _attach_daily_omni_to_request_func_input(input_requests[0], test_input) + _attach_omni_chat_to_request_func_input(input_requests[0], test_input) _attach_seed_tts_to_request_func_input(input_requests[0], test_input) _attach_omniinteract_to_request_func_input(input_requests[0], test_input) @@ -2836,7 +2963,7 @@ async def benchmark( if num_warmups > 0: print(f"Warming up with {num_warmups} requests...") warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups) - warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext() + warmup_semaphore = _async_limiter(max_concurrency) warmup_tasks = [] async def warmup_limited_request_func(): @@ -2854,12 +2981,13 @@ async def benchmark( print("Starting main benchmark run...") + lora_iter: Iterator[str] | None = None if lora_modules: lora_modules_list = list(lora_modules) if lora_assignment == "round-robin": - lora_modules = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))]) + lora_iter = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))]) else: - lora_modules = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))]) + lora_iter = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))]) if profile: print("Starting profiler...") @@ -2877,7 +3005,7 @@ async def benchmark( extra_body=test_extra_body, chat_messages=test_chat_messages, ) - _attach_daily_omni_to_request_func_input(input_requests[0], profile_input) + _attach_omni_chat_to_request_func_input(input_requests[0], profile_input) _attach_seed_tts_to_request_func_input(input_requests[0], profile_input) profile_output = await request_func(request_func_input=profile_input, session=session) if profile_output.success: @@ -2898,7 +3026,7 @@ async def benchmark( pbar = None if disable_tqdm else tqdm(total=len(input_requests)) - semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext() + semaphore = _async_limiter(max_concurrency) async def limited_request_func(request_func_input, session, pbar): async with semaphore: @@ -2969,8 +3097,8 @@ async def benchmark( ) per_request_extra_body = _merge_overrides(extra_body, request.request_overrides) req_model_id, req_model_name = model_id, model_name - if lora_modules: - req_lora_module = next(lora_modules) + if lora_iter is not None: + req_lora_module = next(lora_iter) req_model_id, req_model_name = req_lora_module, req_lora_module request_func_input = RequestFuncInput( @@ -2988,7 +3116,7 @@ async def benchmark( request_id=request_id, chat_messages=request.chat_messages, ) - _attach_daily_omni_to_request_func_input(request, request_func_input) + _attach_omni_chat_to_request_func_input(request, request_func_input) _attach_seed_tts_to_request_func_input(request, request_func_input) _attach_omniinteract_to_request_func_input(request, request_func_input) tasks.append( @@ -3007,6 +3135,8 @@ async def benchmark( omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs) + metrics: Any + actual_output_lens: list[int] | int if task_type == TaskType.GENERATION: metrics, actual_output_lens = calculate_metrics( input_requests=input_requests, @@ -3031,6 +3161,10 @@ async def benchmark( actual_output_lens = 0 if isinstance(metrics, MultiModalsBenchmarkMetrics): + # ``make_dataclass`` types this class as ``type``, so isinstance() + # narrows to ``object`` and attribute access fails. Keep the runtime + # check and read fields through ``Any``. + mm_metrics: Any = metrics def measured_ttft(output: RequestFuncOutput) -> float | None: session_metrics = getattr(output, "duplex_session_metrics", None) @@ -3040,33 +3174,33 @@ async def benchmark( result = { "duration": benchmark_duration, - "completed": metrics.completed, - "failed": metrics.failed, - "total_input_tokens": metrics.total_input, - "total_output_tokens": metrics.total_output, - "request_throughput": metrics.request_throughput, - "request_goodput": metrics.request_goodput if goodput_config_dict else None, - "output_throughput": metrics.output_throughput, - "total_token_throughput": metrics.total_token_throughput, - defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S), - defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES), - defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT), - defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES), - defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT), - defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE), - defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS), - defs.TOTAL_VIDEO_DURATION_S: getattr(metrics, defs.TOTAL_VIDEO_DURATION_S), - defs.TOTAL_VIDEO_FRAMES: getattr(metrics, defs.TOTAL_VIDEO_FRAMES), - defs.VIDEO_THROUGHPUT: getattr(metrics, defs.VIDEO_THROUGHPUT), - defs.MEAN_VIDEO_RTF: getattr(metrics, defs.MEAN_VIDEO_RTF), - defs.MEDIAN_VIDEO_RTF: getattr(metrics, defs.MEDIAN_VIDEO_RTF), - defs.PERCENTILES_VIDEO_RTF: getattr(metrics, defs.PERCENTILES_VIDEO_RTF), - defs.MEAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS), - defs.MEDIAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS), - defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS), - defs.MEAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEAN_PEAK_MEMORY_MB), - defs.MEDIAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB), - defs.PERCENTILES_PEAK_MEMORY_MB: getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB), + "completed": mm_metrics.completed, + "failed": mm_metrics.failed, + "total_input_tokens": mm_metrics.total_input, + "total_output_tokens": mm_metrics.total_output, + "request_throughput": mm_metrics.request_throughput, + "request_goodput": mm_metrics.request_goodput if goodput_config_dict else None, + "output_throughput": mm_metrics.output_throughput, + "total_token_throughput": mm_metrics.total_token_throughput, + defs.TOTAL_AUDIO_DURATION_S: getattr(mm_metrics, defs.TOTAL_AUDIO_DURATION_S), + defs.TOTAL_AUDIO_FRAMES: getattr(mm_metrics, defs.TOTAL_AUDIO_FRAMES), + defs.AUDIO_THROUGHPUT: getattr(mm_metrics, defs.AUDIO_THROUGHPUT), + defs.TOTAL_IMAGES: getattr(mm_metrics, defs.TOTAL_IMAGES), + defs.IMAGE_THROUGHPUT: getattr(mm_metrics, defs.IMAGE_THROUGHPUT), + defs.AVERAGE_PIXELS_PER_IMAGE: getattr(mm_metrics, defs.AVERAGE_PIXELS_PER_IMAGE), + defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(mm_metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS), + defs.TOTAL_VIDEO_DURATION_S: getattr(mm_metrics, defs.TOTAL_VIDEO_DURATION_S), + defs.TOTAL_VIDEO_FRAMES: getattr(mm_metrics, defs.TOTAL_VIDEO_FRAMES), + defs.VIDEO_THROUGHPUT: getattr(mm_metrics, defs.VIDEO_THROUGHPUT), + defs.MEAN_VIDEO_RTF: getattr(mm_metrics, defs.MEAN_VIDEO_RTF), + defs.MEDIAN_VIDEO_RTF: getattr(mm_metrics, defs.MEDIAN_VIDEO_RTF), + defs.PERCENTILES_VIDEO_RTF: getattr(mm_metrics, defs.PERCENTILES_VIDEO_RTF), + defs.MEAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEAN_VIDEO_GENERATION_MS), + defs.MEDIAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEDIAN_VIDEO_GENERATION_MS), + defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.PERCENTILES_VIDEO_GENERATION_MS), + defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB), + defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB), + defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB), "input_lens": [output.prompt_len for output in outputs], "start_times": [output.start_time for output in outputs], "output_lens": actual_output_lens, @@ -3074,9 +3208,9 @@ async def benchmark( "itls": [output.itl for output in outputs], "generated_texts": [output.generated_text for output in outputs], "errors": [output.error for output in outputs], - "max_output_tokens_per_s": metrics.max_output_tokens_per_s, - "max_concurrent_requests": metrics.max_concurrent_requests, - "rtfx": metrics.rtfx, + "max_output_tokens_per_s": mm_metrics.max_output_tokens_per_s, + "max_concurrent_requests": mm_metrics.max_concurrent_requests, + "rtfx": mm_metrics.rtfx, } for sample_count in ( "num_ttft_samples", @@ -3150,6 +3284,21 @@ async def benchmark( result.update(_daily_acc) print_daily_omni_accuracy_summary(_daily_acc) + from vllm_omni.benchmarks.data_modules.videomme_eval import ( + compute_videomme_accuracy_metrics, + print_videomme_accuracy_summary, + ) + + _save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in ( + "1", + "true", + "yes", + ) + _vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm) + if _vm_acc is not None: + result.update(_vm_acc) + print_videomme_accuracy_summary(_vm_acc) + if _seed_tts_capture_pcm_for_wer(): from vllm_omni.benchmarks.data_modules.seed_tts_eval import ( compute_seed_tts_wer_metrics, diff --git a/vllm_omni/benchmarks/serve.py b/vllm_omni/benchmarks/serve.py index 7d05eb24f..5a04b63da 100644 --- a/vllm_omni/benchmarks/serve.py +++ b/vllm_omni/benchmarks/serve.py @@ -43,7 +43,7 @@ def _normalize_endpoint(endpoint: str | None) -> str | None: def _use_endpoint_backend_when_implicit(args: argparse.Namespace) -> None: - explicit_keys = getattr(args, "explicit_keys", frozenset()) + explicit_keys: frozenset[str] = getattr(args, "explicit_keys", frozenset()) if "backend" in explicit_keys: return # Upstream vLLM defaults --backend to "openai". Treat that non-explicit @@ -63,6 +63,8 @@ def main(args: argparse.Namespace) -> dict[str, Any]: os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1" if getattr(args, "daily_omni_save_eval_items", False): os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1" + if getattr(args, "videomme_save_eval_items", False): + os.environ["VIDEOMME_SAVE_EVAL_ITEMS"] = "1" request_timeout_s = getattr(args, "omni_request_timeout_s", None) if request_timeout_s is not None: set_request_timeout_s(request_timeout_s) diff --git a/vllm_omni/config/environment_variable_inventory.py b/vllm_omni/config/environment_variable_inventory.py index 4ba700e84..98ff7d6db 100644 --- a/vllm_omni/config/environment_variable_inventory.py +++ b/vllm_omni/config/environment_variable_inventory.py @@ -228,6 +228,7 @@ _BENCHMARK_TRANSITIONAL = ( "SEED_TTS_WER_EVAL", "SEED_TTS_WER_SAVE_AUDIO_DIR", "SEED_TTS_WER_SAVE_ITEMS", + "VIDEOMME_SAVE_EVAL_ITEMS", "VLLM_DAILY_OMNI_MEDIA_REPO", "VLLM_OMNI_BENCH_AUDIO_CHANNELS", "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S", diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 5e51974a2..8f7319b13 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -1386,7 +1386,7 @@ def normalize_and_validate_diffusion_engine_ingress_kwargs( | orchestrator_field_names() # Coordination fields also live on the typed orchestrator config, not # all of them are present on the CLI-only OrchestratorArgs dataclass. - | frozenset(config_field.name for config_field in fields(VllmOmniOrchestratorConfig)) + | frozenset(config_field.name for config_field in fields(cast(Any, VllmOmniOrchestratorConfig))) ) allowed_fields = stage_consumed_fields | externally_consumed_fields validate_omni_diffusion_kwargs(normalized, allowed_fields, stage_id=stage_id) diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index a04cff400..07090660b 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -47,7 +47,7 @@ stages: fps: 1 num_frames: 128 limit_mm_per_prompt: - # Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps). + # Daily-Omni: up to 64 interleaved image/audio pairs (1fps). image: 64 audio: 64 video: 1 diff --git a/vllm_omni/entrypoints/cli/benchmark/cli_args.py b/vllm_omni/entrypoints/cli/benchmark/cli_args.py index 626951298..bc8ae064f 100644 --- a/vllm_omni/entrypoints/cli/benchmark/cli_args.py +++ b/vllm_omni/entrypoints/cli/benchmark/cli_args.py @@ -192,6 +192,78 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None: ) +def add_videomme_cli_args(parser: argparse.ArgumentParser) -> None: + """Add CLI arguments specific to the Video-MME dataset.""" + group = parser.add_argument_group("Video-MME Dataset Options") + group.add_argument( + "--videomme-parquet", + type=str, + default=None, + help="Path to local Video-MME parquet " + "(e.g. videomme/test-00000-of-00001.parquet). When set, Hub QA loading is skipped.", + ) + group.add_argument( + "--videomme-video-dir", + type=str, + default=None, + help="Directory containing extracted Video-MME videos (videoID.mp4). " + "Typical layout after unzipping videos_chunked_*.zip: /video/. " + "When using file:// URLs, start the server with --allowed-local-media-path " + "covering this directory.", + ) + group.add_argument( + "--videomme-subtitle-dir", + type=str, + default=None, + help="Directory containing Video-MME .srt subtitles (videoID.srt). Used only with --videomme-use-subtitle.", + ) + group.add_argument( + "--videomme-pack-mode", + type=str, + choices=["minicpm-frames", "minicpm-interleave", "video_url"], + default="minicpm-frames", + help="Multimodal packing. " + "'minicpm-frames' (default): OmniEvalKit MiniCPM videomme recipe — sampled frames " + "as image_url only (max_frames=96). " + "'minicpm-interleave': OmniEvalKit videomme_short recipe — 1fps frame/audio pairs " + "(max_frames=64). " + "'video_url': single video_url part (models with native video input).", + ) + group.add_argument( + "--videomme-max-frames", + type=int, + default=None, + help="Override max sampled frames (OmniEvalKit defaults: 96 for minicpm-frames, 64 for minicpm-interleave).", + ) + group.add_argument( + "--videomme-duration", + type=str, + choices=["all", "short", "medium", "long"], + default="all", + help="Filter by Video-MME duration bucket (default: all).", + ) + group.add_argument( + "--videomme-use-subtitle", + action="store_true", + default=False, + help="Prepend subtitle text to the user prompt (Video-MME w/ subs setting).", + ) + group.add_argument( + "--videomme-inline-local-video", + action="store_true", + default=False, + help="Embed local frames/audio as base64 data URLs so the server does not need " + "--allowed-local-media-path. Increases request size; use for small --num-prompts.", + ) + group.add_argument( + "--videomme-save-eval-items", + action="store_true", + default=False, + help="Include per-request Video-MME accuracy rows in the saved JSON under " + "videomme_eval_items. Or set env VIDEOMME_SAVE_EVAL_ITEMS=1.", + ) + + def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None: """Add CLI arguments for Seed-TTS benchmarks.""" group = parser.add_argument_group("Seed-TTS Dataset Options") @@ -266,6 +338,7 @@ _OMNI_BENCH_DATASET_CHOICES = ( "seed-tts-design", "ttsd", "sound-effect", + "videomme", ) @@ -332,6 +405,7 @@ def update_omni_help(parser: argparse.ArgumentParser) -> None: def add_omni_args(parser: argparse.ArgumentParser) -> None: """Register all vLLM-Omni serving benchmark arguments.""" add_daily_omni_cli_args(parser) + add_videomme_cli_args(parser) add_omniinteract_cli_args(parser) add_seed_tts_cli_args(parser) add_multi_stage_cli_args(parser)