[Benchmark][MiniCPM-o] Port Video-MME dataset support to Omni bench s… (#6987)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-20 17:15:53 +08:00
1 parent ea62f86718
commit 9f5521351e
15 files changed
+2432 -59

No files matched your search

+54 -1
View File
@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
The API endpoint exposed externally, to which clients send their requests.
- `--dataset-name`
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
`omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
(`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
- `--num-prompts`
The total number of requests to send, an integer.
@@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
signal, not a transport failure. This local performance test does not score answer accuracy.
### Video-MME Benchmark
Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
to absolute `file://` URLs. For those URLs, start the server with
`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
with the local video root shared by the benchmark client and server):
```bash
vllm serve openbmb/MiniCPM-o-4_5 --omni \
--allowed-local-media-path /path/to/Video-MME \
--stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
```
The override applies to this server invocation only. Alternatively, use
`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
this evaluates a different frame-sampling setting from the 96-frame recipe.
Run the benchmark against that server:
```bash
vllm bench serve --omni \
--backend openai-chat-omni \
--dataset-name videomme \
--dataset-path /path/to/Video-MME \
--videomme-pack-mode minicpm-frames \
--videomme-max-frames 96 \
--model openbmb/MiniCPM-o-4_5 \
--endpoint /v1/chat/completions \
--num-prompts 8 \
--max-concurrency 1 \
--save-result
```
Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
oversampling made the request count differ from unique-question coverage.
`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
it does not establish that a supplied local mirror contains the entire official dataset.
Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
`videomme_eval_items` rows.
### Multi-Modal Benchmark
<details class="admonition abstract" markdown="1">
@@ -47,11 +47,50 @@ from benchmarks.accuracy.text_to_image.gbench import (
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
build_serve_common_argv,
seed_tts_bench_argv,
videomme_bench_argv,
)
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
def test_videomme_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", "lmms-eval/Video-MME")
monkeypatch.delenv("VLLM_VIDEOMME_REPO", raising=False)
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
argv = videomme_bench_argv()
dataset_idx = argv.index("--dataset-path")
assert argv[argv.index("--dataset-name") + 1] == "videomme"
assert argv[dataset_idx + 1] == "lmms-eval/Video-MME"
def test_videomme_bench_argv_uses_local_directory(monkeypatch, tmp_path: Path):
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", str(tmp_path))
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
argv = videomme_bench_argv()
assert argv[argv.index("--dataset-path") + 1] == str(tmp_path.resolve())
def test_sync_dataset_env_preserves_videomme_hf_repo_id(monkeypatch):
ns = argparse.Namespace(
daily_omni_repo=None,
daily_omni_qa_json=None,
daily_omni_video_dir=None,
videomme_repo=None,
videomme_dataset_path="lmms-eval/Video-MME",
seed_tts_dataset_path=None,
seed_tts_root=None,
)
monkeypatch.delenv("VLLM_VIDEOMME_DATASET_PATH", raising=False)
sync_dataset_env_from_ns(ns)
assert os.environ["VLLM_VIDEOMME_DATASET_PATH"] == "lmms-eval/Video-MME"
def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
monkeypatch.setenv("VLLM_SEED_TTS_DATASET_PATH", "zhaochenyang20/seed-tts-eval")
monkeypatch.delenv("VLLM_SEED_TTS_REPO", raising=False)
@@ -0,0 +1,546 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Offline Video-MME sources: local mirrors must not fall back to a Hub id."""
from __future__ import annotations
import os
import zipfile
from argparse import Namespace
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlparse
import pytest
from PIL import Image
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import _validate_videomme
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
VideoMMEDataset,
VideoMMESampleRequest,
cached_videomme_snapshots,
ensure_videomme_hub_root,
ensure_videomme_videos_extracted,
resolve_videomme_local_root,
videomme_local_parquet,
videomme_local_subtitle_dir,
videomme_local_video_dir,
)
from vllm_omni.benchmarks.data_modules.videomme_eval import (
compute_videomme_accuracy_metrics,
extract_characters_regex,
normalize_gold_answer,
)
from vllm_omni.benchmarks.patch.patch import _looks_like_hf_dataset_id, _videomme_repo_from_args, get_samples
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
def _write_parquet_placeholder(root: Path) -> Path:
path = root / "videomme" / "test-00000-of-00001.parquet"
path.parent.mkdir(parents=True)
path.write_bytes(b"parquet")
return path
def test_resolve_local_root_returns_none_for_hub_id() -> None:
assert resolve_videomme_local_root("lmms-eval/Video-MME") is None
assert resolve_videomme_local_root("") is None
assert resolve_videomme_local_root(None) is None
def test_plain_local_mirror(tmp_path: Path) -> None:
pq = _write_parquet_placeholder(tmp_path)
video = tmp_path / "video" / "abc.mp4"
video.parent.mkdir()
video.write_bytes(b"fake")
(tmp_path / "subtitle").mkdir()
(tmp_path / "subtitle" / "abc.srt").write_text("1\n", encoding="utf-8")
root = resolve_videomme_local_root(str(tmp_path))
assert root == tmp_path.resolve()
assert videomme_local_parquet(root) == pq.resolve()
assert videomme_local_video_dir(root) == tmp_path.resolve() / "video"
assert videomme_local_subtitle_dir(root) == tmp_path.resolve() / "subtitle"
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
snapshot = cache_dir / "snapshots" / "deadbeef"
snapshot.mkdir(parents=True)
_write_parquet_placeholder(snapshot)
(cache_dir / "refs").mkdir()
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
root = resolve_videomme_local_root(str(cache_dir))
assert root == snapshot.resolve()
assert videomme_local_parquet(root) is not None
def test_hf_cache_dir_with_unusable_ref_falls_back_to_revision(tmp_path: Path) -> None:
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
snapshot = cache_dir / "snapshots" / "deadbeef"
snapshot.mkdir(parents=True)
_write_parquet_placeholder(snapshot)
(cache_dir / "refs").mkdir()
# An empty ref must not resolve to snapshots/, which holds no dataset files.
(cache_dir / "refs" / "main").write_text("", encoding="utf-8")
assert resolve_videomme_local_root(str(cache_dir)) == snapshot.resolve()
def _stub_failing_hub(monkeypatch: pytest.MonkeyPatch, hub_cache: Path) -> dict[str, object]:
"""Point the Hub cache at ``hub_cache`` and make every snapshot_download miss."""
from huggingface_hub import constants as hf_constants
from huggingface_hub.errors import LocalEntryNotFoundError
from vllm_omni.transformers_utils import repo_utils
seen: dict[str, object] = {}
class _Api:
def snapshot_download(self, **kwargs: object) -> str:
seen.update(kwargs)
raise LocalEntryNotFoundError("no cached snapshot")
monkeypatch.setattr(hf_constants, "HF_HUB_OFFLINE", True)
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(hub_cache))
monkeypatch.setattr(repo_utils, "hf_api", lambda: _Api())
return seen
def test_hub_root_offline_without_cache_points_at_local_options(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
seen = _stub_failing_hub(monkeypatch, tmp_path)
with pytest.raises(FileNotFoundError, match="--videomme-video-dir") as excinfo:
ensure_videomme_hub_root("lmms-lab/Video-MME")
# Offline runs must consult the cache directly instead of resolving over the network.
assert seen["local_files_only"] is True
assert "already cached under another repo id" not in str(excinfo.value)
def test_hub_root_offline_names_snapshot_cached_under_another_repo_id(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
sibling = tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef"
sibling.mkdir(parents=True)
_write_parquet_placeholder(sibling)
_stub_failing_hub(monkeypatch, tmp_path)
with pytest.raises(FileNotFoundError) as excinfo:
ensure_videomme_hub_root("lmms-lab/Video-MME")
assert str(sibling.resolve()) in str(excinfo.value)
def test_cached_videomme_snapshots_skips_partial_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from huggingface_hub import constants as hf_constants
(tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef").mkdir(parents=True)
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
assert cached_videomme_snapshots() == []
def test_extract_videos_rejects_empty_tree(tmp_path: Path) -> None:
(tmp_path / "video").mkdir()
with pytest.raises(FileNotFoundError):
ensure_videomme_videos_extracted(tmp_path)
assert not (tmp_path / ".videomme_videos_extracted").exists()
def test_nested_unzipped_videos_are_discovered(tmp_path: Path) -> None:
nested = tmp_path / "videos" / "videos_chunked_01" / "data" / "xyz.mp4"
nested.parent.mkdir(parents=True)
nested.write_bytes(b"fake")
assert videomme_local_video_dir(tmp_path) == tmp_path / "videos"
def test_extract_characters_regex_matches_official_prefixes() -> None:
assert extract_characters_regex("The best answer is B.") == "B"
assert extract_characters_regex("Answer: C") == "C"
assert extract_characters_regex("I think D is correct") == "D"
assert extract_characters_regex("no letter here at all in this long sentence") is None
def test_normalize_gold_answer() -> None:
assert normalize_gold_answer("a") == "A"
assert normalize_gold_answer("B. content") == "B"
assert normalize_gold_answer("") is None
class _Out:
def __init__(self, text: str, *, success: bool = True) -> None:
self.generated_text = text
self.success = success
self.error = "" if success else "http fail"
def test_compute_videomme_accuracy_metrics() -> None:
reqs = [
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_video_id="v1",
videomme_question_id="q1",
videomme_duration="short",
videomme_domain="Knowledge",
videomme_sub_category="Science",
videomme_task_type="QA",
),
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="B",
videomme_video_id="v2",
videomme_question_id="q2",
videomme_duration="long",
videomme_domain="Knowledge",
videomme_sub_category="Science",
videomme_task_type="QA",
),
]
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("The answer is C")])
assert metrics is not None
assert metrics["videomme_correct"] == 1
assert metrics["videomme_evaluated_ok"] == 2
assert metrics["videomme_accuracy"] == 0.5
assert metrics["videomme_submitted"] == 2
assert metrics["videomme_unique_question_ids"] == 2
assert _validate_videomme(metrics, min_accuracy=0.4) == []
assert "videomme_eval_items" not in metrics
assert metrics["videomme_per_duration_accuracy"]["short"] == 1.0
assert metrics["videomme_per_duration_accuracy"]["long"] == 0.0
def test_compute_videomme_accuracy_metrics_http_fail_excluded_from_default() -> None:
reqs = [
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_question_id="q1",
),
VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="B",
videomme_question_id="q2",
),
]
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("", success=False)])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
assert metrics["videomme_accuracy_incl_http_fail"] == 0.5
assert metrics["videomme_request_failed"] == 1
errs = _validate_videomme(metrics, min_accuracy=0.68)
assert any("videomme_request_failed" in err for err in errs)
def test_compute_videomme_accuracy_metrics_saves_eval_items() -> None:
req = VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_gold_answer="A",
videomme_question_id="q1",
)
metrics = compute_videomme_accuracy_metrics([req], [_Out("A")], include_per_item=True)
assert metrics is not None
assert metrics["videomme_eval_items"][0]["question_id"] == "q1"
def test_videomme_save_eval_items_cli_sets_env(monkeypatch: pytest.MonkeyPatch) -> None:
from vllm_omni.benchmarks import serve
seen: dict[str, str | None] = {}
async def fake_main_async(args: Namespace) -> dict[str, str]:
seen["env"] = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS")
return {"ok": "1"}
monkeypatch.delenv("VIDEOMME_SAVE_EVAL_ITEMS", raising=False)
monkeypatch.setattr(serve, "main_async", fake_main_async)
args = Namespace(
videomme_save_eval_items=True,
seed_tts_wer_eval=False,
seed_tts_wer_save_items=False,
daily_omni_save_eval_items=False,
omni_request_timeout_s=None,
endpoint=None,
backend="openai-chat-omni",
explicit_keys=frozenset(),
extra_body=None,
print_stage=False,
dataset_name="videomme",
)
assert serve.main(args) == {"ok": "1"}
assert seen["env"] == "1"
def _file_uri_path(url: str) -> Path:
parsed = urlparse(url)
assert parsed.scheme == "file"
return Path(unquote(parsed.path))
def test_relative_video_dir_emits_absolute_file_uris_cold_and_warm(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
videos = tmp_path / "videos"
videos.mkdir()
(videos / "vid1.mp4").write_bytes(b"fake")
def fake_extract(cls, video_path, *, include_audio, max_num_frames): # noqa: ANN001
return [Image.new("RGB", (4, 4), color="red")], []
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
monkeypatch.setattr(VideoMMEDataset, "_extract_frames_and_audio", classmethod(fake_extract))
ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
assert ds.video_dir == videos.resolve()
cold = ds._get_minicpm_frame_parts("vid1", include_audio=False)
assert cold is not None
cold_path = _file_uri_path(cold[0]["image_url"]["url"])
assert cold_path.is_absolute()
assert cold_path.is_file()
warm_ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
warm = warm_ds._get_minicpm_frame_parts("vid1", include_audio=False)
assert warm is not None
warm_path = _file_uri_path(warm[0]["image_url"]["url"])
assert warm_path.is_absolute()
assert warm_path == cold_path
def test_videomme_repo_from_args_honors_custom_hub_id() -> None:
args = Namespace(dataset_name="videomme", dataset_path="my-org/custom-videomme", hf_name=None)
assert _videomme_repo_from_args(args, explicit=True) == "my-org/custom-videomme"
def test_videomme_repo_from_args_defaults_when_path_omitted() -> None:
args = Namespace(dataset_name="videomme", dataset_path=None, hf_name=None)
assert _videomme_repo_from_args(args, explicit=True) is None
def test_videomme_repo_from_args_rejects_unsupported_path() -> None:
args = Namespace(dataset_name="videomme", dataset_path="not-a-local-dir-or-hub-id", hf_name=None)
with pytest.raises(ValueError, match="Unsupported Video-MME"):
_videomme_repo_from_args(args, explicit=True)
def test_get_samples_rejects_unsupported_videomme_dataset_path() -> None:
args = Namespace(
dataset_name="videomme",
dataset_path="not-a-local-dir-or-hub-id",
hf_name=None,
backend="openai-chat-omni",
seed=0,
)
with pytest.raises(ValueError, match="Unsupported Video-MME"):
get_samples(args, None)
def test_hf_dataset_name_does_not_auto_detect_custom_repo() -> None:
args = Namespace(dataset_name="hf", dataset_path="my-org/custom-videomme", hf_name=None)
assert _videomme_repo_from_args(args) is None
assert _looks_like_hf_dataset_id("my-org/custom-videomme")
assert not _looks_like_hf_dataset_id("./videos")
official = Namespace(dataset_name="hf", dataset_path="lmms-eval/Video-MME", hf_name=None)
assert _videomme_repo_from_args(official) == "lmms-eval/Video-MME"
@pytest.mark.parametrize("max_frames", [0, -1])
def test_rejects_nonpositive_max_frames(max_frames: int, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
with pytest.raises(ValueError, match="max_frames must be positive"):
VideoMMEDataset(parquet_path="unused.parquet", max_frames=max_frames)
def test_frame_cache_rebuild_overwrites_incomplete_files(tmp_path: Path) -> None:
frame = tmp_path / "frame_0000.jpg"
frame.write_bytes(b"interrupted JPEG")
VideoMMEDataset._emit_media(tmp_path, frame.name, b"complete JPEG", "image_url", "image/jpeg")
assert frame.read_bytes() == b"complete JPEG"
def test_frame_cache_rejects_empty_media(tmp_path: Path) -> None:
(tmp_path / "manifest.json").write_text('{"count": 1}', encoding="utf-8")
(tmp_path / "frame_0000.jpg").touch()
dataset = object.__new__(VideoMMEDataset)
assert dataset._cached_parts_from_disk(tmp_path, include_audio=False) is None
@pytest.mark.parametrize("case", ["duplicates", "missing_gold"])
def test_accuracy_gate_rejects_incomplete_question_coverage(case: str) -> None:
def request(question_id: str, gold: str) -> VideoMMESampleRequest:
return VideoMMESampleRequest(
prompt="q",
prompt_len=1,
expected_output_len=8,
videomme_question_id=question_id,
videomme_gold_answer=gold,
)
requests = [
request("q1", "A"),
request("q1" if case == "duplicates" else "q2", "" if case == "missing_gold" else "A"),
]
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A"), _Out("A")])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
errors = _validate_videomme(metrics, min_accuracy=0.68)
assert any(("unique" if case == "duplicates" else "no_gold") in error for error in errors)
@pytest.fixture
def videomme_mirror(tmp_path: Path) -> Path:
import pandas as pd
parquet = tmp_path / "videomme" / "test-00000-of-00001.parquet"
parquet.parent.mkdir()
pd.DataFrame(
[
{
"videoID": f"v{i}",
"question_id": f"q{i}",
"question": "Which option is correct?",
"options": ["A. first", "B. second", "C. third", "D. fourth"],
"answer": "A",
"duration": duration,
}
for i, duration in enumerate(("short", "long"), start=1)
]
).to_parquet(parquet)
videos = tmp_path / "video"
videos.mkdir()
for i in (1, 2):
(videos / f"v{i}.mp4").write_bytes(b"video")
return tmp_path
def _sample_video_urls(dataset: VideoMMEDataset, monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> list:
monkeypatch.setattr(
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
)
return dataset.sample(Namespace(encode=lambda text: [1]), **kwargs)
@pytest.mark.parametrize("no_oversample", [True, False])
def test_missing_video_cannot_pass_accuracy_gate(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch, no_oversample: bool
) -> None:
(videomme_mirror / "video" / "v2.mp4").unlink()
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", disable_shuffle=True)
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2, no_oversample=no_oversample)
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
assert metrics is not None
assert metrics["videomme_accuracy"] == 1.0
assert metrics["videomme_submitted"] == (1 if no_oversample else 2)
assert metrics["videomme_skipped_rows"] == 1
assert any("videomme_skipped_rows" in error for error in _validate_videomme(metrics, min_accuracy=0.68))
def test_complete_duration_subset_can_pass_accuracy_gate(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", duration_filter="short")
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2700, no_oversample=True)
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
assert metrics is not None
assert metrics["videomme_submitted"] == 1
assert metrics["videomme_skipped_rows"] == 0
assert _validate_videomme(metrics, min_accuracy=0.68) == []
def test_dataset_direct_hub_loading_resolves_video_assets(
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
downloaded = []
def download(repo_id: str) -> Path:
downloaded.append(repo_id)
return videomme_mirror
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.ensure_videomme_hub_root", download)
dataset = VideoMMEDataset(dataset_path="my-org/custom-videomme", pack_mode="video_url")
assert downloaded == ["my-org/custom-videomme"]
assert dataset.video_dir == videomme_mirror / "video"
assert len(dataset.data) == 2
def test_local_source_with_explicit_parquet_keeps_video_root(videomme_mirror: Path) -> None:
dataset = VideoMMEDataset(
dataset_path=str(videomme_mirror),
parquet_path=str(videomme_local_parquet(videomme_mirror)),
pack_mode="video_url",
)
assert dataset.video_dir == videomme_mirror / "video"
assert len(dataset.data) == 2
def test_local_source_without_parquet_fails_locally(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def unexpected_hub(*args, **kwargs):
pytest.fail("An explicitly selected local mirror must not fall back to Hub loading")
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.load_dataset", unexpected_hub)
with pytest.raises(FileNotFoundError, match="No Video-MME parquet"):
VideoMMEDataset(dataset_path=str(tmp_path))
def test_get_samples_prefers_local_root_over_hf_name(videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch) -> None:
args = Namespace(
dataset_name="videomme",
dataset_path=str(videomme_mirror),
hf_name="unused-hf-name",
backend="openai-chat-omni",
seed=0,
videomme_pack_mode="video_url",
num_prompts=2,
request_id_prefix="",
no_oversample=True,
)
monkeypatch.setattr(
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
)
requests = get_samples(args, Namespace(encode=lambda text: [1]))
assert len(requests) == 2
assert {request.videomme_question_id for request in requests} == {"q1", "q2"}
def test_extraction_replaces_truncated_video(tmp_path: Path) -> None:
video = tmp_path / "video" / "v1.mp4"
video.parent.mkdir()
video.write_bytes(b"partial")
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
archive.writestr("nested/v1.mp4", b"complete video")
assert ensure_videomme_videos_extracted(tmp_path) == video.parent
assert video.read_bytes() == b"complete video"
def test_failed_extraction_does_not_publish_partial_video(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
archive.writestr("v1.mp4", b"complete video")
def interrupted_copy(src, dst):
dst.write(b"partial")
raise OSError("interrupted")
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.shutil.copyfileobj", interrupted_copy)
with pytest.raises(OSError, match="interrupted"):
ensure_videomme_videos_extracted(tmp_path)
assert list((tmp_path / "video").iterdir()) == []
assert not (tmp_path / ".videomme_videos_extracted").exists()
+1 -1
View File
@@ -168,7 +168,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
point to existing files, those are used with inline video. Otherwise the benchmark falls back to
@@ -10,6 +10,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
``lmms-eval/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
"""
@@ -26,6 +30,7 @@ from typing import Any, Protocol
DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
DEFAULT_VIDEOMME_HF_REPO = "lmms-eval/Video-MME"
class OmniBenchServerEndpoint(Protocol):
@@ -41,6 +46,7 @@ def build_acc_benchmark_cli_argv(
*,
skip_seed: bool,
skip_daily: bool,
skip_videomme: bool = True,
num_prompts: int | None = None,
max_concurrency: int | None = None,
) -> list[str]:
@@ -50,6 +56,9 @@ def build_acc_benchmark_cli_argv(
``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
"""
n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
@@ -72,6 +81,12 @@ def build_acc_benchmark_cli_argv(
argv.append("--skip-seed-tts")
if skip_daily:
argv.append("--skip-daily-omni")
if skip_videomme:
argv.append("--skip-videomme")
else:
argv.append("--run-videomme")
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
argv.extend(["--videomme-repo", repo])
return argv
@@ -101,6 +116,28 @@ def daily_omni_bench_argv() -> list[str]:
]
def videomme_bench_argv() -> list[str]:
"""CLI args for Video-MME (after ``vllm bench serve --omni``).
Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
* If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
use that local mirror as ``--dataset-path``.
* Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-eval/Video-MME``); the child
bench downloads via ``huggingface_hub.snapshot_download``.
"""
source = (
os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip()
or os.environ.get("VIDEOMME_ROOT", "").strip()
or os.environ.get("VLLM_VIDEOMME_REPO", "").strip()
or DEFAULT_VIDEOMME_HF_REPO
)
path = Path(source).expanduser()
if path.is_dir():
source = str(path.resolve())
return ["--dataset-name", "videomme", "--dataset-path", source]
def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
"""CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Accuracy (and light perf) checks for Qwen3-Omni via ``vllm bench serve --omni``.
The standalone CLI uses small ``--num-prompts`` / ``--max-concurrency`` defaults suitable for
@@ -11,6 +12,9 @@ H100 / MI325) because they launch the live Omni server inside the test.
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
**0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
Prerequisites
-------------
@@ -21,7 +25,8 @@ Prerequisites
On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
only drives the **client** benchmark.
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
``videomme``).
* **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
@@ -35,6 +40,12 @@ Prerequisites
pip install 'vllm-omni[dev]'
* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-eval/Video-MME`` (override with
``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
``huggingface_hub.snapshot_download``. Pass an existing local root with ``--videomme-dataset-path`` /
``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without
``--allowed-local-media-path``, pass ``--videomme-inline-local-video``.
Examples
--------
Pytest (same checks; needs a running server)::
@@ -55,6 +66,24 @@ Skip one suite, tighten gates::
--skip-daily-omni \\
--max-seed-tts-mean-wer 0.35 \\
--min-seed-tts-mean-sim 0.75
MiniCPM-o Video-MME (w/o subs). Start the server with the 96-image stage 0
limit first; replace /path/to/Video-MME with the local dataset root::
vllm serve openbmb/MiniCPM-o-4_5 --omni \\
--allowed-local-media-path /path/to/Video-MME \\
--stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
Then run the opt-in evaluation in another terminal::
python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
--model openbmb/MiniCPM-o-4_5 \\
--videomme-dataset-path /path/to/Video-MME \\
--skip-daily-omni --skip-seed-tts --run-videomme \\
--videomme-pack-mode minicpm-frames \\
--videomme-max-frames 96 \\
--temperature 0 --output-len 128 \\
--min-videomme-accuracy 0.68
"""
from __future__ import annotations
@@ -75,6 +104,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
load_benchmark_result,
run_vllm_bench_subprocess,
seed_tts_bench_argv,
videomme_bench_argv,
)
_REPO_ROOT = Path(__file__).resolve().parents[4]
@@ -102,6 +132,42 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
return errs
def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
errs: list[str] = []
acc = result.get("videomme_accuracy")
if acc is None:
errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
return errs
ev = int(result.get("videomme_evaluated_ok", 0) or 0)
if ev <= 0:
errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
failed = int(result.get("videomme_request_failed", 0) or 0)
if failed > 0:
errs.append(
f"videomme_request_failed={failed}; the accuracy gate requires every "
"gold-labeled request to complete HTTP successfully "
"(videomme_accuracy excludes HTTP failures)."
)
submitted = int(result.get("videomme_submitted", 0) or 0)
unique = int(result.get("videomme_unique_question_ids", 0) or 0)
if submitted <= 0 or unique != submitted:
errs.append(
f"Video-MME coverage is incomplete: submitted={submitted}, unique={unique}; "
"the accuracy gate requires distinct questions, not oversampled replacements."
)
skipped = int(result.get("videomme_skipped_rows", 0) or 0)
if skipped:
errs.append(
f"videomme_skipped_rows={skipped}; fix missing or unreadable source rows before accuracy evaluation."
)
no_gold = int(result.get("videomme_no_gold", 0) or 0)
if no_gold:
errs.append(f"videomme_no_gold={no_gold}; every submitted question must have a gold answer.")
if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
return errs
def _validate_seed_tts(
result: dict[str, Any],
*,
@@ -130,7 +196,7 @@ def _validate_seed_tts(
def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
"""Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
"""Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
repo = getattr(ns, "daily_omni_repo", None)
if repo is not None and str(repo).strip():
os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
@@ -138,6 +204,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
if ns.daily_omni_video_dir is not None:
os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
vm_repo = getattr(ns, "videomme_repo", None)
if vm_repo is not None and str(vm_repo).strip():
os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
vm_path = getattr(ns, "videomme_dataset_path", None)
if vm_path is not None and str(vm_path).strip():
raw = str(vm_path).strip()
p = Path(raw).expanduser()
os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
if ns.seed_tts_dataset_path is not None:
# ``--seed-tts-dataset-path`` accepts either a local directory or a
# Hugging Face repo id. Only resolve to an absolute filesystem path
@@ -157,6 +231,9 @@ def _preserve_benchmark_dataset_env() -> Any:
"VLLM_DAILY_OMNI_REPO",
"VLLM_DAILY_OMNI_QA_JSON",
"VLLM_DAILY_OMNI_VIDEO_DIR",
"VLLM_VIDEOMME_REPO",
"VLLM_VIDEOMME_DATASET_PATH",
"VIDEOMME_ROOT",
"VLLM_SEED_TTS_DATASET_PATH",
"SEED_TTS_ROOT",
)
@@ -225,6 +302,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
return out
def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
ns.result_dir.mkdir(parents=True, exist_ok=True)
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
result_filename = f"omni_acc_videomme_{tag}.json"
extra = json.loads(ns.videomme_extra_body_json)
argv = (
_build_common_args(ns, result_filename=result_filename)
+ videomme_bench_argv()
+ [
"--disable-shuffle",
"--videomme-pack-mode",
ns.videomme_pack_mode,
"--videomme-duration",
ns.videomme_duration,
"--extra-body",
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
]
)
if ns.videomme_max_frames is not None:
argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
if ns.videomme_use_subtitle:
argv.append("--videomme-use-subtitle")
if ns.videomme_inline_local_video:
argv.append("--videomme-inline-local-video")
if ns.videomme_save_eval_items:
argv.append("--videomme-save-eval-items")
if ns.videomme_video_dir is not None:
argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
if ns.videomme_parquet is not None:
argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
if ns.videomme_subtitle_dir is not None:
argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
print("\n$", vllm, *argv, "\n", flush=True)
run_vllm_bench_subprocess(vllm, argv)
out = Path(ns.result_dir) / result_filename
if not out.is_file():
raise FileNotFoundError(f"Expected result JSON at {out}")
return out
def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
ns.result_dir.mkdir(parents=True, exist_ok=True)
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
@@ -312,6 +429,20 @@ def build_arg_parser() -> argparse.ArgumentParser:
p.add_argument("--skip-daily-omni", action="store_true")
p.add_argument("--skip-seed-tts", action="store_true")
# Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
p.set_defaults(skip_videomme=True)
p.add_argument(
"--skip-videomme",
action="store_true",
dest="skip_videomme",
help="Skip Video-MME (default).",
)
p.add_argument(
"--run-videomme",
action="store_false",
dest="skip_videomme",
help="Enable the Video-MME accuracy suite.",
)
p.add_argument(
"--daily-omni-repo",
@@ -358,6 +489,87 @@ def build_arg_parser() -> argparse.ArgumentParser:
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
)
p.add_argument(
"--videomme-repo",
type=str,
default=None,
help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
"Default when no local path is set: lmms-eval/Video-MME. "
"Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
)
p.add_argument(
"--videomme-dataset-path",
type=str,
default=None,
help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
"download; Hub ids are also accepted and passed through as --dataset-path. "
"Sets VLLM_VIDEOMME_DATASET_PATH.",
)
p.add_argument(
"--videomme-parquet",
type=Path,
default=None,
help="Optional explicit path to videomme/test-*.parquet.",
)
p.add_argument(
"--videomme-video-dir",
type=Path,
default=None,
help="Optional explicit directory of extracted Video-MME videos.",
)
p.add_argument(
"--videomme-subtitle-dir",
type=Path,
default=None,
help="Optional subtitle directory (used with --videomme-use-subtitle).",
)
p.add_argument(
"--videomme-pack-mode",
choices=("minicpm-frames", "minicpm-interleave", "video_url"),
default="minicpm-frames",
help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
)
p.add_argument(
"--videomme-max-frames",
type=int,
default=None,
help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
)
p.add_argument(
"--videomme-duration",
choices=("all", "short", "medium", "long"),
default="all",
help="Filter Video-MME by duration bucket (default: all).",
)
p.add_argument(
"--videomme-use-subtitle",
action="store_true",
help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
)
p.add_argument(
"--videomme-inline-local-video",
action="store_true",
help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
)
p.add_argument(
"--videomme-extra-body-json",
default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
help="JSON merged into each chat request for Video-MME.",
)
p.add_argument(
"--videomme-save-eval-items",
action="store_true",
help="Include per-request Video-MME accuracy rows in the saved JSON.",
)
p.add_argument(
"--min-videomme-accuracy",
type=float,
default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
"Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
"override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
)
p.add_argument(
"--seed-tts-dataset-path",
type=str,
@@ -427,7 +639,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
def run_acc_benchmark(ns: argparse.Namespace) -> int:
"""Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
"""Run Daily-Omni / Video-MME / Seed-TTS client benches against a running server; return 0 on success."""
failed: list[str] = []
with _preserve_benchmark_dataset_env():
@@ -451,6 +663,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
flush=True,
)
if not ns.skip_videomme:
path = run_videomme(ns, vllm)
print(f"\n[Video-MME] result JSON: {path}", flush=True)
data = load_benchmark_result(path)
errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
if errs:
failed.extend([f"[Video-MME] {e}" for e in errs])
else:
print(
f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
f"evaluated_ok={data.get('videomme_evaluated_ok')} "
f"per_duration={data.get('videomme_per_duration_accuracy')}",
flush=True,
)
if not ns.skip_seed_tts:
path = run_seed_tts(ns, vllm)
print(f"\n[Seed-TTS] result JSON: {path}", flush=True)
@@ -178,7 +178,7 @@ class TestDeployTopology:
0.18,
]
assert sum(memory_utilizations) <= 0.9 + 1e-6
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
# Keep the default Daily-Omni budget; Video-MME overrides stage 0 explicitly.
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {
"image": 64,
"audio": 64,
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve.
Answer extraction ports ``extract_characters_regex`` from the official Video-MME
script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval).
**Denominator:** responses where no A–D can be extracted are counted as wrong, which
matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The
official standalone script instead drops them from ``answered``; ``videomme_parse_failed``
is reported so that view can be recovered.
"""
from __future__ import annotations
from typing import Any
import regex as re
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest
_VALID = frozenset("ABCD")
VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long")
#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns).
_DIMENSIONS: dict[str, str] = {
"duration": "videomme_duration",
"domain": "videomme_domain",
"sub_category": "videomme_sub_category",
"task": "videomme_task_type",
}
# Official prefix list, with the two accidental string concatenations in the upstream
# tuple (missing commas after "The best option is" / "Best answer:") split apart.
_ANSWER_PREFIXES = (
"The best answer is",
"The correct answer is",
"The answer is",
"The answer",
"The best option is",
"The correct option is",
"Best answer:",
"Best option:",
"Answer:",
"Option:",
"The correct answer",
"The correct option",
)
def extract_characters_regex(text: str | None) -> str | None:
"""Port of official Video-MME / lmms-eval ``extract_characters_regex``."""
if not text:
return None
s = str(text).strip()
for prefix in _ANSWER_PREFIXES:
s = s.replace(prefix, "")
if len(s.split()) > 10 and not re.search(r"[ABCD]", s):
return None
match = re.search(r"[ABCD]", s)
return match.group(0) if match else None
def normalize_gold_answer(gold: str) -> str | None:
g = (gold or "").strip().upper()
if len(g) == 1 and g in _VALID:
return g
m = re.search(r"([ABCD])\b", g)
return m.group(1).upper() if m else None
def _bucket(req: VideoMMESampleRequest, dimension: str) -> str:
value = (getattr(req, _DIMENSIONS[dimension]) or "").strip()
if dimension == "duration":
return value.lower()
return value or "unknown"
def compute_videomme_accuracy_metrics(
input_requests: list[Any],
outputs: list[RequestFuncOutput],
*,
include_per_item: bool = False,
) -> dict[str, Any] | None:
"""If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats.
Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are
excluded from ``videomme_accuracy`` and counted in
``videomme_accuracy_incl_http_fail`` instead. The accuracy wrapper gate
(``_validate_videomme``) additionally rejects HTTP failures, duplicate question
IDs, skipped source rows and missing gold answers. Direct
``vllm bench serve`` still reports both metrics without that completeness
check.
``videomme_submitted`` / ``videomme_unique_question_ids`` show when
oversampling made the request count differ from unique question coverage.
``videomme_skipped_rows`` counts source rows skipped during sampling (for
example, missing or undecodable media), before any oversampling.
"""
if not input_requests or len(input_requests) != len(outputs):
return None
if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests):
return None
stats: dict[str, dict[str, dict[str, int]]] = {
dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {})
for dim in _DIMENSIONS
}
items: list[dict[str, Any]] = []
correct = evaluated = no_gold = request_failed = parse_failed = 0
for req, out in zip(input_requests, outputs, strict=True):
assert isinstance(req, VideoMMESampleRequest)
gold_raw = (req.videomme_gold_answer or "").strip()
buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS}
row: dict[str, Any] = {
"request_id": req.request_id,
"video_id": req.videomme_video_id,
"question_id": req.videomme_question_id,
**buckets,
}
if not gold_raw:
no_gold += 1
if include_per_item:
items.append({**row, "skipped": True, "reason": "no_gold"})
continue
evaluated += 1
gold_norm = normalize_gold_answer(gold_raw)
pred = extract_characters_regex(out.generated_text) if out.success else None
is_correct = pred is not None and pred.upper() == (gold_norm or gold_raw).upper()
if not out.success:
request_failed += 1
else:
if pred is None:
parse_failed += 1
# Only successful responses enter the per-dimension denominators, mirroring
# the overall videomme_accuracy definition.
for dim, key in buckets.items():
bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0})
bucket["total"] += 1
bucket["correct"] += int(is_correct)
if is_correct:
correct += 1
if include_per_item:
items.append(
{
**row,
"gold": gold_raw,
"gold_normalized": gold_norm,
"predicted": pred,
"correct": is_correct,
"parse_failed": out.success and pred is None,
**({"error": (out.error or "")[:500]} if not out.success else {}),
}
)
evaluated_ok = evaluated - request_failed
unique_question_ids = {
(req.videomme_question_id or req.request_id) for req in input_requests if isinstance(req, VideoMMESampleRequest)
}
result: dict[str, Any] = {
"videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None,
"videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None,
"videomme_correct": correct,
"videomme_evaluated": evaluated,
"videomme_evaluated_ok": evaluated_ok,
"videomme_no_gold": no_gold,
"videomme_request_failed": request_failed,
"videomme_parse_failed": parse_failed,
"videomme_submitted": len(input_requests),
"videomme_unique_question_ids": len(unique_question_ids),
"videomme_skipped_rows": max(req.videomme_skipped_rows for req in input_requests),
}
for dim, per_bucket in stats.items():
result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()}
result[f"videomme_per_{dim}_accuracy"] = {
k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items()
}
if include_per_item:
result["videomme_eval_items"] = items
return result
def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None:
"""Pretty-print the Video-MME accuracy block (stdout)."""
acc = metrics.get("videomme_accuracy")
if acc is None and not metrics.get("videomme_evaluated", 0):
return
print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="="))
ok = int(metrics.get("videomme_evaluated_ok", 0) or 0)
if ok and acc is not None:
print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}")
else:
print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)")
for label, key in (
("Submitted (gold present):", "videomme_evaluated"),
("Successful HTTP (denominator):", "videomme_evaluated_ok"),
("Correct:", "videomme_correct"),
("Skipped (no gold):", "videomme_no_gold"),
("HTTP failed:", "videomme_request_failed"),
("Parsed OK but no A-D found:", "videomme_parse_failed"),
("Submitted requests:", "videomme_submitted"),
("Unique questions:", "videomme_unique_question_ids"),
("Skipped source rows:", "videomme_skipped_rows"),
):
print(f"{label:<40} {metrics.get(key, 0):<10}")
for dim, title in (
("duration", "Duration"),
("domain", "Domain"),
("sub_category", "Sub Category"),
("task", "Task Type"),
):
per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {}
if not per_acc:
continue
counts = metrics.get(f"videomme_per_{dim}") or {}
names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc)
print(f"\n--- Accuracy by {title} ---")
for name in names:
st = counts.get(name) or {}
total = int(st.get("total", 0) or 0)
value = per_acc.get(name)
if total and value is not None:
print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}")
else:
print(f"{name}: 0/0 = N/A")
print("=" * 50)
+198 -49
View File
@@ -14,7 +14,7 @@ import time
import traceback
import uuid
import wave
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from pathlib import Path
@@ -67,6 +67,12 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
)
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
VIDEOMME_DEFAULT_HF_REPO,
VideoMMEDataset,
VideoMMESampleRequest,
resolve_videomme_local_root,
)
from vllm_omni.benchmarks.omniinteract import (
VIDEO_FPS,
OmniInteractBenchmarkConfig,
@@ -283,14 +289,14 @@ def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict
return out
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
if not isinstance(sample, DailyOmniSampleRequest):
def _attach_omni_chat_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni / Video-MME."""
if not isinstance(sample, (DailyOmniSampleRequest, VideoMMESampleRequest)):
return
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
if sample.omni_chat_messages is not None:
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
else:
elif isinstance(sample, DailyOmniSampleRequest):
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
@@ -334,6 +340,29 @@ def _attach_omniinteract_to_request_func_input(sample: SampleRequest, rfi: Reque
setattr(rfi, "omniinteract_prepared_input", sample.omniinteract_prepared_input)
def _async_limiter(max_concurrency: int | None) -> contextlib.AbstractAsyncContextManager[object]:
if max_concurrency:
return asyncio.Semaphore(max_concurrency)
return contextlib.nullcontext()
def _as_float(value: object, default: float = 0.0) -> float:
"""Coerce loosely-typed metric dict values to ``float`` for mypy.
Strings are parsed rather than dropped: JSON-sourced session metrics such as
``audio_duration_ms`` may arrive quoted, and silently reporting ``0.0`` would
corrupt the derived TTFT / RTF numbers.
"""
if isinstance(value, bool) or value is None:
return default
if isinstance(value, (int, float, str)):
try:
return float(value)
except ValueError:
return default
return default
def _append_error(existing: str, message: str) -> str:
return f"{existing}\n{message}" if existing else message
@@ -404,9 +433,13 @@ def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None:
for sample in input_requests:
if not isinstance(sample, OmniInteractSampleRequest):
continue
root = sample.omniinteract_options.output_root.resolve()
options = sample.omniinteract_options
case = sample.omniinteract_case
if options is None or case is None:
raise RuntimeError("OmniInteract benchmark input lost its dataset identity")
root = options.output_root.resolve()
roots.add(root)
clear_case_artifacts(sample.omniinteract_options.output_root, sample.omniinteract_case)
clear_case_artifacts(options.output_root, case)
for root in roots:
clear_batch_artifacts(root)
@@ -428,11 +461,55 @@ def _daily_omni_repo_from_args(args) -> str | None:
return None
def _looks_like_hf_dataset_id(value: str) -> bool:
"""True for Hub ids such as ``org/name``; false for local paths."""
raw = value.strip()
if not raw or raw.startswith((".", "~", "/")) or "\\" in raw:
return False
parts = raw.split("/")
return len(parts) == 2 and all(part.strip() and part.strip() not in (".", "..") for part in parts)
def _videomme_repo_from_args(args, *, explicit: bool = False) -> str | None:
"""Resolve a Hugging Face repo id for Video-MME from CLI args.
``--dataset-name hf`` auto-detect only recognizes the official
``lmms-eval/Video-MME`` id so a custom Hub dataset is not silently treated
as Video-MME. Explicit ``--dataset-name videomme`` accepts any ``org/name``
Hub id (for ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO`` overrides) and
raises when ``--dataset-path`` is neither a local directory nor a Hub id.
"""
official = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS}
official.add(VIDEOMME_DEFAULT_HF_REPO.lower())
candidates: list[str] = []
for attr in ("dataset_path", "hf_name"):
val = getattr(args, attr, None)
if isinstance(val, str) and val.strip():
candidates.append(val.strip())
for raw in candidates:
if resolve_videomme_local_root(raw) is not None:
continue
if raw.lower() in official:
return raw
if explicit and _looks_like_hf_dataset_id(raw):
return raw
if explicit:
raise ValueError(
f"Unsupported Video-MME --dataset-path={raw!r}. Pass an existing local "
"directory, a Hugging Face dataset id (org/name), or omit --dataset-path "
f"to use {VIDEOMME_DEFAULT_HF_REPO}."
)
return None
def get_samples(args, tokenizer):
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
is_daily_omni = args.dataset_name == "daily-omni" or (
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
)
is_videomme = args.dataset_name == "videomme" or (
args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None
)
is_seed_tts = args.dataset_name in (
"seed-tts",
"seed-tts-text",
@@ -449,7 +526,7 @@ def get_samples(args, tokenizer):
"openai-realtime-duplex",
"daily-omni",
]
is_omni_dataset = is_daily_omni or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
if not is_omni_backend and not is_omni_dataset:
# Not an omni-related request, delegate to original implementation
@@ -625,6 +702,54 @@ def get_samples(args, tokenizer):
)
return input_requests
if is_videomme:
if args.backend not in ["openai-chat-omni", "daily-omni"]:
raise ValueError(
f"Video-MME dataset requires a multimodal backend that supports video. "
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
)
# Resolve the source identity here; the dataset owns loading and extraction.
local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or (
resolve_videomme_local_root(getattr(args, "hf_name", None))
)
source = (
str(local_root)
if local_root is not None
else (_videomme_repo_from_args(args, explicit=args.dataset_name == "videomme") or VIDEOMME_DEFAULT_HF_REPO)
)
dataset = VideoMMEDataset(
parquet_path=getattr(args, "videomme_parquet", None),
dataset_path=source,
dataset_split=getattr(args, "hf_split", None) or "test",
dataset_subset=getattr(args, "hf_subset", None),
random_seed=args.seed,
video_dir=getattr(args, "videomme_video_dir", None),
subtitle_dir=getattr(args, "videomme_subtitle_dir", None),
pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"),
max_frames=getattr(args, "videomme_max_frames", None),
duration_filter=getattr(args, "videomme_duration", "all"),
use_subtitle=getattr(args, "videomme_use_subtitle", False),
inline_local_video=getattr(args, "videomme_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
no_stream=getattr(args, "no_stream", False),
disable_shuffle=getattr(args, "disable_shuffle", False),
)
out_len = getattr(args, "output_len", None)
if out_len is None:
out_len = getattr(args, "hf_output_len", None)
if out_len is None:
out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN
return dataset.sample(
tokenizer=tokenizer,
num_requests=args.num_prompts,
output_len=out_len,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
if is_seed_tts:
if args.backend not in (
"openai-audio-speech",
@@ -711,7 +836,7 @@ datasets.get_samples = get_samples
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
if _serve_mod is not None:
_serve_mod.get_samples = get_samples
setattr(_serve_mod, "get_samples", get_samples)
@dataclass
@@ -2637,7 +2762,9 @@ async def async_request_openai_realtime_duplex(
output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0
output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0
output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0
output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
output.audio_duration = (
sum((_as_float(metric.get("audio_duration_ms")) for metric in turn_metrics), start=0.0) / 1000.0
)
output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ)
output.latency = request_finished_at - output.start_time
output.tts_turn_pcm_bytes = turn_pcm_bytes
@@ -2811,7 +2938,7 @@ async def benchmark(
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
_attach_omni_chat_to_request_func_input(input_requests[0], test_input)
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
_attach_omniinteract_to_request_func_input(input_requests[0], test_input)
@@ -2836,7 +2963,7 @@ async def benchmark(
if num_warmups > 0:
print(f"Warming up with {num_warmups} requests...")
warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups)
warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
warmup_semaphore = _async_limiter(max_concurrency)
warmup_tasks = []
async def warmup_limited_request_func():
@@ -2854,12 +2981,13 @@ async def benchmark(
print("Starting main benchmark run...")
lora_iter: Iterator[str] | None = None
if lora_modules:
lora_modules_list = list(lora_modules)
if lora_assignment == "round-robin":
lora_modules = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
lora_iter = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
else:
lora_modules = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
lora_iter = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
if profile:
print("Starting profiler...")
@@ -2877,7 +3005,7 @@ async def benchmark(
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
_attach_omni_chat_to_request_func_input(input_requests[0], profile_input)
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
profile_output = await request_func(request_func_input=profile_input, session=session)
if profile_output.success:
@@ -2898,7 +3026,7 @@ async def benchmark(
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
semaphore = _async_limiter(max_concurrency)
async def limited_request_func(request_func_input, session, pbar):
async with semaphore:
@@ -2969,8 +3097,8 @@ async def benchmark(
)
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
req_model_id, req_model_name = model_id, model_name
if lora_modules:
req_lora_module = next(lora_modules)
if lora_iter is not None:
req_lora_module = next(lora_iter)
req_model_id, req_model_name = req_lora_module, req_lora_module
request_func_input = RequestFuncInput(
@@ -2988,7 +3116,7 @@ async def benchmark(
request_id=request_id,
chat_messages=request.chat_messages,
)
_attach_daily_omni_to_request_func_input(request, request_func_input)
_attach_omni_chat_to_request_func_input(request, request_func_input)
_attach_seed_tts_to_request_func_input(request, request_func_input)
_attach_omniinteract_to_request_func_input(request, request_func_input)
tasks.append(
@@ -3007,6 +3135,8 @@ async def benchmark(
omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs)
metrics: Any
actual_output_lens: list[int] | int
if task_type == TaskType.GENERATION:
metrics, actual_output_lens = calculate_metrics(
input_requests=input_requests,
@@ -3031,6 +3161,10 @@ async def benchmark(
actual_output_lens = 0
if isinstance(metrics, MultiModalsBenchmarkMetrics):
# ``make_dataclass`` types this class as ``type``, so isinstance()
# narrows to ``object`` and attribute access fails. Keep the runtime
# check and read fields through ``Any``.
mm_metrics: Any = metrics
def measured_ttft(output: RequestFuncOutput) -> float | None:
session_metrics = getattr(output, "duplex_session_metrics", None)
@@ -3040,33 +3174,33 @@ async def benchmark(
result = {
"duration": benchmark_duration,
"completed": metrics.completed,
"failed": metrics.failed,
"total_input_tokens": metrics.total_input,
"total_output_tokens": metrics.total_output,
"request_throughput": metrics.request_throughput,
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
"output_throughput": metrics.output_throughput,
"total_token_throughput": metrics.total_token_throughput,
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
defs.TOTAL_VIDEO_DURATION_S: getattr(metrics, defs.TOTAL_VIDEO_DURATION_S),
defs.TOTAL_VIDEO_FRAMES: getattr(metrics, defs.TOTAL_VIDEO_FRAMES),
defs.VIDEO_THROUGHPUT: getattr(metrics, defs.VIDEO_THROUGHPUT),
defs.MEAN_VIDEO_RTF: getattr(metrics, defs.MEAN_VIDEO_RTF),
defs.MEDIAN_VIDEO_RTF: getattr(metrics, defs.MEDIAN_VIDEO_RTF),
defs.PERCENTILES_VIDEO_RTF: getattr(metrics, defs.PERCENTILES_VIDEO_RTF),
defs.MEAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS),
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
defs.MEAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEAN_PEAK_MEMORY_MB),
defs.MEDIAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB),
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
"completed": mm_metrics.completed,
"failed": mm_metrics.failed,
"total_input_tokens": mm_metrics.total_input,
"total_output_tokens": mm_metrics.total_output,
"request_throughput": mm_metrics.request_throughput,
"request_goodput": mm_metrics.request_goodput if goodput_config_dict else None,
"output_throughput": mm_metrics.output_throughput,
"total_token_throughput": mm_metrics.total_token_throughput,
defs.TOTAL_AUDIO_DURATION_S: getattr(mm_metrics, defs.TOTAL_AUDIO_DURATION_S),
defs.TOTAL_AUDIO_FRAMES: getattr(mm_metrics, defs.TOTAL_AUDIO_FRAMES),
defs.AUDIO_THROUGHPUT: getattr(mm_metrics, defs.AUDIO_THROUGHPUT),
defs.TOTAL_IMAGES: getattr(mm_metrics, defs.TOTAL_IMAGES),
defs.IMAGE_THROUGHPUT: getattr(mm_metrics, defs.IMAGE_THROUGHPUT),
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(mm_metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(mm_metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
defs.TOTAL_VIDEO_DURATION_S: getattr(mm_metrics, defs.TOTAL_VIDEO_DURATION_S),
defs.TOTAL_VIDEO_FRAMES: getattr(mm_metrics, defs.TOTAL_VIDEO_FRAMES),
defs.VIDEO_THROUGHPUT: getattr(mm_metrics, defs.VIDEO_THROUGHPUT),
defs.MEAN_VIDEO_RTF: getattr(mm_metrics, defs.MEAN_VIDEO_RTF),
defs.MEDIAN_VIDEO_RTF: getattr(mm_metrics, defs.MEDIAN_VIDEO_RTF),
defs.PERCENTILES_VIDEO_RTF: getattr(mm_metrics, defs.PERCENTILES_VIDEO_RTF),
defs.MEAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEAN_VIDEO_GENERATION_MS),
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB),
defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB),
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
"input_lens": [output.prompt_len for output in outputs],
"start_times": [output.start_time for output in outputs],
"output_lens": actual_output_lens,
@@ -3074,9 +3208,9 @@ async def benchmark(
"itls": [output.itl for output in outputs],
"generated_texts": [output.generated_text for output in outputs],
"errors": [output.error for output in outputs],
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
"max_concurrent_requests": metrics.max_concurrent_requests,
"rtfx": metrics.rtfx,
"max_output_tokens_per_s": mm_metrics.max_output_tokens_per_s,
"max_concurrent_requests": mm_metrics.max_concurrent_requests,
"rtfx": mm_metrics.rtfx,
}
for sample_count in (
"num_ttft_samples",
@@ -3150,6 +3284,21 @@ async def benchmark(
result.update(_daily_acc)
print_daily_omni_accuracy_summary(_daily_acc)
from vllm_omni.benchmarks.data_modules.videomme_eval import (
compute_videomme_accuracy_metrics,
print_videomme_accuracy_summary,
)
_save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in (
"1",
"true",
"yes",
)
_vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm)
if _vm_acc is not None:
result.update(_vm_acc)
print_videomme_accuracy_summary(_vm_acc)
if _seed_tts_capture_pcm_for_wer():
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
compute_seed_tts_wer_metrics,
+3 -1
View File
@@ -43,7 +43,7 @@ def _normalize_endpoint(endpoint: str | None) -> str | None:
def _use_endpoint_backend_when_implicit(args: argparse.Namespace) -> None:
explicit_keys = getattr(args, "explicit_keys", frozenset())
explicit_keys: frozenset[str] = getattr(args, "explicit_keys", frozenset())
if "backend" in explicit_keys:
return
# Upstream vLLM defaults --backend to "openai". Treat that non-explicit
@@ -63,6 +63,8 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
if getattr(args, "daily_omni_save_eval_items", False):
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
if getattr(args, "videomme_save_eval_items", False):
os.environ["VIDEOMME_SAVE_EVAL_ITEMS"] = "1"
request_timeout_s = getattr(args, "omni_request_timeout_s", None)
if request_timeout_s is not None:
set_request_timeout_s(request_timeout_s)
@@ -228,6 +228,7 @@ _BENCHMARK_TRANSITIONAL = (
"SEED_TTS_WER_EVAL",
"SEED_TTS_WER_SAVE_AUDIO_DIR",
"SEED_TTS_WER_SAVE_ITEMS",
"VIDEOMME_SAVE_EVAL_ITEMS",
"VLLM_DAILY_OMNI_MEDIA_REPO",
"VLLM_OMNI_BENCH_AUDIO_CHANNELS",
"VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S",
+1 -1
View File
@@ -1386,7 +1386,7 @@ def normalize_and_validate_diffusion_engine_ingress_kwargs(
| orchestrator_field_names()
# Coordination fields also live on the typed orchestrator config, not
# all of them are present on the CLI-only OrchestratorArgs dataclass.
| frozenset(config_field.name for config_field in fields(VllmOmniOrchestratorConfig))
| frozenset(config_field.name for config_field in fields(cast(Any, VllmOmniOrchestratorConfig)))
)
allowed_fields = stage_consumed_fields | externally_consumed_fields
validate_omni_diffusion_kwargs(normalized, allowed_fields, stage_id=stage_id)
+1 -1
View File
@@ -47,7 +47,7 @@ stages:
fps: 1
num_frames: 128
limit_mm_per_prompt:
# Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
# Daily-Omni: up to 64 interleaved image/audio pairs (1fps).
image: 64
audio: 64
video: 1
@@ -192,6 +192,78 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
)
def add_videomme_cli_args(parser: argparse.ArgumentParser) -> None:
"""Add CLI arguments specific to the Video-MME dataset."""
group = parser.add_argument_group("Video-MME Dataset Options")
group.add_argument(
"--videomme-parquet",
type=str,
default=None,
help="Path to local Video-MME parquet "
"(e.g. videomme/test-00000-of-00001.parquet). When set, Hub QA loading is skipped.",
)
group.add_argument(
"--videomme-video-dir",
type=str,
default=None,
help="Directory containing extracted Video-MME videos (videoID.mp4). "
"Typical layout after unzipping videos_chunked_*.zip: <root>/video/. "
"When using file:// URLs, start the server with --allowed-local-media-path "
"covering this directory.",
)
group.add_argument(
"--videomme-subtitle-dir",
type=str,
default=None,
help="Directory containing Video-MME .srt subtitles (videoID.srt). Used only with --videomme-use-subtitle.",
)
group.add_argument(
"--videomme-pack-mode",
type=str,
choices=["minicpm-frames", "minicpm-interleave", "video_url"],
default="minicpm-frames",
help="Multimodal packing. "
"'minicpm-frames' (default): OmniEvalKit MiniCPM videomme recipe — sampled frames "
"as image_url only (max_frames=96). "
"'minicpm-interleave': OmniEvalKit videomme_short recipe — 1fps frame/audio pairs "
"(max_frames=64). "
"'video_url': single video_url part (models with native video input).",
)
group.add_argument(
"--videomme-max-frames",
type=int,
default=None,
help="Override max sampled frames (OmniEvalKit defaults: 96 for minicpm-frames, 64 for minicpm-interleave).",
)
group.add_argument(
"--videomme-duration",
type=str,
choices=["all", "short", "medium", "long"],
default="all",
help="Filter by Video-MME duration bucket (default: all).",
)
group.add_argument(
"--videomme-use-subtitle",
action="store_true",
default=False,
help="Prepend subtitle text to the user prompt (Video-MME w/ subs setting).",
)
group.add_argument(
"--videomme-inline-local-video",
action="store_true",
default=False,
help="Embed local frames/audio as base64 data URLs so the server does not need "
"--allowed-local-media-path. Increases request size; use for small --num-prompts.",
)
group.add_argument(
"--videomme-save-eval-items",
action="store_true",
default=False,
help="Include per-request Video-MME accuracy rows in the saved JSON under "
"videomme_eval_items. Or set env VIDEOMME_SAVE_EVAL_ITEMS=1.",
)
def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
"""Add CLI arguments for Seed-TTS benchmarks."""
group = parser.add_argument_group("Seed-TTS Dataset Options")
@@ -266,6 +338,7 @@ _OMNI_BENCH_DATASET_CHOICES = (
"seed-tts-design",
"ttsd",
"sound-effect",
"videomme",
)
@@ -332,6 +405,7 @@ def update_omni_help(parser: argparse.ArgumentParser) -> None:
def add_omni_args(parser: argparse.ArgumentParser) -> None:
"""Register all vLLM-Omni serving benchmark arguments."""
add_daily_omni_cli_args(parser)
add_videomme_cli_args(parser)
add_omniinteract_cli_args(parser)
add_seed_tts_cli_args(parser)
add_multi_stage_cli_args(parser)