[Benchmark][MiniCPM-o] Port Video-MME dataset support to Omni bench s… (#6987)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
ea62f86718
commit
9f5521351e
15 files changed
+2432
-59
No files matched your search
+54
-1
@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
|
||||
The API endpoint exposed externally, to which clients send their requests.
|
||||
|
||||
- `--dataset-name`
|
||||
The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
|
||||
The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
|
||||
`omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
|
||||
(`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
|
||||
|
||||
- `--num-prompts`
|
||||
The total number of requests to send, an integer.
|
||||
@@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
|
||||
chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
|
||||
signal, not a transport failure. This local performance test does not score answer accuracy.
|
||||
|
||||
### Video-MME Benchmark
|
||||
|
||||
Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
|
||||
OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
|
||||
mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
|
||||
Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
|
||||
`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
|
||||
to absolute `file://` URLs. For those URLs, start the server with
|
||||
`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
|
||||
|
||||
For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
|
||||
96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
|
||||
with the local video root shared by the benchmark client and server):
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--allowed-local-media-path /path/to/Video-MME \
|
||||
--stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
|
||||
```
|
||||
|
||||
The override applies to this server invocation only. Alternatively, use
|
||||
`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
|
||||
this evaluates a different frame-sampling setting from the 96-frame recipe.
|
||||
|
||||
Run the benchmark against that server:
|
||||
|
||||
```bash
|
||||
vllm bench serve --omni \
|
||||
--backend openai-chat-omni \
|
||||
--dataset-name videomme \
|
||||
--dataset-path /path/to/Video-MME \
|
||||
--videomme-pack-mode minicpm-frames \
|
||||
--videomme-max-frames 96 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--endpoint /v1/chat/completions \
|
||||
--num-prompts 8 \
|
||||
--max-concurrency 1 \
|
||||
--save-result
|
||||
```
|
||||
|
||||
Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
|
||||
the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
|
||||
counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
|
||||
oversampling made the request count differ from unique-question coverage.
|
||||
`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
|
||||
media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
|
||||
failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
|
||||
it does not establish that a supplied local mirror contains the entire official dataset.
|
||||
Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
|
||||
`videomme_eval_items` rows.
|
||||
|
||||
### Multi-Modal Benchmark
|
||||
|
||||
<details class="admonition abstract" markdown="1">
|
||||
|
||||
@@ -47,11 +47,50 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_serve_common_argv,
|
||||
seed_tts_bench_argv,
|
||||
videomme_bench_argv,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||
|
||||
|
||||
def test_videomme_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", "lmms-eval/Video-MME")
|
||||
monkeypatch.delenv("VLLM_VIDEOMME_REPO", raising=False)
|
||||
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
|
||||
|
||||
argv = videomme_bench_argv()
|
||||
|
||||
dataset_idx = argv.index("--dataset-path")
|
||||
assert argv[argv.index("--dataset-name") + 1] == "videomme"
|
||||
assert argv[dataset_idx + 1] == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
def test_videomme_bench_argv_uses_local_directory(monkeypatch, tmp_path: Path):
|
||||
monkeypatch.setenv("VLLM_VIDEOMME_DATASET_PATH", str(tmp_path))
|
||||
monkeypatch.delenv("VIDEOMME_ROOT", raising=False)
|
||||
|
||||
argv = videomme_bench_argv()
|
||||
|
||||
assert argv[argv.index("--dataset-path") + 1] == str(tmp_path.resolve())
|
||||
|
||||
|
||||
def test_sync_dataset_env_preserves_videomme_hf_repo_id(monkeypatch):
|
||||
ns = argparse.Namespace(
|
||||
daily_omni_repo=None,
|
||||
daily_omni_qa_json=None,
|
||||
daily_omni_video_dir=None,
|
||||
videomme_repo=None,
|
||||
videomme_dataset_path="lmms-eval/Video-MME",
|
||||
seed_tts_dataset_path=None,
|
||||
seed_tts_root=None,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("VLLM_VIDEOMME_DATASET_PATH", raising=False)
|
||||
sync_dataset_env_from_ns(ns)
|
||||
|
||||
assert os.environ["VLLM_VIDEOMME_DATASET_PATH"] == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_SEED_TTS_DATASET_PATH", "zhaochenyang20/seed-tts-eval")
|
||||
monkeypatch.delenv("VLLM_SEED_TTS_REPO", raising=False)
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Offline Video-MME sources: local mirrors must not fall back to a Hub id."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import zipfile
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import _validate_videomme
|
||||
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
|
||||
VideoMMEDataset,
|
||||
VideoMMESampleRequest,
|
||||
cached_videomme_snapshots,
|
||||
ensure_videomme_hub_root,
|
||||
ensure_videomme_videos_extracted,
|
||||
resolve_videomme_local_root,
|
||||
videomme_local_parquet,
|
||||
videomme_local_subtitle_dir,
|
||||
videomme_local_video_dir,
|
||||
)
|
||||
from vllm_omni.benchmarks.data_modules.videomme_eval import (
|
||||
compute_videomme_accuracy_metrics,
|
||||
extract_characters_regex,
|
||||
normalize_gold_answer,
|
||||
)
|
||||
from vllm_omni.benchmarks.patch.patch import _looks_like_hf_dataset_id, _videomme_repo_from_args, get_samples
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||
|
||||
|
||||
def _write_parquet_placeholder(root: Path) -> Path:
|
||||
path = root / "videomme" / "test-00000-of-00001.parquet"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"parquet")
|
||||
return path
|
||||
|
||||
|
||||
def test_resolve_local_root_returns_none_for_hub_id() -> None:
|
||||
assert resolve_videomme_local_root("lmms-eval/Video-MME") is None
|
||||
assert resolve_videomme_local_root("") is None
|
||||
assert resolve_videomme_local_root(None) is None
|
||||
|
||||
|
||||
def test_plain_local_mirror(tmp_path: Path) -> None:
|
||||
pq = _write_parquet_placeholder(tmp_path)
|
||||
video = tmp_path / "video" / "abc.mp4"
|
||||
video.parent.mkdir()
|
||||
video.write_bytes(b"fake")
|
||||
(tmp_path / "subtitle").mkdir()
|
||||
(tmp_path / "subtitle" / "abc.srt").write_text("1\n", encoding="utf-8")
|
||||
|
||||
root = resolve_videomme_local_root(str(tmp_path))
|
||||
assert root == tmp_path.resolve()
|
||||
assert videomme_local_parquet(root) == pq.resolve()
|
||||
assert videomme_local_video_dir(root) == tmp_path.resolve() / "video"
|
||||
assert videomme_local_subtitle_dir(root) == tmp_path.resolve() / "subtitle"
|
||||
|
||||
|
||||
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_parquet_placeholder(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
root = resolve_videomme_local_root(str(cache_dir))
|
||||
assert root == snapshot.resolve()
|
||||
assert videomme_local_parquet(root) is not None
|
||||
|
||||
|
||||
def test_hf_cache_dir_with_unusable_ref_falls_back_to_revision(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--lmms-eval--Video-MME"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_parquet_placeholder(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
# An empty ref must not resolve to snapshots/, which holds no dataset files.
|
||||
(cache_dir / "refs" / "main").write_text("", encoding="utf-8")
|
||||
|
||||
assert resolve_videomme_local_root(str(cache_dir)) == snapshot.resolve()
|
||||
|
||||
|
||||
def _stub_failing_hub(monkeypatch: pytest.MonkeyPatch, hub_cache: Path) -> dict[str, object]:
|
||||
"""Point the Hub cache at ``hub_cache`` and make every snapshot_download miss."""
|
||||
from huggingface_hub import constants as hf_constants
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError
|
||||
|
||||
from vllm_omni.transformers_utils import repo_utils
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class _Api:
|
||||
def snapshot_download(self, **kwargs: object) -> str:
|
||||
seen.update(kwargs)
|
||||
raise LocalEntryNotFoundError("no cached snapshot")
|
||||
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_OFFLINE", True)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(hub_cache))
|
||||
monkeypatch.setattr(repo_utils, "hf_api", lambda: _Api())
|
||||
return seen
|
||||
|
||||
|
||||
def test_hub_root_offline_without_cache_points_at_local_options(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
seen = _stub_failing_hub(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(FileNotFoundError, match="--videomme-video-dir") as excinfo:
|
||||
ensure_videomme_hub_root("lmms-lab/Video-MME")
|
||||
|
||||
# Offline runs must consult the cache directly instead of resolving over the network.
|
||||
assert seen["local_files_only"] is True
|
||||
assert "already cached under another repo id" not in str(excinfo.value)
|
||||
|
||||
|
||||
def test_hub_root_offline_names_snapshot_cached_under_another_repo_id(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sibling = tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef"
|
||||
sibling.mkdir(parents=True)
|
||||
_write_parquet_placeholder(sibling)
|
||||
_stub_failing_hub(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(FileNotFoundError) as excinfo:
|
||||
ensure_videomme_hub_root("lmms-lab/Video-MME")
|
||||
|
||||
assert str(sibling.resolve()) in str(excinfo.value)
|
||||
|
||||
|
||||
def test_cached_videomme_snapshots_skips_partial_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
(tmp_path / "datasets--lmms-eval--Video-MME" / "snapshots" / "deadbeef").mkdir(parents=True)
|
||||
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
|
||||
assert cached_videomme_snapshots() == []
|
||||
|
||||
|
||||
def test_extract_videos_rejects_empty_tree(tmp_path: Path) -> None:
|
||||
(tmp_path / "video").mkdir()
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
ensure_videomme_videos_extracted(tmp_path)
|
||||
assert not (tmp_path / ".videomme_videos_extracted").exists()
|
||||
|
||||
|
||||
def test_nested_unzipped_videos_are_discovered(tmp_path: Path) -> None:
|
||||
nested = tmp_path / "videos" / "videos_chunked_01" / "data" / "xyz.mp4"
|
||||
nested.parent.mkdir(parents=True)
|
||||
nested.write_bytes(b"fake")
|
||||
|
||||
assert videomme_local_video_dir(tmp_path) == tmp_path / "videos"
|
||||
|
||||
|
||||
def test_extract_characters_regex_matches_official_prefixes() -> None:
|
||||
assert extract_characters_regex("The best answer is B.") == "B"
|
||||
assert extract_characters_regex("Answer: C") == "C"
|
||||
assert extract_characters_regex("I think D is correct") == "D"
|
||||
assert extract_characters_regex("no letter here at all in this long sentence") is None
|
||||
|
||||
|
||||
def test_normalize_gold_answer() -> None:
|
||||
assert normalize_gold_answer("a") == "A"
|
||||
assert normalize_gold_answer("B. content") == "B"
|
||||
assert normalize_gold_answer("") is None
|
||||
|
||||
|
||||
class _Out:
|
||||
def __init__(self, text: str, *, success: bool = True) -> None:
|
||||
self.generated_text = text
|
||||
self.success = success
|
||||
self.error = "" if success else "http fail"
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics() -> None:
|
||||
reqs = [
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_video_id="v1",
|
||||
videomme_question_id="q1",
|
||||
videomme_duration="short",
|
||||
videomme_domain="Knowledge",
|
||||
videomme_sub_category="Science",
|
||||
videomme_task_type="QA",
|
||||
),
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="B",
|
||||
videomme_video_id="v2",
|
||||
videomme_question_id="q2",
|
||||
videomme_duration="long",
|
||||
videomme_domain="Knowledge",
|
||||
videomme_sub_category="Science",
|
||||
videomme_task_type="QA",
|
||||
),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("The answer is C")])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_correct"] == 1
|
||||
assert metrics["videomme_evaluated_ok"] == 2
|
||||
assert metrics["videomme_accuracy"] == 0.5
|
||||
assert metrics["videomme_submitted"] == 2
|
||||
assert metrics["videomme_unique_question_ids"] == 2
|
||||
assert _validate_videomme(metrics, min_accuracy=0.4) == []
|
||||
assert "videomme_eval_items" not in metrics
|
||||
assert metrics["videomme_per_duration_accuracy"]["short"] == 1.0
|
||||
assert metrics["videomme_per_duration_accuracy"]["long"] == 0.0
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics_http_fail_excluded_from_default() -> None:
|
||||
reqs = [
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_question_id="q1",
|
||||
),
|
||||
VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="B",
|
||||
videomme_question_id="q2",
|
||||
),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(reqs, [_Out("A"), _Out("", success=False)])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
assert metrics["videomme_accuracy_incl_http_fail"] == 0.5
|
||||
assert metrics["videomme_request_failed"] == 1
|
||||
errs = _validate_videomme(metrics, min_accuracy=0.68)
|
||||
assert any("videomme_request_failed" in err for err in errs)
|
||||
|
||||
|
||||
def test_compute_videomme_accuracy_metrics_saves_eval_items() -> None:
|
||||
req = VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_gold_answer="A",
|
||||
videomme_question_id="q1",
|
||||
)
|
||||
metrics = compute_videomme_accuracy_metrics([req], [_Out("A")], include_per_item=True)
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_eval_items"][0]["question_id"] == "q1"
|
||||
|
||||
|
||||
def test_videomme_save_eval_items_cli_sets_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.benchmarks import serve
|
||||
|
||||
seen: dict[str, str | None] = {}
|
||||
|
||||
async def fake_main_async(args: Namespace) -> dict[str, str]:
|
||||
seen["env"] = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS")
|
||||
return {"ok": "1"}
|
||||
|
||||
monkeypatch.delenv("VIDEOMME_SAVE_EVAL_ITEMS", raising=False)
|
||||
monkeypatch.setattr(serve, "main_async", fake_main_async)
|
||||
|
||||
args = Namespace(
|
||||
videomme_save_eval_items=True,
|
||||
seed_tts_wer_eval=False,
|
||||
seed_tts_wer_save_items=False,
|
||||
daily_omni_save_eval_items=False,
|
||||
omni_request_timeout_s=None,
|
||||
endpoint=None,
|
||||
backend="openai-chat-omni",
|
||||
explicit_keys=frozenset(),
|
||||
extra_body=None,
|
||||
print_stage=False,
|
||||
dataset_name="videomme",
|
||||
)
|
||||
assert serve.main(args) == {"ok": "1"}
|
||||
assert seen["env"] == "1"
|
||||
|
||||
|
||||
def _file_uri_path(url: str) -> Path:
|
||||
parsed = urlparse(url)
|
||||
assert parsed.scheme == "file"
|
||||
return Path(unquote(parsed.path))
|
||||
|
||||
|
||||
def test_relative_video_dir_emits_absolute_file_uris_cold_and_warm(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
videos = tmp_path / "videos"
|
||||
videos.mkdir()
|
||||
(videos / "vid1.mp4").write_bytes(b"fake")
|
||||
|
||||
def fake_extract(cls, video_path, *, include_audio, max_num_frames): # noqa: ANN001
|
||||
return [Image.new("RGB", (4, 4), color="red")], []
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
|
||||
monkeypatch.setattr(VideoMMEDataset, "_extract_frames_and_audio", classmethod(fake_extract))
|
||||
|
||||
ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
|
||||
assert ds.video_dir == videos.resolve()
|
||||
|
||||
cold = ds._get_minicpm_frame_parts("vid1", include_audio=False)
|
||||
assert cold is not None
|
||||
cold_path = _file_uri_path(cold[0]["image_url"]["url"])
|
||||
assert cold_path.is_absolute()
|
||||
assert cold_path.is_file()
|
||||
|
||||
warm_ds = VideoMMEDataset(parquet_path="dummy.parquet", video_dir="videos", max_frames=1)
|
||||
warm = warm_ds._get_minicpm_frame_parts("vid1", include_audio=False)
|
||||
assert warm is not None
|
||||
warm_path = _file_uri_path(warm[0]["image_url"]["url"])
|
||||
assert warm_path.is_absolute()
|
||||
assert warm_path == cold_path
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_honors_custom_hub_id() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path="my-org/custom-videomme", hf_name=None)
|
||||
assert _videomme_repo_from_args(args, explicit=True) == "my-org/custom-videomme"
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_defaults_when_path_omitted() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path=None, hf_name=None)
|
||||
assert _videomme_repo_from_args(args, explicit=True) is None
|
||||
|
||||
|
||||
def test_videomme_repo_from_args_rejects_unsupported_path() -> None:
|
||||
args = Namespace(dataset_name="videomme", dataset_path="not-a-local-dir-or-hub-id", hf_name=None)
|
||||
with pytest.raises(ValueError, match="Unsupported Video-MME"):
|
||||
_videomme_repo_from_args(args, explicit=True)
|
||||
|
||||
|
||||
def test_get_samples_rejects_unsupported_videomme_dataset_path() -> None:
|
||||
args = Namespace(
|
||||
dataset_name="videomme",
|
||||
dataset_path="not-a-local-dir-or-hub-id",
|
||||
hf_name=None,
|
||||
backend="openai-chat-omni",
|
||||
seed=0,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Unsupported Video-MME"):
|
||||
get_samples(args, None)
|
||||
|
||||
|
||||
def test_hf_dataset_name_does_not_auto_detect_custom_repo() -> None:
|
||||
args = Namespace(dataset_name="hf", dataset_path="my-org/custom-videomme", hf_name=None)
|
||||
assert _videomme_repo_from_args(args) is None
|
||||
assert _looks_like_hf_dataset_id("my-org/custom-videomme")
|
||||
assert not _looks_like_hf_dataset_id("./videos")
|
||||
official = Namespace(dataset_name="hf", dataset_path="lmms-eval/Video-MME", hf_name=None)
|
||||
assert _videomme_repo_from_args(official) == "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_frames", [0, -1])
|
||||
def test_rejects_nonpositive_max_frames(max_frames: int, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(VideoMMEDataset, "load_data", lambda self: None)
|
||||
with pytest.raises(ValueError, match="max_frames must be positive"):
|
||||
VideoMMEDataset(parquet_path="unused.parquet", max_frames=max_frames)
|
||||
|
||||
|
||||
def test_frame_cache_rebuild_overwrites_incomplete_files(tmp_path: Path) -> None:
|
||||
frame = tmp_path / "frame_0000.jpg"
|
||||
frame.write_bytes(b"interrupted JPEG")
|
||||
VideoMMEDataset._emit_media(tmp_path, frame.name, b"complete JPEG", "image_url", "image/jpeg")
|
||||
assert frame.read_bytes() == b"complete JPEG"
|
||||
|
||||
|
||||
def test_frame_cache_rejects_empty_media(tmp_path: Path) -> None:
|
||||
(tmp_path / "manifest.json").write_text('{"count": 1}', encoding="utf-8")
|
||||
(tmp_path / "frame_0000.jpg").touch()
|
||||
dataset = object.__new__(VideoMMEDataset)
|
||||
assert dataset._cached_parts_from_disk(tmp_path, include_audio=False) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["duplicates", "missing_gold"])
|
||||
def test_accuracy_gate_rejects_incomplete_question_coverage(case: str) -> None:
|
||||
def request(question_id: str, gold: str) -> VideoMMESampleRequest:
|
||||
return VideoMMESampleRequest(
|
||||
prompt="q",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
videomme_question_id=question_id,
|
||||
videomme_gold_answer=gold,
|
||||
)
|
||||
|
||||
requests = [
|
||||
request("q1", "A"),
|
||||
request("q1" if case == "duplicates" else "q2", "" if case == "missing_gold" else "A"),
|
||||
]
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A"), _Out("A")])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
errors = _validate_videomme(metrics, min_accuracy=0.68)
|
||||
assert any(("unique" if case == "duplicates" else "no_gold") in error for error in errors)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def videomme_mirror(tmp_path: Path) -> Path:
|
||||
import pandas as pd
|
||||
|
||||
parquet = tmp_path / "videomme" / "test-00000-of-00001.parquet"
|
||||
parquet.parent.mkdir()
|
||||
pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"videoID": f"v{i}",
|
||||
"question_id": f"q{i}",
|
||||
"question": "Which option is correct?",
|
||||
"options": ["A. first", "B. second", "C. third", "D. fourth"],
|
||||
"answer": "A",
|
||||
"duration": duration,
|
||||
}
|
||||
for i, duration in enumerate(("short", "long"), start=1)
|
||||
]
|
||||
).to_parquet(parquet)
|
||||
videos = tmp_path / "video"
|
||||
videos.mkdir()
|
||||
for i in (1, 2):
|
||||
(videos / f"v{i}.mp4").write_bytes(b"video")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _sample_video_urls(dataset: VideoMMEDataset, monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> list:
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
|
||||
)
|
||||
return dataset.sample(Namespace(encode=lambda text: [1]), **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_oversample", [True, False])
|
||||
def test_missing_video_cannot_pass_accuracy_gate(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch, no_oversample: bool
|
||||
) -> None:
|
||||
(videomme_mirror / "video" / "v2.mp4").unlink()
|
||||
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", disable_shuffle=True)
|
||||
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2, no_oversample=no_oversample)
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_accuracy"] == 1.0
|
||||
assert metrics["videomme_submitted"] == (1 if no_oversample else 2)
|
||||
assert metrics["videomme_skipped_rows"] == 1
|
||||
assert any("videomme_skipped_rows" in error for error in _validate_videomme(metrics, min_accuracy=0.68))
|
||||
|
||||
|
||||
def test_complete_duration_subset_can_pass_accuracy_gate(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
dataset = VideoMMEDataset(dataset_path=str(videomme_mirror), pack_mode="video_url", duration_filter="short")
|
||||
requests = _sample_video_urls(dataset, monkeypatch, num_requests=2700, no_oversample=True)
|
||||
metrics = compute_videomme_accuracy_metrics(requests, [_Out("A") for _ in requests])
|
||||
assert metrics is not None
|
||||
assert metrics["videomme_submitted"] == 1
|
||||
assert metrics["videomme_skipped_rows"] == 0
|
||||
assert _validate_videomme(metrics, min_accuracy=0.68) == []
|
||||
|
||||
|
||||
def test_dataset_direct_hub_loading_resolves_video_assets(
|
||||
videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
downloaded = []
|
||||
|
||||
def download(repo_id: str) -> Path:
|
||||
downloaded.append(repo_id)
|
||||
return videomme_mirror
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.ensure_videomme_hub_root", download)
|
||||
dataset = VideoMMEDataset(dataset_path="my-org/custom-videomme", pack_mode="video_url")
|
||||
assert downloaded == ["my-org/custom-videomme"]
|
||||
assert dataset.video_dir == videomme_mirror / "video"
|
||||
assert len(dataset.data) == 2
|
||||
|
||||
|
||||
def test_local_source_with_explicit_parquet_keeps_video_root(videomme_mirror: Path) -> None:
|
||||
dataset = VideoMMEDataset(
|
||||
dataset_path=str(videomme_mirror),
|
||||
parquet_path=str(videomme_local_parquet(videomme_mirror)),
|
||||
pack_mode="video_url",
|
||||
)
|
||||
assert dataset.video_dir == videomme_mirror / "video"
|
||||
assert len(dataset.data) == 2
|
||||
|
||||
|
||||
def test_local_source_without_parquet_fails_locally(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def unexpected_hub(*args, **kwargs):
|
||||
pytest.fail("An explicitly selected local mirror must not fall back to Hub loading")
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.load_dataset", unexpected_hub)
|
||||
with pytest.raises(FileNotFoundError, match="No Video-MME parquet"):
|
||||
VideoMMEDataset(dataset_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_get_samples_prefers_local_root_over_hf_name(videomme_mirror: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
args = Namespace(
|
||||
dataset_name="videomme",
|
||||
dataset_path=str(videomme_mirror),
|
||||
hf_name="unused-hf-name",
|
||||
backend="openai-chat-omni",
|
||||
seed=0,
|
||||
videomme_pack_mode="video_url",
|
||||
num_prompts=2,
|
||||
request_id_prefix="",
|
||||
no_oversample=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.data_modules.videomme_dataset.get_cached_tokenizer", lambda tokenizer: tokenizer
|
||||
)
|
||||
requests = get_samples(args, Namespace(encode=lambda text: [1]))
|
||||
assert len(requests) == 2
|
||||
assert {request.videomme_question_id for request in requests} == {"q1", "q2"}
|
||||
|
||||
|
||||
def test_extraction_replaces_truncated_video(tmp_path: Path) -> None:
|
||||
video = tmp_path / "video" / "v1.mp4"
|
||||
video.parent.mkdir()
|
||||
video.write_bytes(b"partial")
|
||||
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
|
||||
archive.writestr("nested/v1.mp4", b"complete video")
|
||||
assert ensure_videomme_videos_extracted(tmp_path) == video.parent
|
||||
assert video.read_bytes() == b"complete video"
|
||||
|
||||
|
||||
def test_failed_extraction_does_not_publish_partial_video(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with zipfile.ZipFile(tmp_path / "videos_chunked_01.zip", "w") as archive:
|
||||
archive.writestr("v1.mp4", b"complete video")
|
||||
|
||||
def interrupted_copy(src, dst):
|
||||
dst.write(b"partial")
|
||||
raise OSError("interrupted")
|
||||
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.videomme_dataset.shutil.copyfileobj", interrupted_copy)
|
||||
with pytest.raises(OSError, match="interrupted"):
|
||||
ensure_videomme_videos_extracted(tmp_path)
|
||||
assert list((tmp_path / "video").iterdir()) == []
|
||||
assert not (tmp_path / ".videomme_videos_extracted").exists()
|
||||
@@ -168,7 +168,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
|
||||
EnvironmentVariableCategory.INTERNAL: 2,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
|
||||
"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
|
||||
|
||||
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
|
||||
point to existing files, those are used with inline video. Otherwise the benchmark falls back to
|
||||
@@ -10,6 +10,10 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
|
||||
Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
|
||||
and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
|
||||
|
||||
Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
|
||||
``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
|
||||
``lmms-eval/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
|
||||
|
||||
Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
|
||||
and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
|
||||
"""
|
||||
@@ -26,6 +30,7 @@ from typing import Any, Protocol
|
||||
|
||||
DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
|
||||
DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
|
||||
DEFAULT_VIDEOMME_HF_REPO = "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
class OmniBenchServerEndpoint(Protocol):
|
||||
@@ -41,6 +46,7 @@ def build_acc_benchmark_cli_argv(
|
||||
*,
|
||||
skip_seed: bool,
|
||||
skip_daily: bool,
|
||||
skip_videomme: bool = True,
|
||||
num_prompts: int | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> list[str]:
|
||||
@@ -50,6 +56,9 @@ def build_acc_benchmark_cli_argv(
|
||||
``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
|
||||
``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
|
||||
matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
|
||||
|
||||
Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
|
||||
callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
|
||||
"""
|
||||
n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
|
||||
n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
|
||||
@@ -72,6 +81,12 @@ def build_acc_benchmark_cli_argv(
|
||||
argv.append("--skip-seed-tts")
|
||||
if skip_daily:
|
||||
argv.append("--skip-daily-omni")
|
||||
if skip_videomme:
|
||||
argv.append("--skip-videomme")
|
||||
else:
|
||||
argv.append("--run-videomme")
|
||||
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
|
||||
argv.extend(["--videomme-repo", repo])
|
||||
return argv
|
||||
|
||||
|
||||
@@ -101,6 +116,28 @@ def daily_omni_bench_argv() -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def videomme_bench_argv() -> list[str]:
|
||||
"""CLI args for Video-MME (after ``vllm bench serve --omni``).
|
||||
|
||||
Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
|
||||
|
||||
* If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
|
||||
use that local mirror as ``--dataset-path``.
|
||||
* Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-eval/Video-MME``); the child
|
||||
bench downloads via ``huggingface_hub.snapshot_download``.
|
||||
"""
|
||||
source = (
|
||||
os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip()
|
||||
or os.environ.get("VIDEOMME_ROOT", "").strip()
|
||||
or os.environ.get("VLLM_VIDEOMME_REPO", "").strip()
|
||||
or DEFAULT_VIDEOMME_HF_REPO
|
||||
)
|
||||
path = Path(source).expanduser()
|
||||
if path.is_dir():
|
||||
source = str(path.resolve())
|
||||
return ["--dataset-name", "videomme", "--dataset-path", source]
|
||||
|
||||
|
||||
def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
"""CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
|
||||
dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Accuracy (and light perf) checks for Qwen3-Omni via ``vllm bench serve --omni``.
|
||||
|
||||
The standalone CLI uses small ``--num-prompts`` / ``--max-concurrency`` defaults suitable for
|
||||
@@ -11,6 +12,9 @@ H100 / MI325) because they launch the live Omni server inside the test.
|
||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||
3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
|
||||
stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
|
||||
**0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
@@ -21,7 +25,8 @@ Prerequisites
|
||||
On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
|
||||
only drives the **client** benchmark.
|
||||
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
|
||||
``videomme``).
|
||||
|
||||
* **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
|
||||
the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
|
||||
@@ -35,6 +40,12 @@ Prerequisites
|
||||
|
||||
pip install 'vllm-omni[dev]'
|
||||
|
||||
* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-eval/Video-MME`` (override with
|
||||
``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
|
||||
``huggingface_hub.snapshot_download``. Pass an existing local root with ``--videomme-dataset-path`` /
|
||||
``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without
|
||||
``--allowed-local-media-path``, pass ``--videomme-inline-local-video``.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Pytest (same checks; needs a running server)::
|
||||
@@ -55,6 +66,24 @@ Skip one suite, tighten gates::
|
||||
--skip-daily-omni \\
|
||||
--max-seed-tts-mean-wer 0.35 \\
|
||||
--min-seed-tts-mean-sim 0.75
|
||||
|
||||
MiniCPM-o Video-MME (w/o subs). Start the server with the 96-image stage 0
|
||||
limit first; replace /path/to/Video-MME with the local dataset root::
|
||||
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \\
|
||||
--allowed-local-media-path /path/to/Video-MME \\
|
||||
--stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
|
||||
|
||||
Then run the opt-in evaluation in another terminal::
|
||||
|
||||
python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
|
||||
--model openbmb/MiniCPM-o-4_5 \\
|
||||
--videomme-dataset-path /path/to/Video-MME \\
|
||||
--skip-daily-omni --skip-seed-tts --run-videomme \\
|
||||
--videomme-pack-mode minicpm-frames \\
|
||||
--videomme-max-frames 96 \\
|
||||
--temperature 0 --output-len 128 \\
|
||||
--min-videomme-accuracy 0.68
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -75,6 +104,7 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
load_benchmark_result,
|
||||
run_vllm_bench_subprocess,
|
||||
seed_tts_bench_argv,
|
||||
videomme_bench_argv,
|
||||
)
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
@@ -102,6 +132,42 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
|
||||
errs: list[str] = []
|
||||
acc = result.get("videomme_accuracy")
|
||||
if acc is None:
|
||||
errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
|
||||
return errs
|
||||
ev = int(result.get("videomme_evaluated_ok", 0) or 0)
|
||||
if ev <= 0:
|
||||
errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
|
||||
failed = int(result.get("videomme_request_failed", 0) or 0)
|
||||
if failed > 0:
|
||||
errs.append(
|
||||
f"videomme_request_failed={failed}; the accuracy gate requires every "
|
||||
"gold-labeled request to complete HTTP successfully "
|
||||
"(videomme_accuracy excludes HTTP failures)."
|
||||
)
|
||||
submitted = int(result.get("videomme_submitted", 0) or 0)
|
||||
unique = int(result.get("videomme_unique_question_ids", 0) or 0)
|
||||
if submitted <= 0 or unique != submitted:
|
||||
errs.append(
|
||||
f"Video-MME coverage is incomplete: submitted={submitted}, unique={unique}; "
|
||||
"the accuracy gate requires distinct questions, not oversampled replacements."
|
||||
)
|
||||
skipped = int(result.get("videomme_skipped_rows", 0) or 0)
|
||||
if skipped:
|
||||
errs.append(
|
||||
f"videomme_skipped_rows={skipped}; fix missing or unreadable source rows before accuracy evaluation."
|
||||
)
|
||||
no_gold = int(result.get("videomme_no_gold", 0) or 0)
|
||||
if no_gold:
|
||||
errs.append(f"videomme_no_gold={no_gold}; every submitted question must have a gold answer.")
|
||||
if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
|
||||
errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_seed_tts(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
@@ -130,7 +196,7 @@ def _validate_seed_tts(
|
||||
|
||||
|
||||
def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
"""Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
|
||||
"""Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
|
||||
repo = getattr(ns, "daily_omni_repo", None)
|
||||
if repo is not None and str(repo).strip():
|
||||
os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
|
||||
@@ -138,6 +204,14 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
|
||||
if ns.daily_omni_video_dir is not None:
|
||||
os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
|
||||
vm_repo = getattr(ns, "videomme_repo", None)
|
||||
if vm_repo is not None and str(vm_repo).strip():
|
||||
os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
|
||||
vm_path = getattr(ns, "videomme_dataset_path", None)
|
||||
if vm_path is not None and str(vm_path).strip():
|
||||
raw = str(vm_path).strip()
|
||||
p = Path(raw).expanduser()
|
||||
os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
|
||||
if ns.seed_tts_dataset_path is not None:
|
||||
# ``--seed-tts-dataset-path`` accepts either a local directory or a
|
||||
# Hugging Face repo id. Only resolve to an absolute filesystem path
|
||||
@@ -157,6 +231,9 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
"VLLM_DAILY_OMNI_REPO",
|
||||
"VLLM_DAILY_OMNI_QA_JSON",
|
||||
"VLLM_DAILY_OMNI_VIDEO_DIR",
|
||||
"VLLM_VIDEOMME_REPO",
|
||||
"VLLM_VIDEOMME_DATASET_PATH",
|
||||
"VIDEOMME_ROOT",
|
||||
"VLLM_SEED_TTS_DATASET_PATH",
|
||||
"SEED_TTS_ROOT",
|
||||
)
|
||||
@@ -225,6 +302,46 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
return out
|
||||
|
||||
|
||||
def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"omni_acc_videomme_{tag}.json"
|
||||
extra = json.loads(ns.videomme_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
+ videomme_bench_argv()
|
||||
+ [
|
||||
"--disable-shuffle",
|
||||
"--videomme-pack-mode",
|
||||
ns.videomme_pack_mode,
|
||||
"--videomme-duration",
|
||||
ns.videomme_duration,
|
||||
"--extra-body",
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.videomme_max_frames is not None:
|
||||
argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
|
||||
if ns.videomme_use_subtitle:
|
||||
argv.append("--videomme-use-subtitle")
|
||||
if ns.videomme_inline_local_video:
|
||||
argv.append("--videomme-inline-local-video")
|
||||
if ns.videomme_save_eval_items:
|
||||
argv.append("--videomme-save-eval-items")
|
||||
if ns.videomme_video_dir is not None:
|
||||
argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
|
||||
if ns.videomme_parquet is not None:
|
||||
argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
|
||||
if ns.videomme_subtitle_dir is not None:
|
||||
argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
run_vllm_bench_subprocess(vllm, argv)
|
||||
out = Path(ns.result_dir) / result_filename
|
||||
if not out.is_file():
|
||||
raise FileNotFoundError(f"Expected result JSON at {out}")
|
||||
return out
|
||||
|
||||
|
||||
def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
@@ -312,6 +429,20 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
|
||||
p.add_argument("--skip-daily-omni", action="store_true")
|
||||
p.add_argument("--skip-seed-tts", action="store_true")
|
||||
# Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
|
||||
p.set_defaults(skip_videomme=True)
|
||||
p.add_argument(
|
||||
"--skip-videomme",
|
||||
action="store_true",
|
||||
dest="skip_videomme",
|
||||
help="Skip Video-MME (default).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--run-videomme",
|
||||
action="store_false",
|
||||
dest="skip_videomme",
|
||||
help="Enable the Video-MME accuracy suite.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--daily-omni-repo",
|
||||
@@ -358,6 +489,87 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--videomme-repo",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
|
||||
"Default when no local path is set: lmms-eval/Video-MME. "
|
||||
"Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-dataset-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
|
||||
"download; Hub ids are also accepted and passed through as --dataset-path. "
|
||||
"Sets VLLM_VIDEOMME_DATASET_PATH.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-parquet",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit path to videomme/test-*.parquet.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-video-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit directory of extracted Video-MME videos.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-subtitle-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional subtitle directory (used with --videomme-use-subtitle).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-pack-mode",
|
||||
choices=("minicpm-frames", "minicpm-interleave", "video_url"),
|
||||
default="minicpm-frames",
|
||||
help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-max-frames",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-duration",
|
||||
choices=("all", "short", "medium", "long"),
|
||||
default="all",
|
||||
help="Filter Video-MME by duration bucket (default: all).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-use-subtitle",
|
||||
action="store_true",
|
||||
help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-inline-local-video",
|
||||
action="store_true",
|
||||
help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-extra-body-json",
|
||||
default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
|
||||
help="JSON merged into each chat request for Video-MME.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-save-eval-items",
|
||||
action="store_true",
|
||||
help="Include per-request Video-MME accuracy rows in the saved JSON.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--min-videomme-accuracy",
|
||||
type=float,
|
||||
default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
|
||||
help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
|
||||
"Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
|
||||
"override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--seed-tts-dataset-path",
|
||||
type=str,
|
||||
@@ -427,7 +639,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
|
||||
|
||||
|
||||
def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
"""Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
|
||||
"""Run Daily-Omni / Video-MME / Seed-TTS client benches against a running server; return 0 on success."""
|
||||
failed: list[str] = []
|
||||
|
||||
with _preserve_benchmark_dataset_env():
|
||||
@@ -451,6 +663,21 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_videomme:
|
||||
path = run_videomme(ns, vllm)
|
||||
print(f"\n[Video-MME] result JSON: {path}", flush=True)
|
||||
data = load_benchmark_result(path)
|
||||
errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
|
||||
if errs:
|
||||
failed.extend([f"[Video-MME] {e}" for e in errs])
|
||||
else:
|
||||
print(
|
||||
f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
|
||||
f"evaluated_ok={data.get('videomme_evaluated_ok')} "
|
||||
f"per_duration={data.get('videomme_per_duration_accuracy')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_seed_tts:
|
||||
path = run_seed_tts(ns, vllm)
|
||||
print(f"\n[Seed-TTS] result JSON: {path}", flush=True)
|
||||
|
||||
@@ -178,7 +178,7 @@ class TestDeployTopology:
|
||||
0.18,
|
||||
]
|
||||
assert sum(memory_utilizations) <= 0.9 + 1e-6
|
||||
# Daily-Omni minicpm-interleave: up to 64 image/audio items (+ optional video).
|
||||
# Keep the default Daily-Omni budget; Video-MME overrides stage 0 explicitly.
|
||||
assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {
|
||||
"image": 64,
|
||||
"audio": 64,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve.
|
||||
|
||||
Answer extraction ports ``extract_characters_regex`` from the official Video-MME
|
||||
script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval).
|
||||
|
||||
**Denominator:** responses where no A–D can be extracted are counted as wrong, which
|
||||
matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The
|
||||
official standalone script instead drops them from ``answered``; ``videomme_parse_failed``
|
||||
is reported so that view can be recovered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest
|
||||
|
||||
_VALID = frozenset("ABCD")
|
||||
|
||||
VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long")
|
||||
|
||||
#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns).
|
||||
_DIMENSIONS: dict[str, str] = {
|
||||
"duration": "videomme_duration",
|
||||
"domain": "videomme_domain",
|
||||
"sub_category": "videomme_sub_category",
|
||||
"task": "videomme_task_type",
|
||||
}
|
||||
|
||||
# Official prefix list, with the two accidental string concatenations in the upstream
|
||||
# tuple (missing commas after "The best option is" / "Best answer:") split apart.
|
||||
_ANSWER_PREFIXES = (
|
||||
"The best answer is",
|
||||
"The correct answer is",
|
||||
"The answer is",
|
||||
"The answer",
|
||||
"The best option is",
|
||||
"The correct option is",
|
||||
"Best answer:",
|
||||
"Best option:",
|
||||
"Answer:",
|
||||
"Option:",
|
||||
"The correct answer",
|
||||
"The correct option",
|
||||
)
|
||||
|
||||
|
||||
def extract_characters_regex(text: str | None) -> str | None:
|
||||
"""Port of official Video-MME / lmms-eval ``extract_characters_regex``."""
|
||||
if not text:
|
||||
return None
|
||||
s = str(text).strip()
|
||||
for prefix in _ANSWER_PREFIXES:
|
||||
s = s.replace(prefix, "")
|
||||
if len(s.split()) > 10 and not re.search(r"[ABCD]", s):
|
||||
return None
|
||||
match = re.search(r"[ABCD]", s)
|
||||
return match.group(0) if match else None
|
||||
|
||||
|
||||
def normalize_gold_answer(gold: str) -> str | None:
|
||||
g = (gold or "").strip().upper()
|
||||
if len(g) == 1 and g in _VALID:
|
||||
return g
|
||||
m = re.search(r"([ABCD])\b", g)
|
||||
return m.group(1).upper() if m else None
|
||||
|
||||
|
||||
def _bucket(req: VideoMMESampleRequest, dimension: str) -> str:
|
||||
value = (getattr(req, _DIMENSIONS[dimension]) or "").strip()
|
||||
if dimension == "duration":
|
||||
return value.lower()
|
||||
return value or "unknown"
|
||||
|
||||
|
||||
def compute_videomme_accuracy_metrics(
|
||||
input_requests: list[Any],
|
||||
outputs: list[RequestFuncOutput],
|
||||
*,
|
||||
include_per_item: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
"""If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats.
|
||||
|
||||
Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are
|
||||
excluded from ``videomme_accuracy`` and counted in
|
||||
``videomme_accuracy_incl_http_fail`` instead. The accuracy wrapper gate
|
||||
(``_validate_videomme``) additionally rejects HTTP failures, duplicate question
|
||||
IDs, skipped source rows and missing gold answers. Direct
|
||||
``vllm bench serve`` still reports both metrics without that completeness
|
||||
check.
|
||||
|
||||
``videomme_submitted`` / ``videomme_unique_question_ids`` show when
|
||||
oversampling made the request count differ from unique question coverage.
|
||||
``videomme_skipped_rows`` counts source rows skipped during sampling (for
|
||||
example, missing or undecodable media), before any oversampling.
|
||||
"""
|
||||
if not input_requests or len(input_requests) != len(outputs):
|
||||
return None
|
||||
if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests):
|
||||
return None
|
||||
|
||||
stats: dict[str, dict[str, dict[str, int]]] = {
|
||||
dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {})
|
||||
for dim in _DIMENSIONS
|
||||
}
|
||||
items: list[dict[str, Any]] = []
|
||||
correct = evaluated = no_gold = request_failed = parse_failed = 0
|
||||
|
||||
for req, out in zip(input_requests, outputs, strict=True):
|
||||
assert isinstance(req, VideoMMESampleRequest)
|
||||
gold_raw = (req.videomme_gold_answer or "").strip()
|
||||
buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS}
|
||||
row: dict[str, Any] = {
|
||||
"request_id": req.request_id,
|
||||
"video_id": req.videomme_video_id,
|
||||
"question_id": req.videomme_question_id,
|
||||
**buckets,
|
||||
}
|
||||
|
||||
if not gold_raw:
|
||||
no_gold += 1
|
||||
if include_per_item:
|
||||
items.append({**row, "skipped": True, "reason": "no_gold"})
|
||||
continue
|
||||
|
||||
evaluated += 1
|
||||
gold_norm = normalize_gold_answer(gold_raw)
|
||||
pred = extract_characters_regex(out.generated_text) if out.success else None
|
||||
is_correct = pred is not None and pred.upper() == (gold_norm or gold_raw).upper()
|
||||
|
||||
if not out.success:
|
||||
request_failed += 1
|
||||
else:
|
||||
if pred is None:
|
||||
parse_failed += 1
|
||||
# Only successful responses enter the per-dimension denominators, mirroring
|
||||
# the overall videomme_accuracy definition.
|
||||
for dim, key in buckets.items():
|
||||
bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0})
|
||||
bucket["total"] += 1
|
||||
bucket["correct"] += int(is_correct)
|
||||
|
||||
if is_correct:
|
||||
correct += 1
|
||||
|
||||
if include_per_item:
|
||||
items.append(
|
||||
{
|
||||
**row,
|
||||
"gold": gold_raw,
|
||||
"gold_normalized": gold_norm,
|
||||
"predicted": pred,
|
||||
"correct": is_correct,
|
||||
"parse_failed": out.success and pred is None,
|
||||
**({"error": (out.error or "")[:500]} if not out.success else {}),
|
||||
}
|
||||
)
|
||||
|
||||
evaluated_ok = evaluated - request_failed
|
||||
unique_question_ids = {
|
||||
(req.videomme_question_id or req.request_id) for req in input_requests if isinstance(req, VideoMMESampleRequest)
|
||||
}
|
||||
result: dict[str, Any] = {
|
||||
"videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None,
|
||||
"videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None,
|
||||
"videomme_correct": correct,
|
||||
"videomme_evaluated": evaluated,
|
||||
"videomme_evaluated_ok": evaluated_ok,
|
||||
"videomme_no_gold": no_gold,
|
||||
"videomme_request_failed": request_failed,
|
||||
"videomme_parse_failed": parse_failed,
|
||||
"videomme_submitted": len(input_requests),
|
||||
"videomme_unique_question_ids": len(unique_question_ids),
|
||||
"videomme_skipped_rows": max(req.videomme_skipped_rows for req in input_requests),
|
||||
}
|
||||
for dim, per_bucket in stats.items():
|
||||
result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()}
|
||||
result[f"videomme_per_{dim}_accuracy"] = {
|
||||
k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items()
|
||||
}
|
||||
if include_per_item:
|
||||
result["videomme_eval_items"] = items
|
||||
return result
|
||||
|
||||
|
||||
def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None:
|
||||
"""Pretty-print the Video-MME accuracy block (stdout)."""
|
||||
acc = metrics.get("videomme_accuracy")
|
||||
if acc is None and not metrics.get("videomme_evaluated", 0):
|
||||
return
|
||||
|
||||
print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="="))
|
||||
ok = int(metrics.get("videomme_evaluated_ok", 0) or 0)
|
||||
if ok and acc is not None:
|
||||
print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}")
|
||||
else:
|
||||
print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)")
|
||||
for label, key in (
|
||||
("Submitted (gold present):", "videomme_evaluated"),
|
||||
("Successful HTTP (denominator):", "videomme_evaluated_ok"),
|
||||
("Correct:", "videomme_correct"),
|
||||
("Skipped (no gold):", "videomme_no_gold"),
|
||||
("HTTP failed:", "videomme_request_failed"),
|
||||
("Parsed OK but no A-D found:", "videomme_parse_failed"),
|
||||
("Submitted requests:", "videomme_submitted"),
|
||||
("Unique questions:", "videomme_unique_question_ids"),
|
||||
("Skipped source rows:", "videomme_skipped_rows"),
|
||||
):
|
||||
print(f"{label:<40} {metrics.get(key, 0):<10}")
|
||||
|
||||
for dim, title in (
|
||||
("duration", "Duration"),
|
||||
("domain", "Domain"),
|
||||
("sub_category", "Sub Category"),
|
||||
("task", "Task Type"),
|
||||
):
|
||||
per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {}
|
||||
if not per_acc:
|
||||
continue
|
||||
counts = metrics.get(f"videomme_per_{dim}") or {}
|
||||
names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc)
|
||||
print(f"\n--- Accuracy by {title} ---")
|
||||
for name in names:
|
||||
st = counts.get(name) or {}
|
||||
total = int(st.get("total", 0) or 0)
|
||||
value = per_acc.get(name)
|
||||
if total and value is not None:
|
||||
print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}")
|
||||
else:
|
||||
print(f"{name}: 0/0 = N/A")
|
||||
print("=" * 50)
|
||||
@@ -14,7 +14,7 @@ import time
|
||||
import traceback
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -67,6 +67,12 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||
)
|
||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
|
||||
VIDEOMME_DEFAULT_HF_REPO,
|
||||
VideoMMEDataset,
|
||||
VideoMMESampleRequest,
|
||||
resolve_videomme_local_root,
|
||||
)
|
||||
from vllm_omni.benchmarks.omniinteract import (
|
||||
VIDEO_FPS,
|
||||
OmniInteractBenchmarkConfig,
|
||||
@@ -283,14 +289,14 @@ def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict
|
||||
return out
|
||||
|
||||
|
||||
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
|
||||
if not isinstance(sample, DailyOmniSampleRequest):
|
||||
def _attach_omni_chat_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni / Video-MME."""
|
||||
if not isinstance(sample, (DailyOmniSampleRequest, VideoMMESampleRequest)):
|
||||
return
|
||||
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
|
||||
if sample.omni_chat_messages is not None:
|
||||
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
|
||||
else:
|
||||
elif isinstance(sample, DailyOmniSampleRequest):
|
||||
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
|
||||
|
||||
|
||||
@@ -334,6 +340,29 @@ def _attach_omniinteract_to_request_func_input(sample: SampleRequest, rfi: Reque
|
||||
setattr(rfi, "omniinteract_prepared_input", sample.omniinteract_prepared_input)
|
||||
|
||||
|
||||
def _async_limiter(max_concurrency: int | None) -> contextlib.AbstractAsyncContextManager[object]:
|
||||
if max_concurrency:
|
||||
return asyncio.Semaphore(max_concurrency)
|
||||
return contextlib.nullcontext()
|
||||
|
||||
|
||||
def _as_float(value: object, default: float = 0.0) -> float:
|
||||
"""Coerce loosely-typed metric dict values to ``float`` for mypy.
|
||||
|
||||
Strings are parsed rather than dropped: JSON-sourced session metrics such as
|
||||
``audio_duration_ms`` may arrive quoted, and silently reporting ``0.0`` would
|
||||
corrupt the derived TTFT / RTF numbers.
|
||||
"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float, str)):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return default
|
||||
|
||||
|
||||
def _append_error(existing: str, message: str) -> str:
|
||||
return f"{existing}\n{message}" if existing else message
|
||||
|
||||
@@ -404,9 +433,13 @@ def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None:
|
||||
for sample in input_requests:
|
||||
if not isinstance(sample, OmniInteractSampleRequest):
|
||||
continue
|
||||
root = sample.omniinteract_options.output_root.resolve()
|
||||
options = sample.omniinteract_options
|
||||
case = sample.omniinteract_case
|
||||
if options is None or case is None:
|
||||
raise RuntimeError("OmniInteract benchmark input lost its dataset identity")
|
||||
root = options.output_root.resolve()
|
||||
roots.add(root)
|
||||
clear_case_artifacts(sample.omniinteract_options.output_root, sample.omniinteract_case)
|
||||
clear_case_artifacts(options.output_root, case)
|
||||
for root in roots:
|
||||
clear_batch_artifacts(root)
|
||||
|
||||
@@ -428,11 +461,55 @@ def _daily_omni_repo_from_args(args) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_hf_dataset_id(value: str) -> bool:
|
||||
"""True for Hub ids such as ``org/name``; false for local paths."""
|
||||
raw = value.strip()
|
||||
if not raw or raw.startswith((".", "~", "/")) or "\\" in raw:
|
||||
return False
|
||||
parts = raw.split("/")
|
||||
return len(parts) == 2 and all(part.strip() and part.strip() not in (".", "..") for part in parts)
|
||||
|
||||
|
||||
def _videomme_repo_from_args(args, *, explicit: bool = False) -> str | None:
|
||||
"""Resolve a Hugging Face repo id for Video-MME from CLI args.
|
||||
|
||||
``--dataset-name hf`` auto-detect only recognizes the official
|
||||
``lmms-eval/Video-MME`` id so a custom Hub dataset is not silently treated
|
||||
as Video-MME. Explicit ``--dataset-name videomme`` accepts any ``org/name``
|
||||
Hub id (for ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO`` overrides) and
|
||||
raises when ``--dataset-path`` is neither a local directory nor a Hub id.
|
||||
"""
|
||||
official = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS}
|
||||
official.add(VIDEOMME_DEFAULT_HF_REPO.lower())
|
||||
candidates: list[str] = []
|
||||
for attr in ("dataset_path", "hf_name"):
|
||||
val = getattr(args, attr, None)
|
||||
if isinstance(val, str) and val.strip():
|
||||
candidates.append(val.strip())
|
||||
for raw in candidates:
|
||||
if resolve_videomme_local_root(raw) is not None:
|
||||
continue
|
||||
if raw.lower() in official:
|
||||
return raw
|
||||
if explicit and _looks_like_hf_dataset_id(raw):
|
||||
return raw
|
||||
if explicit:
|
||||
raise ValueError(
|
||||
f"Unsupported Video-MME --dataset-path={raw!r}. Pass an existing local "
|
||||
"directory, a Hugging Face dataset id (org/name), or omit --dataset-path "
|
||||
f"to use {VIDEOMME_DEFAULT_HF_REPO}."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_samples(args, tokenizer):
|
||||
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
|
||||
is_daily_omni = args.dataset_name == "daily-omni" or (
|
||||
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
|
||||
)
|
||||
is_videomme = args.dataset_name == "videomme" or (
|
||||
args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None
|
||||
)
|
||||
is_seed_tts = args.dataset_name in (
|
||||
"seed-tts",
|
||||
"seed-tts-text",
|
||||
@@ -449,7 +526,7 @@ def get_samples(args, tokenizer):
|
||||
"openai-realtime-duplex",
|
||||
"daily-omni",
|
||||
]
|
||||
is_omni_dataset = is_daily_omni or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
|
||||
is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
|
||||
|
||||
if not is_omni_backend and not is_omni_dataset:
|
||||
# Not an omni-related request, delegate to original implementation
|
||||
@@ -625,6 +702,54 @@ def get_samples(args, tokenizer):
|
||||
)
|
||||
return input_requests
|
||||
|
||||
if is_videomme:
|
||||
if args.backend not in ["openai-chat-omni", "daily-omni"]:
|
||||
raise ValueError(
|
||||
f"Video-MME dataset requires a multimodal backend that supports video. "
|
||||
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
|
||||
)
|
||||
|
||||
# Resolve the source identity here; the dataset owns loading and extraction.
|
||||
local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or (
|
||||
resolve_videomme_local_root(getattr(args, "hf_name", None))
|
||||
)
|
||||
source = (
|
||||
str(local_root)
|
||||
if local_root is not None
|
||||
else (_videomme_repo_from_args(args, explicit=args.dataset_name == "videomme") or VIDEOMME_DEFAULT_HF_REPO)
|
||||
)
|
||||
dataset = VideoMMEDataset(
|
||||
parquet_path=getattr(args, "videomme_parquet", None),
|
||||
dataset_path=source,
|
||||
dataset_split=getattr(args, "hf_split", None) or "test",
|
||||
dataset_subset=getattr(args, "hf_subset", None),
|
||||
random_seed=args.seed,
|
||||
video_dir=getattr(args, "videomme_video_dir", None),
|
||||
subtitle_dir=getattr(args, "videomme_subtitle_dir", None),
|
||||
pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"),
|
||||
max_frames=getattr(args, "videomme_max_frames", None),
|
||||
duration_filter=getattr(args, "videomme_duration", "all"),
|
||||
use_subtitle=getattr(args, "videomme_use_subtitle", False),
|
||||
inline_local_video=getattr(args, "videomme_inline_local_video", False),
|
||||
trust_remote_code=getattr(args, "trust_remote_code", False),
|
||||
no_stream=getattr(args, "no_stream", False),
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
)
|
||||
|
||||
out_len = getattr(args, "output_len", None)
|
||||
if out_len is None:
|
||||
out_len = getattr(args, "hf_output_len", None)
|
||||
if out_len is None:
|
||||
out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN
|
||||
|
||||
return dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=args.num_prompts,
|
||||
output_len=out_len,
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
|
||||
if is_seed_tts:
|
||||
if args.backend not in (
|
||||
"openai-audio-speech",
|
||||
@@ -711,7 +836,7 @@ datasets.get_samples = get_samples
|
||||
|
||||
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
|
||||
if _serve_mod is not None:
|
||||
_serve_mod.get_samples = get_samples
|
||||
setattr(_serve_mod, "get_samples", get_samples)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2637,7 +2762,9 @@ async def async_request_openai_realtime_duplex(
|
||||
output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0
|
||||
output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0
|
||||
output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0
|
||||
output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
|
||||
output.audio_duration = (
|
||||
sum((_as_float(metric.get("audio_duration_ms")) for metric in turn_metrics), start=0.0) / 1000.0
|
||||
)
|
||||
output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ)
|
||||
output.latency = request_finished_at - output.start_time
|
||||
output.tts_turn_pcm_bytes = turn_pcm_bytes
|
||||
@@ -2811,7 +2938,7 @@ async def benchmark(
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_omni_chat_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_omniinteract_to_request_func_input(input_requests[0], test_input)
|
||||
|
||||
@@ -2836,7 +2963,7 @@ async def benchmark(
|
||||
if num_warmups > 0:
|
||||
print(f"Warming up with {num_warmups} requests...")
|
||||
warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups)
|
||||
warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
|
||||
warmup_semaphore = _async_limiter(max_concurrency)
|
||||
warmup_tasks = []
|
||||
|
||||
async def warmup_limited_request_func():
|
||||
@@ -2854,12 +2981,13 @@ async def benchmark(
|
||||
|
||||
print("Starting main benchmark run...")
|
||||
|
||||
lora_iter: Iterator[str] | None = None
|
||||
if lora_modules:
|
||||
lora_modules_list = list(lora_modules)
|
||||
if lora_assignment == "round-robin":
|
||||
lora_modules = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
|
||||
lora_iter = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
|
||||
else:
|
||||
lora_modules = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
|
||||
lora_iter = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
|
||||
|
||||
if profile:
|
||||
print("Starting profiler...")
|
||||
@@ -2877,7 +3005,7 @@ async def benchmark(
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
|
||||
_attach_omni_chat_to_request_func_input(input_requests[0], profile_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
|
||||
profile_output = await request_func(request_func_input=profile_input, session=session)
|
||||
if profile_output.success:
|
||||
@@ -2898,7 +3026,7 @@ async def benchmark(
|
||||
|
||||
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
|
||||
|
||||
semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
|
||||
semaphore = _async_limiter(max_concurrency)
|
||||
|
||||
async def limited_request_func(request_func_input, session, pbar):
|
||||
async with semaphore:
|
||||
@@ -2969,8 +3097,8 @@ async def benchmark(
|
||||
)
|
||||
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
||||
req_model_id, req_model_name = model_id, model_name
|
||||
if lora_modules:
|
||||
req_lora_module = next(lora_modules)
|
||||
if lora_iter is not None:
|
||||
req_lora_module = next(lora_iter)
|
||||
req_model_id, req_model_name = req_lora_module, req_lora_module
|
||||
|
||||
request_func_input = RequestFuncInput(
|
||||
@@ -2988,7 +3116,7 @@ async def benchmark(
|
||||
request_id=request_id,
|
||||
chat_messages=request.chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(request, request_func_input)
|
||||
_attach_omni_chat_to_request_func_input(request, request_func_input)
|
||||
_attach_seed_tts_to_request_func_input(request, request_func_input)
|
||||
_attach_omniinteract_to_request_func_input(request, request_func_input)
|
||||
tasks.append(
|
||||
@@ -3007,6 +3135,8 @@ async def benchmark(
|
||||
|
||||
omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs)
|
||||
|
||||
metrics: Any
|
||||
actual_output_lens: list[int] | int
|
||||
if task_type == TaskType.GENERATION:
|
||||
metrics, actual_output_lens = calculate_metrics(
|
||||
input_requests=input_requests,
|
||||
@@ -3031,6 +3161,10 @@ async def benchmark(
|
||||
actual_output_lens = 0
|
||||
|
||||
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
||||
# ``make_dataclass`` types this class as ``type``, so isinstance()
|
||||
# narrows to ``object`` and attribute access fails. Keep the runtime
|
||||
# check and read fields through ``Any``.
|
||||
mm_metrics: Any = metrics
|
||||
|
||||
def measured_ttft(output: RequestFuncOutput) -> float | None:
|
||||
session_metrics = getattr(output, "duplex_session_metrics", None)
|
||||
@@ -3040,33 +3174,33 @@ async def benchmark(
|
||||
|
||||
result = {
|
||||
"duration": benchmark_duration,
|
||||
"completed": metrics.completed,
|
||||
"failed": metrics.failed,
|
||||
"total_input_tokens": metrics.total_input,
|
||||
"total_output_tokens": metrics.total_output,
|
||||
"request_throughput": metrics.request_throughput,
|
||||
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
||||
"output_throughput": metrics.output_throughput,
|
||||
"total_token_throughput": metrics.total_token_throughput,
|
||||
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
|
||||
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
|
||||
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
defs.TOTAL_VIDEO_DURATION_S: getattr(metrics, defs.TOTAL_VIDEO_DURATION_S),
|
||||
defs.TOTAL_VIDEO_FRAMES: getattr(metrics, defs.TOTAL_VIDEO_FRAMES),
|
||||
defs.VIDEO_THROUGHPUT: getattr(metrics, defs.VIDEO_THROUGHPUT),
|
||||
defs.MEAN_VIDEO_RTF: getattr(metrics, defs.MEAN_VIDEO_RTF),
|
||||
defs.MEDIAN_VIDEO_RTF: getattr(metrics, defs.MEDIAN_VIDEO_RTF),
|
||||
defs.PERCENTILES_VIDEO_RTF: getattr(metrics, defs.PERCENTILES_VIDEO_RTF),
|
||||
defs.MEAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS),
|
||||
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
|
||||
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
|
||||
defs.MEAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEAN_PEAK_MEMORY_MB),
|
||||
defs.MEDIAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB),
|
||||
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
|
||||
"completed": mm_metrics.completed,
|
||||
"failed": mm_metrics.failed,
|
||||
"total_input_tokens": mm_metrics.total_input,
|
||||
"total_output_tokens": mm_metrics.total_output,
|
||||
"request_throughput": mm_metrics.request_throughput,
|
||||
"request_goodput": mm_metrics.request_goodput if goodput_config_dict else None,
|
||||
"output_throughput": mm_metrics.output_throughput,
|
||||
"total_token_throughput": mm_metrics.total_token_throughput,
|
||||
defs.TOTAL_AUDIO_DURATION_S: getattr(mm_metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||
defs.TOTAL_AUDIO_FRAMES: getattr(mm_metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||
defs.AUDIO_THROUGHPUT: getattr(mm_metrics, defs.AUDIO_THROUGHPUT),
|
||||
defs.TOTAL_IMAGES: getattr(mm_metrics, defs.TOTAL_IMAGES),
|
||||
defs.IMAGE_THROUGHPUT: getattr(mm_metrics, defs.IMAGE_THROUGHPUT),
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(mm_metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(mm_metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
defs.TOTAL_VIDEO_DURATION_S: getattr(mm_metrics, defs.TOTAL_VIDEO_DURATION_S),
|
||||
defs.TOTAL_VIDEO_FRAMES: getattr(mm_metrics, defs.TOTAL_VIDEO_FRAMES),
|
||||
defs.VIDEO_THROUGHPUT: getattr(mm_metrics, defs.VIDEO_THROUGHPUT),
|
||||
defs.MEAN_VIDEO_RTF: getattr(mm_metrics, defs.MEAN_VIDEO_RTF),
|
||||
defs.MEDIAN_VIDEO_RTF: getattr(mm_metrics, defs.MEDIAN_VIDEO_RTF),
|
||||
defs.PERCENTILES_VIDEO_RTF: getattr(mm_metrics, defs.PERCENTILES_VIDEO_RTF),
|
||||
defs.MEAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEAN_VIDEO_GENERATION_MS),
|
||||
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
|
||||
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
|
||||
defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB),
|
||||
defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB),
|
||||
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
|
||||
"input_lens": [output.prompt_len for output in outputs],
|
||||
"start_times": [output.start_time for output in outputs],
|
||||
"output_lens": actual_output_lens,
|
||||
@@ -3074,9 +3208,9 @@ async def benchmark(
|
||||
"itls": [output.itl for output in outputs],
|
||||
"generated_texts": [output.generated_text for output in outputs],
|
||||
"errors": [output.error for output in outputs],
|
||||
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
|
||||
"max_concurrent_requests": metrics.max_concurrent_requests,
|
||||
"rtfx": metrics.rtfx,
|
||||
"max_output_tokens_per_s": mm_metrics.max_output_tokens_per_s,
|
||||
"max_concurrent_requests": mm_metrics.max_concurrent_requests,
|
||||
"rtfx": mm_metrics.rtfx,
|
||||
}
|
||||
for sample_count in (
|
||||
"num_ttft_samples",
|
||||
@@ -3150,6 +3284,21 @@ async def benchmark(
|
||||
result.update(_daily_acc)
|
||||
print_daily_omni_accuracy_summary(_daily_acc)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.videomme_eval import (
|
||||
compute_videomme_accuracy_metrics,
|
||||
print_videomme_accuracy_summary,
|
||||
)
|
||||
|
||||
_save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
_vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm)
|
||||
if _vm_acc is not None:
|
||||
result.update(_vm_acc)
|
||||
print_videomme_accuracy_summary(_vm_acc)
|
||||
|
||||
if _seed_tts_capture_pcm_for_wer():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
compute_seed_tts_wer_metrics,
|
||||
|
||||
@@ -43,7 +43,7 @@ def _normalize_endpoint(endpoint: str | None) -> str | None:
|
||||
|
||||
|
||||
def _use_endpoint_backend_when_implicit(args: argparse.Namespace) -> None:
|
||||
explicit_keys = getattr(args, "explicit_keys", frozenset())
|
||||
explicit_keys: frozenset[str] = getattr(args, "explicit_keys", frozenset())
|
||||
if "backend" in explicit_keys:
|
||||
return
|
||||
# Upstream vLLM defaults --backend to "openai". Treat that non-explicit
|
||||
@@ -63,6 +63,8 @@ def main(args: argparse.Namespace) -> dict[str, Any]:
|
||||
os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
|
||||
if getattr(args, "daily_omni_save_eval_items", False):
|
||||
os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
|
||||
if getattr(args, "videomme_save_eval_items", False):
|
||||
os.environ["VIDEOMME_SAVE_EVAL_ITEMS"] = "1"
|
||||
request_timeout_s = getattr(args, "omni_request_timeout_s", None)
|
||||
if request_timeout_s is not None:
|
||||
set_request_timeout_s(request_timeout_s)
|
||||
|
||||
@@ -228,6 +228,7 @@ _BENCHMARK_TRANSITIONAL = (
|
||||
"SEED_TTS_WER_EVAL",
|
||||
"SEED_TTS_WER_SAVE_AUDIO_DIR",
|
||||
"SEED_TTS_WER_SAVE_ITEMS",
|
||||
"VIDEOMME_SAVE_EVAL_ITEMS",
|
||||
"VLLM_DAILY_OMNI_MEDIA_REPO",
|
||||
"VLLM_OMNI_BENCH_AUDIO_CHANNELS",
|
||||
"VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S",
|
||||
|
||||
@@ -1386,7 +1386,7 @@ def normalize_and_validate_diffusion_engine_ingress_kwargs(
|
||||
| orchestrator_field_names()
|
||||
# Coordination fields also live on the typed orchestrator config, not
|
||||
# all of them are present on the CLI-only OrchestratorArgs dataclass.
|
||||
| frozenset(config_field.name for config_field in fields(VllmOmniOrchestratorConfig))
|
||||
| frozenset(config_field.name for config_field in fields(cast(Any, VllmOmniOrchestratorConfig)))
|
||||
)
|
||||
allowed_fields = stage_consumed_fields | externally_consumed_fields
|
||||
validate_omni_diffusion_kwargs(normalized, allowed_fields, stage_id=stage_id)
|
||||
|
||||
@@ -47,7 +47,7 @@ stages:
|
||||
fps: 1
|
||||
num_frames: 128
|
||||
limit_mm_per_prompt:
|
||||
# Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
|
||||
# Daily-Omni: up to 64 interleaved image/audio pairs (1fps).
|
||||
image: 64
|
||||
audio: 64
|
||||
video: 1
|
||||
|
||||
@@ -192,6 +192,78 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
)
|
||||
|
||||
|
||||
def add_videomme_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
"""Add CLI arguments specific to the Video-MME dataset."""
|
||||
group = parser.add_argument_group("Video-MME Dataset Options")
|
||||
group.add_argument(
|
||||
"--videomme-parquet",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local Video-MME parquet "
|
||||
"(e.g. videomme/test-00000-of-00001.parquet). When set, Hub QA loading is skipped.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-video-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory containing extracted Video-MME videos (videoID.mp4). "
|
||||
"Typical layout after unzipping videos_chunked_*.zip: <root>/video/. "
|
||||
"When using file:// URLs, start the server with --allowed-local-media-path "
|
||||
"covering this directory.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-subtitle-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory containing Video-MME .srt subtitles (videoID.srt). Used only with --videomme-use-subtitle.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-pack-mode",
|
||||
type=str,
|
||||
choices=["minicpm-frames", "minicpm-interleave", "video_url"],
|
||||
default="minicpm-frames",
|
||||
help="Multimodal packing. "
|
||||
"'minicpm-frames' (default): OmniEvalKit MiniCPM videomme recipe — sampled frames "
|
||||
"as image_url only (max_frames=96). "
|
||||
"'minicpm-interleave': OmniEvalKit videomme_short recipe — 1fps frame/audio pairs "
|
||||
"(max_frames=64). "
|
||||
"'video_url': single video_url part (models with native video input).",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-max-frames",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Override max sampled frames (OmniEvalKit defaults: 96 for minicpm-frames, 64 for minicpm-interleave).",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-duration",
|
||||
type=str,
|
||||
choices=["all", "short", "medium", "long"],
|
||||
default="all",
|
||||
help="Filter by Video-MME duration bucket (default: all).",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-use-subtitle",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Prepend subtitle text to the user prompt (Video-MME w/ subs setting).",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-inline-local-video",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Embed local frames/audio as base64 data URLs so the server does not need "
|
||||
"--allowed-local-media-path. Increases request size; use for small --num-prompts.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--videomme-save-eval-items",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Include per-request Video-MME accuracy rows in the saved JSON under "
|
||||
"videomme_eval_items. Or set env VIDEOMME_SAVE_EVAL_ITEMS=1.",
|
||||
)
|
||||
|
||||
|
||||
def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
"""Add CLI arguments for Seed-TTS benchmarks."""
|
||||
group = parser.add_argument_group("Seed-TTS Dataset Options")
|
||||
@@ -266,6 +338,7 @@ _OMNI_BENCH_DATASET_CHOICES = (
|
||||
"seed-tts-design",
|
||||
"ttsd",
|
||||
"sound-effect",
|
||||
"videomme",
|
||||
)
|
||||
|
||||
|
||||
@@ -332,6 +405,7 @@ def update_omni_help(parser: argparse.ArgumentParser) -> None:
|
||||
def add_omni_args(parser: argparse.ArgumentParser) -> None:
|
||||
"""Register all vLLM-Omni serving benchmark arguments."""
|
||||
add_daily_omni_cli_args(parser)
|
||||
add_videomme_cli_args(parser)
|
||||
add_omniinteract_cli_args(parser)
|
||||
add_seed_tts_cli_args(parser)
|
||||
add_multi_stage_cli_args(parser)
|
||||
|
||||
Reference in new issue
Block a user