anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity
Files
e5c0afc99432dba767dcb66f28b0fb9aa43ab1af
vllm-omni/vllm_omni/benchmarks/data_modules/videomme_dataset.py
T

1008 lines
42 KiB
Python
Raw Normal View History

[Benchmark][MiniCPM-o] Port Video-MME dataset support to Omni bench s… (#6987)
2026-09-20 17:15:53 +08:00
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Video-MME dataset loader for vLLM-Omni bench serve.
Video-MME (lmms-eval/Video-MME) is a full-spectrum video MLLM benchmark with
900 videos / 2,700 MCQ pairs across short / medium / long durations.
This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe
(``o_e_Kit/configs/generation_configs.json``):
- ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false``
- ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave``
Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame
timestamps match OmniEvalKit ``_sample_video_frame_indices``.
"""
from __future__ import annotations
import json
import logging
import os
import shutil
import tempfile
import time
import zipfile
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal
import pybase64 as base64
import regex as re
try:
from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
except ImportError:
from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset
from vllm.benchmarks.datasets import SampleRequest
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.hf import get_cached_tokenizer
try:
from datasets import load_dataset
except ImportError:
load_dataset = None
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
MINICPM_OMNI_SYSTEM_TEXT,
_ListDatasetIterator,
_numpy_to_wav_bytes,
_pil_to_jpeg_bytes,
_uniform_sample_indices,
)
logger = logging.getLogger(__name__)
_MINICPM_AUDIO_SR = 16000
# OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults.
VIDEOMME_DEFAULT_MAX_FRAMES = 96
VIDEOMME_SHORT_MAX_FRAMES = 64
VIDEOMME_DEFAULT_HF_REPO = "lmms-eval/Video-MME"
#: Decode forward to reach a target this close, seek to the preceding keyframe beyond it.
_SEEK_THRESHOLD_S = 3.0
#: How often ``sample`` reports progress while warming a cold frame cache.
_SAMPLE_PROGRESS_INTERVAL_S = 30.0
VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"]
VideoMMEDurationFilter = Literal["all", "short", "medium", "long"]
# OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped;
# media parts are prepended as OpenAI content parts, matching ``build_content``).
VIDEOMME_USER_PROMPT_TEMPLATE = (
"Carefully read the following question and select the letter corresponding to the "
"correct answer.Highlight the applicable choices without giving explanations.\n"
"{question}\n"
"Options:\n"
"{options}\n"
"Please select the correct answer from the options above. Only respond with the letter."
)
_OPTION_LETTERS = "ABCDEFGHIJKL"
_OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$")
_VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi")
def _iter_video_files(root: Path) -> Iterator[Path]:
"""Yield video files under ``root``, skipping dot-dirs such as the frame cache."""
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
for name in filenames:
if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES:
yield Path(dirpath) / name
def _resolve_hf_cache_snapshot(path: Path) -> Path:
"""Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir."""
snapshots = path / "snapshots"
if not snapshots.is_dir():
return path
ref = path / "refs" / "main"
if ref.is_file():
try:
revision = ref.read_text(encoding="utf-8").strip()
except OSError:
revision = ""
# An empty or unreadable ref must not resolve to ``snapshots`` itself, which holds
# no dataset files; fall through to the newest revision below.
if revision and (snapshots / revision).is_dir():
return snapshots / revision
revisions = [p for p in snapshots.iterdir() if p.is_dir()]
if revisions:
return max(revisions, key=lambda p: p.stat().st_mtime)
return path
def absolute_path(value: str | Path | None) -> Path | None:
"""Return an absolute path, or ``None`` when ``value`` is empty."""
if value is None:
return None
raw = str(value).strip()
if not raw:
return None
return Path(raw).expanduser().resolve()
def resolve_videomme_local_root(dataset_path: str | None) -> Path | None:
"""Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror.
Hub ids such as ``lmms-eval/Video-MME`` return ``None`` (they are not directories).
"""
raw = (dataset_path or "").strip()
if not raw:
return None
path = Path(raw).expanduser()
if not path.is_dir():
return None
return _resolve_hf_cache_snapshot(path.resolve())
def cached_videomme_snapshots() -> list[Path]:
"""Usable Video-MME snapshots already in the Hub cache, whatever org published them.
The dataset is mirrored under several ids (``lmms-lab``/``lmms-eval``), so an offline
run routinely requests one while a sibling is what actually sits on disk.
"""
from huggingface_hub import constants as hf_constants
cache = Path(hf_constants.HF_HUB_CACHE)
if not cache.is_dir():
return []
found: list[Path] = []
for entry in sorted(cache.iterdir()):
name = entry.name.lower()
if not entry.is_dir() or not name.startswith("datasets--") or not name.endswith("--video-mme"):
continue
snapshot = _resolve_hf_cache_snapshot(entry)
# A repo dir with no parquet is a partial download, not a usable root.
if videomme_local_parquet(snapshot) is not None:
found.append(snapshot)
return found
def ensure_videomme_hub_root(repo_id: str) -> Path:
"""Download the Video-MME dataset snapshot from Hugging Face and return its root.
Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`:
Hub ids trigger ``hf_api().snapshot_download``; callers then extract videos via
:func:`ensure_videomme_videos_extracted`.
Raises:
ImportError: if Hugging Face Hub helpers are not installed.
ValueError: if ``repo_id`` is empty.
FileNotFoundError: if the snapshot is neither cached nor downloadable.
"""
rid = (repo_id or "").strip()
if not rid:
raise ValueError("repo_id is required to download Video-MME from Hugging Face")
try:
from huggingface_hub import constants as hf_constants
from huggingface_hub.errors import (
HfHubHTTPError,
HFValidationError,
LocalEntryNotFoundError,
OfflineModeIsEnabled,
)
from vllm_omni.transformers_utils.repo_utils import hf_api
except ImportError as e:
raise ImportError(
"Install huggingface_hub to download Video-MME from the Hub, or pass a local "
"--dataset-path / --videomme-video-dir with parquet + extracted videos."
) from e
offline = bool(hf_constants.HF_HUB_OFFLINE)
try:
# Pull QA parquet + video/subtitle archives; skip unrelated repo files.
cache = hf_api().snapshot_download(
repo_id=rid,
repo_type="dataset",
# Offline runs otherwise resolve repo metadata over the network first and
# surface a connection failure instead of the missing cache entry.
local_files_only=offline,
allow_patterns=[
"videomme/**",
"*.parquet",
"videos_chunked_*.zip",
"subtitle.zip",
"subtitle/**",
"video/**",
"videos/**",
],
)
# ``HfHubHTTPError`` also covers gated / missing repos; Video-MME requires accepting
# the dataset terms, so a plain token-less run lands here rather than downloading.
except (HFValidationError, HfHubHTTPError, LocalEntryNotFoundError, OfflineModeIsEnabled) as e:
if offline:
reason = f"HF_HUB_OFFLINE is set and it is not cached under {hf_constants.HF_HUB_CACHE}"
else:
reason = f"snapshot_download failed with {type(e).__name__}"
hints = [
"Point --dataset-path at a local Video-MME root (a directory holding "
"videomme/test-*.parquet plus video/ or videos_chunked_*.zip), or pass "
"--videomme-parquet / --videomme-video-dir."
]
cached = cached_videomme_snapshots()
if cached:
hints.append(
"Video-MME is already cached under another repo id: " + ", ".join(str(p) for p in cached) + "."
)
if offline:
hints.append("Unset HF_HUB_OFFLINE to download it instead.")
raise FileNotFoundError(
f"Video-MME is unavailable from Hugging Face repo '{rid}': {reason}. " + " ".join(hints)
) from e
root = _resolve_hf_cache_snapshot(Path(cache).resolve())
logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid)
return root
def resolve_videomme_root(dataset_path: str | None) -> Path:
"""Return a Video-MME root containing parquet and (or zip) video assets.
* Existing local directories (absolute or relative) are used as-is.
* Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and
downloaded via :func:`ensure_videomme_hub_root` (default
``lmms-eval/Video-MME``).
"""
local = resolve_videomme_local_root(dataset_path)
if local is not None:
return local
rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO
return ensure_videomme_hub_root(rid)
def videomme_local_parquet(root: Path) -> Path | None:
for candidate in (
root / "videomme" / "test-00000-of-00001.parquet",
root / "test-00000-of-00001.parquet",
):
if candidate.is_file():
return candidate
return None
def videomme_local_video_dir(root: Path) -> Path | None:
"""Return a directory already holding extracted videos, flat or nested.
Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``,
so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of
the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable.
"""
for name in ("video", "videos", "data"):
candidate = root / name
if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None:
return candidate
return None
def videomme_local_subtitle_dir(root: Path) -> Path | None:
candidate = root / "subtitle"
return candidate if candidate.is_dir() else None
def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None:
"""Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout)."""
name = os.path.basename(member)
if not name:
return
dest = dest_dir / name
if dest.is_file() and dest.stat().st_size == zf.getinfo(member).file_size:
return
dest_dir.mkdir(parents=True, exist_ok=True)
# Publish only after ZipFile has read and checked the complete member.
with tempfile.TemporaryDirectory(prefix=".videomme-", dir=dest_dir) as temporary:
temporary_path = Path(temporary) / name
with zf.open(member) as src, temporary_path.open("wb") as out:
shutil.copyfileobj(src, out)
temporary_path.replace(dest)
def ensure_videomme_videos_extracted(root: Path) -> Path:
"""Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``."""
video_dir = root / "video"
marker = root / ".videomme_videos_extracted"
if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None:
return video_dir
# Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it.
existing = videomme_local_video_dir(root)
if existing is not None and existing != video_dir:
return existing
zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()]
if not zips and not video_dir.is_dir():
raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip")
video_dir.mkdir(parents=True, exist_ok=True)
for zp in zips:
logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir)
with zipfile.ZipFile(zp, "r") as zf:
for member in zf.namelist():
if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")):
_unzip_member_flat(zf, member, video_dir)
if next(_iter_video_files(video_dir), None) is None:
# Marking an empty tree as extracted would make every later request log
# "video not found" and yield a run with zero samples.
raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip")
marker.write_text("ok", encoding="utf-8")
return video_dir
def ensure_videomme_subtitles_extracted(root: Path) -> Path | None:
"""Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present."""
subtitle_dir = root / "subtitle"
if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None:
return subtitle_dir
zip_path = root / "subtitle.zip"
if not zip_path.is_file():
return subtitle_dir if subtitle_dir.is_dir() else None
subtitle_dir.mkdir(parents=True, exist_ok=True)
logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir)
with zipfile.ZipFile(zip_path, "r") as zf:
for member in zf.namelist():
if member.lower().endswith(".srt"):
_unzip_member_flat(zf, member, subtitle_dir)
return subtitle_dir
def _probe_video_duration(video_path: Path) -> float:
"""Container duration in seconds, without decoding the stream.
Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to
an hour, and ``stream.frames == 0`` containers would otherwise force a full decode
just to learn the duration.
"""
import av
with av.open(str(video_path)) as container:
stream = container.streams.video[0]
duration: float | None = None
if stream.duration is not None and stream.time_base is not None:
duration = float(stream.duration * stream.time_base)
elif container.duration is not None:
duration = container.duration / av.time_base
elif stream.frames and stream.average_rate:
duration = stream.frames / float(stream.average_rate)
if not duration or duration <= 0:
raise ValueError(f"Could not determine duration of {video_path}")
return duration
def _decode_frames_at_timestamps(
video_path: Path,
timestamps: list[float],
*,
seek_threshold_s: float = _SEEK_THRESHOLD_S,
) -> list[Any]:
"""Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp.
A single forward pass over the stream — the Daily-Omni approach — is not affordable
here: Video-MME samples points spread across the whole video, so the last target sits
near the end and every frame of an hour-long file would be decoded. Targets within
``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper
than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding
keyframe first.
"""
import av
if not timestamps:
return []
images: list[Any] = []
with av.open(str(video_path)) as container:
stream = container.streams.video[0]
stream.thread_type = "AUTO"
time_base = float(stream.time_base) if stream.time_base else 0.0
start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0
decoder = None
frame = None
frame_time = -1.0
image = None
for ts in timestamps:
if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s:
if time_base > 0:
container.seek(int((ts + start_s) / time_base), stream=stream, backward=True)
else:
container.seek(0, stream=stream, backward=True)
decoder = container.decode(stream)
frame, image, frame_time = None, None, -1.0
while frame is None or frame_time < ts:
nxt = next(decoder, None)
if nxt is None:
break
frame, image = nxt, None
if nxt.pts is not None and time_base > 0:
frame_time = float(nxt.pts) * time_base - start_s
if frame is None:
break
if image is None:
image = frame.to_image()
images.append(image)
if not images:
raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}")
# Container duration can overshoot the real stream; repeat the last frame (decord clamping).
images.extend([images[-1]] * (len(timestamps) - len(images)))
return images
@dataclass
class VideoMMESampleRequest(SampleRequest):
"""``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring."""
videomme_gold_answer: str = ""
videomme_video_id: str = ""
videomme_question_id: str = ""
videomme_duration: str = ""
videomme_domain: str = ""
videomme_sub_category: str = ""
videomme_task_type: str = ""
videomme_skipped_rows: int = 0
omni_extra_body: dict[str, Any] | None = None
omni_chat_messages: list[dict[str, Any]] | None = None
class VideoMMEDataset(BenchmarkDataset):
"""Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing)."""
SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO}
DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO
IS_MULTIMODAL = True
DEFAULT_OUTPUT_LEN = 128
def __init__(
self,
dataset_path: str | None = None,
dataset_split: str = "test",
random_seed: int = 0,
video_dir: str | None = None,
subtitle_dir: str | None = None,
parquet_path: str | None = None,
pack_mode: VideoMMEPackMode = "minicpm-frames",
max_frames: int | None = None,
duration_filter: VideoMMEDurationFilter = "all",
use_subtitle: bool = False,
inline_local_video: bool = False,
trust_remote_code: bool = False,
dataset_subset: str | None = None,
no_stream: bool = False,
**kwargs,
) -> None:
if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"):
raise ValueError(
f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}"
)
if duration_filter not in ("all", "short", "medium", "long"):
raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}")
if parquet_path is None and dataset_path is None:
raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.")
if pack_mode == "minicpm-interleave" and duration_filter != "short":
logger.warning(
"Video-MME 'minicpm-interleave' ports the OmniEvalKit videomme_short recipe. Beyond the "
"short bucket, frames are spread across the whole clip and each paired audio segment covers "
"the gap between them (tens of seconds on medium/long videos), which inflates the audio "
"token count far past the 1s pairing the recipe assumes. Pass --videomme-duration short."
)
self.parquet_path = absolute_path(parquet_path)
self.dataset_path = dataset_path
self.dataset_split = dataset_split
self.dataset_subset = dataset_subset
self._hf_streaming = not no_stream
self.video_dir = absolute_path(video_dir)
self.subtitle_dir = absolute_path(subtitle_dir)
self.pack_mode: VideoMMEPackMode = pack_mode
self.duration_filter: VideoMMEDurationFilter = duration_filter
self.use_subtitle = use_subtitle
self.inline_local_video = inline_local_video
self.trust_remote_code = trust_remote_code
self.max_frames = int(
max_frames
if max_frames is not None
else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES)
)
if self.max_frames <= 0:
raise ValueError("Video-MME max_frames must be positive")
#: In-process memo of content parts; the on-disk frame cache survives across runs.
self._frame_cache: dict[str, list[dict[str, Any]]] = {}
self._video_index: dict[str, Path] | None = None
super().__init__(
dataset_path=dataset_path,
random_seed=random_seed,
**kwargs,
)
self.load_data()
logger.info(
"Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, "
"use_subtitle=%s, video_dir=%s",
str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}",
pack_mode,
self.max_frames,
duration_filter,
use_subtitle,
self.video_dir,
)
# ------------------------------------------------------------------ loading
def load_data(self) -> None:
"""Resolve local/Hub assets once, shared by the CLI and direct callers."""
local_root = resolve_videomme_local_root(self.dataset_path)
if local_root is None and self.parquet_path is None and self.video_dir is None:
local_root = resolve_videomme_root(self.dataset_path)
if local_root is not None:
if self.parquet_path is None:
self.parquet_path = videomme_local_parquet(local_root)
if self.parquet_path is None:
raise FileNotFoundError(f"No Video-MME parquet under {local_root}; pass --videomme-parquet.")
if self.video_dir is None:
self.video_dir = ensure_videomme_videos_extracted(local_root)
if self.use_subtitle and self.subtitle_dir is None:
self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root)
if self.video_dir is None or not self.video_dir.is_dir():
raise ValueError(
"Video-MME requires --videomme-video-dir pointing at extracted videos, "
"or a local/Hub dataset root containing video/ or videos_chunked_*.zip."
)
if self.parquet_path is not None:
self._load_from_parquet(self.parquet_path)
else:
self._load_from_huggingface()
def _load_from_parquet(self, path: Path) -> None:
try:
import pandas as pd
except ImportError as e:
raise ImportError("pandas is required to load Video-MME parquet") from e
if not path.is_file():
raise FileNotFoundError(f"Video-MME parquet not found: {path}")
df = pd.read_parquet(path)
self._set_rows(df.to_dict(orient="records"))
def _load_from_huggingface(self) -> None:
if load_dataset is None:
raise ImportError(
"datasets library is required for HuggingFace Video-MME loading. "
"Install with: pip install datasets, or pass --videomme-parquet."
)
load_kw: dict[str, Any] = {
"split": self.dataset_split,
"streaming": self._hf_streaming,
"trust_remote_code": self.trust_remote_code,
}
if self.dataset_subset is not None:
load_kw["name"] = self.dataset_subset
ds = load_dataset(self.dataset_path, **load_kw)
self._set_rows(list(ds))
def _set_rows(self, rows: list[dict[str, Any]]) -> None:
"""Apply the duration filter and shuffle, then expose rows as ``self.data``."""
if self.duration_filter != "all":
want = self.duration_filter.lower()
rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want]
if not getattr(self, "disable_shuffle", False) and self.random_seed is not None:
import random
rows = rows[:]
random.Random(self.random_seed).shuffle(rows)
self.data = _ListDatasetIterator(rows)
@staticmethod
def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]:
# The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain
# over the alias keys is not usable here.
options: Any = []
for key in ("options", "candidates", "choices"):
value = row.get(key)
if value is not None and hasattr(value, "tolist"):
value = value.tolist()
if value is not None and len(value) > 0:
options = value
break
return {
"video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(),
"question_id": str(row.get("question_id") or "").strip(),
"question": str(row.get("question") or "").strip(),
"options": [str(x) for x in list(options)],
"answer": str(row.get("answer") or "").strip(),
"duration": str(row.get("duration") or "").strip(),
"domain": str(row.get("domain") or "").strip(),
"sub_category": str(row.get("sub_category") or "").strip(),
"task_type": str(row.get("task_type") or "").strip(),
}
# ------------------------------------------------------------------ sampling
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list[SampleRequest]:
if output_len is None:
output_len = self.DEFAULT_OUTPUT_LEN
sampled: list[SampleRequest] = []
cached_tokenizer = get_cached_tokenizer(tokenizer)
# A cold frame cache costs a few seconds per video, so the whole set takes tens of
# minutes before the first request goes out; report progress to keep that legible.
started = time.monotonic()
last_report = started
total = min(num_requests, len(self.data))
skipped_rows = 0
seen = 0
for seen, item in enumerate(self.data, start=1): # noqa: B007
if len(sampled) >= num_requests:
break
req = self._create_sample_request(item, cached_tokenizer, output_len, request_id_prefix, len(sampled))
if req is not None:
sampled.append(req)
else:
skipped_rows += 1
now = time.monotonic()
if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S:
logger.info(
"Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed",
seen,
total,
len(sampled),
now - started,
)
last_report = now
logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started)
if not sampled and num_requests > 0:
# Oversampling an empty list raises a bare IndexError; the usual cause is a
# video_dir that holds none of the videoIDs referenced by the QA rows.
raise ValueError(
f"Video-MME produced no requests from {seen} QA rows "
f"(duration_filter={self.duration_filter!r}, video_dir={self.video_dir}). "
"Check --videomme-duration and that --videomme-video-dir holds the videoIDs the rows reference."
)
for request in sampled:
assert isinstance(request, VideoMMESampleRequest)
request.videomme_skipped_rows = skipped_rows
self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample)
unique_questions = {getattr(req, "videomme_question_id", "") or req.request_id for req in sampled}
if len(sampled) > len(unique_questions):
logger.warning(
"Video-MME submitted %d requests covering %d unique questions "
"(missing media is skipped, then the remainder may be reused). "
"Submitted request count is not unique-question coverage.",
len(sampled),
len(unique_questions),
)
return sampled
def _create_sample_request(
self,
qa_item: dict[str, Any],
tokenizer: TokenizerLike,
output_len: int,
request_id_prefix: str,
index: int,
) -> SampleRequest | None:
fields = self._normalize_fields(qa_item)
if not fields["video_id"] or not fields["question"]:
logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"])
return None
mm_payload, omni_extra = self._compose_multimodal(fields["video_id"])
if not mm_payload:
return None
user_text = self._build_user_prompt(fields)
return VideoMMESampleRequest(
prompt=user_text,
prompt_len=len(tokenizer.encode(user_text)),
expected_output_len=output_len,
multi_modal_data=None,
request_id=f"{request_id_prefix}{index}",
videomme_gold_answer=fields["answer"],
videomme_video_id=fields["video_id"],
videomme_question_id=fields["question_id"],
videomme_duration=fields["duration"],
videomme_domain=fields["domain"],
videomme_sub_category=fields["sub_category"],
videomme_task_type=fields["task_type"],
omni_extra_body=omni_extra,
omni_chat_messages=self._build_openai_messages(mm_payload, user_text),
)
# ------------------------------------------------------------------ prompt
@staticmethod
def _format_options(options: list[str]) -> str:
"""Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line).
The Hub parquet already ships options as ``"A. content"``. Existing labels are
dropped before relabelling so the prompt never reads ``A. A. content``, but only
when *every* option carries one — otherwise contents such as ``A.M. shift`` would
be truncated.
"""
if not options:
return ""
matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options]
relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS))
lines = []
for key, raw, m in zip(_OPTION_LETTERS, options, matches):
content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip()
lines.append(f"{key}. {content}")
return "\n".join(lines)
def _build_user_prompt(self, fields: dict[str, Any]) -> str:
prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format(
question=fields["question"],
options=self._format_options(fields["options"]),
)
if self.use_subtitle:
subs = self._load_subtitle_text(fields["video_id"])
if subs:
prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}"
return prompt
def _load_subtitle_text(self, video_id: str) -> str:
if not self.subtitle_dir:
return ""
path = self.subtitle_dir / f"{video_id}.srt"
if not path.is_file():
return ""
try:
raw = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
return ""
lines = [
s.replace("\\N", " ")
for s in (line.strip() for line in raw.splitlines())
if s and not s.isdigit() and "-->" not in s
]
return "\n".join(lines)
def _build_openai_messages(
self,
mm_payload: dict[str, Any] | list[dict[str, Any]],
user_text: str,
) -> list[dict[str, Any]]:
mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload]
messages: list[dict[str, Any]] = []
if MINICPM_OMNI_SYSTEM_TEXT:
messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]})
messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]})
return messages
# ------------------------------------------------------------------ media
def _video_path_index(self) -> dict[str, Path]:
"""Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses."""
if self._video_index is None:
index: dict[str, Path] = {}
if self.video_dir is not None:
for path in _iter_video_files(self.video_dir):
index.setdefault(path.stem, path)
logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir)
self._video_index = index
return self._video_index
def _resolve_local_video_path(self, video_id: str) -> Path | None:
if not self.video_dir or not video_id:
return None
for p in (
self.video_dir / f"{video_id}.mp4",
self.video_dir / f"{video_id}.MP4",
self.video_dir / f"{video_id}.mkv",
self.video_dir / video_id / f"{video_id}.mp4",
):
if p.is_file():
return p
return self._video_path_index().get(video_id)
def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]:
path = path.expanduser().resolve()
if self.inline_local_video:
b64 = base64.b64encode(path.read_bytes()).decode("ascii")
return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
return {"type": typ, typ: {"url": path.as_uri()}}
def _compose_multimodal(
self,
video_id: str,
) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]:
# OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the
# sample carries a video path: 9 slices per frame would blow past max_model_len and
# <image_id>N</image_id> would label each frame as a separate picture.
extra: dict[str, Any] = {
"mm_processor_kwargs": {
"use_audio_in_video": False,
"max_slice_nums": 1,
"use_image_id": False,
}
}
if self.pack_mode == "video_url":
video_path = self._resolve_local_video_path(video_id)
if video_path is None:
logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir)
return None, None
return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra
parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave")
if not parts:
return None, None
return parts, extra
def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path | None:
"""Where the on-disk frame dump lives, or ``None`` when there is no ``video_dir`` to host it."""
if self.video_dir is None:
return None
tag = f"f{self.max_frames}{'_av' if include_audio else ''}"
return self.video_dir / ".minicpm_videomme_frames" / video_id / tag
def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None:
"""Rebuild content parts from a previous run's frame dump, or ``None`` on a miss."""
manifest = cache_dir / "manifest.json"
if not manifest.is_file():
return None
try:
count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"])
except (OSError, ValueError, KeyError, TypeError):
return None
parts: list[dict[str, Any]] = []
for i in range(count):
frame_path = cache_dir / f"frame_{i:04d}.jpg"
if not frame_path.is_file() or frame_path.stat().st_size == 0:
return None
parts.append({"type": "image_url", "image_url": {"url": frame_path.expanduser().resolve().as_uri()}})
if include_audio:
audio_path = cache_dir / f"audio_{i:04d}.wav"
if not audio_path.is_file() or audio_path.stat().st_size == 0:
return None
parts.append({"type": "audio_url", "audio_url": {"url": audio_path.expanduser().resolve().as_uri()}})
return parts or None
def _get_minicpm_frame_parts(
self,
video_id: str,
*,
include_audio: bool,
) -> list[dict[str, Any]] | None:
cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}"
cached = self._frame_cache.get(cache_key)
if cached is not None:
return cached
# Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS.
cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio)
if cache_dir is not None:
disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio)
if disk_parts is not None:
self._frame_cache[cache_key] = disk_parts
return disk_parts
video_path = self._resolve_local_video_path(video_id)
if video_path is None:
logger.warning(
"Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r",
self.video_dir,
video_id,
)
return None
try:
frames, audio_segments = self._extract_frames_and_audio(
video_path,
include_audio=include_audio,
max_num_frames=self.max_frames,
)
except Exception:
logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path)
return None
if not frames:
logger.warning("No frames extracted for video_id=%r", video_id)
return None
if include_audio and len(audio_segments) != len(frames):
n = min(len(frames), len(audio_segments))
frames, audio_segments = frames[:n], audio_segments[:n]
if cache_dir is not None:
cache_dir.mkdir(parents=True, exist_ok=True)
# A failed rebuild must not leave a manifest advertising a complete cache.
(cache_dir / "manifest.json").unlink(missing_ok=True)
parts: list[dict[str, Any]] = []
for i, frame in enumerate(frames):
parts.append(
self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg")
)
if include_audio:
parts.append(
self._emit_media(
cache_dir,
f"audio_{i:04d}.wav",
_numpy_to_wav_bytes(audio_segments[i]),
"audio_url",
"audio/wav",
)
)
if cache_dir is not None:
(cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8")
self._frame_cache[cache_key] = parts
logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts))
return parts
@staticmethod
def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]:
"""Write ``payload`` to the frame cache and return a file URL part, or inline base64."""
if cache_dir is None:
b64 = base64.b64encode(payload).decode("ascii")
return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
path = (cache_dir / name).expanduser().resolve()
# Reaching this path means the cache missed validation. Replace residual
# files as well: a nonempty file may be from an interrupted write.
path.write_bytes(payload)
return {"type": typ, typ: {"url": path.as_uri()}}
@staticmethod
def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]:
"""Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps."""
if duration > max_num_frames:
return [round(i * 0.1, 1) for i in _uniform_sample_indices(int(duration / 0.1), max_num_frames)]
# OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame.
return [float(i) for i in range(max(1, int(duration)))]
@classmethod
def _extract_frames_and_audio(
cls,
video_path: Path,
*,
include_audio: bool,
max_num_frames: int,
) -> tuple[list[Any], list[Any]]:
import numpy as np
from vllm.multimodal.media.audio import load_audio
duration = _probe_video_duration(video_path)
timestamps = cls._sample_timestamps(duration, max_num_frames)
frames = _decode_frames_at_timestamps(video_path, timestamps)
audio_segments: list[Any] = []
if include_audio:
audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True)
for i, start_time in enumerate(timestamps):
end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration
segment = audio_np[int(start_time * sr) : int(end_time * sr)]
if i == len(timestamps) - 1 and len(segment) < 1600:
segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)])
if len(segment) == 0:
segment = np.zeros(1600, dtype=np.float32)
audio_segments.append(segment)
return frames, audio_segments
Reference in new issue Copy Permalink