1009 lines
42 KiB
Python
1009 lines
42 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||
|
||
"""Video-MME dataset loader for vLLM-Omni bench serve.
|
||
|
||
Video-MME (lmms-eval/Video-MME) is a full-spectrum video MLLM benchmark with
|
||
900 videos / 2,700 MCQ pairs across short / medium / long durations.
|
||
|
||
This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe
|
||
(``o_e_Kit/configs/generation_configs.json``):
|
||
|
||
- ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false``
|
||
- ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave``
|
||
|
||
Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame
|
||
timestamps match OmniEvalKit ``_sample_video_frame_indices``.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import tempfile
|
||
import time
|
||
import zipfile
|
||
from collections.abc import Iterator
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any, Literal
|
||
|
||
import pybase64 as base64
|
||
import regex as re
|
||
|
||
try:
|
||
from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
|
||
except ImportError:
|
||
from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset
|
||
from vllm.benchmarks.datasets import SampleRequest
|
||
from vllm.tokenizers import TokenizerLike
|
||
from vllm.tokenizers.hf import get_cached_tokenizer
|
||
|
||
try:
|
||
from datasets import load_dataset
|
||
except ImportError:
|
||
load_dataset = None
|
||
|
||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||
MINICPM_OMNI_SYSTEM_TEXT,
|
||
_ListDatasetIterator,
|
||
_numpy_to_wav_bytes,
|
||
_pil_to_jpeg_bytes,
|
||
_uniform_sample_indices,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_MINICPM_AUDIO_SR = 16000
|
||
|
||
# OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults.
|
||
VIDEOMME_DEFAULT_MAX_FRAMES = 96
|
||
VIDEOMME_SHORT_MAX_FRAMES = 64
|
||
VIDEOMME_DEFAULT_HF_REPO = "lmms-eval/Video-MME"
|
||
|
||
#: Decode forward to reach a target this close, seek to the preceding keyframe beyond it.
|
||
_SEEK_THRESHOLD_S = 3.0
|
||
|
||
#: How often ``sample`` reports progress while warming a cold frame cache.
|
||
_SAMPLE_PROGRESS_INTERVAL_S = 30.0
|
||
|
||
VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"]
|
||
VideoMMEDurationFilter = Literal["all", "short", "medium", "long"]
|
||
|
||
# OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped;
|
||
# media parts are prepended as OpenAI content parts, matching ``build_content``).
|
||
VIDEOMME_USER_PROMPT_TEMPLATE = (
|
||
"Carefully read the following question and select the letter corresponding to the "
|
||
"correct answer.Highlight the applicable choices without giving explanations.\n"
|
||
"{question}\n"
|
||
"Options:\n"
|
||
"{options}\n"
|
||
"Please select the correct answer from the options above. Only respond with the letter."
|
||
)
|
||
|
||
_OPTION_LETTERS = "ABCDEFGHIJKL"
|
||
_OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$")
|
||
|
||
_VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi")
|
||
|
||
|
||
def _iter_video_files(root: Path) -> Iterator[Path]:
|
||
"""Yield video files under ``root``, skipping dot-dirs such as the frame cache."""
|
||
for dirpath, dirnames, filenames in os.walk(root):
|
||
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
|
||
for name in filenames:
|
||
if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES:
|
||
yield Path(dirpath) / name
|
||
|
||
|
||
def _resolve_hf_cache_snapshot(path: Path) -> Path:
|
||
"""Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir."""
|
||
snapshots = path / "snapshots"
|
||
if not snapshots.is_dir():
|
||
return path
|
||
ref = path / "refs" / "main"
|
||
if ref.is_file():
|
||
try:
|
||
revision = ref.read_text(encoding="utf-8").strip()
|
||
except OSError:
|
||
revision = ""
|
||
# An empty or unreadable ref must not resolve to ``snapshots`` itself, which holds
|
||
# no dataset files; fall through to the newest revision below.
|
||
if revision and (snapshots / revision).is_dir():
|
||
return snapshots / revision
|
||
revisions = [p for p in snapshots.iterdir() if p.is_dir()]
|
||
if revisions:
|
||
return max(revisions, key=lambda p: p.stat().st_mtime)
|
||
return path
|
||
|
||
|
||
def absolute_path(value: str | Path | None) -> Path | None:
|
||
"""Return an absolute path, or ``None`` when ``value`` is empty."""
|
||
if value is None:
|
||
return None
|
||
raw = str(value).strip()
|
||
if not raw:
|
||
return None
|
||
return Path(raw).expanduser().resolve()
|
||
|
||
|
||
def resolve_videomme_local_root(dataset_path: str | None) -> Path | None:
|
||
"""Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror.
|
||
|
||
Hub ids such as ``lmms-eval/Video-MME`` return ``None`` (they are not directories).
|
||
"""
|
||
raw = (dataset_path or "").strip()
|
||
if not raw:
|
||
return None
|
||
path = Path(raw).expanduser()
|
||
if not path.is_dir():
|
||
return None
|
||
return _resolve_hf_cache_snapshot(path.resolve())
|
||
|
||
|
||
def cached_videomme_snapshots() -> list[Path]:
|
||
"""Usable Video-MME snapshots already in the Hub cache, whatever org published them.
|
||
|
||
The dataset is mirrored under several ids (``lmms-lab``/``lmms-eval``), so an offline
|
||
run routinely requests one while a sibling is what actually sits on disk.
|
||
"""
|
||
from huggingface_hub import constants as hf_constants
|
||
|
||
cache = Path(hf_constants.HF_HUB_CACHE)
|
||
if not cache.is_dir():
|
||
return []
|
||
|
||
found: list[Path] = []
|
||
for entry in sorted(cache.iterdir()):
|
||
name = entry.name.lower()
|
||
if not entry.is_dir() or not name.startswith("datasets--") or not name.endswith("--video-mme"):
|
||
continue
|
||
snapshot = _resolve_hf_cache_snapshot(entry)
|
||
# A repo dir with no parquet is a partial download, not a usable root.
|
||
if videomme_local_parquet(snapshot) is not None:
|
||
found.append(snapshot)
|
||
return found
|
||
|
||
|
||
def ensure_videomme_hub_root(repo_id: str) -> Path:
|
||
"""Download the Video-MME dataset snapshot from Hugging Face and return its root.
|
||
|
||
Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`:
|
||
Hub ids trigger ``hf_api().snapshot_download``; callers then extract videos via
|
||
:func:`ensure_videomme_videos_extracted`.
|
||
|
||
Raises:
|
||
ImportError: if Hugging Face Hub helpers are not installed.
|
||
ValueError: if ``repo_id`` is empty.
|
||
FileNotFoundError: if the snapshot is neither cached nor downloadable.
|
||
"""
|
||
rid = (repo_id or "").strip()
|
||
if not rid:
|
||
raise ValueError("repo_id is required to download Video-MME from Hugging Face")
|
||
|
||
try:
|
||
from huggingface_hub import constants as hf_constants
|
||
from huggingface_hub.errors import (
|
||
HfHubHTTPError,
|
||
HFValidationError,
|
||
LocalEntryNotFoundError,
|
||
OfflineModeIsEnabled,
|
||
)
|
||
|
||
from vllm_omni.transformers_utils.repo_utils import hf_api
|
||
except ImportError as e:
|
||
raise ImportError(
|
||
"Install huggingface_hub to download Video-MME from the Hub, or pass a local "
|
||
"--dataset-path / --videomme-video-dir with parquet + extracted videos."
|
||
) from e
|
||
|
||
offline = bool(hf_constants.HF_HUB_OFFLINE)
|
||
try:
|
||
# Pull QA parquet + video/subtitle archives; skip unrelated repo files.
|
||
cache = hf_api().snapshot_download(
|
||
repo_id=rid,
|
||
repo_type="dataset",
|
||
# Offline runs otherwise resolve repo metadata over the network first and
|
||
# surface a connection failure instead of the missing cache entry.
|
||
local_files_only=offline,
|
||
allow_patterns=[
|
||
"videomme/**",
|
||
"*.parquet",
|
||
"videos_chunked_*.zip",
|
||
"subtitle.zip",
|
||
"subtitle/**",
|
||
"video/**",
|
||
"videos/**",
|
||
],
|
||
)
|
||
# ``HfHubHTTPError`` also covers gated / missing repos; Video-MME requires accepting
|
||
# the dataset terms, so a plain token-less run lands here rather than downloading.
|
||
except (HFValidationError, HfHubHTTPError, LocalEntryNotFoundError, OfflineModeIsEnabled) as e:
|
||
if offline:
|
||
reason = f"HF_HUB_OFFLINE is set and it is not cached under {hf_constants.HF_HUB_CACHE}"
|
||
else:
|
||
reason = f"snapshot_download failed with {type(e).__name__}"
|
||
|
||
hints = [
|
||
"Point --dataset-path at a local Video-MME root (a directory holding "
|
||
"videomme/test-*.parquet plus video/ or videos_chunked_*.zip), or pass "
|
||
"--videomme-parquet / --videomme-video-dir."
|
||
]
|
||
cached = cached_videomme_snapshots()
|
||
if cached:
|
||
hints.append(
|
||
"Video-MME is already cached under another repo id: " + ", ".join(str(p) for p in cached) + "."
|
||
)
|
||
if offline:
|
||
hints.append("Unset HF_HUB_OFFLINE to download it instead.")
|
||
|
||
raise FileNotFoundError(
|
||
f"Video-MME is unavailable from Hugging Face repo '{rid}': {reason}. " + " ".join(hints)
|
||
) from e
|
||
|
||
root = _resolve_hf_cache_snapshot(Path(cache).resolve())
|
||
logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid)
|
||
return root
|
||
|
||
|
||
def resolve_videomme_root(dataset_path: str | None) -> Path:
|
||
"""Return a Video-MME root containing parquet and (or zip) video assets.
|
||
|
||
* Existing local directories (absolute or relative) are used as-is.
|
||
* Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and
|
||
downloaded via :func:`ensure_videomme_hub_root` (default
|
||
``lmms-eval/Video-MME``).
|
||
"""
|
||
local = resolve_videomme_local_root(dataset_path)
|
||
if local is not None:
|
||
return local
|
||
rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO
|
||
return ensure_videomme_hub_root(rid)
|
||
|
||
|
||
def videomme_local_parquet(root: Path) -> Path | None:
|
||
for candidate in (
|
||
root / "videomme" / "test-00000-of-00001.parquet",
|
||
root / "test-00000-of-00001.parquet",
|
||
):
|
||
if candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def videomme_local_video_dir(root: Path) -> Path | None:
|
||
"""Return a directory already holding extracted videos, flat or nested.
|
||
|
||
Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``,
|
||
so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of
|
||
the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable.
|
||
"""
|
||
for name in ("video", "videos", "data"):
|
||
candidate = root / name
|
||
if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def videomme_local_subtitle_dir(root: Path) -> Path | None:
|
||
candidate = root / "subtitle"
|
||
return candidate if candidate.is_dir() else None
|
||
|
||
|
||
def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None:
|
||
"""Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout)."""
|
||
name = os.path.basename(member)
|
||
if not name:
|
||
return
|
||
dest = dest_dir / name
|
||
if dest.is_file() and dest.stat().st_size == zf.getinfo(member).file_size:
|
||
return
|
||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||
# Publish only after ZipFile has read and checked the complete member.
|
||
with tempfile.TemporaryDirectory(prefix=".videomme-", dir=dest_dir) as temporary:
|
||
temporary_path = Path(temporary) / name
|
||
with zf.open(member) as src, temporary_path.open("wb") as out:
|
||
shutil.copyfileobj(src, out)
|
||
temporary_path.replace(dest)
|
||
|
||
|
||
def ensure_videomme_videos_extracted(root: Path) -> Path:
|
||
"""Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``."""
|
||
video_dir = root / "video"
|
||
marker = root / ".videomme_videos_extracted"
|
||
if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None:
|
||
return video_dir
|
||
|
||
# Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it.
|
||
existing = videomme_local_video_dir(root)
|
||
if existing is not None and existing != video_dir:
|
||
return existing
|
||
|
||
zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()]
|
||
if not zips and not video_dir.is_dir():
|
||
raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip")
|
||
|
||
video_dir.mkdir(parents=True, exist_ok=True)
|
||
for zp in zips:
|
||
logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir)
|
||
with zipfile.ZipFile(zp, "r") as zf:
|
||
for member in zf.namelist():
|
||
if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")):
|
||
_unzip_member_flat(zf, member, video_dir)
|
||
|
||
if next(_iter_video_files(video_dir), None) is None:
|
||
# Marking an empty tree as extracted would make every later request log
|
||
# "video not found" and yield a run with zero samples.
|
||
raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip")
|
||
marker.write_text("ok", encoding="utf-8")
|
||
return video_dir
|
||
|
||
|
||
def ensure_videomme_subtitles_extracted(root: Path) -> Path | None:
|
||
"""Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present."""
|
||
subtitle_dir = root / "subtitle"
|
||
if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None:
|
||
return subtitle_dir
|
||
|
||
zip_path = root / "subtitle.zip"
|
||
if not zip_path.is_file():
|
||
return subtitle_dir if subtitle_dir.is_dir() else None
|
||
|
||
subtitle_dir.mkdir(parents=True, exist_ok=True)
|
||
logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir)
|
||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||
for member in zf.namelist():
|
||
if member.lower().endswith(".srt"):
|
||
_unzip_member_flat(zf, member, subtitle_dir)
|
||
return subtitle_dir
|
||
|
||
|
||
def _probe_video_duration(video_path: Path) -> float:
|
||
"""Container duration in seconds, without decoding the stream.
|
||
|
||
Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to
|
||
an hour, and ``stream.frames == 0`` containers would otherwise force a full decode
|
||
just to learn the duration.
|
||
"""
|
||
import av
|
||
|
||
with av.open(str(video_path)) as container:
|
||
stream = container.streams.video[0]
|
||
duration: float | None = None
|
||
if stream.duration is not None and stream.time_base is not None:
|
||
duration = float(stream.duration * stream.time_base)
|
||
elif container.duration is not None:
|
||
duration = container.duration / av.time_base
|
||
elif stream.frames and stream.average_rate:
|
||
duration = stream.frames / float(stream.average_rate)
|
||
|
||
if not duration or duration <= 0:
|
||
raise ValueError(f"Could not determine duration of {video_path}")
|
||
return duration
|
||
|
||
|
||
def _decode_frames_at_timestamps(
|
||
video_path: Path,
|
||
timestamps: list[float],
|
||
*,
|
||
seek_threshold_s: float = _SEEK_THRESHOLD_S,
|
||
) -> list[Any]:
|
||
"""Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp.
|
||
|
||
A single forward pass over the stream — the Daily-Omni approach — is not affordable
|
||
here: Video-MME samples points spread across the whole video, so the last target sits
|
||
near the end and every frame of an hour-long file would be decoded. Targets within
|
||
``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper
|
||
than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding
|
||
keyframe first.
|
||
"""
|
||
import av
|
||
|
||
if not timestamps:
|
||
return []
|
||
|
||
images: list[Any] = []
|
||
with av.open(str(video_path)) as container:
|
||
stream = container.streams.video[0]
|
||
stream.thread_type = "AUTO"
|
||
time_base = float(stream.time_base) if stream.time_base else 0.0
|
||
start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0
|
||
|
||
decoder = None
|
||
frame = None
|
||
frame_time = -1.0
|
||
image = None
|
||
|
||
for ts in timestamps:
|
||
if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s:
|
||
if time_base > 0:
|
||
container.seek(int((ts + start_s) / time_base), stream=stream, backward=True)
|
||
else:
|
||
container.seek(0, stream=stream, backward=True)
|
||
decoder = container.decode(stream)
|
||
frame, image, frame_time = None, None, -1.0
|
||
|
||
while frame is None or frame_time < ts:
|
||
nxt = next(decoder, None)
|
||
if nxt is None:
|
||
break
|
||
frame, image = nxt, None
|
||
if nxt.pts is not None and time_base > 0:
|
||
frame_time = float(nxt.pts) * time_base - start_s
|
||
|
||
if frame is None:
|
||
break
|
||
if image is None:
|
||
image = frame.to_image()
|
||
images.append(image)
|
||
|
||
if not images:
|
||
raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}")
|
||
# Container duration can overshoot the real stream; repeat the last frame (decord clamping).
|
||
images.extend([images[-1]] * (len(timestamps) - len(images)))
|
||
return images
|
||
|
||
|
||
@dataclass
|
||
class VideoMMESampleRequest(SampleRequest):
|
||
"""``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring."""
|
||
|
||
videomme_gold_answer: str = ""
|
||
videomme_video_id: str = ""
|
||
videomme_question_id: str = ""
|
||
videomme_duration: str = ""
|
||
videomme_domain: str = ""
|
||
videomme_sub_category: str = ""
|
||
videomme_task_type: str = ""
|
||
videomme_skipped_rows: int = 0
|
||
omni_extra_body: dict[str, Any] | None = None
|
||
omni_chat_messages: list[dict[str, Any]] | None = None
|
||
|
||
|
||
class VideoMMEDataset(BenchmarkDataset):
|
||
"""Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing)."""
|
||
|
||
SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO}
|
||
DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO
|
||
IS_MULTIMODAL = True
|
||
DEFAULT_OUTPUT_LEN = 128
|
||
|
||
def __init__(
|
||
self,
|
||
dataset_path: str | None = None,
|
||
dataset_split: str = "test",
|
||
random_seed: int = 0,
|
||
video_dir: str | None = None,
|
||
subtitle_dir: str | None = None,
|
||
parquet_path: str | None = None,
|
||
pack_mode: VideoMMEPackMode = "minicpm-frames",
|
||
max_frames: int | None = None,
|
||
duration_filter: VideoMMEDurationFilter = "all",
|
||
use_subtitle: bool = False,
|
||
inline_local_video: bool = False,
|
||
trust_remote_code: bool = False,
|
||
dataset_subset: str | None = None,
|
||
no_stream: bool = False,
|
||
**kwargs,
|
||
) -> None:
|
||
if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"):
|
||
raise ValueError(
|
||
f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}"
|
||
)
|
||
if duration_filter not in ("all", "short", "medium", "long"):
|
||
raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}")
|
||
if parquet_path is None and dataset_path is None:
|
||
raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.")
|
||
if pack_mode == "minicpm-interleave" and duration_filter != "short":
|
||
logger.warning(
|
||
"Video-MME 'minicpm-interleave' ports the OmniEvalKit videomme_short recipe. Beyond the "
|
||
"short bucket, frames are spread across the whole clip and each paired audio segment covers "
|
||
"the gap between them (tens of seconds on medium/long videos), which inflates the audio "
|
||
"token count far past the 1s pairing the recipe assumes. Pass --videomme-duration short."
|
||
)
|
||
|
||
self.parquet_path = absolute_path(parquet_path)
|
||
self.dataset_path = dataset_path
|
||
self.dataset_split = dataset_split
|
||
self.dataset_subset = dataset_subset
|
||
self._hf_streaming = not no_stream
|
||
self.video_dir = absolute_path(video_dir)
|
||
self.subtitle_dir = absolute_path(subtitle_dir)
|
||
self.pack_mode: VideoMMEPackMode = pack_mode
|
||
self.duration_filter: VideoMMEDurationFilter = duration_filter
|
||
self.use_subtitle = use_subtitle
|
||
self.inline_local_video = inline_local_video
|
||
self.trust_remote_code = trust_remote_code
|
||
self.max_frames = int(
|
||
max_frames
|
||
if max_frames is not None
|
||
else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES)
|
||
)
|
||
if self.max_frames <= 0:
|
||
raise ValueError("Video-MME max_frames must be positive")
|
||
#: In-process memo of content parts; the on-disk frame cache survives across runs.
|
||
self._frame_cache: dict[str, list[dict[str, Any]]] = {}
|
||
self._video_index: dict[str, Path] | None = None
|
||
|
||
super().__init__(
|
||
dataset_path=dataset_path,
|
||
random_seed=random_seed,
|
||
**kwargs,
|
||
)
|
||
self.load_data()
|
||
logger.info(
|
||
"Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, "
|
||
"use_subtitle=%s, video_dir=%s",
|
||
str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}",
|
||
pack_mode,
|
||
self.max_frames,
|
||
duration_filter,
|
||
use_subtitle,
|
||
self.video_dir,
|
||
)
|
||
|
||
# ------------------------------------------------------------------ loading
|
||
|
||
def load_data(self) -> None:
|
||
"""Resolve local/Hub assets once, shared by the CLI and direct callers."""
|
||
local_root = resolve_videomme_local_root(self.dataset_path)
|
||
if local_root is None and self.parquet_path is None and self.video_dir is None:
|
||
local_root = resolve_videomme_root(self.dataset_path)
|
||
|
||
if local_root is not None:
|
||
if self.parquet_path is None:
|
||
self.parquet_path = videomme_local_parquet(local_root)
|
||
if self.parquet_path is None:
|
||
raise FileNotFoundError(f"No Video-MME parquet under {local_root}; pass --videomme-parquet.")
|
||
if self.video_dir is None:
|
||
self.video_dir = ensure_videomme_videos_extracted(local_root)
|
||
if self.use_subtitle and self.subtitle_dir is None:
|
||
self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root)
|
||
|
||
if self.video_dir is None or not self.video_dir.is_dir():
|
||
raise ValueError(
|
||
"Video-MME requires --videomme-video-dir pointing at extracted videos, "
|
||
"or a local/Hub dataset root containing video/ or videos_chunked_*.zip."
|
||
)
|
||
if self.parquet_path is not None:
|
||
self._load_from_parquet(self.parquet_path)
|
||
else:
|
||
self._load_from_huggingface()
|
||
|
||
def _load_from_parquet(self, path: Path) -> None:
|
||
try:
|
||
import pandas as pd
|
||
except ImportError as e:
|
||
raise ImportError("pandas is required to load Video-MME parquet") from e
|
||
if not path.is_file():
|
||
raise FileNotFoundError(f"Video-MME parquet not found: {path}")
|
||
df = pd.read_parquet(path)
|
||
self._set_rows(df.to_dict(orient="records"))
|
||
|
||
def _load_from_huggingface(self) -> None:
|
||
if load_dataset is None:
|
||
raise ImportError(
|
||
"datasets library is required for HuggingFace Video-MME loading. "
|
||
"Install with: pip install datasets, or pass --videomme-parquet."
|
||
)
|
||
load_kw: dict[str, Any] = {
|
||
"split": self.dataset_split,
|
||
"streaming": self._hf_streaming,
|
||
"trust_remote_code": self.trust_remote_code,
|
||
}
|
||
if self.dataset_subset is not None:
|
||
load_kw["name"] = self.dataset_subset
|
||
ds = load_dataset(self.dataset_path, **load_kw)
|
||
self._set_rows(list(ds))
|
||
|
||
def _set_rows(self, rows: list[dict[str, Any]]) -> None:
|
||
"""Apply the duration filter and shuffle, then expose rows as ``self.data``."""
|
||
if self.duration_filter != "all":
|
||
want = self.duration_filter.lower()
|
||
rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want]
|
||
if not getattr(self, "disable_shuffle", False) and self.random_seed is not None:
|
||
import random
|
||
|
||
rows = rows[:]
|
||
random.Random(self.random_seed).shuffle(rows)
|
||
self.data = _ListDatasetIterator(rows)
|
||
|
||
@staticmethod
|
||
def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]:
|
||
# The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain
|
||
# over the alias keys is not usable here.
|
||
options: Any = []
|
||
for key in ("options", "candidates", "choices"):
|
||
value = row.get(key)
|
||
if value is not None and hasattr(value, "tolist"):
|
||
value = value.tolist()
|
||
if value is not None and len(value) > 0:
|
||
options = value
|
||
break
|
||
return {
|
||
"video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(),
|
||
"question_id": str(row.get("question_id") or "").strip(),
|
||
"question": str(row.get("question") or "").strip(),
|
||
"options": [str(x) for x in list(options)],
|
||
"answer": str(row.get("answer") or "").strip(),
|
||
"duration": str(row.get("duration") or "").strip(),
|
||
"domain": str(row.get("domain") or "").strip(),
|
||
"sub_category": str(row.get("sub_category") or "").strip(),
|
||
"task_type": str(row.get("task_type") or "").strip(),
|
||
}
|
||
|
||
# ------------------------------------------------------------------ sampling
|
||
|
||
def sample(
|
||
self,
|
||
tokenizer: TokenizerLike,
|
||
num_requests: int,
|
||
output_len: int | None = None,
|
||
request_id_prefix: str = "",
|
||
no_oversample: bool = False,
|
||
**kwargs,
|
||
) -> list[SampleRequest]:
|
||
if output_len is None:
|
||
output_len = self.DEFAULT_OUTPUT_LEN
|
||
|
||
sampled: list[SampleRequest] = []
|
||
cached_tokenizer = get_cached_tokenizer(tokenizer)
|
||
# A cold frame cache costs a few seconds per video, so the whole set takes tens of
|
||
# minutes before the first request goes out; report progress to keep that legible.
|
||
started = time.monotonic()
|
||
last_report = started
|
||
total = min(num_requests, len(self.data))
|
||
skipped_rows = 0
|
||
seen = 0
|
||
for seen, item in enumerate(self.data, start=1): # noqa: B007
|
||
if len(sampled) >= num_requests:
|
||
break
|
||
req = self._create_sample_request(item, cached_tokenizer, output_len, request_id_prefix, len(sampled))
|
||
if req is not None:
|
||
sampled.append(req)
|
||
else:
|
||
skipped_rows += 1
|
||
now = time.monotonic()
|
||
if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S:
|
||
logger.info(
|
||
"Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed",
|
||
seen,
|
||
total,
|
||
len(sampled),
|
||
now - started,
|
||
)
|
||
last_report = now
|
||
|
||
logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started)
|
||
if not sampled and num_requests > 0:
|
||
# Oversampling an empty list raises a bare IndexError; the usual cause is a
|
||
# video_dir that holds none of the videoIDs referenced by the QA rows.
|
||
raise ValueError(
|
||
f"Video-MME produced no requests from {seen} QA rows "
|
||
f"(duration_filter={self.duration_filter!r}, video_dir={self.video_dir}). "
|
||
"Check --videomme-duration and that --videomme-video-dir holds the videoIDs the rows reference."
|
||
)
|
||
for request in sampled:
|
||
assert isinstance(request, VideoMMESampleRequest)
|
||
request.videomme_skipped_rows = skipped_rows
|
||
self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample)
|
||
unique_questions = {getattr(req, "videomme_question_id", "") or req.request_id for req in sampled}
|
||
if len(sampled) > len(unique_questions):
|
||
logger.warning(
|
||
"Video-MME submitted %d requests covering %d unique questions "
|
||
"(missing media is skipped, then the remainder may be reused). "
|
||
"Submitted request count is not unique-question coverage.",
|
||
len(sampled),
|
||
len(unique_questions),
|
||
)
|
||
return sampled
|
||
|
||
def _create_sample_request(
|
||
self,
|
||
qa_item: dict[str, Any],
|
||
tokenizer: TokenizerLike,
|
||
output_len: int,
|
||
request_id_prefix: str,
|
||
index: int,
|
||
) -> SampleRequest | None:
|
||
fields = self._normalize_fields(qa_item)
|
||
if not fields["video_id"] or not fields["question"]:
|
||
logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"])
|
||
return None
|
||
|
||
mm_payload, omni_extra = self._compose_multimodal(fields["video_id"])
|
||
if not mm_payload:
|
||
return None
|
||
|
||
user_text = self._build_user_prompt(fields)
|
||
return VideoMMESampleRequest(
|
||
prompt=user_text,
|
||
prompt_len=len(tokenizer.encode(user_text)),
|
||
expected_output_len=output_len,
|
||
multi_modal_data=None,
|
||
request_id=f"{request_id_prefix}{index}",
|
||
videomme_gold_answer=fields["answer"],
|
||
videomme_video_id=fields["video_id"],
|
||
videomme_question_id=fields["question_id"],
|
||
videomme_duration=fields["duration"],
|
||
videomme_domain=fields["domain"],
|
||
videomme_sub_category=fields["sub_category"],
|
||
videomme_task_type=fields["task_type"],
|
||
omni_extra_body=omni_extra,
|
||
omni_chat_messages=self._build_openai_messages(mm_payload, user_text),
|
||
)
|
||
|
||
# ------------------------------------------------------------------ prompt
|
||
|
||
@staticmethod
|
||
def _format_options(options: list[str]) -> str:
|
||
"""Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line).
|
||
|
||
The Hub parquet already ships options as ``"A. content"``. Existing labels are
|
||
dropped before relabelling so the prompt never reads ``A. A. content``, but only
|
||
when *every* option carries one — otherwise contents such as ``A.M. shift`` would
|
||
be truncated.
|
||
"""
|
||
if not options:
|
||
return ""
|
||
matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options]
|
||
relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS))
|
||
lines = []
|
||
for key, raw, m in zip(_OPTION_LETTERS, options, matches):
|
||
content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip()
|
||
lines.append(f"{key}. {content}")
|
||
return "\n".join(lines)
|
||
|
||
def _build_user_prompt(self, fields: dict[str, Any]) -> str:
|
||
prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format(
|
||
question=fields["question"],
|
||
options=self._format_options(fields["options"]),
|
||
)
|
||
if self.use_subtitle:
|
||
subs = self._load_subtitle_text(fields["video_id"])
|
||
if subs:
|
||
prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}"
|
||
return prompt
|
||
|
||
def _load_subtitle_text(self, video_id: str) -> str:
|
||
if not self.subtitle_dir:
|
||
return ""
|
||
path = self.subtitle_dir / f"{video_id}.srt"
|
||
if not path.is_file():
|
||
return ""
|
||
try:
|
||
raw = path.read_text(encoding="utf-8", errors="ignore")
|
||
except OSError:
|
||
return ""
|
||
lines = [
|
||
s.replace("\\N", " ")
|
||
for s in (line.strip() for line in raw.splitlines())
|
||
if s and not s.isdigit() and "-->" not in s
|
||
]
|
||
return "\n".join(lines)
|
||
|
||
def _build_openai_messages(
|
||
self,
|
||
mm_payload: dict[str, Any] | list[dict[str, Any]],
|
||
user_text: str,
|
||
) -> list[dict[str, Any]]:
|
||
mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload]
|
||
messages: list[dict[str, Any]] = []
|
||
if MINICPM_OMNI_SYSTEM_TEXT:
|
||
messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]})
|
||
messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]})
|
||
return messages
|
||
|
||
# ------------------------------------------------------------------ media
|
||
|
||
def _video_path_index(self) -> dict[str, Path]:
|
||
"""Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses."""
|
||
if self._video_index is None:
|
||
index: dict[str, Path] = {}
|
||
if self.video_dir is not None:
|
||
for path in _iter_video_files(self.video_dir):
|
||
index.setdefault(path.stem, path)
|
||
logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir)
|
||
self._video_index = index
|
||
return self._video_index
|
||
|
||
def _resolve_local_video_path(self, video_id: str) -> Path | None:
|
||
if not self.video_dir or not video_id:
|
||
return None
|
||
for p in (
|
||
self.video_dir / f"{video_id}.mp4",
|
||
self.video_dir / f"{video_id}.MP4",
|
||
self.video_dir / f"{video_id}.mkv",
|
||
self.video_dir / video_id / f"{video_id}.mp4",
|
||
):
|
||
if p.is_file():
|
||
return p
|
||
return self._video_path_index().get(video_id)
|
||
|
||
def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]:
|
||
path = path.expanduser().resolve()
|
||
if self.inline_local_video:
|
||
b64 = base64.b64encode(path.read_bytes()).decode("ascii")
|
||
return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
|
||
return {"type": typ, typ: {"url": path.as_uri()}}
|
||
|
||
def _compose_multimodal(
|
||
self,
|
||
video_id: str,
|
||
) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]:
|
||
# OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the
|
||
# sample carries a video path: 9 slices per frame would blow past max_model_len and
|
||
# <image_id>N</image_id> would label each frame as a separate picture.
|
||
extra: dict[str, Any] = {
|
||
"mm_processor_kwargs": {
|
||
"use_audio_in_video": False,
|
||
"max_slice_nums": 1,
|
||
"use_image_id": False,
|
||
}
|
||
}
|
||
|
||
if self.pack_mode == "video_url":
|
||
video_path = self._resolve_local_video_path(video_id)
|
||
if video_path is None:
|
||
logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir)
|
||
return None, None
|
||
return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra
|
||
|
||
parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave")
|
||
if not parts:
|
||
return None, None
|
||
return parts, extra
|
||
|
||
def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path | None:
|
||
"""Where the on-disk frame dump lives, or ``None`` when there is no ``video_dir`` to host it."""
|
||
if self.video_dir is None:
|
||
return None
|
||
tag = f"f{self.max_frames}{'_av' if include_audio else ''}"
|
||
return self.video_dir / ".minicpm_videomme_frames" / video_id / tag
|
||
|
||
def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None:
|
||
"""Rebuild content parts from a previous run's frame dump, or ``None`` on a miss."""
|
||
manifest = cache_dir / "manifest.json"
|
||
if not manifest.is_file():
|
||
return None
|
||
try:
|
||
count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"])
|
||
except (OSError, ValueError, KeyError, TypeError):
|
||
return None
|
||
|
||
parts: list[dict[str, Any]] = []
|
||
for i in range(count):
|
||
frame_path = cache_dir / f"frame_{i:04d}.jpg"
|
||
if not frame_path.is_file() or frame_path.stat().st_size == 0:
|
||
return None
|
||
parts.append({"type": "image_url", "image_url": {"url": frame_path.expanduser().resolve().as_uri()}})
|
||
if include_audio:
|
||
audio_path = cache_dir / f"audio_{i:04d}.wav"
|
||
if not audio_path.is_file() or audio_path.stat().st_size == 0:
|
||
return None
|
||
parts.append({"type": "audio_url", "audio_url": {"url": audio_path.expanduser().resolve().as_uri()}})
|
||
return parts or None
|
||
|
||
def _get_minicpm_frame_parts(
|
||
self,
|
||
video_id: str,
|
||
*,
|
||
include_audio: bool,
|
||
) -> list[dict[str, Any]] | None:
|
||
cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}"
|
||
cached = self._frame_cache.get(cache_key)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
# Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS.
|
||
cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio)
|
||
if cache_dir is not None:
|
||
disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio)
|
||
if disk_parts is not None:
|
||
self._frame_cache[cache_key] = disk_parts
|
||
return disk_parts
|
||
|
||
video_path = self._resolve_local_video_path(video_id)
|
||
if video_path is None:
|
||
logger.warning(
|
||
"Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r",
|
||
self.video_dir,
|
||
video_id,
|
||
)
|
||
return None
|
||
|
||
try:
|
||
frames, audio_segments = self._extract_frames_and_audio(
|
||
video_path,
|
||
include_audio=include_audio,
|
||
max_num_frames=self.max_frames,
|
||
)
|
||
except Exception:
|
||
logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path)
|
||
return None
|
||
|
||
if not frames:
|
||
logger.warning("No frames extracted for video_id=%r", video_id)
|
||
return None
|
||
if include_audio and len(audio_segments) != len(frames):
|
||
n = min(len(frames), len(audio_segments))
|
||
frames, audio_segments = frames[:n], audio_segments[:n]
|
||
|
||
if cache_dir is not None:
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
# A failed rebuild must not leave a manifest advertising a complete cache.
|
||
(cache_dir / "manifest.json").unlink(missing_ok=True)
|
||
|
||
parts: list[dict[str, Any]] = []
|
||
for i, frame in enumerate(frames):
|
||
parts.append(
|
||
self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg")
|
||
)
|
||
if include_audio:
|
||
parts.append(
|
||
self._emit_media(
|
||
cache_dir,
|
||
f"audio_{i:04d}.wav",
|
||
_numpy_to_wav_bytes(audio_segments[i]),
|
||
"audio_url",
|
||
"audio/wav",
|
||
)
|
||
)
|
||
|
||
if cache_dir is not None:
|
||
(cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8")
|
||
self._frame_cache[cache_key] = parts
|
||
logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts))
|
||
return parts
|
||
|
||
@staticmethod
|
||
def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]:
|
||
"""Write ``payload`` to the frame cache and return a file URL part, or inline base64."""
|
||
if cache_dir is None:
|
||
b64 = base64.b64encode(payload).decode("ascii")
|
||
return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}}
|
||
path = (cache_dir / name).expanduser().resolve()
|
||
# Reaching this path means the cache missed validation. Replace residual
|
||
# files as well: a nonempty file may be from an interrupted write.
|
||
path.write_bytes(payload)
|
||
return {"type": typ, typ: {"url": path.as_uri()}}
|
||
|
||
@staticmethod
|
||
def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]:
|
||
"""Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps."""
|
||
if duration > max_num_frames:
|
||
return [round(i * 0.1, 1) for i in _uniform_sample_indices(int(duration / 0.1), max_num_frames)]
|
||
# OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame.
|
||
return [float(i) for i in range(max(1, int(duration)))]
|
||
|
||
@classmethod
|
||
def _extract_frames_and_audio(
|
||
cls,
|
||
video_path: Path,
|
||
*,
|
||
include_audio: bool,
|
||
max_num_frames: int,
|
||
) -> tuple[list[Any], list[Any]]:
|
||
import numpy as np
|
||
from vllm.multimodal.media.audio import load_audio
|
||
|
||
duration = _probe_video_duration(video_path)
|
||
timestamps = cls._sample_timestamps(duration, max_num_frames)
|
||
frames = _decode_frames_at_timestamps(video_path, timestamps)
|
||
|
||
audio_segments: list[Any] = []
|
||
if include_audio:
|
||
audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True)
|
||
for i, start_time in enumerate(timestamps):
|
||
end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration
|
||
segment = audio_np[int(start_time * sr) : int(end_time * sr)]
|
||
if i == len(timestamps) - 1 and len(segment) < 1600:
|
||
segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)])
|
||
if len(segment) == 0:
|
||
segment = np.zeros(1600, dtype=np.float32)
|
||
audio_segments.append(segment)
|
||
|
||
return frames, audio_segments
|