# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Video-MME dataset loader for vLLM-Omni bench serve. Video-MME (lmms-eval/Video-MME) is a full-spectrum video MLLM benchmark with 900 videos / 2,700 MCQ pairs across short / medium / long durations. This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe (``o_e_Kit/configs/generation_configs.json``): - ``videomme`` (default): frames only, ``max_frames=96``, ``load_av=false`` - ``videomme_short``-style: interleaved AV at 1fps via ``minicpm-interleave`` Prompt template and option formatting match OmniEvalKit ``minicpmo.py``; frame timestamps match OmniEvalKit ``_sample_video_frame_indices``. """ from __future__ import annotations import json import logging import os import shutil import tempfile import time import zipfile from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import Any, Literal import pybase64 as base64 import regex as re try: from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest except ImportError: from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset from vllm.benchmarks.datasets import SampleRequest from vllm.tokenizers import TokenizerLike from vllm.tokenizers.hf import get_cached_tokenizer try: from datasets import load_dataset except ImportError: load_dataset = None from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( MINICPM_OMNI_SYSTEM_TEXT, _ListDatasetIterator, _numpy_to_wav_bytes, _pil_to_jpeg_bytes, _uniform_sample_indices, ) logger = logging.getLogger(__name__) _MINICPM_AUDIO_SR = 16000 # OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults. VIDEOMME_DEFAULT_MAX_FRAMES = 96 VIDEOMME_SHORT_MAX_FRAMES = 64 VIDEOMME_DEFAULT_HF_REPO = "lmms-eval/Video-MME" #: Decode forward to reach a target this close, seek to the preceding keyframe beyond it. _SEEK_THRESHOLD_S = 3.0 #: How often ``sample`` reports progress while warming a cold frame cache. _SAMPLE_PROGRESS_INTERVAL_S = 30.0 VideoMMEPackMode = Literal["minicpm-frames", "minicpm-interleave", "video_url"] VideoMMEDurationFilter = Literal["all", "short", "medium", "long"] # OmniEvalKit ``generation_configs.json`` → ``videomme.user_prompt`` (``{media}`` stripped; # media parts are prepended as OpenAI content parts, matching ``build_content``). VIDEOMME_USER_PROMPT_TEMPLATE = ( "Carefully read the following question and select the letter corresponding to the " "correct answer.Highlight the applicable choices without giving explanations.\n" "{question}\n" "Options:\n" "{options}\n" "Please select the correct answer from the options above. Only respond with the letter." ) _OPTION_LETTERS = "ABCDEFGHIJKL" _OPTION_PREFIX_RE = re.compile(r"^([A-L])\s*[.、:：)]\s*(.*)$") _VIDEO_SUFFIXES = (".mp4", ".mkv", ".webm", ".avi") def _iter_video_files(root: Path) -> Iterator[Path]: """Yield video files under ``root``, skipping dot-dirs such as the frame cache.""" for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if not d.startswith(".")] for name in filenames: if os.path.splitext(name)[1].lower() in _VIDEO_SUFFIXES: yield Path(dirpath) / name def _resolve_hf_cache_snapshot(path: Path) -> Path: """Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir.""" snapshots = path / "snapshots" if not snapshots.is_dir(): return path ref = path / "refs" / "main" if ref.is_file(): try: revision = ref.read_text(encoding="utf-8").strip() except OSError: revision = "" # An empty or unreadable ref must not resolve to ``snapshots`` itself, which holds # no dataset files; fall through to the newest revision below. if revision and (snapshots / revision).is_dir(): return snapshots / revision revisions = [p for p in snapshots.iterdir() if p.is_dir()] if revisions: return max(revisions, key=lambda p: p.stat().st_mtime) return path def absolute_path(value: str | Path | None) -> Path | None: """Return an absolute path, or ``None`` when ``value`` is empty.""" if value is None: return None raw = str(value).strip() if not raw: return None return Path(raw).expanduser().resolve() def resolve_videomme_local_root(dataset_path: str | None) -> Path | None: """Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror. Hub ids such as ``lmms-eval/Video-MME`` return ``None`` (they are not directories). """ raw = (dataset_path or "").strip() if not raw: return None path = Path(raw).expanduser() if not path.is_dir(): return None return _resolve_hf_cache_snapshot(path.resolve()) def cached_videomme_snapshots() -> list[Path]: """Usable Video-MME snapshots already in the Hub cache, whatever org published them. The dataset is mirrored under several ids (``lmms-lab``/``lmms-eval``), so an offline run routinely requests one while a sibling is what actually sits on disk. """ from huggingface_hub import constants as hf_constants cache = Path(hf_constants.HF_HUB_CACHE) if not cache.is_dir(): return [] found: list[Path] = [] for entry in sorted(cache.iterdir()): name = entry.name.lower() if not entry.is_dir() or not name.startswith("datasets--") or not name.endswith("--video-mme"): continue snapshot = _resolve_hf_cache_snapshot(entry) # A repo dir with no parquet is a partial download, not a usable root. if videomme_local_parquet(snapshot) is not None: found.append(snapshot) return found def ensure_videomme_hub_root(repo_id: str) -> Path: """Download the Video-MME dataset snapshot from Hugging Face and return its root. Mirrors :func:`vllm_omni.benchmarks.data_modules.seed_tts_dataset.resolve_seed_tts_root`: Hub ids trigger ``hf_api().snapshot_download``; callers then extract videos via :func:`ensure_videomme_videos_extracted`. Raises: ImportError: if Hugging Face Hub helpers are not installed. ValueError: if ``repo_id`` is empty. FileNotFoundError: if the snapshot is neither cached nor downloadable. """ rid = (repo_id or "").strip() if not rid: raise ValueError("repo_id is required to download Video-MME from Hugging Face") try: from huggingface_hub import constants as hf_constants from huggingface_hub.errors import ( HfHubHTTPError, HFValidationError, LocalEntryNotFoundError, OfflineModeIsEnabled, ) from vllm_omni.transformers_utils.repo_utils import hf_api except ImportError as e: raise ImportError( "Install huggingface_hub to download Video-MME from the Hub, or pass a local " "--dataset-path / --videomme-video-dir with parquet + extracted videos." ) from e offline = bool(hf_constants.HF_HUB_OFFLINE) try: # Pull QA parquet + video/subtitle archives; skip unrelated repo files. cache = hf_api().snapshot_download( repo_id=rid, repo_type="dataset", # Offline runs otherwise resolve repo metadata over the network first and # surface a connection failure instead of the missing cache entry. local_files_only=offline, allow_patterns=[ "videomme/**", "*.parquet", "videos_chunked_*.zip", "subtitle.zip", "subtitle/**", "video/**", "videos/**", ], ) # ``HfHubHTTPError`` also covers gated / missing repos; Video-MME requires accepting # the dataset terms, so a plain token-less run lands here rather than downloading. except (HFValidationError, HfHubHTTPError, LocalEntryNotFoundError, OfflineModeIsEnabled) as e: if offline: reason = f"HF_HUB_OFFLINE is set and it is not cached under {hf_constants.HF_HUB_CACHE}" else: reason = f"snapshot_download failed with {type(e).__name__}" hints = [ "Point --dataset-path at a local Video-MME root (a directory holding " "videomme/test-*.parquet plus video/ or videos_chunked_*.zip), or pass " "--videomme-parquet / --videomme-video-dir." ] cached = cached_videomme_snapshots() if cached: hints.append( "Video-MME is already cached under another repo id: " + ", ".join(str(p) for p in cached) + "." ) if offline: hints.append("Unset HF_HUB_OFFLINE to download it instead.") raise FileNotFoundError( f"Video-MME is unavailable from Hugging Face repo '{rid}': {reason}. " + " ".join(hints) ) from e root = _resolve_hf_cache_snapshot(Path(cache).resolve()) logger.info("Video-MME Hub snapshot ready at %s (repo=%s)", root, rid) return root def resolve_videomme_root(dataset_path: str | None) -> Path: """Return a Video-MME root containing parquet and (or zip) video assets. * Existing local directories (absolute or relative) are used as-is. * Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and downloaded via :func:`ensure_videomme_hub_root` (default ``lmms-eval/Video-MME``). """ local = resolve_videomme_local_root(dataset_path) if local is not None: return local rid = (dataset_path or "").strip() or VIDEOMME_DEFAULT_HF_REPO return ensure_videomme_hub_root(rid) def videomme_local_parquet(root: Path) -> Path | None: for candidate in ( root / "videomme" / "test-00000-of-00001.parquet", root / "test-00000-of-00001.parquet", ): if candidate.is_file(): return candidate return None def videomme_local_video_dir(root: Path) -> Path | None: """Return a directory already holding extracted videos, flat or nested. Unzipping ``videos_chunked_*.zip`` in place yields ``videos/videos_chunked_NN/data/*.mp4``, so probing only for ``video/*.mp4`` would miss it and trigger a redundant re-extract of the whole ~95GB archive set. Lookups below walk the tree, so any depth is acceptable. """ for name in ("video", "videos", "data"): candidate = root / name if candidate.is_dir() and next(_iter_video_files(candidate), None) is not None: return candidate return None def videomme_local_subtitle_dir(root: Path) -> Path | None: candidate = root / "subtitle" return candidate if candidate.is_dir() else None def _unzip_member_flat(zf: zipfile.ZipFile, member: str, dest_dir: Path) -> None: """Extract a zip member into ``dest_dir`` using only its basename (VLMEvalKit layout).""" name = os.path.basename(member) if not name: return dest = dest_dir / name if dest.is_file() and dest.stat().st_size == zf.getinfo(member).file_size: return dest_dir.mkdir(parents=True, exist_ok=True) # Publish only after ZipFile has read and checked the complete member. with tempfile.TemporaryDirectory(prefix=".videomme-", dir=dest_dir) as temporary: temporary_path = Path(temporary) / name with zf.open(member) as src, temporary_path.open("wb") as out: shutil.copyfileobj(src, out) temporary_path.replace(dest) def ensure_videomme_videos_extracted(root: Path) -> Path: """Ensure ``video/*.mp4`` exists under ``root`` by unzipping ``videos_chunked_*.zip``.""" video_dir = root / "video" marker = root / ".videomme_videos_extracted" if marker.is_file() and video_dir.is_dir() and next(video_dir.glob("*.mp4"), None) is not None: return video_dir # Honour a tree someone unzipped by hand rather than re-extracting ~95GB alongside it. existing = videomme_local_video_dir(root) if existing is not None and existing != video_dir: return existing zips = [z for z in sorted(root.glob("videos_chunked_*.zip")) if z.is_file()] if not zips and not video_dir.is_dir(): raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip") video_dir.mkdir(parents=True, exist_ok=True) for zp in zips: logger.info("Extracting Video-MME videos from %s -> %s", zp, video_dir) with zipfile.ZipFile(zp, "r") as zf: for member in zf.namelist(): if member.lower().endswith((".mp4", ".mkv", ".webm", ".avi")): _unzip_member_flat(zf, member, video_dir) if next(_iter_video_files(video_dir), None) is None: # Marking an empty tree as extracted would make every later request log # "video not found" and yield a run with zero samples. raise FileNotFoundError(f"No Video-MME videos under {root}: expected video/*.mp4 or videos_chunked_*.zip") marker.write_text("ok", encoding="utf-8") return video_dir def ensure_videomme_subtitles_extracted(root: Path) -> Path | None: """Ensure ``subtitle/*.srt`` exists under ``root`` when ``subtitle.zip`` is present.""" subtitle_dir = root / "subtitle" if subtitle_dir.is_dir() and next(subtitle_dir.glob("*.srt"), None) is not None: return subtitle_dir zip_path = root / "subtitle.zip" if not zip_path.is_file(): return subtitle_dir if subtitle_dir.is_dir() else None subtitle_dir.mkdir(parents=True, exist_ok=True) logger.info("Extracting Video-MME subtitles from %s -> %s", zip_path, subtitle_dir) with zipfile.ZipFile(zip_path, "r") as zf: for member in zf.namelist(): if member.lower().endswith(".srt"): _unzip_member_flat(zf, member, subtitle_dir) return subtitle_dir def _probe_video_duration(video_path: Path) -> float: """Container duration in seconds, without decoding the stream. Frame-counting fallbacks are deliberately avoided: Video-MME long videos run up to an hour, and ``stream.frames == 0`` containers would otherwise force a full decode just to learn the duration. """ import av with av.open(str(video_path)) as container: stream = container.streams.video[0] duration: float | None = None if stream.duration is not None and stream.time_base is not None: duration = float(stream.duration * stream.time_base) elif container.duration is not None: duration = container.duration / av.time_base elif stream.frames and stream.average_rate: duration = stream.frames / float(stream.average_rate) if not duration or duration <= 0: raise ValueError(f"Could not determine duration of {video_path}") return duration def _decode_frames_at_timestamps( video_path: Path, timestamps: list[float], *, seek_threshold_s: float = _SEEK_THRESHOLD_S, ) -> list[Any]: """Decode one RGB ``PIL.Image`` at (or just after) each ascending timestamp. A single forward pass over the stream — the Daily-Omni approach — is not affordable here: Video-MME samples points spread across the whole video, so the last target sits near the end and every frame of an hour-long file would be decoded. Targets within ``seek_threshold_s`` of the decoder position are reached by decoding forward (cheaper than a keyframe round-trip at 1fps spacing); farther ones seek to the preceding keyframe first. """ import av if not timestamps: return [] images: list[Any] = [] with av.open(str(video_path)) as container: stream = container.streams.video[0] stream.thread_type = "AUTO" time_base = float(stream.time_base) if stream.time_base else 0.0 start_s = float(stream.start_time * stream.time_base) if stream.start_time else 0.0 decoder = None frame = None frame_time = -1.0 image = None for ts in timestamps: if decoder is None or ts < frame_time or (ts - frame_time) > seek_threshold_s: if time_base > 0: container.seek(int((ts + start_s) / time_base), stream=stream, backward=True) else: container.seek(0, stream=stream, backward=True) decoder = container.decode(stream) frame, image, frame_time = None, None, -1.0 while frame is None or frame_time < ts: nxt = next(decoder, None) if nxt is None: break frame, image = nxt, None if nxt.pts is not None and time_base > 0: frame_time = float(nxt.pts) * time_base - start_s if frame is None: break if image is None: image = frame.to_image() images.append(image) if not images: raise ValueError(f"Decoded no frames from {video_path} at timestamps {timestamps[:4]}") # Container duration can overshoot the real stream; repeat the last frame (decord clamping). images.extend([images[-1]] * (len(timestamps) - len(images))) return images @dataclass class VideoMMESampleRequest(SampleRequest): """``SampleRequest`` with Video-MME gold labels for post-run accuracy scoring.""" videomme_gold_answer: str = "" videomme_video_id: str = "" videomme_question_id: str = "" videomme_duration: str = "" videomme_domain: str = "" videomme_sub_category: str = "" videomme_task_type: str = "" videomme_skipped_rows: int = 0 omni_extra_body: dict[str, Any] | None = None omni_chat_messages: list[dict[str, Any]] | None = None class VideoMMEDataset(BenchmarkDataset): """Video-MME MCQ dataset for Omni bench serve (MiniCPM-oriented packing).""" SUPPORTED_DATASET_PATHS: set[str] = {VIDEOMME_DEFAULT_HF_REPO} DEFAULT_HF_DATASET_ID = VIDEOMME_DEFAULT_HF_REPO IS_MULTIMODAL = True DEFAULT_OUTPUT_LEN = 128 def __init__( self, dataset_path: str | None = None, dataset_split: str = "test", random_seed: int = 0, video_dir: str | None = None, subtitle_dir: str | None = None, parquet_path: str | None = None, pack_mode: VideoMMEPackMode = "minicpm-frames", max_frames: int | None = None, duration_filter: VideoMMEDurationFilter = "all", use_subtitle: bool = False, inline_local_video: bool = False, trust_remote_code: bool = False, dataset_subset: str | None = None, no_stream: bool = False, **kwargs, ) -> None: if pack_mode not in ("minicpm-frames", "minicpm-interleave", "video_url"): raise ValueError( f"pack_mode must be 'minicpm-frames', 'minicpm-interleave', or 'video_url', got {pack_mode!r}" ) if duration_filter not in ("all", "short", "medium", "long"): raise ValueError(f"duration_filter must be all|short|medium|long, got {duration_filter!r}") if parquet_path is None and dataset_path is None: raise ValueError("Either 'parquet_path' (local) or 'dataset_path' (HF id / local mirror) must be provided.") if pack_mode == "minicpm-interleave" and duration_filter != "short": logger.warning( "Video-MME 'minicpm-interleave' ports the OmniEvalKit videomme_short recipe. Beyond the " "short bucket, frames are spread across the whole clip and each paired audio segment covers " "the gap between them (tens of seconds on medium/long videos), which inflates the audio " "token count far past the 1s pairing the recipe assumes. Pass --videomme-duration short." ) self.parquet_path = absolute_path(parquet_path) self.dataset_path = dataset_path self.dataset_split = dataset_split self.dataset_subset = dataset_subset self._hf_streaming = not no_stream self.video_dir = absolute_path(video_dir) self.subtitle_dir = absolute_path(subtitle_dir) self.pack_mode: VideoMMEPackMode = pack_mode self.duration_filter: VideoMMEDurationFilter = duration_filter self.use_subtitle = use_subtitle self.inline_local_video = inline_local_video self.trust_remote_code = trust_remote_code self.max_frames = int( max_frames if max_frames is not None else (VIDEOMME_SHORT_MAX_FRAMES if pack_mode == "minicpm-interleave" else VIDEOMME_DEFAULT_MAX_FRAMES) ) if self.max_frames <= 0: raise ValueError("Video-MME max_frames must be positive") #: In-process memo of content parts; the on-disk frame cache survives across runs. self._frame_cache: dict[str, list[dict[str, Any]]] = {} self._video_index: dict[str, Path] | None = None super().__init__( dataset_path=dataset_path, random_seed=random_seed, **kwargs, ) self.load_data() logger.info( "Loaded Video-MME: source=%s, pack_mode=%s, max_frames=%d, duration_filter=%s, " "use_subtitle=%s, video_dir=%s", str(self.parquet_path) if self.parquet_path else f"{dataset_path}/{dataset_split}", pack_mode, self.max_frames, duration_filter, use_subtitle, self.video_dir, ) # ------------------------------------------------------------------ loading def load_data(self) -> None: """Resolve local/Hub assets once, shared by the CLI and direct callers.""" local_root = resolve_videomme_local_root(self.dataset_path) if local_root is None and self.parquet_path is None and self.video_dir is None: local_root = resolve_videomme_root(self.dataset_path) if local_root is not None: if self.parquet_path is None: self.parquet_path = videomme_local_parquet(local_root) if self.parquet_path is None: raise FileNotFoundError(f"No Video-MME parquet under {local_root}; pass --videomme-parquet.") if self.video_dir is None: self.video_dir = ensure_videomme_videos_extracted(local_root) if self.use_subtitle and self.subtitle_dir is None: self.subtitle_dir = ensure_videomme_subtitles_extracted(local_root) if self.video_dir is None or not self.video_dir.is_dir(): raise ValueError( "Video-MME requires --videomme-video-dir pointing at extracted videos, " "or a local/Hub dataset root containing video/ or videos_chunked_*.zip." ) if self.parquet_path is not None: self._load_from_parquet(self.parquet_path) else: self._load_from_huggingface() def _load_from_parquet(self, path: Path) -> None: try: import pandas as pd except ImportError as e: raise ImportError("pandas is required to load Video-MME parquet") from e if not path.is_file(): raise FileNotFoundError(f"Video-MME parquet not found: {path}") df = pd.read_parquet(path) self._set_rows(df.to_dict(orient="records")) def _load_from_huggingface(self) -> None: if load_dataset is None: raise ImportError( "datasets library is required for HuggingFace Video-MME loading. " "Install with: pip install datasets, or pass --videomme-parquet." ) load_kw: dict[str, Any] = { "split": self.dataset_split, "streaming": self._hf_streaming, "trust_remote_code": self.trust_remote_code, } if self.dataset_subset is not None: load_kw["name"] = self.dataset_subset ds = load_dataset(self.dataset_path, **load_kw) self._set_rows(list(ds)) def _set_rows(self, rows: list[dict[str, Any]]) -> None: """Apply the duration filter and shuffle, then expose rows as ``self.data``.""" if self.duration_filter != "all": want = self.duration_filter.lower() rows = [r for r in rows if str(r.get("duration") or "").strip().lower() == want] if not getattr(self, "disable_shuffle", False) and self.random_seed is not None: import random rows = rows[:] random.Random(self.random_seed).shuffle(rows) self.data = _ListDatasetIterator(rows) @staticmethod def _normalize_fields(row: dict[str, Any]) -> dict[str, Any]: # The parquet hands back a numpy array, whose truthiness raises, so an ``or`` chain # over the alias keys is not usable here. options: Any = [] for key in ("options", "candidates", "choices"): value = row.get(key) if value is not None and hasattr(value, "tolist"): value = value.tolist() if value is not None and len(value) > 0: options = value break return { "video_id": str(row.get("videoID") or row.get("video_id") or row.get("video") or "").strip(), "question_id": str(row.get("question_id") or "").strip(), "question": str(row.get("question") or "").strip(), "options": [str(x) for x in list(options)], "answer": str(row.get("answer") or "").strip(), "duration": str(row.get("duration") or "").strip(), "domain": str(row.get("domain") or "").strip(), "sub_category": str(row.get("sub_category") or "").strip(), "task_type": str(row.get("task_type") or "").strip(), } # ------------------------------------------------------------------ sampling def sample( self, tokenizer: TokenizerLike, num_requests: int, output_len: int | None = None, request_id_prefix: str = "", no_oversample: bool = False, **kwargs, ) -> list[SampleRequest]: if output_len is None: output_len = self.DEFAULT_OUTPUT_LEN sampled: list[SampleRequest] = [] cached_tokenizer = get_cached_tokenizer(tokenizer) # A cold frame cache costs a few seconds per video, so the whole set takes tens of # minutes before the first request goes out; report progress to keep that legible. started = time.monotonic() last_report = started total = min(num_requests, len(self.data)) skipped_rows = 0 seen = 0 for seen, item in enumerate(self.data, start=1): # noqa: B007 if len(sampled) >= num_requests: break req = self._create_sample_request(item, cached_tokenizer, output_len, request_id_prefix, len(sampled)) if req is not None: sampled.append(req) else: skipped_rows += 1 now = time.monotonic() if now - last_report >= _SAMPLE_PROGRESS_INTERVAL_S: logger.info( "Video-MME preparing requests: %d/%d rows, %d ready, %.0fs elapsed", seen, total, len(sampled), now - started, ) last_report = now logger.info("Created %d sample requests from Video-MME in %.0fs", len(sampled), time.monotonic() - started) if not sampled and num_requests > 0: # Oversampling an empty list raises a bare IndexError; the usual cause is a # video_dir that holds none of the videoIDs referenced by the QA rows. raise ValueError( f"Video-MME produced no requests from {seen} QA rows " f"(duration_filter={self.duration_filter!r}, video_dir={self.video_dir}). " "Check --videomme-duration and that --videomme-video-dir holds the videoIDs the rows reference." ) for request in sampled: assert isinstance(request, VideoMMESampleRequest) request.videomme_skipped_rows = skipped_rows self.maybe_oversample_requests(sampled, num_requests, request_id_prefix, no_oversample) unique_questions = {getattr(req, "videomme_question_id", "") or req.request_id for req in sampled} if len(sampled) > len(unique_questions): logger.warning( "Video-MME submitted %d requests covering %d unique questions " "(missing media is skipped, then the remainder may be reused). " "Submitted request count is not unique-question coverage.", len(sampled), len(unique_questions), ) return sampled def _create_sample_request( self, qa_item: dict[str, Any], tokenizer: TokenizerLike, output_len: int, request_id_prefix: str, index: int, ) -> SampleRequest | None: fields = self._normalize_fields(qa_item) if not fields["video_id"] or not fields["question"]: logger.warning("Skipping Video-MME item without videoID/question: %r", fields["video_id"]) return None mm_payload, omni_extra = self._compose_multimodal(fields["video_id"]) if not mm_payload: return None user_text = self._build_user_prompt(fields) return VideoMMESampleRequest( prompt=user_text, prompt_len=len(tokenizer.encode(user_text)), expected_output_len=output_len, multi_modal_data=None, request_id=f"{request_id_prefix}{index}", videomme_gold_answer=fields["answer"], videomme_video_id=fields["video_id"], videomme_question_id=fields["question_id"], videomme_duration=fields["duration"], videomme_domain=fields["domain"], videomme_sub_category=fields["sub_category"], videomme_task_type=fields["task_type"], omni_extra_body=omni_extra, omni_chat_messages=self._build_openai_messages(mm_payload, user_text), ) # ------------------------------------------------------------------ prompt @staticmethod def _format_options(options: list[str]) -> str: """Match OmniEvalKit ``_build_options_prompt`` (``A. content`` per line). The Hub parquet already ships options as ``"A. content"``. Existing labels are dropped before relabelling so the prompt never reads ``A. A. content``, but only when *every* option carries one — otherwise contents such as ``A.M. shift`` would be truncated. """ if not options: return "" matches = [_OPTION_PREFIX_RE.match(str(o).strip()) for o in options] relabel = all(m is not None and m.group(1) == key for m, key in zip(matches, _OPTION_LETTERS)) lines = [] for key, raw, m in zip(_OPTION_LETTERS, options, matches): content = m.group(2).strip() if (relabel and m is not None) else str(raw).strip() lines.append(f"{key}. {content}") return "\n".join(lines) def _build_user_prompt(self, fields: dict[str, Any]) -> str: prompt = VIDEOMME_USER_PROMPT_TEMPLATE.format( question=fields["question"], options=self._format_options(fields["options"]), ) if self.use_subtitle: subs = self._load_subtitle_text(fields["video_id"]) if subs: prompt = f"This video's subtitles are listed below:\n{subs}\n{prompt}" return prompt def _load_subtitle_text(self, video_id: str) -> str: if not self.subtitle_dir: return "" path = self.subtitle_dir / f"{video_id}.srt" if not path.is_file(): return "" try: raw = path.read_text(encoding="utf-8", errors="ignore") except OSError: return "" lines = [ s.replace("\\N", " ") for s in (line.strip() for line in raw.splitlines()) if s and not s.isdigit() and "-->" not in s ] return "\n".join(lines) def _build_openai_messages( self, mm_payload: dict[str, Any] | list[dict[str, Any]], user_text: str, ) -> list[dict[str, Any]]: mm_list = mm_payload if isinstance(mm_payload, list) else [mm_payload] messages: list[dict[str, Any]] = [] if MINICPM_OMNI_SYSTEM_TEXT: messages.append({"role": "system", "content": [{"type": "text", "text": MINICPM_OMNI_SYSTEM_TEXT}]}) messages.append({"role": "user", "content": [*mm_list, {"type": "text", "text": user_text}]}) return messages # ------------------------------------------------------------------ media def _video_path_index(self) -> dict[str, Path]: """Lazily map ``videoID -> path`` so nested layouts resolve after the flat probe misses.""" if self._video_index is None: index: dict[str, Path] = {} if self.video_dir is not None: for path in _iter_video_files(self.video_dir): index.setdefault(path.stem, path) logger.info("Indexed %d Video-MME video files under %s", len(index), self.video_dir) self._video_index = index return self._video_index def _resolve_local_video_path(self, video_id: str) -> Path | None: if not self.video_dir or not video_id: return None for p in ( self.video_dir / f"{video_id}.mp4", self.video_dir / f"{video_id}.MP4", self.video_dir / f"{video_id}.mkv", self.video_dir / video_id / f"{video_id}.mp4", ): if p.is_file(): return p return self._video_path_index().get(video_id) def _media_part(self, path: Path, *, typ: str, mime: str) -> dict[str, Any]: path = path.expanduser().resolve() if self.inline_local_video: b64 = base64.b64encode(path.read_bytes()).decode("ascii") return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} return {"type": typ, typ: {"url": path.as_uri()}} def _compose_multimodal( self, video_id: str, ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None]: # OmniEvalKit ``minicpmo.py`` uses max_slice_nums=1 / use_image_id=False whenever the # sample carries a video path: 9 slices per frame would blow past max_model_len and # N would label each frame as a separate picture. extra: dict[str, Any] = { "mm_processor_kwargs": { "use_audio_in_video": False, "max_slice_nums": 1, "use_image_id": False, } } if self.pack_mode == "video_url": video_path = self._resolve_local_video_path(video_id) if video_path is None: logger.warning("Video-MME video not found for video_id=%r under %s", video_id, self.video_dir) return None, None return self._media_part(video_path, typ="video_url", mime="video/mp4"), extra parts = self._get_minicpm_frame_parts(video_id, include_audio=self.pack_mode == "minicpm-interleave") if not parts: return None, None return parts, extra def _frame_cache_dir(self, video_id: str, *, include_audio: bool) -> Path | None: """Where the on-disk frame dump lives, or ``None`` when there is no ``video_dir`` to host it.""" if self.video_dir is None: return None tag = f"f{self.max_frames}{'_av' if include_audio else ''}" return self.video_dir / ".minicpm_videomme_frames" / video_id / tag def _cached_parts_from_disk(self, cache_dir: Path, *, include_audio: bool) -> list[dict[str, Any]] | None: """Rebuild content parts from a previous run's frame dump, or ``None`` on a miss.""" manifest = cache_dir / "manifest.json" if not manifest.is_file(): return None try: count = int(json.loads(manifest.read_text(encoding="utf-8"))["count"]) except (OSError, ValueError, KeyError, TypeError): return None parts: list[dict[str, Any]] = [] for i in range(count): frame_path = cache_dir / f"frame_{i:04d}.jpg" if not frame_path.is_file() or frame_path.stat().st_size == 0: return None parts.append({"type": "image_url", "image_url": {"url": frame_path.expanduser().resolve().as_uri()}}) if include_audio: audio_path = cache_dir / f"audio_{i:04d}.wav" if not audio_path.is_file() or audio_path.stat().st_size == 0: return None parts.append({"type": "audio_url", "audio_url": {"url": audio_path.expanduser().resolve().as_uri()}}) return parts or None def _get_minicpm_frame_parts( self, video_id: str, *, include_audio: bool, ) -> list[dict[str, Any]] | None: cache_key = f"{video_id}|audio={int(include_audio)}|frames={self.max_frames}" cached = self._frame_cache.get(cache_key) if cached is not None: return cached # Inline base64 must not be persisted or retained: 900 videos would pin GBs of RSS. cache_dir = None if self.inline_local_video else self._frame_cache_dir(video_id, include_audio=include_audio) if cache_dir is not None: disk_parts = self._cached_parts_from_disk(cache_dir, include_audio=include_audio) if disk_parts is not None: self._frame_cache[cache_key] = disk_parts return disk_parts video_path = self._resolve_local_video_path(video_id) if video_path is None: logger.warning( "Video-MME MiniCPM pack requires local video under video_dir=%s for video_id=%r", self.video_dir, video_id, ) return None try: frames, audio_segments = self._extract_frames_and_audio( video_path, include_audio=include_audio, max_num_frames=self.max_frames, ) except Exception: logger.exception("Failed Video-MME frame extract for video_id=%r path=%s", video_id, video_path) return None if not frames: logger.warning("No frames extracted for video_id=%r", video_id) return None if include_audio and len(audio_segments) != len(frames): n = min(len(frames), len(audio_segments)) frames, audio_segments = frames[:n], audio_segments[:n] if cache_dir is not None: cache_dir.mkdir(parents=True, exist_ok=True) # A failed rebuild must not leave a manifest advertising a complete cache. (cache_dir / "manifest.json").unlink(missing_ok=True) parts: list[dict[str, Any]] = [] for i, frame in enumerate(frames): parts.append( self._emit_media(cache_dir, f"frame_{i:04d}.jpg", _pil_to_jpeg_bytes(frame), "image_url", "image/jpeg") ) if include_audio: parts.append( self._emit_media( cache_dir, f"audio_{i:04d}.wav", _numpy_to_wav_bytes(audio_segments[i]), "audio_url", "audio/wav", ) ) if cache_dir is not None: (cache_dir / "manifest.json").write_text(json.dumps({"count": len(frames)}), encoding="utf-8") self._frame_cache[cache_key] = parts logger.debug("Video-MME packed video_id=%r frames=%d parts=%d", video_id, len(frames), len(parts)) return parts @staticmethod def _emit_media(cache_dir: Path | None, name: str, payload: bytes, typ: str, mime: str) -> dict[str, Any]: """Write ``payload`` to the frame cache and return a file URL part, or inline base64.""" if cache_dir is None: b64 = base64.b64encode(payload).decode("ascii") return {"type": typ, typ: {"url": f"data:{mime};base64,{b64}"}} path = (cache_dir / name).expanduser().resolve() # Reaching this path means the cache missed validation. Replace residual # files as well: a nonempty file may be from an interrupted write. path.write_bytes(payload) return {"type": typ, typ: {"url": path.as_uri()}} @staticmethod def _sample_timestamps(duration: float, max_num_frames: int) -> list[float]: """Port of OmniEvalKit ``_sample_video_frame_indices`` timestamps.""" if duration > max_num_frames: return [round(i * 0.1, 1) for i in _uniform_sample_indices(int(duration / 0.1), max_num_frames)] # OmniEvalKit uses int(duration); clamp to >=1 so sub-second clips still yield a frame. return [float(i) for i in range(max(1, int(duration)))] @classmethod def _extract_frames_and_audio( cls, video_path: Path, *, include_audio: bool, max_num_frames: int, ) -> tuple[list[Any], list[Any]]: import numpy as np from vllm.multimodal.media.audio import load_audio duration = _probe_video_duration(video_path) timestamps = cls._sample_timestamps(duration, max_num_frames) frames = _decode_frames_at_timestamps(video_path, timestamps) audio_segments: list[Any] = [] if include_audio: audio_np, sr = load_audio(str(video_path), sr=_MINICPM_AUDIO_SR, mono=True) for i, start_time in enumerate(timestamps): end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration segment = audio_np[int(start_time * sr) : int(end_time * sr)] if i == len(timestamps) - 1 and len(segment) < 1600: segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)]) if len(segment) == 0: segment = np.zeros(1600, dtype=np.float32) audio_segments.append(segment) return frames, audio_segments