diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 34daa08a9..f66631184 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -85,6 +85,9 @@ steps: commands: - export SEED_TTS_WER_EVAL=1 - export SEED_TTS_EVAL_DEVICE=npu:1 + # Video-MME needs no repo override here: the NPU runners pre-stage the dataset under + # the lmms-eval mirror, which is now VIDEOMME_DEFAULT_HF_REPO, so snapshot_download + # resolves it straight out of the hub cache. - | set +e pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model @@ -115,6 +118,7 @@ steps: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py index e9c9019c0..26ed52ea8 100644 --- a/examples/offline_inference/minicpmo/end2end.py +++ b/examples/offline_inference/minicpmo/end2end.py @@ -16,7 +16,6 @@ import numpy as np import soundfile as sf import vllm from PIL import Image -from vllm import SamplingParams from vllm.assets.audio import AudioAsset from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset, video_to_ndarrays @@ -26,8 +25,6 @@ from vllm.multimodal.media.audio import load_audio from vllm_omni.entrypoints.omni import Omni from vllm_omni.utils.tracking_parser import TrackingArgumentParser -SEED = 42 - default_system = ( "You are MiniCPM-o, a helpful multimodal assistant that can " "understand images, audio and video, and respond in text and speech." @@ -285,27 +282,17 @@ def main(args): omni_kwargs["trust_remote_code"] = True omni = Omni(**omni_kwargs) - # Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled). - thinker_sampling_params = SamplingParams( - temperature=0.0, - top_p=1.0, - top_k=-1, - max_tokens=2048, - seed=SEED, - detokenize=True, - repetition_penalty=1.1, - ) - # Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler; - # waveform is produced in-process by Token2Wav. - talker_sampling_params = SamplingParams( - temperature=0.0, - top_p=1.0, - top_k=-1, - max_tokens=1, - seed=SEED, - detokenize=False, - ) - sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages] + # Per-stage sampling defaults live in the deploy YAML + # (``vllm_omni/deploy/minicpmo_4_5*.yaml``, ``default_sampling_params``), + # which is the single source of truth that tracks the pipeline topology. + # Passing None makes ``resolve_sampling_params_list`` load exactly those, + # so adding or removing a stage cannot desynchronize this example. + # + # This used to hardcode a two-element list built for the old two-stage + # topology (thinker → talker+Token2Wav). Since Code2Wav became its own + # Stage 2, that list was both the wrong length and wrong in content + # (talker ``max_tokens=1`` truncated the codec stream to one frame). + sampling_params_list = None if args.txt_prompts is None: prompts = [query_result.inputs for _ in range(args.num_prompts)] diff --git a/tests/benchmarks/test_daily_omni_local_source.py b/tests/benchmarks/test_daily_omni_local_source.py index 35045a04b..13ae0637a 100644 --- a/tests/benchmarks/test_daily_omni_local_source.py +++ b/tests/benchmarks/test_daily_omni_local_source.py @@ -9,13 +9,17 @@ from pathlib import Path import pytest from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( + DailyOmniDataset, daily_omni_local_qa_json, daily_omni_local_videos_dir, + ensure_daily_omni_hub_root, resolve_daily_omni_local_root, ) pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark] +_HUB_ID = "liarliar/Daily-Omni" + def _write_qa_json(root: Path) -> Path: qa = root / "qa.json" @@ -23,6 +27,28 @@ def _write_qa_json(root: Path) -> Path: return qa +def _stage_offline_hub_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, with_qa: bool = True) -> Path: + """Build a hub cache holding ``liarliar/Daily-Omni`` and cut off all network access. + + Reproduces the air-gapped NPU runner: the dataset is pre-staged with ``hf download``, so it + only exists in the *hub* cache -- never in the ``datasets`` cache that ``load_dataset`` reads. + """ + constants = pytest.importorskip("huggingface_hub").constants + + cache = tmp_path / "hub" + repo_dir = cache / "datasets--liarliar--Daily-Omni" + snapshot = repo_dir / "snapshots" / "deadbeef" + snapshot.mkdir(parents=True) + if with_qa: + _write_qa_json(snapshot) + (repo_dir / "refs").mkdir() + (repo_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8") + + monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache)) + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + return snapshot + + def test_resolve_local_root_returns_none_for_hub_id() -> None: assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None assert resolve_daily_omni_local_root("") is None @@ -90,3 +116,37 @@ def test_missing_qa_json_reports_none(tmp_path: Path) -> None: root = resolve_daily_omni_local_root(str(tmp_path)) assert daily_omni_local_qa_json(root) is None assert daily_omni_local_videos_dir(root) is None + + +def test_hub_id_resolves_to_offline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch) + + root = ensure_daily_omni_hub_root(_HUB_ID) + + assert root == snapshot.resolve() + assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve() + + +def test_dataset_loads_hub_id_without_network(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The regression this file exists for: a Hub id must not require ``load_dataset``.""" + snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch) + # Any use of `datasets` here means we went back to the path that cannot read the hub cache. + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None) + + dataset = DailyOmniDataset(dataset_path=_HUB_ID, random_seed=0) + + assert dataset.qa_json_path == (snapshot / "qa.json").resolve() + assert [row["video_id"] for row in dataset.data] == ["vid0"] + + +def test_uncached_hub_id_reports_both_attempts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _stage_offline_hub_cache(tmp_path, monkeypatch, with_qa=False) + monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None) + + with pytest.raises(RuntimeError) as excinfo: + DailyOmniDataset(dataset_path="someone-else/Daily-Omni", random_seed=0) + + message = str(excinfo.value) + assert "someone-else/Daily-Omni" in message + assert "snapshot_download failed" in message + assert "--daily-omni-qa-json" in message diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index a1416afce..0c942e9c3 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -129,8 +129,8 @@ def _optional_local_videomme_dataset_path() -> str | None: """Return an explicit local Video-MME root, or ``None`` to use the Hub default. Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as - local. Hard-coded workspace paths are intentionally not probed so CI defaults to - ``lmms-lab/Video-MME`` via ``--videomme-repo``. + local. Hard-coded workspace paths are intentionally not probed so CI defaults to the + Hub id in ``VIDEOMME_DEFAULT_HF_REPO`` (``lmms-eval/Video-MME``). """ for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"): raw = os.environ.get(key, "").strip() @@ -210,7 +210,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@hardware_test(res={"cuda": "H100"}, num_cards=1) @pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True) def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None: """Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe.""" diff --git a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py index d0b9f067e..d64e81948 100644 --- a/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py +++ b/tests/e2e/accuracy/qwen3_omni/qwen3_omni_acc_bench_core.py @@ -11,7 +11,7 @@ and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id -``lmms-lab/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives. +``lmms-eval/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives. Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver. @@ -29,7 +29,7 @@ from typing import Any, Protocol DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni" DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval" -DEFAULT_VIDEOMME_HF_REPO = "lmms-lab/Video-MME" +DEFAULT_VIDEOMME_HF_REPO = "lmms-eval/Video-MME" class OmniBenchServerEndpoint(Protocol): @@ -122,7 +122,7 @@ def videomme_bench_argv() -> list[str]: * If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory, use that local mirror as ``--dataset-path``. - * Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-lab/Video-MME``); the child + * Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-eval/Video-MME``); the child bench downloads via ``huggingface_hub.snapshot_download``. """ root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip() diff --git a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py index 27a07b9a9..180cecbc5 100644 --- a/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py +++ b/tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py @@ -35,7 +35,7 @@ Prerequisites the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed. -* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-lab/Video-MME`` (override with +* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-eval/Video-MME`` (override with ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via ``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` / @@ -466,7 +466,7 @@ def build_arg_parser() -> argparse.ArgumentParser: type=str, default=None, help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). " - "Default when no local path is set: lmms-lab/Video-MME. " + "Default when no local path is set: lmms-eval/Video-MME. " "Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.", ) p.add_argument( diff --git a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py index 512e86538..0c3df81fe 100644 --- a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py +++ b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py @@ -7,7 +7,8 @@ Dataset source: https://huggingface.co/datasets/liarliar/Daily-Omni Supports loading QA metadata from: - Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments -- HuggingFace datasets (``dataset_path``): legacy online mode +- HuggingFace Hub id (``dataset_path``): resolved with ``snapshot_download`` to the repo's + ``qa.json``, which reads the hub cache and therefore also works under ``HF_HUB_OFFLINE`` Video/audio files normally come from extracted ``Videos.tar``. When ``--daily-omni-video-dir`` is not set, the first request that needs on-disk media downloads that archive from the Hugging Face @@ -24,9 +25,9 @@ Why ``BenchmarkDataset`` instead of ``HuggingFaceDataset``? This class therefore inherits only ``BenchmarkDataset`` (minimal: ``dataset_path``, ``random_seed``, ``self.data``) and implements **two explicit loaders**: ``_load_from_local_json`` (default path for air-gapped runs) and ``_load_from_huggingface`` - (optional legacy path for users who prefer ``datasets`` + Hub cache). The latter is **not** + (Hub ids, resolved to the repo's ``qa.json`` through the hub cache). The latter is **not** inheritance; it is the same Hub rows as before, factored into a helper so one class can - serve both deployment modes without mandatory ``datasets`` when using ``qa_json_path``. + serve both deployment modes without mandatory ``datasets``. Usage: from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset @@ -38,7 +39,7 @@ Usage: random_seed=42, ) - # HuggingFace mode (legacy, requires network) + # Hub id mode (works offline once the repo is in the HF hub cache) dataset = DailyOmniDataset( dataset_path="liarliar/Daily-Omni", dataset_split="train", @@ -210,6 +211,68 @@ def _resolve_hf_cache_snapshot(path: Path) -> Path: return path +#: QA metadata pulled by :func:`ensure_daily_omni_hub_root`. ``Videos.tar`` is deliberately left +#: out: it is ~60GB and only :func:`ensure_daily_omni_hub_videos_dir` needs it, lazily, once a +#: request actually asks for media. +_DAILY_OMNI_QA_ALLOW_PATTERNS = ["qa.json", "QA.json"] + + +def ensure_daily_omni_hub_root(repo_id: str) -> Path: + """Resolve a Daily-Omni Hub id to its snapshot directory, downloading QA metadata if needed. + + Mirrors :func:`...videomme_dataset.ensure_videomme_hub_root` and Seed-TTS: ``snapshot_download`` + resolves the repo through the **hub cache**, so an air-gapped run (``HF_HUB_OFFLINE=1``) keeps + working as long as the dataset was pre-staged there. ``datasets.load_dataset`` cannot do this — + it only consults the separate *datasets* cache, which a ``hf download`` never populates. + + Raises: + ImportError: if ``huggingface_hub`` is not installed. + ValueError: if ``repo_id`` is empty. + """ + rid = (repo_id or "").strip() + if not rid: + raise ValueError("repo_id is required to download Daily-Omni from Hugging Face") + + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise ImportError( + "Install huggingface_hub to load Daily-Omni from the Hub, or pass a local " + "--daily-omni-qa-json / --daily-omni-video-dir mirror." + ) from e + + cache = snapshot_download( + repo_id=rid, + repo_type="dataset", + allow_patterns=_DAILY_OMNI_QA_ALLOW_PATTERNS, + ) + root = _resolve_hf_cache_snapshot(Path(cache).resolve()) + logger.info("Daily-Omni Hub snapshot ready at %s (repo=%s)", root, rid) + return root + + +def _daily_omni_qa_load_error( + repo_id: str | None, + snapshot_error: Exception | None, + datasets_error: Exception, +) -> RuntimeError: + """Compose an actionable error after both QA-metadata sources failed. + + Offline CI hits this when the dataset is missing from the hub cache *or* staged under a + different repo id, so the message names both attempts and the flags that bypass them. + """ + parts = [f"Could not load Daily-Omni QA metadata for {repo_id!r}."] + if snapshot_error is not None: + parts.append(f"huggingface_hub snapshot_download failed: {type(snapshot_error).__name__}: {snapshot_error}") + parts.append(f"datasets.load_dataset failed: {type(datasets_error).__name__}: {datasets_error}") + parts.append( + "For offline runs, pre-stage the dataset in the HF hub cache " + "(`hf download --repo-type dataset `) and make sure the repo id matches, " + "or pass --daily-omni-qa-json / --daily-omni-video-dir pointing at a local mirror." + ) + return RuntimeError(" ".join(parts)) + + def resolve_daily_omni_local_root(dataset_path: str | None) -> Path | None: """Return the local Daily-Omni mirror root for ``dataset_path``, or ``None`` if not local. @@ -433,7 +496,8 @@ class DailyOmniDataset(BenchmarkDataset): QA metadata can be loaded from: - Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments - - HuggingFace datasets (``dataset_path``): legacy online mode + - HuggingFace Hub id (``dataset_path``): the repo's ``qa.json`` via ``snapshot_download`` + (hub-cache backed, offline-safe); ``datasets.load_dataset`` only as a parquet fallback Video/audio files normally come from extracted ``Videos.tar``. When ``video_dir`` is not set, the first sample that needs on-disk media downloads that archive from the Hugging Face dataset @@ -444,7 +508,8 @@ class DailyOmniDataset(BenchmarkDataset): qa_json_path: Path to local qa.json file (offline mode, preferred). When provided, ``dataset_path`` and ``dataset_split`` are ignored. dataset_path: HuggingFace dataset path (e.g., "liarliar/Daily-Omni"). Used only if - ``qa_json_path`` is not provided (legacy online mode). + ``qa_json_path`` is not provided; resolved through the hub cache, so a pre-staged + repo also loads with ``HF_HUB_OFFLINE=1``. dataset_split: Dataset split to use (default: "train"). Used only in online mode. random_seed: Random seed for shuffling video_dir: Directory containing extracted video files (default: None; may be filled lazily @@ -602,21 +667,42 @@ class DailyOmniDataset(BenchmarkDataset): self.data = _ListDatasetIterator(data) def _load_from_huggingface(self) -> None: - """Load QA rows via ``datasets.load_dataset`` (legacy / convenience path). + """Load QA rows for a Hub id, preferring the repo's ``qa.json`` over ``datasets``. - Kept for backward compatibility: callers can still pass ``dataset_path=liarliar/Daily-Omni`` - and get the same parquet-backed rows as the Hub dataset card, with streaming (or - non-streaming if ``no_stream=True``) and shuffle. + The Hub repo ships ``qa.json`` (1,197 rows) next to ``Videos.tar``, so the primary path is + :func:`ensure_daily_omni_hub_root` → :meth:`_load_from_local_json`: ``snapshot_download`` + reads the hub cache and therefore works under ``HF_HUB_OFFLINE``, the same contract as + Seed-TTS and Video-MME. ``datasets.load_dataset`` remains only as a fallback for mirrors + that publish parquet rows instead of ``qa.json``; it consults the *datasets* cache rather + than the hub cache and streams shards over HTTP, so it cannot serve air-gapped runs. This is intentionally **not** implemented by subclassing ``HuggingFaceDataset``: that base always runs Hub ``load_dataset`` from its constructor and expects a Hub id as the primary API; Daily-Omni instead chooses the source in ``load_data()`` (JSON vs Hub) while sharing one ``sample()`` / request-building implementation for both. """ + snapshot_error: Exception | None = None + try: + root = ensure_daily_omni_hub_root(self.dataset_path) + except Exception as e: # noqa: BLE001 - any failure just falls through to `datasets` + snapshot_error = e + else: + qa_json = daily_omni_local_qa_json(root) + if qa_json is not None: + self.qa_json_path = qa_json + logger.info("Loading Daily-Omni QA metadata from Hub snapshot: %s", qa_json) + self._load_from_local_json() + return + logger.info( + "Hub snapshot %s ships no qa.json; falling back to `datasets.load_dataset`", + root, + ) + if load_dataset is None: - raise ImportError( - "datasets library is required for HuggingFace mode. " - "Install with: pip install datasets, or use local JSON mode instead." + raise _daily_omni_qa_load_error( + self.dataset_path, + snapshot_error, + ImportError("datasets library is not installed (pip install datasets)"), ) load_kw: dict[str, Any] = { @@ -626,7 +712,10 @@ class DailyOmniDataset(BenchmarkDataset): } if self.dataset_subset is not None: load_kw["name"] = self.dataset_subset - ds = load_dataset(self.dataset_path, **load_kw) + try: + ds = load_dataset(self.dataset_path, **load_kw) + except Exception as e: + raise _daily_omni_qa_load_error(self.dataset_path, snapshot_error, e) from e if not getattr(self, "disable_shuffle", False): ds = ds.shuffle(seed=self.random_seed) self.data = ds diff --git a/vllm_omni/benchmarks/data_modules/videomme_dataset.py b/vllm_omni/benchmarks/data_modules/videomme_dataset.py index 68a5263e6..15b8386b6 100644 --- a/vllm_omni/benchmarks/data_modules/videomme_dataset.py +++ b/vllm_omni/benchmarks/data_modules/videomme_dataset.py @@ -1,6 +1,6 @@ """Video-MME dataset loader for vLLM-Omni bench serve. -Video-MME (lmms-lab/Video-MME) is a full-spectrum video MLLM benchmark with +Video-MME (lmms-eval/Video-MME) is a full-spectrum video MLLM benchmark with 900 videos / 2,700 MCQ pairs across short / medium / long durations. This loader follows OpenBMB OmniEvalKit's MiniCPM-o recipe @@ -55,7 +55,7 @@ _MINICPM_AUDIO_SR = 16000 # OmniEvalKit MiniCPM ``videomme`` / ``videomme_short`` defaults. VIDEOMME_DEFAULT_MAX_FRAMES = 96 VIDEOMME_SHORT_MAX_FRAMES = 64 -VIDEOMME_DEFAULT_HF_REPO = "lmms-lab/Video-MME" +VIDEOMME_DEFAULT_HF_REPO = "lmms-eval/Video-MME" #: Decode forward to reach a target this close, seek to the preceding keyframe beyond it. _SEEK_THRESHOLD_S = 3.0 @@ -114,7 +114,7 @@ def _resolve_hf_cache_snapshot(path: Path) -> Path: def resolve_videomme_local_root(dataset_path: str | None) -> Path | None: """Return a local Video-MME root if ``dataset_path`` points at an on-disk mirror. - Hub ids such as ``lmms-lab/Video-MME`` return ``None`` (they are not directories). + Hub ids such as ``lmms-eval/Video-MME`` return ``None`` (they are not directories). """ raw = (dataset_path or "").strip() if not raw: @@ -173,7 +173,7 @@ def resolve_videomme_root(dataset_path: str | None) -> Path: * Existing local directories (absolute or relative) are used as-is. * Otherwise ``dataset_path`` is treated as a Hugging Face dataset id and downloaded via :func:`ensure_videomme_hub_root` (default - ``lmms-lab/Video-MME``). + ``lmms-eval/Video-MME``). """ local = resolve_videomme_local_root(dataset_path) if local is not None: