661 lines
25 KiB
Python
661 lines
25 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""OmniInteract dataset discovery for the standard serving benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import shutil
|
|
import tarfile
|
|
import tempfile
|
|
from collections.abc import Sequence
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
|
|
from vllm.tokenizers import TokenizerLike
|
|
from vllm.transformers_utils.repo_utils import hf_fs
|
|
|
|
OMNIINTERACT_SUBSETS = ("1q1a", "1q1a_math", "1qna")
|
|
OMNIINTERACT_SCENARIO_TAGS = ("realtime", "proactive", "nested", "interrupted", "1qna")
|
|
DEFAULT_OMNIINTERACT_REPO = "lucky-lance/OmniInteract"
|
|
DEFAULT_MAX_VIDEO_DURATION_S = 3600.0
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OmniInteractCase:
|
|
subset: str
|
|
video_rel: str
|
|
video_path: Path
|
|
annotation_path: Path
|
|
scene_type: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OmniInteractEvaluationOptions:
|
|
judge_base_url: str
|
|
judge_model: str
|
|
judge_api_key: str
|
|
judge_timeout_s: float
|
|
judge_max_tokens: int
|
|
workers: int
|
|
output_dir: Path
|
|
skip_existing: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OmniInteractSessionOptions:
|
|
output_root: Path
|
|
timeout_s: float
|
|
media_timeout_s: float
|
|
ref_audio: str
|
|
require_response: bool = False
|
|
max_video_duration_s: float = DEFAULT_MAX_VIDEO_DURATION_S
|
|
evaluation: OmniInteractEvaluationOptions | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OmniInteractPreparedInput:
|
|
duration_s: float
|
|
pcm16: bytes = field(repr=False)
|
|
video_frames: tuple[str | None, ...] = field(repr=False)
|
|
ref_audio_data_url: str = field(repr=False)
|
|
|
|
|
|
@dataclass
|
|
class OmniInteractSampleRequest(SampleRequest):
|
|
omniinteract_case: OmniInteractCase | None = None
|
|
omniinteract_options: OmniInteractSessionOptions | None = None
|
|
omniinteract_prepared_input: OmniInteractPreparedInput | None = field(default=None, repr=False)
|
|
|
|
|
|
class OmniInteractDataset(BenchmarkDataset):
|
|
def __init__(
|
|
self,
|
|
*,
|
|
data_root: str | None,
|
|
dataset_repo: str = DEFAULT_OMNIINTERACT_REPO,
|
|
subsets: Sequence[str] = OMNIINTERACT_SUBSETS,
|
|
random_seed: int = 0,
|
|
disable_shuffle: bool = False,
|
|
scenario_tags: Sequence[str] = (),
|
|
scenario_focus: bool = False,
|
|
video_list: str | Path | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
dataset_path=data_root or dataset_repo,
|
|
random_seed=random_seed,
|
|
disable_shuffle=disable_shuffle,
|
|
)
|
|
self.video_list = Path(video_list).expanduser().resolve() if video_list else None
|
|
self.subsets = tuple(subsets)
|
|
self.scenario_tags = normalize_scenario_tags(scenario_tags)
|
|
self.scenario_focus = bool(scenario_focus)
|
|
if self.video_list is not None:
|
|
if self.scenario_tags:
|
|
raise ValueError("OmniInteract video_list cannot be combined with scenario_tags")
|
|
if self.scenario_focus:
|
|
raise ValueError("OmniInteract video_list cannot be combined with scenario_focus")
|
|
if not self.video_list.is_file():
|
|
raise FileNotFoundError(f"--omniinteract-video-list does not exist: {self.video_list}")
|
|
self.root = None
|
|
else:
|
|
self.root = resolve_omniinteract_root(data_root, dataset_repo)
|
|
if self.scenario_focus and not self.scenario_tags:
|
|
raise ValueError("scenario_focus requires at least one scenario tag")
|
|
|
|
def sample(
|
|
self,
|
|
tokenizer: TokenizerLike | None,
|
|
num_requests: int,
|
|
*,
|
|
request_id_prefix: str = "",
|
|
options: OmniInteractSessionOptions,
|
|
**_: Any,
|
|
) -> list[SampleRequest]:
|
|
del tokenizer
|
|
if self.video_list is not None:
|
|
cases = load_omniinteract_cases_from_video_list(self.video_list, num_prompts=num_requests)
|
|
else:
|
|
assert self.root is not None
|
|
cases = discover_omniinteract_cases(
|
|
self.root,
|
|
self.subsets,
|
|
num_prompts=num_requests,
|
|
seed=self.random_seed,
|
|
disable_shuffle=self.disable_shuffle,
|
|
scenario_tags=self.scenario_tags,
|
|
scenario_focus=self.scenario_focus,
|
|
)
|
|
return [
|
|
OmniInteractSampleRequest(
|
|
prompt="",
|
|
prompt_len=0,
|
|
expected_output_len=0,
|
|
multi_modal_data=None,
|
|
request_id=f"{request_id_prefix}{index}",
|
|
omniinteract_case=case,
|
|
omniinteract_options=options,
|
|
)
|
|
for index, case in enumerate(cases)
|
|
]
|
|
|
|
|
|
def _data_dir(root: Path) -> Path:
|
|
for candidate in (root, root / "data"):
|
|
if any((candidate / subset).is_dir() for subset in OMNIINTERACT_SUBSETS):
|
|
return candidate
|
|
raise FileNotFoundError(f"OmniInteract data not found under {root}")
|
|
|
|
|
|
def _confined_path(root: Path, relative: object, *, field: str) -> Path:
|
|
text, resolved_root = str(relative or ""), root.resolve()
|
|
path = Path(text)
|
|
destination = (root / path).resolve()
|
|
if not text or path.is_absolute() or ".." in path.parts or not destination.is_relative_to(resolved_root):
|
|
raise ValueError(f"Unsafe OmniInteract {field} path: {text!r}")
|
|
return destination
|
|
|
|
|
|
def _safe_extract(archive: tarfile.TarFile, target: Path) -> None:
|
|
root = target.resolve()
|
|
members = archive.getmembers()
|
|
for member in members:
|
|
path = Path(member.name)
|
|
if (
|
|
path.is_absolute()
|
|
or ".." in path.parts
|
|
or not (root / path).resolve().is_relative_to(root)
|
|
or not (member.isdir() or member.isfile())
|
|
):
|
|
raise ValueError(f"Unsafe path in OmniInteract archive: {member.name!r}")
|
|
archive.extractall(target, members=members, filter="data")
|
|
|
|
|
|
def _extract_archive(archive: Path, target: Path) -> Path:
|
|
if target.is_symlink():
|
|
raise ValueError(f"Refusing to extract through symlink: {target}")
|
|
stat = archive.stat()
|
|
published = target / f"{stat.st_size}-{stat.st_mtime_ns}"
|
|
if published.is_symlink():
|
|
raise ValueError(f"Refusing to use symlinked extraction: {published}")
|
|
if published.is_dir():
|
|
try:
|
|
return _data_dir(published)
|
|
except FileNotFoundError:
|
|
pass
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
staging = Path(tempfile.mkdtemp(prefix=".tmp-", dir=target))
|
|
try:
|
|
with tarfile.open(archive, "r:*") as handle:
|
|
_safe_extract(handle, staging)
|
|
root = _data_dir(staging)
|
|
try:
|
|
staging.replace(published)
|
|
return published / root.relative_to(staging)
|
|
except OSError:
|
|
if not published.is_dir():
|
|
raise
|
|
return _data_dir(published)
|
|
finally:
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
|
|
|
|
def resolve_omniinteract_root(data_root: str | None, dataset_repo: str = DEFAULT_OMNIINTERACT_REPO) -> Path:
|
|
"""Resolve a local tree/archive or download the official archive."""
|
|
|
|
if data_root:
|
|
local = Path(data_root).expanduser().resolve()
|
|
if local.is_file():
|
|
return _extract_archive(local, local.parent / f".{local.stem}.vllm_omni_extracted")
|
|
if not local.is_dir():
|
|
raise FileNotFoundError(f"--dataset-path does not exist: {local}")
|
|
try:
|
|
return _data_dir(local)
|
|
except FileNotFoundError:
|
|
for name in ("data.tar.gz", "data.tar"):
|
|
if (archive := local / name).is_file():
|
|
return _extract_archive(archive, local / ".vllm_omni_extracted")
|
|
raise
|
|
|
|
cache = Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface"))
|
|
target = cache / "vllm_omni" / "omniinteract" / dataset_repo.replace("/", "__")
|
|
downloads = target / ".downloads"
|
|
downloads.mkdir(parents=True, exist_ok=True)
|
|
filesystem, errors = hf_fs(), []
|
|
for name in ("data.tar.gz", "data.tar"):
|
|
archive = downloads / name
|
|
try:
|
|
if not archive.is_file():
|
|
descriptor, temporary_name = tempfile.mkstemp(prefix=f".{name}.", dir=downloads)
|
|
os.close(descriptor)
|
|
temporary = Path(temporary_name)
|
|
try:
|
|
filesystem.get_file(f"datasets/{dataset_repo}/{name}", str(temporary))
|
|
if not temporary.stat().st_size:
|
|
raise ValueError("downloaded archive is empty")
|
|
temporary.replace(archive)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
return _extract_archive(archive, target)
|
|
except Exception as exc: # noqa: BLE001, PERF203 - Hub errors vary by version
|
|
errors.append(f"{name}: {exc}")
|
|
raise FileNotFoundError(f"Could not download OmniInteract from {dataset_repo!r}: {'; '.join(errors)}")
|
|
|
|
|
|
def _mapping_cases(root: Path, subset: str) -> list[OmniInteractCase]:
|
|
mapping_path = root / "video_json_map.json"
|
|
if not mapping_path.is_file():
|
|
raise FileNotFoundError(f"Missing OmniInteract mapping: {mapping_path}")
|
|
mapping = json.loads(mapping_path.read_text())
|
|
entries = mapping.get("entries") if isinstance(mapping, dict) else None
|
|
if not isinstance(entries, list):
|
|
raise ValueError(f"Invalid OmniInteract mapping: {mapping_path}")
|
|
cases = []
|
|
for row in entries:
|
|
if not isinstance(row, dict):
|
|
raise ValueError(f"Invalid OmniInteract mapping row in {mapping_path}")
|
|
video_rel, annotation_rel = str(row.get("video") or ""), str(row.get("annotation") or "")
|
|
video = _confined_path(root, video_rel, field="video")
|
|
annotation = _confined_path(root, annotation_rel, field="annotation")
|
|
if not video.is_file() or not annotation.is_file():
|
|
raise FileNotFoundError(f"OmniInteract mapping references missing files: {video_rel!r}, {annotation_rel!r}")
|
|
cases.append(
|
|
OmniInteractCase(
|
|
subset,
|
|
video_rel,
|
|
video,
|
|
annotation,
|
|
str(row.get("scene_type") or "multi_turn").lower(),
|
|
)
|
|
)
|
|
return cases
|
|
|
|
|
|
def _one_to_many_cases(root: Path) -> list[OmniInteractCase]:
|
|
videos, annotations = root / "videos_bench", root / "annotations"
|
|
if not videos.is_dir() or not annotations.is_dir():
|
|
raise FileNotFoundError(f"Invalid OmniInteract 1qna layout under {root}")
|
|
cases = []
|
|
for video in sorted(videos.rglob("*.mp4")):
|
|
if not video.resolve().is_relative_to(videos.resolve()):
|
|
raise ValueError(f"Unsafe OmniInteract video path: {video}")
|
|
relative = video.relative_to(videos)
|
|
annotation = (annotations / relative).with_suffix(".json").resolve()
|
|
if not annotation.is_relative_to(annotations.resolve()):
|
|
raise ValueError(f"Unsafe OmniInteract annotation path: {relative}")
|
|
if not annotation.is_file():
|
|
raise FileNotFoundError(f"Missing OmniInteract annotation: {annotation}")
|
|
cases.append(OmniInteractCase("1qna", f"videos_bench/{relative.as_posix()}", video, annotation, "1qna"))
|
|
return cases
|
|
|
|
|
|
def normalize_scenario_tags(tags: Sequence[str] | None) -> tuple[str, ...]:
|
|
"""Validate and dedupe scenario tags while preserving caller order."""
|
|
|
|
if not tags:
|
|
return ()
|
|
normalized: list[str] = []
|
|
seen: set[str] = set()
|
|
invalid: list[str] = []
|
|
for raw in tags:
|
|
tag = str(raw or "").strip().lower()
|
|
if not tag:
|
|
continue
|
|
if tag not in OMNIINTERACT_SCENARIO_TAGS:
|
|
invalid.append(tag)
|
|
continue
|
|
if tag not in seen:
|
|
seen.add(tag)
|
|
normalized.append(tag)
|
|
if invalid:
|
|
raise ValueError(
|
|
"Unsupported OmniInteract scenario tags: "
|
|
f"{sorted(set(invalid))}; choose from {list(OMNIINTERACT_SCENARIO_TAGS)}"
|
|
)
|
|
return tuple(normalized)
|
|
|
|
|
|
def _truthy(value: object) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, int | float):
|
|
return bool(value)
|
|
return str(value or "").strip().lower() in {"1", "true", "yes", "y", "t"}
|
|
|
|
|
|
def _annotation_rows(root: object) -> list[dict[str, object]]:
|
|
if isinstance(root, list):
|
|
return [row for row in root if isinstance(row, dict)]
|
|
if not isinstance(root, dict):
|
|
return []
|
|
rows: list[dict[str, object]] = []
|
|
items = root.get("items")
|
|
if isinstance(items, list):
|
|
rows.extend(row for row in items if isinstance(row, dict))
|
|
answers = root.get("answers")
|
|
if isinstance(answers, list):
|
|
rows.extend(row for row in answers if isinstance(row, dict))
|
|
conversations = root.get("conversations")
|
|
if isinstance(conversations, list):
|
|
rows.extend(row for row in conversations if isinstance(row, dict))
|
|
return rows
|
|
|
|
|
|
def case_scenario_tags(case: OmniInteractCase) -> frozenset[str]:
|
|
"""Infer paper-table scenario tags from a case's scene type and annotation."""
|
|
|
|
tags: set[str] = set()
|
|
scene = str(case.scene_type or "").strip().lower()
|
|
if scene == "1qna":
|
|
tags.add("1qna")
|
|
elif scene == "nested":
|
|
tags.add("nested")
|
|
|
|
try:
|
|
payload = json.loads(case.annotation_path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
raise ValueError(f"Invalid OmniInteract annotation: {case.annotation_path}") from exc
|
|
|
|
for row in _annotation_rows(payload):
|
|
question_type = str(row.get("question_type") or row.get("label") or "").strip().lower()
|
|
if scene == "multi_turn":
|
|
if question_type == "realtime":
|
|
tags.add("realtime")
|
|
elif question_type == "proactive":
|
|
tags.add("proactive")
|
|
if _truthy(row.get("is_interrupted")) or _truthy(row.get("interrupted")):
|
|
tags.add("interrupted")
|
|
return frozenset(tags)
|
|
|
|
|
|
def _select_with_scenario_tags(
|
|
cases: list[OmniInteractCase],
|
|
*,
|
|
num_prompts: int,
|
|
seed: int,
|
|
disable_shuffle: bool,
|
|
scenario_tags: Sequence[str],
|
|
scenario_focus: bool,
|
|
) -> list[OmniInteractCase]:
|
|
tags = normalize_scenario_tags(scenario_tags)
|
|
if not tags:
|
|
ordered = list(cases)
|
|
if not disable_shuffle:
|
|
random.Random(seed).shuffle(ordered)
|
|
if num_prompts:
|
|
if num_prompts > len(ordered):
|
|
logger.warning(
|
|
"Requested %d OmniInteract prompts but only %d are available; using all cases",
|
|
num_prompts,
|
|
len(ordered),
|
|
)
|
|
return ordered[:num_prompts]
|
|
return ordered
|
|
|
|
tag_sets = {case.video_path: case_scenario_tags(case) for case in cases}
|
|
if scenario_focus:
|
|
pool = [case for case in cases if tag_sets[case.video_path] & set(tags)]
|
|
missing = [tag for tag in tags if not any(tag in tag_sets[case.video_path] for case in pool)]
|
|
if missing:
|
|
raise ValueError(
|
|
f"No OmniInteract cases match scenario tags {missing} under the selected subsets (focus mode)"
|
|
)
|
|
if not pool:
|
|
raise ValueError(f"No OmniInteract cases match scenario tags {list(tags)}")
|
|
ordered = list(pool)
|
|
if not disable_shuffle:
|
|
random.Random(seed).shuffle(ordered)
|
|
if num_prompts:
|
|
if num_prompts > len(ordered):
|
|
logger.warning(
|
|
"Requested %d OmniInteract prompts but only %d match tags %s; using all matches",
|
|
num_prompts,
|
|
len(ordered),
|
|
list(tags),
|
|
)
|
|
return ordered[:num_prompts]
|
|
return ordered
|
|
|
|
rng = random.Random(seed)
|
|
buckets: dict[str, list[OmniInteractCase]] = {tag: [] for tag in tags}
|
|
for case in cases:
|
|
owned = tag_sets[case.video_path]
|
|
for tag in tags:
|
|
if tag in owned:
|
|
buckets[tag].append(case)
|
|
for tag, bucket in buckets.items():
|
|
if not bucket:
|
|
logger.warning(
|
|
"No OmniInteract cases available for scenario tag %r under the selected subsets",
|
|
tag,
|
|
)
|
|
elif not disable_shuffle:
|
|
rng.shuffle(bucket)
|
|
|
|
selected: list[OmniInteractCase] = []
|
|
selected_paths: set[Path] = set()
|
|
covered_tags: set[str] = set()
|
|
requested = set(tags)
|
|
for tag in tags:
|
|
if tag in covered_tags:
|
|
continue
|
|
for case in buckets[tag]:
|
|
if case.video_path in selected_paths:
|
|
continue
|
|
selected.append(case)
|
|
selected_paths.add(case.video_path)
|
|
covered_tags.update(tag_sets[case.video_path] & requested)
|
|
break
|
|
|
|
remaining = [case for case in cases if case.video_path not in selected_paths]
|
|
if not disable_shuffle:
|
|
rng.shuffle(remaining)
|
|
|
|
if num_prompts == 0:
|
|
return selected + remaining
|
|
|
|
if len(selected) > num_prompts:
|
|
logger.warning(
|
|
"OmniInteract cover tags need %d cases but --num-prompts=%d; truncating coverage set",
|
|
len(selected),
|
|
num_prompts,
|
|
)
|
|
return selected[:num_prompts]
|
|
|
|
return selected + remaining[: num_prompts - len(selected)]
|
|
|
|
|
|
def discover_omniinteract_cases(
|
|
root: Path,
|
|
subsets: Sequence[str],
|
|
*,
|
|
num_prompts: int,
|
|
seed: int = 0,
|
|
disable_shuffle: bool = False,
|
|
scenario_tags: Sequence[str] = (),
|
|
scenario_focus: bool = False,
|
|
) -> list[OmniInteractCase]:
|
|
invalid = set(subsets) - set(OMNIINTERACT_SUBSETS)
|
|
if invalid:
|
|
raise ValueError(f"Unsupported OmniInteract subsets: {sorted(invalid)}")
|
|
if not subsets:
|
|
raise ValueError("At least one OmniInteract subset is required")
|
|
if len(set(subsets)) != len(subsets):
|
|
raise ValueError("OmniInteract subsets must not contain duplicates")
|
|
if num_prompts < 0:
|
|
raise ValueError("num_prompts must be non-negative")
|
|
tags = normalize_scenario_tags(scenario_tags)
|
|
if scenario_focus and not tags:
|
|
raise ValueError("scenario_focus requires at least one scenario tag")
|
|
data_root, cases = _data_dir(root.resolve()), []
|
|
for subset in subsets:
|
|
subset_root = data_root / subset
|
|
selected = _one_to_many_cases(subset_root) if subset == "1qna" else _mapping_cases(subset_root, subset)
|
|
if not selected:
|
|
raise ValueError(f"No OmniInteract sessions found for requested subset {subset!r}")
|
|
cases.extend(selected)
|
|
if len({case.video_path for case in cases}) != len(cases):
|
|
raise ValueError("OmniInteract dataset contains duplicate video paths")
|
|
return _select_with_scenario_tags(
|
|
cases,
|
|
num_prompts=num_prompts,
|
|
seed=seed,
|
|
disable_shuffle=disable_shuffle,
|
|
scenario_tags=tags,
|
|
scenario_focus=scenario_focus,
|
|
)
|
|
|
|
|
|
def _subset_root_for_video(video_path: Path, subset: str) -> Path:
|
|
"""Find the subset data root that owns an absolute video path."""
|
|
|
|
if subset not in OMNIINTERACT_SUBSETS:
|
|
raise ValueError(f"Unsupported OmniInteract subset: {subset!r}")
|
|
parts = video_path.resolve().parts
|
|
for index in range(len(parts) - 1, -1, -1):
|
|
if parts[index] != subset:
|
|
continue
|
|
candidate = Path(*parts[: index + 1])
|
|
if subset == "1qna":
|
|
if (candidate / "videos_bench").is_dir() and (candidate / "annotations").is_dir():
|
|
return candidate
|
|
elif (candidate / "video_json_map.json").is_file():
|
|
return candidate
|
|
raise ValueError(f"Could not locate OmniInteract subset root for {subset!r} from {video_path}")
|
|
|
|
|
|
def _case_from_video_list_row(row: dict[str, object], *, line_no: int) -> OmniInteractCase:
|
|
"""Rebuild an ``OmniInteractCase`` from one ``sampled_cases.jsonl`` row."""
|
|
|
|
subset = str(row.get("subset") or "")
|
|
video_text = str(row.get("video_path") or "")
|
|
if not subset or not video_text:
|
|
raise ValueError(f"OmniInteract video_list line {line_no} requires subset and video_path")
|
|
video_path = Path(video_text).expanduser().resolve()
|
|
if not video_path.is_file():
|
|
raise FileNotFoundError(f"OmniInteract video_list line {line_no} video missing: {video_path}")
|
|
|
|
subset_root = _subset_root_for_video(video_path, subset)
|
|
if subset == "1qna":
|
|
videos_root = (subset_root / "videos_bench").resolve()
|
|
annotations_root = (subset_root / "annotations").resolve()
|
|
if not video_path.is_relative_to(videos_root):
|
|
raise ValueError(f"OmniInteract video_list line {line_no} is outside videos_bench: {video_path}")
|
|
relative = video_path.relative_to(videos_root)
|
|
annotation = (subset_root / "annotations" / relative).with_suffix(".json").resolve()
|
|
if not annotation.is_relative_to(annotations_root):
|
|
raise ValueError(f"OmniInteract video_list line {line_no} unsafe annotation path: {relative}")
|
|
if not annotation.is_file():
|
|
raise FileNotFoundError(f"OmniInteract video_list line {line_no} annotation missing: {annotation}")
|
|
case = OmniInteractCase(
|
|
subset,
|
|
f"videos_bench/{relative.as_posix()}",
|
|
video_path,
|
|
annotation,
|
|
"1qna",
|
|
)
|
|
else:
|
|
by_path = {case.video_path.resolve(): case for case in _mapping_cases(subset_root, subset)}
|
|
case = by_path.get(video_path)
|
|
if case is None:
|
|
raise ValueError(f"OmniInteract video_list line {line_no} is not in {subset} mapping: {video_path}")
|
|
|
|
expected_name = str(row.get("output_name") or "")
|
|
if expected_name and expected_name != official_output_name(case):
|
|
raise ValueError(
|
|
f"OmniInteract video_list line {line_no} output_name mismatch: "
|
|
f"{expected_name!r} != {official_output_name(case)!r}"
|
|
)
|
|
return case
|
|
|
|
|
|
def load_omniinteract_cases_from_video_list(
|
|
video_list: str | Path,
|
|
*,
|
|
num_prompts: int,
|
|
) -> list[OmniInteractCase]:
|
|
"""Load cases from a ``sampled_cases.jsonl``-style list, preserving order.
|
|
|
|
``num_prompts`` caps the prefix of the list. ``0`` or a value larger than the
|
|
list length selects every row.
|
|
"""
|
|
|
|
if num_prompts < 0:
|
|
raise ValueError("num_prompts must be non-negative")
|
|
path = Path(video_list).expanduser().resolve()
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"--omniinteract-video-list does not exist: {path}")
|
|
|
|
cases: list[OmniInteractCase] = []
|
|
seen: set[Path] = set()
|
|
with path.open(encoding="utf-8") as handle:
|
|
for line_no, raw in enumerate(handle, start=1):
|
|
text = raw.strip()
|
|
if not text:
|
|
continue
|
|
try:
|
|
row = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"OmniInteract video_list line {line_no} is not valid JSON") from exc
|
|
if not isinstance(row, dict):
|
|
raise ValueError(f"OmniInteract video_list line {line_no} must be a JSON object")
|
|
case = _case_from_video_list_row(row, line_no=line_no)
|
|
if case.video_path in seen:
|
|
raise ValueError(f"OmniInteract video_list contains duplicate video_path: {case.video_path}")
|
|
seen.add(case.video_path)
|
|
cases.append(case)
|
|
|
|
if not cases:
|
|
raise ValueError(f"OmniInteract video_list is empty: {path}")
|
|
if num_prompts == 0 or num_prompts >= len(cases):
|
|
return cases
|
|
return cases[:num_prompts]
|
|
|
|
|
|
def official_output_name(case: OmniInteractCase) -> str:
|
|
"""Match Lucky-Lance/OmniInteract ``collect_all_videos`` output_name layout.
|
|
|
|
``1q1a`` / ``1q1a_math`` use the mapping-relative video path with ``/`` →
|
|
``__``. ``1qna`` strips the ``videos_bench/`` prefix before the same rewrite.
|
|
"""
|
|
|
|
relative = case.video_rel.replace("\\", "/")
|
|
if case.subset == "1qna":
|
|
prefix = "videos_bench/"
|
|
if relative.startswith(prefix):
|
|
relative = relative[len(prefix) :]
|
|
stem = Path(relative).with_suffix("").as_posix().replace("/", "__")
|
|
return f"{case.subset}/{stem}"
|
|
|
|
|
|
def sampled_case_row(case: OmniInteractCase) -> dict[str, str]:
|
|
"""One official MiniCPM-o ``--video_list`` row for a sampled OmniInteract case.
|
|
|
|
``video_path`` reuses the absolute path already resolved from ``--dataset-path``
|
|
(local tree or Hub extract). Official
|
|
``batch_inference_minicpmo.py --video_list`` accepts these rows as-is.
|
|
"""
|
|
|
|
return {
|
|
"video_path": str(case.video_path.resolve()),
|
|
"output_name": official_output_name(case),
|
|
"subset": case.subset,
|
|
}
|
|
|
|
|
|
def case_manifest(case: OmniInteractCase, output_dir: Path) -> dict[str, Any]:
|
|
return {
|
|
"sample_id": f"{case.subset}__{output_dir.name}",
|
|
"video": str(case.video_path),
|
|
"gt_json": str(case.annotation_path.resolve()),
|
|
"model_json": str((output_dir / "wav_transcript.json").resolve()),
|
|
"scene_type": "1QnA" if case.scene_type == "1qna" else case.scene_type,
|
|
}
|