diff --git a/tests/benchmarks/test_daily_omni_pack_modes.py b/tests/benchmarks/test_daily_omni_pack_modes.py
new file mode 100644
index 000000000..0fd231857
--- /dev/null
+++ b/tests/benchmarks/test_daily_omni_pack_modes.py
@@ -0,0 +1,237 @@
+"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
+
+The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
+OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
+interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and no Qwen system prompt.
+These tests pin that contract so a refactor cannot silently regress the score.
+
+vllm stubs are installed by tests/benchmarks/conftest.py before collection.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
+_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
+
+if _MODULE_NAME not in sys.modules:
+ _spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
+ _mod = importlib.util.module_from_spec(_spec)
+ sys.modules[_MODULE_NAME] = _mod
+ _spec.loader.exec_module(_mod)
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( # noqa: E402
+ DailyOmniDataset,
+ _uniform_sample_indices,
+)
+
+_VIDEO_ID = "vid001"
+_NUM_FRAMES = 5
+
+
+@pytest.fixture
+def video_dir(tmp_path: Path) -> Path:
+ """Official Daily-Omni ``Videos/`` layout with placeholder media files."""
+ per_video = tmp_path / "Videos" / _VIDEO_ID
+ per_video.mkdir(parents=True)
+ (per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
+ (per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
+ return tmp_path / "Videos"
+
+
+@pytest.fixture
+def qa_json(tmp_path: Path) -> Path:
+ path = tmp_path / "qa.json"
+ path.write_text(
+ json.dumps(
+ [
+ {
+ "video_id": _VIDEO_ID,
+ "question": "What happens first?",
+ "choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
+ "answer": "A",
+ "task_type": "AV Event Alignment",
+ "video_duration": "30s",
+ }
+ ]
+ ),
+ encoding="utf-8",
+ )
+ return path
+
+
+def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
+ return DailyOmniDataset(
+ qa_json_path=str(qa_json),
+ video_dir=str(video_dir),
+ disable_shuffle=True,
+ **kwargs,
+ )
+
+
+def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
+ """Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
+ from PIL import Image
+
+ def fake_extract(video_path, *, audio_path, include_audio, **_kw):
+ counter.append(1)
+ frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
+ segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
+ return frames, segments
+
+ monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
+
+
+# ---------------------------------------------------------------------------
+# _uniform_sample_indices
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (7, 7), (3, 5), (1, 1)])
+def test_uniform_sample_indices_matches_numpy_linspace(n: int, k: int) -> None:
+ got = _uniform_sample_indices(n, k)
+ if n <= k:
+ assert got == list(range(n))
+ else:
+ assert got == [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
+
+
+def test_uniform_sample_indices_non_positive_k() -> None:
+ assert _uniform_sample_indices(10, 0) == []
+ assert _uniform_sample_indices(10, -1) == []
+
+
+# ---------------------------------------------------------------------------
+# minicpm-interleave packing
+# ---------------------------------------------------------------------------
+
+
+def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
+ ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+ _stub_extract(monkeypatch, ds, [])
+
+ parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ assert position == "first"
+ assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
+ assert extra["mm_processor_kwargs"] == {
+ "use_audio_in_video": False,
+ "max_slice_nums": 1,
+ "use_image_id": False,
+ }
+
+
+def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
+ ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
+ _stub_extract(monkeypatch, ds, [])
+
+ parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
+
+
+def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
+ ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+ _stub_extract(monkeypatch, ds, [])
+
+ parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
+ for part in parts:
+ url = part[part["type"]]["url"]
+ assert url.startswith("file://")
+ assert Path(url[len("file://") :]).parent == segment_dir.resolve()
+ assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
+
+
+def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
+ ds = _make_dataset(
+ qa_json,
+ video_dir,
+ pack_mode="minicpm-interleave",
+ input_mode="all",
+ inline_local_video=True,
+ )
+ _stub_extract(monkeypatch, ds, [])
+
+ parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
+ assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
+ assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
+
+
+def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
+ """Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
+ file_calls: list[int] = []
+ ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+ _stub_extract(monkeypatch, ds_file, file_calls)
+ ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
+ ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
+ assert len(file_calls) == 1
+ assert ds_file._minicpm_interleave_cache
+
+ inline_calls: list[int] = []
+ ds_inline = _make_dataset(
+ qa_json,
+ video_dir,
+ pack_mode="minicpm-interleave",
+ input_mode="all",
+ inline_local_video=True,
+ )
+ _stub_extract(monkeypatch, ds_inline, inline_calls)
+ ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
+ ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
+ assert len(inline_calls) == 2
+ assert not ds_inline._minicpm_interleave_cache
+
+
+def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
+ empty = tmp_path / "empty_videos"
+ empty.mkdir()
+ ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
+
+ parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ assert parts is None
+ assert extra is None
+
+
+# ---------------------------------------------------------------------------
+# System prompt selection
+# ---------------------------------------------------------------------------
+
+
+def test_minicpm_interleave_omits_qwen_system_message(monkeypatch, qa_json, video_dir) -> None:
+ ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+ _stub_extract(monkeypatch, ds, [])
+ parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+ messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
+
+ assert [m["role"] for m in messages] == ["user"]
+
+
+def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
+ ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
+
+ messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
+
+ assert [m["role"] for m in messages] == ["system", "user"]
+ assert "Qwen" in messages[0]["content"][0]["text"]
+
+
+def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
+ with pytest.raises(ValueError, match="pack_mode must be"):
+ _make_dataset(qa_json, video_dir, pack_mode="bogus")
diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py
index feb60df31..bce016d67 100644
--- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py
+++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py
@@ -24,8 +24,6 @@ from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
- load_deploy_config,
- merge_pipeline_deploy,
)
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
@@ -138,45 +136,6 @@ class TestPipelineTopology:
class TestDeployTopology:
- @pytest.mark.parametrize(("filename", "devices"), _DEPLOY_LAYOUTS.items())
- def test_deploy_resolves_three_stage_pipeline(self, filename: str, devices: list[str]) -> None:
- deploy = load_deploy_config(_DEPLOY_DIR / filename)
- pipeline = OMNI_PIPELINES[deploy.pipeline]
- stages = merge_pipeline_deploy(pipeline, deploy)
-
- assert deploy.pipeline == _PIPELINE_KEY
- assert deploy.async_chunk is True
- assert [stage.stage_id for stage in stages] == [0, 1, 2]
- assert [stage.yaml_runtime["devices"] for stage in stages] == devices
- assert "additional_config" not in stages[1].yaml_engine_args
- assert "skip_mm_profiling" not in stages[0].yaml_engine_args
- assert stages[1].yaml_engine_args["skip_mm_profiling"] is True
- assert stages[2].yaml_engine_args["skip_mm_profiling"] is True
- assert stages[1].yaml_extras["output_connectors"]["to_stage_2"] == "connector_of_shared_memory"
- assert stages[2].yaml_extras["input_connectors"]["from_stage_1"] == "connector_of_shared_memory"
- connector = deploy.connectors["connector_of_shared_memory"]
- assert connector["name"] == "SharedMemoryConnector"
- assert connector["extra"]["codec_chunk_frames"] == 25
- assert connector["extra"]["codec_left_context_frames"] == 3
- assert connector["extra"]["connector_get_max_wait_first_chunk"] == 3000
- assert connector["extra"]["connector_get_max_wait"] == 300
- expected_processor = "tts2code2wav_async_chunk" if deploy.async_chunk else "tts2code2wav_full_payload"
- assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor)
- if filename == "minicpmo_4_5.yaml":
- assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [16, 16, 16]
- assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
- 0.6,
- 0.1,
- 0.1,
- ]
- assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}}
- elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}:
- assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
- 0.9,
- 0.55,
- 0.35,
- ]
-
def test_pipeline_exposes_full_and_async_payload_hooks(self) -> None:
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
talker = pipeline.get_stage(1)
diff --git a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py
index 65918414f..fd8f32f30 100644
--- a/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py
+++ b/vllm_omni/benchmarks/data_modules/daily_omni_dataset.py
@@ -52,11 +52,14 @@ Usage:
"""
import base64
+import io
import json
import logging
+import math
import os
import shutil
import tarfile
+import wave
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal
@@ -77,6 +80,10 @@ except ImportError:
logger = logging.getLogger(__name__)
+# Official MiniCPM ``get_video_frame_audio_segments`` default (env ``MAX_NUM_FRAMES``).
+_MINICPM_MAX_NUM_FRAMES = int(os.environ.get("MAX_NUM_FRAMES", "64"))
+_MINICPM_AUDIO_SR = 16000
+
def _daily_omni_hf_cache_root() -> Path:
return Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")).expanduser().resolve()
@@ -203,12 +210,63 @@ class _ListDatasetIterator:
# Aligns with Lliar-liar/Daily-Omni CLI ``--input_mode`` (test_model/*/testmodel.py).
DailyOmniInputMode = Literal["all", "visual", "audio"]
+# How multimodal parts are packed into OpenAI chat messages:
+# - ``qwen``: one ``video_url`` + one ``audio_url`` (upstream Daily-Omni / Qwen protocol).
+# - ``minicpm-interleave``: 1fps ``[image_i, audio_i, ...]`` matching OpenBMB MiniCPM-o
+# ``get_video_frame_audio_segments`` + interleaved omni contents (paper ~80% recipe).
+DailyOmniPackMode = Literal["qwen", "minicpm-interleave"]
+
# ``build_conversation()`` in Daily-Omni ``test_model/Qwen2.5-Omni/testmodel.py`` (verbatim).
DAILY_OMNI_SYSTEM_TEXT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
+# MiniCPM-o ``get_sys_prompt(mode="omni")`` without ref-audio is an empty system string.
+MINICPM_OMNI_SYSTEM_TEXT = ""
+
+
+def _uniform_sample_indices(n: int, k: int) -> list[int]:
+ """Evenly pick ``k`` indices from ``0..n-1`` (MiniCPM ``uniform_sample`` on index lists)."""
+ if k <= 0:
+ return []
+ if n <= k:
+ return list(range(n))
+ # Match numpy linspace(..., dtype=int) used upstream.
+ try:
+ import numpy as np
+
+ return [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
+ except ImportError:
+ if k == 1:
+ return [0]
+ return [int(round(i * (n - 1) / (k - 1))) for i in range(k)]
+
+
+def _numpy_to_wav_bytes(audio_np: Any, sample_rate: int = _MINICPM_AUDIO_SR) -> bytes:
+ """Encode a mono float/int numpy waveform as 16-bit PCM WAV bytes."""
+ import numpy as np
+
+ arr = np.asarray(audio_np)
+ if arr.ndim > 1:
+ arr = arr.reshape(-1)
+ if arr.dtype != np.int16:
+ arr = np.clip(arr.astype(np.float64), -1.0, 1.0)
+ arr = (arr * 32767.0).astype(np.int16)
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(sample_rate)
+ wf.writeframes(arr.tobytes())
+ return buf.getvalue()
+
+
+def _pil_to_jpeg_bytes(image: Any, quality: int = 85) -> bytes:
+ buf = io.BytesIO()
+ image.save(buf, format="JPEG", quality=quality)
+ return buf.getvalue()
+
@dataclass
class DailyOmniSampleRequest(SampleRequest):
@@ -262,6 +320,11 @@ class DailyOmniDataset(BenchmarkDataset):
``all`` — video + WAV (default; official audio-visual protocol);
``visual`` — video only;
``audio`` — extracted WAV only (requires ``{video_id}/{video_id}_audio.wav`` under ``video_dir``).
+ pack_mode: Multimodal packing recipe:
+ ``qwen`` — one ``video_url`` + optional ``audio_url`` (default; Daily-Omni/Qwen);
+ ``minicpm-interleave`` — MiniCPM-o paper recipe: 1fps frames interleaved with
+ matching 1s audio segments as ``image_url``/``audio_url`` pairs (required to approach
+ OpenBMB ~80% Daily-Omni; needs local video+WAV under ``video_dir``).
max_duration_seconds: Reserved for future ffprobe-based filtering; currently **not applied**
when building requests (metadata ``video_duration`` is still passed through for eval).
dataset_subset: Optional HuggingFace subset name (``load_dataset(..., name=...)``); used by bench
@@ -270,7 +333,8 @@ class DailyOmniDataset(BenchmarkDataset):
inline_local_video: If True, embed local MP4 as ``data:video/mp4;base64,...`` in requests so
the API server does not need ``--allowed-local-media-path`` (large JSON; use for small runs).
When ``input_mode`` is ``audio`` or ``all``, local WAV is embedded the same way
- (``data:audio/wav;base64,...``).
+ (``data:audio/wav;base64,...``). For ``minicpm-interleave``, embeds JPEG/WAV segment
+ data URLs instead of the raw MP4.
trust_remote_code: Whether to trust remote code when loading HuggingFace dataset
(online mode only).
"""
@@ -291,6 +355,7 @@ class DailyOmniDataset(BenchmarkDataset):
random_seed: int = 0,
video_dir: str | None = None,
input_mode: DailyOmniInputMode = "all",
+ pack_mode: DailyOmniPackMode = "qwen",
inline_local_video: bool = False,
trust_remote_code: bool = False,
max_duration_seconds: float | None = None,
@@ -300,6 +365,8 @@ class DailyOmniDataset(BenchmarkDataset):
) -> None:
if input_mode not in ("all", "visual", "audio"):
raise ValueError(f"input_mode must be 'all', 'visual', or 'audio', got {input_mode!r}")
+ if pack_mode not in ("qwen", "minicpm-interleave"):
+ raise ValueError(f"pack_mode must be 'qwen' or 'minicpm-interleave', got {pack_mode!r}")
# Validate arguments: need either local JSON or HF path
if qa_json_path is None and dataset_path is None:
@@ -318,11 +385,14 @@ class DailyOmniDataset(BenchmarkDataset):
self.video_dir = Path(video_dir) if video_dir else None
self.inline_local_video = inline_local_video
self.input_mode: DailyOmniInputMode = input_mode
+ self.pack_mode: DailyOmniPackMode = pack_mode
self.max_duration_seconds = max_duration_seconds
self.trust_remote_code = trust_remote_code
#: In-process cache of ffprobe durations only (no disk persistence).
self._video_durations: dict[str, float] = {}
+ #: Cache of MiniCPM 1fps interleaved expansions: video_id -> content parts.
+ self._minicpm_interleave_cache: dict[str, list[dict[str, Any]]] = {}
# Initialize parent BenchmarkDataset
super().__init__(
@@ -336,11 +406,13 @@ class DailyOmniDataset(BenchmarkDataset):
# Verify dataset info
logger.info(
- "Loaded Daily-Omni dataset: mode=%s, source=%s, random_seed=%d, input_mode=%s, max_duration=%s",
+ "Loaded Daily-Omni dataset: mode=%s, source=%s, random_seed=%d, "
+ "input_mode=%s, pack_mode=%s, max_duration=%s",
"local_json" if self.qa_json_path else "huggingface",
str(self.qa_json_path) if self.qa_json_path else f"{dataset_path}/{dataset_split}",
random_seed,
input_mode,
+ pack_mode,
f"{max_duration_seconds}s" if max_duration_seconds else "unlimited",
)
@@ -575,7 +647,8 @@ class DailyOmniDataset(BenchmarkDataset):
messages = self._build_daily_omni_openai_messages(mm_payload, question, choice)
user_text = self._official_daily_omni_user_prompt(question, choice)
# Text-only length estimate (same as before: no MM token count in bench).
- prompt_len = len(tokenizer.encode(f"{DAILY_OMNI_SYSTEM_TEXT}\n{user_text}"))
+ system_text = self._system_text_for_pack_mode()
+ prompt_len = len(tokenizer.encode(f"{system_text}\n{user_text}" if system_text else user_text))
return DailyOmniSampleRequest(
prompt=user_text,
@@ -766,10 +839,11 @@ class DailyOmniDataset(BenchmarkDataset):
video_id: str,
video_url: str | None,
) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None, Literal["first", "last"]]:
- """Build ``multi_modal_data`` and request extras for the active ``input_mode``.
+ """Build multimodal OpenAI content parts and request extras for the active modes."""
+ if self.pack_mode == "minicpm-interleave":
+ return self._compose_minicpm_interleaved_multimodal(video_id, video_url)
- Mirrors upstream Daily-Omni: separate video + WAV with ``use_audio_in_video=False``.
- """
+ # Qwen / upstream Daily-Omni: separate video + WAV with ``use_audio_in_video=False``.
extra: dict[str, Any] = {"mm_processor_kwargs": {"use_audio_in_video": False}}
mode = self.input_mode
@@ -787,6 +861,219 @@ class DailyOmniDataset(BenchmarkDataset):
return None, None, "first"
return [v, a], extra, "first"
+ def _compose_minicpm_interleaved_multimodal(
+ self,
+ video_id: str,
+ video_url: str | None,
+ ) -> tuple[list[dict[str, Any]] | None, dict[str, Any] | None, Literal["first", "last"]]:
+ """Pack media like MiniCPM-o official omni chat: 1fps ``[frame_i, audio_i, ...]``.
+
+ OpenBMB's HF / utils path expands ``video_url`` with ``use_audio=True`` into interleaved
+ PIL frames and 1s audio ndarrays. vLLM-Omni does not implement that expansion, so the
+ bench client builds ordered ``image_url`` / ``audio_url`` parts instead. MiniCPM-o 5.0's
+ OpenAI chat template preserves content-part order, so placeholders stay time-aligned.
+
+ ``max_slice_nums=1`` and ``use_image_id=False`` mirror MiniCPM-o's omni path
+ (``modeling_minicpmo.py`` passes both for interleaved frames): slicing every frame with
+ the config default of 9 would blow past ``max_model_len``, and ``N``
+ prefixes would label each frame as a separate picture instead of a video timeline.
+ """
+ del video_url # Hub HTTP URLs are not used; local extract is required.
+ extra: dict[str, Any] = {
+ "mm_processor_kwargs": {
+ "use_audio_in_video": False,
+ "max_slice_nums": 1,
+ "use_image_id": False,
+ }
+ }
+ mode = self.input_mode
+ if mode == "audio":
+ # Fall back to whole-WAV for audio-only ablations.
+ a = self._get_audio_content(video_id)
+ return ([a] if a else None), extra, "first"
+
+ parts = self._get_minicpm_interleaved_parts(video_id, include_audio=(mode == "all"))
+ if not parts:
+ return None, None, "first"
+ return parts, extra, "first"
+
+ def _get_minicpm_interleaved_parts(
+ self,
+ video_id: str,
+ *,
+ include_audio: bool,
+ ) -> list[dict[str, Any]] | None:
+ """Return cached or freshly extracted MiniCPM-style interleaved OpenAI content parts."""
+ cache_key = f"{video_id}|audio={int(include_audio)}|inline={int(self.inline_local_video)}"
+ cached = self._minicpm_interleave_cache.get(cache_key)
+ if cached is not None:
+ return cached
+
+ self._lazy_ensure_hub_media_dir()
+ video_path = self._resolve_local_video_path(video_id)
+ if video_path is None:
+ logger.warning(
+ "MiniCPM interleave pack requires local video under video_dir=%s for video_id=%r",
+ self.video_dir,
+ video_id,
+ )
+ return None
+ audio_path = self._resolve_local_audio_path(video_id) if include_audio else None
+ if include_audio and audio_path is None:
+ logger.warning(
+ "MiniCPM interleave pack (input_mode=all) missing WAV for video_id=%r under %s",
+ video_id,
+ self.video_dir,
+ )
+ return None
+
+ try:
+ frames, audio_segments = self._extract_minicpm_frame_audio_segments(
+ video_path,
+ audio_path=audio_path,
+ include_audio=include_audio,
+ )
+ except Exception:
+ logger.exception(
+ "Failed MiniCPM 1fps extract for video_id=%r path=%s",
+ video_id,
+ video_path,
+ )
+ return None
+
+ if not frames:
+ logger.warning("No frames extracted for video_id=%r", video_id)
+ return None
+ if include_audio and len(audio_segments) != len(frames):
+ logger.warning(
+ "Frame/audio length mismatch for video_id=%r: frames=%d audios=%d",
+ video_id,
+ len(frames),
+ len(audio_segments),
+ )
+ n = min(len(frames), len(audio_segments))
+ frames = frames[:n]
+ audio_segments = audio_segments[:n]
+
+ cache_root: Path | None = None
+ if not self.inline_local_video:
+ assert self.video_dir is not None
+ cache_root = self.video_dir / ".minicpm_daily_omni_interleave" / video_id
+ cache_root.mkdir(parents=True, exist_ok=True)
+
+ parts: list[dict[str, Any]] = []
+ for i, frame in enumerate(frames):
+ jpeg = _pil_to_jpeg_bytes(frame)
+ if self.inline_local_video:
+ b64 = base64.b64encode(jpeg).decode("ascii")
+ parts.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
+ }
+ )
+ else:
+ assert cache_root is not None
+ frame_path = cache_root / f"frame_{i:04d}.jpg"
+ if not frame_path.is_file() or frame_path.stat().st_size == 0:
+ frame_path.write_bytes(jpeg)
+ parts.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": frame_path.expanduser().resolve().as_uri()},
+ }
+ )
+
+ if include_audio:
+ wav = _numpy_to_wav_bytes(audio_segments[i])
+ if self.inline_local_video:
+ b64 = base64.b64encode(wav).decode("ascii")
+ parts.append(
+ {
+ "type": "audio_url",
+ "audio_url": {"url": f"data:audio/wav;base64,{b64}"},
+ }
+ )
+ else:
+ assert cache_root is not None
+ audio_seg_path = cache_root / f"audio_{i:04d}.wav"
+ if not audio_seg_path.is_file() or audio_seg_path.stat().st_size == 0:
+ audio_seg_path.write_bytes(wav)
+ parts.append(
+ {
+ "type": "audio_url",
+ "audio_url": {"url": audio_seg_path.expanduser().resolve().as_uri()},
+ }
+ )
+
+ # File URLs are tiny; cache them so repeated QA on the same video skips re-extract.
+ # Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS.
+ if not self.inline_local_video:
+ self._minicpm_interleave_cache[cache_key] = parts
+ logger.debug(
+ "MiniCPM interleave packed video_id=%r frames=%d include_audio=%s parts=%d",
+ video_id,
+ len(frames),
+ include_audio,
+ len(parts),
+ )
+ return parts
+
+ @staticmethod
+ def _extract_minicpm_frame_audio_segments(
+ video_path: Path,
+ *,
+ audio_path: Path | None,
+ include_audio: bool,
+ max_num_frames: int = _MINICPM_MAX_NUM_FRAMES,
+ ) -> tuple[list[Any], list[Any]]:
+ """Port of MiniCPM ``get_video_frame_audio_segments`` (stack_frames=1, 1fps)."""
+ import numpy as np
+ from decord import VideoReader, cpu
+ from PIL import Image
+ from vllm.multimodal.media.audio import load_audio
+
+ vr = VideoReader(str(video_path), ctx=cpu(0))
+ avg_fps = float(vr.get_avg_fps()) or 1.0
+ duration = len(vr) / avg_fps
+ num_seconds = max(1, int(math.ceil(duration)))
+ second_timestamps = list(range(num_seconds))
+
+ if duration > max_num_frames:
+ timestamps = [round(i * 0.1, 1) for i in range(int(duration / 0.1))]
+ frame_idx = [min(int(ts * avg_fps), len(vr) - 1) for ts in timestamps]
+ pick = _uniform_sample_indices(len(frame_idx), max_num_frames)
+ frame_idx = [frame_idx[i] for i in pick]
+ timestamps = [timestamps[i] for i in pick]
+ else:
+ frame_idx = [min(int(i * avg_fps), len(vr) - 1) for i in range(num_seconds)]
+ timestamps = second_timestamps
+
+ video = vr.get_batch(frame_idx).asnumpy()
+ frames = [Image.fromarray(v.astype("uint8")).convert("RGB") for v in video]
+
+ audio_segments: list[Any] = []
+ if include_audio:
+ load_path = str(audio_path) if audio_path is not None else str(video_path)
+ audio_np, sr = load_audio(load_path, sr=_MINICPM_AUDIO_SR, mono=True)
+ for i, start_time in enumerate(timestamps):
+ end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration
+ start_sample = int(start_time * sr)
+ end_sample = int(end_time * sr)
+ segment = audio_np[start_sample:end_sample]
+ if i == len(timestamps) - 1 and len(segment) < 1600:
+ segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)])
+ if len(segment) == 0:
+ segment = np.zeros(1600, dtype=np.float32)
+ audio_segments.append(segment)
+
+ return frames, audio_segments
+
+ def _system_text_for_pack_mode(self) -> str:
+ if self.pack_mode == "minicpm-interleave":
+ return MINICPM_OMNI_SYSTEM_TEXT
+ return DAILY_OMNI_SYSTEM_TEXT
+
@staticmethod
def _media_desc_for_official_prompt(mode: DailyOmniInputMode) -> str:
"""``media_desc`` in upstream ``build_conversation``."""
@@ -835,14 +1122,16 @@ class DailyOmniDataset(BenchmarkDataset):
question: str,
choice: Any,
) -> list[dict[str, Any]]:
- """Map upstream conversation to OpenAI Chat Completions ``messages`` (video_url / audio_url parts)."""
+ """Map upstream conversation to OpenAI Chat Completions ``messages``."""
user_text = self._official_daily_omni_user_prompt(question, choice)
mm_list: list[dict[str, Any]] = mm_payload if isinstance(mm_payload, list) else [mm_payload]
user_content: list[dict[str, Any]] = [*mm_list, {"type": "text", "text": user_text}]
- return [
- {"role": "system", "content": [{"type": "text", "text": DAILY_OMNI_SYSTEM_TEXT}]},
- {"role": "user", "content": user_content},
- ]
+ system_text = self._system_text_for_pack_mode()
+ messages: list[dict[str, Any]] = []
+ if system_text:
+ messages.append({"role": "system", "content": [{"type": "text", "text": system_text}]})
+ messages.append({"role": "user", "content": user_content})
+ return messages
def sample_by_task_type(
self,
@@ -900,6 +1189,7 @@ class DailyOmniDataset(BenchmarkDataset):
f"dataset_split={self.dataset_split!r}, "
f"video_dir={self.video_dir!r}, "
f"input_mode={self.input_mode!r}, "
+ f"pack_mode={self.pack_mode!r}, "
f"inline_local_video={self.inline_local_video!r}, "
f"max_duration_seconds={self.max_duration_seconds}, "
f"random_seed={self.random_seed}"
diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py
index 00af6ee23..2d816d718 100644
--- a/vllm_omni/benchmarks/patch/patch.py
+++ b/vllm_omni/benchmarks/patch/patch.py
@@ -280,6 +280,7 @@ def get_samples(args, tokenizer):
random_seed=args.seed,
video_dir=video_dir,
input_mode=getattr(args, "daily_omni_input_mode", "all"),
+ pack_mode=getattr(args, "daily_omni_pack_mode", "qwen"),
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
disable_shuffle=getattr(args, "disable_shuffle", False),
@@ -310,6 +311,7 @@ def get_samples(args, tokenizer):
random_seed=args.seed,
video_dir=video_dir,
input_mode=getattr(args, "daily_omni_input_mode", "all"),
+ pack_mode=getattr(args, "daily_omni_pack_mode", "qwen"),
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
no_stream=getattr(args, "no_stream", False),
diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml
index 7e74c4081..36ab6144c 100644
--- a/vllm_omni/deploy/minicpmo_4_5.yaml
+++ b/vllm_omni/deploy/minicpmo_4_5.yaml
@@ -25,9 +25,10 @@ stages:
max_model_len: 32768
devices: "0"
limit_mm_per_prompt:
- video:
- count: 1
- num_frames: 32
+ # Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
+ image: 64
+ audio: 64
+ video: 1
default_sampling_params:
temperature: 0.0
top_p: 1.0
@@ -35,7 +36,8 @@ stages:
max_tokens: 2048
seed: 42
detokenize: true
- repetition_penalty: 1.1
+ # 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
+ repetition_penalty: 1.0
- stage_id: 1
max_num_seqs: 16
diff --git a/vllm_omni/entrypoints/cli/benchmark/cli_args.py b/vllm_omni/entrypoints/cli/benchmark/cli_args.py
index 7f7671496..a68651140 100644
--- a/vllm_omni/entrypoints/cli/benchmark/cli_args.py
+++ b/vllm_omni/entrypoints/cli/benchmark/cli_args.py
@@ -95,6 +95,18 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
"'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false "
"and matches official separate video/audio streams.",
)
+ group.add_argument(
+ "--daily-omni-pack-mode",
+ type=str,
+ choices=["qwen", "minicpm-interleave"],
+ default="qwen",
+ help="How to pack multimodal parts into OpenAI chat messages. "
+ "'qwen' (default): one video_url + one audio_url (Daily-Omni/Qwen protocol). "
+ "'minicpm-interleave': MiniCPM-o official recipe — 1fps frames interleaved with "
+ "matching 1s audio segments as image_url/audio_url pairs (needed to approach "
+ "OpenBMB ~80%% Daily-Omni; requires local Videos extract). For MiniCPM-o 4.5 "
+ "string chat templates also start the server with --interleave-mm-strings.",
+ )
group.add_argument(
"--daily-omni-save-eval-items",
action="store_true",
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py
index 6f6841db3..a01e97f8f 100644
--- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py
@@ -3422,10 +3422,23 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
) -> bool:
return False
- def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
+ def get_image_prompt_texts(
+ self,
+ image_size: ImageSize,
+ image_idx: int = 0,
+ max_slice_nums: int | None = None,
+ use_image_id: bool | None = None,
+ ) -> str:
+ # ``process_images`` forwards ``mm_processor_kwargs`` to the pixel path, so the
+ # placeholder grid must honor the same ``max_slice_nums``; otherwise a request that
+ # sets it (e.g. interleaved omni frames with max_slice_nums=1) produces fewer image
+ # features than placeholder ```` slots. ``use_image_id`` is likewise per-request
+ # so callers can match MiniCPM-o's omni path, which emits no ```` prefix.
return self.info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
+ max_slice_nums=max_slice_nums,
+ **({} if use_image_id is None else {"use_image_id": use_image_id}),
)
def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str:
@@ -3615,13 +3628,21 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
additional_placeholders.append((modality, sub_pattern))
placeholders += additional_placeholders
+ image_max_slice_nums = hf_processor_mm_kwargs.get("max_slice_nums")
+ image_use_image_id = hf_processor_mm_kwargs.get("use_image_id")
+
def get_image_replacement(item_idx: int):
images = mm_items.get_items("image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))
image_size = images.get_image_size(item_idx)
return PromptUpdateDetails.select_text(
- self.get_image_prompt_texts(image_size, item_idx),
+ self.get_image_prompt_texts(
+ image_size,
+ item_idx,
+ max_slice_nums=None if image_max_slice_nums is None else int(image_max_slice_nums), # type: ignore[arg-type]
+ use_image_id=None if image_use_image_id is None else bool(image_use_image_id),
+ ),
"",
)