[Bugfix][MiniCPM-o] Align Daily-Omni bench with MiniCPM interleaved AV recipe (#5606)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/benchmarks/test_daily_omni_pack_modes.py

@@ -0,0 +1,237 @@
+"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
+
+The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
+OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
+interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and no Qwen system prompt.
+These tests pin that contract so a refactor cannot silently regress the score.
+
+vllm stubs are installed by tests/benchmarks/conftest.py before collection.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
+_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
+
+if _MODULE_NAME not in sys.modules:
+    _spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
+    _mod = importlib.util.module_from_spec(_spec)
+    sys.modules[_MODULE_NAME] = _mod
+    _spec.loader.exec_module(_mod)
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (  # noqa: E402
+    DailyOmniDataset,
+    _uniform_sample_indices,
+)
+
+_VIDEO_ID = "vid001"
+_NUM_FRAMES = 5
+
+
+@pytest.fixture
+def video_dir(tmp_path: Path) -> Path:
+    """Official Daily-Omni ``Videos/`` layout with placeholder media files."""
+    per_video = tmp_path / "Videos" / _VIDEO_ID
+    per_video.mkdir(parents=True)
+    (per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
+    (per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
+    return tmp_path / "Videos"
+
+
+@pytest.fixture
+def qa_json(tmp_path: Path) -> Path:
+    path = tmp_path / "qa.json"
+    path.write_text(
+        json.dumps(
+            [
+                {
+                    "video_id": _VIDEO_ID,
+                    "question": "What happens first?",
+                    "choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
+                    "answer": "A",
+                    "task_type": "AV Event Alignment",
+                    "video_duration": "30s",
+                }
+            ]
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
+    return DailyOmniDataset(
+        qa_json_path=str(qa_json),
+        video_dir=str(video_dir),
+        disable_shuffle=True,
+        **kwargs,
+    )
+
+
+def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
+    """Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
+    from PIL import Image
+
+    def fake_extract(video_path, *, audio_path, include_audio, **_kw):
+        counter.append(1)
+        frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
+        segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
+        return frames, segments
+
+    monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
+
+
+# ---------------------------------------------------------------------------
+# _uniform_sample_indices
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (7, 7), (3, 5), (1, 1)])
+def test_uniform_sample_indices_matches_numpy_linspace(n: int, k: int) -> None:
+    got = _uniform_sample_indices(n, k)
+    if n <= k:
+        assert got == list(range(n))
+    else:
+        assert got == [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
+
+
+def test_uniform_sample_indices_non_positive_k() -> None:
+    assert _uniform_sample_indices(10, 0) == []
+    assert _uniform_sample_indices(10, -1) == []
+
+
+# ---------------------------------------------------------------------------
+# minicpm-interleave packing
+# ---------------------------------------------------------------------------
+
+
+def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
+    ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+    _stub_extract(monkeypatch, ds, [])
+
+    parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    assert position == "first"
+    assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
+    assert extra["mm_processor_kwargs"] == {
+        "use_audio_in_video": False,
+        "max_slice_nums": 1,
+        "use_image_id": False,
+    }
+
+
+def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
+    ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
+    _stub_extract(monkeypatch, ds, [])
+
+    parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
+
+
+def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
+    ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+    _stub_extract(monkeypatch, ds, [])
+
+    parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
+    for part in parts:
+        url = part[part["type"]]["url"]
+        assert url.startswith("file://")
+        assert Path(url[len("file://") :]).parent == segment_dir.resolve()
+    assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
+
+
+def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
+    ds = _make_dataset(
+        qa_json,
+        video_dir,
+        pack_mode="minicpm-interleave",
+        input_mode="all",
+        inline_local_video=True,
+    )
+    _stub_extract(monkeypatch, ds, [])
+
+    parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
+    assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
+    assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
+
+
+def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
+    """Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
+    file_calls: list[int] = []
+    ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+    _stub_extract(monkeypatch, ds_file, file_calls)
+    ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
+    ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
+    assert len(file_calls) == 1
+    assert ds_file._minicpm_interleave_cache
+
+    inline_calls: list[int] = []
+    ds_inline = _make_dataset(
+        qa_json,
+        video_dir,
+        pack_mode="minicpm-interleave",
+        input_mode="all",
+        inline_local_video=True,
+    )
+    _stub_extract(monkeypatch, ds_inline, inline_calls)
+    ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
+    ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
+    assert len(inline_calls) == 2
+    assert not ds_inline._minicpm_interleave_cache
+
+
+def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
+    empty = tmp_path / "empty_videos"
+    empty.mkdir()
+    ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
+
+    parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    assert parts is None
+    assert extra is None
+
+
+# ---------------------------------------------------------------------------
+# System prompt selection
+# ---------------------------------------------------------------------------
+
+
+def test_minicpm_interleave_omits_qwen_system_message(monkeypatch, qa_json, video_dir) -> None:
+    ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
+    _stub_extract(monkeypatch, ds, [])
+    parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
+
+    messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
+
+    assert [m["role"] for m in messages] == ["user"]
+
+
+def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
+    ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
+
+    messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
+
+    assert [m["role"] for m in messages] == ["system", "user"]
+    assert "Qwen" in messages[0]["content"][0]["text"]
+
+
+def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
+    with pytest.raises(ValueError, match="pack_mode must be"):
+        _make_dataset(qa_json, video_dir, pack_mode="bogus")

tests/model_executor/models/minicpmo_4_5/test_pipeline.py

@@ -24,8 +24,6 @@ from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
 from vllm_omni.config.stage_config import (
     PipelineConfig,
     StageExecutionType,
-    load_deploy_config,
-    merge_pipeline_deploy,
 )
 from vllm_omni.model_executor.models.registry import _OMNI_MODELS
 
@@ -138,45 +136,6 @@ class TestPipelineTopology:
 
 
 class TestDeployTopology:
-    @pytest.mark.parametrize(("filename", "devices"), _DEPLOY_LAYOUTS.items())
-    def test_deploy_resolves_three_stage_pipeline(self, filename: str, devices: list[str]) -> None:
-        deploy = load_deploy_config(_DEPLOY_DIR / filename)
-        pipeline = OMNI_PIPELINES[deploy.pipeline]
-        stages = merge_pipeline_deploy(pipeline, deploy)
-
-        assert deploy.pipeline == _PIPELINE_KEY
-        assert deploy.async_chunk is True
-        assert [stage.stage_id for stage in stages] == [0, 1, 2]
-        assert [stage.yaml_runtime["devices"] for stage in stages] == devices
-        assert "additional_config" not in stages[1].yaml_engine_args
-        assert "skip_mm_profiling" not in stages[0].yaml_engine_args
-        assert stages[1].yaml_engine_args["skip_mm_profiling"] is True
-        assert stages[2].yaml_engine_args["skip_mm_profiling"] is True
-        assert stages[1].yaml_extras["output_connectors"]["to_stage_2"] == "connector_of_shared_memory"
-        assert stages[2].yaml_extras["input_connectors"]["from_stage_1"] == "connector_of_shared_memory"
-        connector = deploy.connectors["connector_of_shared_memory"]
-        assert connector["name"] == "SharedMemoryConnector"
-        assert connector["extra"]["codec_chunk_frames"] == 25
-        assert connector["extra"]["codec_left_context_frames"] == 3
-        assert connector["extra"]["connector_get_max_wait_first_chunk"] == 3000
-        assert connector["extra"]["connector_get_max_wait"] == 300
-        expected_processor = "tts2code2wav_async_chunk" if deploy.async_chunk else "tts2code2wav_full_payload"
-        assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor)
-        if filename == "minicpmo_4_5.yaml":
-            assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [16, 16, 16]
-            assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
-                0.6,
-                0.1,
-                0.1,
-            ]
-            assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}}
-        elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}:
-            assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
-                0.9,
-                0.55,
-                0.35,
-            ]
-
     def test_pipeline_exposes_full_and_async_payload_hooks(self) -> None:
         pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
         talker = pipeline.get_stage(1)

vllm_omni/benchmarks/data_modules/daily_omni_dataset.py

@@ -52,11 +52,14 @@ Usage:
 """
 
 import base64
+import io
 import json
 import logging
+import math
 import os
 import shutil
 import tarfile
+import wave
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Literal
@@ -77,6 +80,10 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
+# Official MiniCPM ``get_video_frame_audio_segments`` default (env ``MAX_NUM_FRAMES``).
+_MINICPM_MAX_NUM_FRAMES = int(os.environ.get("MAX_NUM_FRAMES", "64"))
+_MINICPM_AUDIO_SR = 16000
+
 
 def _daily_omni_hf_cache_root() -> Path:
     return Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")).expanduser().resolve()
@@ -203,12 +210,63 @@ class _ListDatasetIterator:
 # Aligns with Lliar-liar/Daily-Omni CLI ``--input_mode`` (test_model/*/testmodel.py).
 DailyOmniInputMode = Literal["all", "visual", "audio"]
 
+# How multimodal parts are packed into OpenAI chat messages:
+# - ``qwen``: one ``video_url`` + one ``audio_url`` (upstream Daily-Omni / Qwen protocol).
+# - ``minicpm-interleave``: 1fps ``[image_i, audio_i, ...]`` matching OpenBMB MiniCPM-o
+#   ``get_video_frame_audio_segments`` + interleaved omni contents (paper ~80% recipe).
+DailyOmniPackMode = Literal["qwen", "minicpm-interleave"]
+
 # ``build_conversation()`` in Daily-Omni ``test_model/Qwen2.5-Omni/testmodel.py`` (verbatim).
 DAILY_OMNI_SYSTEM_TEXT = (
     "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
     "capable of perceiving auditory and visual inputs, as well as generating text and speech."
 )
 
+# MiniCPM-o ``get_sys_prompt(mode="omni")`` without ref-audio is an empty system string.
+MINICPM_OMNI_SYSTEM_TEXT = ""
+
+
+def _uniform_sample_indices(n: int, k: int) -> list[int]:
+    """Evenly pick ``k`` indices from ``0..n-1`` (MiniCPM ``uniform_sample`` on index lists)."""
+    if k <= 0:
+        return []
+    if n <= k:
+        return list(range(n))
+    # Match numpy linspace(..., dtype=int) used upstream.
+    try:
+        import numpy as np
+
+        return [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
+    except ImportError:
+        if k == 1:
+            return [0]
+        return [int(round(i * (n - 1) / (k - 1))) for i in range(k)]
+
+
+def _numpy_to_wav_bytes(audio_np: Any, sample_rate: int = _MINICPM_AUDIO_SR) -> bytes:
+    """Encode a mono float/int numpy waveform as 16-bit PCM WAV bytes."""
+    import numpy as np
+
+    arr = np.asarray(audio_np)
+    if arr.ndim > 1:
+        arr = arr.reshape(-1)
+    if arr.dtype != np.int16:
+        arr = np.clip(arr.astype(np.float64), -1.0, 1.0)
+        arr = (arr * 32767.0).astype(np.int16)
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(arr.tobytes())
+    return buf.getvalue()
+
+
+def _pil_to_jpeg_bytes(image: Any, quality: int = 85) -> bytes:
+    buf = io.BytesIO()
+    image.save(buf, format="JPEG", quality=quality)
+    return buf.getvalue()
+
 
 @dataclass
 class DailyOmniSampleRequest(SampleRequest):
@@ -262,6 +320,11 @@ class DailyOmniDataset(BenchmarkDataset):
             ``all`` — video + WAV (default; official audio-visual protocol);
             ``visual`` — video only;
             ``audio`` — extracted WAV only (requires ``{video_id}/{video_id}_audio.wav`` under ``video_dir``).
+        pack_mode: Multimodal packing recipe:
+            ``qwen`` — one ``video_url`` + optional ``audio_url`` (default; Daily-Omni/Qwen);
+            ``minicpm-interleave`` — MiniCPM-o paper recipe: 1fps frames interleaved with
+            matching 1s audio segments as ``image_url``/``audio_url`` pairs (required to approach
+            OpenBMB ~80% Daily-Omni; needs local video+WAV under ``video_dir``).
         max_duration_seconds: Reserved for future ffprobe-based filtering; currently **not applied**
             when building requests (metadata ``video_duration`` is still passed through for eval).
         dataset_subset: Optional HuggingFace subset name (``load_dataset(..., name=...)``); used by bench
@@ -270,7 +333,8 @@ class DailyOmniDataset(BenchmarkDataset):
         inline_local_video: If True, embed local MP4 as ``data:video/mp4;base64,...`` in requests so
             the API server does not need ``--allowed-local-media-path`` (large JSON; use for small runs).
             When ``input_mode`` is ``audio`` or ``all``, local WAV is embedded the same way
-            (``data:audio/wav;base64,...``).
+            (``data:audio/wav;base64,...``). For ``minicpm-interleave``, embeds JPEG/WAV segment
+            data URLs instead of the raw MP4.
         trust_remote_code: Whether to trust remote code when loading HuggingFace dataset
             (online mode only).
     """
@@ -291,6 +355,7 @@ class DailyOmniDataset(BenchmarkDataset):
         random_seed: int = 0,
         video_dir: str | None = None,
         input_mode: DailyOmniInputMode = "all",
+        pack_mode: DailyOmniPackMode = "qwen",
         inline_local_video: bool = False,
         trust_remote_code: bool = False,
         max_duration_seconds: float | None = None,
@@ -300,6 +365,8 @@ class DailyOmniDataset(BenchmarkDataset):
     ) -> None:
         if input_mode not in ("all", "visual", "audio"):
             raise ValueError(f"input_mode must be 'all', 'visual', or 'audio', got {input_mode!r}")
+        if pack_mode not in ("qwen", "minicpm-interleave"):
+            raise ValueError(f"pack_mode must be 'qwen' or 'minicpm-interleave', got {pack_mode!r}")
 
         # Validate arguments: need either local JSON or HF path
         if qa_json_path is None and dataset_path is None:
@@ -318,11 +385,14 @@ class DailyOmniDataset(BenchmarkDataset):
         self.video_dir = Path(video_dir) if video_dir else None
         self.inline_local_video = inline_local_video
         self.input_mode: DailyOmniInputMode = input_mode
+        self.pack_mode: DailyOmniPackMode = pack_mode
         self.max_duration_seconds = max_duration_seconds
         self.trust_remote_code = trust_remote_code
 
         #: In-process cache of ffprobe durations only (no disk persistence).
         self._video_durations: dict[str, float] = {}
+        #: Cache of MiniCPM 1fps interleaved expansions: video_id -> content parts.
+        self._minicpm_interleave_cache: dict[str, list[dict[str, Any]]] = {}
 
         # Initialize parent BenchmarkDataset
         super().__init__(
@@ -336,11 +406,13 @@ class DailyOmniDataset(BenchmarkDataset):
 
         # Verify dataset info
         logger.info(
-            "Loaded Daily-Omni dataset: mode=%s, source=%s, random_seed=%d, input_mode=%s, max_duration=%s",
+            "Loaded Daily-Omni dataset: mode=%s, source=%s, random_seed=%d, "
+            "input_mode=%s, pack_mode=%s, max_duration=%s",
             "local_json" if self.qa_json_path else "huggingface",
             str(self.qa_json_path) if self.qa_json_path else f"{dataset_path}/{dataset_split}",
             random_seed,
             input_mode,
+            pack_mode,
             f"{max_duration_seconds}s" if max_duration_seconds else "unlimited",
         )
 
@@ -575,7 +647,8 @@ class DailyOmniDataset(BenchmarkDataset):
         messages = self._build_daily_omni_openai_messages(mm_payload, question, choice)
         user_text = self._official_daily_omni_user_prompt(question, choice)
         # Text-only length estimate (same as before: no MM token count in bench).
-        prompt_len = len(tokenizer.encode(f"{DAILY_OMNI_SYSTEM_TEXT}\n{user_text}"))
+        system_text = self._system_text_for_pack_mode()
+        prompt_len = len(tokenizer.encode(f"{system_text}\n{user_text}" if system_text else user_text))
 
         return DailyOmniSampleRequest(
             prompt=user_text,
@@ -766,10 +839,11 @@ class DailyOmniDataset(BenchmarkDataset):
         video_id: str,
         video_url: str | None,
     ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None, Literal["first", "last"]]:
-        """Build ``multi_modal_data`` and request extras for the active ``input_mode``.
+        """Build multimodal OpenAI content parts and request extras for the active modes."""
+        if self.pack_mode == "minicpm-interleave":
+            return self._compose_minicpm_interleaved_multimodal(video_id, video_url)
 
-        Mirrors upstream Daily-Omni: separate video + WAV with ``use_audio_in_video=False``.
-        """
+        # Qwen / upstream Daily-Omni: separate video + WAV with ``use_audio_in_video=False``.
         extra: dict[str, Any] = {"mm_processor_kwargs": {"use_audio_in_video": False}}
         mode = self.input_mode
 
@@ -787,6 +861,219 @@ class DailyOmniDataset(BenchmarkDataset):
             return None, None, "first"
         return [v, a], extra, "first"
 
+    def _compose_minicpm_interleaved_multimodal(
+        self,
+        video_id: str,
+        video_url: str | None,
+    ) -> tuple[list[dict[str, Any]] | None, dict[str, Any] | None, Literal["first", "last"]]:
+        """Pack media like MiniCPM-o official omni chat: 1fps ``[frame_i, audio_i, ...]``.
+
+        OpenBMB's HF / utils path expands ``video_url`` with ``use_audio=True`` into interleaved
+        PIL frames and 1s audio ndarrays. vLLM-Omni does not implement that expansion, so the
+        bench client builds ordered ``image_url`` / ``audio_url`` parts instead. MiniCPM-o 5.0's
+        OpenAI chat template preserves content-part order, so placeholders stay time-aligned.
+
+        ``max_slice_nums=1`` and ``use_image_id=False`` mirror MiniCPM-o's omni path
+        (``modeling_minicpmo.py`` passes both for interleaved frames): slicing every frame with
+        the config default of 9 would blow past ``max_model_len``, and ``<image_id>N</image_id>``
+        prefixes would label each frame as a separate picture instead of a video timeline.
+        """
+        del video_url  # Hub HTTP URLs are not used; local extract is required.
+        extra: dict[str, Any] = {
+            "mm_processor_kwargs": {
+                "use_audio_in_video": False,
+                "max_slice_nums": 1,
+                "use_image_id": False,
+            }
+        }
+        mode = self.input_mode
+        if mode == "audio":
+            # Fall back to whole-WAV for audio-only ablations.
+            a = self._get_audio_content(video_id)
+            return ([a] if a else None), extra, "first"
+
+        parts = self._get_minicpm_interleaved_parts(video_id, include_audio=(mode == "all"))
+        if not parts:
+            return None, None, "first"
+        return parts, extra, "first"
+
+    def _get_minicpm_interleaved_parts(
+        self,
+        video_id: str,
+        *,
+        include_audio: bool,
+    ) -> list[dict[str, Any]] | None:
+        """Return cached or freshly extracted MiniCPM-style interleaved OpenAI content parts."""
+        cache_key = f"{video_id}|audio={int(include_audio)}|inline={int(self.inline_local_video)}"
+        cached = self._minicpm_interleave_cache.get(cache_key)
+        if cached is not None:
+            return cached
+
+        self._lazy_ensure_hub_media_dir()
+        video_path = self._resolve_local_video_path(video_id)
+        if video_path is None:
+            logger.warning(
+                "MiniCPM interleave pack requires local video under video_dir=%s for video_id=%r",
+                self.video_dir,
+                video_id,
+            )
+            return None
+        audio_path = self._resolve_local_audio_path(video_id) if include_audio else None
+        if include_audio and audio_path is None:
+            logger.warning(
+                "MiniCPM interleave pack (input_mode=all) missing WAV for video_id=%r under %s",
+                video_id,
+                self.video_dir,
+            )
+            return None
+
+        try:
+            frames, audio_segments = self._extract_minicpm_frame_audio_segments(
+                video_path,
+                audio_path=audio_path,
+                include_audio=include_audio,
+            )
+        except Exception:
+            logger.exception(
+                "Failed MiniCPM 1fps extract for video_id=%r path=%s",
+                video_id,
+                video_path,
+            )
+            return None
+
+        if not frames:
+            logger.warning("No frames extracted for video_id=%r", video_id)
+            return None
+        if include_audio and len(audio_segments) != len(frames):
+            logger.warning(
+                "Frame/audio length mismatch for video_id=%r: frames=%d audios=%d",
+                video_id,
+                len(frames),
+                len(audio_segments),
+            )
+            n = min(len(frames), len(audio_segments))
+            frames = frames[:n]
+            audio_segments = audio_segments[:n]
+
+        cache_root: Path | None = None
+        if not self.inline_local_video:
+            assert self.video_dir is not None
+            cache_root = self.video_dir / ".minicpm_daily_omni_interleave" / video_id
+            cache_root.mkdir(parents=True, exist_ok=True)
+
+        parts: list[dict[str, Any]] = []
+        for i, frame in enumerate(frames):
+            jpeg = _pil_to_jpeg_bytes(frame)
+            if self.inline_local_video:
+                b64 = base64.b64encode(jpeg).decode("ascii")
+                parts.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
+                    }
+                )
+            else:
+                assert cache_root is not None
+                frame_path = cache_root / f"frame_{i:04d}.jpg"
+                if not frame_path.is_file() or frame_path.stat().st_size == 0:
+                    frame_path.write_bytes(jpeg)
+                parts.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": frame_path.expanduser().resolve().as_uri()},
+                    }
+                )
+
+            if include_audio:
+                wav = _numpy_to_wav_bytes(audio_segments[i])
+                if self.inline_local_video:
+                    b64 = base64.b64encode(wav).decode("ascii")
+                    parts.append(
+                        {
+                            "type": "audio_url",
+                            "audio_url": {"url": f"data:audio/wav;base64,{b64}"},
+                        }
+                    )
+                else:
+                    assert cache_root is not None
+                    audio_seg_path = cache_root / f"audio_{i:04d}.wav"
+                    if not audio_seg_path.is_file() or audio_seg_path.stat().st_size == 0:
+                        audio_seg_path.write_bytes(wav)
+                    parts.append(
+                        {
+                            "type": "audio_url",
+                            "audio_url": {"url": audio_seg_path.expanduser().resolve().as_uri()},
+                        }
+                    )
+
+        # File URLs are tiny; cache them so repeated QA on the same video skips re-extract.
+        # Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS.
+        if not self.inline_local_video:
+            self._minicpm_interleave_cache[cache_key] = parts
+        logger.debug(
+            "MiniCPM interleave packed video_id=%r frames=%d include_audio=%s parts=%d",
+            video_id,
+            len(frames),
+            include_audio,
+            len(parts),
+        )
+        return parts
+
+    @staticmethod
+    def _extract_minicpm_frame_audio_segments(
+        video_path: Path,
+        *,
+        audio_path: Path | None,
+        include_audio: bool,
+        max_num_frames: int = _MINICPM_MAX_NUM_FRAMES,
+    ) -> tuple[list[Any], list[Any]]:
+        """Port of MiniCPM ``get_video_frame_audio_segments`` (stack_frames=1, 1fps)."""
+        import numpy as np
+        from decord import VideoReader, cpu
+        from PIL import Image
+        from vllm.multimodal.media.audio import load_audio
+
+        vr = VideoReader(str(video_path), ctx=cpu(0))
+        avg_fps = float(vr.get_avg_fps()) or 1.0
+        duration = len(vr) / avg_fps
+        num_seconds = max(1, int(math.ceil(duration)))
+        second_timestamps = list(range(num_seconds))
+
+        if duration > max_num_frames:
+            timestamps = [round(i * 0.1, 1) for i in range(int(duration / 0.1))]
+            frame_idx = [min(int(ts * avg_fps), len(vr) - 1) for ts in timestamps]
+            pick = _uniform_sample_indices(len(frame_idx), max_num_frames)
+            frame_idx = [frame_idx[i] for i in pick]
+            timestamps = [timestamps[i] for i in pick]
+        else:
+            frame_idx = [min(int(i * avg_fps), len(vr) - 1) for i in range(num_seconds)]
+            timestamps = second_timestamps
+
+        video = vr.get_batch(frame_idx).asnumpy()
+        frames = [Image.fromarray(v.astype("uint8")).convert("RGB") for v in video]
+
+        audio_segments: list[Any] = []
+        if include_audio:
+            load_path = str(audio_path) if audio_path is not None else str(video_path)
+            audio_np, sr = load_audio(load_path, sr=_MINICPM_AUDIO_SR, mono=True)
+            for i, start_time in enumerate(timestamps):
+                end_time = timestamps[i + 1] if i < len(timestamps) - 1 else duration
+                start_sample = int(start_time * sr)
+                end_sample = int(end_time * sr)
+                segment = audio_np[start_sample:end_sample]
+                if i == len(timestamps) - 1 and len(segment) < 1600:
+                    segment = np.concatenate([segment, np.zeros(1600 - len(segment), dtype=segment.dtype)])
+                if len(segment) == 0:
+                    segment = np.zeros(1600, dtype=np.float32)
+                audio_segments.append(segment)
+
+        return frames, audio_segments
+
+    def _system_text_for_pack_mode(self) -> str:
+        if self.pack_mode == "minicpm-interleave":
+            return MINICPM_OMNI_SYSTEM_TEXT
+        return DAILY_OMNI_SYSTEM_TEXT
+
     @staticmethod
     def _media_desc_for_official_prompt(mode: DailyOmniInputMode) -> str:
         """``media_desc`` in upstream ``build_conversation``."""
@@ -835,14 +1122,16 @@ class DailyOmniDataset(BenchmarkDataset):
         question: str,
         choice: Any,
     ) -> list[dict[str, Any]]:
-        """Map upstream conversation to OpenAI Chat Completions ``messages`` (video_url / audio_url parts)."""
+        """Map upstream conversation to OpenAI Chat Completions ``messages``."""
         user_text = self._official_daily_omni_user_prompt(question, choice)
         mm_list: list[dict[str, Any]] = mm_payload if isinstance(mm_payload, list) else [mm_payload]
         user_content: list[dict[str, Any]] = [*mm_list, {"type": "text", "text": user_text}]
-        return [
-            {"role": "system", "content": [{"type": "text", "text": DAILY_OMNI_SYSTEM_TEXT}]},
-            {"role": "user", "content": user_content},
-        ]
+        system_text = self._system_text_for_pack_mode()
+        messages: list[dict[str, Any]] = []
+        if system_text:
+            messages.append({"role": "system", "content": [{"type": "text", "text": system_text}]})
+        messages.append({"role": "user", "content": user_content})
+        return messages
 
     def sample_by_task_type(
         self,
@@ -900,6 +1189,7 @@ class DailyOmniDataset(BenchmarkDataset):
             f"dataset_split={self.dataset_split!r}, "
             f"video_dir={self.video_dir!r}, "
             f"input_mode={self.input_mode!r}, "
+            f"pack_mode={self.pack_mode!r}, "
             f"inline_local_video={self.inline_local_video!r}, "
             f"max_duration_seconds={self.max_duration_seconds}, "
             f"random_seed={self.random_seed}"

vllm_omni/benchmarks/patch/patch.py

@@ -280,6 +280,7 @@ def get_samples(args, tokenizer):
                 random_seed=args.seed,
                 video_dir=video_dir,
                 input_mode=getattr(args, "daily_omni_input_mode", "all"),
+                pack_mode=getattr(args, "daily_omni_pack_mode", "qwen"),
                 inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
                 trust_remote_code=getattr(args, "trust_remote_code", False),
                 disable_shuffle=getattr(args, "disable_shuffle", False),
@@ -310,6 +311,7 @@ def get_samples(args, tokenizer):
                 random_seed=args.seed,
                 video_dir=video_dir,
                 input_mode=getattr(args, "daily_omni_input_mode", "all"),
+                pack_mode=getattr(args, "daily_omni_pack_mode", "qwen"),
                 inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
                 trust_remote_code=getattr(args, "trust_remote_code", False),
                 no_stream=getattr(args, "no_stream", False),

vllm_omni/deploy/minicpmo_4_5.yaml

@@ -25,9 +25,10 @@ stages:
     max_model_len: 32768
     devices: "0"
     limit_mm_per_prompt:
-      video:
-        count: 1
-        num_frames: 32
+      # Daily-Omni MiniCPM pack: up to 64 interleaved image/audio pairs (1fps).
+      image: 64
+      audio: 64
+      video: 1
     default_sampling_params:
       temperature: 0.0
       top_p: 1.0
@@ -35,7 +36,8 @@ stages:
       max_tokens: 2048
       seed: 42
       detokenize: true
-      repetition_penalty: 1.1
+      # 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
+      repetition_penalty: 1.0
 
   - stage_id: 1
     max_num_seqs: 16

vllm_omni/entrypoints/cli/benchmark/cli_args.py

@@ -95,6 +95,18 @@ def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
         "'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false "
         "and matches official separate video/audio streams.",
     )
+    group.add_argument(
+        "--daily-omni-pack-mode",
+        type=str,
+        choices=["qwen", "minicpm-interleave"],
+        default="qwen",
+        help="How to pack multimodal parts into OpenAI chat messages. "
+        "'qwen' (default): one video_url + one audio_url (Daily-Omni/Qwen protocol). "
+        "'minicpm-interleave': MiniCPM-o official recipe — 1fps frames interleaved with "
+        "matching 1s audio segments as image_url/audio_url pairs (needed to approach "
+        "OpenBMB ~80%% Daily-Omni; requires local Videos extract). For MiniCPM-o 4.5 "
+        "string chat templates also start the server with --interleave-mm-strings.",
+    )
     group.add_argument(
         "--daily-omni-save-eval-items",
         action="store_true",

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py

@@ -3422,10 +3422,23 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
     ) -> bool:
         return False
 
-    def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
+    def get_image_prompt_texts(
+        self,
+        image_size: ImageSize,
+        image_idx: int = 0,
+        max_slice_nums: int | None = None,
+        use_image_id: bool | None = None,
+    ) -> str:
+        # ``process_images`` forwards ``mm_processor_kwargs`` to the pixel path, so the
+        # placeholder grid must honor the same ``max_slice_nums``; otherwise a request that
+        # sets it (e.g. interleaved omni frames with max_slice_nums=1) produces fewer image
+        # features than placeholder ``<unk>`` slots. ``use_image_id`` is likewise per-request
+        # so callers can match MiniCPM-o's omni path, which emits no ``<image_id>`` prefix.
         return self.info.get_slice_image_placeholder(
             image_size,
             image_idx=image_idx,
+            max_slice_nums=max_slice_nums,
+            **({} if use_image_id is None else {"use_image_id": use_image_id}),
         )
 
     def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str:
@@ -3615,13 +3628,21 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
                 additional_placeholders.append((modality, sub_pattern))
         placeholders += additional_placeholders
 
+        image_max_slice_nums = hf_processor_mm_kwargs.get("max_slice_nums")
+        image_use_image_id = hf_processor_mm_kwargs.get("use_image_id")
+
         def get_image_replacement(item_idx: int):
             images = mm_items.get_items("image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))
 
             image_size = images.get_image_size(item_idx)
 
             return PromptUpdateDetails.select_text(
-                self.get_image_prompt_texts(image_size, item_idx),
+                self.get_image_prompt_texts(
+                    image_size,
+                    item_idx,
+                    max_slice_nums=None if image_max_slice_nums is None else int(image_max_slice_nums),  # type: ignore[arg-type]
+                    use_image_id=None if image_use_image_id is None else bool(image_use_image_id),
+                ),
                 "<unk>",
             )