diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md
new file mode 100644
index 000000000..a34764652
--- /dev/null
+++ b/examples/online_serving/minicpmo/README.md
@@ -0,0 +1,65 @@
+# vLLM-Omni · MiniCPM-o 4.5 Online Demo
+
+Gradio-based web UI for **MiniCPM-o 4.5** served via `vllm-omni`'s
+OpenAI-compatible endpoints.
+
+The UI supports:
+
+- **Inputs**: text prompt + optional image, audio (file or mic), video.
+- **Outputs**: text + speech (WAV player).
+
+## 1. Start the backend server
+
+The deploy config auto-loads via `--omni`; the default
+`vllm_omni/deploy/minicpmo_4_5.yaml` targets a 2-GPU layout (thinker on
+GPU 0, talker + t2w sharing GPU 1).  For other hardware layouts pick
+one of the deploy variants below.
+
+| deploy config | GPUs | Notes |
+|---|---|---|
+| `minicpmo_4_5.yaml` (default) | 2 | Thinker on GPU0, talker+t2w on GPU1. |
+| `minicpmo_4_5_3gpu.yaml` | 3 | Thinker 2-way TP on GPU0/1, talker+t2w share GPU2. |
+| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
+
+Default (2-GPU):
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+    --trust-remote-code \
+    --host 0.0.0.0 --port 8099
+```
+
+Other layouts via `--deploy-config`:
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+    --deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \
+    --trust-remote-code \
+    --host 0.0.0.0 --port 8099
+```
+
+## 2. Launch the Gradio demo
+
+```bash
+bash examples/online_serving/minicpmo/run_gradio_demo.sh
+
+# Or run the python entry point directly:
+python examples/online_serving/minicpmo/gradio_demo.py \
+    --minicpmo45-api-base http://localhost:8099/v1 \
+    --minicpmo45-model openbmb/MiniCPM-o-4_5 \
+    --port 7862
+```
+
+Open `http://<host>:7862` in a browser.
+
+## Notes
+
+- **TTS trigger**: the demo sets
+  `extra_body.chat_template_kwargs.use_tts_template=True`, which appends
+  `<|tts_bos|>` to the assistant prefix.
+- Uncheck **"Generate speech output (TTS)"** to get text-only responses
+  (faster).
+- The audio output is the raw WAV returned by the stage-1 talker +
+  Token2Wav; sample rate is 24 kHz.
+- Video input is forwarded as a base64 `video_url` entry; the server needs
+  decord/torchvision to decode it.
diff --git a/examples/online_serving/minicpmo/gradio_demo.py b/examples/online_serving/minicpmo/gradio_demo.py
new file mode 100644
index 000000000..67bdb2876
--- /dev/null
+++ b/examples/online_serving/minicpmo/gradio_demo.py
@@ -0,0 +1,580 @@
+"""Gradio demo for MiniCPM-o 4.5 online serving through vllm-omni.
+
+Supports:
+- Text / Image / Audio / Video inputs (OpenAI chat-completions multimodal format).
+- Text + audio outputs, returned side-by-side in the UI.
+
+Usage:
+    # run a 4.5 server on :8099
+    python gradio_demo.py \
+        --minicpmo45-api-base http://localhost:8099/v1 \
+        --minicpmo45-model openbmb/MiniCPM-o-4_5 \
+        --port 7862
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import os
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import numpy as np
+import soundfile as sf
+from openai import OpenAI
+from PIL import Image
+
+MINICPMO45 = "MiniCPM-o-4.5"
+
+# Default reference audio per model, relative to the model path. Used to build
+# the audio_assistant system prompt so that the LLM adopts the "speak in this
+# voice" persona (matches the official MiniCPM-o demo).
+_DEFAULT_REF_AUDIO = {
+    MINICPMO45: "assets/HT_ref_audio.wav",
+}
+
+# audio_assistant prompt text, mirrors MiniCPMO.get_sys_prompt("audio_assistant")
+# in the MiniCPM-o-Demo reference implementation.
+_AUDIO_ASSISTANT_PROMPT = {
+    "zh": {
+        "prefix": "模仿音频样本的音色并生成新的内容。",
+        "suffix": (
+            "你的任务是用这种声音模式来当一个助手。请认真、高质量地回复用户的问题。"
+            "请用高自然度的方式和用户聊天。你是由面壁智能开发的人工智能助手：面壁小钢炮。"
+        ),
+    },
+    "en": {
+        "prefix": "Use the voice in the audio prompt to synthesize new content.",
+        "suffix": "You are a helpful assistant with the above voice style.",
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# Media helpers
+# ---------------------------------------------------------------------------
+
+
+def image_to_base64_data_url(image: Image.Image) -> str:
+    buf = io.BytesIO()
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    image.save(buf, format="JPEG", quality=92)
+    b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+    return f"data:image/jpeg;base64,{b64}"
+
+
+def audio_to_base64_data_url(audio_np: np.ndarray, sample_rate: int) -> str:
+    if audio_np.dtype != np.int16:
+        if audio_np.dtype in (np.float32, np.float64):
+            audio_np = np.clip(audio_np, -1.0, 1.0)
+            audio_np = (audio_np * 32767).astype(np.int16)
+        else:
+            audio_np = audio_np.astype(np.int16)
+    buf = io.BytesIO()
+    sf.write(buf, audio_np, sample_rate, format="WAV")
+    b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+    return f"data:audio/wav;base64,{b64}"
+
+
+def ref_audio_to_data_url(path: str, target_sr: int = 16000, max_s: float | None = 8.0) -> str:
+    """Load a file, downmix to mono, resample to 16kHz, optionally truncate, and
+    return a data-URL usable as audio_url content in the chat API.
+
+    MiniCPM-o expects 16kHz mono audio input; long reference audios can waste
+    tokens so we cap at max_s seconds by default (3-10s is the recommended range).
+    """
+    data, sr = sf.read(path)
+    if data.ndim > 1:
+        data = data[:, 0]
+    data = data.astype(np.float32)
+    if sr != target_sr:
+        try:
+            # Demo client only; librosa here resamples audio before sending to
+            # the server.  The fallback below covers environments without it.
+            import librosa  # noqa: TID251
+
+            data = librosa.resample(data, orig_sr=sr, target_sr=target_sr)
+        except ImportError:
+            from math import gcd
+
+            g = gcd(sr, target_sr)
+            up, down = target_sr // g, sr // g
+            idx = (np.arange(len(data) * up) / up).astype(np.int64)
+            data = data[idx][::down]
+        sr = target_sr
+    if max_s is not None and len(data) > int(max_s * sr):
+        data = data[: int(max_s * sr)]
+    return audio_to_base64_data_url(data, sr)
+
+
+def video_to_base64_data_url(path: str) -> str:
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Video not found: {path}")
+    ext = p.suffix.lower()
+    mime = {
+        ".mp4": "video/mp4",
+        ".webm": "video/webm",
+        ".mov": "video/quicktime",
+        ".avi": "video/x-msvideo",
+        ".mkv": "video/x-matroska",
+    }.get(ext, "video/mp4")
+    b64 = base64.b64encode(p.read_bytes()).decode("utf-8")
+    return f"data:{mime};base64,{b64}"
+
+
+def process_audio_input(audio_input: Any | None) -> tuple[np.ndarray, int] | None:
+    """Normalize Gradio audio input to (np.ndarray mono float32, sample_rate).
+
+    Gradio's `gr.Audio(type="numpy")` yields (sr, ndarray); we also accept
+    filesystem paths and a few nested tuple variants that appear in different
+    Gradio versions.
+    """
+    if audio_input is None:
+        return None
+
+    def _from_path(p: str) -> tuple[np.ndarray, int] | None:
+        if not p:
+            return None
+        fp = Path(p)
+        if not fp.exists():
+            return None
+        data, sr = sf.read(fp)
+        if data.ndim > 1:
+            data = data[:, 0]
+        return data.astype(np.float32), int(sr)
+
+    audio_np: np.ndarray | None = None
+    sr: int | None = None
+
+    if isinstance(audio_input, tuple) and len(audio_input) == 2:
+        a, b = audio_input
+        if isinstance(a, int | float) and isinstance(b, np.ndarray):
+            sr, audio_np = int(a), b
+        elif isinstance(a, str):
+            loaded = _from_path(a)
+            if loaded is not None:
+                audio_np, sr = loaded
+    elif isinstance(audio_input, str):
+        loaded = _from_path(audio_input)
+        if loaded is not None:
+            audio_np, sr = loaded
+
+    if audio_np is None or sr is None:
+        return None
+    if audio_np.ndim > 1:
+        audio_np = audio_np[:, 0]
+    return audio_np.astype(np.float32), sr
+
+
+# ---------------------------------------------------------------------------
+# Per-model inference config
+# ---------------------------------------------------------------------------
+
+
+class ModelEndpoint:
+    """Endpoint + TTS-trigger convention for one MiniCPM-o variant."""
+
+    def __init__(self, name: str, api_base: str, model_path: str):
+        self.name = name
+        self.api_base = api_base
+        self.model_path = model_path
+        self.client = OpenAI(api_key="EMPTY", base_url=api_base)
+        self._ref_audio_data_url: str | None = None
+
+    @property
+    def ref_audio_path(self) -> str | None:
+        rel = _DEFAULT_REF_AUDIO.get(self.name)
+        if not rel:
+            return None
+        full = os.path.join(self.model_path, rel)
+        return full if os.path.exists(full) else None
+
+    def get_ref_audio_data_url(self) -> str | None:
+        """Lazily load and cache the default reference audio as a 16kHz mono
+        data URL. Returns None if the reference audio file is missing."""
+        if self._ref_audio_data_url is not None:
+            return self._ref_audio_data_url
+        p = self.ref_audio_path
+        if not p:
+            print(f"[{self.name}] no default reference audio found; falling back to text-only system prompt")
+            return None
+        try:
+            self._ref_audio_data_url = ref_audio_to_data_url(p)
+            print(f"[{self.name}] loaded reference audio: {p}")
+        except Exception as e:
+            print(f"[{self.name}] failed to load reference audio {p}: {e}")
+            return None
+        return self._ref_audio_data_url
+
+    def build_extras(self) -> dict[str, Any]:
+        """Extra kwargs passed to chat.completions.create for this model.
+
+        4.5: use_tts_template=True chat-template flag appends <|tts_bos|>.
+        """
+        if self.name == MINICPMO45:
+            return {
+                "extra_body": {
+                    "chat_template_kwargs": {"use_tts_template": True},
+                    "modalities": ["text", "audio"],
+                },
+            }
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# System prompt (matches MiniCPMO.get_sys_prompt("audio_assistant") in the
+# official MiniCPM-o-Demo). When TTS is disabled we fall back to a plain
+# text-only system prompt.
+# ---------------------------------------------------------------------------
+
+
+def build_audio_assistant_system(
+    ref_audio_data_url: str | None,
+    language: str = "zh",
+) -> dict[str, Any]:
+    """Return an OpenAI-style system message carrying the reference audio."""
+    txt = _AUDIO_ASSISTANT_PROMPT.get(language, _AUDIO_ASSISTANT_PROMPT["zh"])
+    if ref_audio_data_url is None:
+        return {
+            "role": "system",
+            "content": (
+                "You are MiniCPM-o, a helpful multimodal assistant that can "
+                "understand images, audio and video, and respond in text and speech."
+            ),
+        }
+    return {
+        "role": "system",
+        "content": [
+            {"type": "text", "text": txt["prefix"]},
+            {"type": "audio_url", "audio_url": {"url": ref_audio_data_url}},
+            {"type": "text", "text": txt["suffix"]},
+        ],
+    }
+
+
+# ---------------------------------------------------------------------------
+# Core inference function
+# ---------------------------------------------------------------------------
+
+
+def run_inference(
+    endpoint: ModelEndpoint,
+    user_prompt: str,
+    audio_tuple: tuple[np.ndarray, int] | None,
+    image_pil: Image.Image | None,
+    video_path: str | None,
+    enable_tts: bool,
+    max_tokens: int,
+    temperature: float,
+) -> Iterable[tuple[str, tuple[int, np.ndarray] | None]]:
+    user_prompt = user_prompt or ""
+    if not user_prompt.strip() and not audio_tuple and not image_pil and not video_path:
+        yield "Please provide a text prompt or at least one multimodal input.", None
+        return
+
+    try:
+        content: list[dict[str, Any]] = []
+
+        if audio_tuple is not None:
+            audio_np, sr = audio_tuple
+            content.append(
+                {"type": "audio_url", "audio_url": {"url": audio_to_base64_data_url(audio_np, sr)}},
+            )
+
+        if image_pil is not None:
+            img = image_pil if image_pil.mode == "RGB" else image_pil.convert("RGB")
+            content.append(
+                {"type": "image_url", "image_url": {"url": image_to_base64_data_url(img)}},
+            )
+
+        if video_path:
+            content.append(
+                {"type": "video_url", "video_url": {"url": video_to_base64_data_url(video_path)}},
+            )
+
+        if user_prompt.strip():
+            content.append({"type": "text", "text": user_prompt})
+
+        if enable_tts:
+            ref_url = endpoint.get_ref_audio_data_url()
+            system_msg = build_audio_assistant_system(ref_url, language="zh")
+        else:
+            system_msg = {
+                "role": "system",
+                "content": (
+                    "You are MiniCPM-o, a helpful multimodal assistant that can "
+                    "understand images, audio and video, and respond in text and speech."
+                ),
+            }
+
+        messages = [
+            system_msg,
+            {"role": "user", "content": content},
+        ]
+
+        kwargs: dict[str, Any] = {
+            "model": endpoint.model_path,
+            "messages": messages,
+            "max_tokens": int(max_tokens),
+            "temperature": float(temperature),
+        }
+        if enable_tts:
+            extras = endpoint.build_extras()
+            # Merge extra_body dicts cleanly.
+            extra_body = dict(extras.get("extra_body", {}))
+            kwargs["extra_body"] = extra_body
+        else:
+            # Text-only: suppress audio output.
+            kwargs["extra_body"] = {"modalities": ["text"]}
+
+        completion = endpoint.client.chat.completions.create(**kwargs)
+
+        text_parts: list[str] = []
+        audio_out: tuple[int, np.ndarray] | None = None
+
+        for choice in completion.choices:
+            msg = choice.message
+            if getattr(msg, "content", None):
+                text_parts.append(msg.content)
+            audio_obj = getattr(msg, "audio", None)
+            if audio_obj and getattr(audio_obj, "data", None):
+                raw = base64.b64decode(audio_obj.data)
+                if len(raw) > 80:
+                    data, sr = sf.read(io.BytesIO(raw))
+                    if data.ndim > 1:
+                        data = data[:, 0]
+                    audio_out = (int(sr), data.astype(np.float32))
+
+        text_response = "\n\n".join(t for t in text_parts if t) or "(no text returned)"
+        yield text_response, audio_out
+    except Exception as exc:  # noqa: BLE001
+        yield f"Inference failed: {type(exc).__name__}: {exc}", None
+
+
+# ---------------------------------------------------------------------------
+# UI
+# ---------------------------------------------------------------------------
+
+DEFAULT_PROMPTS = [
+    "你好，请用一句话介绍北京。",
+    "Describe what you see in the image.",
+    "Transcribe the audio and translate it into English.",
+    "What is happening in this video?",
+]
+
+
+def build_interface(endpoints: dict[str, ModelEndpoint], default_model: str) -> gr.Blocks:
+    model_choices = list(endpoints.keys())
+
+    css = """
+    #generate-btn button { width: 100%; font-size: 1.05rem; }
+    .media-row > div { min-height: 260px; }
+    .model-banner { color: #555; font-size: 0.9rem; }
+    """
+
+    with gr.Blocks(css=css, title="vLLM-Omni · MiniCPM-o Demo") as demo:
+        gr.Markdown(
+            "# vLLM-Omni · MiniCPM-o Online Demo\n"
+            "Multimodal chat (text / image / audio / video in; text + speech out) for "
+            "MiniCPM-o 4.5 served by vllm-omni."
+        )
+
+        with gr.Row():
+            model_dd = gr.Dropdown(
+                label="Model",
+                choices=model_choices,
+                value=default_model,
+                scale=2,
+            )
+            endpoint_info = gr.Markdown(
+                elem_classes="model-banner",
+                value=f"`{endpoints[default_model].api_base}`  →  `{endpoints[default_model].model_path}`",
+            )
+
+        with gr.Row():
+            prompt_box = gr.Textbox(
+                label="Text Prompt",
+                placeholder="Ask the model anything. Combine with image / audio / video below.",
+                lines=3,
+                value=DEFAULT_PROMPTS[0],
+                scale=3,
+            )
+            with gr.Column(scale=1):
+                tts_checkbox = gr.Checkbox(label="Generate speech output (TTS)", value=True)
+                max_tokens_slider = gr.Slider(label="Max tokens", minimum=32, maximum=4096, value=1024, step=32)
+                temperature_slider = gr.Slider(label="Temperature", minimum=0.0, maximum=1.5, value=0.7, step=0.05)
+
+        with gr.Row(elem_classes="media-row"):
+            image_input = gr.Image(label="Image (optional)", type="pil", sources=["upload"])
+            audio_input = gr.Audio(
+                label="Audio (optional)",
+                type="numpy",
+                sources=["upload", "microphone"],
+            )
+            video_input = gr.Video(label="Video (optional)", sources=["upload"])
+
+        with gr.Row():
+            generate_btn = gr.Button("Generate", variant="primary", elem_id="generate-btn")
+            clear_btn = gr.Button("Clear")
+
+        with gr.Row():
+            text_output = gr.Textbox(label="Text Output", lines=10, scale=2)
+            audio_output = gr.Audio(label="Audio Output", interactive=False, scale=1)
+
+        gr.Examples(
+            examples=[
+                ["你好，请用一句话介绍北京。"],
+                ["Describe what you see in detail."],
+                ["请先识别这段语音的内容，再用中文回复说话人。"],
+                ["What is happening in this video?"],
+            ],
+            inputs=[prompt_box],
+            label="Quick prompts",
+        )
+
+        def _on_model_change(name: str) -> str:
+            ep = endpoints[name]
+            return f"`{ep.api_base}`  →  `{ep.model_path}`"
+
+        model_dd.change(_on_model_change, inputs=[model_dd], outputs=[endpoint_info])
+
+        def _run(
+            model_name: str,
+            prompt: str,
+            image_pil: Image.Image | None,
+            audio_in: Any,
+            video_path: str | None,
+            enable_tts: bool,
+            max_tokens: int,
+            temperature: float,
+        ):
+            ep = endpoints[model_name]
+            audio_tuple = process_audio_input(audio_in)
+            yield from run_inference(
+                ep,
+                prompt,
+                audio_tuple,
+                image_pil,
+                video_path,
+                enable_tts,
+                max_tokens,
+                temperature,
+            )
+
+        generate_btn.click(
+            _run,
+            inputs=[
+                model_dd,
+                prompt_box,
+                image_input,
+                audio_input,
+                video_input,
+                tts_checkbox,
+                max_tokens_slider,
+                temperature_slider,
+            ],
+            outputs=[text_output, audio_output],
+        )
+
+        def _clear():
+            return "", None, None, None, "", None
+
+        clear_btn.click(
+            _clear,
+            outputs=[
+                prompt_box,
+                image_input,
+                audio_input,
+                video_input,
+                text_output,
+                audio_output,
+            ],
+        )
+
+        demo.queue()
+    return demo
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--minicpmo45-api-base", default=os.environ.get("MINICPMO45_API_BASE", "http://localhost:8099/v1"))
+    p.add_argument(
+        "--minicpmo45-model",
+        default=os.environ.get("MINICPMO45_MODEL", "openbmb/MiniCPM-o-4_5"),
+    )
+    p.add_argument("--host", default="0.0.0.0")
+    p.add_argument("--port", type=int, default=7862)
+    p.add_argument("--share", action="store_true")
+    p.add_argument(
+        "--ssl-certfile",
+        default=os.environ.get("GRADIO_SSL_CERTFILE", ""),
+        help="Path to TLS certificate (PEM). Browsers require HTTPS to access the microphone.",
+    )
+    p.add_argument(
+        "--ssl-keyfile",
+        default=os.environ.get("GRADIO_SSL_KEYFILE", ""),
+        help="Path to TLS private key (PEM).",
+    )
+    p.add_argument(
+        "--ssl-verify",
+        action="store_true",
+        default=os.environ.get("GRADIO_SSL_VERIFY", "0") == "1",
+        help="Require browser to verify the certificate (off by default; self-signed certs should stay off).",
+    )
+    return p.parse_args()
+
+
+def _ping(api_base: str, timeout: float = 3.0) -> bool:
+    import urllib.request
+
+    try:
+        with urllib.request.urlopen(api_base.rstrip("/").replace("/v1", "/health"), timeout=timeout):
+            return True
+    except Exception:
+        return False
+
+
+def main() -> None:
+    args = parse_args()
+    endpoints: dict[str, ModelEndpoint] = {}
+
+    if args.minicpmo45_api_base and args.minicpmo45_model:
+        ok = _ping(args.minicpmo45_api_base)
+        status = "reachable" if ok else "not reachable (will error on generate)"
+        print(f"[4.5] {args.minicpmo45_api_base}  ({status})")
+        endpoints[MINICPMO45] = ModelEndpoint(MINICPMO45, args.minicpmo45_api_base, args.minicpmo45_model)
+
+    if not endpoints:
+        raise SystemExit("No endpoints configured. Pass --minicpmo45-api-base/--minicpmo45-model.")
+
+    default_model = next(iter(endpoints.keys()))
+    demo = build_interface(endpoints, default_model)
+
+    launch_kwargs: dict = {
+        "server_name": args.host,
+        "server_port": args.port,
+        "share": args.share,
+    }
+    if args.ssl_certfile and args.ssl_keyfile:
+        launch_kwargs.update(
+            ssl_certfile=args.ssl_certfile,
+            ssl_keyfile=args.ssl_keyfile,
+            ssl_verify=args.ssl_verify,
+        )
+        print(f"[tls] cert={args.ssl_certfile} key={args.ssl_keyfile} verify={args.ssl_verify}")
+    demo.launch(**launch_kwargs)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/online_serving/minicpmo/run_gradio_demo.sh b/examples/online_serving/minicpmo/run_gradio_demo.sh
new file mode 100755
index 000000000..22cc9f182
--- /dev/null
+++ b/examples/online_serving/minicpmo/run_gradio_demo.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Launch the MiniCPM-o 4.5 gradio demo.
+#
+# Prereq:
+#   Start a vllm-omni OpenAI server for MiniCPM-o 4.5 on :8099 (see the
+#   8x4090 stage config under vllm_omni/model_executor/stage_configs).
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+: "${MINICPMO45_API_BASE:=http://localhost:8099/v1}"
+: "${MINICPMO45_MODEL:=openbmb/MiniCPM-o-4_5}"
+: "${GRADIO_HOST:=0.0.0.0}"
+: "${GRADIO_PORT:=7862}"
+# HTTPS (browsers require a secure context for microphone access).
+# Set GRADIO_SSL_CERTFILE / GRADIO_SSL_KEYFILE to enable TLS.
+: "${GRADIO_SSL_CERTFILE:=}"
+: "${GRADIO_SSL_KEYFILE:=}"
+
+export MINICPMO45_API_BASE MINICPMO45_MODEL
+
+SSL_ARGS=()
+if [ -n "$GRADIO_SSL_CERTFILE" ] && [ -n "$GRADIO_SSL_KEYFILE" ] \
+   && [ -f "$GRADIO_SSL_CERTFILE" ] && [ -f "$GRADIO_SSL_KEYFILE" ]; then
+  SSL_ARGS=(--ssl-certfile "$GRADIO_SSL_CERTFILE" --ssl-keyfile "$GRADIO_SSL_KEYFILE")
+  echo "HTTPS enabled: cert=$GRADIO_SSL_CERTFILE key=$GRADIO_SSL_KEYFILE"
+else
+  echo "HTTPS disabled (cert/key not found). Microphone won't work on remote browsers."
+fi
+
+exec python "$HERE/gradio_demo.py" \
+    --minicpmo45-api-base "$MINICPMO45_API_BASE" \
+    --minicpmo45-model "$MINICPMO45_MODEL" \
+    --host "$GRADIO_HOST" \
+    --port "$GRADIO_PORT" \
+    "${SSL_ARGS[@]}"
diff --git a/pyproject.toml b/pyproject.toml
index 306f2e7cc..203475ad4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,6 +76,18 @@ local = [
     "FlagEmbedding",
 ]
 
+# Optional runtime dependencies for the MiniCPM-o 4.5 talker pipeline.
+# The talker loads MiniCPMTTS via `trust_remote_code` from the model repo,
+# and that module performs `from stepaudio2 import Token2wav` against the
+# MiniCPM-o-flavored Token2wav vocoder (PyPI: `stepaudio2-minicpmo`, NOT
+# upstream `stepfun-ai/Step-Audio2` — the upstream signature rejects
+# `n_timesteps` and crashes at construction). A clean `vllm-omni` install
+# does not pull this in by default to avoid forcing the dep onto users
+# who only need other models; opt in with `pip install vllm-omni[minicpmo]`.
+minicpmo = [
+    "stepaudio2-minicpmo",
+]
+
 docs = [
     "mkdocs>=1.5.0",
     "mkdocs-api-autonav",
@@ -101,6 +113,13 @@ Documentation = "https://vllm-omni.readthedocs.io"
 [project.scripts]
 vllm-omni = "vllm_omni.entrypoints.cli.main:main"
 
+# Register vllm-omni's model registrations on every vllm subprocess. vllm
+# auto-loads anything advertised under `vllm.general_plugins` in its own
+# `load_general_plugins()`, so worker subprocesses spawned by vllm (which
+# only import vllm, not vllm_omni) will still pick up the omni archs.
+[project.entry-points."vllm.general_plugins"]
+vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"
+
 
 [tool.setuptools.packages.find]
 where = ["."]
@@ -149,6 +168,10 @@ ignore = [
 [tool.ruff.lint.per-file-ignores]
 "examples/**" = ["E501"]  # Allow long lines in examples
 "tests/**" = ["E501"]  # Allow long lines in tests
+# MiniCPM-o reference modeling code vendored from upstream HF/transformers/MiniCPM-o.
+# These files keep upstream conventions (star imports, long lines, math-symbol
+# argument names) so they stay easy to diff against upstream releases.
+"vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py" = ["E402", "E501", "F403", "F405", "N803"]
 
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
 "librosa".msg = "The librosa module is banned, use vllm.multimodal helpers instead"
diff --git a/tests/model_executor/models/minicpmo_4_5/__init__.py b/tests/model_executor/models/minicpmo_4_5/__init__.py
new file mode 100644
index 000000000..208f01a7c
--- /dev/null
+++ b/tests/model_executor/models/minicpmo_4_5/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
diff --git a/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py
new file mode 100644
index 000000000..d52bcafa1
--- /dev/null
+++ b/tests/model_executor/models/minicpmo_4_5/test_llm2tts.py
@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for the MiniCPM-o 4.5 stage 0 -> stage 1 bridge.
+
+Covers ``vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts``:
+
+  - empty ``source_outputs`` raises
+  - latent fallback to ``hidden_states`` when ``multimodal_output`` is empty
+  - both inputs missing -> raises
+  - additional_information payload carries the keys the talker expects
+    (prompt_embeds, prompt_token_ids, llm_output_token_ids, llm_output_text)
+  - dummy talker prompt ``[BOS, PAD, EOS] = [1, 0, 2]`` (single prefill step)
+  - MiniCPM-o 4.5 TTS region detection on 151703 / 151704 tokens
+  - MiniCPM-o 2.6 fallback detection on 151691 / 151692 when no 4.5 markers
+  - No TTS markers present -> no ``tts_token_ids`` / ``tts_hidden_states`` keys
+  - prompt arg is normalized to a list and ``multi_modal_data`` is gated by
+    ``requires_multimodal_data``
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import llm2tts
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+_HIDDEN_DIM = 4
+
+
+def _make_thinker_output(
+    *,
+    prompt_token_ids: list[int],
+    output_token_ids: list[int],
+    text: str = "",
+    request_id: str = "req-0",
+    latent: torch.Tensor | None = None,
+    hidden_states: torch.Tensor | None = None,
+):
+    """Construct a minimal mock of a thinker engine output entry.
+
+    The real ``llm2tts`` only reads a tight slice of fields:
+      - top-level: ``request_id``, ``prompt_token_ids``, ``outputs[0]``
+      - per-output: ``multimodal_output`` (dict), ``hidden_states`` (opt),
+        ``text``, ``token_ids``
+    """
+    output = SimpleNamespace(
+        multimodal_output={"latent": latent} if latent is not None else {},
+        token_ids=output_token_ids,
+        text=text,
+    )
+    if hidden_states is not None:
+        output.hidden_states = hidden_states
+    return SimpleNamespace(
+        request_id=request_id,
+        prompt_token_ids=prompt_token_ids,
+        outputs=[output],
+    )
+
+
+class TestInputValidation:
+    def test_empty_source_outputs_raises(self) -> None:
+        with pytest.raises(ValueError, match="source_outputs cannot be empty"):
+            llm2tts([], prompt=None)
+
+    def test_missing_latent_and_hidden_states_raises(self) -> None:
+        bad = _make_thinker_output(
+            prompt_token_ids=[10, 11],
+            output_token_ids=[20],
+        )
+        with pytest.raises(ValueError, match="No latent or hidden_states"):
+            llm2tts([bad], prompt=None)
+
+
+class TestBasicShape:
+    def test_returns_one_entry_per_input(self) -> None:
+        hidden = torch.zeros((3, _HIDDEN_DIM))
+        out = llm2tts(
+            [
+                _make_thinker_output(prompt_token_ids=[10, 11], output_token_ids=[20], hidden_states=hidden),
+                _make_thinker_output(
+                    prompt_token_ids=[12], output_token_ids=[21, 22], hidden_states=hidden, request_id="req-1"
+                ),
+            ],
+            prompt=None,
+        )
+        assert len(out) == 2
+
+    def test_talker_prompt_token_ids_dummy_bos_pad_eos(self) -> None:
+        hidden = torch.zeros((2, _HIDDEN_DIM))
+        out = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
+            prompt=None,
+        )
+        # The talker AR framework needs *some* tokens to do a single prefill;
+        # this is the agreed [BOS, PAD, EOS] minimal payload.
+        assert out[0]["prompt_token_ids"] == [1, 0, 2]
+
+    def test_additional_information_carries_thinker_outputs(self) -> None:
+        prompt_ids = [10, 11, 12]
+        out_ids = [20, 21]
+        hidden = torch.randn(len(prompt_ids) + len(out_ids), _HIDDEN_DIM)
+
+        result = llm2tts(
+            [
+                _make_thinker_output(
+                    prompt_token_ids=prompt_ids,
+                    output_token_ids=out_ids,
+                    text="hello",
+                    hidden_states=hidden,
+                )
+            ],
+            prompt=None,
+        )
+        ai = result[0]["additional_information"]
+        assert ai["prompt_token_ids"] == prompt_ids
+        assert ai["llm_output_token_ids"] == out_ids
+        assert ai["llm_output_text"] == ["hello"]
+        # prompt_embeds = float32 view of the prompt-portion of hidden states.
+        assert ai["prompt_embeds"].dtype == torch.float32
+        assert ai["prompt_embeds"].shape == (len(prompt_ids), _HIDDEN_DIM)
+        assert torch.equal(ai["prompt_embeds"], hidden[: len(prompt_ids)].to(torch.float32))
+
+    def test_latent_in_multimodal_output_takes_precedence(self) -> None:
+        # When both ``multimodal_output["latent"]`` and ``hidden_states`` are
+        # present, the latent payload must win (this is the steady-state path
+        # produced by the thinker stage).
+        prompt_ids = [10, 11]
+        out_ids = [20]
+        latent = torch.ones((len(prompt_ids) + len(out_ids), _HIDDEN_DIM))
+        hidden = torch.zeros_like(latent)
+
+        result = llm2tts(
+            [
+                _make_thinker_output(
+                    prompt_token_ids=prompt_ids,
+                    output_token_ids=out_ids,
+                    latent=latent,
+                    hidden_states=hidden,
+                )
+            ],
+            prompt=None,
+        )
+        ai = result[0]["additional_information"]
+        # latent (ones) won over hidden_states (zeros)
+        assert torch.equal(ai["prompt_embeds"], latent[: len(prompt_ids)].to(torch.float32))
+
+
+class TestTtsRegionDetection:
+    """The bridge auto-detects which MiniCPM-o generation produced the output
+    by sniffing for 4.5-specific BOS/EOS token ids (151703 / 151704); when
+    those are absent it falls back to the 2.6 ids (151691 / 151692).
+
+    This is a regression guard: a single off-by-one or wrong branch
+    precedence here causes the talker to receive an empty / wrong-region
+    slice and emit silent audio.
+    """
+
+    def _run(self, prompt_token_ids, output_token_ids):
+        total = len(prompt_token_ids) + len(output_token_ids)
+        hidden = torch.arange(total * _HIDDEN_DIM, dtype=torch.float32).reshape(total, _HIDDEN_DIM)
+        result = llm2tts(
+            [
+                _make_thinker_output(
+                    prompt_token_ids=prompt_token_ids,
+                    output_token_ids=output_token_ids,
+                    hidden_states=hidden,
+                )
+            ],
+            prompt=None,
+        )
+        return result[0]["additional_information"], hidden
+
+    def test_4_5_markers_detected(self) -> None:
+        # prompt:        [10, 11]
+        # output:        [151703, 30, 31, 151704, 40]
+        # full sequence: [10, 11, 151703, 30, 31, 151704, 40]
+        #                  0   1     2    3   4     5     6
+        # 4.5 BOS at idx 2 -> slice starts at 3; EOS at idx 5 -> slice ends at 5.
+        ai, hidden = self._run([10, 11], [151703, 30, 31, 151704, 40])
+        assert "tts_token_ids" in ai and "tts_hidden_states" in ai
+        assert ai["tts_token_ids"].tolist() == [30, 31]
+        assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
+
+    def test_4_5_takes_precedence_when_both_markers_present(self) -> None:
+        # If both 2.6 and 4.5 markers appear (shouldn't really happen on a
+        # real checkpoint, but exercises the loop's break), the 4.5 markers
+        # win because the source loop pins on the first 4.5 id seen.
+        # sequence index: 0   1   2     3   4     5    6   7    8
+        ai, hidden = self._run([10], [151691, 70, 151692, 151703, 30, 151704, 40])
+        assert ai["tts_token_ids"].tolist() == [30]
+        assert torch.equal(ai["tts_hidden_states"], hidden[5:6])
+
+    def test_2_6_fallback(self) -> None:
+        # No 4.5 markers anywhere -> use 2.6 BOS/EOS pair.
+        # full sequence: [10, 151691, 30, 31, 151692, 40]
+        #                  0     1    2   3     4     5
+        # BOS at idx 1 -> slice starts at 2; EOS at idx 4 -> slice ends at 4.
+        ai, hidden = self._run([10], [151691, 30, 31, 151692, 40])
+        assert ai["tts_token_ids"].tolist() == [30, 31]
+        assert torch.equal(ai["tts_hidden_states"], hidden[2:4])
+
+    def test_bos_without_eos_runs_to_end(self) -> None:
+        # When BOS is found but EOS is missing (typical for an in-flight or
+        # truncated decode), the slice should extend to the end of the
+        # hidden-state matrix instead of being dropped.
+        # sequence: [10, 11, 151703, 30, 31]
+        ai, hidden = self._run([10, 11], [151703, 30, 31])
+        assert ai["tts_token_ids"].tolist() == [30, 31]
+        assert torch.equal(ai["tts_hidden_states"], hidden[3:5])
+
+    def test_no_tts_markers_omits_slice_keys(self) -> None:
+        # If neither marker pair is present, the bridge should NOT populate
+        # ``tts_token_ids`` / ``tts_hidden_states`` — the talker should fall
+        # through to the dummy path.
+        ai, _ = self._run([10, 11], [20, 21, 22])
+        assert "tts_token_ids" not in ai
+        assert "tts_hidden_states" not in ai
+
+
+class TestPromptAndMultiModal:
+    def test_prompt_can_be_single_dict_not_a_list(self) -> None:
+        hidden = torch.zeros((2, _HIDDEN_DIM))
+        # A single (non-list) prompt should be auto-wrapped without raising.
+        llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
+            prompt={"multi_modal_data": {"audio": "ignored"}},
+            requires_multimodal_data=False,
+        )
+
+    def test_multimodal_dropped_when_not_requested(self) -> None:
+        hidden = torch.zeros((2, _HIDDEN_DIM))
+        out = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
+            prompt={"multi_modal_data": {"audio": "should-be-ignored"}},
+            requires_multimodal_data=False,
+        )
+        assert out[0]["multi_modal_data"] is None
+
+    def test_multimodal_forwarded_when_requested(self) -> None:
+        hidden = torch.zeros((2, _HIDDEN_DIM))
+        mm = {"audio": "forward-me"}
+        out = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
+            prompt={"multi_modal_data": mm},
+            requires_multimodal_data=True,
+        )
+        assert out[0]["multi_modal_data"] == mm
+
+    def test_streaming_context_is_accepted_and_ignored(self) -> None:
+        # The framework passes ``streaming_context`` for streaming-capable
+        # pipelines; MiniCPM-o 4.5 is turn-taking so the bridge must accept
+        # but discard it without crashing.
+        hidden = torch.zeros((2, _HIDDEN_DIM))
+        out = llm2tts(
+            [_make_thinker_output(prompt_token_ids=[10], output_token_ids=[20], hidden_states=hidden)],
+            prompt=None,
+            streaming_context=object(),
+        )
+        assert len(out) == 1
diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py
new file mode 100644
index 000000000..c51db9fcc
--- /dev/null
+++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py
@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for the MiniCPM-o 4.5 pipeline registration.
+
+Covers:
+  - pipeline declared in the central registry
+  - lazy loader returns the expected ``PipelineConfig``
+  - 2-stage topology (thinker LLM_AR + talker LLM_AR with audio output)
+  - stage 1 routes through ``llm2tts`` custom input processor
+  - ``hf_architectures`` covers both the shared ``MiniCPMO`` alias and the
+    explicit 4.5 arch
+  - ``hf_config_predicate`` selects MiniCPM-o 4.5 only and rejects 2.6
+    checkpoints (regression guard for the shared-arch routing collision).
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
+from vllm_omni.config.stage_config import (
+    _PIPELINE_REGISTRY,
+    PipelineConfig,
+    StageExecutionType,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+_PIPELINE_KEY = "minicpmo_4_5"
+
+
+class TestRegistryDeclaration:
+    def test_declared_in_omni_pipelines(self) -> None:
+        assert _PIPELINE_KEY in _OMNI_PIPELINES
+
+    def test_visible_in_central_registry(self) -> None:
+        assert _PIPELINE_KEY in _PIPELINE_REGISTRY
+
+    def test_lazy_load_returns_pipeline_config(self) -> None:
+        pipeline = _PIPELINE_REGISTRY[_PIPELINE_KEY]
+        assert isinstance(pipeline, PipelineConfig)
+        assert pipeline.model_type == _PIPELINE_KEY
+        assert pipeline.model_arch == "MiniCPMO45OmniForConditionalGeneration"
+
+
+class TestPipelineTopology:
+    @pytest.fixture(scope="class")
+    def pipeline(self) -> PipelineConfig:
+        return _PIPELINE_REGISTRY[_PIPELINE_KEY]
+
+    def test_two_stages(self, pipeline: PipelineConfig) -> None:
+        assert len(pipeline.stages) == 2
+        assert [s.stage_id for s in pipeline.stages] == [0, 1]
+
+    def test_topology_validates(self, pipeline: PipelineConfig) -> None:
+        # ``validate`` returns a list of structural errors; empty == valid.
+        assert pipeline.validate() == []
+
+    def test_thinker_stage(self, pipeline: PipelineConfig) -> None:
+        thinker = pipeline.get_stage(0)
+        assert thinker is not None
+        assert thinker.model_stage == "llm"
+        assert thinker.execution_type == StageExecutionType.LLM_AR
+        assert thinker.input_sources == ()
+        assert thinker.final_output is True
+        assert thinker.final_output_type == "text"
+        assert thinker.owns_tokenizer is True
+        assert thinker.requires_multimodal_data is True
+
+    def test_talker_stage(self, pipeline: PipelineConfig) -> None:
+        talker = pipeline.get_stage(1)
+        assert talker is not None
+        assert talker.model_stage == "tts"
+        assert talker.execution_type == StageExecutionType.LLM_AR
+        # talker consumes thinker output
+        assert talker.input_sources == (0,)
+        assert talker.final_output is True
+        assert talker.final_output_type == "audio"
+        assert talker.engine_output_type == "audio"
+        # scope KV cache / mrope sizing to talker sub-config
+        assert talker.hf_config_name == "tts_config"
+
+    def test_talker_routes_through_llm2tts(self, pipeline: PipelineConfig) -> None:
+        talker = pipeline.get_stage(1)
+        assert talker is not None
+        # stage 1's custom_process_input_func is what bridges thinker
+        # hidden_states + token ids into the talker; if this drifts the
+        # talker silently goes through the dummy path.
+        assert talker.custom_process_input_func == (
+            "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.llm2tts"
+        )
+
+
+class TestArchAliases:
+    """``hf_architectures`` must cover both the shared and explicit names."""
+
+    @pytest.fixture(scope="class")
+    def pipeline(self) -> PipelineConfig:
+        return _PIPELINE_REGISTRY[_PIPELINE_KEY]
+
+    def test_shared_minicpmo_alias_present(self, pipeline: PipelineConfig) -> None:
+        # MiniCPM-o 4.5 ships ``architectures=["MiniCPMO"]`` in its HF config.
+        # Without this alias the arch-fallback path in StageConfigFactory
+        # cannot resolve the pipeline.
+        assert "MiniCPMO" in pipeline.hf_architectures
+
+    def test_explicit_4_5_arch_present(self, pipeline: PipelineConfig) -> None:
+        # Reserve the explicit arch name for future repos that opt into it.
+        assert "MiniCPMO45OmniForConditionalGeneration" in pipeline.hf_architectures
+
+
+class TestHfConfigPredicate:
+    """Regression guard for the 2.6 / 4.5 shared-arch routing collision.
+
+    Both MiniCPM-o 2.6 and 4.5 ship ``architectures=["MiniCPMO"]`` in HF
+    config. The 4.5 pipeline uses ``hf_config_predicate`` to opt in only
+    when ``config.version == "4.5"``; without it, a 2.6 checkpoint would
+    intersect on the shared arch and get misrouted into the 4.5 pipeline.
+    """
+
+    @pytest.fixture(scope="class")
+    def predicate(self):
+        pipeline = _PIPELINE_REGISTRY[_PIPELINE_KEY]
+        assert pipeline.hf_config_predicate is not None, (
+            "MiniCPM-o 4.5 pipeline must declare hf_config_predicate to "
+            "avoid misrouting MiniCPM-o 2.6 checkpoints into the 4.5 path."
+        )
+        return pipeline.hf_config_predicate
+
+    def test_accepts_4_5_string(self, predicate) -> None:
+        assert predicate(SimpleNamespace(version="4.5")) is True
+
+    def test_rejects_2_6_string(self, predicate) -> None:
+        assert predicate(SimpleNamespace(version="2.6")) is False
+
+    def test_rejects_missing_version(self, predicate) -> None:
+        # 1.0 / older checkpoints do not carry ``version`` at all.
+        assert predicate(SimpleNamespace()) is False
+
+    def test_rejects_empty_version(self, predicate) -> None:
+        assert predicate(SimpleNamespace(version="")) is False
diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py
index 938dd74e7..d5ea7b38f 100644
--- a/vllm_omni/config/pipeline_registry.py
+++ b/vllm_omni/config/pipeline_registry.py
@@ -130,6 +130,10 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = {
         "vllm_omni.model_executor.models.moss_tts_nano.pipeline",
         "MOSS_TTS_NANO_PIPELINE",
     ),
+    "minicpmo_4_5": (
+        "vllm_omni.model_executor.models.minicpmo_4_5.pipeline",
+        "MINICPMO_4_5_PIPELINE",
+    ),
     "higgs_audio_v2": (
         "vllm_omni.model_executor.models.higgs_audio_v2.pipeline",
         "HIGGS_AUDIO_V2_PIPELINE",
diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py
index 5142ba234..93453006f 100644
--- a/vllm_omni/config/stage_config.py
+++ b/vllm_omni/config/stage_config.py
@@ -7,6 +7,7 @@ from __future__ import annotations
 import dataclasses
 import re
 import warnings
+from collections.abc import Callable
 from dataclasses import asdict, dataclass, field, fields
 from enum import Enum
 from pathlib import Path
@@ -242,6 +243,18 @@ class PipelineConfig:
     # matches ``hf_config.architectures[*]`` against this tuple to route
     # to the correct pipeline. Leave empty for models with unique model_type.
     hf_architectures: tuple[str, ...] = ()
+    # Optional second-stage predicate for resolving an arch-name collision
+    # between sibling model generations that ship the same
+    # ``architectures=[...]`` entry. When the arch-fallback in
+    # ``StageConfigFactory.create_from_model`` finds an intersection with
+    # ``hf_architectures``, it additionally evaluates this predicate against
+    # the loaded ``hf_config`` and only selects this pipeline when it
+    # returns ``True``. Leave ``None`` to skip the extra check (default).
+    # Example: MiniCPM-o 4.5 and 2.6 both ship ``architectures=["MiniCPMO"]``
+    # but differ on the ``version`` field, so the 4.5 pipeline declares
+    # ``hf_config_predicate=lambda c: getattr(c, "version", "") == "4.5"``
+    # to avoid misrouting 2.6 checkpoints.
+    hf_config_predicate: Callable[[Any], bool] | None = None
     # Diffusers pipeline class name: for models that ship a ``model_index.json``
     # (no root ``config.json``), the ``_class_name`` field is matched against
     # this value to auto-detect the pipeline.  Only needed for diffusers-style
@@ -1179,12 +1192,35 @@ class StageConfigFactory:
         # --- HF architecture fallback: some models report a generic
         # model_type that collides with another model. Match by the
         # hf_architectures declared on each registered PipelineConfig.
+        # When a pipeline declares ``hf_config_predicate`` (e.g. to
+        # disambiguate generation-revision collisions like MiniCPM-o 2.6 vs
+        # 4.5 which share ``architectures=["MiniCPMO"]``), the predicate
+        # must also accept the loaded ``hf_config``.
         if hf_config is not None:
             hf_archs = set(getattr(hf_config, "architectures", []) or [])
             if hf_archs:
                 for registered in _PIPELINE_REGISTRY.values():
-                    if hf_archs.intersection(registered.hf_architectures):
-                        return cls._create_from_registry(registered.model_type, cli_overrides, deploy_config_path)
+                    if not hf_archs.intersection(registered.hf_architectures):
+                        continue
+                    predicate = registered.hf_config_predicate
+                    if predicate is not None:
+                        try:
+                            if not predicate(hf_config):
+                                logger.debug(
+                                    "Pipeline %r matched on architectures %s but its "
+                                    "hf_config_predicate rejected the loaded config; "
+                                    "continuing fallback search.",
+                                    registered.model_type,
+                                    sorted(hf_archs.intersection(registered.hf_architectures)),
+                                )
+                                continue
+                        except Exception:
+                            logger.exception(
+                                "Pipeline %r hf_config_predicate raised; skipping.",
+                                registered.model_type,
+                            )
+                            continue
+                    return cls._create_from_registry(registered.model_type, cli_overrides, deploy_config_path)
 
         # --- Legacy path: load from pipeline YAML ---
         pipeline = cls._load_pipeline(model, trust_remote_code=trust_remote_code)
diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml
new file mode 100644
index 000000000..980615479
--- /dev/null
+++ b/vllm_omni/deploy/minicpmo_4_5.yaml
@@ -0,0 +1,51 @@
+# MiniCPM-o 4.5 default deploy: 2-GPU layout.
+#
+# Stage 0 (thinker): GPU 0 — multimodal understanding, ~70% mem.
+# Stage 1 (talker) : GPU 1 — MiniCPMTTS + Token2Wav, ~75% mem
+#                            (covers both the talker LM and the in-process
+#                            Token2Wav vocoder that emits the waveform).
+#
+# Other GPU layouts: minicpmo_4_5_3gpu.yaml, minicpmo_4_5_8x4090.yaml.
+async_chunk: false
+
+trust_remote_code: true
+enable_prefix_caching: false
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.7
+    enforce_eager: true
+    tensor_parallel_size: 1
+    max_num_batched_tokens: 16384
+    devices: "0"
+    limit_mm_per_prompt:
+      video: 1
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 2048
+      seed: 42
+      detokenize: true
+      repetition_penalty: 1.1
+
+  - stage_id: 1
+    # Talker only consumes runtime_additional_information[0]; any value > 1
+    # makes concurrent requests share request-0's audio (mm_outputs fallback).
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.75
+    enforce_eager: true
+    max_num_batched_tokens: 8192
+    devices: "1"
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 1
+      seed: 42
+      detokenize: false
+
+# MiniCPM-o 4.5 has only been verified on CUDA devices.  Add per-platform
+# overrides here when other backends (ROCm/XPU/NPU) are verified end-to-end.
+platforms: {}
diff --git a/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml
new file mode 100644
index 000000000..f5c0b03a6
--- /dev/null
+++ b/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml
@@ -0,0 +1,48 @@
+# MiniCPM-o 4.5 deploy: 3-GPU layout (thinker TP=2 + talker on GPU 2).
+#
+# Stage 0 (thinker): GPU 0,1 — 2-way tensor parallel, ~70% mem each.
+# Stage 1 (talker) : GPU 2   — MiniCPMTTS + Token2Wav, ~75% mem
+#                              (covers both the talker LM and the in-process
+#                              Token2Wav vocoder that emits the waveform).
+pipeline: minicpmo_4_5
+async_chunk: false
+
+trust_remote_code: true
+enable_prefix_caching: false
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.7
+    enforce_eager: true
+    tensor_parallel_size: 2
+    max_num_batched_tokens: 16384
+    devices: "0,1"
+    limit_mm_per_prompt:
+      video: 1
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 2048
+      seed: 42
+      detokenize: true
+      repetition_penalty: 1.1
+
+  - stage_id: 1
+    # Talker only consumes runtime_additional_information[0]; any value > 1
+    # makes concurrent requests share request-0's audio (mm_outputs fallback).
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.75
+    enforce_eager: true
+    max_num_batched_tokens: 8192
+    devices: "2"
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 1
+      seed: 42
+      detokenize: false
+
+platforms: {}
diff --git a/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml
new file mode 100644
index 000000000..dd249c70a
--- /dev/null
+++ b/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml
@@ -0,0 +1,60 @@
+# MiniCPM-o 4.5 deploy: 8x4090 layout (24GB cards).
+#
+# Stage 0 (thinker): GPU 0,1,2,3 — 4-way TP, ~85% mem (≈20.4 GB / card,
+#                                  thinker weights take ~17.5 GB per card
+#                                  on 4090, leaving ~3 GB for KV cache /
+#                                  activations).
+# Stage 1 (talker) : GPU 4       — MiniCPMTTS + Token2Wav, ~90% mem
+#                                  (covers both the talker LM and the
+#                                  in-process Token2Wav vocoder that emits
+#                                  the waveform).
+#
+# max_model_len is pinned to 4096 because the upstream config.json
+# advertises max_position_embeddings=40960 (tuned for H20 141GB); on a
+# 24 GB 4090 a 40k-token KV cache request OOMs the worker process
+# before serving the first request. 8192 also OOMs in practice with
+# this thinker model + vision/audio encoders + Whisper preallocation,
+# so 4096 is the empirically-verified safe ceiling.
+pipeline: minicpmo_4_5
+async_chunk: false
+
+trust_remote_code: true
+enable_prefix_caching: false
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.85
+    enforce_eager: true
+    tensor_parallel_size: 4
+    max_model_len: 4096
+    max_num_batched_tokens: 4096
+    devices: "0,1,2,3"
+    limit_mm_per_prompt:
+      video: 1
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 2048
+      seed: 42
+      detokenize: true
+      repetition_penalty: 1.1
+
+  - stage_id: 1
+    # Talker only consumes runtime_additional_information[0]; any value > 1
+    # makes concurrent requests share request-0's audio (mm_outputs fallback).
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.9
+    enforce_eager: true
+    max_num_batched_tokens: 8192
+    devices: "4"
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 1
+      seed: 42
+      detokenize: false
+
+platforms: {}
diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py
index c4298298e..7fcb7d0ef 100644
--- a/vllm_omni/engine/arg_utils.py
+++ b/vllm_omni/engine/arg_utils.py
@@ -78,10 +78,13 @@ def register_omni_models_to_vllm():
 
     _register_omni_hf_configs()
 
-    supported_archs = ModelRegistry.get_supported_archs()
+    # Always register every arch declared in _OMNI_MODELS — these are the
+    # archs vllm-omni explicitly owns, so we want our implementation to
+    # win even if a future vllm release ships a same-named built-in.
+    # vllm's ModelRegistry.register_model already emits a warning and
+    # overrides on collision; we let it handle that.
     for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items():
-        if arch not in supported_archs:
-            ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")
+        ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")
 
 
 @dataclass
diff --git a/vllm_omni/entrypoints/cli/main.py b/vllm_omni/entrypoints/cli/main.py
index 44d809466..81b9e8061 100644
--- a/vllm_omni/entrypoints/cli/main.py
+++ b/vllm_omni/entrypoints/cli/main.py
@@ -44,11 +44,18 @@ def main():
             description="vLLM OMNI CLI",
             epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
         )
+        try:
+            _omni_version = importlib.metadata.version("vllm_omni")
+        except importlib.metadata.PackageNotFoundError:
+            try:
+                from vllm_omni.version import __version__ as _omni_version  # type: ignore
+            except Exception:
+                _omni_version = "dev"
         parser.add_argument(
             "-v",
             "--version",
             action="version",
-            version=importlib.metadata.version("vllm_omni"),
+            version=_omni_version,
         )
         subparsers = parser.add_subparsers(required=False, dest="subparser")
         cmds = {}
diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py
index daeb22cbd..7afba4d45 100644
--- a/vllm_omni/entrypoints/openai/api_server.py
+++ b/vllm_omni/entrypoints/openai/api_server.py
@@ -578,6 +578,31 @@ async def build_async_omni_from_stage_config(
 
     async_omni: EngineClient | None = None
 
+    # Pre-load the model config so HuggingFace registers `transformers_modules`
+    # in this process — only when the user has explicitly opted in via
+    # `--trust-remote-code`. Stage workers consume the same flag through their
+    # deploy config, but the API server process also needs the dynamic modules
+    # for ZMQ pickle deserialization of stage outputs that reference them
+    # (e.g. trust_remote_code models like MiniCPM-o).
+    if getattr(args, "trust_remote_code", False) and getattr(args, "model", None):
+        try:
+            import os
+
+            from transformers import AutoConfig
+
+            # Hide GPUs so the custom config code doesn't allocate CUDA memory.
+            saved = os.environ.get("CUDA_VISIBLE_DEVICES")
+            os.environ["CUDA_VISIBLE_DEVICES"] = ""
+            try:
+                AutoConfig.from_pretrained(args.model, trust_remote_code=True)
+            finally:
+                if saved is not None:
+                    os.environ["CUDA_VISIBLE_DEVICES"] = saved
+                else:
+                    os.environ.pop("CUDA_VISIBLE_DEVICES", None)
+        except Exception as e:
+            logger.debug("Pre-loading transformers_modules failed: %s", e)
+
     try:
         kwargs = vars(args).copy()
         kwargs.pop("model", None)
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py b/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py
new file mode 100644
index 000000000..7284c5b7f
--- /dev/null
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py
@@ -0,0 +1,9 @@
+from .minicpmo_4_5_omni import MiniCPMO45OmniForConditionalGeneration
+from .minicpmo_4_5_omni_llm import MiniCPMO45OmniLLMForConditionalGeneration
+from .minicpmo_4_5_omni_tts import MiniCPMO45OmniTTSForConditionalGeneration
+
+__all__ = [
+    "MiniCPMO45OmniForConditionalGeneration",
+    "MiniCPMO45OmniLLMForConditionalGeneration",
+    "MiniCPMO45OmniTTSForConditionalGeneration",
+]
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py
new file mode 100644
index 000000000..7746b423e
--- /dev/null
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py
@@ -0,0 +1,384 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from:
+# https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py
+#
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Iterable
+from functools import cached_property
+
+import torch
+import torch.nn as nn
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
+from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.sequence import IntermediateTensors
+from vllm.v1.outputs import SamplerOutput
+from vllm.v1.sample.metadata import SamplingMetadata
+
+from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
+    MiniCPMO45OmniLLMDummyInputsBuilder,
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    MiniCPMO45OmniLLMProcessingInfo,
+    MiniCPMOConfig,
+)
+from vllm_omni.model_executor.models.output_templates import OmniOutput
+from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
+from vllm_omni.platforms import current_omni_platform
+
+logger = init_logger(__name__)
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    info=MiniCPMO45OmniLLMProcessingInfo,
+    dummy_inputs=MiniCPMO45OmniLLMDummyInputsBuilder,
+)
+class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
+    """MiniCPM-o 4.5 Omni model for conditional generation.
+
+    Two-stage pipeline:
+    - thinker (model_stage="llm"): image / video / audio encoders + 3D
+      resampler + the omni LLM that emits text + hidden states.
+    - talker  (model_stage="tts"): MiniCPMTTS + the in-process Token2Wav
+      vocoder that emits the final audio waveform directly.
+    """
+
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+        if modality.startswith("image"):
+            return "(<image>./</image>)"
+        if modality.startswith("video"):
+            return "(<video>./</video>)"
+        if modality.startswith("audio"):
+            return "(<audio>./</audio>)"
+        raise ValueError("Only image, video or audio modality is supported")
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        super().__init__()
+        self.have_multimodal_outputs = True
+        config: MiniCPMOConfig = vllm_config.model_config.hf_config
+        multimodal_config = vllm_config.model_config.multimodal_config
+        # keep vllm_config for later submodule init
+        self.vllm_config = vllm_config
+
+        # Store configs
+        self.config = config
+        self.multimodal_config = multimodal_config
+
+        self.model_stage = vllm_config.model_config.model_stage
+
+        if self.model_stage == "llm":
+            # Initialize thinker model (image preprocessing + vision encoder + 3D resampler)
+            self.thinker = init_vllm_registered_model(
+                vllm_config=vllm_config,
+                prefix=maybe_prefix(prefix, "thinker"),
+                hf_config=config,
+                # Registry key — must match the entries declared in
+                # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS.
+                architectures=["MiniCPMO45OmniLLMForConditionalGeneration"],
+            )
+            self.model = self.thinker
+            self.talker = None
+
+        elif self.model_stage == "tts":
+            self.thinker = None
+            # Initialize talker model — runs MiniCPMTTS + the in-process
+            # Token2Wav vocoder and emits the final audio waveform directly.
+            self.talker = init_vllm_registered_model(
+                vllm_config=vllm_config,
+                prefix=maybe_prefix(prefix, "talker"),
+                hf_config=config,
+                # Registry key — must match the entries declared in
+                # vllm_omni/model_executor/models/registry.py::_OMNI_MODELS.
+                architectures=["MiniCPMO45OmniTTSForConditionalGeneration"],
+            )
+            # Initialize multimodal components if needed
+            if hasattr(self.talker, "init_multi_modal"):
+                self.talker.init_multi_modal(config)
+            self.model = self.talker
+
+        else:
+            raise ValueError(f"Invalid model stage: {self.model_stage}. Must be one of: 'llm', 'tts'")
+
+        # Set up intermediate tensors
+        self.make_empty_intermediate_tensors = (
+            (self.thinker.make_empty_intermediate_tensors)
+            if self.model_stage == "llm" and self.thinker is not None
+            else lambda: None
+        )
+
+        self._language_model_names = ["model"]
+
+    @cached_property
+    def sampler(self):
+        if hasattr(self.model, "sampler"):
+            return self.model.sampler
+        from vllm.v1.sample.sampler import Sampler
+
+        return Sampler()
+
+    # -------------------- Device utilities --------------------
+    @staticmethod
+    def _module_device(module: nn.Module) -> torch.device:
+        try:
+            return next(module.parameters()).device
+        except StopIteration:
+            # No parameters; fall back to buffers or cpu
+            for _, buf in module.named_buffers(recurse=True):
+                return buf.device
+            return torch.device("cpu")
+
+    def move_submodules_to_devices(
+        self,
+        *,
+        thinker_device: str | torch.device | None = None,
+        talker_device: str | torch.device | None = None,
+    ) -> None:
+        """Optionally move thinker / talker to different devices.
+
+        Example:
+            model.move_submodules_to_devices(
+                thinker_device='cuda:0',
+                talker_device='cuda:1',
+            )
+        """
+        if thinker_device is not None and self.thinker is not None:
+            self.thinker.to(thinker_device)
+        if talker_device is not None and self.talker is not None:
+            self.talker.to(talker_device)
+
+    def get_input_embeddings(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings=None,
+    ) -> torch.Tensor:
+        return self.model.get_input_embeddings(input_ids, multimodal_embeddings)
+
+    def embed_input_ids(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings=None,
+        *,
+        is_multimodal=None,
+    ) -> torch.Tensor:
+        if self.model_stage == "tts":
+            return self.get_input_embeddings(input_ids)
+        return super().embed_input_ids(input_ids, multimodal_embeddings, is_multimodal=is_multimodal)
+
+    def get_multimodal_embeddings(self, **kwargs):
+        # Delegate to the active stage submodule when it implements MM encoding.
+        mm_fn = getattr(self.model, "get_multimodal_embeddings", None)
+        if mm_fn is not None:
+            return mm_fn(**kwargs)
+        return []
+
+    def embed_multimodal(self, **kwargs: object):
+        """vLLM V1 encoder profiling calls this; the inherited Protocol stub returns None."""
+        return self.get_multimodal_embeddings(**kwargs)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        sampling_metadata: SamplingMetadata | None = None,
+        logits_index: int | None = None,
+        sampler=None,
+        additional_information: dict[str, object] | None = None,
+        **kwargs: object,
+    ) -> torch.Tensor | IntermediateTensors | OmniOutput:
+        """
+        Forward pass for MiniCPM-o Omni model.
+
+        Workflow:
+        1) Thinker (model_stage="llm"): Image / video / audio encoders +
+           3D resampler + omni LLM → text + hidden states.
+        2) Talker (model_stage="tts"): MiniCPMTTS + the in-process
+           Token2Wav vocoder → audio waveform (final pipeline output).
+        """
+        if self.model_stage == "llm":
+            # Normalize to batched inputs if caller provides 1D/2D unbatched tensors
+            # TODO: Remove this hack when NPU supports batched inputs properly
+            added_batch_dim = False
+            if input_ids is not None and input_ids.ndim == 1:
+                input_ids = input_ids.unsqueeze(0)
+                added_batch_dim = True
+            if positions is not None and positions.ndim == 1:
+                positions = positions.unsqueeze(0)
+                added_batch_dim = True
+            if inputs_embeds is not None and inputs_embeds.ndim == 2:
+                inputs_embeds = inputs_embeds.unsqueeze(0)
+                added_batch_dim = True
+            thinker_dev = self._module_device(self.thinker)
+
+            # if input_ids is None, set it to a zero tensor
+            if input_ids is None:
+                input_ids = torch.zeros(inputs_embeds.shape[1], dtype=torch.long, device=thinker_dev).unsqueeze(0)
+                added_batch_dim = True
+
+            # Ensure inputs on thinker's device
+            if input_ids is not None and input_ids.device != thinker_dev:
+                input_ids = input_ids.to(thinker_dev)
+            if positions is not None and positions.device != thinker_dev:
+                positions = positions.to(thinker_dev)
+            if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
+                inputs_embeds = inputs_embeds.to(thinker_dev)
+
+            if current_omni_platform.is_npu():
+                # TODO: remove this hack when NPU supports batched inputs properly
+                thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
+                thinker_positions = positions[0] if positions.ndim > 1 else positions
+                thinker_inputs_embeds = (
+                    inputs_embeds[0] if inputs_embeds is not None and added_batch_dim else inputs_embeds
+                )
+            else:
+                thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
+                thinker_positions = positions[0] if positions is not None and added_batch_dim else positions
+                thinker_inputs_embeds = (
+                    inputs_embeds[0] if inputs_embeds is not None and added_batch_dim else inputs_embeds
+                )
+
+            # Run thinker
+            thinker_output = self.thinker(
+                input_ids=thinker_input_ids,
+                positions=thinker_positions,
+                intermediate_tensors=intermediate_tensors,
+                inputs_embeds=thinker_inputs_embeds,
+                **kwargs,
+            )
+
+            if isinstance(thinker_output, tuple):
+                embeds, text_hidden_states = thinker_output
+            else:
+                text_hidden_states = thinker_output
+
+            # Prepare hidden states for downstream stages
+            # Ensure correct shape: (batch_size, seq_len, hidden_dim)
+            if added_batch_dim:
+                text_hidden_states = text_hidden_states.squeeze(0)
+
+            # Return hidden states with latent in multimodal_outputs for stage_input_processors
+            return OmniOutput(
+                text_hidden_states=text_hidden_states,
+                multimodal_outputs={"latent": text_hidden_states},
+            )
+
+        # Talker stage: runs MiniCPMTTS + the in-process Token2Wav vocoder and
+        # emits the final audio waveform directly.
+        if self.model_stage == "tts":
+            if input_ids is not None:
+                num_tokens = input_ids.shape[0]
+                device = input_ids.device
+            elif inputs_embeds is not None:
+                num_tokens = inputs_embeds.shape[0]
+                device = inputs_embeds.device
+            else:
+                num_tokens = 1
+                device = torch.device("cuda")
+            hidden_dim = self.config.hidden_size if hasattr(self.config, "hidden_size") else 2560
+
+            # Profile/dummy run: both input_ids and inputs_embeds are None.
+            # Note: SupportsMultiModal preprocessing converts input_ids to
+            # inputs_embeds, so input_ids=None alone does NOT indicate a dummy run.
+            if input_ids is None and inputs_embeds is None:
+                dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
+                return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=None)
+
+            runtime_info = kwargs.get("runtime_additional_information")
+            talker_info = {}
+            if runtime_info and isinstance(runtime_info, list) and len(runtime_info) > 0:
+                talker_info = runtime_info[0] if isinstance(runtime_info[0], dict) else {}
+
+            with torch.inference_mode():
+                talker_result = self.talker(
+                    input_ids=input_ids,
+                    positions=positions,
+                    inputs_embeds=inputs_embeds,
+                    additional_information=talker_info,
+                )
+
+            dummy_hidden = torch.zeros(num_tokens, hidden_dim, device=device)
+
+            # Talker returns a (mel_spec, waveform_or_None) tuple. MiniCPM-o
+            # 4.5 emits only the waveform (mel_spec is always None); keep the
+            # 2-slot unpack so a future mel-emitting variant can plug in
+            # without changing the wrapper.
+            mm_out: dict = {}
+            if isinstance(talker_result, tuple) and len(talker_result) == 2:
+                _, waveform = talker_result
+                if waveform is not None:
+                    mm_out["model_outputs"] = [waveform]
+
+            return OmniOutput(
+                text_hidden_states=dummy_hidden,
+                multimodal_outputs=mm_out if mm_out else None,
+            )
+
+        raise ValueError(f"Unsupported model stage: {self.model_stage}")
+
+    def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
+        # Handle OmniOutput type
+        if isinstance(hidden_states, OmniOutput):
+            hidden_states = hidden_states.text_hidden_states
+
+        # Use model for logits computation
+        return self.model.compute_logits(hidden_states)
+
+    def sample(
+        self,
+        logits: torch.Tensor,
+        sampling_metadata: SamplingMetadata,
+    ) -> SamplerOutput | None:
+        # Use model for sampling
+        return self.model.sample(logits, sampling_metadata)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+        """Load weights for the active stage of the omni model."""
+        loaded_weights = set()
+        thinker_weights = []
+        talker_weights = []
+
+        # MiniCPM-o checkpoint prefixes → stage mapping:
+        #   thinker: vpm, resampler, llm, apm, audio_projection_layer
+        #   talker:  tts (MiniCPMTTS); the Token2Wav vocoder weights load
+        #            separately inside the talker module from the
+        #            ``assets/token2wav`` subdirectory and do not appear
+        #            in this iterator.
+        for k, v in weights:
+            if k.startswith(("vpm.", "resampler.", "llm.", "apm.", "audio_projection_layer.")):
+                thinker_weights.append((k, v))
+            elif k.startswith("tts."):
+                talker_weights.append((k, v))
+            else:
+                logger.warning("Unknown weight prefix: %s, skipping", k)
+
+        # Load thinker weights
+        if self.thinker is not None and thinker_weights:
+            thinker_loaded = self.thinker.load_weights(thinker_weights)
+            thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
+            loaded_weights.update(thinker_loaded)
+
+        # Load talker weights
+        if self.talker is not None and talker_weights:
+            talker_loaded = self.talker.load_weights(talker_weights)
+            talker_loaded = add_prefix_to_loaded_weights(talker_loaded, "talker")
+            loaded_weights.update(talker_loaded)
+
+        return loaded_weights
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py
new file mode 100644
index 000000000..bf6e9942f
--- /dev/null
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py
@@ -0,0 +1,4408 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from:
+# https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py
+#
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import os
+import warnings
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from functools import partial
+from typing import Annotated, Any, Literal, TypeAlias
+
+import numpy as np
+import PIL
+import PIL.Image
+import PIL.ImageSequence
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from PIL import Image
+from torch import Tensor
+from torch.nn.init import _calculate_fan_in_and_fan_out, trunc_normal_
+from transformers import AutoImageProcessor, PretrainedConfig
+from transformers.cache_utils import DynamicCache, EncoderDecoderCache
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.image_transforms import to_channel_dimension_format
+from transformers.image_utils import (
+    ChannelDimension,
+    infer_channel_dimension_format,
+    is_torch_tensor,
+    to_numpy_array,
+    valid_images,
+)
+from transformers.integrations import is_deepspeed_zero3_enabled
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.whisper.modeling_whisper import ACT2FN
+
+try:
+    from transformers.models.whisper.modeling_whisper import WHISPER_ATTENTION_CLASSES
+except ImportError:
+    from transformers.models.whisper.modeling_whisper import (
+        WhisperAttention,
+    )
+
+    WHISPER_ATTENTION_CLASSES = {
+        "eager": WhisperAttention,
+        "sdpa": WhisperAttention,  # fallback to eager
+    }
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+from transformers.models.whisper.modeling_whisper import WhisperConfig, WhisperEncoder
+from transformers.utils import (
+    TensorType,
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_torch_device,
+    is_torch_dtype,
+    logging,
+    replace_return_docstrings,
+    requires_backends,
+)
+from vllm.config import VllmConfig
+from vllm.config.multimodal import (
+    AudioDummyOptions,
+    BaseDummyOptions,
+    ImageDummyOptions,
+    VideoDummyOptions,
+)
+from vllm.logger import init_logger
+from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMRoPE, SupportsMultiModal, SupportsPP
+from vllm.model_executor.models.utils import (
+    flatten_bn,
+    init_vllm_registered_model,
+    maybe_prefix,
+)
+
+try:
+    from vllm.model_executor.models.utils import merge_multimodal_embeddings
+except ImportError:
+    from vllm.model_executor.models.utils import (
+        _merge_multimodal_embeddings as merge_multimodal_embeddings,
+    )
+try:
+    from vllm.utils import flatten_2d_lists
+except ImportError:
+
+    def flatten_2d_lists(nested):
+        """vLLM >=0.18 removed vllm.utils.flatten_2d_lists; keep MiniCPM-o vision path working."""
+        return [x for row in nested for x in row]
+
+
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+    ImageItem,
+    MultiModalFieldConfig,
+    MultiModalKwargsItems,
+    NestedTensors,
+)
+
+# vllm >=0.21 moved ModalityData / MultiModalDataDict from
+# vllm.multimodal.inputs to vllm.inputs (still re-exported from
+# vllm.multimodal.parse). Fall back to the old location for older vllm.
+try:
+    from vllm.inputs import ModalityData, MultiModalDataDict
+except ImportError:
+    from vllm.multimodal.inputs import (  # type: ignore[no-redef]
+        ModalityData,
+        MultiModalDataDict,
+    )
+from vllm.multimodal.parse import (
+    AudioItem,
+    AudioProcessorItems,
+    DictEmbeddingItems,
+    ImageProcessorItems,
+    ImageSize,
+    ModalityDataItems,
+    MultiModalDataItems,
+    MultiModalDataParser,
+    VideoItem,
+    VideoProcessorItems,
+)
+from vllm.multimodal.processing import (
+    BaseMultiModalProcessor,
+    BaseProcessingInfo,
+    PromptReplacement,
+    PromptUpdate,
+    PromptUpdateDetails,
+)
+
+try:
+    from vllm.multimodal.processing import BaseDummyInputsBuilder
+except ImportError:
+    from vllm.multimodal.profiling import BaseDummyInputsBuilder
+from vllm.sequence import IntermediateTensors
+
+
+def encode_tokens(tokenizer, prompt: str) -> list[int]:
+    """Tokenize ``prompt`` without adding special tokens.
+
+    Prefer vllm's ``vllm.transformers_utils.tokenizer.encode_tokens`` when
+    available; falls back to the equivalent ``tokenizer.encode`` call so we
+    keep working across vllm versions that do not ship that helper.
+
+    The import is performed lazily inside the function so the docs builder
+    (griffe / mkdocs strict mode) does not try to statically resolve the
+    upstream alias, which can fail across vllm releases that rename or move
+    the helper.
+    """
+    try:
+        from vllm.transformers_utils.tokenizer import encode_tokens as _impl
+    except ImportError:
+        return tokenizer.encode(prompt, add_special_tokens=False)
+    return _impl(tokenizer, prompt)
+
+
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+logger = init_logger(__name__)
+hf_logger = logging.get_logger(__name__)
+
+# Flash attention imports (optional)
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input
+
+
+# ============== Configuration Classes ==============
+
+
+class SiglipVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
+    Siglip vision encoder according to the specified arguments, defining the model architecture.
+    """
+
+    model_type = "siglip_vision_model"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        image_size=224,
+        patch_size=16,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        # get the vision config dict if we are loading from SiglipConfig
+        if config_dict.get("model_type") == "siglip":
+            config_dict = config_dict["vision_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            hf_logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class MiniCPMVSliceConfig(PretrainedConfig):
+    model_type = "minicpmv"
+
+    def __init__(
+        self,
+        patch_size=14,
+        max_slice_nums=9,
+        scale_resolution=448,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.patch_size = patch_size
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike, **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        if config_dict.get("model_type") == "minicpmv":
+            config_dict = config_dict["slice_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            hf_logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class ConditionalChatTTSConfig(PretrainedConfig):
+    model_type = "conditional_chattts"
+
+    def __init__(
+        self,
+        llm_dim: int = 2560,
+        hidden_size: int = 768,
+        intermediate_size: int = 3072,
+        num_attention_heads: int = 12,
+        num_hidden_layers: int = 20,
+        max_position_embeddings: int = 4096,
+        num_audio_tokens: int = 626,
+        num_text_tokens: int = 21178,
+        num_mel_bins: int = 100,
+        num_vq: int = 4,
+        use_speaker_embedding: bool = True,
+        use_llm_hidden_state: bool = False,
+        spk_emb_token_id: int = 21143,
+        num_spk_embs: int = 1,
+        audio_bos_token_id: int = 21132,
+        text_eos_token_id: int = 21133,
+        use_text: bool = True,
+        streaming: bool = True,
+        streaming_text_chunk_size: int = 10,
+        streaming_text_reserved_len: int = 300,
+        streaming_audio_chunk_size: int = 50,
+        attn_implementation: str = "sdpa",
+        use_mlp: bool = True,
+        aug_loss_weight: bool = True,
+        do_sample: bool = True,
+        top_p: float = 0.7,
+        top_k: int = 20,
+        repetition_penalty: float = 1.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.llm_dim = llm_dim
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.max_position_embeddings = max_position_embeddings
+        self.num_audio_tokens = num_audio_tokens
+        self.num_text_tokens = num_text_tokens
+        self.num_mel_bins = num_mel_bins
+        self.num_vq = num_vq
+        self.use_speaker_embedding = use_speaker_embedding
+        self.use_llm_hidden_state = use_llm_hidden_state
+        self.spk_emb_token_id = spk_emb_token_id
+        self.num_spk_embs = num_spk_embs
+        self.audio_bos_token_id = audio_bos_token_id
+        self.text_eos_token_id = text_eos_token_id
+        self.use_text = use_text
+        self.streaming = streaming
+        self.streaming_text_chunk_size = streaming_text_chunk_size
+        self.streaming_text_reserved_len = streaming_text_reserved_len
+        self.streaming_audio_chunk_size = streaming_audio_chunk_size
+        self.attn_implementation = attn_implementation
+        self.use_mlp = use_mlp
+        self.aug_loss_weight = aug_loss_weight
+        self.do_sample = do_sample
+        self.top_p = top_p
+        self.top_k = top_k
+        self.repetition_penalty = repetition_penalty
+
+
+class MiniCPMOConfig(Qwen2Config):
+    model_type = "minicpmo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    default_vision_config = {
+        "hidden_size": 1152,
+        "image_size": 980,
+        "intermediate_size": 4304,
+        "model_type": "siglip",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 27,
+        "patch_size": 14,
+    }
+
+    def __init__(
+        self,
+        use_cache=True,
+        query_num=64,
+        image_size=448,
+        drop_vision_last_layer=True,
+        batch_vision_input=True,
+        slice_config=None,
+        vision_config=None,
+        audio_config=None,
+        tts_config=None,
+        use_image_id=True,
+        vision_batch_size=16,
+        audio_pool_step=2,
+        audio_chunk_length=1.0,
+        stream_input=False,
+        init_vision=True,
+        init_audio=True,
+        init_tts=True,
+        **kwargs,
+    ):
+        self.use_cache = use_cache
+        self.query_num = query_num
+        self.image_size = image_size
+        self.drop_vision_last_layer = drop_vision_last_layer
+        self.batch_vision_input = batch_vision_input
+        self.use_image_id = use_image_id
+        self.vision_batch_size = vision_batch_size
+        self.audio_pool_step = audio_pool_step
+        self.audio_chunk_length = audio_chunk_length
+        self.stream_input = stream_input
+        self.init_vision = init_vision
+        self.init_audio = init_audio
+        self.init_tts = init_tts
+
+        if slice_config is None:
+            self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
+        else:
+            self.slice_config = MiniCPMVSliceConfig(**slice_config)
+        self.slice_mode = True
+
+        # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
+        if vision_config is None:
+            self.vision_config = SiglipVisionConfig(**self.default_vision_config)
+            hf_logger.info("vision_config is None, using default vision config")
+        elif isinstance(vision_config, dict):
+            self.vision_config = SiglipVisionConfig(**vision_config)
+        elif isinstance(vision_config, SiglipVisionConfig):
+            self.vision_config = vision_config
+
+        # same as openai/whisper-medium add use_cache
+        if audio_config is None:
+            self.audio_config = WhisperConfig()
+        elif isinstance(audio_config, dict):
+            self.audio_config = WhisperConfig(**audio_config)
+        elif isinstance(audio_config, WhisperConfig):
+            self.audio_config = audio_config
+
+        if tts_config is None:
+            self.tts_config = ConditionalChatTTSConfig()
+        elif isinstance(tts_config, dict):
+            tts_model_type = tts_config.get("model_type", "conditional_chattts")
+            if tts_model_type == "conditional_chattts":
+                self.tts_config = ConditionalChatTTSConfig(**tts_config)
+            else:
+                self.tts_config = PretrainedConfig(**tts_config)
+        elif isinstance(tts_config, ConditionalChatTTSConfig):
+            self.tts_config = tts_config
+        else:
+            self.tts_config = tts_config
+
+        self.patch_size = self.vision_config.patch_size
+
+        super().__init__(**kwargs)
+
+
+# ============== Image Processing Classes ==============
+
+
+def recursive_converter(converter, value):
+    if isinstance(value, list):
+        new_value = []
+        for v in value:
+            new_value += [recursive_converter(converter, v)]
+        return new_value
+    else:
+        return converter(value)
+
+
+class MiniCPMOBatchFeature(BatchFeature):
+    r"""
+    Extend from BatchFeature for supporting various image size
+    """
+
+    def __init__(self, data: dict[str, Any] | None = None, tensor_type: None | str | TensorType = None):
+        super().__init__(data)
+        self.convert_to_tensors(tensor_type=tensor_type)
+
+    def convert_to_tensors(self, tensor_type: str | TensorType | None = None):
+        if tensor_type is None:
+            return self
+
+        is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)
+
+        def converter(value):
+            try:
+                if not is_tensor(value):
+                    tensor = as_tensor(value)
+                    return tensor
+            except:  # noqa E722
+                if key == "overflowing_values":
+                    raise ValueError("Unable to create tensor returning overflowing values of different lengths. ")
+                raise ValueError(
+                    "Unable to create tensor, you should probably activate padding "
+                    "with 'padding=True' to have batched tensors with the same length."
+                )
+
+        for key, value in self.items():
+            self[key] = recursive_converter(converter, value)
+        return self
+
+    def to(self, *args, **kwargs) -> "MiniCPMOBatchFeature":
+        requires_backends(self, ["torch"])
+        import torch
+
+        def cast_tensor(v):
+            # check if v is a floating point
+            if torch.is_floating_point(v):
+                # cast and send to device
+                return v.to(*args, **kwargs)
+            elif device is not None:
+                return v.to(device=device)
+            else:
+                return v
+
+        new_data = {}
+        device = kwargs.get("device")
+        # Check if the args are a device or a dtype
+        if device is None and len(args) > 0:
+            # device should be always the first argument
+            arg = args[0]
+            if is_torch_dtype(arg):
+                # The first argument is a dtype
+                pass
+            elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
+                device = arg
+            else:
+                # it's something else
+                raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
+        # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
+        for k, v in self.items():
+            new_data[k] = recursive_converter(cast_tensor, v)
+        self.data = new_data
+        return self
+
+
+class MiniCPMVImageProcessor(BaseImageProcessor):
+    model_input_names = ["pixel_values"]
+
+    def __init__(self, max_slice_nums=9, scale_resolution=448, patch_size=14, **kwargs):
+        super().__init__(**kwargs)
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+        self.patch_size = patch_size
+        self.use_image_id = kwargs.pop("use_image_id", False)
+        self.image_feature_size = kwargs.pop("image_feature_size", 64)
+        self.im_start_token = kwargs.pop("im_start", "<image>")
+        self.im_end_token = kwargs.pop("im_end", "</image>")
+        self.slice_start_token = kwargs.pop("slice_start", "<slice>")
+        self.slice_end_token = kwargs.pop("slice_end", "</slice>")
+        self.unk_token = kwargs.pop("unk", "<unk>")
+        self.im_id_start = kwargs.pop("im_id_start", "<image_id>")
+        self.im_id_end = kwargs.pop("im_id_end", "</image_id>")
+        self.slice_mode = kwargs.pop("slice_mode", True)
+
+        self.mean = np.array(kwargs.pop("norm_mean", [0.5, 0.5, 0.5]))
+        self.std = np.array(kwargs.pop("norm_std", [0.5, 0.5, 0.5]))
+        self.version = kwargs.pop("version", 2.0)
+
+    def ensure_divide(self, length, patch_size):
+        return max(round(length / patch_size) * patch_size, patch_size)
+
+    def find_best_resize(self, original_size, scale_resolution, patch_size, allow_upscale=False):
+        width, height = original_size
+        if (width * height > scale_resolution * scale_resolution) or allow_upscale:
+            r = width / height
+            height = int(scale_resolution / math.sqrt(r))
+            width = int(height * r)
+        best_width = self.ensure_divide(width, patch_size)
+        best_height = self.ensure_divide(height, patch_size)
+        return (best_width, best_height)
+
+    def get_refine_size(self, original_size, grid, scale_resolution, patch_size, allow_upscale=False):
+        width, height = original_size
+        grid_x, grid_y = grid
+
+        refine_width = self.ensure_divide(width, grid_x)
+        refine_height = self.ensure_divide(height, grid_y)
+
+        grid_width = refine_width / grid_x
+        grid_height = refine_height / grid_y
+
+        best_grid_size = self.find_best_resize(
+            (grid_width, grid_height), scale_resolution, patch_size, allow_upscale=allow_upscale
+        )
+        refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)
+        return refine_size
+
+    def split_to_patches(self, image, grid):
+        patches = []
+        width, height = image.size
+        grid_x = int(width / grid[0])
+        grid_y = int(height / grid[1])
+        for i in range(0, height, grid_y):
+            images = []
+            for j in range(0, width, grid_x):
+                box = (j, i, j + grid_x, i + grid_y)
+                patch = image.crop(box)
+                images.append(patch)
+            patches.append(images)
+        return patches
+
+    def slice_image(self, image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False):
+        original_size = image.size
+        source_image = None
+        best_grid = self.get_sliced_grid(original_size, max_slice_nums, never_split)
+        patches = []
+
+        if best_grid is None:
+            # dont need to slice, upsample
+            best_size = self.find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=True)
+            source_image = image.resize(best_size, resample=Image.Resampling.BICUBIC)
+        else:
+            # source image, down-sampling and ensure divided by patch_size
+            best_resize = self.find_best_resize(original_size, scale_resolution, patch_size)
+            source_image = image.copy().resize(best_resize, resample=Image.Resampling.BICUBIC)
+            refine_size = self.get_refine_size(
+                original_size, best_grid, scale_resolution, patch_size, allow_upscale=True
+            )
+            refine_image = image.resize(refine_size, resample=Image.Resampling.BICUBIC)
+            patches = self.split_to_patches(refine_image, best_grid)
+
+        return source_image, patches, best_grid
+
+    def get_grid_placeholder(self, grid):
+        if grid is None:
+            return ""
+        slice_image_placeholder = (
+            self.slice_start_token + self.unk_token * self.image_feature_size + self.slice_end_token
+        )
+
+        cols = grid[0]
+        rows = grid[1]
+        slices = []
+        for i in range(rows):
+            lines = []
+            for j in range(cols):
+                lines.append(slice_image_placeholder)
+            slices.append("".join(lines))
+
+        slice_placeholder = "\n".join(slices)
+        return slice_placeholder
+
+    def get_image_id_placeholder(self, idx=0):
+        return f"{self.im_id_start}{idx}{self.im_id_end}"
+
+    def get_sliced_images(self, image, max_slice_nums=None):
+        slice_images = []
+
+        if not self.slice_mode:
+            return [image]
+
+        max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)
+        assert max_slice_nums > 0
+        source_image, patches, sliced_grid = self.slice_image(
+            image,
+            max_slice_nums,
+            self.scale_resolution,
+            self.patch_size,  # default: 9  # default: 448  # default: 14
+        )
+
+        slice_images.append(source_image)
+        if len(patches) > 0:
+            for i in range(len(patches)):
+                for j in range(len(patches[0])):
+                    slice_images.append(patches[i][j])
+        return slice_images
+
+    def get_sliced_grid(self, image_size, max_slice_nums, never_split=False):
+        original_width, original_height = image_size
+        log_ratio = math.log(original_width / original_height)
+        ratio = original_width * original_height / (self.scale_resolution * self.scale_resolution)
+        multiple = min(math.ceil(ratio), max_slice_nums)
+        if multiple <= 1 or never_split:
+            return None
+        candidate_split_grids_nums = []
+        for i in [multiple - 1, multiple, multiple + 1]:
+            if i == 1 or i > max_slice_nums:
+                continue
+            candidate_split_grids_nums.append(i)
+
+        candidate_grids = []
+        for split_grids_nums in candidate_split_grids_nums:
+            m = 1
+            while m <= split_grids_nums:
+                if split_grids_nums % m == 0:
+                    candidate_grids.append([m, split_grids_nums // m])
+                m += 1
+
+        best_grid = [1, 1]
+        min_error = float("inf")
+        for grid in candidate_grids:
+            error = abs(log_ratio - math.log(grid[0] / grid[1]))
+            if error < min_error:
+                best_grid = grid
+                min_error = error
+
+        return best_grid
+
+    def get_slice_image_placeholder(self, image_size, image_idx=0, max_slice_nums=None, use_image_id=None):
+        max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)
+        assert max_slice_nums > 0
+        grid = self.get_sliced_grid(image_size=image_size, max_slice_nums=max_slice_nums)
+
+        image_placeholder = self.im_start_token + self.unk_token * self.image_feature_size + self.im_end_token
+        use_image_id = self.use_image_id if use_image_id is None else bool(use_image_id)
+        if use_image_id:
+            final_placeholder = self.get_image_id_placeholder(image_idx) + image_placeholder
+        else:
+            final_placeholder = image_placeholder
+
+        if self.slice_mode:
+            final_placeholder = final_placeholder + self.get_grid_placeholder(grid=grid)
+        return final_placeholder
+
+    def to_pil_image(self, image, rescale=None) -> PIL.Image.Image:
+        """
+        Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if
+        needed.
+
+        Args:
+            image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`):
+                The image to convert to the PIL Image format.
+            rescale (`bool`, *optional*):
+                Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will
+                default to `True` if the image type is a floating type, `False` otherwise.
+        """
+        if isinstance(image, PIL.Image.Image):
+            return image
+        if is_torch_tensor(image):
+            image = image.numpy()
+
+        if isinstance(image, np.ndarray):
+            if rescale is None:
+                # rescale default to the array being of floating type.
+                rescale = isinstance(image.flat[0], np.floating)
+            # If the channel as been moved to first dim, we put it back at the end.
+            if image.ndim == 3 and image.shape[0] in [1, 3]:
+                image = image.transpose(1, 2, 0)
+            if rescale:
+                image = image * 255
+            image = image.astype(np.uint8)
+            return PIL.Image.fromarray(image)
+        return image
+
+    def reshape_by_patch(self, image):
+        """
+        :param image: shape [3, H, W]
+        :param patch_size:
+        :return: [3, patch_size, HW/patch_size]
+        """
+        image = torch.from_numpy(image)
+        patch_size = self.patch_size
+        patches = torch.nn.functional.unfold(image, (patch_size, patch_size), stride=(patch_size, patch_size))
+
+        patches = patches.reshape(image.size(0), patch_size, patch_size, -1)
+        patches = patches.permute(0, 1, 3, 2).reshape(image.size(0), patch_size, -1)
+        return patches.numpy()
+
+    def preprocess(
+        self,
+        images: Image.Image | list[Image.Image] | list[list[Image.Image]],
+        do_pad: bool | None = True,
+        max_slice_nums: int = None,
+        return_tensors: str | TensorType | None = None,
+        **kwargs,
+    ) -> MiniCPMOBatchFeature:
+        if isinstance(images, Image.Image):
+            images_list = [[images]]
+        elif isinstance(images[0], Image.Image):
+            images_list = [images]
+        else:
+            images_list = images
+
+        new_images_list = []
+        image_sizes_list = []
+        tgt_sizes_list = []
+
+        for _images in images_list:
+            if _images is None or len(_images) == 0:
+                new_images_list.append([])
+                image_sizes_list.append([])
+                tgt_sizes_list.append([])
+                continue
+            if not valid_images(_images):
+                raise ValueError(
+                    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                    "torch.Tensor, tf.Tensor or jax.ndarray."
+                )
+
+            _images = [self.to_pil_image(image).convert("RGB") for image in _images]
+            input_data_format = infer_channel_dimension_format(np.array(_images[0]))
+
+            new_images = []
+            image_sizes = [image.size for image in _images]
+            tgt_sizes = []
+            for image in _images:
+                image_patches = self.get_sliced_images(image, max_slice_nums)
+                image_patches = [to_numpy_array(image).astype(np.float32) / 255 for image in image_patches]
+                image_patches = [
+                    self.normalize(image=image, mean=self.mean, std=self.std, input_data_format=input_data_format)
+                    for image in image_patches
+                ]
+                image_patches = [
+                    to_channel_dimension_format(image, ChannelDimension.FIRST, input_channel_dim=input_data_format)
+                    for image in image_patches
+                ]
+                for slice_image in image_patches:
+                    new_images.append(self.reshape_by_patch(slice_image))
+                    tgt_sizes.append(
+                        np.array((slice_image.shape[1] // self.patch_size, slice_image.shape[2] // self.patch_size))
+                    )
+
+            if tgt_sizes:
+                tgt_sizes = np.vstack(tgt_sizes)
+
+            new_images_list.append(new_images)
+            image_sizes_list.append(image_sizes)
+            tgt_sizes_list.append(tgt_sizes)
+        return MiniCPMOBatchFeature(
+            data={"pixel_values": new_images_list, "image_sizes": image_sizes_list, "tgt_sizes": tgt_sizes_list},
+            tensor_type=return_tensors,
+        )
+
+
+AutoImageProcessor.register("MiniCPMVImageProcessor", MiniCPMVImageProcessor)
+
+
+# ============== SigLIP Vision Transformer Classes ==============
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+            "The distribution of values may be incorrect.",
+            stacklevel=2,
+        )
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)  # noqa: E741
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    if tensor.dtype in [torch.float16, torch.bfloat16]:
+        # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
+        og_dtype = tensor.dtype
+        tensor = tensor.to(torch.float32)
+        tensor.erfinv_()
+        tensor = tensor.to(og_dtype)
+    else:
+        tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.0))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    if tensor.dtype == torch.float16:
+        # The `clamp_` op is not (yet?) defined in float16+cpu
+        tensor = tensor.to(torch.float32)
+        tensor.clamp_(min=a, max=b)
+        tensor = tensor.to(torch.float16)
+    else:
+        tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+    tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
+) -> torch.Tensor:
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \\leq \text{mean} \\leq b`.
+    NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+    bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+    and the result is subsequently scaled and shifted by the mean and std args.
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        _trunc_normal_(tensor, 0, 1.0, a, b)
+        tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == "fan_in":
+        denom = fan_in
+    elif mode == "fan_out":
+        denom = fan_out
+    elif mode == "fan_avg":
+        denom = (fan_in + fan_out) / 2
+
+    variance = scale / denom
+
+    if distribution == "truncated_normal":
+        # constant is stddev of standard normal truncated to (-2, 2)
+        trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+    elif distribution == "normal":
+        with torch.no_grad():
+            tensor.normal_(std=math.sqrt(variance))
+    elif distribution == "uniform":
+        bound = math.sqrt(3 * variance)
+        with torch.no_grad():
+            tensor.uniform_(-bound, bound)
+    else:
+        raise ValueError(f"invalid distribution {distribution}")
+
+
+def lecun_normal_(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+def default_flax_embed_init(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="normal")
+
+
+@dataclass
+# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->Siglip
+class SiglipVisionModelOutput(ModelOutput):
+    """
+    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
+    Args:
+        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
+            The image embeddings obtained by applying the projection layer to the pooler_output.
+        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+    """
+
+    image_embeds: torch.FloatTensor | None = None
+    last_hidden_state: torch.FloatTensor = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+class SiglipVisionEmbeddings(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+        )
+
+        self.num_patches_per_side = self.image_size // self.patch_size
+        self.num_patches = self.num_patches_per_side**2
+        self.num_positions = self.num_patches
+        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        patch_attention_mask: torch.BoolTensor,
+        tgt_sizes: torch.IntTensor | None = None,
+    ) -> torch.Tensor:
+        batch_size = pixel_values.size(0)
+
+        patch_embeds = self.patch_embedding(pixel_values)
+        embeddings = patch_embeds.flatten(2).transpose(1, 2)
+
+        max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
+        max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, max_im_w // self.patch_size
+        boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side)
+        position_ids = torch.full(
+            size=(
+                batch_size,
+                max_nb_patches_h * max_nb_patches_w,
+            ),
+            fill_value=0,
+        )
+
+        for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
+            if tgt_sizes is not None:
+                nb_patches_h = tgt_sizes[batch_idx][0]
+                nb_patches_w = tgt_sizes[batch_idx][1]
+            else:
+                nb_patches_h = p_attn_mask[:, 0].sum()
+                nb_patches_w = p_attn_mask[0].sum()
+
+            fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
+            fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
+
+            bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True)
+            bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True)
+
+            pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten()
+            position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
+
+        position_ids = position_ids.to(self.position_embedding.weight.device)
+
+        embeddings = embeddings + self.position_embedding(position_ids)
+        return embeddings
+
+
+class SiglipAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+        self.dropout = config.attention_dropout
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = False,
+    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+        """Input shape: Batch x Time x Channel"""
+
+        batch_size, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        k_v_seq_len = key_states.shape[-2]
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
+
+        if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class SiglipFlashAttention2(SiglipAttention):
+    """
+    Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_causal = False  # Hack to make sure we don't use a causal mask
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_value: tuple[torch.Tensor] | None = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        # cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        # if past_key_value is not None:
+        #     cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+        #     key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = self.dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (LlamaRMSNorm handles it correctly)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                "The input hidden states seems to be silently casted in float32, this might be related to the fact"
+                " you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+    def _flash_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+
+        # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+        causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
+class SiglipMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
+class SiglipEncoderLayer(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+        self.self_attn = SiglipAttention(config) if not self._use_flash_attention_2 else SiglipFlashAttention2(config)
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = SiglipMLP(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        output_attentions: bool | None = False,
+    ) -> tuple[torch.FloatTensor]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                Input to the layer of shape `(batch, seq_len, embed_dim)`.
+            attention_mask (`torch.FloatTensor`):
+                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*, defaults to `False`):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states, attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class SiglipPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = SiglipVisionConfig
+    base_model_prefix = "siglip"
+    supports_gradient_checkpointing = True
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+
+        if isinstance(module, SiglipVisionEmbeddings):
+            width = self.config.hidden_size
+            nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
+        elif isinstance(module, nn.Embedding):
+            default_flax_embed_init(module.weight)
+        elif isinstance(module, SiglipAttention):
+            nn.init.normal_(module.q_proj.weight)
+            nn.init.normal_(module.k_proj.weight)
+            nn.init.normal_(module.v_proj.weight)
+            nn.init.normal_(module.out_proj.weight)
+            nn.init.zeros_(module.q_proj.bias)
+            nn.init.zeros_(module.k_proj.bias)
+            nn.init.zeros_(module.v_proj.bias)
+            nn.init.zeros_(module.out_proj.bias)
+        elif isinstance(module, SiglipMLP):
+            nn.init.normal_(module.fc1.weight)
+            nn.init.normal_(module.fc2.weight)
+            nn.init.normal_(module.fc1.bias, std=1e-6)
+            nn.init.normal_(module.fc2.bias, std=1e-6)
+        elif isinstance(module, nn.Linear | nn.Conv2d):
+            lecun_normal_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+SIGLIP_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
+            Initializing with a config file does not load the weights associated with the model, only the
+            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+SIGLIP_VISION_INPUTS_DOCSTRING = r"""
+    Args:
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
+class SiglipEncoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`SiglipEncoderLayer`].
+    Args:
+        config: SiglipConfig
+    """
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    # Ignore copy
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+    ) -> tuple | BaseModelOutput:
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+                than the model's internal embedding lookup matrix.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+                - 1 for tokens that are **not masked**,
+                - 0 for tokens that are **masked**.
+                [What are attention masks?](../glossary#attention-mask)
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    encoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)
+
+
+@add_start_docstrings("""The vision model from SigLIP without any head or projection on top.""", SIGLIP_START_DOCSTRING)
+class SiglipVisionTransformer(SiglipPreTrainedModel):
+    config_class = SiglipVisionConfig
+    main_input_name = "pixel_values"
+    _supports_flash_attn_2 = True
+    _no_split_modules = []
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__(config)
+        self.config = config
+        embed_dim = config.hidden_size
+
+        self.embeddings = SiglipVisionEmbeddings(config)
+        self.encoder = SiglipEncoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.embeddings.patch_embedding
+
+    @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig)
+    def forward(
+        self,
+        pixel_values,
+        patch_attention_mask: torch.BoolTensor | None = None,
+        tgt_sizes: torch.IntTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        Returns:
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        batch_size = pixel_values.size(0)
+        if patch_attention_mask is None:
+            patch_attention_mask = torch.ones(
+                size=(
+                    batch_size,
+                    pixel_values.size(2) // self.config.patch_size,
+                    pixel_values.size(3) // self.config.patch_size,
+                ),
+                dtype=torch.bool,
+                device=pixel_values.device,
+            )
+
+        hidden_states = self.embeddings(
+            pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, tgt_sizes=tgt_sizes
+        )
+
+        patch_attention_mask = patch_attention_mask.view(batch_size, -1)
+        # The call to `_upad_input` in `_flash_attention_forward` is expensive
+        # So when the `patch_attention_mask` is full of 1s (i.e. attending to the whole sequence),
+        # avoiding passing the attention_mask, which is equivalent to attending to the full sequence
+        if not torch.any(~patch_attention_mask):
+            attention_mask = None
+        else:
+            attention_mask = (
+                _prepare_4d_attention_mask(patch_attention_mask, hidden_states.dtype)
+                if not self._use_flash_attention_2
+                else patch_attention_mask
+            )
+
+        encoder_outputs = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        last_hidden_state = encoder_outputs[0]
+        last_hidden_state = self.post_layernorm(last_hidden_state)
+
+        if not return_dict:
+            return (last_hidden_state, None) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=None,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+# ============== Resampler Classes ==============
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+from torch import nn
+from torch.nn.functional import *
+from torch.nn.modules.activation import *
+
+
+def get_2d_sincos_pos_embed(embed_dim, image_size):
+    """
+    image_size: image_size or (image_height, image_width)
+    return:
+    pos_embed: [image_height, image_width, embed_dim]
+    """
+    if isinstance(image_size, int):
+        grid_h_size, grid_w_size = image_size, image_size
+    else:
+        grid_h_size, grid_w_size = image_size[0], image_size[1]
+
+    grid_h = np.arange(grid_h_size, dtype=np.float32)
+    grid_w = np.arange(grid_w_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[0])  # (H, W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[1])  # (H, W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=-1)  # (H, W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid_new(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (H, W)
+    out: (H, W, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / 10000**omega  # (D/2,)
+
+    out = np.einsum("hw,d->hwd", pos, omega)  # (H, W, D/2), outer product
+
+    emb_sin = np.sin(out)  # (H, W, D/2)
+    emb_cos = np.cos(out)  # (H, W, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=-1)  # (H, W, D)
+    return emb
+
+
+class Resampler(nn.Module):
+    """
+    A 2D perceiver-resampler network with one cross attention layers by
+       given learnable queries and 2d sincos pos_emb
+    Outputs:
+        A tensor with the shape of (batch_size, num_queries, embed_dim)
+    """
+
+    def __init__(
+        self,
+        num_queries,
+        embed_dim,
+        num_heads,
+        kv_dim=None,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        adaptive=False,
+        max_size=(70, 70),
+    ):
+        super().__init__()
+        self.num_queries = num_queries
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.adaptive = adaptive
+        self.max_size = max_size
+
+        self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
+
+        if kv_dim is not None and kv_dim != embed_dim:
+            self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
+        else:
+            self.kv_proj = nn.Identity()
+
+        self.attn = MultiheadAttention(embed_dim, num_heads)
+        self.ln_q = norm_layer(embed_dim)
+        self.ln_kv = norm_layer(embed_dim)
+
+        self.ln_post = norm_layer(embed_dim)
+        self.proj = nn.Parameter((embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
+
+        self._set_2d_pos_cache(self.max_size)
+
+    def _set_2d_pos_cache(self, max_size, device="cpu"):
+        if is_deepspeed_zero3_enabled():
+            device = "cuda"
+        pos_embed = torch.from_numpy(get_2d_sincos_pos_embed(self.embed_dim, max_size)).float().to(device)
+        self.register_buffer("pos_embed", pos_embed, persistent=False)
+
+    def _adjust_pos_cache(self, tgt_sizes, device):
+        max_h = torch.max(tgt_sizes[:, 0])
+        max_w = torch.max(tgt_sizes[:, 1])
+        if max_h > self.max_size[0] or max_w > self.max_size[1]:
+            self.max_size = [max(max_h, self.max_size[0]), max(max_w, self.max_size[1])]
+            self._set_2d_pos_cache(self.max_size, device)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def forward(self, x, tgt_sizes=None):
+        assert x.shape[0] == tgt_sizes.shape[0]
+        bs = x.shape[0]
+
+        device = x.device
+        dtype = x.dtype
+
+        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]
+
+        self._adjust_pos_cache(tgt_sizes, device=device)
+
+        if self.pos_embed.device != device:
+            self.pos_embed = self.pos_embed.to(device)
+
+        max_patch_len = torch.max(patch_len)
+        key_padding_mask = torch.zeros((bs, max_patch_len), dtype=torch.bool, device=device)
+
+        pos_embed = []
+        for i in range(bs):
+            tgt_h, tgt_w = tgt_sizes[i]
+            pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype))  # patches * D
+            key_padding_mask[i, patch_len[i] :] = True
+
+        pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed, batch_first=True, padding_value=0.0).permute(
+            1, 0, 2
+        )  # BLD => L * B * D
+
+        x = self.kv_proj(x)  # B * L * D
+        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D
+
+        q = self.ln_q(self.query)  # Q * D
+
+        out = self.attn(
+            self._repeat(q, bs),  # Q * B * D
+            x + pos_embed,  # L * B * D +  L * B * D
+            x,
+            key_padding_mask=key_padding_mask,
+        )[0]
+        #  out: Q * B * D
+        x = out.permute(1, 0, 2)  # B * Q * D
+
+        x = self.ln_post(x)
+        x = x @ self.proj
+        return x
+
+    def _repeat(self, query, N: int):
+        return query.unsqueeze(1).repeat(1, N, 1)
+
+
+class MultiheadAttention(nn.MultiheadAttention):
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        bias=True,
+        add_bias_kv=False,
+        add_zero_attn=False,
+        kdim=None,
+        vdim=None,
+        batch_first=False,
+        device=None,
+        dtype=None,
+    ):
+        super().__init__(
+            embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype
+        )
+
+        # rewrite out_proj layer，with nn.Linear
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+
+    def forward(
+        self,
+        query: Tensor,
+        key: Tensor,
+        value: Tensor,
+        key_padding_mask: Tensor | None = None,
+        need_weights: bool = True,
+        attn_mask: Tensor | None = None,
+        average_attn_weights: bool = True,
+        is_causal: bool = False,
+    ) -> tuple[Tensor, Tensor | None]:
+        why_not_fast_path = ""
+        if (
+            (attn_mask is not None and torch.is_floating_point(attn_mask))
+            or (key_padding_mask is not None)
+            and torch.is_floating_point(key_padding_mask)
+        ):
+            why_not_fast_path = "floating-point masks are not supported for fast path."
+
+        is_batched = query.dim() == 3
+
+        key_padding_mask = _canonical_mask(
+            mask=key_padding_mask,
+            mask_name="key_padding_mask",
+            other_type=F._none_or_dtype(attn_mask),
+            other_name="attn_mask",
+            target_type=query.dtype,
+        )
+
+        attn_mask = _canonical_mask(
+            mask=attn_mask,
+            mask_name="attn_mask",
+            other_type=None,
+            other_name="",
+            target_type=query.dtype,
+            check_other=False,
+        )
+
+        if not is_batched:
+            why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}"
+        elif query is not key or key is not value:
+            # When lifting this restriction, don't forget to either
+            # enforce that the dtypes all match or test cases where
+            # they don't!
+            why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)"
+        elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype:
+            why_not_fast_path = (
+                f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match"
+            )
+        elif self.in_proj_weight is None:
+            why_not_fast_path = "in_proj_weight was None"
+        elif query.dtype != self.in_proj_weight.dtype:
+            # this case will fail anyway, but at least they'll get a useful error message.
+            why_not_fast_path = (
+                f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match"
+            )
+        elif self.training:
+            why_not_fast_path = "training is enabled"
+        elif (self.num_heads % 2) != 0:
+            why_not_fast_path = "self.num_heads is not even"
+        elif not self.batch_first:
+            why_not_fast_path = "batch_first was not True"
+        elif self.bias_k is not None:
+            why_not_fast_path = "self.bias_k was not None"
+        elif self.bias_v is not None:
+            why_not_fast_path = "self.bias_v was not None"
+        elif self.add_zero_attn:
+            why_not_fast_path = "add_zero_attn was enabled"
+        elif not self._qkv_same_embed_dim:
+            why_not_fast_path = "_qkv_same_embed_dim was not True"
+        elif query.is_nested and (key_padding_mask is not None or attn_mask is not None):
+            why_not_fast_path = "supplying both src_key_padding_mask and src_mask at the same time \
+                                 is not supported with NestedTensor input"
+        elif torch.is_autocast_enabled():
+            why_not_fast_path = "autocast is enabled"
+
+        if not why_not_fast_path:
+            tensor_args = (
+                query,
+                key,
+                value,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.out_proj.weight,
+                self.out_proj.bias,
+            )
+            # We have to use list comprehensions below because TorchScript does not support
+            # generator expressions.
+            if torch.overrides.has_torch_function(tensor_args):
+                why_not_fast_path = "some Tensor argument has_torch_function"
+            elif _is_make_fx_tracing():
+                why_not_fast_path = "we are running make_fx tracing"
+            elif not all(_check_arg_device(x) for x in tensor_args):
+                why_not_fast_path = (
+                    "some Tensor argument's device is neither one of "
+                    f"cpu, cuda or {torch.utils.backend_registration._privateuse1_backend_name}"
+                )
+            elif torch.is_grad_enabled() and any(_arg_requires_grad(x) for x in tensor_args):
+                why_not_fast_path = (
+                    "grad is enabled and at least one of query or the "
+                    "input/output projection weights or biases requires_grad"
+                )
+            if not why_not_fast_path:
+                merged_mask, mask_type = self.merge_masks(attn_mask, key_padding_mask, query)
+
+                if self.in_proj_bias is not None and self.in_proj_weight is not None:
+                    return torch._native_multi_head_attention(
+                        query,
+                        key,
+                        value,
+                        self.embed_dim,
+                        self.num_heads,
+                        self.in_proj_weight,
+                        self.in_proj_bias,
+                        self.out_proj.weight,
+                        self.out_proj.bias,
+                        merged_mask,
+                        need_weights,
+                        average_attn_weights,
+                        mask_type,
+                    )
+
+        any_nested = query.is_nested or key.is_nested or value.is_nested
+        assert not any_nested, (
+            "MultiheadAttention does not support NestedTensor outside of its fast path. "
+            + f"The fast path was not hit because {why_not_fast_path}"
+        )
+
+        if self.batch_first and is_batched:
+            # make sure that the transpose op does not affect the "is" property
+            if key is value:
+                if query is key:
+                    query = key = value = query.transpose(1, 0)
+                else:
+                    query, key = (x.transpose(1, 0) for x in (query, key))
+                    value = key
+            else:
+                query, key, value = (x.transpose(1, 0) for x in (query, key, value))
+
+        if not self._qkv_same_embed_dim:
+            attn_output, attn_output_weights = self.multi_head_attention_forward(
+                query,
+                key,
+                value,
+                self.embed_dim,
+                self.num_heads,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.bias_k,
+                self.bias_v,
+                self.add_zero_attn,
+                self.dropout,
+                self.out_proj.weight,
+                self.out_proj.bias,
+                training=self.training,
+                key_padding_mask=key_padding_mask,
+                need_weights=need_weights,
+                attn_mask=attn_mask,
+                use_separate_proj_weight=True,
+                q_proj_weight=self.q_proj_weight,
+                k_proj_weight=self.k_proj_weight,
+                v_proj_weight=self.v_proj_weight,
+                average_attn_weights=average_attn_weights,
+                is_causal=is_causal,
+            )
+        else:
+            attn_output, attn_output_weights = self.multi_head_attention_forward(
+                query,
+                key,
+                value,
+                self.embed_dim,
+                self.num_heads,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.bias_k,
+                self.bias_v,
+                self.add_zero_attn,
+                self.dropout,
+                self.out_proj.weight,
+                self.out_proj.bias,
+                training=self.training,
+                key_padding_mask=key_padding_mask,
+                need_weights=need_weights,
+                attn_mask=attn_mask,
+                average_attn_weights=average_attn_weights,
+                is_causal=is_causal,
+            )
+        if self.batch_first and is_batched:
+            return attn_output.transpose(1, 0), attn_output_weights
+        else:
+            return attn_output, attn_output_weights
+
+    def multi_head_attention_forward(
+        self,
+        query: Tensor,
+        key: Tensor,
+        value: Tensor,
+        embed_dim_to_check: int,
+        num_heads: int,
+        in_proj_weight: Tensor | None,
+        in_proj_bias: Tensor | None,
+        bias_k: Tensor | None,
+        bias_v: Tensor | None,
+        add_zero_attn: bool,
+        dropout_p: float,
+        out_proj_weight: Tensor,
+        out_proj_bias: Tensor | None,
+        training: bool = True,
+        key_padding_mask: Tensor | None = None,
+        need_weights: bool = True,
+        attn_mask: Tensor | None = None,
+        use_separate_proj_weight: bool = False,
+        q_proj_weight: Tensor | None = None,
+        k_proj_weight: Tensor | None = None,
+        v_proj_weight: Tensor | None = None,
+        static_k: Tensor | None = None,
+        static_v: Tensor | None = None,
+        average_attn_weights: bool = True,
+        is_causal: bool = False,
+    ) -> tuple[Tensor, Tensor | None]:
+        is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)
+
+        # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
+        # is batched, run the computation and before returning squeeze the
+        # batch dimension so that the output doesn't carry this temporary batch dimension.
+        if not is_batched:
+            # unsqueeze if the input is unbatched
+            query = query.unsqueeze(1)
+            key = key.unsqueeze(1)
+            value = value.unsqueeze(1)
+            if key_padding_mask is not None:
+                key_padding_mask = key_padding_mask.unsqueeze(0)
+
+        # set up shape vars
+        tgt_len, bsz, embed_dim = query.shape
+        src_len, _, _ = key.shape
+
+        key_padding_mask = _canonical_mask(
+            mask=key_padding_mask,
+            mask_name="key_padding_mask",
+            other_type=F._none_or_dtype(attn_mask),
+            other_name="attn_mask",
+            target_type=query.dtype,
+        )
+
+        if is_causal and attn_mask is None:
+            raise RuntimeError(
+                "Need attn_mask if specifying the is_causal hint. "
+                "You may use the Transformer module method "
+                "`generate_square_subsequent_mask` to create this mask."
+            )
+
+        if is_causal and key_padding_mask is None and not need_weights:
+            # when we have a kpm or need weights, we need attn_mask
+            # Otherwise, we use the is_causal hint go as is_causal
+            # indicator to SDPA.
+            attn_mask = None
+        else:
+            attn_mask = _canonical_mask(
+                mask=attn_mask,
+                mask_name="attn_mask",
+                other_type=None,
+                other_name="",
+                target_type=query.dtype,
+                check_other=False,
+            )
+
+            if key_padding_mask is not None:
+                # We have the attn_mask, and use that to merge kpm into it.
+                # Turn off use of is_causal hint, as the merged mask is no
+                # longer causal.
+                is_causal = False
+
+        assert embed_dim == embed_dim_to_check, (
+            f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
+        )
+        if isinstance(embed_dim, torch.Tensor):
+            # embed_dim can be a tensor when JIT tracing
+            head_dim = embed_dim.div(num_heads, rounding_mode="trunc")
+        else:
+            head_dim = embed_dim // num_heads
+        assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
+        if use_separate_proj_weight:
+            # allow MHA to have different embedding dimensions when separate projection weights are used
+            assert key.shape[:2] == value.shape[:2], (
+                f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
+            )
+        else:
+            assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
+
+        #
+        # compute in-projection
+        #
+        if not use_separate_proj_weight:
+            assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"
+            q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
+        else:
+            assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
+            assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
+            assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
+            if in_proj_bias is None:
+                b_q = b_k = b_v = None
+            else:
+                b_q, b_k, b_v = in_proj_bias.chunk(3)
+            q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
+
+        # prep attention mask
+
+        if attn_mask is not None:
+            # ensure attn_mask's dim is 3
+            if attn_mask.dim() == 2:
+                correct_2d_size = (tgt_len, src_len)
+                if attn_mask.shape != correct_2d_size:
+                    raise RuntimeError(
+                        f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}."
+                    )
+                attn_mask = attn_mask.unsqueeze(0)
+            elif attn_mask.dim() == 3:
+                correct_3d_size = (bsz * num_heads, tgt_len, src_len)
+                if attn_mask.shape != correct_3d_size:
+                    raise RuntimeError(
+                        f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}."
+                    )
+            else:
+                raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
+
+        # add bias along batch dimension (currently second)
+        if bias_k is not None and bias_v is not None:
+            assert static_k is None, "bias cannot be added to static key."
+            assert static_v is None, "bias cannot be added to static value."
+            k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
+            v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
+            if attn_mask is not None:
+                attn_mask = pad(attn_mask, (0, 1))
+            if key_padding_mask is not None:
+                key_padding_mask = pad(key_padding_mask, (0, 1))
+        else:
+            assert bias_k is None
+            assert bias_v is None
+
+        #
+        # reshape q, k, v for multihead attention and make em batch first
+        #
+        q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
+        if static_k is None:
+            k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+        else:
+            # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+            assert static_k.size(0) == bsz * num_heads, (
+                f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
+            )
+            assert static_k.size(2) == head_dim, f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
+            k = static_k
+        if static_v is None:
+            v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+        else:
+            # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+            assert static_v.size(0) == bsz * num_heads, (
+                f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
+            )
+            assert static_v.size(2) == head_dim, f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
+            v = static_v
+
+        # add zero attention along batch dimension (now first)
+        if add_zero_attn:
+            zero_attn_shape = (bsz * num_heads, 1, head_dim)
+            k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
+            v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
+            if attn_mask is not None:
+                attn_mask = pad(attn_mask, (0, 1))
+            if key_padding_mask is not None:
+                key_padding_mask = pad(key_padding_mask, (0, 1))
+
+        # update source sequence length after adjustments
+        src_len = k.size(1)
+
+        # merge key padding and attention masks
+        if key_padding_mask is not None:
+            assert key_padding_mask.shape == (
+                bsz,
+                src_len,
+            ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
+            key_padding_mask = (
+                key_padding_mask.view(bsz, 1, 1, src_len)
+                .expand(-1, num_heads, -1, -1)
+                .reshape(bsz * num_heads, 1, src_len)
+            )
+            if attn_mask is None:
+                attn_mask = key_padding_mask
+            else:
+                attn_mask = attn_mask + key_padding_mask
+
+        # adjust dropout probability
+        if not training:
+            dropout_p = 0.0
+
+        #
+        # (deep breath) calculate attention and out projection
+        #
+
+        if need_weights:
+            B, Nt, E = q.shape
+            q_scaled = q / math.sqrt(E)
+
+            assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"
+
+            if attn_mask is not None:
+                attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))
+            else:
+                attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
+            attn_output_weights = softmax(attn_output_weights, dim=-1)
+            if dropout_p > 0.0:
+                attn_output_weights = dropout(attn_output_weights, p=dropout_p)
+
+            attn_output = torch.bmm(attn_output_weights, v)
+
+            attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
+            attn_output = self.out_proj(attn_output)
+            attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+
+            # optionally average attention weights over heads
+            attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
+            if average_attn_weights:
+                attn_output_weights = attn_output_weights.mean(dim=1)
+
+            if not is_batched:
+                # squeeze the output if input was unbatched
+                attn_output = attn_output.squeeze(1)
+                attn_output_weights = attn_output_weights.squeeze(0)
+            return attn_output, attn_output_weights
+        else:
+            # attn_mask can be either (L,S) or (N*num_heads, L, S)
+            # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
+            # in order to match the input for SDPA of (N, num_heads, L, S)
+            if attn_mask is not None:
+                if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
+                    attn_mask = attn_mask.unsqueeze(0)
+                else:
+                    attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
+
+            q = q.view(bsz, num_heads, tgt_len, head_dim)
+            k = k.view(bsz, num_heads, src_len, head_dim)
+            v = v.view(bsz, num_heads, src_len, head_dim)
+
+            attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)
+            attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
+
+            attn_output = self.out_proj(attn_output)
+            attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+            if not is_batched:
+                # squeeze the output if input was unbatched
+                attn_output = attn_output.squeeze(1)
+            return attn_output, None
+
+
+def _mha_shape_check(
+    query: Tensor,
+    key: Tensor,
+    value: Tensor,
+    key_padding_mask: Tensor | None,
+    attn_mask: Tensor | None,
+    num_heads: int,
+):
+    # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask`
+    # and returns if the input is batched or not.
+    # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor.
+
+    # Shape check.
+    if query.dim() == 3:
+        # Batched Inputs
+        is_batched = True
+        assert key.dim() == 3 and value.dim() == 3, (
+            "For batched (3-D) `query`, expected `key` and `value` to be 3-D"
+            f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"
+        )
+        if key_padding_mask is not None:
+            assert key_padding_mask.dim() == 2, (
+                "For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D"
+                f" but found {key_padding_mask.dim()}-D tensor instead"
+            )
+        if attn_mask is not None:
+            assert attn_mask.dim() in (2, 3), (
+                "For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
+                f" but found {attn_mask.dim()}-D tensor instead"
+            )
+    elif query.dim() == 2:
+        # Unbatched Inputs
+        is_batched = False
+        assert key.dim() == 2 and value.dim() == 2, (
+            "For unbatched (2-D) `query`, expected `key` and `value` to be 2-D"
+            f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"
+        )
+
+        if key_padding_mask is not None:
+            assert key_padding_mask.dim() == 1, (
+                "For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D"
+                f" but found {key_padding_mask.dim()}-D tensor instead"
+            )
+
+        if attn_mask is not None:
+            assert attn_mask.dim() in (2, 3), (
+                "For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
+                f" but found {attn_mask.dim()}-D tensor instead"
+            )
+            if attn_mask.dim() == 3:
+                expected_shape = (num_heads, query.shape[0], key.shape[0])
+                assert attn_mask.shape == expected_shape, (
+                    f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}"
+                )
+    else:
+        raise AssertionError(
+            f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor"
+        )
+
+    return is_batched
+
+
+def _canonical_mask(
+    mask: Tensor | None,
+    mask_name: str,
+    other_type: DType | None,
+    other_name: str,
+    target_type: DType,
+    check_other: bool = True,
+) -> Tensor | None:
+    if mask is not None:
+        _mask_dtype = mask.dtype
+        _mask_is_float = torch.is_floating_point(mask)
+        if _mask_dtype != torch.bool and not _mask_is_float:
+            raise AssertionError(f"only bool and floating types of {mask_name} are supported")
+        if check_other and other_type is not None:
+            if _mask_dtype != other_type:
+                warnings.warn(
+                    f"Support for mismatched {mask_name} and {other_name} "
+                    "is deprecated. Use same type for both instead."
+                )
+        if not _mask_is_float:
+            mask = torch.zeros_like(mask, dtype=target_type).masked_fill_(mask, float("-inf"))
+    return mask
+
+
+def _in_projection_packed(
+    q: Tensor,
+    k: Tensor,
+    v: Tensor,
+    w: Tensor,
+    b: Tensor | None = None,
+) -> list[Tensor]:
+    r"""
+    Performs the in-projection step of the attention operation, using packed weights.
+    Output is a triple containing projection tensors for query, key and value.
+    Args:
+        q, k, v: query, key and value tensors to be projected. For self-attention,
+            these are typically the same tensor; for encoder-decoder attention,
+            k and v are typically the same tensor. (We take advantage of these
+            identities for performance if they are present.) Regardless, q, k and v
+            must share a common embedding dimension; otherwise their shapes may vary.
+        w: projection weights for q, k and v, packed into a single tensor. Weights
+            are packed along dimension 0, in q, k, v order.
+        b: optional projection biases for q, k and v, packed into a single tensor
+            in q, k, v order.
+    Shape:
+        Inputs:
+        - q: :math:`(..., E)` where E is the embedding dimension
+        - k: :math:`(..., E)` where E is the embedding dimension
+        - v: :math:`(..., E)` where E is the embedding dimension
+        - w: :math:`(E * 3, E)` where E is the embedding dimension
+        - b: :math:`E * 3` where E is the embedding dimension
+        Output:
+        - in output list :math:`[q', k', v']`, each output tensor will have the
+            same shape as the corresponding input tensor.
+    """
+    E = q.size(-1)
+    if k is v:
+        if q is k:
+            # self-attention
+            proj = linear(q, w, b)
+            # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()
+            proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
+            return proj[0], proj[1], proj[2]
+        else:
+            # encoder-decoder attention
+            w_q, w_kv = w.split([E, E * 2])
+            if b is None:
+                b_q = b_kv = None
+            else:
+                b_q, b_kv = b.split([E, E * 2])
+            q_proj = linear(q, w_q, b_q)
+            kv_proj = linear(k, w_kv, b_kv)
+            # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()
+            kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
+            return (q_proj, kv_proj[0], kv_proj[1])
+    else:
+        w_q, w_k, w_v = w.chunk(3)
+        if b is None:
+            b_q = b_k = b_v = None
+        else:
+            b_q, b_k, b_v = b.chunk(3)
+        return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
+
+
+def _in_projection(
+    q: Tensor,
+    k: Tensor,
+    v: Tensor,
+    w_q: Tensor,
+    w_k: Tensor,
+    w_v: Tensor,
+    b_q: Tensor | None = None,
+    b_k: Tensor | None = None,
+    b_v: Tensor | None = None,
+) -> tuple[Tensor, Tensor, Tensor]:
+    r"""
+    Performs the in-projection step of the attention operation. This is simply
+    a triple of linear projections, with shape constraints on the weights which
+    ensure embedding dimension uniformity in the projected outputs.
+    Output is a triple containing projection tensors for query, key and value.
+    Args:
+        q, k, v: query, key and value tensors to be projected.
+        w_q, w_k, w_v: weights for q, k and v, respectively.
+        b_q, b_k, b_v: optional biases for q, k and v, respectively.
+    Shape:
+        Inputs:
+        - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any
+            number of leading dimensions.
+        - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any
+            number of leading dimensions.
+        - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any
+            number of leading dimensions.
+        - w_q: :math:`(Eq, Eq)`
+        - w_k: :math:`(Eq, Ek)`
+        - w_v: :math:`(Eq, Ev)`
+        - b_q: :math:`(Eq)`
+        - b_k: :math:`(Eq)`
+        - b_v: :math:`(Eq)`
+        Output: in output triple :math:`(q', k', v')`,
+         - q': :math:`[Qdims..., Eq]`
+         - k': :math:`[Kdims..., Eq]`
+         - v': :math:`[Vdims..., Eq]`
+    """
+    Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1)
+    assert w_q.shape == (Eq, Eq), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}"
+    assert w_k.shape == (Eq, Ek), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}"
+    assert w_v.shape == (Eq, Ev), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}"
+    assert b_q is None or b_q.shape == (Eq,), f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}"
+    assert b_k is None or b_k.shape == (Eq,), f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}"
+    assert b_v is None or b_v.shape == (Eq,), f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}"
+    return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
+
+
+# Copied from transformers.models.whisper.modeling_whisper.WhisperEncoderLayer and add use_cache for streaming inference
+class MiniCPMWhisperEncoderLayer(nn.Module):
+    def __init__(self, config: WhisperConfig, layer_idx: int = None):
+        super().__init__()
+        self.embed_dim = config.d_model
+        self.self_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation](
+            embed_dim=self.embed_dim,
+            num_heads=config.encoder_attention_heads,
+            dropout=config.attention_dropout,
+            config=config,
+            layer_idx=layer_idx,
+        )
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
+        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        layer_head_mask: torch.Tensor,
+        output_attentions: bool = False,
+        past_key_values: EncoderDecoderCache | None = None,
+        use_cache: bool | None = False,
+    ) -> torch.Tensor:
+        r"""
+        Args:
+            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, embed_dim)`):
+                Hidden states to be fed into the encoder layer.
+            attention_mask (`torch.FloatTensor` of shape `(batch_size, 1, tgt_len, src_len)`):
+                Attention mask where padding elements are indicated by large negative values.
+            layer_head_mask (`torch.FloatTensor` of shape `(encoder_attention_heads,)`):
+                Mask to nullify selected heads of the attention modules.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attention weights.
+            past_key_values (`EncoderDecoderCache`, *optional*):
+                Past key-value pairs used for incremental decoding.
+            use_cache (`bool`, *optional*):
+                Whether or not to return updated `past_key_values` for caching.
+
+        Returns:
+            A tuple of shape `(hidden_states, optional(attn_weights), optional(past_key_values))`.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+        attn_out = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            layer_head_mask=layer_head_mask,
+            output_attentions=output_attentions,
+            past_key_value=past_key_values,
+        )
+        hidden_states = attn_out[0]
+        attn_weights = attn_out[1] if len(attn_out) > 1 else None
+        past_key_values = attn_out[2] if len(attn_out) > 2 else None
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        if hidden_states.dtype == torch.float16 and (
+            torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
+        ):
+            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
+            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        if use_cache:
+            outputs += (past_key_values,)
+
+        return outputs
+
+
+# Copied from from transformers.models.whisper.modeling_whisper.WhisperEncoder and add use_cache for streaming inference
+class MiniCPMWhisperEncoder(WhisperEncoder):
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+        self.layers = nn.ModuleList(
+            [MiniCPMWhisperEncoderLayer(config, layer_idx=i) for i in range(config.encoder_layers)]
+        )
+
+    def forward(
+        self,
+        input_features,
+        attention_mask=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        past_key_values: EncoderDecoderCache | None = None,
+        use_cache: bool | None = None,
+    ) -> BaseModelOutputWithPast | tuple:
+        r"""
+        Forward pass of the Whisper encoder.
+
+        Args:
+            input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`):
+                Float values of log-mel features extracted from the raw audio waveform. Typically generated
+                by a feature extractor (e.g., `WhisperFeatureExtractor`) that processes `.flac` or `.wav`
+                files into padded 2D mel spectrogram frames. These features are projected via convolution layers
+                (`conv1` and `conv2`) and then transformed into embeddings for the encoder.
+
+            attention_mask (`torch.Tensor`, *optional*):
+                Not used by Whisper for masking `input_features`, but included for API compatibility with
+                other models. If provided, it is simply ignored within the model. By default, Whisper
+                effectively ignores silence in the input log-mel spectrogram.
+
+            head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
+                Mask to nullify selected attention heads. The elements should be either 1 or 0, where:
+                - 1 indicates the head is **not masked**,
+                - 0 indicates the head is **masked** (i.e., the attention head is dropped).
+
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attention tensors of all encoder layers. If set to `True`, the
+                returned tuple (or `BaseModelOutputWithPast`) will contain an additional element with
+                attention weights for each encoder layer.
+
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. If set to `True`, the returned
+                tuple (or `BaseModelOutputWithPast`) will contain a tuple of hidden states, including the
+                initial embedding output as well as the outputs of each layer.
+
+            return_dict (`bool`, *optional*):
+                Whether or not to return a `BaseModelOutputWithPast` (a subclass of `ModelOutput`) instead
+                of a plain tuple. If set to `True`, the output will be a `BaseModelOutputWithPast` object,
+                otherwise it will be a tuple.
+
+            past_key_values (`EncoderDecoderCache`, *optional*):
+                When using caching for faster inference, this is an object that stores the key-value pairs
+                for attention states. If provided, the model will append new states to the existing cache
+                and return the updated cache. This speeds up sequential decoding or chunked inference.
+
+                - If `past_key_values` is `None`, no past states are used or returned.
+                - If `past_key_values` is not `None` and `use_cache=True`, the model will use the provided
+                cache and return the updated cache (as `next_encoder_cache`).
+
+            use_cache (`bool`, *optional*):
+                Whether or not the model should use caching (`past_key_values`) to speed up processing
+                during inference. When set to `True`, the model will:
+                - Inspect and use `past_key_values` if provided.
+                - Return updated `past_key_values` (under the name `next_encoder_cache` in
+                    `BaseModelOutputWithPast`).
+
+        Returns:
+            `BaseModelOutputWithPast` or `tuple` (depending on `return_dict`):
+                If `return_dict=True`, a `BaseModelOutputWithPast` is returned, which contains:
+                - **last_hidden_state** (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                The output of the final encoder layer.
+                - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_hidden_states=True`):
+                Hidden states of the model at each layer (including the initial projection).
+                - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_attentions=True`):
+                Attention weights from each encoder layer.
+                - **past_key_values** (an object of type `EncoderDecoderCache` or `None`, *optional*):
+                Updated cache of key-value pairs if `use_cache=True`.
+
+                If `return_dict=False`, a tuple is returned, where the format is:
+                `(last_hidden_state, hidden_states, attentions)`, with `hidden_states` and `attentions`
+                only present if their respective `output_*` arguments are set to `True`.
+
+        Example:
+            >>> from transformers import AutoFeatureExtractor, WhisperConfig, WhisperForConditionalGeneration
+            >>> import torch
+
+            >>> # Load a feature extractor and a Whisper model
+            >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-tiny.en")
+            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
+
+            >>> # Assume you have audio (list of floats or numpy array) loaded from a file
+            >>> # Then extract the mel features:
+            >>> input_features = feature_extractor(audio, sampling_rate=16000, return_tensors="pt").input_features
+
+            >>> # Forward pass
+            >>> outputs = model.encoder(
+            ...     input_features=input_features,
+            ...     output_hidden_states=True,
+            ...     output_attentions=True,
+            ...     use_cache=True
+            ... )
+
+            >>> # Retrieve the last hidden state
+            >>> last_hidden_state = outputs.last_hidden_state
+            >>> print(last_hidden_state.shape)
+            torch.Size([batch_size, seq_length, hidden_size])
+
+            >>> # Retrieve the intermediate hidden states if output_hidden_states=True
+            >>> all_encoder_hidden_states = outputs.hidden_states
+
+            >>> # Retrieve attention weights if output_attentions=True
+            >>> all_encoder_attentions = outputs.attentions
+
+            >>> # Retrieve updated past key values if use_cache=True
+            >>> encoder_cache = outputs.past_key_values
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # Ignore copy
+        input_features = input_features.to(dtype=self.conv1.weight.dtype, device=self.conv1.weight.device)
+
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+
+        embed_pos = self.embed_positions.weight
+        past_key_values_length = 0
+        if use_cache:
+            if past_key_values is None:
+                past_key_values = EncoderDecoderCache(DynamicCache(), DynamicCache())
+            elif isinstance(past_key_values, list):
+                past_key_values = EncoderDecoderCache(DynamicCache.from_legacy_cache(past_key_values), DynamicCache())
+            elif isinstance(past_key_values, DynamicCache):
+                past_key_values = EncoderDecoderCache(past_key_values, DynamicCache())
+            else:
+                pass
+            past_key_values_length = past_key_values.self_attention_cache.get_usable_length(inputs_embeds.shape[1])
+            if inputs_embeds.shape[1] + past_key_values_length > embed_pos.shape[0]:
+                logger.warning("seems the audio is longer than 30s. repeating the last part of the audio")
+                embed_pos_front = embed_pos[past_key_values_length:, :]
+                embed_pos = torch.cat(
+                    (
+                        embed_pos_front,
+                        torch.repeat_interleave(
+                            embed_pos[-1, :].unsqueeze(0),
+                            inputs_embeds.shape[1] - embed_pos.shape[0] + past_key_values_length,
+                            dim=0,
+                        ),
+                    )
+                )
+            else:
+                embed_pos = embed_pos[past_key_values_length : inputs_embeds.shape[1] + past_key_values_length, :]
+        else:
+            embed_pos = embed_pos[: inputs_embeds.shape[1], :]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (len(self.layers)), (
+                f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+            )
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            # Ignore copy
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                        past_key_values,
+                        use_cache,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                        output_attentions=output_attentions,
+                        past_key_values=past_key_values,
+                        use_cache=use_cache,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_encoder_cache = layer_outputs[2 if output_attentions else 1]
+            else:
+                next_encoder_cache = None
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+            past_key_values=next_encoder_cache,
+        )
+
+
+class MiniCPMOOmniImageFeatureInputs(TensorSchema):
+    """
+    Dimensions:
+        - ni: Number of images
+        - nq: Number of queries (from resampler)
+        - embed_dim: Embedding dimension
+    """
+
+    type: Literal["image_features"]
+    image_embeds: Annotated[
+        torch.Tensor | list[torch.Tensor],
+        TensorShape("ni", "nq", "embed_dim"),
+    ]
+
+
+def _minicpmo_omni_llm_field_config(hf_inputs: Mapping[str, torch.Tensor]):
+    """Create field config for MiniCPM-o thinker multimodal inputs."""
+    # For MiniCPM-o, pixel_values come preprocessed from image processor
+    # tgt_sizes indicates the size of each image after patch embedding
+    pixel_values = hf_inputs.get("pixel_values")
+    tgt_sizes = hf_inputs.get("tgt_sizes")
+
+    # Handle empty case - return empty tensor configs like qwen2.5
+    if pixel_values is None or tgt_sizes is None or (isinstance(pixel_values, list) and len(pixel_values) == 0):
+        # Return empty configs with empty tensors
+        return dict(
+            pixel_values=MultiModalFieldConfig.flat_from_sizes("image", torch.empty((0,), dtype=torch.long)),
+            tgt_sizes=MultiModalFieldConfig.batched("image"),
+            image_embeds=MultiModalFieldConfig.flat_from_sizes("image", torch.empty((0,), dtype=torch.long)),
+        )
+
+    # Calculate number of images and slices per image
+    if isinstance(pixel_values, list):
+        num_images = len(pixel_values)
+        # Calculate number of slices per image (each image can have multiple slices)
+        pixel_value_sizes = torch.tensor(
+            [len(pv) if isinstance(pv, list) else 1 for pv in pixel_values], dtype=torch.long
+        )
+    else:
+        num_images = 1
+        pixel_value_sizes = torch.tensor([1], dtype=torch.long)
+
+    # query_num is the output size from resampler
+    query_num = hf_inputs.get("query_num", 64)
+    image_grid_sizes = torch.full((num_images,), query_num, dtype=torch.long)
+
+    return dict(
+        pixel_values=MultiModalFieldConfig.flat_from_sizes("image", pixel_value_sizes),
+        tgt_sizes=MultiModalFieldConfig.batched("image"),
+        image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
+    )
+
+
+def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
+    version_float = getattr(config, "version", None)
+
+    # The old configs do not include version number
+    # TODO: Remove this after the HF repos are updated
+    if version_float is None:
+        if config.hidden_size == 2304 and config.query_num == 64:
+            return (2, 0)
+        return (2, 5)
+    version_str = str(version_float)
+    return tuple(int(x) for x in version_str.split("."))
+
+
+_MAX_FRAMES_PER_VIDEO = 64
+
+
+class MiniCPMO45OmniLLMProcessingInfo(BaseProcessingInfo):
+    """Processing info for MiniCPM-o thinker multimodal processor."""
+
+    audio_pattern = "(<audio>./</audio>)"
+    image_pattern = "(<image>./</image>)"
+    video_pattern = "(<video>./</video>)"
+
+    def get_tokenizer(self):
+        """Return tokenizer; load lazily when ctx.tokenizer is None (e.g. skip_tokenizer_init=True)."""
+        if self.ctx.tokenizer is not None:
+            return self.ctx.tokenizer
+        if not hasattr(self, "_lazy_tokenizer") or self._lazy_tokenizer is None:
+            from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
+
+            self._lazy_tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
+        return self._lazy_tokenizer
+
+    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+        mm_limits = {"image": None, "audio": None}
+        if self.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
+            mm_limits["video"] = None
+
+        return mm_limits
+
+    def get_audio_placeholder(
+        self,
+        audio_lens: int,
+        chunk_input: bool = True,
+        chunk_length: int = 1,
+    ) -> str:
+        hf_processor = self.get_hf_processor()
+
+        return hf_processor.get_audio_placeholder(
+            audio_lens,
+            chunk_input=chunk_input,
+            chunk_length=chunk_length,
+        )
+
+    def get_default_audio_pool_step(self) -> int:
+        return getattr(self.get_hf_config(), "audio_pool_step", 2)
+
+    def get_default_audio_sampling_rate(self) -> int:
+        return 16000
+
+    def get_data_parser(self) -> MultiModalDataParser:
+        # vLLM >=0.16 expects ``BaseProcessingInfo.get_data_parser`` to construct
+        # the per-model multi-modal data parser.
+        return MiniCPMOMultiModalDataParser(target_sr=self.get_default_audio_sampling_rate())
+
+    def get_chunk_length(self) -> int:
+        return self.get_hf_config().audio_chunk_length
+
+    def get_max_audio_tokens_per_chunk(self) -> int:
+        pool_step = self.get_default_audio_pool_step()
+        fbank_feat_in_chunk = 100
+        cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
+        return (cnn_feat_in_chunk - pool_step) // pool_step + 1
+
+    def get_max_audio_chunks_with_most_features(self) -> int:
+        return 30
+
+    def get_max_audio_tokens(self) -> int:
+        num_chunks = self.get_max_audio_chunks_with_most_features()
+        return self.get_max_audio_tokens_per_chunk() * num_chunks
+
+    def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
+        sampling_rate = self.get_default_audio_sampling_rate()
+        num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk()
+        return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1
+
+    def get_num_frames_with_most_features(
+        self,
+        seq_len: int,
+        mm_counts: Mapping[str, int],
+    ) -> int:
+        max_images = mm_counts.get("image", 0)
+        max_videos = mm_counts.get("video", 0)
+        max_audios = mm_counts.get("audio", 0)
+
+        max_image_tokens = self.get_max_image_tokens() * max_images
+        max_audio_tokens = self.get_max_audio_tokens() * max_audios
+        max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens - max_audio_tokens)
+        max_frames_per_video = min(max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO)
+
+        return max(max_frames_per_video, 1)
+
+    def get_hf_config(self):
+        return self.ctx.get_hf_config()
+
+    def get_hf_processor(self, **kwargs: object):
+        # When skip_tokenizer_init=True (e.g. in worker), ctx.tokenizer is None.
+        # HF MiniCPM processor requires a tokenizer; use get_tokenizer() which loads lazily.
+        if self.ctx.tokenizer is None:
+            from vllm.transformers_utils.processor import cached_processor_from_config
+
+            tokenizer = self.get_tokenizer()
+            hf_processor = cached_processor_from_config(
+                self.ctx.model_config,
+                tokenizer=tokenizer,
+                **kwargs,
+            )
+        else:
+            hf_processor = self.ctx.get_hf_processor(**kwargs)
+
+        # NumPy arrays are considered as Iterable but not Sequence in
+        # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
+        image_processor = hf_processor.image_processor  # type: ignore
+        for attr in ("mean", "std"):
+            val = getattr(image_processor, attr)
+            if isinstance(val, np.ndarray):
+                setattr(image_processor, attr, val.tolist())
+
+        return hf_processor
+
+    def get_hf_image_processor(self) -> MiniCPMVImageProcessor:
+        """Get the MiniCPM-o image processor."""
+        config = self.get_hf_config()
+        return MiniCPMVImageProcessor(
+            max_slice_nums=config.slice_config.max_slice_nums,
+            scale_resolution=config.image_size,
+            patch_size=config.vision_config.patch_size,
+            use_image_id=config.use_image_id,
+            image_feature_size=config.query_num,
+        )
+
+    def get_model_version(self):
+        return get_version_by_config(self.get_hf_config())
+
+    def get_image_processor(self, **kwargs: object):
+        return self.get_hf_processor(**kwargs).image_processor
+
+    def get_slice_image_placeholder(
+        self,
+        image_size: ImageSize,
+        # For MiniCPM V/O 2.6
+        image_idx: int = 0,
+        max_slice_nums: int | None = None,
+        use_image_id: bool = True,
+    ) -> str:
+        image_processor = self.get_image_processor()
+        version = self.get_model_version()
+
+        if version == (2, 0) or version == (2, 5):
+            return image_processor.get_slice_image_placeholder(image_size)
+
+        return image_processor.get_slice_image_placeholder(
+            image_size,
+            image_idx=image_idx,
+            max_slice_nums=max_slice_nums,
+            use_image_id=use_image_id,
+        )
+
+    def get_sliced_grid(
+        self,
+        image_size: ImageSize,
+        # For MiniCPM V/O 2.6
+        max_slice_nums: int | None = None,
+    ) -> tuple[int, int] | None:
+        image_processor = self.get_image_processor()
+        version = self.get_model_version()
+
+        if version == (2, 0) or version == (2, 5):
+            return image_processor.get_sliced_grid(image_size)
+
+        if max_slice_nums is None:
+            max_slice_nums = image_processor.max_slice_nums
+
+        return image_processor.get_sliced_grid(
+            image_size,
+            max_slice_nums=max_slice_nums,
+        )
+
+    def get_num_image_tokens(
+        self,
+        image_size: ImageSize,
+        max_slice_nums: int | None = None,
+    ) -> int:
+        image_processor = self.get_image_processor()
+
+        grid = self.get_sliced_grid(
+            image_size,
+            max_slice_nums=max_slice_nums,
+        )
+        if grid is None:
+            ncols = nrows = 0
+        else:
+            ncols, nrows = grid
+
+        return (ncols * nrows + 1) * image_processor.image_feature_size
+
+    def get_max_image_tokens(self) -> int:
+        image_size = self.get_image_size_with_most_features()
+        return self.get_num_image_tokens(image_size)
+
+    def get_image_max_slice_num(self) -> int:
+        return getattr(self.get_hf_config(), "max_slice_num", 9)
+
+    def get_image_size_with_most_features(self) -> ImageSize:
+        image_size = getattr(self.get_hf_config(), "image_size", 448)
+        max_slice_num = self.get_image_max_slice_num()
+        return ImageSize(width=image_size, height=image_size * max_slice_num)
+
+    def get_max_video_frame_tokens(self) -> int:
+        frame_size = self.get_video_frame_size_with_most_features()
+
+        return self.get_num_image_tokens(
+            frame_size,
+            max_slice_nums=self.get_video_max_slice_num(),
+        )
+
+    def get_max_video_tokens(
+        self,
+        seq_len: int,
+        mm_counts: Mapping[str, int],
+    ) -> int:
+        num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts)
+        num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames
+        return num_video_tokens_total
+
+    def get_video_max_slice_num(self) -> int:
+        return 1
+
+    def get_video_frame_size_with_most_features(self) -> ImageSize:
+        image_size = getattr(self.get_hf_config(), "image_size", 448)
+        max_slice_num = self.get_video_max_slice_num()
+        return ImageSize(width=image_size, height=image_size * max_slice_num)
+
+    def get_max_video_frames(self, max_tokens: int) -> int:
+        num_frame_tokens = self.get_max_video_frame_tokens()
+        num_frames = max_tokens // num_frame_tokens
+        return num_frames
+
+
+class MiniCPMO45OmniLLMDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMO45OmniLLMProcessingInfo]):
+    """Builder for dummy inputs for MiniCPM-o thinker."""
+
+    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+        num_audios = mm_counts.get("audio", 0)
+        num_images = mm_counts.get("image", 0)
+        num_videos = mm_counts.get("video", 0)
+
+        audio_token: str = self.info.audio_pattern
+        image_token: str = self.info.image_pattern
+        video_token: str = self.info.video_pattern
+
+        return audio_token * num_audios + image_token * num_images + video_token * num_videos
+
+    def get_dummy_mm_data(
+        self,
+        seq_len: int,
+        mm_counts: Mapping[str, int],
+        mm_options: Mapping[str, BaseDummyOptions] | None = None,
+    ) -> MultiModalDataDict:
+        mm_options = mm_options or {}
+        num_images = mm_counts.get("image", 0)
+        num_videos = mm_counts.get("video", 0)
+        num_audios = mm_counts.get("audio", 0)
+
+        image_width, image_height = self.info.get_image_size_with_most_features()
+        video_width, video_height = self.info.get_video_frame_size_with_most_features()
+        num_video_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts)
+
+        image_opts = mm_options.get("image")
+        image_overrides = image_opts if isinstance(image_opts, ImageDummyOptions) else None
+        audio_opts = mm_options.get("audio")
+        audio_overrides = audio_opts if isinstance(audio_opts, AudioDummyOptions) else None
+        video_opts = mm_options.get("video")
+        video_overrides = video_opts if isinstance(video_opts, VideoDummyOptions) else None
+
+        if video_overrides is not None and video_overrides.num_frames:
+            num_video_frames = min(num_video_frames, video_overrides.num_frames)
+        if video_overrides is not None:
+            if video_overrides.width:
+                video_width = min(video_width, video_overrides.width)
+            if video_overrides.height:
+                video_height = min(video_height, video_overrides.height)
+
+        audio_len = self.info.get_max_audio_chunks_with_most_features() * self.info.get_default_audio_sampling_rate()
+
+        return {
+            "image": self._get_dummy_images(
+                width=image_width,
+                height=image_height,
+                num_images=num_images,
+                overrides=image_overrides,
+            ),
+            "audio": self._get_dummy_audios(
+                length=audio_len,
+                num_audios=num_audios,
+                overrides=audio_overrides,
+            ),
+            "video": [
+                self._get_dummy_images(
+                    width=video_width,
+                    height=video_height,
+                    num_images=num_video_frames,
+                )
+            ]
+            * num_videos,
+        }
+
+
+class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
+    def __init__(
+        self,
+        data: Mapping[str, torch.Tensor],
+        fields_factory: Callable[
+            [Mapping[str, torch.Tensor]],
+            Mapping[str, MultiModalFieldConfig],
+        ],
+    ) -> None:
+        super().__init__(
+            data,
+            modality="image",
+            required_fields={"image_embeds", "image_sizes"},
+            fields_factory=fields_factory,
+        )
+
+    def get_image_size(self, index: int) -> ImageSize:
+        image_size = self.get(index)["image_sizes"].tolist()
+        return ImageSize(width=image_size[0], height=image_size[1])
+
+
+class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
+    def __init__(
+        self,
+        data: Mapping[str, torch.Tensor],
+        fields_factory: Callable[
+            [Mapping[str, torch.Tensor]],
+            Mapping[str, MultiModalFieldConfig],
+        ],
+    ) -> None:
+        super().__init__(
+            data,
+            modality="video",
+            required_fields={"video_embeds", "video_image_sizes"},
+            fields_factory=fields_factory,
+        )
+
+    def get_frame_size(self, index: int) -> ImageSize:
+        frame_size = self.get(index)["video_image_sizes"].tolist()
+        return ImageSize(width=frame_size[0], height=frame_size[1])
+
+    def get_num_frames(self, index: int) -> int:
+        return len(self.get(index)["video_image_sizes"])
+
+
+class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
+    def __init__(
+        self,
+        data: Mapping[str, torch.Tensor],
+        fields_factory: Callable[
+            [Mapping[str, torch.Tensor]],
+            Mapping[str, MultiModalFieldConfig],
+        ],
+    ) -> None:
+        super().__init__(
+            data,
+            modality="image",
+            required_fields={"audio_embeds"},
+            fields_factory=fields_factory,
+        )
+
+
+def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
+    return dict(
+        **_minicpmv_field_config(hf_inputs),
+        audio_features=MultiModalFieldConfig.batched("audio"),
+        audio_feature_lens=MultiModalFieldConfig.batched("audio"),
+        audio_embeds=MultiModalFieldConfig.batched("audio"),
+    )
+
+
+def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
+    return dict(
+        pixel_values=MultiModalFieldConfig.batched("image"),
+        image_sizes=MultiModalFieldConfig.batched("image"),
+        tgt_sizes=MultiModalFieldConfig.batched("image"),
+        image_embeds=MultiModalFieldConfig.batched("image"),
+        video_pixel_values=MultiModalFieldConfig.batched("video"),
+        video_image_sizes=MultiModalFieldConfig.batched("video"),
+        video_tgt_sizes=MultiModalFieldConfig.batched("video"),
+        video_embeds=MultiModalFieldConfig.batched("video"),
+    )
+
+
+class MiniCPMOMultiModalDataParser(MultiModalDataParser):
+    def _parse_image_data(
+        self,
+        data: dict[str, torch.Tensor] | ModalityData[ImageItem],
+    ) -> ModalityDataItems[Any, Any] | None:
+        if isinstance(data, dict):
+            return MiniCPMVImageEmbeddingItems(
+                data,
+                fields_factory=_minicpmv_field_config,
+            )
+
+        return super()._parse_image_data(data)
+
+    def _parse_video_data(
+        self,
+        data: dict[str, torch.Tensor] | ModalityData[VideoItem],
+    ) -> ModalityDataItems[Any, Any] | None:
+        if isinstance(data, dict):
+            return MiniCPMVVideoEmbeddingItems(
+                data,
+                fields_factory=_minicpmv_field_config,
+            )
+
+        return super()._parse_video_data(data)
+
+    def _parse_audio_data(
+        self,
+        data: dict[str, torch.Tensor] | ModalityData[AudioItem],
+    ) -> ModalityDataItems[Any, Any] | None:
+        if isinstance(data, dict):
+            return MiniCPMOAudioEmbeddingItems(
+                data,
+                fields_factory=_minicpmo_field_config,
+            )
+
+        return super()._parse_audio_data(data)
+
+
+class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45OmniLLMProcessingInfo]):
+    """Multimodal processor for MiniCPM-o thinker stage."""
+
+    def _apply_hf_processor_main(
+        self,
+        prompt: str | list[int],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+        tokenization_kwargs: Mapping[str, object],
+        *,
+        enable_hf_prompt_update: bool,
+    ) -> tuple[list[int], BatchFeature, bool]:
+        """
+        MiniCPM-O reimplements this to avoid calling HF processor with text-only
+        when enable_hf_prompt_update=False. The MiniCPM processor asserts
+        len(image_tags) == len(image_sizes) and fails if given placeholder text
+        without corresponding image data (e.g. during profiling/cache-miss path).
+        """
+        use_tts = hf_processor_mm_kwargs.get("use_tts", False)
+        hf_processor_mm_kwargs = {k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_tts"}
+        if isinstance(prompt, str):
+            if use_tts:
+                prompt = prompt + self._TTS_SUFFIX
+            if enable_hf_prompt_update:
+                return self._apply_hf_processor_text_mm(
+                    prompt_text=prompt,
+                    mm_items=mm_items,
+                    hf_processor_mm_kwargs=hf_processor_mm_kwargs,
+                    tokenization_kwargs=tokenization_kwargs,
+                )
+            tokenizer = self.info.get_tokenizer()
+            prompt_ids = encode_tokens(tokenizer, prompt)
+        else:
+            prompt_ids = self._apply_hf_processor_tokens_only(prompt)
+            if use_tts:
+                tokenizer = self.info.get_tokenizer()
+                tts_ids = tokenizer.convert_tokens_to_ids(["<|spk_bos|>", "<|spk|>", "<|spk_eos|>", "<|tts_bos|>"])
+                prompt_ids = list(prompt_ids) + tts_ids
+
+        mm_processed_data = self._apply_hf_processor_mm_only(
+            mm_items=mm_items,
+            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
+            tokenization_kwargs=tokenization_kwargs,
+        )
+
+        return prompt_ids, mm_processed_data, False
+
+    _TTS_SUFFIX = "<|spk_bos|><|spk|><|spk_eos|><|tts_bos|>"
+
+    def _call_hf_processor(
+        self,
+        prompt: str,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> BatchFeature:
+        tokenizer = self.info.get_tokenizer()
+
+        use_tts = mm_kwargs.get("use_tts", False)
+        mm_kwargs = {k: v for k, v in mm_kwargs.items() if k != "use_tts"}
+        if use_tts:
+            prompt = prompt + self._TTS_SUFFIX
+
+        input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
+        mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
+
+        return BatchFeature(
+            {
+                "input_ids": input_ids,
+                **mm_inputs,
+            }
+        )
+
+    def _hf_processor_applies_updates(
+        self,
+        prompt_text: str,
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+        tokenization_kwargs: Mapping[str, object],
+    ) -> bool:
+        return False
+
+    def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
+        return self.info.get_slice_image_placeholder(
+            image_size,
+            image_idx=image_idx,
+        )
+
+    def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str:
+        return (
+            self.info.get_slice_image_placeholder(
+                image_size=image_size,
+                image_idx=0,
+                max_slice_nums=self.info.get_video_max_slice_num(),
+                use_image_id=False,
+            )
+            * num_frames
+        )
+
+    def get_audio_prompt_texts(
+        self,
+        audio_lens: int,
+        chunk_input: bool = False,  # it should be False to avoid different process
+        chunk_length: int = 1,
+    ) -> str:
+        return self.info.get_audio_placeholder(
+            audio_lens,
+            chunk_input=chunk_input,
+            chunk_length=chunk_length,
+        )
+
+    def process_audios(
+        self,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> Mapping[str, NestedTensors]:
+        if (audios := mm_data.get("audios")) is None:
+            return {}
+
+        parsed_audios = self.data_parser.parse_mm_data({"audio": audios}).get_items(
+            "audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
+        )
+
+        if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
+            audio_inputs = {}
+        else:
+            audio_inputs = self._base_call_hf_processor(
+                prompts=[self.info.audio_pattern] * len(parsed_audios),
+                mm_data={"audios": [[audio] for audio in parsed_audios]},
+                mm_kwargs={**mm_kwargs, "chunk_input": True},
+                tok_kwargs=tok_kwargs,
+                out_keys={"audio_features", "audio_feature_lens"},
+            )
+
+            unpadded_audio_features = [
+                feat[:, :feature_len]
+                for feat, feature_len in zip(
+                    audio_inputs["audio_features"],
+                    audio_inputs["audio_feature_lens"],
+                )
+            ]
+            audio_inputs["audio_features"] = unpadded_audio_features
+
+        return audio_inputs
+
+    def process_images(
+        self,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> Mapping[str, NestedTensors]:
+        if (images := mm_data.get("images")) is None:
+            return {}
+
+        parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
+            "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
+        )
+
+        if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
+            image_inputs = {}
+        else:
+            image_inputs = self._base_call_hf_processor(
+                prompts=[self.info.image_pattern] * len(parsed_images),
+                mm_data={"images": [[image] for image in parsed_images]},
+                mm_kwargs=mm_kwargs,
+                tok_kwargs=tok_kwargs,
+                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
+            )
+        return image_inputs
+
+    def process_videos(
+        self,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> Mapping[str, NestedTensors]:
+        if (videos := mm_data.get("videos")) is None:
+            return {}
+
+        parsed_videos = self.data_parser.parse_mm_data({"video": videos}).get_items(
+            "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
+        )
+
+        if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
+            video_inputs = {}
+        else:
+            video_inputs = self._base_call_hf_processor(
+                prompts=[self.info.image_pattern * len(video) for video in parsed_videos],
+                mm_data={"images": list(parsed_videos)},
+                mm_kwargs={
+                    **mm_kwargs,
+                    "max_slice_nums": self.info.get_video_max_slice_num(),
+                },
+                tok_kwargs=tok_kwargs,
+                out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
+            )
+
+        video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
+
+        return video_inputs
+
+    def process_mm_inputs(
+        self,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> Mapping[str, NestedTensors]:
+        return {
+            **self.process_images(mm_data, mm_kwargs, tok_kwargs),
+            **self.process_videos(mm_data, mm_kwargs, tok_kwargs),
+            **self.process_audios(mm_data, mm_kwargs, tok_kwargs),
+        }
+
+    def _base_call_hf_processor(
+        self,
+        prompts: list[str],
+        mm_data: Mapping[str, Sequence[object]],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+        *,
+        out_keys: set[str],
+    ) -> dict[str, NestedTensors]:
+        mm_kwargs = {k: v for k, v in mm_kwargs.items() if k != "use_tts"}
+        # This processor supports zipping prompt and mm_data together
+        if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
+            inputs = super()._call_hf_processor(
+                prompt=prompts,  # type: ignore
+                mm_data=mm_data,
+                mm_kwargs=mm_kwargs,
+                tok_kwargs=tok_kwargs,
+            )
+        else:
+            inputs = defaultdict[str, list[torch.Tensor]](list)
+
+            for i, prompt in enumerate(prompts):
+                inputs_one = super()._call_hf_processor(
+                    prompt=prompt,
+                    mm_data={k: v[i] for k, v in mm_data.items()},
+                    mm_kwargs=mm_kwargs,
+                    tok_kwargs=tok_kwargs,
+                )
+
+                for k, v in inputs_one.items():
+                    assert len(v) == 1, (k, len(v))
+                    inputs[k].append(v[0])
+
+        return {k: inputs[k] for k in out_keys}
+
+    def _get_prompt_updates(
+        self,
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+        out_mm_kwargs: MultiModalKwargsItems,
+    ) -> Sequence[PromptUpdate]:
+        placeholders = [
+            ("image", self.info.image_pattern),
+            ("video", self.info.video_pattern),
+        ]
+
+        # hard code for inconsistency of encode-decode image_pattern
+        additional_placeholders = []
+        tokenizer = self.info.get_tokenizer()
+        for modality, pattern in placeholders:
+            sub_pattern = tokenizer.decode(tokenizer.encode(pattern, add_special_tokens=False))
+            if sub_pattern != pattern:
+                additional_placeholders.append((modality, sub_pattern))
+        placeholders += additional_placeholders
+
+        def get_image_replacement(item_idx: int):
+            images = mm_items.get_items("image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))
+
+            image_size = images.get_image_size(item_idx)
+
+            return PromptUpdateDetails.select_text(
+                self.get_image_prompt_texts(image_size, item_idx),
+                "<unk>",
+            )
+
+        def get_video_replacement(item_idx: int):
+            videos = mm_items.get_items("video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems))
+
+            frame_size = videos.get_frame_size(item_idx)
+            num_frames = videos.get_num_frames(item_idx)
+
+            return PromptUpdateDetails.select_text(
+                self.get_video_prompt_texts(frame_size, num_frames),
+                "<unk>",
+            )
+
+        get_replacement = {
+            "image": get_image_replacement,
+            "video": get_video_replacement,
+        }
+        base_updates = [
+            PromptReplacement(modality=modality, target=pattern, replacement=get_replacement[modality])
+            for modality, pattern in placeholders
+        ]
+
+        audio_placeholder = self.info.audio_pattern
+
+        def get_audio_replacement(item_idx: int):
+            audios = mm_items.get_items("audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems))
+
+            if isinstance(audios, MiniCPMOAudioEmbeddingItems):
+                single_audio_embeds = audios.get(item_idx)["audio_embeds"]
+                audio_len = self.info.get_audio_len_by_num_chunks(sum(map(len, single_audio_embeds)))
+            else:
+                audio_len = audios.get_audio_length(item_idx)
+
+            return PromptUpdateDetails.select_text(
+                self.get_audio_prompt_texts(audio_len),
+                "<unk>",
+            )
+
+        return [
+            *base_updates,
+            PromptReplacement(
+                modality="audio",
+                target=audio_placeholder,
+                replacement=get_audio_replacement,
+            ),
+        ]
+
+    def _get_mm_fields_config(
+        self,
+        hf_inputs: BatchFeature,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> Mapping[str, MultiModalFieldConfig]:
+        return _minicpmo_field_config(hf_inputs)
+
+
+class MultiModalProjector(nn.Module):
+    def __init__(self, in_dim: int, out_dim: int):
+        super().__init__()
+        self.linear1 = nn.Linear(in_features=in_dim, out_features=out_dim, bias=True)
+        self.relu = nn.ReLU()
+        self.linear2 = nn.Linear(in_features=out_dim, out_features=out_dim, bias=True)
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.relu(self.linear1(audio_features))
+        hidden_states = self.linear2(hidden_states)
+        return hidden_states
+
+
+class MiniCPMOAudioFeatureInputs(TensorSchema):
+    """
+    Dimensions:
+        - bns: Batch size * number of audios * number of slices
+        - bn: Batch size * number of audios
+        - c: Number of channels
+        - l: Length
+        - s: Number of slices
+    """
+
+    type: Literal["audio_features"] = "audio_features"
+
+    audio_features: Annotated[
+        torch.Tensor | list[torch.Tensor],
+        TensorShape("bns", "c", "l", dynamic_dims={"l"}),
+    ]
+    """
+    Slice here means chunk. Audio that is too long will be split into slices,
+    which is the same as image. Padding is used therefore `audio_features` is
+    `torch.Tensor`.
+    """
+
+    audio_feature_lens: Annotated[
+        torch.Tensor | list[torch.Tensor],
+        TensorShape("bn", "s"),
+    ]
+    """
+    This should be feature length of each audio slice,
+    which equals to `audio_features.shape[-1]`
+    """
+
+
+class MiniCPMOAudioEmbeddingInputs(TensorSchema):
+    """
+    Dimensions:
+        - bn: Batch size * number of audios
+        - s: Number of slices
+        - h: Hidden size (must match language model backbone)
+
+    Length of each slice may vary, so pass it as a list.
+    """
+
+    type: Literal["audio_embeds"] = "audio_embeds"
+
+    audio_embeds: Annotated[
+        torch.Tensor | list[torch.Tensor],
+        TensorShape("bn", "s", "h", dynamic_dims={"s"}),
+    ]
+
+
+class MiniCPMVImagePixelInputs(TensorSchema):
+    """
+    Dimensions:
+        - bns: Batch size * number of images * number of slices
+        - bn: Batch size * number of images
+        - c: Number of channels
+        - h: Height
+        - w: Width
+    """
+
+    type: Literal["pixel_values"] = "pixel_values"
+
+    # Note that the patch size may vary, so we pass it as a list instead of a
+    # batched tensor.
+    pixel_values: Annotated[
+        list[torch.Tensor],
+        TensorShape("bns", "c", "h", "w", dynamic_dims={"h", "w"}),
+    ]
+    tgt_sizes: Annotated[
+        torch.Tensor,
+        TensorShape("bns", 2),  # This should be in `(height, width)` format.
+    ]
+    num_slices: Annotated[
+        torch.Tensor,
+        TensorShape("bn"),
+    ]
+
+
+class MiniCPMVImageEmbeddingInputs(TensorSchema):
+    """
+    Dimensions:
+        - bn: Batch size * number of images
+        - ns: Number of slices
+        - hs: Hidden size (must match language model backbone)
+    """
+
+    type: Literal["image_embeds"]
+    image_embeds: Annotated[
+        torch.Tensor | list[torch.Tensor],
+        TensorShape("bn", "ns", "hs"),
+    ]
+
+
+MiniCPMOAudioInputs: TypeAlias = MiniCPMOAudioFeatureInputs | MiniCPMOAudioEmbeddingInputs
+
+MiniCPMVImageInputs = MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    info=MiniCPMO45OmniLLMProcessingInfo,
+    dummy_inputs=MiniCPMO45OmniLLMDummyInputsBuilder,
+)
+class MiniCPMO45OmniLLMForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP, SupportsMRoPE):
+    """MiniCPM-o Thinker model: Image preprocessing + Vision encoder + 3D Resampler + LLM.
+
+    This model processes images through:
+    1. Image preprocessing (MiniCPMVImageProcessor)
+    2. Vision encoder (SiglipVisionTransformer)
+    3. 3D Resampler (Resampler to convert vision features to fixed-size queries)
+    4. LLM (Qwen2ForCausalLM for text generation)
+    """
+
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+        if modality.startswith("image"):
+            return "(<image>./</image>)"
+        if modality.startswith("video"):
+            return "(<video>./</video>)"
+        if modality.startswith("audio"):
+            return "(<audio>./</audio>)"
+        raise ValueError("Only image, video or audio modality is supported")
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        super().__init__()
+        config: MiniCPMOConfig = vllm_config.model_config.hf_config
+        multimodal_config = vllm_config.model_config.multimodal_config
+
+        self.config = config
+        self.multimodal_config = multimodal_config
+
+        # Initialize image processor
+        self.image_processor = MiniCPMVImageProcessor(
+            max_slice_nums=config.slice_config.max_slice_nums,
+            scale_resolution=config.image_size,
+            patch_size=config.vision_config.patch_size,
+            use_image_id=config.use_image_id,
+            image_feature_size=config.query_num,
+        )
+
+        # Initialize vision encoder (SigLIP)
+        if multimodal_config.get_limit_per_prompt("image"):
+            # Set attention implementation
+            if hasattr(vllm_config, "model_config") and hasattr(vllm_config.model_config, "_attn_implementation"):
+                config.vision_config._attn_implementation = vllm_config.model_config._attn_implementation
+            else:
+                config.vision_config._attn_implementation = "eager"
+
+            self.vpm = SiglipVisionTransformer(config.vision_config)
+            # Drop last layer if configured
+            if config.drop_vision_last_layer:
+                self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
+
+            # Set embed_dim and patch_size attributes for compatibility
+            setattr(self.vpm, "embed_dim", self.vpm.embeddings.embed_dim)
+            setattr(self.vpm, "patch_size", self.vpm.embeddings.patch_size)
+        else:
+            self.vpm = None
+
+        config_dict = config.to_dict()
+        use_qwen3 = config_dict.get("attention_bias") is False
+        if use_qwen3:
+            from transformers import Qwen3Config as _Qwen3Config
+
+            text_config = _Qwen3Config.from_dict(config_dict)
+            llm_arch = "Qwen3ForCausalLM"
+        else:
+            text_config = Qwen2Config.from_dict(config_dict)
+            llm_arch = "Qwen2ForCausalLM"
+        self.llm = init_vllm_registered_model(
+            vllm_config=vllm_config,
+            prefix=maybe_prefix(prefix, "llm"),
+            hf_config=text_config,
+            architectures=[llm_arch],
+        )
+
+        embed_dim = self.llm.config.hidden_size
+
+        # Initialize 3D Resampler
+        if self.vpm is not None:
+            vision_dim = self.vpm.embed_dim
+            self.resampler = Resampler(
+                num_queries=config.query_num,
+                embed_dim=embed_dim,  # LLM embedding dimension
+                num_heads=embed_dim // 128,  # Default from MiniCPM-o
+                kv_dim=vision_dim,  # Vision encoder dimension
+                adaptive=True,  # Enable adaptive resampling
+            )
+        else:
+            self.resampler = None
+
+        # Initialize audio encoder (APM) and audio projection
+        if getattr(config, "init_audio", True) and hasattr(config, "audio_config") and config.audio_config is not None:
+            self.apm = MiniCPMWhisperEncoder(config.audio_config)
+            audio_output_dim = int(self.apm.config.encoder_ffn_dim // 4)
+            self.audio_avg_pooler = nn.AvgPool1d(config.audio_pool_step, stride=config.audio_pool_step)
+            self.audio_projection_layer = MultiModalProjector(in_dim=audio_output_dim, out_dim=embed_dim)
+            self.audio_encoder_layer = -1
+        else:
+            self.apm = None
+            self.audio_avg_pooler = None
+            self.audio_projection_layer = None
+            self.audio_encoder_layer = None
+
+        self.mm_token_ids = set[int]()
+        self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors
+
+    def get_language_model(self) -> torch.nn.Module:
+        return self.llm
+
+    def _process_image_input(self, image_input: dict[str, torch.Tensor]) -> list[torch.Tensor]:
+        """Process image input through vision encoder and resampler.
+
+        Args:
+            image_input: Dict containing:
+                - pixel_values: List of preprocessed image tensors or tensor
+                - tgt_sizes: Target sizes for each image after patch embedding
+
+        Returns:
+            List of image embeddings (one per image)
+        """
+        if self.vpm is None or self.resampler is None:
+            return []
+
+        pixel_values_list = image_input.get("pixel_values")
+        tgt_sizes = image_input.get("tgt_sizes")
+
+        if pixel_values_list is None or tgt_sizes is None:
+            return []
+
+        # Normalize input format
+        if not isinstance(pixel_values_list, list):
+            pixel_values_list = [pixel_values_list]
+        if not isinstance(tgt_sizes, list | torch.Tensor):
+            tgt_sizes = [tgt_sizes] if tgt_sizes is not None else []
+
+        # Convert to tensors if needed
+        if isinstance(tgt_sizes, list):
+            tgt_sizes = torch.vstack(
+                [torch.tensor(ts) if not isinstance(ts, torch.Tensor) else ts for ts in tgt_sizes]
+            ).type(torch.int32)
+
+        dtype = self.llm.model.embed_tokens.weight.dtype
+        device = self.llm.model.embed_tokens.weight.device
+
+        # Process all images
+        all_pixel_values = []
+        img_cnt = []
+        for pixel_values in pixel_values_list:
+            if isinstance(pixel_values, list):
+                img_cnt.append(len(pixel_values))
+                all_pixel_values.extend(
+                    [pv.flatten(end_dim=1).permute(1, 0) if pv.ndim > 2 else pv for pv in pixel_values]
+                )
+            else:
+                img_cnt.append(1)
+                if pixel_values.ndim > 2:
+                    all_pixel_values.append(pixel_values.flatten(end_dim=1).permute(1, 0))
+                else:
+                    all_pixel_values.append(pixel_values)
+
+        if not all_pixel_values:
+            return []
+
+        # Stack and reshape pixel values
+        tgt_sizes = (
+            [ts for ts in tgt_sizes if isinstance(ts, torch.Tensor)] if isinstance(tgt_sizes, list) else tgt_sizes
+        )
+        if isinstance(tgt_sizes, list):
+            tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
+
+        max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
+
+        all_pixel_values = torch.nn.utils.rnn.pad_sequence(all_pixel_values, batch_first=True, padding_value=0.0)
+        B, L, _ = all_pixel_values.shape
+        all_pixel_values = all_pixel_values.permute(0, 2, 1).reshape(B, 3, -1, L)
+
+        # Create patch attention mask
+        patch_attn_mask = torch.zeros((B, 1, max_patches), dtype=torch.bool, device=device)
+        for i in range(B):
+            patch_attn_mask[i, 0, : tgt_sizes[i][0] * tgt_sizes[i][1]] = True
+
+        # Process through vision encoder with batching
+        all_pixel_values = all_pixel_values.type(dtype)
+        vision_batch_size = self.config.vision_batch_size
+
+        if B > vision_batch_size:
+            vision_embeddings = []
+            for i in range(0, B, vision_batch_size):
+                start_idx = i
+                end_idx = min(i + vision_batch_size, B)
+                tmp_emb = self.vpm(
+                    all_pixel_values[start_idx:end_idx],
+                    patch_attention_mask=patch_attn_mask[start_idx:end_idx],
+                    tgt_sizes=tgt_sizes[start_idx:end_idx],
+                ).last_hidden_state
+                vision_embeddings.append(tmp_emb)
+            vision_embedding = torch.cat(vision_embeddings, dim=0)
+        else:
+            vision_embedding = self.vpm(
+                all_pixel_values,
+                patch_attention_mask=patch_attn_mask,
+                tgt_sizes=tgt_sizes,
+            ).last_hidden_state
+
+        # Apply resampler
+        image_embeds = self.resampler(vision_embedding, tgt_sizes)  # (B, query_num, embed_dim)
+
+        # Split back into per-image embeddings
+        image_embeddings = []
+        start = 0
+        for cnt in img_cnt:
+            if cnt > 0:
+                image_embeddings.append(image_embeds[start : start + cnt])
+                start += cnt
+            else:
+                image_embeddings.append([])
+
+        return image_embeddings
+
+    # def _parse_and_validate_vision_input(
+    #     self,
+    #     modality: str,
+    #     **kwargs: object,
+    # ) -> MiniCPMVImageInputs | None:
+    #     pixel_values = kwargs.pop("pixel_values", None)
+    #     image_embeds = kwargs.pop("image_embeds", None)
+    #
+    #     if pixel_values is None and image_embeds is None:
+    #         return None
+    #
+    #     if image_embeds is not None:
+    #         return MiniCPMVImageEmbeddingInputs(
+    #             type="image_embeds",
+    #             image_embeds=image_embeds,
+    #         )
+    #
+    #     tgt_sizes = kwargs.pop("tgt_sizes")
+    #
+    #     num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
+    #     pixel_values_flat = flatten_bn(pixel_values)
+    #     tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
+    #
+    #     return MiniCPMVImagePixelInputs(
+    #         type="pixel_values",
+    #         pixel_values=pixel_values_flat,
+    #         tgt_sizes=tgt_sizes_flat,
+    #         num_slices=num_slices_flat,
+    #     )
+
+    def _parse_and_validate_vision_input(
+        self,
+        modality: str,
+        **kwargs: object,
+    ) -> MiniCPMVImageInputs | None:
+        pixel_values = kwargs.pop("pixel_values", None)
+        image_embeds = kwargs.pop("image_embeds", None)
+
+        if pixel_values is None and image_embeds is None:
+            return None
+
+        if image_embeds is not None:
+            return MiniCPMVImageEmbeddingInputs(
+                type="image_embeds",
+                image_embeds=image_embeds,
+            )
+
+        tgt_sizes = kwargs.pop("tgt_sizes")
+
+        # vLLM 0.18 profiling batches pixel_values as a dense tensor; the HF path
+        # uses nested lists [[slice_tensor, ...], ...] per image/video.
+        if isinstance(pixel_values, torch.Tensor):
+            t = pixel_values
+            if t.ndim == 5:
+                t = flatten_bn(t)
+            if t.ndim == 4:
+                if t.shape[1] != 3:
+                    raise ValueError(f"pixel_values tensor must have shape (N, 3, H, W), got {tuple(t.shape)}")
+                n = t.shape[0]
+                pixel_values = [[t[i].contiguous() for i in range(n)]]
+            elif t.ndim == 3 and t.shape[1] == 3:
+                n = t.shape[0]
+                pixel_values = [[t[i].contiguous() for i in range(n)]]
+            else:
+                raise ValueError(f"Unsupported pixel_values tensor for vision parsing: shape={tuple(t.shape)}")
+
+        # Match vLLM MiniCPMV: count slices per image with len(ps), and flatten
+        # only the outer nesting. Do NOT use flatten_2d_lists here — flatten_bn
+        # on a flat list of (3,H,W) tensors would iterate dim 0 and split RGB.
+        num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
+        pixel_values_flat = flatten_bn(pixel_values)
+
+        if isinstance(tgt_sizes, torch.Tensor):
+            ts = tgt_sizes
+            if ts.ndim == 3:
+                tgt_sizes_flat = ts.flatten(0, 1).contiguous()
+            elif ts.ndim == 2:
+                tgt_sizes_flat = ts.contiguous()
+            else:
+                raise ValueError(f"Unsupported tgt_sizes tensor shape {tuple(ts.shape)}")
+        else:
+            tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
+
+        return MiniCPMVImagePixelInputs(
+            type="pixel_values",
+            pixel_values=pixel_values_flat,
+            tgt_sizes=tgt_sizes_flat,
+            num_slices=num_slices_flat,
+        )
+
+    def _parse_and_validate_audio_input(self, **kwargs: object) -> MiniCPMOAudioInputs | None:
+        audio_features = kwargs.pop("audio_features", None)
+        audio_embeds = kwargs.pop("audio_embeds", None)
+
+        if audio_features is None and audio_embeds is None:
+            return None
+
+        if audio_embeds is not None:
+            return MiniCPMOAudioEmbeddingInputs(
+                type="audio_embeds",
+                audio_embeds=audio_embeds,
+            )
+
+        audio_feature_lens = kwargs.pop("audio_feature_lens")
+
+        # Flatten first two dims to match schema expectations:
+        #   audio_features:    (batch, slices, c, l) → (bns, c, l)  [4D→3D]
+        #   audio_feature_lens: (batch, n, s)        → (bn, s)      [3D→2D]
+        if isinstance(audio_features, torch.Tensor) and audio_features.ndim == 4:
+            audio_features = audio_features.reshape(-1, *audio_features.shape[2:])
+        if isinstance(audio_feature_lens, torch.Tensor) and audio_feature_lens.ndim == 3:
+            audio_feature_lens = audio_feature_lens.reshape(-1, *audio_feature_lens.shape[2:])
+
+        return MiniCPMOAudioFeatureInputs(
+            type="audio_features",
+            audio_features=audio_features,
+            audio_feature_lens=audio_feature_lens,
+        )
+
+    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
+        modalities = {}
+
+        # Preserve the order of modalities if there are multiple of them
+        # from the order of kwargs.
+        for input_key in kwargs:
+            if input_key in ("pixel_values", "image_embeds") and "images" not in modalities:
+                modalities["images"] = self._parse_and_validate_vision_input("images", **kwargs)
+            if input_key in ("video_pixel_values", "video_embeds") and "videos" not in modalities:
+                modalities["videos"] = self._parse_and_validate_vision_input(
+                    "videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
+                )
+            if input_key in ("audio_features", "audio_embeds") and "audios" not in modalities:
+                modalities["audios"] = self._parse_and_validate_audio_input(**kwargs)
+
+        return modalities
+
+    # V2.6 compatible
+    def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
+        pixel_values = data["pixel_values"]
+        tgt_sizes = data["tgt_sizes"]
+
+        B = len(pixel_values)
+        P = pixel_values[0].shape[-2]
+        L = max(item.shape[-1] for item in pixel_values)
+        device = pixel_values[0].device
+        dtype = pixel_values[0].dtype
+
+        all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device)
+        for i, pixel_values_item in enumerate(pixel_values):
+            L_item = pixel_values_item.shape[-1]
+            all_pixel_values[i, ..., :L_item] = pixel_values_item
+
+        num_patches = tgt_sizes.prod(-1)
+        max_patches = num_patches.max().item()
+        assert isinstance(max_patches, int)
+
+        patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
+        for i, num_patches_item in enumerate(num_patches):
+            patch_attn_mask[i, :num_patches_item] = True
+
+        vision_embedding = self.vpm(
+            all_pixel_values,
+            patch_attention_mask=patch_attn_mask.unsqueeze(1),
+            tgt_sizes=tgt_sizes,
+        )
+
+        if not isinstance(vision_embedding, torch.Tensor):
+            vision_embedding = vision_embedding.last_hidden_state
+
+        return self.resampler(vision_embedding, tgt_sizes)
+
+    def _process_vision_input(
+        self,
+        image_input: MiniCPMVImageInputs,
+    ) -> torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...]:
+        if image_input["type"] == "image_embeds":
+            return image_input["image_embeds"]
+
+        image_features_flat = self.get_vision_hidden_states(image_input)
+
+        num_slices = image_input["num_slices"]
+        return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())]
+
+    def subsequent_chunk_mask(
+        self,
+        size: int,
+        chunk_size: int,
+        num_left_chunks: int = -1,
+        device: torch.device = torch.device("cpu"),
+        num_lookhead: int = 0,
+    ) -> torch.Tensor:
+        """Create mask for subsequent steps (size, size) with chunk size,
+        this is for streaming encoder
+
+        Args:
+            size (int): size of mask
+            chunk_size (int): size of chunk
+            num_left_chunks (int): number of left chunks
+                <0: use full chunk
+                >=0: use num_left_chunks
+            device (torch.device): "cpu" or "cuda" or torch.Tensor.device
+
+        Returns:
+            torch.Tensor: mask
+
+        """
+        ret = torch.zeros(size, size, device=device, dtype=torch.bool)
+        for i in range(size):
+            if num_left_chunks < 0:
+                start = 0
+            else:
+                start = max((i // chunk_size - num_left_chunks) * chunk_size, 0)
+            ending = min((i // chunk_size + 1) * chunk_size + num_lookhead, size)
+            ret[i, start:ending] = True
+        return ret
+
+    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
+        input_lengths_after_cnn = (input_lengths - 1) // 2 + 1
+        input_lengths_after_pooling = (
+            input_lengths_after_cnn - self.config.audio_pool_step
+        ) // self.config.audio_pool_step + 1
+        input_lengths_after_pooling = input_lengths_after_pooling.to(dtype=torch.int32)
+
+        return input_lengths_after_cnn, input_lengths_after_pooling
+
+    def get_audio_hidden_states(self, data: MiniCPMOAudioFeatureInputs) -> list[torch.Tensor]:
+        chunk_length = self.config.audio_chunk_length
+
+        # (bs, 80, frames) or [], multi audios need filled in advance
+        wavforms_raw = data["audio_features"]
+        if isinstance(wavforms_raw, list):
+            B = len(wavforms_raw)
+            C = wavforms_raw[0].shape[-2]
+            L = max(item.shape[-1] for item in wavforms_raw)
+            device = wavforms_raw[0].device
+            dtype = wavforms_raw[0].dtype
+
+            wavforms = torch.zeros((B, C, L), dtype=dtype, device=device)
+            for i, wavforms_item in enumerate(wavforms_raw):
+                L_item = wavforms_item.shape[-1]
+                wavforms[i, ..., :L_item] = wavforms_item
+        else:
+            wavforms = wavforms_raw
+
+        # list, [[x1, x2], [y1], [z1]]
+        audio_feature_lens_raw = data["audio_feature_lens"]
+        if isinstance(audio_feature_lens_raw, torch.Tensor):
+            audio_feature_lens_raw = audio_feature_lens_raw.unbind(0)
+
+        audio_feature_lens = torch.hstack(audio_feature_lens_raw)
+        batch_size, _, max_mel_seq_len = wavforms.shape
+        max_seq_len = (max_mel_seq_len - 1) // 2 + 1
+
+        # Create a sequence tensor of shape (batch_size, max_seq_len)
+        seq_range = (
+            torch.arange(
+                0,
+                max_seq_len,
+                dtype=audio_feature_lens.dtype,
+                device=audio_feature_lens.device,
+            )
+            .unsqueeze(0)
+            .expand(batch_size, max_seq_len)
+        )
+        lengths_expand = audio_feature_lens.unsqueeze(1).expand(batch_size, max_seq_len)
+        # Create mask
+        padding_mask = seq_range >= lengths_expand  # 1 for padded values
+
+        audio_attention_mask_ = padding_mask.view(batch_size, 1, 1, max_seq_len).expand(
+            batch_size, 1, max_seq_len, max_seq_len
+        )
+        audio_attention_mask = audio_attention_mask_.to(
+            dtype=self.apm.conv1.weight.dtype, device=self.apm.conv1.weight.device
+        )
+
+        if chunk_length > 0:
+            chunk_num_frame = int(chunk_length * 50)
+            chunk_mask = self.subsequent_chunk_mask(
+                size=max_seq_len,
+                chunk_size=chunk_num_frame,
+                num_left_chunks=-1,
+                device=audio_attention_mask_.device,
+            )
+            audio_attention_mask_ = torch.logical_or(audio_attention_mask_, torch.logical_not(chunk_mask))
+
+        audio_attention_mask[audio_attention_mask_] = float("-inf")
+        audio_states = self.apm(wavforms, attention_mask=audio_attention_mask, output_hidden_states=True).hidden_states[
+            self.audio_encoder_layer
+        ]
+        audio_embeds = self.audio_projection_layer(audio_states)
+
+        audio_embeds = audio_embeds.transpose(1, 2)
+        audio_embeds = self.audio_avg_pooler(audio_embeds)
+        audio_embeds = audio_embeds.transpose(1, 2)
+
+        _, feature_lens_after_pooling = self._get_feat_extract_output_lengths(audio_feature_lens)
+
+        num_audio_tokens = feature_lens_after_pooling
+
+        final_audio_embeds = list[torch.Tensor]()
+        idx = 0
+        for i in range(len(audio_feature_lens_raw)):
+            target_audio_embeds_lst = list[torch.Tensor]()
+            for _ in range(len(audio_feature_lens_raw[i])):
+                target_audio_embeds_lst.append(audio_embeds[idx, : num_audio_tokens[idx], :])
+                idx += 1
+
+            final_audio_embeds.append(torch.cat(target_audio_embeds_lst))
+
+        return final_audio_embeds
+
+    def _process_audio_input(
+        self,
+        audio_input: MiniCPMOAudioInputs,
+    ) -> torch.Tensor | list[torch.Tensor]:
+        if audio_input["type"] == "audio_embeds":
+            return audio_input["audio_embeds"]
+
+        return self.get_audio_hidden_states(audio_input)
+
+    def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
+        mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
+        if not mm_input_by_modality:
+            return []
+
+        # The result multimodal_embeddings is tuple of tensors, with each
+        # tensor correspoending to a multimodal data item (image or video).
+        multimodal_embeddings: tuple[torch.Tensor, ...] = ()
+
+        # NOTE: It is important to iterate over the keys in this dictionary
+        # to preserve the order of the modalities.
+        for modality in mm_input_by_modality:
+            multimodal_input = mm_input_by_modality[modality]
+            if modality == "images":
+                image_embeddings = self._process_vision_input(multimodal_input)
+                multimodal_embeddings += tuple(image_embeddings)
+            if modality == "videos":
+                video_embeddings = self._process_vision_input(multimodal_input)
+                multimodal_embeddings += tuple(video_embeddings)
+            if modality == "audios":
+                audio_embeddings = self._process_audio_input(multimodal_input)
+                multimodal_embeddings += tuple(audio_embeddings)
+        return multimodal_embeddings
+
+    def get_input_embeddings(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings: MultiModalEmbeddings | None = None,
+    ) -> torch.Tensor:
+        """Get input embeddings combining text and multimodal features."""
+        inputs_embeds = self.llm.get_input_embeddings(input_ids)
+
+        if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
+            unk_token_id = 128244
+            inputs_embeds = merge_multimodal_embeddings(
+                input_ids,
+                inputs_embeds,
+                multimodal_embeddings,
+                [unk_token_id],
+            )
+
+        return inputs_embeds
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        **kwargs: object,
+    ) -> torch.Tensor | IntermediateTensors:
+        """Forward pass through thinker model."""
+        text_inputs_embeds = None
+
+        if intermediate_tensors is not None:
+            inputs_embeds = None
+
+        # NOTE: In v1, inputs_embeds is always generated at model runner
+        elif inputs_embeds is None:
+            multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
+            inputs_embeds = self.get_input_embeddings(input_ids, multimodal_embeddings)
+            text_inputs_embeds = self.get_input_embeddings(
+                input_ids,
+                (
+                    [(torch.zeros_like(embeddings), "image") for embeddings in multimodal_embeddings]
+                    if multimodal_embeddings is not None
+                    else None
+                ),
+            )
+            input_ids = None
+        else:
+            text_inputs_embeds = inputs_embeds
+
+        # Forward through language model
+        hidden_states = self.llm.model(input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds)
+
+        return text_inputs_embeds, hidden_states.unsqueeze(0) if hidden_states.ndim == 2 else hidden_states
+
+    def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
+        """Compute logits from hidden states."""
+        return self.llm.compute_logits(hidden_states)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+        """Load weights for thinker model components."""
+        loaded_weights = set()
+        vision_weights = []
+        resampler_weights = []
+        language_model_weights = []
+        apm_weights = []
+        audio_projection_weights = []
+
+        for k, v in weights:
+            if k.startswith("vpm."):
+                vision_weights.append((k, v))
+            elif k.startswith("resampler."):
+                resampler_weights.append((k, v))
+            elif k.startswith("llm."):
+                language_model_weights.append((k, v))
+            elif k.startswith("apm."):
+                apm_weights.append((k, v))
+            elif k.startswith("audio_projection_layer."):
+                audio_projection_weights.append((k, v))
+            else:
+                logger.warning("Unknown thinker weight: %s, skipping", k)
+
+        # --- 视觉编码器 ---
+        if self.vpm is not None and vision_weights:
+            clean = [(k.replace("vpm.", ""), v) for k, v in vision_weights]
+            missing, unexpected = self.vpm.load_state_dict(dict(clean), strict=False)
+            if missing:
+                logger.warning("Vision missing: %s", missing)
+            if unexpected:
+                logger.warning("Vision unexpected: %s", unexpected)
+            for k, _ in vision_weights:
+                suffix = k.replace("vpm.", "")
+                loaded_weights.add("vpm." + suffix)
+
+        # --- Resampler ---
+        if self.resampler is not None and resampler_weights:
+            clean = [(k.replace("resampler.", ""), v) for k, v in resampler_weights]
+            missing, unexpected = self.resampler.load_state_dict(dict(clean), strict=False)
+            if missing:
+                logger.warning("Resampler missing: %s", missing)
+            if unexpected:
+                logger.warning("Resampler unexpected: %s", unexpected)
+            loaded_weights.update("resampler." + k for k, _ in clean)
+
+        # --- 语言模型 (strip "llm." → "model.*" / "lm_head.*") ---
+        if language_model_weights:
+            stripped = [(k.replace("llm.", "", 1), v) for k, v in language_model_weights]
+            lm_loaded = self.llm.load_weights(stripped)
+            loaded_weights.update("llm." + k for k in lm_loaded)
+
+        # --- 音频编码器 (apm) ---
+        if self.apm is not None and apm_weights:
+            clean = [(k.replace("apm.", ""), v) for k, v in apm_weights]
+            missing, unexpected = self.apm.load_state_dict(dict(clean), strict=False)
+            if missing:
+                logger.warning("APM missing: %s", missing)
+            if unexpected:
+                logger.warning("APM unexpected: %s", unexpected)
+            loaded_weights.update("apm." + k for k, _ in clean)
+
+        # --- 音频投影层 ---
+        if self.audio_projection_layer is not None and audio_projection_weights:
+            clean = [(k.replace("audio_projection_layer.", ""), v) for k, v in audio_projection_weights]
+            missing, unexpected = self.audio_projection_layer.load_state_dict(dict(clean), strict=False)
+            if missing:
+                logger.warning("AudioProj missing: %s", missing)
+            if unexpected:
+                logger.warning("AudioProj unexpected: %s", unexpected)
+            loaded_weights.update("audio_projection_layer." + k for k, _ in clean)
+
+        return loaded_weights
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py
new file mode 100644
index 000000000..c7208752b
--- /dev/null
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py
@@ -0,0 +1,456 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from:
+# https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/main/modeling_minicpmo.py
+"""MiniCPM-o 4.5 Talker + Token2Wav: MiniCPMTTS with hidden_text_merge condition.
+
+Pipeline:
+  1. Receive thinker hidden_states + full token IDs via additional_information
+  2. Extract tts_bos..tts_eos region
+  3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge)
+  4. Run MiniCPMTTS.generate() -> discrete audio tokens
+  5. Run Token2wav(tokens) -> waveform bytes -> numpy array
+"""
+
+import io
+import logging
+import os
+from collections.abc import Iterable
+
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from vllm.config import VllmConfig
+from vllm.model_executor.models.interfaces import SupportsPP
+
+try:
+    from stepaudio2 import Token2wav as _Token2wav
+
+    _stepaudio2_available = True
+except ImportError:
+    _Token2wav = None
+    _stepaudio2_available = False
+
+logger = logging.getLogger(__name__)
+
+
+def _install_torchaudio_soundfile_shim() -> None:
+    """Monkey-patch torchaudio.load to use soundfile instead of the default
+    torchcodec backend, which requires libtorchcodec/ffmpeg shared libs that
+    may be missing on the deployment machine."""
+    try:
+        import torchaudio
+
+        if getattr(torchaudio, "_soundfile_shim_installed", False):
+            return
+        _orig_load = torchaudio.load
+
+        def _patched_load(uri, *args, **kwargs):
+            try:
+                return _orig_load(uri, *args, **kwargs)
+            except Exception:
+                import numpy as _np
+                import soundfile as _sf
+
+                data, sr = _sf.read(uri, dtype="float32", always_2d=True)
+                wav = torch.from_numpy(_np.ascontiguousarray(data.T))
+                return wav, sr
+
+        torchaudio.load = _patched_load
+        torchaudio._soundfile_shim_installed = True
+        logger.info("Installed torchaudio.load soundfile shim")
+    except Exception as _e:
+        logger.warning("Could not install torchaudio shim: %s", _e)
+
+
+_install_torchaudio_soundfile_shim()
+
+
+class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
+    """MiniCPM-o 4.5 Talker: MiniCPMTTS + Token2wav in a single forward pass."""
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+        super().__init__()
+        from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import MiniCPMOConfig
+
+        config: MiniCPMOConfig = vllm_config.model_config.hf_config
+        self.config = config
+        self.vllm_config = vllm_config
+
+        self.tts = None
+        self.audio_tokenizer = None
+        self._assets_loaded = False
+
+        tts_config = getattr(config, "tts_config", None)
+        if tts_config is not None:
+            self._tts_config = tts_config
+            self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687)
+            self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692)
+            self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562)
+            self._hidden_size = getattr(tts_config, "hidden_size", 768)
+            self._normalize = getattr(tts_config, "normalize_projected_hidden", True)
+        else:
+            self._tts_config = None
+
+    def _lazy_init_tts(self):
+        if self._assets_loaded or self._tts_config is None:
+            return
+        try:
+            model_path = self.vllm_config.model_config.model
+            import os
+            import sys
+
+            if model_path not in sys.path:
+                sys.path.insert(0, model_path)
+            from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+            MiniCPMTTS = get_class_from_dynamic_module("modeling_minicpmo.MiniCPMTTS", model_path)
+
+            # MiniCPMTTS.__init__ reads `config.top_p / top_k / repetition_penalty`
+            # directly (modeling_minicpmo.py L4112-4114), but the model repo's
+            # config.json `tts_config` block does not declare these fields and
+            # PretrainedConfig in recent transformers no longer surfaces
+            # generation-style params on `self.config`. Inject the defaults the
+            # upstream code itself ships with (modeling_minicpmo.py L2212-2214,
+            # L3132-3133) so attribute access does not raise.
+            for _attr, _default in (("top_p", 0.8), ("top_k", 100), ("repetition_penalty", 1.02)):
+                if not hasattr(self._tts_config, _attr):
+                    setattr(self._tts_config, _attr, _default)
+
+            prev_dtype = torch.get_default_dtype()
+            torch.set_default_dtype(torch.float32)
+            try:
+                self.tts_obj = MiniCPMTTS(config=self._tts_config, audio_tokenizer=None)
+            finally:
+                torch.set_default_dtype(prev_dtype)
+            self.emb_text = self.tts_obj.emb_text
+            self.projector_semantic = self.tts_obj.projector_semantic
+
+            token2wav_dir = os.path.join(model_path, "assets", "token2wav")
+            if os.path.isdir(token2wav_dir):
+                if not _stepaudio2_available:
+                    raise ImportError(
+                        "MiniCPM-o 4.5 token2wav stage requires the `stepaudio2` Python "
+                        "module (a MiniCPM-o-flavored Token2wav vocoder, NOT the upstream "
+                        "stepfun-ai/Step-Audio2 — the upstream signature does not accept "
+                        "n_timesteps and will fail at __init__). Install via:\n"
+                        "    pip install 'vllm-omni[minicpmo]'   # recommended, declared as PR extra\n"
+                        "Equivalent direct installs of the same `from stepaudio2 import Token2wav`\n"
+                        "entry point used by openbmb/MiniCPM-o-4_5/modeling_minicpmo.py:\n"
+                        "    pip install stepaudio2-minicpmo     # bare token2wav package\n"
+                        "    pip install 'minicpmo-utils[all]'   # MiniCPM-o umbrella (also brings image/video deps)"
+                    )
+                prev_dtype2 = torch.get_default_dtype()
+                torch.set_default_dtype(torch.float32)
+                try:
+                    # NB: this must be the MiniCPM-o-flavored Token2wav from
+                    # the `stepaudio2-minicpmo` PyPI package (or the
+                    # `minicpmo-utils[all]` umbrella), not the upstream
+                    # `stepfun-ai/Step-Audio2` repo. The MiniCPM-o variant's
+                    # __init__ accepts n_timesteps; the upstream signature is
+                    # (model_path, float16=False) and will raise
+                    # TypeError on n_timesteps. See ImportError message below
+                    # for installation guidance.
+                    self.audio_tokenizer = _Token2wav(token2wav_dir, float16=False, n_timesteps=10)
+                finally:
+                    torch.set_default_dtype(prev_dtype2)
+                self.tts_obj.audio_tokenizer = self.audio_tokenizer
+                logger.info("Loaded Token2wav from %s", token2wav_dir)
+            # Only mark init as complete after every step succeeds, so a
+            # partial failure leaves the next call free to retry the full
+            # init instead of short-circuiting back to a silent empty path.
+            self._assets_loaded = True
+        except ImportError:
+            # Surface missing dependencies directly so users can act on them
+            # instead of getting a silent None waveform downstream.
+            raise
+        except Exception:
+            # Re-raise non-import init failures (bad token2wav assets, missing
+            # weights, OOM during Token2wav construction, etc.) so the server
+            # fails loudly at startup / first request instead of returning
+            # silent empty audio for every subsequent request.
+            logger.error("Failed to init 4.5 TTS", exc_info=True)
+            raise
+
+    def generate_speech(
+        self,
+        tts_token_ids: torch.Tensor,
+        tts_hidden_states: torch.Tensor,
+    ) -> np.ndarray | None:
+        """Run full 4.5 TTS pipeline using original MiniCPMTTS.generate."""
+        self._lazy_init_tts()
+        if not hasattr(self, "tts_obj") or self.tts_obj is None:
+            logger.warning("generate_speech: tts_obj not initialized")
+            return None
+
+        tts = self.tts_obj
+        device = tts.emb_text.weight.device
+        dtype = tts.emb_text.weight.dtype
+
+        llm_embeds = tts.emb_text(tts_token_ids.to(device))
+        hidden_embeds = tts.projector_semantic(tts_hidden_states.to(device=device, dtype=dtype))
+        if getattr(tts.config, "normalize_projected_hidden", False):
+            hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1)
+        tts_embeds = llm_embeds + hidden_embeds
+
+        text_eos = tts.emb_text(torch.tensor([tts.config.text_eos_token_id], device=device, dtype=torch.long))
+        audio_bos = tts.emb_text(torch.tensor([tts.audio_bos_token_id], device=device, dtype=torch.long))
+        spk_embeds = torch.zeros(0, tts.config.hidden_size, device=device, dtype=tts_embeds.dtype)
+
+        inputs_embeds = torch.cat([spk_embeds, tts_embeds, text_eos, audio_bos], dim=0).unsqueeze(0)
+        logger.info("generate_speech: inputs_embeds shape=%s", list(inputs_embeds.shape))
+
+        # Scale max_new_token with input text length to avoid mid-stream truncation on long
+        # responses (default 2048 can only cover ~300 text tokens at ~6x audio/text ratio).
+        # Empirically 511 text tokens → 1951 audio tokens (~3.8x) finishes cleanly, so use 10x
+        # as a safe upper bound with a floor of 2048 and a hard cap of 16384 to bound latency/mem.
+        num_text = int(tts_token_ids.shape[-1]) if tts_token_ids.ndim > 0 else 0
+        max_new_token = max(2048, min(16384, num_text * 10))
+
+        eos_token = torch.tensor([tts.config.num_audio_tokens - 1], dtype=torch.long, device=device)
+        outputs = tts.generate(
+            inputs_embeds=inputs_embeds,
+            eos_token=eos_token,
+            max_new_token=max_new_token,
+            show_tqdm=False,
+        )
+        generated_tokens = outputs.new_ids.squeeze(-1)
+        logger.info(
+            "generate_speech: generated %d audio tokens (cap=%d, text_tokens=%d)",
+            generated_tokens.shape[-1],
+            max_new_token,
+            num_text,
+        )
+
+        if self.audio_tokenizer is None:
+            logger.warning("No audio_tokenizer")
+            return None
+
+        import torchaudio
+
+        model_path = self.vllm_config.model_config.model
+        default_ref = os.path.join(model_path, "assets", "HT_ref_audio.wav")
+        prompt_wav_path = default_ref if os.path.exists(default_ref) else None
+
+        _orig_save = torchaudio.save
+
+        def _patched_save(uri, src, sample_rate, **kw):
+            kw.pop("backend", None)
+            if hasattr(uri, "write"):
+                sf.write(uri, src.cpu().numpy().T, sample_rate, format="WAV")
+                return
+            return _orig_save(uri, src, sample_rate, backend="soundfile", **kw)
+
+        torchaudio.save = _patched_save
+        prev_dtype = torch.get_default_dtype()
+        torch.set_default_dtype(torch.float32)
+        try:
+            with torch.amp.autocast("cuda", enabled=False):
+                token_list = generated_tokens.squeeze(0).tolist()
+                num_tokens = len(token_list)
+
+                # For long outputs, the one-shot vocoder path
+                # (Token2wav.__call__ -> flow.inference) runs full O(N^2) self-
+                # attention over all audio tokens and OOMs on a 24GB card once
+                # N exceeds a few thousand (e.g. 4964 tokens needs ~3GiB for a
+                # single attention matmul). Switch to the chunked / streaming
+                # vocoder (set_stream_cache + stream) which truncates the flow
+                # attention caches to prompt_len + 100 steps on every chunk,
+                # keeping peak memory bounded regardless of total length.
+                STREAM_THRESHOLD = int(os.environ.get("MINICPMO45_TTS_STREAM_THRESHOLD", "2500"))  # ~100s @ 25Hz
+                CHUNK_SIZE = int(os.environ.get("MINICPMO45_TTS_STREAM_CHUNK", "50"))  # ~2s per chunk
+                MIN_TAIL = 6  # must exceed flow.pre_lookahead_len (typically 3)
+
+                if num_tokens <= STREAM_THRESHOLD:
+                    wav_bytes = self.audio_tokenizer(token_list, prompt_wav_path)
+                    waveform, sr = sf.read(io.BytesIO(wav_bytes))
+                    waveform = waveform.astype(np.float32)
+                else:
+                    # Build chunk boundaries, merging a too-small tail into the
+                    # previous chunk so every chunk satisfies MIN_TAIL.
+                    boundaries = []
+                    i = 0
+                    while i < num_tokens:
+                        end = min(i + CHUNK_SIZE, num_tokens)
+                        if 0 < num_tokens - end < MIN_TAIL:
+                            end = num_tokens
+                        boundaries.append((i, end))
+                        i = end
+
+                    logger.info(
+                        "generate_speech: streaming vocoder, %d tokens -> %d chunks (chunk=%d)",
+                        num_tokens,
+                        len(boundaries),
+                        CHUNK_SIZE,
+                    )
+
+                    stream_cache, hift_cache_dict = self.audio_tokenizer.set_stream_cache(prompt_wav_path)
+                    self.audio_tokenizer.stream_cache = stream_cache
+                    self.audio_tokenizer.hift_cache_dict = hift_cache_dict
+
+                    try:
+                        pieces = []
+                        for idx, (s, e) in enumerate(boundaries):
+                            is_last = idx == len(boundaries) - 1
+                            wav_np = self.audio_tokenizer.stream(
+                                token_list[s:e],
+                                prompt_wav_path,
+                                last_chunk=is_last,
+                                return_waveform=True,
+                            )
+                            pieces.append(np.asarray(wav_np).reshape(-1))
+                        waveform = np.concatenate(pieces, axis=0).astype(np.float32)
+                        sr = 24000
+                    finally:
+                        # Free per-request streaming state so the next request starts clean
+                        self.audio_tokenizer.stream_cache = None
+                        self.audio_tokenizer.hift_cache_dict = {}
+        finally:
+            torch.set_default_dtype(prev_dtype)
+            torchaudio.save = _orig_save
+
+        logger.info("generate_speech: waveform %d samples, sr=%d", waveform.shape[0], sr)
+        return waveform
+
+    def _generate_tokens(self, inputs_embeds: torch.Tensor, max_new_token: int = 2048) -> torch.Tensor | None:
+        """Autoregressive generation of audio tokens using the TTS LlamaModel."""
+        device = inputs_embeds.device
+        eos_token = self._num_audio_tokens - 1
+        condition_length = inputs_embeds.shape[1]
+        num_vq = len(self.emb_code)
+
+        new_tokens = torch.zeros(1, max_new_token, num_vq, device=device, dtype=torch.long)
+        past_key_values = None
+        finished = False
+
+        for t in range(max_new_token):
+            if t == 0:
+                emb = inputs_embeds
+                position_ids = torch.arange(condition_length, device=device).unsqueeze(0)
+            else:
+                code_emb = [self.emb_code[q](new_tokens[:, t - 1 : t, q]) for q in range(num_vq)]
+                emb = torch.stack(code_emb, -1).sum(-1)
+                position_ids = torch.tensor([[condition_length + t - 1]], device=device)
+
+            outputs = self.tts_model(
+                inputs_embeds=emb,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=True,
+            )
+            hidden = outputs.last_hidden_state
+            past_key_values = outputs.past_key_values
+
+            logits = torch.stack([self.head_code[q](hidden[:, -1]) for q in range(num_vq)], dim=-1)
+            logits = logits.float() / 0.8
+
+            if t < 50:
+                logits[:, eos_token, :] = -float("inf")
+
+            probs = F.softmax(logits, dim=1)
+            idx = torch.multinomial(probs.view(-1, probs.shape[1]), 1).view(1, num_vq)
+            new_tokens[:, t] = idx
+
+            if (idx == eos_token).any():
+                finished = True
+                break
+
+        return new_tokens[:, : t + 1 if finished else t, :]
+
+    def _dummy_hidden_states(
+        self,
+        input_ids: torch.Tensor | None,
+        positions: torch.Tensor | None,
+        inputs_embeds: torch.Tensor | None,
+    ) -> torch.Tensor:
+        """Shape-correct zero tensor for vllm KV cache profiling.
+
+        vllm's gpu_model_runner._dummy_run takes forward()'s return value as
+        ``hidden_states`` and does ``hidden_states[logit_indices_device]``;
+        returning None on the dummy path crashes with
+        ``TypeError: 'NoneType' object is not subscriptable``.
+        """
+        for ref in (input_ids, positions, inputs_embeds):
+            if isinstance(ref, torch.Tensor):
+                num_tokens = int(ref.shape[0]) if ref.ndim >= 1 else 1
+                device = ref.device
+                break
+        else:
+            num_tokens = 1
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        hidden_size = int(getattr(self, "_hidden_size", 768) or 768)
+        return torch.zeros((num_tokens, hidden_size), device=device, dtype=torch.bfloat16)
+
+    def forward(
+        self,
+        input_ids=None,
+        positions=None,
+        intermediate_tensors=None,
+        inputs_embeds=None,
+        additional_information=None,
+        **kwargs,
+    ):
+        if additional_information is None:
+            additional_information = {}
+
+        tts_token_ids = additional_information.get("tts_token_ids")
+        tts_hidden_states = additional_information.get("tts_hidden_states")
+        tts_text = additional_information.get("llm_output_text", [""])
+        if isinstance(tts_text, list):
+            tts_text = tts_text[0] if tts_text else ""
+
+        if tts_token_ids is None or tts_hidden_states is None:
+            # KV cache profiling / dummy run path — no real TTS input yet.
+            logger.debug("4.5 Talker: dummy forward (missing tts_token_ids/tts_hidden_states)")
+            return self._dummy_hidden_states(input_ids, positions, inputs_embeds)
+
+        logger.info("4.5 Talker: generating speech for %d tokens", tts_token_ids.shape[0])
+        waveform = self.generate_speech(tts_token_ids, tts_hidden_states)
+        # Tuple layout: (mel_spec, waveform). 4.5 talker emits only waveform,
+        # so mel_spec stays None; the wrapper unpacks in this order and
+        # packages the waveform into ``multimodal_outputs["model_outputs"]``.
+        if waveform is not None:
+            return None, torch.tensor(waveform, dtype=torch.float32)
+        return None, None
+
+    def compute_logits(self, hidden_states, *args, **kwargs):
+        return torch.zeros(1, 2, device=hidden_states.device if isinstance(hidden_states, torch.Tensor) else "cuda")
+
+    def sample(self, logits, sampling_metadata):
+        return None
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+        loaded = set()
+        tts_weights = {}
+        for k, v in weights:
+            if k.startswith("tts."):
+                tts_weights[k.replace("tts.", "", 1)] = v
+                # vllm sanity-checks `loaded` against `named_parameters()`.
+                # The submodule is attached at `self.tts_obj`, not `self.tts`,
+                # so report the loaded name under the on-module path.
+                loaded.add(k.replace("tts.", "tts_obj.", 1))
+
+        if tts_weights and self._tts_config is not None:
+            self._lazy_init_tts()
+            if hasattr(self, "tts_obj") and self.tts_obj is not None:
+                missing, unexpected = self.tts_obj.load_state_dict(tts_weights, strict=False)
+                if missing:
+                    logger.warning("TTS missing keys (%d): %s", len(missing), missing[:5])
+                if unexpected:
+                    logger.warning("TTS unexpected keys (%d): %s", len(unexpected), unexpected[:5])
+                self.tts_obj = self.tts_obj.to("cuda")
+                self.emb_text = self.tts_obj.emb_text
+                self.projector_semantic = self.tts_obj.projector_semantic
+                logger.info("Loaded %d TTS weights, moved to cuda", len(tts_weights))
+
+        return loaded
+
+    def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs):
+        if hasattr(self, "emb_text") and self.emb_text is not None:
+            return self.emb_text(input_ids)
+        return torch.zeros(input_ids.shape[0], 1)
+
+    def embed_input_ids(self, input_ids, **kwargs):
+        return self.get_input_embeddings(input_ids, **kwargs)
diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py
new file mode 100644
index 000000000..701e014e6
--- /dev/null
+++ b/vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""MiniCPM-o 4.5 pipeline topology (frozen).
+
+Stage 0: Thinker — multimodal understanding + text generation.
+Stage 1: Talker  — MiniCPMTTS + Token2Wav, emits the final audio waveform.
+
+The thinker -> talker bridge passes the hidden states + token ids extracted
+from the thinker output through ``minicpmo_4_5_omni.llm2tts``; the talker
+runs MiniCPMTTS and the on-device Token2wav vocoder in the same process and
+returns the waveform directly as the pipeline's final audio output.
+"""
+
+from vllm_omni.config.stage_config import (
+    PipelineConfig,
+    StageExecutionType,
+    StagePipelineConfig,
+)
+
+_PROC = "vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni"
+
+
+MINICPMO_4_5_PIPELINE = PipelineConfig(
+    model_type="minicpmo_4_5",
+    model_arch="MiniCPMO45OmniForConditionalGeneration",
+    # MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and
+    # `architectures=["MiniCPMO"]` — both shared verbatim with older MiniCPM-o
+    # 1.0 / 2.6 checkpoints. The only field distinguishing the generations is
+    # the top-level ``version`` string, so we register both the shared
+    # ``MiniCPMO`` arch (for auto-detection) and the 4.5-specific arch (for
+    # repos that opt into the explicit name later), then pin the routing to
+    # 4.5 via ``hf_config_predicate``. Without the predicate, loading a 2.6
+    # checkpoint would also intersect ``["MiniCPMO"]`` here and get routed
+    # into the 4.5 pipeline, which would then fail at load time.
+    hf_architectures=("MiniCPMO", "MiniCPMO45OmniForConditionalGeneration"),
+    hf_config_predicate=lambda c: str(getattr(c, "version", "")) == "4.5",
+    stages=(
+        StagePipelineConfig(
+            stage_id=0,
+            model_stage="llm",
+            execution_type=StageExecutionType.LLM_AR,
+            input_sources=(),
+            final_output=True,
+            final_output_type="text",
+            owns_tokenizer=True,
+            requires_multimodal_data=True,
+            engine_output_type="latent",
+            sampling_constraints={"detokenize": True},
+        ),
+        StagePipelineConfig(
+            stage_id=1,
+            model_stage="tts",
+            # Stage 1 shares the top-level wrapper class
+            # (``MiniCPMO45OmniForConditionalGeneration``) inherited from
+            # ``model_arch`` above. The wrapper dispatches on ``model_stage``
+            # and, for ``"tts"``, instantiates the standalone TTS submodule
+            # (``MiniCPMO45OmniTTSForConditionalGeneration``) internally.
+            # Routing through the wrapper is required so that the runner-side
+            # ``runtime_additional_information`` payload reaches the talker
+            # (the standalone TTS class only reads ``additional_information``,
+            # so wiring stage 1 directly to it would always trigger the dummy
+            # path) and so the resulting waveform is packaged as
+            # ``OmniOutput.multimodal_outputs["model_outputs"]`` instead of
+            # being returned as a bare tuple that the AR runner would mistake
+            # for hidden states. ``hf_config_name="tts_config"`` keeps KV
+            # cache / mrope sizing scoped to the talker sub-config.
+            execution_type=StageExecutionType.LLM_AR,
+            input_sources=(0,),
+            final_output=True,
+            final_output_type="audio",
+            hf_config_name="tts_config",
+            engine_output_type="audio",
+            custom_process_input_func=f"{_PROC}.llm2tts",
+            sampling_constraints={"detokenize": False},
+        ),
+    ),
+)
diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py
index 9ef3c853a..d642c6efb 100644
--- a/vllm_omni/model_executor/models/registry.py
+++ b/vllm_omni/model_executor/models/registry.py
@@ -250,6 +250,22 @@ _OMNI_MODELS = {
         "ming_flash_omni",
         "MingFlashOmniForConditionalGeneration",
     ),
+    # MiniCPM-o 4.5 Omni models
+    "MiniCPMO45OmniForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni",
+        "MiniCPMO45OmniForConditionalGeneration",
+    ),
+    "MiniCPMO45OmniLLMForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni_llm",
+        "MiniCPMO45OmniLLMForConditionalGeneration",
+    ),
+    "MiniCPMO45OmniTTSForConditionalGeneration": (
+        "minicpmo_4_5",
+        "minicpmo_4_5_omni_tts",
+        "MiniCPMO45OmniTTSForConditionalGeneration",
+    ),
 }
 
 
diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
new file mode 100644
index 000000000..799914846
--- /dev/null
+++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Stage input processor for MiniCPM-o 4.5: Thinker (LLM) -> Talker (TTS).
+
+This is the original vLLM-Omni bridge: it converts the thinker stage's
+hidden states + token ids into the talker stage's prompt payload. The
+talker model itself is adapted from openbmb/MiniCPM-o-4_5 (see the headers
+on vllm_omni/model_executor/models/minicpmo_4_5/*.py).
+"""
+
+from typing import Any
+
+import torch
+from vllm.inputs import TextPrompt
+
+from vllm_omni.inputs.data import OmniTokensPrompt
+
+
+def llm2tts(
+    source_outputs: list[Any],
+    prompt: OmniTokensPrompt | TextPrompt | dict | list | None = None,
+    requires_multimodal_data: bool = False,
+    streaming_context: Any | None = None,
+):
+    """Convert thinker stage output to talker stage input for MiniCPMO Omni.
+
+    The signature matches the framework's ``custom_process_input_func`` call
+    convention used by ``StageEngineCoreClientBase.process_engine_inputs``:
+
+        (source_outputs, prompt, requires_multimodal_data, streaming_context)
+
+    ``source_outputs`` is the already-resolved list of upstream engine
+    outputs (one entry per request), so we do not need to look anything up
+    via ``stage_list[source_stage_id].engine_outputs``.
+
+    Extracts from thinker output:
+      - Full hidden states (prompt + generated) for speaker embedding extraction
+      - Prompt token IDs (for finding spk_bos/spk_eos positions)
+      - Generated token IDs (for decoding TTS text)
+
+    The talker model will:
+      1. Find <|spk_bos|>/<|spk_eos|> positions in prompt_token_ids
+      2. Extract speaker embedding from hidden states at those positions
+      3. Decode generated text and extract TTS content
+      4. Run ConditionalChatTTS pipeline
+    """
+    del streaming_context  # not used by MiniCPM-o 4.5 turn-taking pipeline
+
+    if not source_outputs:
+        raise ValueError("source_outputs cannot be empty")
+
+    llm_outputs = source_outputs
+    tts_inputs = []
+
+    if not isinstance(prompt, list):
+        prompt = [prompt]
+
+    multi_modal_data = {
+        llm_output.request_id: p.get("multi_modal_data", None) if isinstance(p, dict) else None
+        for llm_output, p in zip(llm_outputs, prompt)
+    }
+
+    for i, llm_output in enumerate(llm_outputs):
+        output = llm_output.outputs[0]
+        prompt_token_ids = llm_output.prompt_token_ids
+        llm_output_ids = output.token_ids
+        prompt_token_ids_len = len(prompt_token_ids)
+
+        latent = output.multimodal_output.get("latent", None)
+        if latent is None:
+            latent = output.hidden_states if hasattr(output, "hidden_states") else None
+            if latent is None:
+                raise ValueError("No latent or hidden_states found in thinker output")
+
+        thinker_hidden_states = latent.clone().detach()
+
+        # Split hidden states: prompt portion has speaker embedding,
+        # generated portion has the text content
+        prompt_hidden = thinker_hidden_states[:prompt_token_ids_len].to(torch.float32)
+
+        # Extract decoded text from thinker output for TTS text extraction
+        thinker_text = getattr(output, "text", "") or ""
+
+        # Build full token sequence and extract TTS region
+        full_token_ids = list(prompt_token_ids) + (
+            list(llm_output_ids) if not isinstance(llm_output_ids, list) else llm_output_ids
+        )
+        full_hidden = thinker_hidden_states.to(torch.float32)
+
+        # Detect TTS token IDs (4.5: 151703/151704, 2.6: 151691/151692)
+        tts_bos_id, tts_eos_id = 151691, 151692
+        for _id in [151703, 151704]:
+            if _id in full_token_ids:
+                tts_bos_id, tts_eos_id = 151703, 151704
+                break
+
+        tts_bos_idx = tts_eos_idx = None
+        for idx_t, tid in enumerate(full_token_ids):
+            if tid == tts_bos_id:
+                tts_bos_idx = idx_t + 1
+            elif tid == tts_eos_id:
+                tts_eos_idx = idx_t
+
+        tts_token_ids_slice = tts_hidden_slice = None
+        if tts_bos_idx is not None and full_hidden.shape[0] > tts_bos_idx:
+            end_idx = tts_eos_idx if tts_eos_idx is not None else full_hidden.shape[0]
+            tts_token_ids_slice = torch.tensor(full_token_ids[tts_bos_idx:end_idx], dtype=torch.long)
+            tts_hidden_slice = full_hidden[tts_bos_idx:end_idx]
+
+        additional_information = {
+            "prompt_embeds": prompt_hidden,
+            "prompt_token_ids": list(prompt_token_ids),
+            "llm_output_token_ids": list(llm_output_ids) if not isinstance(llm_output_ids, list) else llm_output_ids,
+            "llm_output_text": [thinker_text],
+        }
+        if tts_token_ids_slice is not None:
+            additional_information["tts_token_ids"] = tts_token_ids_slice
+        if tts_hidden_slice is not None:
+            additional_information["tts_hidden_states"] = tts_hidden_slice
+
+        # Minimal prompt token IDs: the talker's AR framework needs *some* tokens
+        # to do a single prefill step. We use [BOS, PAD, EOS] as a dummy.
+        tts_inputs.append(
+            OmniTokensPrompt(
+                prompt_token_ids=[1, 0, 2],
+                additional_information=additional_information,
+                multi_modal_data=(
+                    multi_modal_data[llm_output.request_id]
+                    if requires_multimodal_data and multi_modal_data.get(llm_output.request_id) is not None
+                    else None
+                ),
+                mm_processor_kwargs=None,
+            )
+        )
+
+    return tts_inputs