diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md
new file mode 100644
index 000000000..bb6ce2568
--- /dev/null
+++ b/examples/offline_inference/minicpmo/README.md
@@ -0,0 +1,91 @@
+# MiniCPM-o 4.5: Offline inference
+
+Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2Wav**
+(24 kHz speech). Deploy config auto-loads from
+`vllm_omni/deploy/minicpmo_4_5.yaml` (2-GPU default).
+
+## Setup
+
+- Install talker deps: `pip install 'vllm-omni[minicpmo]'` (or `stepaudio2-minicpmo`)
+- `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO)
+- See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning
+
+## Run examples
+
+### Single prompt
+
+```bash
+cd examples/offline_inference/minicpmo
+bash run_single_prompt.sh
+```
+
+### Multiple prompts
+
+```bash
+bash run_multiple_prompts.sh
+```
+
+### Thinker tensor parallel (3-GPU)
+
+```bash
+bash run_single_prompt_tp.sh
+```
+
+### Modality control
+
+Text-only (skip talker / no `<|tts_bos|>`):
+
+```bash
+python end2end.py --query-type use_audio --modalities text
+```
+
+Text + speech (default — appends `<|tts_bos|>`):
+
+```bash
+python end2end.py --query-type text --modalities text,audio
+```
+
+### Local media files
+
+```bash
+python end2end.py --query-type use_video --video-path /path/to/video.mp4
+python end2end.py --query-type use_image --image-path /path/to/image.jpg
+python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+python end2end.py --query-type use_mixed_modalities \
+ --video-path /path/to/video.mp4 \
+ --image-path /path/to/image.jpg \
+ --audio-path /path/to/audio.wav
+```
+
+Supported `--query-type` values:
+
+| Query type | Inputs |
+|---|---|
+| `text` | Text only |
+| `use_image` | Image + text |
+| `use_audio` | Audio + text |
+| `use_video` | Video + text |
+| `use_multi_audios` | Two audio clips |
+| `use_mixed_modalities` | Audio + image + video |
+
+### Custom deploy config
+
+```bash
+python end2end.py --query-type text \
+ --deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml
+```
+
+## Notes
+
+- Speech requires `<|tts_bos|>` on the assistant prefix (offline equivalent of
+ online `chat_template_kwargs.use_tts_template=true`). Without it, the talker
+ gets an empty TTS span.
+- Output WAV is **24 kHz mono**.
+- Placeholders in the prompt are MiniCPM-style:
+ `(./)`, `()`, `()`.
+- Default layout needs **2 GPUs**. Async chunking is off in the bundled YAMLs.
+
+## Online serving
+
+See [`examples/online_serving/minicpmo/`](../../online_serving/minicpmo/) and
+the recipe [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md).
diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py
new file mode 100644
index 000000000..e9c9019c0
--- /dev/null
+++ b/examples/offline_inference/minicpmo/end2end.py
@@ -0,0 +1,543 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Offline end-to-end inference for MiniCPM-o 4.5 (thinker + talker/Token2Wav).
+
+MiniCPM-o uses placeholders ``(./)``, ``()``,
+and ``()``. Speech output requires a ``<|tts_bos|>`` suffix on
+the assistant prefix (equivalent to online ``chat_template_kwargs.use_tts_template``).
+"""
+
+import os
+import time
+from typing import NamedTuple
+
+import numpy as np
+import soundfile as sf
+import vllm
+from PIL import Image
+from vllm import SamplingParams
+from vllm.assets.audio import AudioAsset
+from vllm.assets.image import ImageAsset
+from vllm.assets.video import VideoAsset, video_to_ndarrays
+from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.media.audio import load_audio
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.utils.tracking_parser import TrackingArgumentParser
+
+SEED = 42
+
+default_system = (
+ "You are MiniCPM-o, a helpful multimodal assistant that can "
+ "understand images, audio and video, and respond in text and speech."
+)
+
+
+class QueryResult(NamedTuple):
+ inputs: dict
+ limit_mm_per_prompt: dict[str, int]
+
+
+def _assistant_prefix(use_tts: bool) -> str:
+ # Matches HF chat_template: assistant header, optional empty , then TTS bos.
+ prefix = "<|im_start|>assistant\n"
+ if use_tts:
+ prefix += "<|tts_bos|>"
+ return prefix
+
+
+def _build_prompt(user_body: str, use_tts: bool) -> str:
+ return (
+ f"<|im_start|>system\n{default_system}<|im_end|>\n"
+ f"<|im_start|>user\n{user_body}<|im_end|>\n"
+ f"{_assistant_prefix(use_tts)}"
+ )
+
+
+def get_text_query(question: str | None = None, use_tts: bool = True) -> QueryResult:
+ if question is None:
+ question = "Say hello, then introduce vLLM-Omni in one sentence."
+ return QueryResult(
+ inputs={"prompt": _build_prompt(question, use_tts=use_tts)},
+ limit_mm_per_prompt={},
+ )
+
+
+def get_image_query(
+ question: str | None = None,
+ image_path: str | None = None,
+ use_tts: bool = True,
+) -> QueryResult:
+ if question is None:
+ question = "What is the content of this image?"
+ user_body = f"(./)\n{question}"
+
+ if image_path:
+ if not os.path.exists(image_path):
+ raise FileNotFoundError(f"Image file not found: {image_path}")
+ image_data = convert_image_mode(Image.open(image_path), "RGB")
+ else:
+ image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+ return QueryResult(
+ inputs={
+ "prompt": _build_prompt(user_body, use_tts=use_tts),
+ "multi_modal_data": {"image": image_data},
+ },
+ limit_mm_per_prompt={"image": 1},
+ )
+
+
+def get_audio_query(
+ question: str | None = None,
+ audio_path: str | None = None,
+ sampling_rate: int = 16000,
+ use_tts: bool = True,
+) -> QueryResult:
+ if question is None:
+ question = "What is the content of this audio?"
+ user_body = f"()\n{question}"
+
+ if audio_path:
+ if not os.path.exists(audio_path):
+ raise FileNotFoundError(f"Audio file not found: {audio_path}")
+ audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
+ audio_data = (audio_signal.astype(np.float32), sr)
+ else:
+ audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
+ return QueryResult(
+ inputs={
+ "prompt": _build_prompt(user_body, use_tts=use_tts),
+ "multi_modal_data": {"audio": audio_data},
+ },
+ limit_mm_per_prompt={"audio": 1},
+ )
+
+
+def get_video_query(
+ question: str | None = None,
+ video_path: str | None = None,
+ num_frames: int = 16,
+ use_tts: bool = True,
+) -> QueryResult:
+ if question is None:
+ question = "Why is this video funny?"
+ user_body = f"()\n{question}"
+
+ if video_path:
+ if not os.path.exists(video_path):
+ raise FileNotFoundError(f"Video file not found: {video_path}")
+ video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+ else:
+ video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
+ return QueryResult(
+ inputs={
+ "prompt": _build_prompt(user_body, use_tts=use_tts),
+ "multi_modal_data": {"video": video_frames},
+ },
+ limit_mm_per_prompt={"video": 1},
+ )
+
+
+def get_mixed_modalities_query(
+ video_path: str | None = None,
+ image_path: str | None = None,
+ audio_path: str | None = None,
+ num_frames: int = 16,
+ sampling_rate: int = 16000,
+ use_tts: bool = True,
+) -> QueryResult:
+ question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
+ user_body = f"()\n(./)\n()\n{question}"
+
+ if video_path:
+ if not os.path.exists(video_path):
+ raise FileNotFoundError(f"Video file not found: {video_path}")
+ video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+ else:
+ video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
+ if image_path:
+ if not os.path.exists(image_path):
+ raise FileNotFoundError(f"Image file not found: {image_path}")
+ image_data = convert_image_mode(Image.open(image_path), "RGB")
+ else:
+ image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+ if audio_path:
+ if not os.path.exists(audio_path):
+ raise FileNotFoundError(f"Audio file not found: {audio_path}")
+ audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
+ audio_data = (audio_signal.astype(np.float32), sr)
+ else:
+ audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
+ return QueryResult(
+ inputs={
+ "prompt": _build_prompt(user_body, use_tts=use_tts),
+ "multi_modal_data": {
+ "audio": audio_data,
+ "image": image_data,
+ "video": video_frames,
+ },
+ },
+ limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
+ )
+
+
+def get_multi_audios_query(use_tts: bool = True) -> QueryResult:
+ question = "Are these two audio clips the same?"
+ user_body = f"()\n()\n{question}"
+ return QueryResult(
+ inputs={
+ "prompt": _build_prompt(user_body, use_tts=use_tts),
+ "multi_modal_data": {
+ "audio": [
+ AudioAsset("winning_call").audio_and_sample_rate,
+ AudioAsset("mary_had_lamb").audio_and_sample_rate,
+ ],
+ },
+ },
+ limit_mm_per_prompt={"audio": 2},
+ )
+
+
+query_map = {
+ "text": get_text_query,
+ "use_audio": get_audio_query,
+ "use_image": get_image_query,
+ "use_video": get_video_query,
+ "use_multi_audios": get_multi_audios_query,
+ "use_mixed_modalities": get_mixed_modalities_query,
+}
+
+
+def _wants_tts(modalities: str | None) -> bool:
+ """TTS bos is needed whenever audio is among the requested output modalities."""
+ if modalities is None:
+ return True
+ parts = {p.strip() for p in modalities.split(",") if p.strip()}
+ return "audio" in parts
+
+
+def main(args):
+ model_name = args.model
+ print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20)
+
+ video_path = getattr(args, "video_path", None)
+ image_path = getattr(args, "image_path", None)
+ audio_path = getattr(args, "audio_path", None)
+ use_tts = _wants_tts(args.modalities)
+
+ query_func = query_map[args.query_type]
+ if args.query_type == "use_video":
+ query_result = query_func(
+ video_path=video_path,
+ num_frames=getattr(args, "num_frames", 16),
+ use_tts=use_tts,
+ )
+ elif args.query_type == "use_image":
+ query_result = query_func(image_path=image_path, use_tts=use_tts)
+ elif args.query_type == "use_audio":
+ query_result = query_func(
+ audio_path=audio_path,
+ sampling_rate=getattr(args, "sampling_rate", 16000),
+ use_tts=use_tts,
+ )
+ elif args.query_type == "use_mixed_modalities":
+ query_result = query_func(
+ video_path=video_path,
+ image_path=image_path,
+ audio_path=audio_path,
+ num_frames=getattr(args, "num_frames", 16),
+ sampling_rate=getattr(args, "sampling_rate", 16000),
+ use_tts=use_tts,
+ )
+ elif args.query_type == "use_multi_audios":
+ query_result = query_func(use_tts=use_tts)
+ else:
+ query_result = query_func(use_tts=use_tts)
+
+ omni_kwargs = vars(args).copy()
+ # Drop example-only CLI keys that are not Omni / engine args.
+ for key in (
+ "query_type",
+ "output_wav",
+ "output_dir",
+ "num_prompts",
+ "txt_prompts",
+ "video_path",
+ "image_path",
+ "audio_path",
+ "num_frames",
+ "sampling_rate",
+ "modalities",
+ "py_generator",
+ "enable_profiler",
+ "profiler_stages",
+ "log_dir",
+ ):
+ omni_kwargs.pop(key, None)
+ omni_kwargs["model"] = model_name
+ omni_kwargs["trust_remote_code"] = True
+ omni = Omni(**omni_kwargs)
+
+ # Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled).
+ thinker_sampling_params = SamplingParams(
+ temperature=0.0,
+ top_p=1.0,
+ top_k=-1,
+ max_tokens=2048,
+ seed=SEED,
+ detokenize=True,
+ repetition_penalty=1.1,
+ )
+ # Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler;
+ # waveform is produced in-process by Token2Wav.
+ talker_sampling_params = SamplingParams(
+ temperature=0.0,
+ top_p=1.0,
+ top_k=-1,
+ max_tokens=1,
+ seed=SEED,
+ detokenize=False,
+ )
+ sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages]
+
+ if args.txt_prompts is None:
+ prompts = [query_result.inputs for _ in range(args.num_prompts)]
+ else:
+ assert args.query_type == "text", "txt-prompts is only supported for text query type"
+ with open(args.txt_prompts, encoding="utf-8") as f:
+ lines = [ln.strip() for ln in f.readlines()]
+ prompts = [get_text_query(ln, use_tts=use_tts).inputs for ln in lines if ln != ""]
+ print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
+
+ if args.modalities is not None:
+ output_modalities = args.modalities.split(",")
+ for prompt in prompts:
+ prompt["modalities"] = output_modalities
+
+ profiler_enabled = args.enable_profiler
+ if profiler_enabled:
+ omni.start_profile(stages=args.profiler_stages)
+
+ omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
+ output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
+ os.makedirs(output_dir, exist_ok=True)
+
+ total_requests = len(prompts)
+ processed_count = 0
+ print(f"query type: {args.query_type}")
+ print(f"use_tts (tts_bos): {use_tts}")
+
+ for stage_outputs in omni_generator:
+ output = stage_outputs.request_output
+ if stage_outputs.final_output_type == "text":
+ request_id = output.request_id
+ text_output = output.outputs[0].text
+ out_txt = os.path.join(output_dir, f"{request_id}.txt")
+ lines = [
+ "Prompt:\n",
+ str(output.prompt) + "\n",
+ "vllm_text_output:\n",
+ str(text_output).strip() + "\n",
+ ]
+ try:
+ with open(out_txt, "w", encoding="utf-8") as f:
+ f.writelines(lines)
+ except Exception as e:
+ print(f"[Warn] Failed writing text file {out_txt}: {e}")
+ print(f"Request ID: {request_id}, Text saved to {out_txt}")
+ elif stage_outputs.final_output_type == "audio":
+ request_id = output.request_id
+ audio_tensor = output.outputs[0].multimodal_output["audio"]
+ output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
+
+ if isinstance(audio_tensor, list):
+ import torch
+
+ audio_tensor = torch.cat(
+ [(t if isinstance(t, torch.Tensor) else torch.tensor(t)).flatten() for t in audio_tensor]
+ )
+ audio_numpy = audio_tensor.float().detach().cpu().numpy()
+ if audio_numpy.ndim > 1:
+ audio_numpy = audio_numpy.flatten()
+
+ # MiniCPM-o 4.5 Token2Wav emits 24 kHz mono.
+ sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV")
+ print(f"Request ID: {request_id}, Saved audio to {output_wav}")
+
+ processed_count += 1
+ if profiler_enabled and processed_count >= total_requests:
+ print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
+ omni.stop_profile(stages=args.profiler_stages)
+ print("[Info] Waiting 30s for workers to write trace files to disk...")
+ time.sleep(30)
+ print("[Info] Trace export wait time finished.")
+ omni.close()
+
+
+def parse_args():
+ parser = TrackingArgumentParser(
+ description="Offline inference demo for MiniCPM-o 4.5 (text / image / audio / video → text + speech)"
+ )
+ parser.add_argument(
+ "--model",
+ type=str,
+ default="openbmb/MiniCPM-o-4_5",
+ help="Model name or path.",
+ )
+ parser.add_argument(
+ "--query-type",
+ "-q",
+ type=str,
+ default="text",
+ choices=query_map.keys(),
+ help="Query type.",
+ )
+ parser.add_argument(
+ "--log-stats",
+ action="store_true",
+ default=False,
+ help="Enable writing detailed statistics (default: disabled)",
+ )
+ parser.add_argument(
+ "--stage-init-timeout",
+ type=int,
+ default=300,
+ help="Timeout for initializing a single stage in seconds (default: 300)",
+ )
+ parser.add_argument(
+ "--batch-timeout",
+ type=int,
+ default=5,
+ help="Timeout for batching in seconds (default: 5)",
+ )
+ parser.add_argument(
+ "--init-timeout",
+ type=int,
+ default=300,
+ help="Timeout for initializing stages in seconds (default: 300)",
+ )
+ parser.add_argument(
+ "--shm-threshold-bytes",
+ type=int,
+ default=65536,
+ help="Threshold for using shared memory in bytes (default: 65536)",
+ )
+ parser.add_argument(
+ "--output-wav",
+ default="output_audio",
+ help="[Deprecated] Output wav directory (use --output-dir).",
+ )
+ parser.add_argument(
+ "--output-dir",
+ type=str,
+ default=None,
+ help="Output directory for text/audio results.",
+ )
+ parser.add_argument(
+ "--num-prompts",
+ type=int,
+ default=1,
+ help="Number of prompts to generate.",
+ )
+ parser.add_argument(
+ "--txt-prompts",
+ type=str,
+ default=None,
+ help="Path to a .txt file with one prompt per line (preferred).",
+ )
+ parser.add_argument(
+ "--stage-configs-path",
+ type=str,
+ default=None,
+ help="Path to a stage configs file (deprecated; prefer --deploy-config).",
+ )
+ parser.add_argument(
+ "--deploy-config",
+ type=str,
+ default=None,
+ help="Path to a deploy YAML (default: auto-load minicpmo_4_5.yaml).",
+ )
+ parser.add_argument(
+ "--video-path",
+ "-v",
+ type=str,
+ default=None,
+ help="Path to local video file. If not provided, uses default video asset.",
+ )
+ parser.add_argument(
+ "--image-path",
+ "-i",
+ type=str,
+ default=None,
+ help="Path to local image file. If not provided, uses default image asset.",
+ )
+ parser.add_argument(
+ "--audio-path",
+ "-a",
+ type=str,
+ default=None,
+ help="Path to local audio file. If not provided, uses default audio asset.",
+ )
+ parser.add_argument(
+ "--num-frames",
+ type=int,
+ default=16,
+ help="Number of frames to extract from video (default: 16).",
+ )
+ parser.add_argument(
+ "--sampling-rate",
+ type=int,
+ default=16000,
+ help="Sampling rate for audio loading (default: 16000).",
+ )
+ parser.add_argument(
+ "--log-dir",
+ type=str,
+ default="logs",
+ help="Log directory (default: logs).",
+ )
+ parser.add_argument(
+ "--modalities",
+ type=str,
+ default=None,
+ help="Output modalities (comma-separated), e.g. text or text,audio.",
+ )
+ parser.add_argument(
+ "--py-generator",
+ action="store_true",
+ default=False,
+ help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
+ )
+ parser.add_argument(
+ "--enable-profiler",
+ action="store_true",
+ default=False,
+ help="Enables profiling when set.",
+ )
+ parser.add_argument(
+ "--profiler-stages",
+ type=int,
+ nargs="*",
+ default=None,
+ help="List of stage IDs to profile. If not set, profiles all stages.",
+ )
+ parser.add_argument(
+ "--dtype",
+ type=str,
+ default="auto",
+ help="Model dtype (auto, half, float16, bfloat16, float, float32).",
+ )
+
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ main(args)
diff --git a/examples/offline_inference/minicpmo/run_multiple_prompts.sh b/examples/offline_inference/minicpmo/run_multiple_prompts.sh
new file mode 100755
index 000000000..f130fece3
--- /dev/null
+++ b/examples/offline_inference/minicpmo/run_multiple_prompts.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Multi-prompt offline batch (py_generator mode).
+set -euo pipefail
+cd "$(dirname "$0")"
+
+python end2end.py --output-wav output_audio \
+ --query-type text \
+ --txt-prompts ../qwen3_omni/text_prompts_10.txt \
+ --py-generator
diff --git a/examples/offline_inference/minicpmo/run_single_prompt.sh b/examples/offline_inference/minicpmo/run_single_prompt.sh
new file mode 100755
index 000000000..3a43ddda9
--- /dev/null
+++ b/examples/offline_inference/minicpmo/run_single_prompt.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Single-prompt offline smoke test (text → text + 24 kHz speech).
+set -euo pipefail
+cd "$(dirname "$0")"
+
+python end2end.py --output-wav output_audio \
+ --query-type text
diff --git a/examples/offline_inference/minicpmo/run_single_prompt_tp.sh b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh
new file mode 100755
index 000000000..88f488d2c
--- /dev/null
+++ b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Single-prompt offline run with thinker tensor-parallel (3-GPU layout).
+# Thinker on GPU 0,1 (TP=2); talker + Token2Wav on GPU 2.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+REPO_ROOT="$(cd ../../.. && pwd)"
+
+python end2end.py --output-wav output_audio \
+ --query-type use_audio \
+ --deploy-config "${REPO_ROOT}/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml" \
+ --stage-init-timeout 300
diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md
index 334b5ed11..e1c084224 100644
--- a/examples/online_serving/minicpmo/README.md
+++ b/examples/online_serving/minicpmo/README.md
@@ -1,14 +1,19 @@
-# vLLM-Omni · MiniCPM-o 4.5 Online Demo
+# MiniCPM-o 4.5: Online serving
-Gradio-based web UI for **MiniCPM-o 4.5** served via `vllm-omni`'s
-OpenAI-compatible endpoints.
+OpenAI-compatible `/v1/chat/completions` serving for **MiniCPM-o 4.5**, plus a
+Gradio UI and curl / Python clients.
-The UI supports:
+Inputs: text / image / audio / video. Outputs: text and optional **24 kHz** speech.
-- **Inputs**: text prompt + optional image, audio (file or mic), video.
-- **Outputs**: text + speech (WAV player).
+## Installation
-## 1. Start the backend server
+Please refer to [README.md](../../../README.md). Install the talker extra:
+
+```bash
+pip install 'vllm-omni[minicpmo]'
+```
+
+## Launch the server
The deploy config auto-loads via `--omni`; the default
`vllm_omni/deploy/minicpmo_4_5.yaml` targets a single-GPU layout (thinker
@@ -30,37 +35,125 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \
--host 0.0.0.0 --port 8099
```
-Other layouts via `--deploy-config`:
+Other layouts:
```bash
vllm serve openbmb/MiniCPM-o-4_5 --omni \
- --deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \
+ --deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \
--trust-remote-code \
--host 0.0.0.0 --port 8099
```
-## 2. Launch the Gradio demo
+### Stage-based CLI (optional)
+
+Stage 0 (thinker + API) and stage 1 (talker) can run in separate processes:
```bash
-bash examples/online_serving/minicpmo/run_gradio_demo.sh
+# Stage 0
+CUDA_VISIBLE_DEVICES=0 vllm serve openbmb/MiniCPM-o-4_5 --omni \
+ --trust-remote-code --port 8099 --stage-id 0 \
+ --omni-master-address 127.0.0.1 --omni-master-port 26000
-# Or run the python entry point directly:
-python examples/online_serving/minicpmo/gradio_demo.py \
+# Stage 1 (headless)
+CUDA_VISIBLE_DEVICES=1 vllm serve openbmb/MiniCPM-o-4_5 --omni \
+ --trust-remote-code --stage-id 1 --headless \
+ --omni-master-address 127.0.0.1 --omni-master-port 26000
+```
+
+### Per-stage overrides
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \
+ --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}'
+```
+
+## Send multimodal requests
+
+```bash
+cd examples/online_serving/minicpmo
+```
+
+### curl
+
+```bash
+bash run_curl_multimodal_generation.sh text
+bash run_curl_multimodal_generation.sh use_image
+bash run_curl_multimodal_generation.sh use_audio '["text"]' # text-only
+```
+
+Text + speech smoke test (TTS needs top-level `chat_template_kwargs`):
+
+```bash
+curl http://localhost:8099/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "openbmb/MiniCPM-o-4_5",
+ "messages": [{"role": "user", "content": "Say hello, then introduce vLLM in one sentence."}],
+ "modalities": ["text", "audio"],
+ "chat_template_kwargs": {"use_tts_template": true}
+ }'
+```
+
+### Python client
+
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py \
+ --query-type use_image \
+ --port 8099 \
+ --host localhost
+
+# Text-only (faster; no <|tts_bos|>)
+python openai_chat_completion_client_for_multimodal_generation.py \
+ --query-type text \
+ --modalities text \
+ --prompt "Briefly introduce yourself."
+```
+
+Shared helpers also work if you pass MiniCPM defaults yourself:
+
+```bash
+python ../openai_chat_completion_client_for_multimodal_generation.py \
+ --model openbmb/MiniCPM-o-4_5 \
+ --query-type text \
+ --port 8099
+```
+
+(Note: the shared client does **not** set `use_tts_template`; prefer the
+MiniCPM-specific client above for speech.)
+
+### Gradio demo
+
+```bash
+bash run_gradio_demo.sh
+
+# Or:
+python gradio_demo.py \
--minicpmo45-api-base http://localhost:8099/v1 \
--minicpmo45-model openbmb/MiniCPM-o-4_5 \
--port 7862
```
-Open `http://:7862` in a browser.
+Open `http://:7862`. Uncheck **"Generate speech output (TTS)"** for
+text-only responses.
+
+## Modality control
+
+| Modalities | Output |
+|---|---|
+| `["text"]` | Text only (no TTS bos) |
+| `["text", "audio"]` / unset | Text + 24 kHz speech |
+
+Speech requires `chat_template_kwargs.use_tts_template=true` so the chat
+template appends `<|tts_bos|>`. For **curl**, put that field at the request
+root; nested `extra_body` is ignored. The OpenAI Python SDK may use
+`extra_body` because it merges those fields into the root.
## Notes
-- **TTS trigger**: the demo sets
- `extra_body.chat_template_kwargs.use_tts_template=True`, which appends
- `<|tts_bos|>` to the assistant prefix.
-- Uncheck **"Generate speech output (TTS)"** to get text-only responses
- (faster).
-- The audio output is the raw WAV returned by the stage-1 talker +
- Token2Wav; sample rate is 24 kHz.
-- Video input is forwarded as a base64 `video_url` entry; the server needs
- decord/torchvision to decode it.
+- Stage 1 is capped at `max_num_seqs: 1` in the deploy YAML (talker shares
+ request-0 audio metadata).
+- Output audio is base64 WAV in `message.audio.data` (24 kHz mono).
+- Offline counterpart:
+ [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
+- Recipe:
+ [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
diff --git a/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py
new file mode 100644
index 000000000..45afe7e5d
--- /dev/null
+++ b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py
@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""OpenAI chat-completions client for MiniCPM-o 4.5 online serving.
+
+Thin wrapper around the shared multimodal client helpers, with MiniCPM-specific
+defaults (port 8099, system prompt, ``chat_template_kwargs.use_tts_template``).
+"""
+
+from __future__ import annotations
+
+import base64
+import concurrent.futures
+import os
+import sys
+
+from openai import OpenAI
+
+from vllm_omni.utils.tracking_parser import TrackingArgumentParser
+
+# Reuse prompt builders / URL helpers from the shared online client.
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_ONLINE_ROOT = os.path.abspath(os.path.join(_HERE, ".."))
+if _ONLINE_ROOT not in sys.path:
+ sys.path.insert(0, _ONLINE_ROOT)
+
+from openai_chat_completion_client_for_multimodal_generation import ( # noqa: E402
+ _build_prompt_for_query_type,
+ _parse_csv_arg,
+ make_audio_output_filename,
+ query_map,
+)
+
+SEED = 42
+
+default_system = (
+ "You are MiniCPM-o, a helpful multimodal assistant that can "
+ "understand images, audio and video, and respond in text and speech."
+)
+
+
+def get_system_prompt() -> dict:
+ return {
+ "role": "system",
+ "content": [{"type": "text", "text": default_system}],
+ }
+
+
+def _wants_tts(modalities: list[str] | None) -> bool:
+ if modalities is None:
+ return True
+ return "audio" in modalities
+
+
+def run_multimodal_generation(args, client: OpenAI) -> None:
+ model_name = args.model
+ video_path = getattr(args, "video_path", None)
+ image_path = getattr(args, "image_path", None)
+ audio_path = getattr(args, "audio_path", None)
+ custom_prompt = getattr(args, "prompt", None)
+
+ if args.modalities is not None:
+ output_modalities = args.modalities.split(",")
+ else:
+ output_modalities = None
+
+ use_tts = _wants_tts(output_modalities)
+ num_concurrent_requests = args.num_concurrent_requests
+ prompt_list = _parse_csv_arg(getattr(args, "prompts", None))
+
+ request_payloads = []
+ for idx in range(num_concurrent_requests):
+ per_req_prompt = (
+ prompt_list[idx]
+ if idx < len(prompt_list)
+ else (custom_prompt if idx == 0 or not prompt_list else prompt_list[-1])
+ )
+ prompt = _build_prompt_for_query_type(
+ query_type=args.query_type,
+ custom_prompt=per_req_prompt,
+ video_path=video_path,
+ image_path=image_path,
+ audio_path=audio_path,
+ )
+ extra_body: dict = {
+ "chat_template_kwargs": {"use_tts_template": use_tts},
+ }
+ if args.query_type == "use_audio_in_video":
+ # MiniCPM-o 4.5 does not use Qwen's use_audio_in_video path; keep
+ # the flag only if the shared query type is selected explicitly.
+ extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
+ request_payloads.append({"prompt": prompt, "extra_body": extra_body})
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
+ futures = [
+ executor.submit(
+ client.chat.completions.create,
+ messages=[
+ get_system_prompt(),
+ payload["prompt"],
+ ],
+ model=model_name,
+ modalities=output_modalities,
+ extra_body=payload["extra_body"],
+ stream=args.stream,
+ )
+ for payload in request_payloads
+ ]
+ chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)]
+
+ assert len(chat_completions) == num_concurrent_requests
+ count = 0
+ if not args.stream:
+ for chat_completion in chat_completions:
+ request_id = getattr(chat_completion, "id", None)
+ for choice in chat_completion.choices:
+ if choice.message.audio:
+ audio_data = base64.b64decode(choice.message.audio.data)
+ audio_file_path = make_audio_output_filename(request_id=request_id, index=count)
+ with open(audio_file_path, "wb") as f:
+ f.write(audio_data)
+ print(f"Audio saved to {audio_file_path}")
+ count += 1
+ elif choice.message.content:
+ print("Chat completion output from text:", choice.message.content)
+ else:
+ printed_content = False
+ for chat_completion in chat_completions:
+ for chunk in chat_completion:
+ for choice in chunk.choices:
+ if hasattr(choice, "delta"):
+ content = getattr(choice.delta, "content", None)
+ else:
+ content = None
+
+ if getattr(chunk, "modality", None) == "audio" and content:
+ audio_data = base64.b64decode(content)
+ request_id = getattr(chunk, "id", None)
+ audio_file_path = make_audio_output_filename(request_id=request_id, index=count)
+ with open(audio_file_path, "wb") as f:
+ f.write(audio_data)
+ print(f"\nAudio saved to {audio_file_path}")
+ count += 1
+ elif getattr(chunk, "modality", None) == "text":
+ if not printed_content:
+ printed_content = True
+ print("\ncontent:", end="", flush=True)
+ print(content, end="", flush=True)
+
+
+def parse_args():
+ parser = TrackingArgumentParser(description="MiniCPM-o 4.5 online multimodal chat client")
+ parser.add_argument(
+ "--query-type",
+ "-q",
+ type=str,
+ default="text",
+ choices=query_map.keys(),
+ help="Query type.",
+ )
+ parser.add_argument(
+ "--model",
+ "-m",
+ type=str,
+ default="openbmb/MiniCPM-o-4_5",
+ help="Model Name / Path",
+ )
+ parser.add_argument(
+ "--video-path",
+ "-v",
+ type=str,
+ default=None,
+ help="Path to local video file or URL.",
+ )
+ parser.add_argument(
+ "--image-path",
+ "-i",
+ type=str,
+ default=None,
+ help="Path to local image file or URL.",
+ )
+ parser.add_argument(
+ "--audio-path",
+ "-a",
+ type=str,
+ default=None,
+ help="Path to local audio file or URL.",
+ )
+ parser.add_argument(
+ "--prompt",
+ "-p",
+ type=str,
+ default=None,
+ help="Custom text prompt/question.",
+ )
+ parser.add_argument(
+ "--modalities",
+ type=str,
+ default=None,
+ help="Output modalities, e.g. text or text,audio.",
+ )
+ parser.add_argument(
+ "--stream",
+ action="store_true",
+ help="Stream the response.",
+ )
+ parser.add_argument(
+ "--num-concurrent-requests",
+ type=int,
+ default=1,
+ help="Number of concurrent requests to send.",
+ )
+ parser.add_argument(
+ "--port",
+ type=int,
+ default=8099,
+ help="Port of the vLLM Omni API server.",
+ )
+ parser.add_argument(
+ "--host",
+ type=str,
+ default="localhost",
+ help="Host/IP of the vLLM Omni API server.",
+ )
+ parser.add_argument(
+ "--prompts",
+ type=str,
+ default=None,
+ help="Comma-separated prompts for concurrent requests.",
+ )
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ openai_api_base = f"http://{args.host}:{args.port}/v1"
+ client = OpenAI(api_key="EMPTY", base_url=openai_api_base)
+ run_multimodal_generation(args, client)
diff --git a/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh
new file mode 100755
index 000000000..426a2df00
--- /dev/null
+++ b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Default query type
+QUERY_TYPE="${1:-text}"
+
+# Default modalities argument (JSON). Use null for server default (text+audio).
+MODALITIES="${2:-null}"
+
+# Validate query type
+if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then
+ echo "Error: Invalid query type '$QUERY_TYPE'"
+ echo "Usage: $0 [text|use_audio|use_image|use_video] [modalities]"
+ echo " text: Text query"
+ echo " use_audio: Audio + text query"
+ echo " use_image: Image + text query"
+ echo " use_video: Video + text query"
+ echo " modalities: JSON list or null (default: null → text+audio)"
+ echo " examples: '[\"text\"]' '[\"text\",\"audio\"]'"
+ exit 1
+fi
+
+HOST="${HOST:-localhost}"
+PORT="${PORT:-8099}"
+MODEL="${MODEL:-openbmb/MiniCPM-o-4_5}"
+
+thinker_sampling_params='{
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "top_k": -1,
+ "max_tokens": 2048,
+ "seed": 42,
+ "detokenize": true,
+ "repetition_penalty": 1.1
+}'
+
+talker_sampling_params='{
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "top_k": -1,
+ "max_tokens": 1,
+ "seed": 42,
+ "detokenize": false
+}'
+# Above is optional; defaults live in vllm_omni/deploy/minicpmo_4_5.yaml.
+
+MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
+CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
+SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
+
+case "$QUERY_TYPE" in
+ text)
+ user_content='[
+ {
+ "type": "text",
+ "text": "Say hello, then introduce vLLM-Omni in one sentence."
+ }
+ ]'
+ ;;
+ use_audio)
+ user_content='[
+ {
+ "type": "audio_url",
+ "audio_url": {
+ "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
+ }
+ },
+ {
+ "type": "text",
+ "text": "What is the content of this audio?"
+ }
+ ]'
+ ;;
+ use_image)
+ user_content='[
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
+ }
+ },
+ {
+ "type": "text",
+ "text": "What is the content of this image?"
+ }
+ ]'
+ ;;
+ use_video)
+ user_content='[
+ {
+ "type": "video_url",
+ "video_url": {
+ "url": "'"$SAMPLE_VIDEO_URL"'"
+ }
+ },
+ {
+ "type": "text",
+ "text": "Why is this video funny?"
+ }
+ ]'
+ ;;
+esac
+
+sampling_params_list='[
+ '"$thinker_sampling_params"',
+ '"$talker_sampling_params"'
+]'
+
+# TTS speech path needs use_tts_template at the request root (curl does not
+# flatten nested extra_body). Skip it when the caller asked for text-only.
+USE_TTS_TEMPLATE=true
+if [[ "$MODALITIES" == '["text"]' || "$MODALITIES" == "['text']" ]]; then
+ USE_TTS_TEMPLATE=false
+fi
+
+echo "Running query type: $QUERY_TYPE (host=$HOST port=$PORT model=$MODEL)"
+echo "modalities=$MODALITIES use_tts_template=$USE_TTS_TEMPLATE"
+echo ""
+
+request_body=$(cat < Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech)
+> Online serving and offline inference for omni multimodal chat
+> (text / image / audio / video → text + 24 kHz speech)
## Summary
@@ -9,7 +10,7 @@
- Task: Omni multimodal chat — accepts text / image / audio / video input;
emits text and 24 kHz mono speech in the same response
- Mode: Online serving via the OpenAI-compatible `/v1/chat/completions`
- API, plus a bundled Gradio demo (text + speech UI)
+ API (plus Gradio demo), and offline inference via `Omni.generate`
- Maintainer: [`@tc-mb`](https://github.com/tc-mb) (MiniCPM-V / MiniCPM-o team)
## When to use this recipe
@@ -37,6 +38,8 @@ shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs):
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
- Online example + Gradio demo:
[`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/)
+- Offline end-to-end example:
+ [`examples/offline_inference/minicpmo/`](../../examples/offline_inference/minicpmo/)
- Pipeline / talker source:
[`vllm_omni/model_executor/models/minicpmo_4_5/`](../../vllm_omni/model_executor/models/minicpmo_4_5/)
- Stage-input processor (thinker → talker bridge):