From 64bcdc2186837533bf6572953a25625187ef043a Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 21 Jul 2026 11:37:15 +0800 Subject: [PATCH] [Example][MiniCPM-o] Add offline/online MiniCPM-o 4.5 examples (#5222) Signed-off-by: amy-why-3459 Co-authored-by: Cursor --- examples/offline_inference/minicpmo/README.md | 91 +++ .../offline_inference/minicpmo/end2end.py | 543 ++++++++++++++++++ .../minicpmo/run_multiple_prompts.sh | 9 + .../minicpmo/run_single_prompt.sh | 7 + .../minicpmo/run_single_prompt_tp.sh | 12 + examples/online_serving/minicpmo/README.md | 139 ++++- ...letion_client_for_multimodal_generation.py | 237 ++++++++ .../run_curl_multimodal_generation.sh | 154 +++++ recipes/OpenBMB/MiniCPM-o-4_5.md | 7 +- 9 files changed, 1174 insertions(+), 25 deletions(-) create mode 100644 examples/offline_inference/minicpmo/README.md create mode 100644 examples/offline_inference/minicpmo/end2end.py create mode 100755 examples/offline_inference/minicpmo/run_multiple_prompts.sh create mode 100755 examples/offline_inference/minicpmo/run_single_prompt.sh create mode 100755 examples/offline_inference/minicpmo/run_single_prompt_tp.sh create mode 100644 examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py create mode 100755 examples/online_serving/minicpmo/run_curl_multimodal_generation.sh diff --git a/examples/offline_inference/minicpmo/README.md b/examples/offline_inference/minicpmo/README.md new file mode 100644 index 000000000..bb6ce2568 --- /dev/null +++ b/examples/offline_inference/minicpmo/README.md @@ -0,0 +1,91 @@ +# MiniCPM-o 4.5: Offline inference + +Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2Wav** +(24 kHz speech). Deploy config auto-loads from +`vllm_omni/deploy/minicpmo_4_5.yaml` (2-GPU default). + +## Setup + +- Install talker deps: `pip install 'vllm-omni[minicpmo]'` (or `stepaudio2-minicpmo`) +- `--trust-remote-code` is always passed by `end2end.py` (required for MiniCPMO) +- See [stage configuration docs](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory tuning + +## Run examples + +### Single prompt + +```bash +cd examples/offline_inference/minicpmo +bash run_single_prompt.sh +``` + +### Multiple prompts + +```bash +bash run_multiple_prompts.sh +``` + +### Thinker tensor parallel (3-GPU) + +```bash +bash run_single_prompt_tp.sh +``` + +### Modality control + +Text-only (skip talker / no `<|tts_bos|>`): + +```bash +python end2end.py --query-type use_audio --modalities text +``` + +Text + speech (default — appends `<|tts_bos|>`): + +```bash +python end2end.py --query-type text --modalities text,audio +``` + +### Local media files + +```bash +python end2end.py --query-type use_video --video-path /path/to/video.mp4 +python end2end.py --query-type use_image --image-path /path/to/image.jpg +python end2end.py --query-type use_audio --audio-path /path/to/audio.wav +python end2end.py --query-type use_mixed_modalities \ + --video-path /path/to/video.mp4 \ + --image-path /path/to/image.jpg \ + --audio-path /path/to/audio.wav +``` + +Supported `--query-type` values: + +| Query type | Inputs | +|---|---| +| `text` | Text only | +| `use_image` | Image + text | +| `use_audio` | Audio + text | +| `use_video` | Video + text | +| `use_multi_audios` | Two audio clips | +| `use_mixed_modalities` | Audio + image + video | + +### Custom deploy config + +```bash +python end2end.py --query-type text \ + --deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml +``` + +## Notes + +- Speech requires `<|tts_bos|>` on the assistant prefix (offline equivalent of + online `chat_template_kwargs.use_tts_template=true`). Without it, the talker + gets an empty TTS span. +- Output WAV is **24 kHz mono**. +- Placeholders in the prompt are MiniCPM-style: + `(./)`, `()`, `()`. +- Default layout needs **2 GPUs**. Async chunking is off in the bundled YAMLs. + +## Online serving + +See [`examples/online_serving/minicpmo/`](../../online_serving/minicpmo/) and +the recipe [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md). diff --git a/examples/offline_inference/minicpmo/end2end.py b/examples/offline_inference/minicpmo/end2end.py new file mode 100644 index 000000000..e9c9019c0 --- /dev/null +++ b/examples/offline_inference/minicpmo/end2end.py @@ -0,0 +1,543 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Offline end-to-end inference for MiniCPM-o 4.5 (thinker + talker/Token2Wav). + +MiniCPM-o uses placeholders ``(./)``, ``()``, +and ``()``. Speech output requires a ``<|tts_bos|>`` suffix on +the assistant prefix (equivalent to online ``chat_template_kwargs.use_tts_template``). +""" + +import os +import time +from typing import NamedTuple + +import numpy as np +import soundfile as sf +import vllm +from PIL import Image +from vllm import SamplingParams +from vllm.assets.audio import AudioAsset +from vllm.assets.image import ImageAsset +from vllm.assets.video import VideoAsset, video_to_ndarrays +from vllm.multimodal.image import convert_image_mode +from vllm.multimodal.media.audio import load_audio + +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +SEED = 42 + +default_system = ( + "You are MiniCPM-o, a helpful multimodal assistant that can " + "understand images, audio and video, and respond in text and speech." +) + + +class QueryResult(NamedTuple): + inputs: dict + limit_mm_per_prompt: dict[str, int] + + +def _assistant_prefix(use_tts: bool) -> str: + # Matches HF chat_template: assistant header, optional empty , then TTS bos. + prefix = "<|im_start|>assistant\n" + if use_tts: + prefix += "<|tts_bos|>" + return prefix + + +def _build_prompt(user_body: str, use_tts: bool) -> str: + return ( + f"<|im_start|>system\n{default_system}<|im_end|>\n" + f"<|im_start|>user\n{user_body}<|im_end|>\n" + f"{_assistant_prefix(use_tts)}" + ) + + +def get_text_query(question: str | None = None, use_tts: bool = True) -> QueryResult: + if question is None: + question = "Say hello, then introduce vLLM-Omni in one sentence." + return QueryResult( + inputs={"prompt": _build_prompt(question, use_tts=use_tts)}, + limit_mm_per_prompt={}, + ) + + +def get_image_query( + question: str | None = None, + image_path: str | None = None, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "What is the content of this image?" + user_body = f"(./)\n{question}" + + if image_path: + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image file not found: {image_path}") + image_data = convert_image_mode(Image.open(image_path), "RGB") + else: + image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB") + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"image": image_data}, + }, + limit_mm_per_prompt={"image": 1}, + ) + + +def get_audio_query( + question: str | None = None, + audio_path: str | None = None, + sampling_rate: int = 16000, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "What is the content of this audio?" + user_body = f"()\n{question}" + + if audio_path: + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + audio_signal, sr = load_audio(audio_path, sr=sampling_rate) + audio_data = (audio_signal.astype(np.float32), sr) + else: + audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"audio": audio_data}, + }, + limit_mm_per_prompt={"audio": 1}, + ) + + +def get_video_query( + question: str | None = None, + video_path: str | None = None, + num_frames: int = 16, + use_tts: bool = True, +) -> QueryResult: + if question is None: + question = "Why is this video funny?" + user_body = f"()\n{question}" + + if video_path: + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + video_frames = video_to_ndarrays(video_path, num_frames=num_frames) + else: + video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": {"video": video_frames}, + }, + limit_mm_per_prompt={"video": 1}, + ) + + +def get_mixed_modalities_query( + video_path: str | None = None, + image_path: str | None = None, + audio_path: str | None = None, + num_frames: int = 16, + sampling_rate: int = 16000, + use_tts: bool = True, +) -> QueryResult: + question = "What is recited in the audio? What is the content of this image? Why is this video funny?" + user_body = f"()\n(./)\n()\n{question}" + + if video_path: + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + video_frames = video_to_ndarrays(video_path, num_frames=num_frames) + else: + video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays + + if image_path: + if not os.path.exists(image_path): + raise FileNotFoundError(f"Image file not found: {image_path}") + image_data = convert_image_mode(Image.open(image_path), "RGB") + else: + image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB") + + if audio_path: + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + audio_signal, sr = load_audio(audio_path, sr=sampling_rate) + audio_data = (audio_signal.astype(np.float32), sr) + else: + audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate + + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": { + "audio": audio_data, + "image": image_data, + "video": video_frames, + }, + }, + limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1}, + ) + + +def get_multi_audios_query(use_tts: bool = True) -> QueryResult: + question = "Are these two audio clips the same?" + user_body = f"()\n()\n{question}" + return QueryResult( + inputs={ + "prompt": _build_prompt(user_body, use_tts=use_tts), + "multi_modal_data": { + "audio": [ + AudioAsset("winning_call").audio_and_sample_rate, + AudioAsset("mary_had_lamb").audio_and_sample_rate, + ], + }, + }, + limit_mm_per_prompt={"audio": 2}, + ) + + +query_map = { + "text": get_text_query, + "use_audio": get_audio_query, + "use_image": get_image_query, + "use_video": get_video_query, + "use_multi_audios": get_multi_audios_query, + "use_mixed_modalities": get_mixed_modalities_query, +} + + +def _wants_tts(modalities: str | None) -> bool: + """TTS bos is needed whenever audio is among the requested output modalities.""" + if modalities is None: + return True + parts = {p.strip() for p in modalities.split(",") if p.strip()} + return "audio" in parts + + +def main(args): + model_name = args.model + print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20) + + video_path = getattr(args, "video_path", None) + image_path = getattr(args, "image_path", None) + audio_path = getattr(args, "audio_path", None) + use_tts = _wants_tts(args.modalities) + + query_func = query_map[args.query_type] + if args.query_type == "use_video": + query_result = query_func( + video_path=video_path, + num_frames=getattr(args, "num_frames", 16), + use_tts=use_tts, + ) + elif args.query_type == "use_image": + query_result = query_func(image_path=image_path, use_tts=use_tts) + elif args.query_type == "use_audio": + query_result = query_func( + audio_path=audio_path, + sampling_rate=getattr(args, "sampling_rate", 16000), + use_tts=use_tts, + ) + elif args.query_type == "use_mixed_modalities": + query_result = query_func( + video_path=video_path, + image_path=image_path, + audio_path=audio_path, + num_frames=getattr(args, "num_frames", 16), + sampling_rate=getattr(args, "sampling_rate", 16000), + use_tts=use_tts, + ) + elif args.query_type == "use_multi_audios": + query_result = query_func(use_tts=use_tts) + else: + query_result = query_func(use_tts=use_tts) + + omni_kwargs = vars(args).copy() + # Drop example-only CLI keys that are not Omni / engine args. + for key in ( + "query_type", + "output_wav", + "output_dir", + "num_prompts", + "txt_prompts", + "video_path", + "image_path", + "audio_path", + "num_frames", + "sampling_rate", + "modalities", + "py_generator", + "enable_profiler", + "profiler_stages", + "log_dir", + ): + omni_kwargs.pop(key, None) + omni_kwargs["model"] = model_name + omni_kwargs["trust_remote_code"] = True + omni = Omni(**omni_kwargs) + + # Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled). + thinker_sampling_params = SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=2048, + seed=SEED, + detokenize=True, + repetition_penalty=1.1, + ) + # Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler; + # waveform is produced in-process by Token2Wav. + talker_sampling_params = SamplingParams( + temperature=0.0, + top_p=1.0, + top_k=-1, + max_tokens=1, + seed=SEED, + detokenize=False, + ) + sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages] + + if args.txt_prompts is None: + prompts = [query_result.inputs for _ in range(args.num_prompts)] + else: + assert args.query_type == "text", "txt-prompts is only supported for text query type" + with open(args.txt_prompts, encoding="utf-8") as f: + lines = [ln.strip() for ln in f.readlines()] + prompts = [get_text_query(ln, use_tts=use_tts).inputs for ln in lines if ln != ""] + print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}") + + if args.modalities is not None: + output_modalities = args.modalities.split(",") + for prompt in prompts: + prompt["modalities"] = output_modalities + + profiler_enabled = args.enable_profiler + if profiler_enabled: + omni.start_profile(stages=args.profiler_stages) + + omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator) + output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav + os.makedirs(output_dir, exist_ok=True) + + total_requests = len(prompts) + processed_count = 0 + print(f"query type: {args.query_type}") + print(f"use_tts (tts_bos): {use_tts}") + + for stage_outputs in omni_generator: + output = stage_outputs.request_output + if stage_outputs.final_output_type == "text": + request_id = output.request_id + text_output = output.outputs[0].text + out_txt = os.path.join(output_dir, f"{request_id}.txt") + lines = [ + "Prompt:\n", + str(output.prompt) + "\n", + "vllm_text_output:\n", + str(text_output).strip() + "\n", + ] + try: + with open(out_txt, "w", encoding="utf-8") as f: + f.writelines(lines) + except Exception as e: + print(f"[Warn] Failed writing text file {out_txt}: {e}") + print(f"Request ID: {request_id}, Text saved to {out_txt}") + elif stage_outputs.final_output_type == "audio": + request_id = output.request_id + audio_tensor = output.outputs[0].multimodal_output["audio"] + output_wav = os.path.join(output_dir, f"output_{request_id}.wav") + + if isinstance(audio_tensor, list): + import torch + + audio_tensor = torch.cat( + [(t if isinstance(t, torch.Tensor) else torch.tensor(t)).flatten() for t in audio_tensor] + ) + audio_numpy = audio_tensor.float().detach().cpu().numpy() + if audio_numpy.ndim > 1: + audio_numpy = audio_numpy.flatten() + + # MiniCPM-o 4.5 Token2Wav emits 24 kHz mono. + sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV") + print(f"Request ID: {request_id}, Saved audio to {output_wav}") + + processed_count += 1 + if profiler_enabled and processed_count >= total_requests: + print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...") + omni.stop_profile(stages=args.profiler_stages) + print("[Info] Waiting 30s for workers to write trace files to disk...") + time.sleep(30) + print("[Info] Trace export wait time finished.") + omni.close() + + +def parse_args(): + parser = TrackingArgumentParser( + description="Offline inference demo for MiniCPM-o 4.5 (text / image / audio / video → text + speech)" + ) + parser.add_argument( + "--model", + type=str, + default="openbmb/MiniCPM-o-4_5", + help="Model name or path.", + ) + parser.add_argument( + "--query-type", + "-q", + type=str, + default="text", + choices=query_map.keys(), + help="Query type.", + ) + parser.add_argument( + "--log-stats", + action="store_true", + default=False, + help="Enable writing detailed statistics (default: disabled)", + ) + parser.add_argument( + "--stage-init-timeout", + type=int, + default=300, + help="Timeout for initializing a single stage in seconds (default: 300)", + ) + parser.add_argument( + "--batch-timeout", + type=int, + default=5, + help="Timeout for batching in seconds (default: 5)", + ) + parser.add_argument( + "--init-timeout", + type=int, + default=300, + help="Timeout for initializing stages in seconds (default: 300)", + ) + parser.add_argument( + "--shm-threshold-bytes", + type=int, + default=65536, + help="Threshold for using shared memory in bytes (default: 65536)", + ) + parser.add_argument( + "--output-wav", + default="output_audio", + help="[Deprecated] Output wav directory (use --output-dir).", + ) + parser.add_argument( + "--output-dir", + type=str, + default=None, + help="Output directory for text/audio results.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1, + help="Number of prompts to generate.", + ) + parser.add_argument( + "--txt-prompts", + type=str, + default=None, + help="Path to a .txt file with one prompt per line (preferred).", + ) + parser.add_argument( + "--stage-configs-path", + type=str, + default=None, + help="Path to a stage configs file (deprecated; prefer --deploy-config).", + ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Path to a deploy YAML (default: auto-load minicpmo_4_5.yaml).", + ) + parser.add_argument( + "--video-path", + "-v", + type=str, + default=None, + help="Path to local video file. If not provided, uses default video asset.", + ) + parser.add_argument( + "--image-path", + "-i", + type=str, + default=None, + help="Path to local image file. If not provided, uses default image asset.", + ) + parser.add_argument( + "--audio-path", + "-a", + type=str, + default=None, + help="Path to local audio file. If not provided, uses default audio asset.", + ) + parser.add_argument( + "--num-frames", + type=int, + default=16, + help="Number of frames to extract from video (default: 16).", + ) + parser.add_argument( + "--sampling-rate", + type=int, + default=16000, + help="Sampling rate for audio loading (default: 16000).", + ) + parser.add_argument( + "--log-dir", + type=str, + default="logs", + help="Log directory (default: logs).", + ) + parser.add_argument( + "--modalities", + type=str, + default=None, + help="Output modalities (comma-separated), e.g. text or text,audio.", + ) + parser.add_argument( + "--py-generator", + action="store_true", + default=False, + help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.", + ) + parser.add_argument( + "--enable-profiler", + action="store_true", + default=False, + help="Enables profiling when set.", + ) + parser.add_argument( + "--profiler-stages", + type=int, + nargs="*", + default=None, + help="List of stage IDs to profile. If not set, profiles all stages.", + ) + parser.add_argument( + "--dtype", + type=str, + default="auto", + help="Model dtype (auto, half, float16, bfloat16, float, float32).", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/examples/offline_inference/minicpmo/run_multiple_prompts.sh b/examples/offline_inference/minicpmo/run_multiple_prompts.sh new file mode 100755 index 000000000..f130fece3 --- /dev/null +++ b/examples/offline_inference/minicpmo/run_multiple_prompts.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Multi-prompt offline batch (py_generator mode). +set -euo pipefail +cd "$(dirname "$0")" + +python end2end.py --output-wav output_audio \ + --query-type text \ + --txt-prompts ../qwen3_omni/text_prompts_10.txt \ + --py-generator diff --git a/examples/offline_inference/minicpmo/run_single_prompt.sh b/examples/offline_inference/minicpmo/run_single_prompt.sh new file mode 100755 index 000000000..3a43ddda9 --- /dev/null +++ b/examples/offline_inference/minicpmo/run_single_prompt.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Single-prompt offline smoke test (text → text + 24 kHz speech). +set -euo pipefail +cd "$(dirname "$0")" + +python end2end.py --output-wav output_audio \ + --query-type text diff --git a/examples/offline_inference/minicpmo/run_single_prompt_tp.sh b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh new file mode 100755 index 000000000..88f488d2c --- /dev/null +++ b/examples/offline_inference/minicpmo/run_single_prompt_tp.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Single-prompt offline run with thinker tensor-parallel (3-GPU layout). +# Thinker on GPU 0,1 (TP=2); talker + Token2Wav on GPU 2. +set -euo pipefail +cd "$(dirname "$0")" + +REPO_ROOT="$(cd ../../.. && pwd)" + +python end2end.py --output-wav output_audio \ + --query-type use_audio \ + --deploy-config "${REPO_ROOT}/vllm_omni/deploy/minicpmo_4_5_3gpu.yaml" \ + --stage-init-timeout 300 diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 334b5ed11..e1c084224 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -1,14 +1,19 @@ -# vLLM-Omni · MiniCPM-o 4.5 Online Demo +# MiniCPM-o 4.5: Online serving -Gradio-based web UI for **MiniCPM-o 4.5** served via `vllm-omni`'s -OpenAI-compatible endpoints. +OpenAI-compatible `/v1/chat/completions` serving for **MiniCPM-o 4.5**, plus a +Gradio UI and curl / Python clients. -The UI supports: +Inputs: text / image / audio / video. Outputs: text and optional **24 kHz** speech. -- **Inputs**: text prompt + optional image, audio (file or mic), video. -- **Outputs**: text + speech (WAV player). +## Installation -## 1. Start the backend server +Please refer to [README.md](../../../README.md). Install the talker extra: + +```bash +pip install 'vllm-omni[minicpmo]' +``` + +## Launch the server The deploy config auto-loads via `--omni`; the default `vllm_omni/deploy/minicpmo_4_5.yaml` targets a single-GPU layout (thinker @@ -30,37 +35,125 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \ --host 0.0.0.0 --port 8099 ``` -Other layouts via `--deploy-config`: +Other layouts: ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni \ - --deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \ + --deploy-config vllm_omni/deploy/minicpmo_4_5_3gpu.yaml \ --trust-remote-code \ --host 0.0.0.0 --port 8099 ``` -## 2. Launch the Gradio demo +### Stage-based CLI (optional) + +Stage 0 (thinker + API) and stage 1 (talker) can run in separate processes: ```bash -bash examples/online_serving/minicpmo/run_gradio_demo.sh +# Stage 0 +CUDA_VISIBLE_DEVICES=0 vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --trust-remote-code --port 8099 --stage-id 0 \ + --omni-master-address 127.0.0.1 --omni-master-port 26000 -# Or run the python entry point directly: -python examples/online_serving/minicpmo/gradio_demo.py \ +# Stage 1 (headless) +CUDA_VISIBLE_DEVICES=1 vllm serve openbmb/MiniCPM-o-4_5 --omni \ + --trust-remote-code --stage-id 1 --headless \ + --omni-master-address 127.0.0.1 --omni-master-port 26000 +``` + +### Per-stage overrides + +```bash +vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code --port 8099 \ + --stage-overrides '{"0": {"gpu_memory_utilization": 0.65}}' +``` + +## Send multimodal requests + +```bash +cd examples/online_serving/minicpmo +``` + +### curl + +```bash +bash run_curl_multimodal_generation.sh text +bash run_curl_multimodal_generation.sh use_image +bash run_curl_multimodal_generation.sh use_audio '["text"]' # text-only +``` + +Text + speech smoke test (TTS needs top-level `chat_template_kwargs`): + +```bash +curl http://localhost:8099/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "openbmb/MiniCPM-o-4_5", + "messages": [{"role": "user", "content": "Say hello, then introduce vLLM in one sentence."}], + "modalities": ["text", "audio"], + "chat_template_kwargs": {"use_tts_template": true} + }' +``` + +### Python client + +```bash +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type use_image \ + --port 8099 \ + --host localhost + +# Text-only (faster; no <|tts_bos|>) +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type text \ + --modalities text \ + --prompt "Briefly introduce yourself." +``` + +Shared helpers also work if you pass MiniCPM defaults yourself: + +```bash +python ../openai_chat_completion_client_for_multimodal_generation.py \ + --model openbmb/MiniCPM-o-4_5 \ + --query-type text \ + --port 8099 +``` + +(Note: the shared client does **not** set `use_tts_template`; prefer the +MiniCPM-specific client above for speech.) + +### Gradio demo + +```bash +bash run_gradio_demo.sh + +# Or: +python gradio_demo.py \ --minicpmo45-api-base http://localhost:8099/v1 \ --minicpmo45-model openbmb/MiniCPM-o-4_5 \ --port 7862 ``` -Open `http://:7862` in a browser. +Open `http://:7862`. Uncheck **"Generate speech output (TTS)"** for +text-only responses. + +## Modality control + +| Modalities | Output | +|---|---| +| `["text"]` | Text only (no TTS bos) | +| `["text", "audio"]` / unset | Text + 24 kHz speech | + +Speech requires `chat_template_kwargs.use_tts_template=true` so the chat +template appends `<|tts_bos|>`. For **curl**, put that field at the request +root; nested `extra_body` is ignored. The OpenAI Python SDK may use +`extra_body` because it merges those fields into the root. ## Notes -- **TTS trigger**: the demo sets - `extra_body.chat_template_kwargs.use_tts_template=True`, which appends - `<|tts_bos|>` to the assistant prefix. -- Uncheck **"Generate speech output (TTS)"** to get text-only responses - (faster). -- The audio output is the raw WAV returned by the stage-1 talker + - Token2Wav; sample rate is 24 kHz. -- Video input is forwarded as a base64 `video_url` entry; the server needs - decord/torchvision to decode it. +- Stage 1 is capped at `max_num_seqs: 1` in the deploy YAML (talker shares + request-0 audio metadata). +- Output audio is base64 WAV in `message.audio.data` (24 kHz mono). +- Offline counterpart: + [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/) +- Recipe: + [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md) diff --git a/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py new file mode 100644 index 000000000..45afe7e5d --- /dev/null +++ b/examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""OpenAI chat-completions client for MiniCPM-o 4.5 online serving. + +Thin wrapper around the shared multimodal client helpers, with MiniCPM-specific +defaults (port 8099, system prompt, ``chat_template_kwargs.use_tts_template``). +""" + +from __future__ import annotations + +import base64 +import concurrent.futures +import os +import sys + +from openai import OpenAI + +from vllm_omni.utils.tracking_parser import TrackingArgumentParser + +# Reuse prompt builders / URL helpers from the shared online client. +_HERE = os.path.dirname(os.path.abspath(__file__)) +_ONLINE_ROOT = os.path.abspath(os.path.join(_HERE, "..")) +if _ONLINE_ROOT not in sys.path: + sys.path.insert(0, _ONLINE_ROOT) + +from openai_chat_completion_client_for_multimodal_generation import ( # noqa: E402 + _build_prompt_for_query_type, + _parse_csv_arg, + make_audio_output_filename, + query_map, +) + +SEED = 42 + +default_system = ( + "You are MiniCPM-o, a helpful multimodal assistant that can " + "understand images, audio and video, and respond in text and speech." +) + + +def get_system_prompt() -> dict: + return { + "role": "system", + "content": [{"type": "text", "text": default_system}], + } + + +def _wants_tts(modalities: list[str] | None) -> bool: + if modalities is None: + return True + return "audio" in modalities + + +def run_multimodal_generation(args, client: OpenAI) -> None: + model_name = args.model + video_path = getattr(args, "video_path", None) + image_path = getattr(args, "image_path", None) + audio_path = getattr(args, "audio_path", None) + custom_prompt = getattr(args, "prompt", None) + + if args.modalities is not None: + output_modalities = args.modalities.split(",") + else: + output_modalities = None + + use_tts = _wants_tts(output_modalities) + num_concurrent_requests = args.num_concurrent_requests + prompt_list = _parse_csv_arg(getattr(args, "prompts", None)) + + request_payloads = [] + for idx in range(num_concurrent_requests): + per_req_prompt = ( + prompt_list[idx] + if idx < len(prompt_list) + else (custom_prompt if idx == 0 or not prompt_list else prompt_list[-1]) + ) + prompt = _build_prompt_for_query_type( + query_type=args.query_type, + custom_prompt=per_req_prompt, + video_path=video_path, + image_path=image_path, + audio_path=audio_path, + ) + extra_body: dict = { + "chat_template_kwargs": {"use_tts_template": use_tts}, + } + if args.query_type == "use_audio_in_video": + # MiniCPM-o 4.5 does not use Qwen's use_audio_in_video path; keep + # the flag only if the shared query type is selected explicitly. + extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True} + request_payloads.append({"prompt": prompt, "extra_body": extra_body}) + + with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: + futures = [ + executor.submit( + client.chat.completions.create, + messages=[ + get_system_prompt(), + payload["prompt"], + ], + model=model_name, + modalities=output_modalities, + extra_body=payload["extra_body"], + stream=args.stream, + ) + for payload in request_payloads + ] + chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)] + + assert len(chat_completions) == num_concurrent_requests + count = 0 + if not args.stream: + for chat_completion in chat_completions: + request_id = getattr(chat_completion, "id", None) + for choice in chat_completion.choices: + if choice.message.audio: + audio_data = base64.b64decode(choice.message.audio.data) + audio_file_path = make_audio_output_filename(request_id=request_id, index=count) + with open(audio_file_path, "wb") as f: + f.write(audio_data) + print(f"Audio saved to {audio_file_path}") + count += 1 + elif choice.message.content: + print("Chat completion output from text:", choice.message.content) + else: + printed_content = False + for chat_completion in chat_completions: + for chunk in chat_completion: + for choice in chunk.choices: + if hasattr(choice, "delta"): + content = getattr(choice.delta, "content", None) + else: + content = None + + if getattr(chunk, "modality", None) == "audio" and content: + audio_data = base64.b64decode(content) + request_id = getattr(chunk, "id", None) + audio_file_path = make_audio_output_filename(request_id=request_id, index=count) + with open(audio_file_path, "wb") as f: + f.write(audio_data) + print(f"\nAudio saved to {audio_file_path}") + count += 1 + elif getattr(chunk, "modality", None) == "text": + if not printed_content: + printed_content = True + print("\ncontent:", end="", flush=True) + print(content, end="", flush=True) + + +def parse_args(): + parser = TrackingArgumentParser(description="MiniCPM-o 4.5 online multimodal chat client") + parser.add_argument( + "--query-type", + "-q", + type=str, + default="text", + choices=query_map.keys(), + help="Query type.", + ) + parser.add_argument( + "--model", + "-m", + type=str, + default="openbmb/MiniCPM-o-4_5", + help="Model Name / Path", + ) + parser.add_argument( + "--video-path", + "-v", + type=str, + default=None, + help="Path to local video file or URL.", + ) + parser.add_argument( + "--image-path", + "-i", + type=str, + default=None, + help="Path to local image file or URL.", + ) + parser.add_argument( + "--audio-path", + "-a", + type=str, + default=None, + help="Path to local audio file or URL.", + ) + parser.add_argument( + "--prompt", + "-p", + type=str, + default=None, + help="Custom text prompt/question.", + ) + parser.add_argument( + "--modalities", + type=str, + default=None, + help="Output modalities, e.g. text or text,audio.", + ) + parser.add_argument( + "--stream", + action="store_true", + help="Stream the response.", + ) + parser.add_argument( + "--num-concurrent-requests", + type=int, + default=1, + help="Number of concurrent requests to send.", + ) + parser.add_argument( + "--port", + type=int, + default=8099, + help="Port of the vLLM Omni API server.", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Host/IP of the vLLM Omni API server.", + ) + parser.add_argument( + "--prompts", + type=str, + default=None, + help="Comma-separated prompts for concurrent requests.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + openai_api_base = f"http://{args.host}:{args.port}/v1" + client = OpenAI(api_key="EMPTY", base_url=openai_api_base) + run_multimodal_generation(args, client) diff --git a/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh new file mode 100755 index 000000000..426a2df00 --- /dev/null +++ b/examples/online_serving/minicpmo/run_curl_multimodal_generation.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Default query type +QUERY_TYPE="${1:-text}" + +# Default modalities argument (JSON). Use null for server default (text+audio). +MODALITIES="${2:-null}" + +# Validate query type +if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then + echo "Error: Invalid query type '$QUERY_TYPE'" + echo "Usage: $0 [text|use_audio|use_image|use_video] [modalities]" + echo " text: Text query" + echo " use_audio: Audio + text query" + echo " use_image: Image + text query" + echo " use_video: Video + text query" + echo " modalities: JSON list or null (default: null → text+audio)" + echo " examples: '[\"text\"]' '[\"text\",\"audio\"]'" + exit 1 +fi + +HOST="${HOST:-localhost}" +PORT="${PORT:-8099}" +MODEL="${MODEL:-openbmb/MiniCPM-o-4_5}" + +thinker_sampling_params='{ + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": 2048, + "seed": 42, + "detokenize": true, + "repetition_penalty": 1.1 +}' + +talker_sampling_params='{ + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "max_tokens": 1, + "seed": 42, + "detokenize": false +}' +# Above is optional; defaults live in vllm_omni/deploy/minicpmo_4_5.yaml. + +MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" +CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg" +SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4" + +case "$QUERY_TYPE" in + text) + user_content='[ + { + "type": "text", + "text": "Say hello, then introduce vLLM-Omni in one sentence." + } + ]' + ;; + use_audio) + user_content='[ + { + "type": "audio_url", + "audio_url": { + "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" + } + }, + { + "type": "text", + "text": "What is the content of this audio?" + } + ]' + ;; + use_image) + user_content='[ + { + "type": "image_url", + "image_url": { + "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" + } + }, + { + "type": "text", + "text": "What is the content of this image?" + } + ]' + ;; + use_video) + user_content='[ + { + "type": "video_url", + "video_url": { + "url": "'"$SAMPLE_VIDEO_URL"'" + } + }, + { + "type": "text", + "text": "Why is this video funny?" + } + ]' + ;; +esac + +sampling_params_list='[ + '"$thinker_sampling_params"', + '"$talker_sampling_params"' +]' + +# TTS speech path needs use_tts_template at the request root (curl does not +# flatten nested extra_body). Skip it when the caller asked for text-only. +USE_TTS_TEMPLATE=true +if [[ "$MODALITIES" == '["text"]' || "$MODALITIES" == "['text']" ]]; then + USE_TTS_TEMPLATE=false +fi + +echo "Running query type: $QUERY_TYPE (host=$HOST port=$PORT model=$MODEL)" +echo "modalities=$MODALITIES use_tts_template=$USE_TTS_TEMPLATE" +echo "" + +request_body=$(cat < Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) +> Online serving and offline inference for omni multimodal chat +> (text / image / audio / video → text + 24 kHz speech) ## Summary @@ -9,7 +10,7 @@ - Task: Omni multimodal chat — accepts text / image / audio / video input; emits text and 24 kHz mono speech in the same response - Mode: Online serving via the OpenAI-compatible `/v1/chat/completions` - API, plus a bundled Gradio demo (text + speech UI) + API (plus Gradio demo), and offline inference via `Omni.generate` - Maintainer: [`@tc-mb`](https://github.com/tc-mb) (MiniCPM-V / MiniCPM-o team) ## When to use this recipe @@ -37,6 +38,8 @@ shot. The recipe covers the shipped GPU layouts (single / 2 / 3 / 8 GPUs): [`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml) - Online example + Gradio demo: [`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/) +- Offline end-to-end example: + [`examples/offline_inference/minicpmo/`](../../examples/offline_inference/minicpmo/) - Pipeline / talker source: [`vllm_omni/model_executor/models/minicpmo_4_5/`](../../vllm_omni/model_executor/models/minicpmo_4_5/) - Stage-input processor (thinker → talker bridge):