From 0a51a18570d8df8e65e9d4f9be688ce686c67ce6 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:35:13 +0800 Subject: [PATCH] [Bugfix] Leave MiniCPM-o vocoder memory headroom (#5621) Signed-off-by: lishunyang12 Signed-off-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: lishunyang12 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- examples/online_serving/minicpmo/README.md | 8 +++++--- recipes/OpenBMB/MiniCPM-o-4_5.md | 11 +++++++---- .../models/minicpmo_4_5/test_pipeline.py | 8 +++++--- vllm_omni/deploy/minicpmo_4_5.yaml | 5 +++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 94fae5fa7..aea9a2b6a 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -23,9 +23,11 @@ including `librosa`. The deploy config auto-loads via `--omni`. The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on -logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits -at most four sequences per stage and bounds startup video profiling to 32 -frames per video. For throughput,`minicpmo_4_5_2gpu.yaml` gives the Thinker +logical device 0 with memory budgets of 55%, 15%, and 15%, leaving headroom +for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The +profile admits at most four sequences per stage and bounds startup video +profiling to 32 frames per video. For throughput, +`minicpmo_4_5_2gpu.yaml` gives the Thinker GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That profile admits at most four concurrent sequences per stage. diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 340c89581..f87ee5b19 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -77,7 +77,9 @@ a fallback because it would duplicate state machines and correctness paths. The default [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) co-locates Thinker, codec-only Talker, and Code2Wav on GPU 0. Their -`gpu_memory_utilization` budgets are 0.55, 0.22, and 0.22. All stages admit +`gpu_memory_utilization` budgets are 0.55, 0.15, and 0.15. The remaining +device memory is left available for runtime workspaces such as the HiFi-GAN +vocoder's cuDNN kernels. All stages admit up to four sequences. Startup video profiling is bounded to 32 frames per video. Use the two-GPU profile for production throughput. @@ -202,9 +204,10 @@ speech output (TTS)"** checkbox on / off. #### Notes -- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.22, and - 0.22 of GPU 0. The larger Thinker share protects its multimodal KV cache; - all three model processes still share one CUDA device. +- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.15, and + 0.15 of GPU 0. The larger Thinker share protects its multimodal KV cache, + while the unreserved memory remains available to runtime kernels; all three + model processes still share one CUDA device. - `--trust-remote-code` is required — the HF repo ships a custom `MiniCPMO` config / model class. - Stage 0 Thinker and Stage 1 Talker enable vLLM CUDA Graphs. Stage 2 remains diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index b6f11ff5d..84b5ca11c 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -163,11 +163,13 @@ class TestDeployTopology: assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor) if filename == "minicpmo_4_5.yaml": assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4] - assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ + memory_utilizations = [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] + assert memory_utilizations == [ 0.55, - 0.22, - 0.22, + 0.15, + 0.15, ] + assert sum(memory_utilizations) <= 0.9 + 1e-6 assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}} elif filename in {"minicpmo_4_5_2gpu.yaml"}: assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index ea1fdaf2f..abe8b5ad2 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -38,7 +38,8 @@ stages: - stage_id: 1 max_num_seqs: 4 - gpu_memory_utilization: 0.22 + # Leave device-level headroom for the colocated HiFi-GAN cuDNN workspace. + gpu_memory_utilization: 0.15 enforce_eager: false max_num_batched_tokens: 8192 devices: "0" @@ -58,7 +59,7 @@ stages: - stage_id: 2 max_num_seqs: 4 - gpu_memory_utilization: 0.22 + gpu_memory_utilization: 0.15 enforce_eager: true enable_chunked_prefill: false max_num_batched_tokens: 65536