# MiniCPM-o 4.5 deploy: 8x4090 layout (24GB cards). # # Stage 0 (thinker): GPU 0,1,2,3 — 4-way TP, ~85% mem (≈20.4 GB / card, # thinker weights take ~17.5 GB per card # on 4090, leaving ~3 GB for KV cache / # activations). # Stage 1 (talker) : GPU 4 — MiniCPMTTS codec generation, ~90% mem. # Stage 2 (code2wav): GPU 5 — codec-to-waveform generation, ~90% mem. # # max_model_len is pinned to 4096 because the upstream config.json # advertises max_position_embeddings=40960 (tuned for H20 141GB); on a # 24 GB 4090 a 40k-token KV cache request OOMs the worker process # before serving the first request. 8192 also OOMs in practice with # this thinker model + vision/audio encoders + Whisper preallocation, # so 4096 is the empirically-verified safe ceiling. pipeline: minicpmo_4_5 async_chunk: true trust_remote_code: true enable_prefix_caching: false connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 connector_get_max_wait: 300 codec_chunk_frames: 25 codec_left_context_frames: 3 # HiFT capture buckets are derived from codec_chunk_frames and # codec_left_context_frames. (#5869) # A future initial_codec_chunk_frames shape will use lazy capture unless # HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it. enable_hift_graph: true enable_cfm_graph: true cfm_max_graphs: 32 stages: - stage_id: 0 max_num_seqs: 1 gpu_memory_utilization: 0.85 enforce_eager: true tensor_parallel_size: 4 max_model_len: 4096 max_num_batched_tokens: 4096 devices: "0,1,2,3" interleave_mm_strings: true media_io_kwargs: video: fps: 1 num_frames: 128 limit_mm_per_prompt: video: 1 default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 max_tokens: 2048 seed: 42 detokenize: true repetition_penalty: 1.1 - stage_id: 1 max_num_seqs: 1 gpu_memory_utilization: 0.9 enforce_eager: true max_num_batched_tokens: 8192 devices: "4" output_connectors: to_stage_2: connector_of_shared_memory default_sampling_params: temperature: 0.8 top_p: 0.8 top_k: 100 repetition_penalty: 1.02 min_tokens: 50 max_tokens: 4096 seed: 42 detokenize: false - stage_id: 2 max_num_seqs: 1 gpu_memory_utilization: 0.9 enforce_eager: true enable_chunked_prefill: false max_num_batched_tokens: 65536 max_model_len: 65536 devices: "5" input_connectors: from_stage_1: connector_of_shared_memory default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 max_tokens: 65536 detokenize: true platforms: {}