diff --git a/vllm_omni/deploy/voxcpm2.yaml b/vllm_omni/deploy/voxcpm2.yaml index 4e3f7f3f6..5d366521b 100644 --- a/vllm_omni/deploy/voxcpm2.yaml +++ b/vllm_omni/deploy/voxcpm2.yaml @@ -1,5 +1,6 @@ # VoxCPM2 deploy: single-stage AR pipeline with per-request state batching. -# Verified on 1x H20 141GB. +# Runs on large (H20 141GB) and small (24GB L4) cards alike: the KV cache is +# pinned to a right-sized budget instead of ballooning to fill the card. # # VoxCPM2-local runtime knobs are passed through hf_overrides. async_chunk: false @@ -9,6 +10,16 @@ stages: - stage_id: 0 max_num_seqs: 32 gpu_memory_utilization: 0.9 + # Right-size the KV cache instead of letting it fill gpu_memory_utilization. + # max_num_seqs(32) * max_model_len(4096) ~= 131k tokens; 6 GiB holds ~175k + # (42x concurrency, above the 32-seq scheduler cap, so no throughput loss). + # The remaining VRAM stays free for the VoxCPM2 diffusion side-path + # (LocDiT / AudioVAE / code2wav + batch CUDA-graph pools), which allocates + # outside vLLM's KV accounting during inference. Keeps peak ~13 GiB on any + # card and avoids per-rebase OOM tuning on small CI GPUs. + # Assumes prefix caching off and max_num_seqs*max_model_len <= ~175k tokens; + # raise this cap if either grows or enable_prefix_caching is turned on. + kv_cache_memory_bytes: 6442450944 # 6 GiB enforce_eager: true trust_remote_code: true enable_prefix_caching: false