53 lines
2.1 KiB
YAML
53 lines
2.1 KiB
YAML
# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching.
|
|
# Runs on large (H20 141GB) and small (24GB L4) cards alike: the KV cache is
|
|
# pinned to a right-sized budget instead of ballooning to fill the card.
|
|
#
|
|
# VoxCPM2-local runtime knobs are passed through hf_overrides.
|
|
async_chunk: false
|
|
dtype: bfloat16
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 8
|
|
gpu_memory_utilization: 0.9
|
|
# Right-size the KV cache instead of letting it fill gpu_memory_utilization.
|
|
# max_num_seqs(8) * max_model_len(4096) ~= 32k tokens; 6 GiB leaves headroom
|
|
# for longer live requests while keeping VoxCPM2 TTS decode batches small
|
|
# enough to avoid the high ITL observed with large admission caps.
|
|
# The remaining VRAM stays free for the VoxCPM2 diffusion side-path
|
|
# (LocDiT / AudioVAE / code2wav + batch CUDA-graph pools), which allocates
|
|
# outside vLLM's KV accounting during inference. Keeps peak ~13 GiB on any
|
|
# card and avoids per-rebase OOM tuning on small CI GPUs.
|
|
# Assumes prefix caching off and typical live decode concurrency <= ~8;
|
|
# raise this cap if either grows or enable_prefix_caching is turned on.
|
|
kv_cache_memory_bytes: 6442450944 # 6 GiB
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: true
|
|
max_num_batched_tokens: 4096
|
|
max_model_len: 4096
|
|
devices: "0"
|
|
engine_extras:
|
|
hf_overrides:
|
|
voxcpm2_runtime_config:
|
|
audio_emit_every: 1
|
|
vae_decode_every: 3
|
|
enable_batched_cfm: true
|
|
enable_batched_vae_decode: true
|
|
coalesce_audio_d2h: true
|
|
enable_loc_dit_fused_qkv: true
|
|
enable_loc_dit_fused_mlp: true
|
|
enable_loc_dit_zero_dt_cache: true
|
|
enable_loc_dit_skip_qkv_contig: true
|
|
enable_loc_dit_reduce_overhead_no_cg: true
|
|
enable_unified_decode_graph: true
|
|
unified_decode_graph_max_batch_size: 8
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 4096
|
|
seed: 42
|
|
repetition_penalty: 1.0
|