Sy03andYueqian Lin
3bc3f1a7d0
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
54 lines
2.1 KiB
YAML
54 lines
2.1 KiB
YAML
# Opt-in CUDA profile for MOSS-TTS-Local-Transformer-v1.5 on a large-memory GPU.
|
|
# Both stages share GPU 0. Validated with one H200 and Seed-TTS EN1088.
|
|
base_config: moss_tts_local_mrv2.yaml
|
|
|
|
connectors:
|
|
shm:
|
|
extra:
|
|
# Coalesce ready codec streams before submitting a small execution batch.
|
|
# This is a dispatch target, not a cap: execution capacity remains 128.
|
|
generation_min_batch_size: 32
|
|
generation_max_wait_ms: 12
|
|
# Optional; preserves the first/steady sizes while adding a middle chunk.
|
|
# codec_chunk_ramp: [1, 4, 15]
|
|
|
|
# Optional codec attention-ring headroom (see recipes/OpenMOSS/MOSS-TTS.md):
|
|
# sizes each ring as context + one chunk so chunked streaming matches
|
|
# whole-sequence decoding. Requires additional state/graph memory; evaluate
|
|
# throughput and audio quality before enabling for your context and capacity.
|
|
# Add codec_ring_headroom: 1 under connectors.shm.extra above.
|
|
|
|
platforms:
|
|
cuda:
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 128
|
|
max_num_batched_tokens: 512
|
|
hf_overrides:
|
|
mrv2_gpu_slot_state: true
|
|
# Throughput-oriented pair; measure first-packet latency for your load.
|
|
mrv2_batch_prefill: true
|
|
mrv2_direct_tokens: true
|
|
# Mixed prefill/decode fits the same bounded token budget as capture.
|
|
compilation_config:
|
|
mode: 3
|
|
cudagraph_mode: FULL
|
|
max_cudagraph_capture_size: 512
|
|
cudagraph_num_of_warmups: 1
|
|
# Leave room for the codec state pool and its batch-128 CUDA graphs.
|
|
kv_cache_memory_bytes: 34359738368 # 32 GiB
|
|
- stage_id: 1
|
|
max_num_seqs: 128
|
|
hf_overrides:
|
|
codec_attention_backend: triton_slot
|
|
compilation_config:
|
|
mode: 3
|
|
backend: inductor
|
|
# The codec owns capture/replay after Inductor compiles its decoder.
|
|
cudagraph_mode: FULL
|
|
cudagraph_capture_sizes: [1, 2, 4, 6, 8, 12, 16, 24, 32, 64, 128]
|
|
cudagraph_num_of_warmups: 1
|
|
inductor_compile_config:
|
|
combo_kernels: false
|
|
benchmark_combo_kernel: false
|