Sy03andYueqian Lin
3bc3f1a7d0
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
67 lines
2.7 KiB
YAML
67 lines
2.7 KiB
YAML
# Opt-in CUDA profile for MOSS-TTS-Local-Transformer-v1.5 tuned for first-packet
|
|
# latency at client concurrency 128 on one H200 (Seed-TTS EN1088, closed loop).
|
|
# Both stages share GPU 0. Use moss_tts_local_mrv2_high_concurrency.yaml for
|
|
# throughput-first serving.
|
|
#
|
|
# Optional launch-level settings (see the full command in recipes/OpenMOSS/MOSS-TTS.md):
|
|
# - CUDA MPS for the GPU (stage 0 below lowers its own MPS client priority);
|
|
# - VLLM_OMNI_EVENT_DRIVEN_ORCH=1;
|
|
# - --api-server-count 4 with VLLM_WORKER_MULTIPROC_METHOD=spawn and
|
|
# VLLM_OMNI_MOSS_REF_CODES_SHARED_DIR=/dev/shm/<dir> so frontends share reference codes;
|
|
# - VLLM_OMNI_CONNECTOR_RECV_POLL_MS=1 so Stage 1 picks up first chunks sooner.
|
|
base_config: moss_tts_local_mrv2.yaml
|
|
|
|
connectors:
|
|
shm:
|
|
extra:
|
|
generation_min_batch_size: 32
|
|
generation_max_wait_ms: 12
|
|
# Stage 1 decodes each first chunk from its receive thread on a small
|
|
# dedicated graph set and emits it directly, so first chunks no longer
|
|
# wait for the engine loop or a regular codec step.
|
|
codec_first_chunk_fast_path: 1
|
|
# Regular codec replays wait for an in-flight first-chunk decode.
|
|
codec_first_chunk_gate: 1
|
|
# Short regular codec steps bound contention with first-chunk decodes.
|
|
generation_max_regular_batch: 16
|
|
# Optional ramp. Both the first-chunk graph and regular graphs follow it.
|
|
# codec_chunk_ramp: [1, 4, 15]
|
|
|
|
platforms:
|
|
cuda:
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 128
|
|
# Larger prefill budget admits bursts of new requests in fewer steps.
|
|
max_num_batched_tokens: 2048
|
|
hf_overrides:
|
|
mrv2_gpu_slot_state: true
|
|
mrv2_batch_prefill: true
|
|
mrv2_direct_tokens: true
|
|
# Sample each frame right after the forward that produced it.
|
|
mrv2_eager_mtp: true
|
|
compilation_config:
|
|
mode: 3
|
|
cudagraph_mode: FULL
|
|
max_cudagraph_capture_size: 2048
|
|
cudagraph_num_of_warmups: 1
|
|
kv_cache_memory_bytes: 34359738368 # 32 GiB
|
|
env:
|
|
# Under MPS, let codec kernels schedule ahead of the talker's.
|
|
CUDA_MPS_CLIENT_PRIORITY: "1"
|
|
- stage_id: 1
|
|
max_num_seqs: 128
|
|
# Keep synchronous scheduling; benchmark async scheduling separately.
|
|
async_scheduling: false
|
|
hf_overrides:
|
|
codec_attention_backend: triton_slot
|
|
compilation_config:
|
|
mode: 3
|
|
backend: inductor
|
|
cudagraph_mode: FULL
|
|
cudagraph_capture_sizes: [1, 2, 4, 6, 8, 12, 16, 24, 32, 64, 128]
|
|
cudagraph_num_of_warmups: 1
|
|
inductor_compile_config:
|
|
combo_kernels: false
|
|
benchmark_combo_kernel: false
|