77 lines
2.8 KiB
YAML
77 lines
2.8 KiB
YAML
# CosyVoice3 deploy: 2-stage talker → flow-matching code2wav.
|
|
#
|
|
# Default mode is async-chunk streaming through SharedMemoryConnector. Pass
|
|
# ``--no-async-chunk`` to the serve CLI for the legacy sync path (stage 1
|
|
# builds flow input from the full speech-token sequence via ``text2flow``).
|
|
# The shared-memory connector definition and input/output_connectors
|
|
# references stay in the yaml unconditionally; the runtime only activates
|
|
# them when ``async_chunk: true``.
|
|
#
|
|
# enforce_eager:
|
|
# * Stage 0 (talker) — enforce_eager: false (CUDA graphs enabled).
|
|
# * Stage 1 (code2wav) — enforce_eager: true (CUDA graphs don't work with
|
|
# dynamic conv shapes).
|
|
#
|
|
# Streaming continuity is GPU-class sensitive — TUNE PER GPU.
|
|
# The defaults below (max_num_seqs: 8, codec_chunk_frames: 15) are tuned for
|
|
# H100, where streaming continuity at concurrency 4 stays ~100%. They push
|
|
# more audio per scheduler step through the SharedMemoryConnector than a
|
|
# slower GPU can keep up with: on H20-3e the same config regressed continuity
|
|
# to ~78% at c=4 (vs ~86% with smaller batch/chunk), because code2wav can't
|
|
# produce mel chunks fast enough to feed the streaming output → buffer
|
|
# underruns → audible gaps.
|
|
# On a slower GPU class, lower max_num_seqs (e.g. 4) and/or codec_chunk_frames
|
|
# first. Raising connector_get_sleep_s also relaxes the consumer's polling
|
|
# cadence, but trades against TTFP — keep it as a last resort.
|
|
pipeline: cosyvoice3
|
|
async_chunk: true
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
codec_streaming: true
|
|
connector_get_sleep_s: 0.01
|
|
connector_get_max_wait_first_chunk: 3000
|
|
connector_get_max_wait: 300
|
|
codec_chunk_frames: 15
|
|
codec_left_context_frames: 25
|
|
codec_vocab_size: 6561
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 8
|
|
gpu_memory_utilization: 0.4
|
|
enforce_eager: false
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "0"
|
|
output_connectors:
|
|
to_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
max_tokens: 2048
|
|
top_k: 25
|
|
top_p: 0.8
|
|
# near-identity repetition penalty forces vLLM to track
|
|
# output_token_ids for RAS (stop-token logit logsumexp).
|
|
repetition_penalty: 1.0001
|
|
disable_hybrid_kv_cache_manager: true
|
|
mm_processor_cache_gb: 0
|
|
skip_mm_profiling: true
|
|
|
|
- stage_id: 1
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 8
|
|
gpu_memory_utilization: 0.2
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
max_model_len: 32768
|
|
devices: "0"
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
max_tokens: 2048
|
|
disable_hybrid_kv_cache_manager: true
|
|
skip_mm_profiling: true
|