96 lines
3.3 KiB
YAML
96 lines
3.3 KiB
YAML
# Qwen3-TTS deploy: talker → code2wav via shared-memory chunk streaming.
|
|
# Verified on 1x H100.
|
|
#
|
|
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
|
# vllm_omni/config/stage_config.py). Notable choices for this model:
|
|
# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk
|
|
# latency tuning (not correctness) — small per-step batches keep
|
|
# first-chunk latency low.
|
|
# * Stage 1 (code2wav) sets max_num_batched_tokens=65536 for correctness:
|
|
# codec prefill length (Q * num_frames) exceeds the 32k default.
|
|
# * Stage 0 does not set enforce_eager — talker runs cudagraph by default.
|
|
# Stage 1 defaults to false so Code2Wav can use its inner cudagraph;
|
|
# set true to disable that path in eager mode. NPU platform flips stage 0 to true where
|
|
# cudagraph is not yet verified.
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
shm_threshold_bytes: 65536
|
|
codec_streaming: true
|
|
connector_get_sleep_s: 0.01
|
|
connector_get_max_wait_first_chunk: 3000
|
|
connector_get_max_wait: 300
|
|
# Must match the decoder sliding attention window.
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 72
|
|
# Emit only the first audio chunk early, then return to codec_chunk_frames.
|
|
initial_codec_chunk_frames: 1
|
|
# Common Stage1 decode buckets:
|
|
# no-ref first/steady chunks: 25 / 97 frames
|
|
# Base ref-context first/steady chunks: 73 / 169 frames
|
|
# decoder internal non-streaming chunks: 325 frames
|
|
decode_cudagraph_capture_sizes: [25, 73, 97, 169, 325]
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_seqs: 10
|
|
gpu_memory_utilization: 0.3
|
|
trust_remote_code: true
|
|
# Prefix caching on by default. The HS opt-out (talker only consumes the
|
|
# last token's hidden state) and deferred mm-output writes added in this
|
|
# PR make per-step bookkeeping negligible, and any text repetition across
|
|
# requests (same voice or shared ref_audio) is a net throughput win.
|
|
# Disable explicitly if memory headroom is tight on a single-GPU deploy.
|
|
enable_prefix_caching: true
|
|
async_scheduling: true
|
|
max_num_batched_tokens: 512
|
|
max_model_len: 4096
|
|
devices: "0"
|
|
output_connectors:
|
|
to_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_k: 50
|
|
max_tokens: 4096
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
subtalker_sampling_params:
|
|
do_sample: true
|
|
temperature: 0.9
|
|
top_k: 50
|
|
top_p: 1.0
|
|
|
|
- stage_id: 1
|
|
max_num_seqs: 10
|
|
gpu_memory_utilization: 0.3
|
|
enforce_eager: false
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: true
|
|
# Must be divisible by num_code_groups and cover (left_context + chunk).
|
|
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
|
|
# headroom past 32k.
|
|
max_num_batched_tokens: 65536
|
|
# async_chunk appends windows per step; max_model_len must cover the
|
|
# accumulated flat codec stream.
|
|
max_model_len: 65536
|
|
devices: "0"
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
seed: 42
|
|
repetition_penalty: 1.0
|
|
|
|
platforms:
|
|
npu:
|
|
stages:
|
|
- stage_id: 0
|
|
enforce_eager: true
|