# Qwen3-TTS deploy: talker → code2wav via shared-memory chunk streaming. # Verified on 1x H100. # # Fields omitted from a stage fall back to StageDeployConfig defaults (see # vllm_omni/config/stage_config.py). Notable choices for this model: # * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk # latency tuning (not correctness) — small per-step batches keep # first-chunk latency low. # * Stage 1 (code2wav) sets max_num_batched_tokens=65536 for correctness: # codec prefill length (Q * num_frames) exceeds the 32k default. # * Stage 0 does not set enforce_eager — talker runs cudagraph by default. # Stage 1 sets true because its codec generation loop is not # cudagraph-compatible. NPU platform flips stage 0 to true where # cudagraph is not yet verified. async_chunk: true connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: shm_threshold_bytes: 65536 codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 connector_get_max_wait: 300 # Must match the decoder sliding attention window. codec_chunk_frames: 25 codec_left_context_frames: 72 stages: - stage_id: 0 max_num_seqs: 10 gpu_memory_utilization: 0.3 async_scheduling: true max_num_batched_tokens: 512 max_model_len: 4096 devices: "0" output_connectors: to_stage_1: connector_of_shared_memory default_sampling_params: temperature: 0.9 top_k: 50 max_tokens: 4096 seed: 42 repetition_penalty: 1.05 subtalker_sampling_params: do_sample: true temperature: 0.9 top_k: 50 top_p: 1.0 - stage_id: 1 max_num_seqs: 1 gpu_memory_utilization: 0.3 enforce_eager: true async_scheduling: true # Must be divisible by num_code_groups and cover (left_context + chunk). # Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep # headroom past 32k. max_num_batched_tokens: 65536 # async_chunk appends windows per step; max_model_len must cover the # accumulated flat codec stream. max_model_len: 65536 devices: "0" input_connectors: from_stage_0: connector_of_shared_memory default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 max_tokens: 65536 seed: 42 repetition_penalty: 1.0