# Qwen3-TTS high-concurrency deploy profile. # # This is intentionally separate from qwen3_tts.yaml. Use it only when running # sustained high-concurrency serving on two GPUs, for example: # # vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \ # --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml # # Profile validated for the c64 / PROMPTS=512 performance experiments: # Stage 0 talker on GPU 0 with S0=64, Stage 1 Code2Wav on GPU 1 with S1=10. # async_chunk: true connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: codec_streaming: true connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 connector_get_max_wait: 300 codec_chunk_frames: 25 codec_left_context_frames: 72 # Stage0 code-predictor prefix CUDA graphs for the c64 hot path. # These keys are consumed by Qwen3-TTS talker and ignored by Code2Wav. code_predictor_prefix_graphs: true code_predictor_prefix_graph_buckets: [64] code_predictor_prefix_graph_seq_lens: [2, 3, 4, 5, 6, 7, 8] # Keep voice-clone reference context bounded so Stage1 chunk lengths are # stable across different reference-audio durations. ref_code_context_frames: 72 # Emit only the first audio chunk early, then return to codec_chunk_frames. initial_codec_chunk_frames: 1 # Chunk ramp (opt-in): gradual size increase for first N chunks to # eliminate the IC→steady cliff (1→25). Uncomment to enable. # Each entry is the size of that chunk index; indices past the table # fall back to codec_chunk_frames. When enabled, ramp overrides # initial_codec_chunk_frames (including the per-request API field). # Example: # codec_chunk_ramp: [4, 4, 8, 16, 25] # async_chunk derives stateful ICL/xvec graph shapes from the ramp when # enabled, otherwise from the fixed chunk settings above, and ignores # decode_cudagraph_capture_sizes. Keep B>1 # captures opt-in; c64 e2e validation did not show a stable win. decode_cudagraph_batch_sizes: [1] decode_batch_max_size: 1 decode_enable_tf32: false stages: - stage_id: 0 max_num_seqs: 64 gpu_memory_utilization: 0.3 trust_remote_code: true enable_prefix_caching: false async_scheduling: true max_num_batched_tokens: 512 max_model_len: 4096 devices: "0" output_connectors: to_stage_1: connector_of_shared_memory default_sampling_params: temperature: 0.9 top_k: 50 max_tokens: 4096 # Official qwen-tts generates the talker with min_new_tokens=2; without # it, checkpoints that sample codec EOS on the first frame return empty # audio (HTTP 400). See #4962. min_tokens: 2 repetition_penalty: 1.05 subtalker_sampling_params: do_sample: true temperature: 0.9 top_k: 50 top_p: 1.0 - stage_id: 1 max_num_seqs: 10 gpu_memory_utilization: 0.3 enforce_eager: true trust_remote_code: true enable_prefix_caching: false async_scheduling: true max_num_batched_tokens: 65536 max_model_len: 65536 devices: "1" input_connectors: from_stage_0: connector_of_shared_memory default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 max_tokens: 65536 repetition_penalty: 1.0 platforms: npu: stages: - stage_id: 0 enforce_eager: true