# Qwen3-Omni-MoE production deploy, verified on 2x H100 (stage 0 on cuda:0, # stages 1+2 on cuda:1). # # Fields omitted from a stage fall back to StageDeployConfig defaults (see # vllm_omni/config/stage_config.py). Notable implicit defaults for this # model: # * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens — # chat-sized prefill fits in the 32768 default. # * Stages 0/1 do not set enforce_eager — cudagraph runs by default # (false). Stage 2 (code2wav) keeps enforce_eager=false so Code2Wav can # use its inner cudagraph on CUDA. ROCm flips stage 2 to eager because # MIOpen conv_transpose1d is not capture-safe in the Code2Wav graph path. # * Platform sections flip enforce_eager per-stage where platform # cudagraph support differs. async_chunk: true connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: initial_codec_chunk_frames: 4 codec_chunk_frames: 25 codec_left_context_frames: 25 stages: - stage_id: 0 max_num_batched_tokens: 32768 max_num_seqs: 64 gpu_memory_utilization: 0.9 trust_remote_code: true enable_prefix_caching: false devices: "0" default_sampling_params: temperature: 0.0 max_tokens: 2048 - stage_id: 1 max_num_batched_tokens: 32768 max_num_seqs: 64 gpu_memory_utilization: 0.6 trust_remote_code: true enable_prefix_caching: false devices: "1" input_connectors: from_stage_0: connector_of_shared_memory default_sampling_params: temperature: 0.9 top_k: 50 max_tokens: 4096 repetition_penalty: 1.05 - stage_id: 2 max_num_batched_tokens: 65536 max_num_seqs: 64 gpu_memory_utilization: 0.1 enforce_eager: false trust_remote_code: true enable_prefix_caching: false enable_chunked_prefill: false async_scheduling: false devices: "1" input_connectors: from_stage_1: connector_of_shared_memory default_sampling_params: temperature: 0.0 top_p: 1.0 top_k: -1 max_tokens: 65536 repetition_penalty: 1.1 platforms: # The supported Qwen3-Omni ModelOpt checkpoint quantizes the thinker only. Clear the # checkpoint-level metadata for the BF16 audio stages on the verified MUSA # profile before vLLM performs its normal quantization auto-detection. musa: stages: - stage_id: 1 hf_overrides: quantization_config: null - stage_id: 2 hf_overrides: quantization_config: null npu: stages: - stage_id: 0 gpu_memory_utilization: 0.6 tensor_parallel_size: 2 max_num_batched_tokens: 8192 compilation_config: cudagraph_mode: PIECEWISE # Qwen3-Omni not FULL-cudagraph-safe on NPU devices: "0,1" - stage_id: 1 gpu_memory_utilization: 0.6 max_num_batched_tokens: 8192 compilation_config: cudagraph_mode: PIECEWISE # code_predictor MaskedSelect / inner NPU graph replay devices: "2" - stage_id: 2 gpu_memory_utilization: 0.3 compilation_config: cudagraph_mode: PIECEWISE devices: "2" # Code2Wav decode uses UpSample1d, which calls F.conv_transpose1d. # On ROCm this routes through MIOpen and is not capture-safe during # vLLM's outer graph capture, so stage 2 runs eager to avoid MIOpen/HIP # graph-capture failures. rocm: stages: - stage_id: 2 enforce_eager: true xpu: stages: - stage_id: 0 tensor_parallel_size: 4 enforce_eager: true max_cudagraph_capture_size: 0 devices: "0,1,2,3" - stage_id: 1 enforce_eager: true max_cudagraph_capture_size: 0 devices: "4" - stage_id: 2 gpu_memory_utilization: 0.3 max_cudagraph_capture_size: 0 devices: "4"