vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml

# Qwen3-Omni-MoE production deploy, verified on 2x H100 (stage 0 on cuda:0,
# stages 1+2 on cuda:1).
#
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
# vllm_omni/config/stage_config.py). Notable implicit defaults for this
# model:
#   * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens —
#     chat-sized prefill fits in the 32768 default.
#   * Stages 0/1 do not set enforce_eager — cudagraph runs by default
#     (false). Stage 2 (code2wav) keeps enforce_eager=false so Code2Wav can
#     use its inner cudagraph on CUDA. ROCm flips stage 2 to eager because
#     MIOpen conv_transpose1d is not capture-safe in the Code2Wav graph path.
#   * Platform sections flip enforce_eager per-stage where platform
#     cudagraph support differs.
async_chunk: true

connectors:
  connector_of_shared_memory:
    name: SharedMemoryConnector
    extra:
      initial_codec_chunk_frames: 4
      codec_chunk_frames: 25
      codec_left_context_frames: 25

stages:
  - stage_id: 0
    max_num_batched_tokens: 32768
    max_num_seqs: 64
    gpu_memory_utilization: 0.9
    moe_backend: triton
    trust_remote_code: true
    enable_prefix_caching: false
    devices: "0"
    default_sampling_params:
      temperature: 0.0
      max_tokens: 2048

  - stage_id: 1
    max_num_batched_tokens: 32768
    max_num_seqs: 64
    gpu_memory_utilization: 0.6
    moe_backend: triton
    trust_remote_code: true
    enable_prefix_caching: false
    devices: "1"
    input_connectors:
      from_stage_0: connector_of_shared_memory
    default_sampling_params:
      # The talker is the only stochastic stage (the thinker and code2wav both
      # decode greedily at temperature 0). Without a seed, every request draws
      # fresh codec tokens, and a small fraction of draws derail into fluent
      # nonsense part-way through an utterance while the thinker text stays
      # correct -- the audio-vs-text nightly failures in #6090. This seed was
      # present from the original deploy schema and was dropped as collateral
      # in #4986; keep it so Qwen3-Omni speech is reproducible for a given
      # request. It bounds sampling only: it does not make the talker immune
      # to the underlying degeneration, which is tracked separately.
      seed: 42
      temperature: 0.9
      top_k: 50
      max_tokens: 4096
      repetition_penalty: 1.05

  - stage_id: 2
    max_num_batched_tokens: 65536
    max_num_seqs: 64
    gpu_memory_utilization: 0.1
    enforce_eager: false
    trust_remote_code: true
    enable_prefix_caching: false
    enable_chunked_prefill: false
    async_scheduling: false
    devices: "1"
    input_connectors:
      from_stage_1: connector_of_shared_memory
    default_sampling_params:
      temperature: 0.0
      top_p: 1.0
      top_k: -1
      max_tokens: 65536
      repetition_penalty: 1.1

platforms:
  # vLLM 0.27's compiled native MRoPE path can vertically fuse the thinker's
  # Q/K RMSNorm reductions into the rotary operations and materially change
  # Qwen3-Omni audio output (#6090). Keep MRoPE on its CUDA path instead.
  cuda:
    stages:
      - stage_id: 0
        compilation_config:
          custom_ops: ["+rotary_embedding"]

  # The supported Qwen3-Omni ModelOpt checkpoint quantizes the thinker only. Clear the
  # checkpoint-level metadata for the BF16 audio stages on the verified MUSA
  # profile before vLLM performs its normal quantization auto-detection.
  musa:
    stages:
      - stage_id: 1
        hf_overrides:
          quantization_config: null
      - stage_id: 2
        hf_overrides:
          quantization_config: null

  npu:
    stages:
      - stage_id: 0
        gpu_memory_utilization: 0.6
        tensor_parallel_size: 2
        max_num_batched_tokens: 8192
        compilation_config:
          cudagraph_mode: PIECEWISE   # Qwen3-Omni not FULL-cudagraph-safe on NPU
        devices: "0,1"
      - stage_id: 1
        gpu_memory_utilization: 0.6
        max_num_batched_tokens: 8192
        compilation_config:
          cudagraph_mode: PIECEWISE   # code_predictor MaskedSelect / inner NPU graph replay
        devices: "2"
      - stage_id: 2
        gpu_memory_utilization: 0.3
        compilation_config:
          cudagraph_mode: PIECEWISE
        devices: "2"

  # Code2Wav decode uses UpSample1d, which calls F.conv_transpose1d.
  # On ROCm this routes through MIOpen and is not capture-safe during
  # vLLM's outer graph capture, so stage 2 runs eager to avoid MIOpen/HIP
  # graph-capture failures.
  rocm:
    stages:
      - stage_id: 2
        enforce_eager: true

  xpu:
    stages:
      - stage_id: 0
        tensor_parallel_size: 4
        enforce_eager: true
        max_cudagraph_capture_size: 0
        devices: "0,1,2,3"
      - stage_id: 1
        enforce_eager: true
        max_cudagraph_capture_size: 0
        devices: "4"
      - stage_id: 2
        gpu_memory_utilization: 0.3
        enforce_eager: true
        max_cudagraph_capture_size: 0
        devices: "4"