e04210d690
Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
129 lines
3.8 KiB
YAML
129 lines
3.8 KiB
YAML
# Qwen3-Omni-MoE production deploy, verified on 2x H100 (stage 0 on cuda:0,
|
|
# stages 1+2 on cuda:1).
|
|
#
|
|
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
|
# vllm_omni/config/stage_config.py). Notable implicit defaults for this
|
|
# model:
|
|
# * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens —
|
|
# chat-sized prefill fits in the 32768 default.
|
|
# * Stages 0/1 do not set enforce_eager — cudagraph runs by default
|
|
# (false). Stage 2 (code2wav) keeps enforce_eager=false so Code2Wav can
|
|
# use its inner cudagraph on CUDA. ROCm flips stage 2 to eager because
|
|
# MIOpen conv_transpose1d is not capture-safe in the Code2Wav graph path.
|
|
# * Platform sections flip enforce_eager per-stage where platform
|
|
# cudagraph support differs.
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
initial_codec_chunk_frames: 4
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 25
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.9
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "0"
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
max_tokens: 2048
|
|
|
|
- stage_id: 1
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.6
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "1"
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_k: 50
|
|
max_tokens: 4096
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 2
|
|
max_num_batched_tokens: 65536
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.1
|
|
enforce_eager: false
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
enable_chunked_prefill: false
|
|
async_scheduling: false
|
|
devices: "1"
|
|
input_connectors:
|
|
from_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
repetition_penalty: 1.1
|
|
|
|
platforms:
|
|
# The supported Qwen3-Omni ModelOpt checkpoint quantizes the thinker only. Clear the
|
|
# checkpoint-level metadata for the BF16 audio stages on the verified MUSA
|
|
# profile before vLLM performs its normal quantization auto-detection.
|
|
musa:
|
|
stages:
|
|
- stage_id: 1
|
|
hf_overrides:
|
|
quantization_config: null
|
|
- stage_id: 2
|
|
hf_overrides:
|
|
quantization_config: null
|
|
|
|
npu:
|
|
stages:
|
|
- stage_id: 0
|
|
gpu_memory_utilization: 0.6
|
|
tensor_parallel_size: 2
|
|
max_num_batched_tokens: 8192
|
|
compilation_config:
|
|
cudagraph_mode: PIECEWISE # Qwen3-Omni not FULL-cudagraph-safe on NPU
|
|
devices: "0,1"
|
|
- stage_id: 1
|
|
gpu_memory_utilization: 0.6
|
|
max_num_batched_tokens: 8192
|
|
compilation_config:
|
|
cudagraph_mode: PIECEWISE # code_predictor MaskedSelect / inner NPU graph replay
|
|
devices: "2"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
compilation_config:
|
|
cudagraph_mode: PIECEWISE
|
|
devices: "2"
|
|
|
|
# Code2Wav decode uses UpSample1d, which calls F.conv_transpose1d.
|
|
# On ROCm this routes through MIOpen and is not capture-safe during
|
|
# vLLM's outer graph capture, so stage 2 runs eager to avoid MIOpen/HIP
|
|
# graph-capture failures.
|
|
rocm:
|
|
stages:
|
|
- stage_id: 2
|
|
enforce_eager: true
|
|
|
|
xpu:
|
|
stages:
|
|
- stage_id: 0
|
|
tensor_parallel_size: 4
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "0,1,2,3"
|
|
- stage_id: 1
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|