Ziming Huangandtzhouam
220ce0b4ea
Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com> Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: tzhouam <tzhouam@connect.ust.hk>
106 lines
2.7 KiB
YAML
106 lines
2.7 KiB
YAML
# Qwen3-Omni-MoE production deploy, verified on 2x H100 (stage 0 on cuda:0,
|
|
# stages 1+2 on cuda:1).
|
|
#
|
|
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
|
# vllm_omni/config/stage_config.py). Notable implicit defaults for this
|
|
# model:
|
|
# * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens —
|
|
# chat-sized prefill fits in the 32768 default.
|
|
# * Stages 0/1 do not set enforce_eager — cudagraph runs by default
|
|
# (false). Stage 2 (code2wav) sets true because its generation loop
|
|
# is cudagraph-incompatible.
|
|
# * Platform sections flip enforce_eager per-stage where platform
|
|
# cudagraph support differs.
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 25
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.9
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "0"
|
|
default_sampling_params:
|
|
temperature: 0.4
|
|
top_p: 0.9
|
|
top_k: 1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 1
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.6
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "1"
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_k: 50
|
|
max_tokens: 4096
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 2
|
|
max_num_batched_tokens: 65536
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.1
|
|
enforce_eager: false
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
enable_chunked_prefill: false
|
|
async_scheduling: false
|
|
devices: "1"
|
|
input_connectors:
|
|
from_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
seed: 42
|
|
repetition_penalty: 1.1
|
|
|
|
platforms:
|
|
npu:
|
|
stages:
|
|
- stage_id: 0
|
|
gpu_memory_utilization: 0.6
|
|
tensor_parallel_size: 2
|
|
max_num_batched_tokens: 8192
|
|
devices: "0,1"
|
|
- stage_id: 1
|
|
gpu_memory_utilization: 0.6
|
|
max_num_batched_tokens: 8192
|
|
devices: "2"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
devices: "2"
|
|
|
|
xpu:
|
|
stages:
|
|
- stage_id: 0
|
|
tensor_parallel_size: 4
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "0,1,2,3"
|
|
- stage_id: 1
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|