Files
vllm-omni/vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml
T
1e5f288a91 [FEAT] support multi-stage deployment (#2396)
Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
2026-05-06 19:33:18 +08:00

102 lines
2.6 KiB
YAML

# Qwen3-Omni-MoE multi-replica deploy, verified on 3x H20-96G
# (stage 0 on cuda:0, stages 1+2 scaled out across cuda:1,2).
#
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
# vllm_omni/config/stage_config.py). Notable implicit defaults for this
# layout:
# * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens:
# chat-sized prefill fits in the 32768 default.
# * Stage 1 (talker) does not set enforce_eager: cudagraph runs by
# default (false).
# * Stage 2 (code2wav) sets enforce_eager=true because its generation
# loop is cudagraph-incompatible.
# * Stages 1/2 use num_replicas=2 and share devices "1,2". Replica
# routing is handled by StagePool with request-level affinity.
async_chunk: true
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
codec_chunk_frames: 25
codec_left_context_frames: 25
stages:
- stage_id: 0
gpu_memory_utilization: 0.9
devices: "0"
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 2048
seed: 42
repetition_penalty: 1.05
- stage_id: 1
gpu_memory_utilization: 0.6
devices: "1,2"
num_replicas: 2
input_connectors:
from_stage_0: connector_of_shared_memory
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
repetition_penalty: 1.05
- stage_id: 2
gpu_memory_utilization: 0.1
max_num_seqs: 1
enforce_eager: true
async_scheduling: false
max_num_batched_tokens: 51200
devices: "1,2"
num_replicas: 2
input_connectors:
from_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
repetition_penalty: 1.1
platforms:
npu:
stages:
- stage_id: 0
gpu_memory_utilization: 0.6
tensor_parallel_size: 2
devices: "0,1"
- stage_id: 1
gpu_memory_utilization: 0.6
enforce_eager: true
devices: "2"
- stage_id: 2
gpu_memory_utilization: 0.3
devices: "2"
rocm:
stages:
- stage_id: 0
enforce_eager: true
xpu:
stages:
- stage_id: 0
tensor_parallel_size: 4
enforce_eager: true
max_cudagraph_capture_size: 0
devices: "0,1,2,3"
- stage_id: 1
enforce_eager: true
max_cudagraph_capture_size: 0
devices: "4"
- stage_id: 2
gpu_memory_utilization: 0.3
max_cudagraph_capture_size: 0
devices: "4"