Ziming Huangandtzhouam
220ce0b4ea
Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com> Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: tzhouam <tzhouam@connect.ust.hk>
109 lines
2.9 KiB
YAML
109 lines
2.9 KiB
YAML
# Qwen3-Omni-MoE multi-replica deploy, verified on 3x H20-96G
|
|
# (stage 0 on cuda:0, stages 1+2 scaled out across cuda:1,2).
|
|
#
|
|
# Fields omitted from a stage fall back to StageDeployConfig defaults (see
|
|
# vllm_omni/config/stage_config.py). Notable implicit defaults for this
|
|
# layout:
|
|
# * Stages 0/1 (thinker, talker) do not set max_num_batched_tokens:
|
|
# chat-sized prefill fits in the 32768 default.
|
|
# * Stage 1 (talker) does not set enforce_eager: cudagraph runs by
|
|
# default (false).
|
|
# * Stage 2 (code2wav) sets enforce_eager=true because its generation
|
|
# loop is cudagraph-incompatible.
|
|
# * Stages 1/2 use num_replicas=2 and share devices "1,2". Replica
|
|
# routing is handled by StagePool with request-level affinity.
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 25
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.9
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "0"
|
|
default_sampling_params:
|
|
temperature: 0.4
|
|
top_p: 0.9
|
|
top_k: 1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 1
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.6
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "1,2"
|
|
num_replicas: 2
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_k: 50
|
|
max_tokens: 4096
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 2
|
|
max_num_batched_tokens: 65536
|
|
max_num_seqs: 64
|
|
gpu_memory_utilization: 0.1
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
enable_chunked_prefill: false
|
|
devices: "1,2"
|
|
num_replicas: 2
|
|
input_connectors:
|
|
from_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
seed: 42
|
|
repetition_penalty: 1.1
|
|
|
|
platforms:
|
|
npu:
|
|
stages:
|
|
- stage_id: 0
|
|
gpu_memory_utilization: 0.6
|
|
tensor_parallel_size: 2
|
|
max_num_batched_tokens: 8192
|
|
devices: "0,1"
|
|
- stage_id: 1
|
|
gpu_memory_utilization: 0.6
|
|
max_num_batched_tokens: 8192
|
|
devices: "2"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
devices: "2"
|
|
|
|
xpu:
|
|
stages:
|
|
- stage_id: 0
|
|
tensor_parallel_size: 4
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "0,1,2,3"
|
|
- stage_id: 1
|
|
enforce_eager: true
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
max_cudagraph_capture_size: 0
|
|
devices: "4"
|