Sy03andYueqian Lin
58007c0084
Signed-off-by: Sysy <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
98 lines
3.0 KiB
YAML
98 lines
3.0 KiB
YAML
# Qwen2.5-Omni deploy: CUDA defaults + platform overrides, verified on 2x H100.
|
|
# Stage 2 disables flashinfer autotune because its DiT block never invokes
|
|
# flashinfer; the autotune dummy run OOMs the shared cuda:0 device otherwise.
|
|
#
|
|
# Fields omitted from a stage fall back to StageDeployConfig dataclass
|
|
# defaults (see vllm_omni/config/stage_config.py). For instance, every
|
|
# stage here uses vLLM's default max_num_batched_tokens=32768 because
|
|
# chat-sized prefill comfortably fits; only models with codec prefill
|
|
# (Qwen3-Omni, Qwen3-TTS) need to bump it above 32k.
|
|
#
|
|
# enforce_eager policy across the three deploy YAMLs:
|
|
# * code2wav / generation stages: always true (cudagraph incompatible with
|
|
# the custom generation loop — set explicitly everywhere).
|
|
# * AR stages (thinker, talker): model-dependent. Qwen2.5-Omni runs eager
|
|
# on CUDA (thinker uses custom ops that don't trace cleanly); NPU / XPU
|
|
# overrides are documented below. Qwen3-Omni / Qwen3-TTS AR stages use the
|
|
# default (false = cudagraph on).
|
|
async_chunk: false
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.8
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
mm_processor_cache_gb: 0
|
|
devices: "0"
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.1
|
|
|
|
- stage_id: 1
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.8
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
devices: "1"
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_p: 0.8
|
|
top_k: 40
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 2
|
|
max_num_batched_tokens: 32768
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.15
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
enable_prefix_caching: false
|
|
enable_flashinfer_autotune: false
|
|
async_scheduling: false
|
|
devices: "0"
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.1
|
|
|
|
platforms:
|
|
npu:
|
|
stages:
|
|
# NPU has cudagraph support for the thinker, unlike GPU which still
|
|
# only runs eager.
|
|
- stage_id: 0
|
|
enforce_eager: false
|
|
- stage_id: 2
|
|
# 3-NPU layout: stage 2 lives on its own card.
|
|
devices: "2"
|
|
|
|
xpu:
|
|
stages:
|
|
# PyTorch 2.13 XPU Dynamo raises duplicate-handler assertions while
|
|
# compiling both Qwen2.5 AR stages. Keep them eager until that toolchain
|
|
# issue is fixed; stage 2 already inherits enforce_eager: true above.
|
|
- stage_id: 0
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: true
|
|
- stage_id: 1
|
|
gpu_memory_utilization: 0.5
|
|
enforce_eager: true
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
# Stage 2 colocates with stage 1's device on XPU.
|
|
devices: "1"
|