e1b0b83238
Signed-off-by: inkcherry <mingzhi.liu@amd.com> Signed-off-by: zejwang <zejwang@amd.com> Signed-off-by: junkang1991 <junkangchow@gmail.com> Signed-off-by: Zejian Wang <zejianwang@sjtu.edu.cn> Co-authored-by: zejwang <zejwang@amd.com> Co-authored-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Zejian Wang <zejianwang@sjtu.edu.cn> Co-authored-by: Zejian Wang <Zejian.Wang@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
108 lines
3.9 KiB
YAML
108 lines
3.9 KiB
YAML
# Qwen3-Omni-MoE intra-node deploy using Mori XGMI transfer.
|
|
#
|
|
# Targets a single AMD Instinct MI300X OAM node (8x GPU, 192GB HBM3,
|
|
# gfx942). Each stage occupies one dedicated GPU (3 GPUs total) and
|
|
# stage-to-stage streaming chunks are shipped via Mori's RDMA IOEngine
|
|
# (with ``backend_type: xgmi`` selecting the AMD Infinity Fabric
|
|
# GPU-to-GPU direct path for intranode transfers).
|
|
#
|
|
# Pipeline wiring (stage_id, execution_type, model_stage, input_sources,
|
|
# custom processors, async-chunk builders) lives in
|
|
# ``vllm_omni/model_executor/models/qwen3_omni/pipeline.py`` and is
|
|
# merged into the stages below by the deploy loader.
|
|
#
|
|
# Why this yaml exists:
|
|
# * On CUDA/H100 Qwen3-Omni-MoE ships via ``SharedMemoryConnector``
|
|
# (see ``qwen3_omni_moe.yaml``). On MI300X the SHM path still works
|
|
# but XGMI RDMA can substantially reduce stage-transition latency
|
|
# for the thinker->talker hidden-state stream and the talker->code2wav
|
|
# codec-frame stream, both of which run in async-chunk mode.
|
|
# * This config swaps the connector definition to
|
|
# ``MoriTransferEngineConnector`` while keeping the pipeline topology,
|
|
# sampling params, and async_chunk=true gating identical to the CUDA
|
|
# deploy, so the chunk_transfer_adapter path (scheduler-owned
|
|
# background put/get threads) is fully exercised.
|
|
#
|
|
# Usage:
|
|
# vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --log-stats \
|
|
# --deploy-config vllm_omni/deploy/qwen3_omni_moe_mori_intranode.yaml
|
|
#
|
|
# Dependencies: mori (``pip install mori``), torch-rocm>=2.9, vllm-omni
|
|
# with #2383 deploy schema, ibverbs/RDMA NICs exposed via /dev/infiniband.
|
|
#
|
|
# Limitation: TP=1-only. Connector raises NotImplementedError on
|
|
# tp_world_size > 1 (chunk-path endpoint derivation isn't rank-aware yet).
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
mori_connector:
|
|
name: MoriTransferEngineConnector
|
|
extra:
|
|
host: "auto"
|
|
zmq_port: 50051
|
|
backend_type: "xgmi" # AMD Infinity Fabric GPU-to-GPU direct
|
|
device_name: "" # leave empty to honour $MORI_RDMA_DEVICES
|
|
memory_pool_size: 536870912 # 512 MB, matches Qwen2.5-Omni Mori deploy
|
|
memory_pool_device: "cuda" # GPU memory required for XGMI RDMA transfers
|
|
xgmi_num_streams: 64
|
|
xgmi_num_events: 64
|
|
qp_per_transfer: 1
|
|
num_worker_threads: 1
|
|
post_batch_size: -1
|
|
# Chunk-transfer knobs consumed by talker2code2wav_async_chunk:
|
|
# 25 decoded codec frames per connector.put (~40 ms at 625 Hz
|
|
# codec rate) with a 25-frame left context to satisfy the
|
|
# code2wav receptive field. Kept identical to the SHM deploy so
|
|
# accuracy stays comparable when swapping backends.
|
|
codec_chunk_frames: 25
|
|
codec_left_context_frames: 25
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: true # ROCm override: avoid flashinfer autotune path
|
|
devices: "0"
|
|
output_connectors:
|
|
to_stage_1: mori_connector
|
|
default_sampling_params:
|
|
temperature: 0.4
|
|
top_p: 0.9
|
|
top_k: 1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 1
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: true
|
|
devices: "1"
|
|
input_connectors:
|
|
from_stage_0: mori_connector
|
|
output_connectors:
|
|
to_stage_2: mori_connector
|
|
default_sampling_params:
|
|
temperature: 0.9
|
|
top_k: 50
|
|
max_tokens: 4096
|
|
seed: 42
|
|
repetition_penalty: 1.05
|
|
|
|
- stage_id: 2
|
|
gpu_memory_utilization: 0.3
|
|
max_num_seqs: 1
|
|
enforce_eager: true
|
|
async_scheduling: false
|
|
# Codec prefill length (Q * num_frames) exceeds 32k default;
|
|
# matches qwen3_omni_moe.yaml code2wav sizing.
|
|
max_num_batched_tokens: 51200
|
|
devices: "2"
|
|
input_connectors:
|
|
from_stage_1: mori_connector
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
top_k: -1
|
|
max_tokens: 65536
|
|
seed: 42
|
|
repetition_penalty: 1.1
|