Files
vllm-omni/vllm_omni/deploy/hunyuan_image3.yaml
T
2026-05-14 11:17:15 +08:00

120 lines
3.0 KiB
YAML

# HunyuanImage-3.0-Instruct deploy: AR (stage 0) + DiT (stage 1)
# with AR-to-DiT KV reuse. The base CUDA layout uses 2 GPUs for AR
# and 2 GPUs for DiT.
pipeline: hunyuan_image_3_moe
async_chunk: false
trust_remote_code: true
connectors:
rdma_connector:
name: MooncakeTransferEngineConnector
extra:
host: "auto"
zmq_port: 50051
protocol: "rdma"
device_name: ""
memory_pool_size: 4294967296
memory_pool_device: "cpu"
shared_memory_connector:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
stages:
- stage_id: 0
final_output: true
final_output_type: text
max_num_seqs: 1
gpu_memory_utilization: 0.95
enforce_eager: true
max_num_batched_tokens: 32768
devices: "0,1"
tensor_parallel_size: 2
hf_overrides:
rope_parameters:
mrope_section: [0, 32, 32]
rope_type: default
omni_kv_config:
need_send_cache: true
output_connectors:
to_stage_1: rdma_connector
default_sampling_params:
temperature: 0.0
top_p: 1
top_k: -1
max_tokens: 8192
detokenize: true
skip_special_tokens: false
- stage_id: 1
max_num_seqs: 1
enforce_eager: true
devices: "2,3"
distributed_executor_backend: "mp"
omni_kv_config:
need_recv_cache: true
parallel_config:
tensor_parallel_size: 2
enable_expert_parallel: true
input_connectors:
from_stage_0: rdma_connector
default_sampling_params:
num_inference_steps: 50
guidance_scale: 0
edges:
- from: 0
to: 1
window_size: -1
max_inflight: 1
platforms:
npu:
stages:
- stage_id: 0
gpu_memory_utilization: 0.8
devices: "0,1,2,3"
tensor_parallel_size: 4
max_num_batched_tokens: 8192
- stage_id: 1
gpu_memory_utilization: 0.65
devices: "4,5,6,7"
max_num_batched_tokens: 8192
parallel_config:
tensor_parallel_size: 4
enable_expert_parallel: false
sequence_parallel_size: 1
ulysses_degree: 1
xpu:
stages:
- stage_id: 0
gpu_memory_utilization: 0.95
devices: "0,1,2,3,4,5,6,7"
tensor_parallel_size: 8
max_num_batched_tokens: 32784
quantization: fp8
enable_expert_parallel: true
worker_cls: vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker
default_sampling_params:
max_tokens: 2048
seed: 42
repetition_penalty: 1.1
- stage_id: 1
gpu_memory_utilization: 0.9
devices: "0,1,2,3,4,5,6,7"
quantization: fp8
parallel_config:
pipeline_parallel_size: 1
data_parallel_size: 1
tensor_parallel_size: 8
enable_expert_parallel: true
sequence_parallel_size: 1
ulysses_degree: 1
ring_degree: 1
cfg_parallel_size: 1
vae_patch_parallel_size: 1
use_hsdp: false
hsdp_shard_size: -1
hsdp_replicate_size: 1