Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk>
37 lines
1.1 KiB
YAML
37 lines
1.1 KiB
YAML
# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
|
|
# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
|
|
# thinker component and do not support audio output.
|
|
#
|
|
# Single stage: Thinker (multimodal understanding + text generation)
|
|
|
|
# The following config has been verified on 2x H100-80G GPUs.
|
|
stage_args:
|
|
- stage_id: 0
|
|
runtime:
|
|
devices: "0,1"
|
|
engine_args:
|
|
model_stage: thinker
|
|
max_num_seqs: 1
|
|
model_arch: Qwen3OmniMoeForConditionalGeneration
|
|
worker_type: ar
|
|
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: true
|
|
trust_remote_code: true
|
|
engine_output_type: text
|
|
distributed_executor_backend: "mp"
|
|
enable_prefix_caching: false
|
|
hf_config_name: thinker_config
|
|
tensor_parallel_size: 2
|
|
final_output: true
|
|
final_output_type: text
|
|
is_comprehension: true
|
|
default_sampling_params:
|
|
temperature: 0.4
|
|
top_p: 0.9
|
|
top_k: 1
|
|
max_tokens: 2048
|
|
seed: 42
|
|
detokenize: True
|
|
repetition_penalty: 1.05
|