From c852267a1acc476388282fc26801fd69879fd718 Mon Sep 17 00:00:00 2001 From: Roger Wang Date: Sun, 30 Nov 2025 20:01:20 -0800 Subject: [PATCH] [Misc] Update default stage config for qwen3-omni (#124) Signed-off-by: Roger Wang --- docs/configuration/stage_configs.md | 3 + .../examples/offline_inference/qwen3_omni.md | 4 - .../qwen3_omni/qwen3_omni_moe_tp.yaml | 94 ------------------- .../qwen3_omni/run_single_prompt_tp.sh | 3 +- vllm_omni/entrypoints/async_omni.py | 2 +- .../stage_configs/qwen2_5_omni.yaml | 2 + .../stage_configs/qwen3_omni_moe.yaml | 11 ++- 7 files changed, 13 insertions(+), 106 deletions(-) delete mode 100644 examples/offline_inference/qwen3_omni/qwen3_omni_moe_tp.yaml diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index d582df801..306e6b1a8 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -2,6 +2,9 @@ In vLLM-Omni, the target model is separated into multiple stages, which are processed by different LLMEngines, DiffusionEngines or other types of engines. Depending on different types of stages, such as Autoregressive (AR) stage or Diffusion transformer (DiT) stage, each can choose corresponding schedulers, model workers to load with the Engines in a plug-in fashion. +!!! note + Default stage config YAMLs (for example, `vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml` and `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`) are bundled and loaded automatically when `stage_configs_path` is not provided. They have been verified to work on 1xH100 for Qwen2.5-Omni and 2xH100 for Qwen3-Omni. + Therefore, as a core part of vLLM-Omni, the stage configs for a model have several main functions: - Claim partition of stages and their corresponding class implementation in `model_executor/models`. diff --git a/docs/user_guide/examples/offline_inference/qwen3_omni.md b/docs/user_guide/examples/offline_inference/qwen3_omni.md index 58ea77b2b..9a53fcf89 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_omni.md +++ b/docs/user_guide/examples/offline_inference/qwen3_omni.md @@ -46,10 +46,6 @@ sudo apt install ffmpeg ``````py --8<-- "/mnt/vllm_open_release/vllm-omni-cursor/vllm-omni/examples/offline_inference/qwen3_omni/end2end.py" `````` -??? abstract "qwen3_omni_moe_tp.yaml" - ``````yaml - --8<-- "/mnt/vllm_open_release/vllm-omni-cursor/vllm-omni/examples/offline_inference/qwen3_omni/qwen3_omni_moe_tp.yaml" - `````` ??? abstract "run_multiple_prompts.sh" ``````sh --8<-- "/mnt/vllm_open_release/vllm-omni-cursor/vllm-omni/examples/offline_inference/qwen3_omni/run_multiple_prompts.sh" diff --git a/examples/offline_inference/qwen3_omni/qwen3_omni_moe_tp.yaml b/examples/offline_inference/qwen3_omni/qwen3_omni_moe_tp.yaml deleted file mode 100644 index 8d9c2900e..000000000 --- a/examples/offline_inference/qwen3_omni/qwen3_omni_moe_tp.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Stage config for running Qwen3-Omni-MoE with 3-stage architecture -# Stage 0: Thinker (multimodal understanding + text generation) -# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes) -# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform) - -stage_args: - - stage_id: 0 - runtime: - devices: "0,1" - max_batch_size: 1 - engine_args: - model_stage: thinker - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler - gpu_memory_utilization: 0.8 - enforce_eager: true - trust_remote_code: true - engine_output_type: latent # Output hidden states for talker - distributed_executor_backend: "mp" - enable_prefix_caching: false - hf_config_name: thinker_config - tensor_parallel_size: 2 - final_output: true - final_output_type: text - is_comprehension: true - default_sampling_params: - temperature: 0.4 - top_p: 0.9 - top_k: 1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 - - - stage_id: 1 - runtime: - devices: "1" - max_batch_size: 1 - engine_args: - model_stage: talker - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler - gpu_memory_utilization: 0.5 - enforce_eager: true - trust_remote_code: true - engine_output_type: latent # Output codec codes for code2wav - # tensor_parallel_size: 2 - enable_prefix_caching: false - distributed_executor_backend: "mp" - hf_config_name: talker_config - engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker - # final_output: true - # final_output_type: text - default_sampling_params: - temperature: 0.9 - top_k: 50 - max_tokens: 4096 - seed: 42 - detokenize: False - repetition_penalty: 1.05 - stop_token_ids: [2150] - - - stage_id: 2 - runtime: - devices: "1" - max_batch_size: 1 - engine_args: - model_stage: code2wav - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.generation_scheduler.GenerationScheduler - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: audio # Final output: audio waveform - gpu_memory_utilization: 0.1 - distributed_executor_backend: "mp" - max_num_batched_tokens: 1000000 - hf_config_name: thinker_config - engine_input_source: [1] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 65536 - seed: 42 - detokenize: True - repetition_penalty: 1.1 diff --git a/examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh b/examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh index eb25b6bc0..f49ec4814 100644 --- a/examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh +++ b/examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh @@ -1,6 +1,5 @@ python end2end.py --output-wav output_audio \ --query-type use_audio \ - --init-sleep-seconds 90 \ - --stage-configs-path qwen3_omni_moe_tp.yaml + --init-sleep-seconds 90 # init-sleep-seconds works to avoid two vLLM stages initialized at the same time within a card. diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 040c18876..50a6ff748 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -90,7 +90,7 @@ class AsyncOmni(EngineClient): stage_configs_path: Optional[str] = None, log_stats: bool = False, log_file: Optional[str] = None, - init_sleep_seconds: int = 20, + init_sleep_seconds: int = 30, shm_threshold_bytes: int = 65536, batch_timeout: int = 10, init_timeout: int = 60000, diff --git a/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml b/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml index c378a5b48..8441db598 100644 --- a/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml @@ -1,4 +1,6 @@ # stage config for running qwen2.5-omni with architecture of OmniLLM. + +# The following config has been verified on 1x H100-80G GPU. stage_args: - stage_id: 0 runtime: diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml index 8584b4c6d..0c9fe7ccb 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml @@ -3,24 +3,25 @@ # Stage 1: Talker (text embeddings → 8-layer RVQ codec codes) # Stage 2: Code2Wav (8-layer RVQ codes → audio waveform) +# The following config has been verified on 2x H100-80G GPUs. stage_args: - stage_id: 0 runtime: - devices: "0" + devices: "0,1" max_batch_size: 1 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler - gpu_memory_utilization: 0.8 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false hf_config_name: thinker_config - # tensor_parallel_size: 2 # Commented out - using single GPU for thinker + tensor_parallel_size: 2 final_output: true final_output_type: text is_comprehension: true @@ -42,7 +43,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler - gpu_memory_utilization: 0.5 + gpu_memory_utilization: 0.3 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -65,7 +66,7 @@ stage_args: - stage_id: 2 runtime: - devices: "1" + devices: "0" max_batch_size: 1 engine_args: model_stage: code2wav