diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index e372692a8..b4c05bdbe 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -306,7 +306,7 @@ The gradio script supports the following arguments: `````` ??? abstract "qwen3_omni_moe_thinking.yaml" ``````yaml - --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml" + --8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml" `````` ??? abstract "run_curl_multimodal_generation.sh" ``````sh diff --git a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml b/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml deleted file mode 100644 index 7ce88f80a..000000000 --- a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output) -# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the -# thinker component and do not support audio output. -# -# Single stage: Thinker (multimodal understanding + text generation) - -# The following config has been verified on 2x H100-80G GPUs. -stage_args: - - stage_id: 0 - runtime: - devices: "0,1" - engine_args: - model_stage: thinker - max_num_seqs: 1 - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.9 - enforce_eager: true - trust_remote_code: true - engine_output_type: text - distributed_executor_backend: "mp" - enable_prefix_caching: true - hf_config_name: thinker_config - tensor_parallel_size: 2 - final_output: true - final_output_type: text - is_comprehension: true - default_sampling_params: - temperature: 0.4 - top_p: 0.9 - top_k: 1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 diff --git a/examples/online_serving/qwen3_omni/run_gradio_demo.sh b/examples/online_serving/qwen3_omni/run_gradio_demo.sh index 73ce273d9..465304534 100755 --- a/examples/online_serving/qwen3_omni/run_gradio_demo.sh +++ b/examples/online_serving/qwen3_omni/run_gradio_demo.sh @@ -13,7 +13,7 @@ set -e MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct" SERVER_PORT=8091 GRADIO_PORT=7861 -STAGE_CONFIGS_PATH="" +DEPLOY_CONFIG="" SERVER_HOST="0.0.0.0" GRADIO_IP="127.0.0.1" GRADIO_SHARE=false @@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do GRADIO_PORT="$2" shift 2 ;; - --stage-configs-path) - STAGE_CONFIGS_PATH="$2" + --deploy-config) + DEPLOY_CONFIG="$2" shift 2 ;; --server-host) @@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do echo " --model MODEL Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)" echo " --server-port PORT Port for vLLM server (default: 8091)" echo " --gradio-port PORT Port for Gradio demo (default: 7861)" - echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)" + echo " --deploy-config PATH Path to custom deploy config YAML file (optional)" echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)" echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)" echo " --share Share Gradio demo publicly" @@ -87,8 +87,8 @@ echo "==========================================" # Build vLLM server command SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST") -if [ -n "$STAGE_CONFIGS_PATH" ]; then - SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH") +if [ -n "$DEPLOY_CONFIG" ]; then + SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG") fi # Function to cleanup on exit diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 3e482f48f..a811868c1 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1233,6 +1233,45 @@ stages: assert deploy.connectors is not None assert deploy.platforms is not None + def test_qwen3_omni_thinker_only_deploy_config_uses_resolved_pipeline(self): + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe_thinking.yaml")) + with patch( + "vllm_omni.config.config_factory.get_config", + return_value=Q3_OMNI_THINKER_HF_CONFIG, + ): + pipeline = StageConfigFactory.get_pipeline_config( + "Qwen/Qwen3-Omni-30B-A3B-Thinking", + trust_remote_code=False, + deploy_config_path=str(deploy_path), + ) + + assert pipeline is not None + assert pipeline.model_type == "qwen3_omni_moe_thinker_only" + + deploy = load_deploy_config(deploy_path) + stages = merge_pipeline_deploy(pipeline, deploy) + + assert len(stages) == 1 + thinker = stages[0] + assert thinker.model_stage == "thinker" + assert thinker.final_output is True + assert thinker.final_output_type == "text" + assert thinker.worker_type == "ar" + assert thinker.scheduler_cls == "vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler" + assert thinker.hf_config_name == "thinker_config" + assert thinker.is_comprehension is True + assert thinker.yaml_runtime["devices"] == "0,1" + assert thinker.yaml_engine_args["engine_output_type"] == "text" + assert thinker.yaml_engine_args["tensor_parallel_size"] == 2 + assert thinker.yaml_engine_args["max_num_seqs"] == 1 + assert thinker.yaml_engine_args["gpu_memory_utilization"] == 0.9 + assert thinker.yaml_engine_args["enforce_eager"] is True + assert thinker.yaml_engine_args["async_scheduling"] is False + assert thinker.yaml_engine_args["distributed_executor_backend"] == "mp" + assert thinker.yaml_engine_args["enable_prefix_caching"] is True + assert thinker.yaml_engine_args["async_chunk"] is False + assert thinker.yaml_extras["default_sampling_params"]["detokenize"] is True + def test_load_voxtral_tts_deploy_config_schema_fields(self): deploy_path = Path(get_deploy_config_path("voxtral_tts.yaml")) deploy = load_deploy_config(deploy_path) diff --git a/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml b/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml new file mode 100644 index 000000000..7ccc64750 --- /dev/null +++ b/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml @@ -0,0 +1,21 @@ +# Deploy config for Qwen3-Omni-MoE thinker-only checkpoints. +# Verified on 2x H100-80G GPUs. +pipeline: qwen3_omni_moe +distributed_executor_backend: mp +enable_prefix_caching: true + +stages: + - stage_id: 0 + devices: "0,1" + max_num_seqs: 1 + gpu_memory_utilization: 0.9 + enforce_eager: true + async_scheduling: false + tensor_parallel_size: 2 + default_sampling_params: + temperature: 0.4 + top_p: 0.9 + top_k: 1 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.05 diff --git a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py index 9baecdd3e..c93e764ef 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py +++ b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py @@ -88,7 +88,7 @@ QWEN3_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig( owns_tokenizer=True, requires_multimodal_data=True, hf_config_name="thinker_config", - engine_output_type="latent", + engine_output_type="text", sampling_constraints={"detokenize": True}, ), ),