[Refactor] Migrate Qwen3 Omni thinker-only deploy config (#5285)
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
1 parent
089bb88706
commit
1752e8ee5b
6 files changed
+68
-44
No files matched your search
@@ -306,7 +306,7 @@ The gradio script supports the following arguments:
|
||||
``````
|
||||
??? abstract "qwen3_omni_moe_thinking.yaml"
|
||||
``````yaml
|
||||
--8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
|
||||
--8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml"
|
||||
``````
|
||||
??? abstract "run_curl_multimodal_generation.sh"
|
||||
``````sh
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
|
||||
# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
|
||||
# thinker component and do not support audio output.
|
||||
#
|
||||
# Single stage: Thinker (multimodal understanding + text generation)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: text
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: true
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
@@ -13,7 +13,7 @@ set -e
|
||||
MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
SERVER_PORT=8091
|
||||
GRADIO_PORT=7861
|
||||
STAGE_CONFIGS_PATH=""
|
||||
DEPLOY_CONFIG=""
|
||||
SERVER_HOST="0.0.0.0"
|
||||
GRADIO_IP="127.0.0.1"
|
||||
GRADIO_SHARE=false
|
||||
@@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do
|
||||
GRADIO_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--stage-configs-path)
|
||||
STAGE_CONFIGS_PATH="$2"
|
||||
--deploy-config)
|
||||
DEPLOY_CONFIG="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-host)
|
||||
@@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do
|
||||
echo " --model MODEL Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)"
|
||||
echo " --server-port PORT Port for vLLM server (default: 8091)"
|
||||
echo " --gradio-port PORT Port for Gradio demo (default: 7861)"
|
||||
echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)"
|
||||
echo " --deploy-config PATH Path to custom deploy config YAML file (optional)"
|
||||
echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)"
|
||||
echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)"
|
||||
echo " --share Share Gradio demo publicly"
|
||||
@@ -87,8 +87,8 @@ echo "=========================================="
|
||||
|
||||
# Build vLLM server command
|
||||
SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
|
||||
if [ -n "$STAGE_CONFIGS_PATH" ]; then
|
||||
SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
|
||||
if [ -n "$DEPLOY_CONFIG" ]; then
|
||||
SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG")
|
||||
fi
|
||||
|
||||
# Function to cleanup on exit
|
||||
|
||||
@@ -1233,6 +1233,45 @@ stages:
|
||||
assert deploy.connectors is not None
|
||||
assert deploy.platforms is not None
|
||||
|
||||
def test_qwen3_omni_thinker_only_deploy_config_uses_resolved_pipeline(self):
|
||||
deploy_path = Path(get_deploy_config_path("qwen3_omni_moe_thinking.yaml"))
|
||||
with patch(
|
||||
"vllm_omni.config.config_factory.get_config",
|
||||
return_value=Q3_OMNI_THINKER_HF_CONFIG,
|
||||
):
|
||||
pipeline = StageConfigFactory.get_pipeline_config(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Thinking",
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=str(deploy_path),
|
||||
)
|
||||
|
||||
assert pipeline is not None
|
||||
assert pipeline.model_type == "qwen3_omni_moe_thinker_only"
|
||||
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
|
||||
assert len(stages) == 1
|
||||
thinker = stages[0]
|
||||
assert thinker.model_stage == "thinker"
|
||||
assert thinker.final_output is True
|
||||
assert thinker.final_output_type == "text"
|
||||
assert thinker.worker_type == "ar"
|
||||
assert thinker.scheduler_cls == "vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler"
|
||||
assert thinker.hf_config_name == "thinker_config"
|
||||
assert thinker.is_comprehension is True
|
||||
assert thinker.yaml_runtime["devices"] == "0,1"
|
||||
assert thinker.yaml_engine_args["engine_output_type"] == "text"
|
||||
assert thinker.yaml_engine_args["tensor_parallel_size"] == 2
|
||||
assert thinker.yaml_engine_args["max_num_seqs"] == 1
|
||||
assert thinker.yaml_engine_args["gpu_memory_utilization"] == 0.9
|
||||
assert thinker.yaml_engine_args["enforce_eager"] is True
|
||||
assert thinker.yaml_engine_args["async_scheduling"] is False
|
||||
assert thinker.yaml_engine_args["distributed_executor_backend"] == "mp"
|
||||
assert thinker.yaml_engine_args["enable_prefix_caching"] is True
|
||||
assert thinker.yaml_engine_args["async_chunk"] is False
|
||||
assert thinker.yaml_extras["default_sampling_params"]["detokenize"] is True
|
||||
|
||||
def test_load_voxtral_tts_deploy_config_schema_fields(self):
|
||||
deploy_path = Path(get_deploy_config_path("voxtral_tts.yaml"))
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# Deploy config for Qwen3-Omni-MoE thinker-only checkpoints.
|
||||
# Verified on 2x H100-80G GPUs.
|
||||
pipeline: qwen3_omni_moe
|
||||
distributed_executor_backend: mp
|
||||
enable_prefix_caching: true
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0,1"
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: true
|
||||
async_scheduling: false
|
||||
tensor_parallel_size: 2
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
repetition_penalty: 1.05
|
||||
@@ -88,7 +88,7 @@ QWEN3_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
hf_config_name="thinker_config",
|
||||
engine_output_type="latent",
|
||||
engine_output_type="text",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
|
||||
Reference in new issue
Block a user