From bfd3cad5ee9d6ac38cb4cf50bba7bd91f1c136c7 Mon Sep 17 00:00:00 2001 From: Gao Han Date: Sun, 30 Nov 2025 21:30:17 +0800 Subject: [PATCH] [Bugfix] Fix bug of online serving for qwen2.5-omni (#118) Signed-off-by: Gaohan123 --- docs/api/README.md | 2 + .../examples/online_serving/qwen3_omni.md | 4 +- .../qwen2_5_omni/run_single_prompt.sh | 2 +- examples/online_serving/qwen3_omni/README.md | 4 +- ...letion_client_for_multimodal_generation.py | 1 - .../run_curl_multimodal_generation.sh | 86 +++++++------------ vllm_omni/diffusion/worker/npu/__init__.py | 7 ++ vllm_omni/entrypoints/omni_stage.py | 2 + vllm_omni/utils/platform_utils.py | 4 +- 9 files changed, 50 insertions(+), 62 deletions(-) create mode 100644 vllm_omni/diffusion/worker/npu/__init__.py diff --git a/docs/api/README.md b/docs/api/README.md index a8049b011..d9c4face4 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -104,6 +104,8 @@ Worker classes and model runners for distributed inference. - [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][] - [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][] +- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][] +- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][] - [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][] - [vllm_omni.worker.gpu_ar_worker.GPUARWorker][] - [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][] diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index 3d1ba770b..34d82d17f 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -26,12 +26,12 @@ cd examples/online_serving Send request via python ```bash -python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities +python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image ``` Send request via curl ```bash -bash run_curl_multimodal_generation.sh mixed_modalities +bash run_curl_multimodal_generation.sh use_image ``` ### FAQ diff --git a/examples/offline_inference/qwen2_5_omni/run_single_prompt.sh b/examples/offline_inference/qwen2_5_omni/run_single_prompt.sh index 2f2b3ae75..26cbf0172 100644 --- a/examples/offline_inference/qwen2_5_omni/run_single_prompt.sh +++ b/examples/offline_inference/qwen2_5_omni/run_single_prompt.sh @@ -1,2 +1,2 @@ python end2end.py --output-wav output_audio \ - --query-type text + --query-type mixed_modalities diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index 757db1e3a..9ef40e1fe 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -23,12 +23,12 @@ cd examples/online_serving Send request via python ```bash -python openai_chat_completion_client_for_multimodal_generation.py --query-type use_video +python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image ``` Send request via curl ```bash -bash run_curl_multimodal_generation.sh use_audio_in_video +bash run_curl_multimodal_generation.sh use_image ``` ### FAQ diff --git a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py index d4522b2a4..3fb1a71e6 100644 --- a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py +++ b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py @@ -197,7 +197,6 @@ def run_multimodal_generation(args) -> None: count = 0 for choice in chat_completion.choices: - print(f"choice: {choice}") if choice.message.audio: audio_data = base64.b64decode(choice.message.audio.data) audio_file_path = f"audio_{count}.wav" diff --git a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh index 092db0b42..37faee06a 100644 --- a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh +++ b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh @@ -2,16 +2,16 @@ set -euo pipefail # Default query type -QUERY_TYPE="${1:-use_audio_in_video}" +QUERY_TYPE="${1:-use_video}" # Validate query type -if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then +if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then echo "Error: Invalid query type '$QUERY_TYPE'" - echo "Usage: $0 [mixed_modalities|use_audio_in_video|multi_audios|text]" - echo " mixed_modalities: Audio + Image + Video + Text query" - echo " use_audio_in_video: Video + Text query (with audio extraction from video)" - echo " multi_audios: Two audio clips + Text query" + echo "Usage: $0 [text|use_audio|use_image|use_video]" echo " text: Text query" + echo " use_audio: Audio + Text query" + echo " use_image: Image + Text query" + echo " use_video: Video + Text query" exit 1 fi @@ -23,7 +23,6 @@ thinker_sampling_params='{ "top_k": 1, "max_tokens": 16384, "seed": 42, - "detokenize": true, "repetition_penalty": 1.05, "stop_token_ids": [151645] }' @@ -33,7 +32,7 @@ talker_sampling_params='{ "top_k": 50, "max_tokens": 4096, "seed": 42, - "detokenize": true, + "detokenize": false, "repetition_penalty": 1.05, "stop_token_ids": [2150] }' @@ -51,7 +50,6 @@ code2wav_sampling_params='{ # Define URLs for assets MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg" -WINNING_CALL_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/winning_call.ogg" CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg" SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4" @@ -71,7 +69,7 @@ case "$QUERY_TYPE" in ]' mm_processor_kwargs="{}" ;; - mixed_modalities) + use_audio) user_content='[ { "type": "audio_url", @@ -79,21 +77,9 @@ case "$QUERY_TYPE" in "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" } }, - { - "type": "image_url", - "image_url": { - "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" - } - }, - { - "type": "video_url", - "video_url": { - "url": "'"$SAMPLE_VIDEO_URL"'" - } - }, { "type": "text", - "text": "What is recited in the audio? What is the content of this image? Why is this video funny?" + "text": "What is the content of this audio?" } ]' sampling_params_list='[ @@ -103,7 +89,27 @@ case "$QUERY_TYPE" in ]' mm_processor_kwargs="{}" ;; - use_audio_in_video) + use_image) + user_content='[ + { + "type": "image_url", + "image_url": { + "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'" + } + }, + { + "type": "text", + "text": "What is the content of this image?" + } + ]' + sampling_params_list='[ + '"$thinker_sampling_params"', + '"$talker_sampling_params"', + '"$code2wav_sampling_params"' + ]' + mm_processor_kwargs="{}" + ;; + use_video) user_content='[ { "type": "video_url", @@ -113,35 +119,7 @@ case "$QUERY_TYPE" in }, { "type": "text", - "text": "Describe the content of the video, then convert what the baby say into text." - } - ]' - sampling_params_list='[ - '"$thinker_sampling_params"', - '"$talker_sampling_params"', - '"$code2wav_sampling_params"' - ]' - mm_processor_kwargs='{ - "use_audio_in_video": true - }' - ;; - multi_audios) - user_content='[ - { - "type": "audio_url", - "audio_url": { - "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'" - } - }, - { - "type": "audio_url", - "audio_url": { - "url": "'"$WINNING_CALL_AUDIO_URL"'" - } - }, - { - "type": "text", - "text": "Are these two audio clips the same?" + "text": "Why is this video funny?" } ]' sampling_params_list='[ @@ -161,7 +139,7 @@ output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \ -H "Content-Type: application/json" \ -d @- < str: return "CUDA_VISIBLE_DEVICES" # fallback -def get_diffusion_worker_class(): +def get_diffusion_worker_class() -> type: """Get the appropriate diffusion WorkerProc class based on current device type. Returns: - The WorkerProc class for the detected device type. + The WorkerProc class for the detected device type (either NPUWorkerProc or WorkerProc). Raises: ImportError: If the worker module for the detected device is not available.