[Bugfix] Fix bug of online serving for qwen2.5-omni (#118)

Signed-off-by: Gaohan123 <gaohan19@huawei.com>
2025-11-30 21:30:17 +08:00
1 parent 0d110312df
commit bfd3cad5ee
@@ -104,6 +104,8 @@ Worker classes and model runners for distributed inference.

 - [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
 - [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
 - [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
 - [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
 - [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
@@ -26,12 +26,12 @@ cd examples/online_serving

 Send request via python
 ```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
+python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
 ```

 Send request via curl
 ```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
+bash run_curl_multimodal_generation.sh use_image
 ```

 ### FAQ
@@ -1,2 +1,2 @@
 python end2end.py --output-wav output_audio \
-                  --query-type text
+                  --query-type mixed_modalities
@@ -23,12 +23,12 @@ cd examples/online_serving

 Send request via python
 ```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type use_video
+python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
 ```

 Send request via curl
 ```bash
-bash run_curl_multimodal_generation.sh use_audio_in_video
+bash run_curl_multimodal_generation.sh use_image
 ```

 ### FAQ
@@ -197,7 +197,6 @@ def run_multimodal_generation(args) -> None:

    count = 0
    for choice in chat_completion.choices:
-        print(f"choice: {choice}")
        if choice.message.audio:
            audio_data = base64.b64decode(choice.message.audio.data)
            audio_file_path = f"audio_{count}.wav"
@@ -2,16 +2,16 @@
 set -euo pipefail

 # Default query type
-QUERY_TYPE="${1:-use_audio_in_video}"
+QUERY_TYPE="${1:-use_video}"

 # Validate query type
-if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then
+if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then
    echo "Error: Invalid query type '$QUERY_TYPE'"
-    echo "Usage: $0 [mixed_modalities|use_audio_in_video|multi_audios|text]"
-    echo "  mixed_modalities: Audio + Image + Video + Text query"
-    echo "  use_audio_in_video: Video + Text query (with audio extraction from video)"
-    echo "  multi_audios: Two audio clips + Text query"
+    echo "Usage: $0 [text|use_audio|use_image|use_video]"
    echo "  text: Text query"
+    echo "  use_audio: Audio + Text query"
+    echo "  use_image: Image + Text query"
+    echo "  use_video: Video + Text query"
    exit 1
 fi

@@ -23,7 +23,6 @@ thinker_sampling_params='{
  "top_k": 1,
  "max_tokens": 16384,
  "seed": 42,
-  "detokenize": true,
  "repetition_penalty": 1.05,
  "stop_token_ids": [151645]
 }'
@@ -33,7 +32,7 @@ talker_sampling_params='{
  "top_k": 50,
  "max_tokens": 4096,
  "seed": 42,
-  "detokenize": true,
+  "detokenize": false,
  "repetition_penalty": 1.05,
  "stop_token_ids": [2150]
 }'
@@ -51,7 +50,6 @@ code2wav_sampling_params='{

 # Define URLs for assets
 MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
-WINNING_CALL_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/winning_call.ogg"
 CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
 SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"

@@ -71,7 +69,7 @@ case "$QUERY_TYPE" in
    ]'
    mm_processor_kwargs="{}"
    ;;
-  mixed_modalities)
+  use_audio)
    user_content='[
        {
          "type": "audio_url",
@@ -79,21 +77,9 @@ case "$QUERY_TYPE" in
            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
          }
        },
-        {
-          "type": "image_url",
-          "image_url": {
-            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
-          }
-        },
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
        {
          "type": "text",
-          "text": "What is recited in the audio? What is the content of this image? Why is this video funny?"
+          "text": "What is the content of this audio?"
        }
      ]'
    sampling_params_list='[
@@ -103,7 +89,27 @@ case "$QUERY_TYPE" in
    ]'
    mm_processor_kwargs="{}"
    ;;
-  use_audio_in_video)
+  use_image)
+    user_content='[
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "What is the content of this image?"
+        }
+      ]'
+    sampling_params_list='[
+      '"$thinker_sampling_params"',
+      '"$talker_sampling_params"',
+      '"$code2wav_sampling_params"'
+    ]'
+    mm_processor_kwargs="{}"
+    ;;
+  use_video)
    user_content='[
        {
          "type": "video_url",
@@ -113,35 +119,7 @@ case "$QUERY_TYPE" in
        },
        {
          "type": "text",
-          "text": "Describe the content of the video, then convert what the baby say into text."
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs='{
-      "use_audio_in_video": true
-    }'
-    ;;
-  multi_audios)
-    user_content='[
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$WINNING_CALL_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Are these two audio clips the same?"
+          "text": "Why is this video funny?"
        }
      ]'
    sampling_params_list='[
@@ -161,7 +139,7 @@ output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d @- <<EOF
 {
-  "model": "Qwen/Qwen2.5-Omni-7B",
+  "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
  "sampling_params_list": $sampling_params_list,
  "mm_processor_kwargs": $mm_processor_kwargs,
  "messages": [
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""NPU worker classes for diffusion models."""
+
+from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorker, NPUWorkerProc
+
+__all__ = ["NPUWorker", "NPUWorkerProc"]
@@ -295,6 +295,8 @@ class OmniStage:
                raise ValueError("engine_input_source is empty")
            source_stage_id = self.engine_input_source[0]
            source_outputs = stage_list[source_stage_id].engine_outputs
+            if not isinstance(prompt, list):
+                prompt = [prompt]
            multi_modal_data = {
                source_output.request_id: p.get("multi_modal_data", None)
                for source_output, p in zip(source_outputs, prompt)
@@ -32,11 +32,11 @@ def get_device_control_env_var() -> str:
    return "CUDA_VISIBLE_DEVICES"  # fallback


-def get_diffusion_worker_class():
+def get_diffusion_worker_class() -> type:
    """Get the appropriate diffusion WorkerProc class based on current device type.

    Returns:
-        The WorkerProc class for the detected device type.
+        The WorkerProc class for the detected device type (either NPUWorkerProc or WorkerProc).

    Raises:
        ImportError: If the worker module for the detected device is not available.