From 3ea7fd280b0e5b6f7b7f05262eb1cf30445af07b Mon Sep 17 00:00:00 2001
From: Samit <285365963@qq.com>
Date: Sun, 30 Nov 2025 20:44:04 +0800
Subject: [PATCH] [Feature] Support Gradio Demo for Qwen3-Omni (#116)

Signed-off-by: samithuang <285365963@qq.com>
---
 examples/online_serving/qwen3_omni/README.md  |  4 ++--
 .../online_serving/qwen3_omni/gradio_demo.py  | 24 ++++++++++---------
 .../run_curl_multimodal_generation.sh         |  2 +-
 3 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md
index 6e4b27ea3..757db1e3a 100644
--- a/examples/online_serving/qwen3_omni/README.md
+++ b/examples/online_serving/qwen3_omni/README.md
@@ -23,12 +23,12 @@ cd examples/online_serving
 
 Send request via python
 ```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
+python openai_chat_completion_client_for_multimodal_generation.py --query-type use_video
 ```
 
 Send request via curl
 ```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
+bash run_curl_multimodal_generation.sh use_audio_in_video
 ```
 
 ### FAQ
diff --git a/examples/online_serving/qwen3_omni/gradio_demo.py b/examples/online_serving/qwen3_omni/gradio_demo.py
index d6a2237e9..b6d802f85 100644
--- a/examples/online_serving/qwen3_omni/gradio_demo.py
+++ b/examples/online_serving/qwen3_omni/gradio_demo.py
@@ -42,19 +42,20 @@ SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
                 "seed": SEED,
             },
             "talker": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
+                "temperature": 0.9,
+                "top_k": 50,
                 "max_tokens": 4096,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
+                "seed": SEED,
+                "detokenize": False,
+                "repetition_penalty": 1.05,
                 "stop_token_ids": [2150],
             },
             "code2wav": {
                 "temperature": 0.0,
                 "top_p": 1.0,
                 "top_k": -1,
-                "max_tokens": 2048,
+                "max_tokens": 4096 * 16,
+                "seed": SEED,
                 "detokenize": True,
                 "repetition_penalty": 1.1,
             },
@@ -310,11 +311,11 @@ async def run_inference_async_omni_llm(
             )
             prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
             if audio_data:
-                prompt += "<|audio_bos|><|AUDIO|><|audio_eos|>"
+                prompt += "<|audio_start|><|audio_pad|><|audio_end|>"
             if image_data:
-                prompt += "<|vision_bos|><|IMAGE|><|vision_eos|>"
+                prompt += "<|vision_start|><|image_pad|><|vision_end|>"
             if video_payload is not None:
-                prompt += "<|vision_bos|><|VIDEO|><|vision_eos|>"
+                prompt += "<|vision_start|><|video_pad|><|vision_end|>"
             if user_prompt.strip():
                 prompt += f"{user_prompt}"
             prompt += "<|im_end|>\n<|im_start|>assistant\n"
@@ -344,14 +345,15 @@ async def run_inference_async_omni_llm(
                             text_outputs.append(output.text)
             elif stage_outputs.final_output_type == "audio":
                 # multimodal_output is on the RequestOutput object
-                # See vllm_omni/entrypoints/openai/serving_chat.py:680 for reference
+                # See vllm_omni/entrypoints/openai/serving_chat.py:733 for reference
                 if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
                     if "audio" in request_output.multimodal_output:
                         audio_tensor = request_output.multimodal_output["audio"]
                         # Ensure audio is 1D (flatten if needed)
                         if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
                             audio_tensor = audio_tensor.flatten()
-                        audio_np = audio_tensor.detach().cpu().numpy()
+                        # Convert BFloat16 to Float32 before numpy conversion (numpy doesn't support BFloat16)
+                        audio_np = audio_tensor.float().detach().cpu().numpy()
                         audio_output = (
                             24000,  # sampling rate in Hz
                             audio_np,
diff --git a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh
index 95c85370c..092db0b42 100644
--- a/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh
+++ b/examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh
@@ -2,7 +2,7 @@
 set -euo pipefail
 
 # Default query type
-QUERY_TYPE="${1:-mixed_modalities}"
+QUERY_TYPE="${1:-use_audio_in_video}"
 
 # Validate query type
 if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then