From 094de95548b14f5142d31475614de3061c69ac41 Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Fri, 19 Dec 2025 18:09:43 +0800 Subject: [PATCH] [Misc] Update qwen-omni gradio demo to use api server (#378) Signed-off-by: samithuang <285365963@qq.com> --- .../examples/online_serving/qwen2_5_omni.md | 48 +- .../examples/online_serving/qwen3_omni.md | 50 +- .../online_serving/qwen2_5_omni/README.md | 48 +- .../qwen2_5_omni/gradio_demo.py | 454 +++++++---------- .../qwen2_5_omni/run_gradio_demo.sh | 212 ++++++++ examples/online_serving/qwen3_omni/README.md | 49 +- .../online_serving/qwen3_omni/gradio_demo.py | 457 +++++++----------- .../qwen3_omni/run_gradio_demo.sh | 212 ++++++++ 8 files changed, 967 insertions(+), 563 deletions(-) create mode 100755 examples/online_serving/qwen2_5_omni/run_gradio_demo.sh create mode 100755 examples/online_serving/qwen3_omni/run_gradio_demo.sh diff --git a/docs/user_guide/examples/online_serving/qwen2_5_omni.md b/docs/user_guide/examples/online_serving/qwen2_5_omni.md index 9fb7680dc..548d44118 100644 --- a/docs/user_guide/examples/online_serving/qwen2_5_omni.md +++ b/docs/user_guide/examples/online_serving/qwen2_5_omni.md @@ -87,20 +87,60 @@ This Web UI demo allows users to interact with the model through a web browser. ### Running Gradio Demo -Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by +The Gradio demo connects to a vLLM API server. You have two options: + +#### Option 1: One-step Launch Script (Recommended) + +The convenience script launches both the vLLM server and Gradio demo together: ```bash -python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861 +./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861 +``` + +This script will: +1. Start the vLLM server in the background +2. Wait for the server to be ready +3. Launch the Gradio demo +4. Handle cleanup when you press Ctrl+C + +The script supports the following arguments: +- `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B) +- `--server-port`: Port for vLLM server (default: 8091) +- `--gradio-port`: Port for Gradio demo (default: 7861) +- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--server-host`: Host for vLLM server (default: 0.0.0.0) +- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1) +- `--share`: Share Gradio demo publicly (creates a public link) + +#### Option 2: Manual Launch (Two-Step Process) + +**Step 1: Launch the vLLM API server** + +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 +``` + +If you have custom stage configs file: +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +**Step 2: Run the Gradio demo** + +In a separate terminal: + +```bash +python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --api-base http://localhost:8091/v1 --port 7861 ``` Then open `http://localhost:7861/` on your local browser to interact with the web UI. The gradio script supports the following arguments: -- `--model`: Model name +- `--model`: Model name/path (should match the server model) +- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) -- `--stage-configs-path`: Path to custom stage configs YAML file (optional) - `--share`: Share the Gradio demo publicly (creates a public link) ### FAQ diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index 84f7a71b1..a86ba6d24 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -95,21 +95,61 @@ This Web UI demo allows users to interact with the model through a web browser. ### Running Gradio Demo -Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by: +The Gradio demo connects to a vLLM API server. You have two options: + +#### Option 1: One-step Launch Script (Recommended) + +The convenience script launches both the vLLM server and Gradio demo together: ```bash -python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861 +./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861 +``` + +This script will: +1. Start the vLLM server in the background +2. Wait for the server to be ready +3. Launch the Gradio demo +4. Handle cleanup when you press Ctrl+C + +The script supports the following arguments: +- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct) +- `--server-port`: Port for vLLM server (default: 8091) +- `--gradio-port`: Port for Gradio demo (default: 7861) +- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--server-host`: Host for vLLM server (default: 0.0.0.0) +- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1) +- `--share`: Share Gradio demo publicly (creates a public link) + +#### Option 2: Manual Launch (Two-Step Process) + +**Step 1: Launch the vLLM API server** + +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 +``` + +If you have custom stage configs file: +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +**Step 2: Run the Gradio demo** + +In a separate terminal: + +```bash +python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --api-base http://localhost:8091/v1 --port 7861 ``` Then open `http://localhost:7861/` on your local browser to interact with the web UI. - The gradio script supports the following arguments: -- `--model`: Model name +- `--model`: Model name/path (should match the server model) +- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) -- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--share`: Share the Gradio demo publicly (creates a public link) - `--share`: Share the Gradio demo publicly (creates a public link) ## Example materials diff --git a/examples/online_serving/qwen2_5_omni/README.md b/examples/online_serving/qwen2_5_omni/README.md index 8d928aa6b..ebaf6a2ca 100644 --- a/examples/online_serving/qwen2_5_omni/README.md +++ b/examples/online_serving/qwen2_5_omni/README.md @@ -84,20 +84,60 @@ This Web UI demo allows users to interact with the model through a web browser. ### Running Gradio Demo -Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by +The Gradio demo connects to a vLLM API server. You have two options: + +#### Option 1: One-step Launch Script (Recommended) + +The convenience script launches both the vLLM server and Gradio demo together: ```bash -python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861 +./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861 +``` + +This script will: +1. Start the vLLM server in the background +2. Wait for the server to be ready +3. Launch the Gradio demo +4. Handle cleanup when you press Ctrl+C + +The script supports the following arguments: +- `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B) +- `--server-port`: Port for vLLM server (default: 8091) +- `--gradio-port`: Port for Gradio demo (default: 7861) +- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--server-host`: Host for vLLM server (default: 0.0.0.0) +- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1) +- `--share`: Share Gradio demo publicly (creates a public link) + +#### Option 2: Manual Launch (Two-Step Process) + +**Step 1: Launch the vLLM API server** + +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 +``` + +If you have custom stage configs file: +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +**Step 2: Run the Gradio demo** + +In a separate terminal: + +```bash +python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --api-base http://localhost:8091/v1 --port 7861 ``` Then open `http://localhost:7861/` on your local browser to interact with the web UI. The gradio script supports the following arguments: -- `--model`: Model name +- `--model`: Model name/path (should match the server model) +- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) -- `--stage-configs-path`: Path to custom stage configs YAML file (optional) - `--share`: Share the Gradio demo publicly (creates a public link) ### FAQ diff --git a/examples/online_serving/qwen2_5_omni/gradio_demo.py b/examples/online_serving/qwen2_5_omni/gradio_demo.py index f60f6f492..82775ac95 100644 --- a/examples/online_serving/qwen2_5_omni/gradio_demo.py +++ b/examples/online_serving/qwen2_5_omni/gradio_demo.py @@ -1,32 +1,19 @@ import argparse +import base64 +import io import os import random -import signal -import sys from pathlib import Path -from types import SimpleNamespace from typing import Any import gradio as gr import numpy as np import soundfile as sf import torch +from openai import OpenAI from PIL import Image -from vllm.assets.video import video_get_metadata, video_to_ndarrays -from vllm.sampling_params import SamplingParams - -from vllm_omni.entrypoints.async_omni import AsyncOmni - -# Import utils from offline inference example -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni")) -try: - from utils import make_omni_prompt -except ImportError: - # Fallback if utils doesn't exist - we'll build prompts directly - make_omni_prompt = None SEED = 42 -ASYNC_INIT_TIMEOUT = 600 SUPPORTED_MODELS: dict[str, dict[str, Any]] = { "Qwen/Qwen2.5-Omni-7B": { @@ -76,7 +63,12 @@ def parse_args(): parser.add_argument( "--model", default="Qwen/Qwen2.5-Omni-7B", - help="Path to local model checkpoint.", + help="Model name/path (should match the server model).", + ) + parser.add_argument( + "--api-base", + default="http://localhost:8091/v1", + help="Base URL for the vLLM API server.", ) parser.add_argument( "--ip", @@ -85,80 +77,9 @@ def parse_args(): ) parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.") parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.") - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="Path to custom stage configs YAML file (optional).", - ) - parser.add_argument( - "--log-stats", - action="store_true", - help="Enable statistics logging for AsyncOmni.", - ) - parser.add_argument( - "--log-file", - type=str, - default=None, - help="Path prefix for AsyncOmni log files.", - ) - parser.add_argument( - "--init-sleep-seconds", - type=int, - default=30, - help="Seconds to sleep between starting stage processes.", - ) - parser.add_argument( - "--shm-threshold-bytes", - type=int, - default=65536, - help="Threshold in bytes for using shared memory IPC.", - ) - parser.add_argument( - "--batch-timeout", - type=int, - default=10, - help="Batching timeout (seconds) inside each stage.", - ) - parser.add_argument( - "--init-timeout", - type=int, - default=ASYNC_INIT_TIMEOUT, - help="Timeout (seconds) for initializing all stages.", - ) - parser.add_argument( - "--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend" - ) - parser.add_argument( - "--ray-address", - type=str, - default=None, - help="Address of the Ray cluster.", - ) return parser.parse_args() -def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace: - """Construct the minimal CLI args Namespace expected by AsyncOmni.""" - return argparse.Namespace( - model=base_args.model, - stage_configs_path=getattr(base_args, "stage_configs_path", None), - log_stats=bool(getattr(base_args, "log_stats", False)), - log_file=getattr(base_args, "log_file", None), - init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)), - shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)), - batch_timeout=int(getattr(base_args, "batch_timeout", 10)), - init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)), - worker_backend=getattr(base_args, "worker_backend", "multi_process"), - ray_address=getattr(base_args, "ray_address", None), - ) - - -def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]: - """Build SamplingParams objects by reusing the dict definitions.""" - return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)] - - def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]: """Build sampling params as dict for HTTP API mode.""" model_conf = SUPPORTED_MODELS.get(model_key) @@ -174,15 +95,63 @@ def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]: return sampling_params -def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace: - # The prompt builder expects a minimal namespace with these attributes. - return SimpleNamespace( - model=base_args.model, - prompt_type="text", - tokenize=True, - use_torchvision=True, - legacy_omni_video=False, - ) +def image_to_base64_data_url(image: Image.Image) -> str: + """Convert PIL Image to base64 data URL.""" + buffered = io.BytesIO() + # Convert to RGB if needed + if image.mode != "RGB": + image = image.convert("RGB") + image.save(buffered, format="JPEG") + img_bytes = buffered.getvalue() + img_b64 = base64.b64encode(img_bytes).decode("utf-8") + return f"data:image/jpeg;base64,{img_b64}" + + +def audio_to_base64_data_url(audio_data: tuple[np.ndarray, int]) -> str: + """Convert audio (numpy array, sample_rate) to base64 data URL.""" + audio_np, sample_rate = audio_data + # Convert to int16 format for WAV + if audio_np.dtype != np.int16: + # Normalize to [-1, 1] range if needed + if audio_np.dtype == np.float32 or audio_np.dtype == np.float64: + audio_np = np.clip(audio_np, -1.0, 1.0) + audio_np = (audio_np * 32767).astype(np.int16) + else: + audio_np = audio_np.astype(np.int16) + + # Write to WAV bytes + buffered = io.BytesIO() + sf.write(buffered, audio_np, sample_rate, format="WAV") + wav_bytes = buffered.getvalue() + wav_b64 = base64.b64encode(wav_bytes).decode("utf-8") + return f"data:audio/wav;base64,{wav_b64}" + + +def video_to_base64_data_url(video_file: str) -> str: + """Convert video file to base64 data URL.""" + video_path = Path(video_file) + if not video_path.exists(): + raise FileNotFoundError(f"Video file not found: {video_file}") + + # Detect MIME type from extension + video_path_lower = str(video_path).lower() + if video_path_lower.endswith(".mp4"): + mime_type = "video/mp4" + elif video_path_lower.endswith(".webm"): + mime_type = "video/webm" + elif video_path_lower.endswith(".mov"): + mime_type = "video/quicktime" + elif video_path_lower.endswith(".avi"): + mime_type = "video/x-msvideo" + elif video_path_lower.endswith(".mkv"): + mime_type = "video/x-matroska" + else: + mime_type = "video/mp4" + + with open(video_path, "rb") as f: + video_bytes = f.read() + video_b64 = base64.b64encode(video_bytes).decode("utf-8") + return f"data:{mime_type};base64,{video_b64}" def process_audio_file( @@ -262,44 +231,10 @@ def process_image_file(image_file: Image.Image | None) -> Image.Image | None: return image_file -def process_video_file( - video_file: str | None, - enable_audio_in_video: bool = False, - max_frames: int = 32, -) -> tuple[np.ndarray, dict[str, Any], tuple[np.ndarray, int] | None] | None: - """Process video file and optionally extract audio track.""" - if video_file is None: - return None - - video_path = Path(video_file) - if not video_path.exists(): - print(f"Video file not found: {video_path}") - return None - - try: - frames = video_to_ndarrays(str(video_path), num_frames=max_frames) - metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0]) - except Exception as exc: # pylint: disable=broad-except - print(f"Failed to decode video {video_path}: {exc}") - return None - - audio_tuple: tuple[np.ndarray, int] | None = None - if enable_audio_in_video: - try: - import librosa # type: ignore import - - audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000) - audio_tuple = (audio_signal.astype(np.float32), sampling_rate) - except Exception as exc: # pylint: disable=broad-except - print(f"Failed to extract audio from video {video_path}: {exc}") - - return frames, metadata, audio_tuple - - -async def run_inference_async_omni( - omni: AsyncOmni, - sampling_params: list[SamplingParams], - prompt_args_template: SimpleNamespace, +def run_inference_api( + client: OpenAI, + model: str, + sampling_params_dict: list[dict], user_prompt: str, audio_file: tuple[str, tuple[int, np.ndarray]] | None = None, image_file: Image.Image | None = None, @@ -307,118 +242,117 @@ async def run_inference_async_omni( use_audio_in_video: bool = False, output_modalities: str | None = None, ): - """Run inference using AsyncOmni directly with multimodal support.""" + """Run inference using OpenAI API client with multimodal support.""" if not user_prompt.strip() and not audio_file and not image_file and not video_file: return "Please provide at least a text prompt or multimodal input.", None - if output_modalities is not None: - output_modalities = output_modalities.split(",") - try: - # Build prompt with multimodal data - prompt_args = SimpleNamespace(**prompt_args_template.__dict__) - - # Process multimodal inputs - multi_modal_data = {} - mm_processor_kwargs = {} + # Build message content list + content_list = [] # Process audio audio_data = process_audio_file(audio_file) if audio_data is not None: - multi_modal_data["audio"] = audio_data + audio_url = audio_to_base64_data_url(audio_data) + content_list.append( + { + "type": "audio_url", + "audio_url": {"url": audio_url}, + } + ) # Process image - image_data = process_image_file(image_file) - if image_data is not None: - multi_modal_data["image"] = image_data + if image_file is not None: + image_data = process_image_file(image_file) + if image_data is not None: + image_url = image_to_base64_data_url(image_data) + content_list.append( + { + "type": "image_url", + "image_url": {"url": image_url}, + } + ) # Process video - video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video) - if video_payload is not None: - video_frames, video_metadata, extracted_audio = video_payload - video_entry: Any - if video_metadata: - video_entry = (video_frames, video_metadata) - else: - video_entry = video_frames - multi_modal_data["video"] = video_entry - if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data: - multi_modal_data["audio"] = extracted_audio - mm_processor_kwargs["use_audio_in_video"] = True - - # Build the prompt input - if make_omni_prompt is not None: - omni_prompt = make_omni_prompt(prompt_args, user_prompt) - # Add multimodal data if present - if multi_modal_data: - if isinstance(omni_prompt, dict): - omni_prompt["multi_modal_data"] = multi_modal_data - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs - else: - # If make_omni_prompt returns a string, we need to wrap it - omni_prompt = { - "prompt": omni_prompt, - "multi_modal_data": multi_modal_data, - } - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs - else: - # Fallback: build prompt directly - default_system = ( - "You are Qwen, a virtual human developed by the Qwen Team, " - "Alibaba Group, capable of perceiving auditory and visual inputs, " - "as well as generating text and speech." - ) - prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n" - if audio_data: - prompt += "<|audio_bos|><|AUDIO|><|audio_eos|>" - if image_data: - prompt += "<|vision_bos|><|IMAGE|><|vision_eos|>" - if video_payload is not None: - prompt += "<|vision_bos|><|VIDEO|><|vision_eos|>" - if user_prompt.strip(): - prompt += f"{user_prompt}" - prompt += "<|im_end|>\n<|im_start|>assistant\n" - - omni_prompt = { - "prompt": prompt, - "multi_modal_data": multi_modal_data, + mm_processor_kwargs = {} + if video_file is not None: + video_url = video_to_base64_data_url(video_file) + video_content = { + "type": "video_url", + "video_url": {"url": video_url}, } - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs + if use_audio_in_video: + video_content["video_url"]["num_frames"] = 32 # Default max frames + mm_processor_kwargs["use_audio_in_video"] = True + content_list.append(video_content) - request_id = "0" + # Add text prompt + if user_prompt.strip(): + content_list.append( + { + "type": "text", + "text": user_prompt, + } + ) + + # Build messages + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are Qwen, a virtual human developed by the Qwen Team, " + "Alibaba Group, capable of perceiving auditory and visual inputs, " + "as well as generating text and speech." + ), + } + ], + }, + { + "role": "user", + "content": content_list, + }, + ] + + # Build extra_body + extra_body = { + "sampling_params_list": sampling_params_dict, + } + if mm_processor_kwargs: + extra_body["mm_processor_kwargs"] = mm_processor_kwargs + + # Parse output modalities + if output_modalities is not None: + output_modalities_list = [m.strip() for m in output_modalities.split(",")] + else: + output_modalities_list = None + + # Call API + chat_completion = client.chat.completions.create( + messages=messages, + model=model, + modalities=output_modalities_list, + extra_body=extra_body, + ) + + # Extract outputs text_outputs: list[str] = [] audio_output = None - async for stage_outputs in omni.generate( - prompt=omni_prompt, - request_id=request_id, - sampling_params_list=sampling_params, - output_modalities=output_modalities, - ): - # stage_outputs.request_output is a RequestOutput object, not a list - request_output = stage_outputs.request_output - if stage_outputs.final_output_type == "text": - if request_output.outputs: - for output in request_output.outputs: - if output.text: - text_outputs.append(output.text) - elif stage_outputs.final_output_type == "audio": - # multimodal_output is on the RequestOutput object - # See vllm_omni/entrypoints/openai/serving_chat.py:680 for reference - if hasattr(request_output, "multimodal_output") and request_output.multimodal_output: - if "audio" in request_output.multimodal_output: - audio_tensor = request_output.multimodal_output["audio"] - # Ensure audio is 1D (flatten if needed) - if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1: - audio_tensor = audio_tensor.flatten() - audio_np = audio_tensor.detach().cpu().numpy() - audio_output = ( - 24000, # sampling rate in Hz - audio_np, - ) + for choice in chat_completion.choices: + if choice.message.content: + text_outputs.append(choice.message.content) + if choice.message.audio: + # Decode base64 audio + audio_data = base64.b64decode(choice.message.audio.data) + # Load audio from bytes + audio_np, sample_rate = sf.read(io.BytesIO(audio_data)) + # Convert to mono if needed + if audio_np.ndim > 1: + audio_np = audio_np[:, 0] + audio_output = (int(sample_rate), audio_np.astype(np.float32)) text_response = "\n\n".join(text_outputs) if text_outputs else "No text output." return text_response, audio_output @@ -427,14 +361,13 @@ async def run_inference_async_omni( def build_interface( - omni: AsyncOmni, - sampling_params: list[SamplingParams], - prompt_args_template: SimpleNamespace, + client: OpenAI, model: str, + sampling_params_dict: list[dict], ): - """Build Gradio interface for AsyncOmni mode.""" + """Build Gradio interface for API server mode.""" - async def run_inference( + def run_inference( user_prompt: str, audio_file: tuple[str, tuple[int, np.ndarray]] | None, image_file: Image.Image | None, @@ -442,10 +375,10 @@ def build_interface( use_audio_in_video: bool, output_modalities: str | None = None, ): - return await run_inference_async_omni( - omni, - sampling_params, - prompt_args_template, + return run_inference_api( + client, + model, + sampling_params_dict, user_prompt, audio_file, image_file, @@ -550,41 +483,27 @@ def build_interface( def main(): args = parse_args() - omni = None model_name = "/".join(args.model.split("/")[-2:]) assert model_name in SUPPORTED_MODELS, ( f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}" ) - # Register signal handlers for graceful shutdown - def signal_handler(sig, frame): - print("\nReceived interrupt signal, shutting down...") - if omni is not None: - try: - omni.shutdown() - except Exception as e: - print(f"Error during shutdown: {e}") - sys.exit(0) + # Initialize OpenAI client + print(f"Connecting to API server at: {args.api_base}") + client = OpenAI( + api_key="EMPTY", + base_url=args.api_base, + ) + print("✓ Connected to API server") - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - print(f"Initializing AsyncOmni with model: {args.model}") - if args.stage_configs_path: - print(f"Using custom stage configs: {args.stage_configs_path}") - - sampling_params = build_sampling_params(SEED, model_name) - cli_args = build_async_omni_cli_args(args) - omni = AsyncOmni(model=args.model, cli_args=cli_args) - print("✓ AsyncOmni initialized successfully") - prompt_args_template = create_prompt_args(args) + # Build sampling params + sampling_params_dict = build_sampling_params_dict(SEED, model_name) demo = build_interface( - omni, - sampling_params, - prompt_args_template, + client, args.model, + sampling_params_dict, ) try: demo.launch( @@ -594,13 +513,6 @@ def main(): ) except KeyboardInterrupt: print("\nShutting down...") - finally: - # Cleanup - if omni is not None: - try: - omni.shutdown() - except Exception as e: - print(f"Error during cleanup: {e}") if __name__ == "__main__": diff --git a/examples/online_serving/qwen2_5_omni/run_gradio_demo.sh b/examples/online_serving/qwen2_5_omni/run_gradio_demo.sh new file mode 100755 index 000000000..296cc7fa5 --- /dev/null +++ b/examples/online_serving/qwen2_5_omni/run_gradio_demo.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# Convenience script to launch both vLLM server and Gradio demo for Qwen2.5-Omni +# +# Usage: +# ./run_gradio_demo.sh [OPTIONS] +# +# Example: +# ./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861 + +set -e + +# Default values +MODEL="Qwen/Qwen2.5-Omni-7B" +SERVER_PORT=8091 +GRADIO_PORT=7861 +STAGE_CONFIGS_PATH="" +SERVER_HOST="0.0.0.0" +GRADIO_IP="127.0.0.1" +GRADIO_SHARE=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --model) + MODEL="$2" + shift 2 + ;; + --server-port) + SERVER_PORT="$2" + shift 2 + ;; + --gradio-port) + GRADIO_PORT="$2" + shift 2 + ;; + --stage-configs-path) + STAGE_CONFIGS_PATH="$2" + shift 2 + ;; + --server-host) + SERVER_HOST="$2" + shift 2 + ;; + --gradio-ip) + GRADIO_IP="$2" + shift 2 + ;; + --share) + GRADIO_SHARE=true + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --model MODEL Model name/path (default: Qwen/Qwen2.5-Omni-7B)" + echo " --server-port PORT Port for vLLM server (default: 8091)" + echo " --gradio-port PORT Port for Gradio demo (default: 7861)" + echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)" + echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)" + echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)" + echo " --share Share Gradio demo publicly" + echo " --help Show this help message" + echo "" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_BASE="http://localhost:${SERVER_PORT}/v1" +HEALTH_URL="http://localhost:${SERVER_PORT}/health" + +echo "==========================================" +echo "Starting vLLM-Omni Gradio Demo" +echo "==========================================" +echo "Model: $MODEL" +echo "Server: http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio: http://${GRADIO_IP}:${GRADIO_PORT}" +echo "==========================================" + +# Build vLLM server command +SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST") +if [ -n "$STAGE_CONFIGS_PATH" ]; then + SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH") +fi + +# Function to cleanup on exit +cleanup() { + echo "" + echo "Shutting down..." + if [ -n "$SERVER_PID" ]; then + echo "Stopping vLLM server (PID: $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + if [ -n "$GRADIO_PID" ]; then + echo "Stopping Gradio demo (PID: $GRADIO_PID)..." + kill "$GRADIO_PID" 2>/dev/null || true + wait "$GRADIO_PID" 2>/dev/null || true + fi + echo "Cleanup complete" + exit 0 +} + +# Set up signal handlers +trap cleanup SIGINT SIGTERM + +# Start vLLM server with output shown in real-time and saved to log +echo "" +echo "Starting vLLM server..." +LOG_FILE="/tmp/vllm_server_${SERVER_PORT}.log" +"${SERVER_CMD[@]}" 2>&1 | tee "$LOG_FILE" & +SERVER_PID=$! + +# Start a background process to monitor the log for startup completion +STARTUP_COMPLETE=false +TAIL_PID="" + +# Function to cleanup tail process +cleanup_tail() { + if [ -n "$TAIL_PID" ]; then + kill "$TAIL_PID" 2>/dev/null || true + wait "$TAIL_PID" 2>/dev/null || true + fi +} + +# Wait for server to be ready by checking log output +echo "" +echo "Waiting for vLLM server to be ready (checking for 'Application startup complete' message)..." +echo "" + +# Monitor log file for startup completion message +MAX_WAIT=300 # 5 minutes timeout as fallback +ELAPSED=0 + +# Use a temporary file to track startup completion +STARTUP_FLAG="/tmp/vllm_startup_flag_${SERVER_PORT}.tmp" +rm -f "$STARTUP_FLAG" + +# Start monitoring in background +( + tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG" +) & +TAIL_PID=$! + +while [ $ELAPSED -lt $MAX_WAIT ]; do + # Check if startup flag file exists (startup complete) + if [ -f "$STARTUP_FLAG" ]; then + cleanup_tail + echo "" + echo "✓ vLLM server is ready!" + STARTUP_COMPLETE=true + break + fi + + # Check if server process is still running + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + cleanup_tail + echo "" + echo "Error: vLLM server failed to start (process terminated)" + wait "$SERVER_PID" 2>/dev/null || true + exit 1 + fi + + sleep 1 + ELAPSED=$((ELAPSED + 1)) +done + +cleanup_tail +rm -f "$STARTUP_FLAG" + +if [ "$STARTUP_COMPLETE" != "true" ]; then + echo "" + echo "Error: vLLM server did not complete startup within ${MAX_WAIT} seconds" + kill "$SERVER_PID" 2>/dev/null || true + exit 1 +fi + +# Start Gradio demo +echo "" +echo "Starting Gradio demo..." +cd "$SCRIPT_DIR" +GRADIO_CMD=("python" "gradio_demo.py" "--model" "$MODEL" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT") +if [ "$GRADIO_SHARE" = true ]; then + GRADIO_CMD+=("--share") +fi + +"${GRADIO_CMD[@]}" > /tmp/gradio_demo.log 2>&1 & +GRADIO_PID=$! + +echo "" +echo "==========================================" +echo "Both services are running!" +echo "==========================================" +echo "vLLM Server: http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio Demo: http://${GRADIO_IP}:${GRADIO_PORT}" +echo "" +echo "Press Ctrl+C to stop both services" +echo "==========================================" +echo "" + +# Wait for either process to exit +wait $SERVER_PID $GRADIO_PID || true + +cleanup diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index 660f413e7..41e954264 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -92,19 +92,58 @@ This Web UI demo allows users to interact with the model through a web browser. ### Running Gradio Demo -Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by: +The Gradio demo connects to a vLLM API server. You have two options: + +#### Option 1: One-step Launch Script (Recommended) + +The convenience script launches both the vLLM server and Gradio demo together: ```bash -python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861 +./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861 +``` + +This script will: +1. Start the vLLM server in the background +2. Wait for the server to be ready +3. Launch the Gradio demo +4. Handle cleanup when you press Ctrl+C + +The script supports the following arguments: +- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct) +- `--server-port`: Port for vLLM server (default: 8091) +- `--gradio-port`: Port for Gradio demo (default: 7861) +- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--server-host`: Host for vLLM server (default: 0.0.0.0) +- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1) +- `--share`: Share Gradio demo publicly (creates a public link) + +#### Option 2: Manual Launch (Two-Step Process) + +**Step 1: Launch the vLLM API server** + +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 +``` + +If you have custom stage configs file: +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +**Step 2: Run the Gradio demo** + +In a separate terminal: + +```bash +python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --api-base http://localhost:8091/v1 --port 7861 ``` Then open `http://localhost:7861/` on your local browser to interact with the web UI. - The gradio script supports the following arguments: -- `--model`: Model name +- `--model`: Model name/path (should match the server model) +- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) -- `--stage-configs-path`: Path to custom stage configs YAML file (optional) - `--share`: Share the Gradio demo publicly (creates a public link) diff --git a/examples/online_serving/qwen3_omni/gradio_demo.py b/examples/online_serving/qwen3_omni/gradio_demo.py index f30c0386b..eee0c75f6 100644 --- a/examples/online_serving/qwen3_omni/gradio_demo.py +++ b/examples/online_serving/qwen3_omni/gradio_demo.py @@ -1,32 +1,19 @@ import argparse +import base64 +import io import os import random -import signal -import sys from pathlib import Path -from types import SimpleNamespace from typing import Any import gradio as gr import numpy as np import soundfile as sf import torch +from openai import OpenAI from PIL import Image -from vllm.assets.video import video_get_metadata, video_to_ndarrays -from vllm.sampling_params import SamplingParams - -from vllm_omni.entrypoints.async_omni import AsyncOmni - -# Import utils from offline inference example -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen3_omni")) -try: - from utils import make_omni_prompt -except ImportError: - # Fallback if utils doesn't exist - we'll build prompts directly - make_omni_prompt = None SEED = 42 -ASYNC_INIT_TIMEOUT = 600 SUPPORTED_MODELS: dict[str, dict[str, Any]] = { "Qwen/Qwen3-Omni-30B-A3B-Instruct": { @@ -79,7 +66,12 @@ def parse_args(): parser.add_argument( "--model", default="Qwen/Qwen3-Omni-30B-A3B-Instruct", - help="Path to local model checkpoint.", + help="Model name/path (should match the server model).", + ) + parser.add_argument( + "--api-base", + default="http://localhost:8091/v1", + help="Base URL for the vLLM API server.", ) parser.add_argument( "--ip", @@ -88,80 +80,9 @@ def parse_args(): ) parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.") parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.") - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help="Path to custom stage configs YAML file (optional).", - ) - parser.add_argument( - "--log-stats", - action="store_true", - help="Enable statistics logging for AsyncOmni.", - ) - parser.add_argument( - "--log-file", - type=str, - default=None, - help="Path prefix for AsyncOmni log files.", - ) - parser.add_argument( - "--init-sleep-seconds", - type=int, - default=30, - help="Seconds to sleep between starting stage processes.", - ) - parser.add_argument( - "--shm-threshold-bytes", - type=int, - default=65536, - help="Threshold in bytes for using shared memory IPC.", - ) - parser.add_argument( - "--batch-timeout", - type=int, - default=10, - help="Batching timeout (seconds) inside each stage.", - ) - parser.add_argument( - "--init-timeout", - type=int, - default=ASYNC_INIT_TIMEOUT, - help="Timeout (seconds) for initializing all stages.", - ) - parser.add_argument( - "--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend" - ) - parser.add_argument( - "--ray-address", - type=str, - default=None, - help="Address of the Ray cluster.", - ) return parser.parse_args() -def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace: - """Construct the minimal CLI args Namespace expected by AsyncOmni.""" - return argparse.Namespace( - model=base_args.model, - stage_configs_path=getattr(base_args, "stage_configs_path", None), - log_stats=bool(getattr(base_args, "log_stats", False)), - log_file=getattr(base_args, "log_file", None), - init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)), - shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)), - batch_timeout=int(getattr(base_args, "batch_timeout", 10)), - init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)), - worker_backend=getattr(base_args, "worker_backend", "multi_process"), - ray_address=getattr(base_args, "ray_address", None), - ) - - -def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]: - """Build SamplingParams objects by reusing the dict definitions.""" - return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)] - - def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]: """Build sampling params as dict for HTTP API mode.""" model_conf = SUPPORTED_MODELS.get(model_key) @@ -177,15 +98,63 @@ def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]: return sampling_params -def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace: - # The prompt builder expects a minimal namespace with these attributes. - return SimpleNamespace( - model=base_args.model, - prompt_type="text", - tokenize=True, - use_torchvision=True, - legacy_omni_video=False, - ) +def image_to_base64_data_url(image: Image.Image) -> str: + """Convert PIL Image to base64 data URL.""" + buffered = io.BytesIO() + # Convert to RGB if needed + if image.mode != "RGB": + image = image.convert("RGB") + image.save(buffered, format="JPEG") + img_bytes = buffered.getvalue() + img_b64 = base64.b64encode(img_bytes).decode("utf-8") + return f"data:image/jpeg;base64,{img_b64}" + + +def audio_to_base64_data_url(audio_data: tuple[np.ndarray, int]) -> str: + """Convert audio (numpy array, sample_rate) to base64 data URL.""" + audio_np, sample_rate = audio_data + # Convert to int16 format for WAV + if audio_np.dtype != np.int16: + # Normalize to [-1, 1] range if needed + if audio_np.dtype == np.float32 or audio_np.dtype == np.float64: + audio_np = np.clip(audio_np, -1.0, 1.0) + audio_np = (audio_np * 32767).astype(np.int16) + else: + audio_np = audio_np.astype(np.int16) + + # Write to WAV bytes + buffered = io.BytesIO() + sf.write(buffered, audio_np, sample_rate, format="WAV") + wav_bytes = buffered.getvalue() + wav_b64 = base64.b64encode(wav_bytes).decode("utf-8") + return f"data:audio/wav;base64,{wav_b64}" + + +def video_to_base64_data_url(video_file: str) -> str: + """Convert video file to base64 data URL.""" + video_path = Path(video_file) + if not video_path.exists(): + raise FileNotFoundError(f"Video file not found: {video_file}") + + # Detect MIME type from extension + video_path_lower = str(video_path).lower() + if video_path_lower.endswith(".mp4"): + mime_type = "video/mp4" + elif video_path_lower.endswith(".webm"): + mime_type = "video/webm" + elif video_path_lower.endswith(".mov"): + mime_type = "video/quicktime" + elif video_path_lower.endswith(".avi"): + mime_type = "video/x-msvideo" + elif video_path_lower.endswith(".mkv"): + mime_type = "video/x-matroska" + else: + mime_type = "video/mp4" + + with open(video_path, "rb") as f: + video_bytes = f.read() + video_b64 = base64.b64encode(video_bytes).decode("utf-8") + return f"data:{mime_type};base64,{video_b64}" def process_audio_file( @@ -265,44 +234,10 @@ def process_image_file(image_file: Image.Image | None) -> Image.Image | None: return image_file -def process_video_file( - video_file: str | None, - enable_audio_in_video: bool = False, - max_frames: int = 32, -) -> tuple[np.ndarray, dict[str, Any], tuple[np.ndarray, int] | None] | None: - """Process video file and optionally extract audio track.""" - if video_file is None: - return None - - video_path = Path(video_file) - if not video_path.exists(): - print(f"Video file not found: {video_path}") - return None - - try: - frames = video_to_ndarrays(str(video_path), num_frames=max_frames) - metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0]) - except Exception as exc: # pylint: disable=broad-except - print(f"Failed to decode video {video_path}: {exc}") - return None - - audio_tuple: tuple[np.ndarray, int] | None = None - if enable_audio_in_video: - try: - import librosa # type: ignore import - - audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000) - audio_tuple = (audio_signal.astype(np.float32), sampling_rate) - except Exception as exc: # pylint: disable=broad-except - print(f"Failed to extract audio from video {video_path}: {exc}") - - return frames, metadata, audio_tuple - - -async def run_inference_async_omni( - omni: AsyncOmni, - sampling_params: list[SamplingParams], - prompt_args_template: SimpleNamespace, +def run_inference_api( + client: OpenAI, + model: str, + sampling_params_dict: list[dict], user_prompt: str, audio_file: tuple[str, tuple[int, np.ndarray]] | None = None, image_file: Image.Image | None = None, @@ -310,121 +245,117 @@ async def run_inference_async_omni( use_audio_in_video: bool = False, output_modalities: str | None = None, ): - """Run inference using AsyncOmni directly with multimodal support.""" + """Run inference using OpenAI API client with multimodal support.""" if not user_prompt.strip() and not audio_file and not image_file and not video_file: return "Please provide at least a text prompt or multimodal input.", None - if output_modalities is not None: - output_modalities = output_modalities.split(",") - try: - # Build prompt with multimodal data - prompt_args = SimpleNamespace(**prompt_args_template.__dict__) - - # Process multimodal inputs - multi_modal_data = {} - mm_processor_kwargs = {} + # Build message content list + content_list = [] # Process audio audio_data = process_audio_file(audio_file) if audio_data is not None: - multi_modal_data["audio"] = [audio_data] if not isinstance(audio_data, list) else audio_data + audio_url = audio_to_base64_data_url(audio_data) + content_list.append( + { + "type": "audio_url", + "audio_url": {"url": audio_url}, + } + ) # Process image - image_data = process_image_file(image_file) - if image_data is not None: - multi_modal_data["image"] = image_data + if image_file is not None: + image_data = process_image_file(image_file) + if image_data is not None: + image_url = image_to_base64_data_url(image_data) + content_list.append( + { + "type": "image_url", + "image_url": {"url": image_url}, + } + ) # Process video - video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video) - if video_payload is not None: - video_frames, video_metadata, extracted_audio = video_payload - video_entry: Any - if video_metadata: - video_entry = (video_frames, video_metadata) - else: - video_entry = video_frames - multi_modal_data["video"] = video_entry - if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data: - multi_modal_data["audio"] = ( - [extracted_audio] if not isinstance(extracted_audio, list) else extracted_audio - ) - mm_processor_kwargs["use_audio_in_video"] = True - - # Build the prompt input - if make_omni_prompt is not None: - omni_prompt = make_omni_prompt(prompt_args, user_prompt) - # Add multimodal data if present - if multi_modal_data: - if isinstance(omni_prompt, dict): - omni_prompt["multi_modal_data"] = multi_modal_data - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs - else: - # If make_omni_prompt returns a string, we need to wrap it - omni_prompt = { - "prompt": omni_prompt, - "multi_modal_data": multi_modal_data, - } - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs - else: - # Fallback: build prompt directly - default_system = ( - "You are Qwen, a virtual human developed by the Qwen Team, " - "Alibaba Group, capable of perceiving auditory and visual inputs, " - "as well as generating text and speech." - ) - prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n" - if audio_data: - prompt += "<|audio_start|><|audio_pad|><|audio_end|>" - if image_data: - prompt += "<|vision_start|><|image_pad|><|vision_end|>" - if video_payload is not None: - prompt += "<|vision_start|><|video_pad|><|vision_end|>" - if user_prompt.strip(): - prompt += f"{user_prompt}" - prompt += "<|im_end|>\n<|im_start|>assistant\n" - - omni_prompt = { - "prompt": prompt, - "multi_modal_data": multi_modal_data, + mm_processor_kwargs = {} + if video_file is not None: + video_url = video_to_base64_data_url(video_file) + video_content = { + "type": "video_url", + "video_url": {"url": video_url}, } - if mm_processor_kwargs: - omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs + if use_audio_in_video: + video_content["video_url"]["num_frames"] = 32 # Default max frames + mm_processor_kwargs["use_audio_in_video"] = True + content_list.append(video_content) - request_id = "0" + # Add text prompt + if user_prompt.strip(): + content_list.append( + { + "type": "text", + "text": user_prompt, + } + ) + + # Build messages + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": ( + "You are Qwen, a virtual human developed by the Qwen Team, " + "Alibaba Group, capable of perceiving auditory and visual inputs, " + "as well as generating text and speech." + ), + } + ], + }, + { + "role": "user", + "content": content_list, + }, + ] + + # Build extra_body + extra_body = { + "sampling_params_list": sampling_params_dict, + } + if mm_processor_kwargs: + extra_body["mm_processor_kwargs"] = mm_processor_kwargs + + # Parse output modalities + if output_modalities is not None: + output_modalities_list = [m.strip() for m in output_modalities.split(",")] + else: + output_modalities_list = None + + # Call API + chat_completion = client.chat.completions.create( + messages=messages, + model=model, + modalities=output_modalities_list, + extra_body=extra_body, + ) + + # Extract outputs text_outputs: list[str] = [] audio_output = None - async for stage_outputs in omni.generate( - prompt=omni_prompt, - request_id=request_id, - sampling_params_list=sampling_params, - output_modalities=output_modalities, - ): - # stage_outputs.request_output is a RequestOutput object, not a list - request_output = stage_outputs.request_output - if stage_outputs.final_output_type == "text": - if request_output.outputs: - for output in request_output.outputs: - if output.text: - text_outputs.append(output.text) - elif stage_outputs.final_output_type == "audio": - # multimodal_output is on the RequestOutput object - # See vllm_omni/entrypoints/openai/serving_chat.py:733 for reference - if hasattr(request_output, "multimodal_output") and request_output.multimodal_output: - if "audio" in request_output.multimodal_output: - audio_tensor = request_output.multimodal_output["audio"] - # Ensure audio is 1D (flatten if needed) - if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1: - audio_tensor = audio_tensor.flatten() - # Convert BFloat16 to Float32 before numpy conversion (numpy doesn't support BFloat16) - audio_np = audio_tensor.float().detach().cpu().numpy() - audio_output = ( - 24000, # sampling rate in Hz - audio_np, - ) + for choice in chat_completion.choices: + if choice.message.content: + text_outputs.append(choice.message.content) + if choice.message.audio: + # Decode base64 audio + audio_data = base64.b64decode(choice.message.audio.data) + # Load audio from bytes + audio_np, sample_rate = sf.read(io.BytesIO(audio_data)) + # Convert to mono if needed + if audio_np.ndim > 1: + audio_np = audio_np[:, 0] + audio_output = (int(sample_rate), audio_np.astype(np.float32)) text_response = "\n\n".join(text_outputs) if text_outputs else "No text output." return text_response, audio_output @@ -433,14 +364,13 @@ async def run_inference_async_omni( def build_interface( - omni: AsyncOmni, - sampling_params: list[SamplingParams], - prompt_args_template: SimpleNamespace, + client: OpenAI, model: str, + sampling_params_dict: list[dict], ): - """Build Gradio interface for AsyncOmni mode.""" + """Build Gradio interface for API server mode.""" - async def run_inference( + def run_inference( user_prompt: str, audio_file: tuple[str, tuple[int, np.ndarray]] | None, image_file: Image.Image | None, @@ -448,10 +378,10 @@ def build_interface( use_audio_in_video: bool, output_modalities: str | None = None, ): - return await run_inference_async_omni( - omni, - sampling_params, - prompt_args_template, + return run_inference_api( + client, + model, + sampling_params_dict, user_prompt, audio_file, image_file, @@ -556,41 +486,27 @@ def build_interface( def main(): args = parse_args() - omni = None model_name = "/".join(args.model.split("/")[-2:]) assert model_name in SUPPORTED_MODELS, ( f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}" ) - # Register signal handlers for graceful shutdown - def signal_handler(sig, frame): - print("\nReceived interrupt signal, shutting down...") - if omni is not None: - try: - omni.shutdown() - except Exception as e: - print(f"Error during shutdown: {e}") - sys.exit(0) + # Initialize OpenAI client + print(f"Connecting to API server at: {args.api_base}") + client = OpenAI( + api_key="EMPTY", + base_url=args.api_base, + ) + print("✓ Connected to API server") - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - print(f"Initializing AsyncOmni with model: {args.model}") - if args.stage_configs_path: - print(f"Using custom stage configs: {args.stage_configs_path}") - - sampling_params = build_sampling_params(SEED, model_name) - cli_args = build_async_omni_cli_args(args) - omni = AsyncOmni(model=args.model, cli_args=cli_args) - print("✓ AsyncOmni initialized successfully") - prompt_args_template = create_prompt_args(args) + # Build sampling params + sampling_params_dict = build_sampling_params_dict(SEED, model_name) demo = build_interface( - omni, - sampling_params, - prompt_args_template, + client, args.model, + sampling_params_dict, ) try: demo.launch( @@ -600,13 +516,6 @@ def main(): ) except KeyboardInterrupt: print("\nShutting down...") - finally: - # Cleanup - if omni is not None: - try: - omni.shutdown() - except Exception as e: - print(f"Error during cleanup: {e}") if __name__ == "__main__": diff --git a/examples/online_serving/qwen3_omni/run_gradio_demo.sh b/examples/online_serving/qwen3_omni/run_gradio_demo.sh new file mode 100755 index 000000000..73ce273d9 --- /dev/null +++ b/examples/online_serving/qwen3_omni/run_gradio_demo.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# Convenience script to launch both vLLM server and Gradio demo for Qwen3-Omni +# +# Usage: +# ./run_gradio_demo.sh [OPTIONS] +# +# Example: +# ./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861 + +set -e + +# Default values +MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct" +SERVER_PORT=8091 +GRADIO_PORT=7861 +STAGE_CONFIGS_PATH="" +SERVER_HOST="0.0.0.0" +GRADIO_IP="127.0.0.1" +GRADIO_SHARE=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --model) + MODEL="$2" + shift 2 + ;; + --server-port) + SERVER_PORT="$2" + shift 2 + ;; + --gradio-port) + GRADIO_PORT="$2" + shift 2 + ;; + --stage-configs-path) + STAGE_CONFIGS_PATH="$2" + shift 2 + ;; + --server-host) + SERVER_HOST="$2" + shift 2 + ;; + --gradio-ip) + GRADIO_IP="$2" + shift 2 + ;; + --share) + GRADIO_SHARE=true + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --model MODEL Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)" + echo " --server-port PORT Port for vLLM server (default: 8091)" + echo " --gradio-port PORT Port for Gradio demo (default: 7861)" + echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)" + echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)" + echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)" + echo " --share Share Gradio demo publicly" + echo " --help Show this help message" + echo "" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_BASE="http://localhost:${SERVER_PORT}/v1" +HEALTH_URL="http://localhost:${SERVER_PORT}/health" + +echo "==========================================" +echo "Starting vLLM-Omni Gradio Demo" +echo "==========================================" +echo "Model: $MODEL" +echo "Server: http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio: http://${GRADIO_IP}:${GRADIO_PORT}" +echo "==========================================" + +# Build vLLM server command +SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST") +if [ -n "$STAGE_CONFIGS_PATH" ]; then + SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH") +fi + +# Function to cleanup on exit +cleanup() { + echo "" + echo "Shutting down..." + if [ -n "$SERVER_PID" ]; then + echo "Stopping vLLM server (PID: $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + if [ -n "$GRADIO_PID" ]; then + echo "Stopping Gradio demo (PID: $GRADIO_PID)..." + kill "$GRADIO_PID" 2>/dev/null || true + wait "$GRADIO_PID" 2>/dev/null || true + fi + echo "Cleanup complete" + exit 0 +} + +# Set up signal handlers +trap cleanup SIGINT SIGTERM + +# Start vLLM server with output shown in real-time and saved to log +echo "" +echo "Starting vLLM server..." +LOG_FILE="/tmp/vllm_server_${SERVER_PORT}.log" +"${SERVER_CMD[@]}" 2>&1 | tee "$LOG_FILE" & +SERVER_PID=$! + +# Start a background process to monitor the log for startup completion +STARTUP_COMPLETE=false +TAIL_PID="" + +# Function to cleanup tail process +cleanup_tail() { + if [ -n "$TAIL_PID" ]; then + kill "$TAIL_PID" 2>/dev/null || true + wait "$TAIL_PID" 2>/dev/null || true + fi +} + +# Wait for server to be ready by checking log output +echo "" +echo "Waiting for vLLM server to be ready (checking for 'Application startup complete' message)..." +echo "" + +# Monitor log file for startup completion message +MAX_WAIT=300 # 5 minutes timeout as fallback +ELAPSED=0 + +# Use a temporary file to track startup completion +STARTUP_FLAG="/tmp/vllm_startup_flag_${SERVER_PORT}.tmp" +rm -f "$STARTUP_FLAG" + +# Start monitoring in background +( + tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG" +) & +TAIL_PID=$! + +while [ $ELAPSED -lt $MAX_WAIT ]; do + # Check if startup flag file exists (startup complete) + if [ -f "$STARTUP_FLAG" ]; then + cleanup_tail + echo "" + echo "✓ vLLM server is ready!" + STARTUP_COMPLETE=true + break + fi + + # Check if server process is still running + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + cleanup_tail + echo "" + echo "Error: vLLM server failed to start (process terminated)" + wait "$SERVER_PID" 2>/dev/null || true + exit 1 + fi + + sleep 1 + ELAPSED=$((ELAPSED + 1)) +done + +cleanup_tail +rm -f "$STARTUP_FLAG" + +if [ "$STARTUP_COMPLETE" != "true" ]; then + echo "" + echo "Error: vLLM server did not complete startup within ${MAX_WAIT} seconds" + kill "$SERVER_PID" 2>/dev/null || true + exit 1 +fi + +# Start Gradio demo +echo "" +echo "Starting Gradio demo..." +cd "$SCRIPT_DIR" +GRADIO_CMD=("python" "gradio_demo.py" "--model" "$MODEL" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT") +if [ "$GRADIO_SHARE" = true ]; then + GRADIO_CMD+=("--share") +fi + +"${GRADIO_CMD[@]}" > /tmp/gradio_demo.log 2>&1 & +GRADIO_PID=$! + +echo "" +echo "==========================================" +echo "Both services are running!" +echo "==========================================" +echo "vLLM Server: http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio Demo: http://${GRADIO_IP}:${GRADIO_PORT}" +echo "" +echo "Press Ctrl+C to stop both services" +echo "==========================================" +echo "" + +# Wait for either process to exit +wait $SERVER_PID $GRADIO_PID || true + +cleanup