[Misc] Update qwen-omni gradio demo to use api server (#378)
Signed-off-by: samithuang <285365963@qq.com>
This commit is contained in:
1 parent
628c0a9217
commit
094de95548
8 files changed
+967
-563
No files matched your search
@@ -87,20 +87,60 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
The Gradio demo connects to a vLLM API server. You have two options:
|
||||
|
||||
#### Option 1: One-step Launch Script (Recommended)
|
||||
|
||||
The convenience script launches both the vLLM server and Gradio demo together:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861
|
||||
./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861
|
||||
```
|
||||
|
||||
This script will:
|
||||
1. Start the vLLM server in the background
|
||||
2. Wait for the server to be ready
|
||||
3. Launch the Gradio demo
|
||||
4. Handle cleanup when you press Ctrl+C
|
||||
|
||||
The script supports the following arguments:
|
||||
- `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--server-host`: Host for vLLM server (default: 0.0.0.0)
|
||||
- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
|
||||
- `--share`: Share Gradio demo publicly (creates a public link)
|
||||
|
||||
#### Option 2: Manual Launch (Two-Step Process)
|
||||
|
||||
**Step 1: Launch the vLLM API server**
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
|
||||
In a separate terminal:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --api-base http://localhost:8091/v1 --port 7861
|
||||
```
|
||||
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name
|
||||
- `--model`: Model name/path (should match the server model)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
|
||||
### FAQ
|
||||
|
||||
@@ -95,21 +95,61 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by:
|
||||
The Gradio demo connects to a vLLM API server. You have two options:
|
||||
|
||||
#### Option 1: One-step Launch Script (Recommended)
|
||||
|
||||
The convenience script launches both the vLLM server and Gradio demo together:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
|
||||
./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861
|
||||
```
|
||||
|
||||
This script will:
|
||||
1. Start the vLLM server in the background
|
||||
2. Wait for the server to be ready
|
||||
3. Launch the Gradio demo
|
||||
4. Handle cleanup when you press Ctrl+C
|
||||
|
||||
The script supports the following arguments:
|
||||
- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--server-host`: Host for vLLM server (default: 0.0.0.0)
|
||||
- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
|
||||
- `--share`: Share Gradio demo publicly (creates a public link)
|
||||
|
||||
#### Option 2: Manual Launch (Two-Step Process)
|
||||
|
||||
**Step 1: Launch the vLLM API server**
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
|
||||
In a separate terminal:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --api-base http://localhost:8091/v1 --port 7861
|
||||
```
|
||||
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name
|
||||
- `--model`: Model name/path (should match the server model)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
|
||||
## Example materials
|
||||
|
||||
@@ -84,20 +84,60 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
|
||||
The Gradio demo connects to a vLLM API server. You have two options:
|
||||
|
||||
#### Option 1: One-step Launch Script (Recommended)
|
||||
|
||||
The convenience script launches both the vLLM server and Gradio demo together:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861
|
||||
./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861
|
||||
```
|
||||
|
||||
This script will:
|
||||
1. Start the vLLM server in the background
|
||||
2. Wait for the server to be ready
|
||||
3. Launch the Gradio demo
|
||||
4. Handle cleanup when you press Ctrl+C
|
||||
|
||||
The script supports the following arguments:
|
||||
- `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--server-host`: Host for vLLM server (default: 0.0.0.0)
|
||||
- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
|
||||
- `--share`: Share Gradio demo publicly (creates a public link)
|
||||
|
||||
#### Option 2: Manual Launch (Two-Step Process)
|
||||
|
||||
**Step 1: Launch the vLLM API server**
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
|
||||
In a separate terminal:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --api-base http://localhost:8091/v1 --port 7861
|
||||
```
|
||||
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name
|
||||
- `--model`: Model name/path (should match the server model)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
|
||||
### FAQ
|
||||
|
||||
@@ -1,32 +1,19 @@
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import gradio as gr
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from openai import OpenAI
|
||||
from PIL import Image
|
||||
from vllm.assets.video import video_get_metadata, video_to_ndarrays
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
# Import utils from offline inference example
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni"))
|
||||
try:
|
||||
from utils import make_omni_prompt
|
||||
except ImportError:
|
||||
# Fallback if utils doesn't exist - we'll build prompts directly
|
||||
make_omni_prompt = None
|
||||
|
||||
SEED = 42
|
||||
ASYNC_INIT_TIMEOUT = 600
|
||||
|
||||
SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
|
||||
"Qwen/Qwen2.5-Omni-7B": {
|
||||
@@ -76,7 +63,12 @@ def parse_args():
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
help="Path to local model checkpoint.",
|
||||
help="Model name/path (should match the server model).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-base",
|
||||
default="http://localhost:8091/v1",
|
||||
help="Base URL for the vLLM API server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ip",
|
||||
@@ -85,80 +77,9 @@ def parse_args():
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.")
|
||||
parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to custom stage configs YAML file (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
help="Enable statistics logging for AsyncOmni.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path prefix for AsyncOmni log files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Seconds to sleep between starting stage processes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold in bytes for using shared memory IPC.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Batching timeout (seconds) inside each stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=ASYNC_INIT_TIMEOUT,
|
||||
help="Timeout (seconds) for initializing all stages.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ray-address",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Address of the Ray cluster.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
|
||||
"""Construct the minimal CLI args Namespace expected by AsyncOmni."""
|
||||
return argparse.Namespace(
|
||||
model=base_args.model,
|
||||
stage_configs_path=getattr(base_args, "stage_configs_path", None),
|
||||
log_stats=bool(getattr(base_args, "log_stats", False)),
|
||||
log_file=getattr(base_args, "log_file", None),
|
||||
init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
|
||||
shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
|
||||
batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
|
||||
init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
|
||||
worker_backend=getattr(base_args, "worker_backend", "multi_process"),
|
||||
ray_address=getattr(base_args, "ray_address", None),
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
|
||||
"""Build SamplingParams objects by reusing the dict definitions."""
|
||||
return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
|
||||
|
||||
|
||||
def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
|
||||
"""Build sampling params as dict for HTTP API mode."""
|
||||
model_conf = SUPPORTED_MODELS.get(model_key)
|
||||
@@ -174,15 +95,63 @@ def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
|
||||
return sampling_params
|
||||
|
||||
|
||||
def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace:
|
||||
# The prompt builder expects a minimal namespace with these attributes.
|
||||
return SimpleNamespace(
|
||||
model=base_args.model,
|
||||
prompt_type="text",
|
||||
tokenize=True,
|
||||
use_torchvision=True,
|
||||
legacy_omni_video=False,
|
||||
)
|
||||
def image_to_base64_data_url(image: Image.Image) -> str:
|
||||
"""Convert PIL Image to base64 data URL."""
|
||||
buffered = io.BytesIO()
|
||||
# Convert to RGB if needed
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
image.save(buffered, format="JPEG")
|
||||
img_bytes = buffered.getvalue()
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
return f"data:image/jpeg;base64,{img_b64}"
|
||||
|
||||
|
||||
def audio_to_base64_data_url(audio_data: tuple[np.ndarray, int]) -> str:
|
||||
"""Convert audio (numpy array, sample_rate) to base64 data URL."""
|
||||
audio_np, sample_rate = audio_data
|
||||
# Convert to int16 format for WAV
|
||||
if audio_np.dtype != np.int16:
|
||||
# Normalize to [-1, 1] range if needed
|
||||
if audio_np.dtype == np.float32 or audio_np.dtype == np.float64:
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
|
||||
# Write to WAV bytes
|
||||
buffered = io.BytesIO()
|
||||
sf.write(buffered, audio_np, sample_rate, format="WAV")
|
||||
wav_bytes = buffered.getvalue()
|
||||
wav_b64 = base64.b64encode(wav_bytes).decode("utf-8")
|
||||
return f"data:audio/wav;base64,{wav_b64}"
|
||||
|
||||
|
||||
def video_to_base64_data_url(video_file: str) -> str:
|
||||
"""Convert video file to base64 data URL."""
|
||||
video_path = Path(video_file)
|
||||
if not video_path.exists():
|
||||
raise FileNotFoundError(f"Video file not found: {video_file}")
|
||||
|
||||
# Detect MIME type from extension
|
||||
video_path_lower = str(video_path).lower()
|
||||
if video_path_lower.endswith(".mp4"):
|
||||
mime_type = "video/mp4"
|
||||
elif video_path_lower.endswith(".webm"):
|
||||
mime_type = "video/webm"
|
||||
elif video_path_lower.endswith(".mov"):
|
||||
mime_type = "video/quicktime"
|
||||
elif video_path_lower.endswith(".avi"):
|
||||
mime_type = "video/x-msvideo"
|
||||
elif video_path_lower.endswith(".mkv"):
|
||||
mime_type = "video/x-matroska"
|
||||
else:
|
||||
mime_type = "video/mp4"
|
||||
|
||||
with open(video_path, "rb") as f:
|
||||
video_bytes = f.read()
|
||||
video_b64 = base64.b64encode(video_bytes).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{video_b64}"
|
||||
|
||||
|
||||
def process_audio_file(
|
||||
@@ -262,44 +231,10 @@ def process_image_file(image_file: Image.Image | None) -> Image.Image | None:
|
||||
return image_file
|
||||
|
||||
|
||||
def process_video_file(
|
||||
video_file: str | None,
|
||||
enable_audio_in_video: bool = False,
|
||||
max_frames: int = 32,
|
||||
) -> tuple[np.ndarray, dict[str, Any], tuple[np.ndarray, int] | None] | None:
|
||||
"""Process video file and optionally extract audio track."""
|
||||
if video_file is None:
|
||||
return None
|
||||
|
||||
video_path = Path(video_file)
|
||||
if not video_path.exists():
|
||||
print(f"Video file not found: {video_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
frames = video_to_ndarrays(str(video_path), num_frames=max_frames)
|
||||
metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0])
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Failed to decode video {video_path}: {exc}")
|
||||
return None
|
||||
|
||||
audio_tuple: tuple[np.ndarray, int] | None = None
|
||||
if enable_audio_in_video:
|
||||
try:
|
||||
import librosa # type: ignore import
|
||||
|
||||
audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000)
|
||||
audio_tuple = (audio_signal.astype(np.float32), sampling_rate)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Failed to extract audio from video {video_path}: {exc}")
|
||||
|
||||
return frames, metadata, audio_tuple
|
||||
|
||||
|
||||
async def run_inference_async_omni(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
def run_inference_api(
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
sampling_params_dict: list[dict],
|
||||
user_prompt: str,
|
||||
audio_file: tuple[str, tuple[int, np.ndarray]] | None = None,
|
||||
image_file: Image.Image | None = None,
|
||||
@@ -307,118 +242,117 @@ async def run_inference_async_omni(
|
||||
use_audio_in_video: bool = False,
|
||||
output_modalities: str | None = None,
|
||||
):
|
||||
"""Run inference using AsyncOmni directly with multimodal support."""
|
||||
"""Run inference using OpenAI API client with multimodal support."""
|
||||
if not user_prompt.strip() and not audio_file and not image_file and not video_file:
|
||||
return "Please provide at least a text prompt or multimodal input.", None
|
||||
|
||||
if output_modalities is not None:
|
||||
output_modalities = output_modalities.split(",")
|
||||
|
||||
try:
|
||||
# Build prompt with multimodal data
|
||||
prompt_args = SimpleNamespace(**prompt_args_template.__dict__)
|
||||
|
||||
# Process multimodal inputs
|
||||
multi_modal_data = {}
|
||||
mm_processor_kwargs = {}
|
||||
# Build message content list
|
||||
content_list = []
|
||||
|
||||
# Process audio
|
||||
audio_data = process_audio_file(audio_file)
|
||||
if audio_data is not None:
|
||||
multi_modal_data["audio"] = audio_data
|
||||
audio_url = audio_to_base64_data_url(audio_data)
|
||||
content_list.append(
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_url},
|
||||
}
|
||||
)
|
||||
|
||||
# Process image
|
||||
image_data = process_image_file(image_file)
|
||||
if image_data is not None:
|
||||
multi_modal_data["image"] = image_data
|
||||
if image_file is not None:
|
||||
image_data = process_image_file(image_file)
|
||||
if image_data is not None:
|
||||
image_url = image_to_base64_data_url(image_data)
|
||||
content_list.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
}
|
||||
)
|
||||
|
||||
# Process video
|
||||
video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video)
|
||||
if video_payload is not None:
|
||||
video_frames, video_metadata, extracted_audio = video_payload
|
||||
video_entry: Any
|
||||
if video_metadata:
|
||||
video_entry = (video_frames, video_metadata)
|
||||
else:
|
||||
video_entry = video_frames
|
||||
multi_modal_data["video"] = video_entry
|
||||
if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data:
|
||||
multi_modal_data["audio"] = extracted_audio
|
||||
mm_processor_kwargs["use_audio_in_video"] = True
|
||||
|
||||
# Build the prompt input
|
||||
if make_omni_prompt is not None:
|
||||
omni_prompt = make_omni_prompt(prompt_args, user_prompt)
|
||||
# Add multimodal data if present
|
||||
if multi_modal_data:
|
||||
if isinstance(omni_prompt, dict):
|
||||
omni_prompt["multi_modal_data"] = multi_modal_data
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
else:
|
||||
# If make_omni_prompt returns a string, we need to wrap it
|
||||
omni_prompt = {
|
||||
"prompt": omni_prompt,
|
||||
"multi_modal_data": multi_modal_data,
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
else:
|
||||
# Fallback: build prompt directly
|
||||
default_system = (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
)
|
||||
prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
|
||||
if audio_data:
|
||||
prompt += "<|audio_bos|><|AUDIO|><|audio_eos|>"
|
||||
if image_data:
|
||||
prompt += "<|vision_bos|><|IMAGE|><|vision_eos|>"
|
||||
if video_payload is not None:
|
||||
prompt += "<|vision_bos|><|VIDEO|><|vision_eos|>"
|
||||
if user_prompt.strip():
|
||||
prompt += f"{user_prompt}"
|
||||
prompt += "<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
omni_prompt = {
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": multi_modal_data,
|
||||
mm_processor_kwargs = {}
|
||||
if video_file is not None:
|
||||
video_url = video_to_base64_data_url(video_file)
|
||||
video_content = {
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
if use_audio_in_video:
|
||||
video_content["video_url"]["num_frames"] = 32 # Default max frames
|
||||
mm_processor_kwargs["use_audio_in_video"] = True
|
||||
content_list.append(video_content)
|
||||
|
||||
request_id = "0"
|
||||
# Add text prompt
|
||||
if user_prompt.strip():
|
||||
content_list.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": user_prompt,
|
||||
}
|
||||
)
|
||||
|
||||
# Build messages
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": content_list,
|
||||
},
|
||||
]
|
||||
|
||||
# Build extra_body
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_dict,
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
extra_body["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
# Parse output modalities
|
||||
if output_modalities is not None:
|
||||
output_modalities_list = [m.strip() for m in output_modalities.split(",")]
|
||||
else:
|
||||
output_modalities_list = None
|
||||
|
||||
# Call API
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model,
|
||||
modalities=output_modalities_list,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
|
||||
# Extract outputs
|
||||
text_outputs: list[str] = []
|
||||
audio_output = None
|
||||
|
||||
async for stage_outputs in omni.generate(
|
||||
prompt=omni_prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params,
|
||||
output_modalities=output_modalities,
|
||||
):
|
||||
# stage_outputs.request_output is a RequestOutput object, not a list
|
||||
request_output = stage_outputs.request_output
|
||||
if stage_outputs.final_output_type == "text":
|
||||
if request_output.outputs:
|
||||
for output in request_output.outputs:
|
||||
if output.text:
|
||||
text_outputs.append(output.text)
|
||||
elif stage_outputs.final_output_type == "audio":
|
||||
# multimodal_output is on the RequestOutput object
|
||||
# See vllm_omni/entrypoints/openai/serving_chat.py:680 for reference
|
||||
if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
|
||||
if "audio" in request_output.multimodal_output:
|
||||
audio_tensor = request_output.multimodal_output["audio"]
|
||||
# Ensure audio is 1D (flatten if needed)
|
||||
if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
|
||||
audio_tensor = audio_tensor.flatten()
|
||||
audio_np = audio_tensor.detach().cpu().numpy()
|
||||
audio_output = (
|
||||
24000, # sampling rate in Hz
|
||||
audio_np,
|
||||
)
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.content:
|
||||
text_outputs.append(choice.message.content)
|
||||
if choice.message.audio:
|
||||
# Decode base64 audio
|
||||
audio_data = base64.b64decode(choice.message.audio.data)
|
||||
# Load audio from bytes
|
||||
audio_np, sample_rate = sf.read(io.BytesIO(audio_data))
|
||||
# Convert to mono if needed
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
audio_output = (int(sample_rate), audio_np.astype(np.float32))
|
||||
|
||||
text_response = "\n\n".join(text_outputs) if text_outputs else "No text output."
|
||||
return text_response, audio_output
|
||||
@@ -427,14 +361,13 @@ async def run_inference_async_omni(
|
||||
|
||||
|
||||
def build_interface(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
sampling_params_dict: list[dict],
|
||||
):
|
||||
"""Build Gradio interface for AsyncOmni mode."""
|
||||
"""Build Gradio interface for API server mode."""
|
||||
|
||||
async def run_inference(
|
||||
def run_inference(
|
||||
user_prompt: str,
|
||||
audio_file: tuple[str, tuple[int, np.ndarray]] | None,
|
||||
image_file: Image.Image | None,
|
||||
@@ -442,10 +375,10 @@ def build_interface(
|
||||
use_audio_in_video: bool,
|
||||
output_modalities: str | None = None,
|
||||
):
|
||||
return await run_inference_async_omni(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
return run_inference_api(
|
||||
client,
|
||||
model,
|
||||
sampling_params_dict,
|
||||
user_prompt,
|
||||
audio_file,
|
||||
image_file,
|
||||
@@ -550,41 +483,27 @@ def build_interface(
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
omni = None
|
||||
|
||||
model_name = "/".join(args.model.split("/")[-2:])
|
||||
assert model_name in SUPPORTED_MODELS, (
|
||||
f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}"
|
||||
)
|
||||
|
||||
# Register signal handlers for graceful shutdown
|
||||
def signal_handler(sig, frame):
|
||||
print("\nReceived interrupt signal, shutting down...")
|
||||
if omni is not None:
|
||||
try:
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during shutdown: {e}")
|
||||
sys.exit(0)
|
||||
# Initialize OpenAI client
|
||||
print(f"Connecting to API server at: {args.api_base}")
|
||||
client = OpenAI(
|
||||
api_key="EMPTY",
|
||||
base_url=args.api_base,
|
||||
)
|
||||
print("✓ Connected to API server")
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
print(f"Initializing AsyncOmni with model: {args.model}")
|
||||
if args.stage_configs_path:
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
cli_args = build_async_omni_cli_args(args)
|
||||
omni = AsyncOmni(model=args.model, cli_args=cli_args)
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
# Build sampling params
|
||||
sampling_params_dict = build_sampling_params_dict(SEED, model_name)
|
||||
|
||||
demo = build_interface(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
client,
|
||||
args.model,
|
||||
sampling_params_dict,
|
||||
)
|
||||
try:
|
||||
demo.launch(
|
||||
@@ -594,13 +513,6 @@ def main():
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("\nShutting down...")
|
||||
finally:
|
||||
# Cleanup
|
||||
if omni is not None:
|
||||
try:
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during cleanup: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
#!/bin/bash
|
||||
# Convenience script to launch both vLLM server and Gradio demo for Qwen2.5-Omni
|
||||
#
|
||||
# Usage:
|
||||
# ./run_gradio_demo.sh [OPTIONS]
|
||||
#
|
||||
# Example:
|
||||
# ./run_gradio_demo.sh --model Qwen/Qwen2.5-Omni-7B --server-port 8091 --gradio-port 7861
|
||||
|
||||
set -e
|
||||
|
||||
# Default values
|
||||
MODEL="Qwen/Qwen2.5-Omni-7B"
|
||||
SERVER_PORT=8091
|
||||
GRADIO_PORT=7861
|
||||
STAGE_CONFIGS_PATH=""
|
||||
SERVER_HOST="0.0.0.0"
|
||||
GRADIO_IP="127.0.0.1"
|
||||
GRADIO_SHARE=false
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--model)
|
||||
MODEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-port)
|
||||
SERVER_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--gradio-port)
|
||||
GRADIO_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--stage-configs-path)
|
||||
STAGE_CONFIGS_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-host)
|
||||
SERVER_HOST="$2"
|
||||
shift 2
|
||||
;;
|
||||
--gradio-ip)
|
||||
GRADIO_IP="$2"
|
||||
shift 2
|
||||
;;
|
||||
--share)
|
||||
GRADIO_SHARE=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --model MODEL Model name/path (default: Qwen/Qwen2.5-Omni-7B)"
|
||||
echo " --server-port PORT Port for vLLM server (default: 8091)"
|
||||
echo " --gradio-port PORT Port for Gradio demo (default: 7861)"
|
||||
echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)"
|
||||
echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)"
|
||||
echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)"
|
||||
echo " --share Share Gradio demo publicly"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
API_BASE="http://localhost:${SERVER_PORT}/v1"
|
||||
HEALTH_URL="http://localhost:${SERVER_PORT}/health"
|
||||
|
||||
echo "=========================================="
|
||||
echo "Starting vLLM-Omni Gradio Demo"
|
||||
echo "=========================================="
|
||||
echo "Model: $MODEL"
|
||||
echo "Server: http://${SERVER_HOST}:${SERVER_PORT}"
|
||||
echo "Gradio: http://${GRADIO_IP}:${GRADIO_PORT}"
|
||||
echo "=========================================="
|
||||
|
||||
# Build vLLM server command
|
||||
SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
|
||||
if [ -n "$STAGE_CONFIGS_PATH" ]; then
|
||||
SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
|
||||
fi
|
||||
|
||||
# Function to cleanup on exit
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Shutting down..."
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
echo "Stopping vLLM server (PID: $SERVER_PID)..."
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "$GRADIO_PID" ]; then
|
||||
echo "Stopping Gradio demo (PID: $GRADIO_PID)..."
|
||||
kill "$GRADIO_PID" 2>/dev/null || true
|
||||
wait "$GRADIO_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Cleanup complete"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Set up signal handlers
|
||||
trap cleanup SIGINT SIGTERM
|
||||
|
||||
# Start vLLM server with output shown in real-time and saved to log
|
||||
echo ""
|
||||
echo "Starting vLLM server..."
|
||||
LOG_FILE="/tmp/vllm_server_${SERVER_PORT}.log"
|
||||
"${SERVER_CMD[@]}" 2>&1 | tee "$LOG_FILE" &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Start a background process to monitor the log for startup completion
|
||||
STARTUP_COMPLETE=false
|
||||
TAIL_PID=""
|
||||
|
||||
# Function to cleanup tail process
|
||||
cleanup_tail() {
|
||||
if [ -n "$TAIL_PID" ]; then
|
||||
kill "$TAIL_PID" 2>/dev/null || true
|
||||
wait "$TAIL_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Wait for server to be ready by checking log output
|
||||
echo ""
|
||||
echo "Waiting for vLLM server to be ready (checking for 'Application startup complete' message)..."
|
||||
echo ""
|
||||
|
||||
# Monitor log file for startup completion message
|
||||
MAX_WAIT=300 # 5 minutes timeout as fallback
|
||||
ELAPSED=0
|
||||
|
||||
# Use a temporary file to track startup completion
|
||||
STARTUP_FLAG="/tmp/vllm_startup_flag_${SERVER_PORT}.tmp"
|
||||
rm -f "$STARTUP_FLAG"
|
||||
|
||||
# Start monitoring in background
|
||||
(
|
||||
tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG"
|
||||
) &
|
||||
TAIL_PID=$!
|
||||
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
# Check if startup flag file exists (startup complete)
|
||||
if [ -f "$STARTUP_FLAG" ]; then
|
||||
cleanup_tail
|
||||
echo ""
|
||||
echo "✓ vLLM server is ready!"
|
||||
STARTUP_COMPLETE=true
|
||||
break
|
||||
fi
|
||||
|
||||
# Check if server process is still running
|
||||
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
cleanup_tail
|
||||
echo ""
|
||||
echo "Error: vLLM server failed to start (process terminated)"
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
ELAPSED=$((ELAPSED + 1))
|
||||
done
|
||||
|
||||
cleanup_tail
|
||||
rm -f "$STARTUP_FLAG"
|
||||
|
||||
if [ "$STARTUP_COMPLETE" != "true" ]; then
|
||||
echo ""
|
||||
echo "Error: vLLM server did not complete startup within ${MAX_WAIT} seconds"
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start Gradio demo
|
||||
echo ""
|
||||
echo "Starting Gradio demo..."
|
||||
cd "$SCRIPT_DIR"
|
||||
GRADIO_CMD=("python" "gradio_demo.py" "--model" "$MODEL" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT")
|
||||
if [ "$GRADIO_SHARE" = true ]; then
|
||||
GRADIO_CMD+=("--share")
|
||||
fi
|
||||
|
||||
"${GRADIO_CMD[@]}" > /tmp/gradio_demo.log 2>&1 &
|
||||
GRADIO_PID=$!
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Both services are running!"
|
||||
echo "=========================================="
|
||||
echo "vLLM Server: http://${SERVER_HOST}:${SERVER_PORT}"
|
||||
echo "Gradio Demo: http://${GRADIO_IP}:${GRADIO_PORT}"
|
||||
echo ""
|
||||
echo "Press Ctrl+C to stop both services"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Wait for either process to exit
|
||||
wait $SERVER_PID $GRADIO_PID || true
|
||||
|
||||
cleanup
|
||||
@@ -92,19 +92,58 @@ This Web UI demo allows users to interact with the model through a web browser.
|
||||
|
||||
### Running Gradio Demo
|
||||
|
||||
Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by:
|
||||
The Gradio demo connects to a vLLM API server. You have two options:
|
||||
|
||||
#### Option 1: One-step Launch Script (Recommended)
|
||||
|
||||
The convenience script launches both the vLLM server and Gradio demo together:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
|
||||
./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861
|
||||
```
|
||||
|
||||
This script will:
|
||||
1. Start the vLLM server in the background
|
||||
2. Wait for the server to be ready
|
||||
3. Launch the Gradio demo
|
||||
4. Handle cleanup when you press Ctrl+C
|
||||
|
||||
The script supports the following arguments:
|
||||
- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--server-host`: Host for vLLM server (default: 0.0.0.0)
|
||||
- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
|
||||
- `--share`: Share Gradio demo publicly (creates a public link)
|
||||
|
||||
#### Option 2: Manual Launch (Two-Step Process)
|
||||
|
||||
**Step 1: Launch the vLLM API server**
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
|
||||
In a separate terminal:
|
||||
|
||||
```bash
|
||||
python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --api-base http://localhost:8091/v1 --port 7861
|
||||
```
|
||||
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name
|
||||
- `--model`: Model name/path (should match the server model)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
@@ -1,32 +1,19 @@
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import gradio as gr
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from openai import OpenAI
|
||||
from PIL import Image
|
||||
from vllm.assets.video import video_get_metadata, video_to_ndarrays
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
# Import utils from offline inference example
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen3_omni"))
|
||||
try:
|
||||
from utils import make_omni_prompt
|
||||
except ImportError:
|
||||
# Fallback if utils doesn't exist - we'll build prompts directly
|
||||
make_omni_prompt = None
|
||||
|
||||
SEED = 42
|
||||
ASYNC_INIT_TIMEOUT = 600
|
||||
|
||||
SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct": {
|
||||
@@ -79,7 +66,12 @@ def parse_args():
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
help="Path to local model checkpoint.",
|
||||
help="Model name/path (should match the server model).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-base",
|
||||
default="http://localhost:8091/v1",
|
||||
help="Base URL for the vLLM API server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ip",
|
||||
@@ -88,80 +80,9 @@ def parse_args():
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.")
|
||||
parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to custom stage configs YAML file (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
help="Enable statistics logging for AsyncOmni.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path prefix for AsyncOmni log files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Seconds to sleep between starting stage processes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold in bytes for using shared memory IPC.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Batching timeout (seconds) inside each stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=ASYNC_INIT_TIMEOUT,
|
||||
help="Timeout (seconds) for initializing all stages.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ray-address",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Address of the Ray cluster.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
|
||||
"""Construct the minimal CLI args Namespace expected by AsyncOmni."""
|
||||
return argparse.Namespace(
|
||||
model=base_args.model,
|
||||
stage_configs_path=getattr(base_args, "stage_configs_path", None),
|
||||
log_stats=bool(getattr(base_args, "log_stats", False)),
|
||||
log_file=getattr(base_args, "log_file", None),
|
||||
init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
|
||||
shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
|
||||
batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
|
||||
init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
|
||||
worker_backend=getattr(base_args, "worker_backend", "multi_process"),
|
||||
ray_address=getattr(base_args, "ray_address", None),
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
|
||||
"""Build SamplingParams objects by reusing the dict definitions."""
|
||||
return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
|
||||
|
||||
|
||||
def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
|
||||
"""Build sampling params as dict for HTTP API mode."""
|
||||
model_conf = SUPPORTED_MODELS.get(model_key)
|
||||
@@ -177,15 +98,63 @@ def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
|
||||
return sampling_params
|
||||
|
||||
|
||||
def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace:
|
||||
# The prompt builder expects a minimal namespace with these attributes.
|
||||
return SimpleNamespace(
|
||||
model=base_args.model,
|
||||
prompt_type="text",
|
||||
tokenize=True,
|
||||
use_torchvision=True,
|
||||
legacy_omni_video=False,
|
||||
)
|
||||
def image_to_base64_data_url(image: Image.Image) -> str:
|
||||
"""Convert PIL Image to base64 data URL."""
|
||||
buffered = io.BytesIO()
|
||||
# Convert to RGB if needed
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
image.save(buffered, format="JPEG")
|
||||
img_bytes = buffered.getvalue()
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
return f"data:image/jpeg;base64,{img_b64}"
|
||||
|
||||
|
||||
def audio_to_base64_data_url(audio_data: tuple[np.ndarray, int]) -> str:
|
||||
"""Convert audio (numpy array, sample_rate) to base64 data URL."""
|
||||
audio_np, sample_rate = audio_data
|
||||
# Convert to int16 format for WAV
|
||||
if audio_np.dtype != np.int16:
|
||||
# Normalize to [-1, 1] range if needed
|
||||
if audio_np.dtype == np.float32 or audio_np.dtype == np.float64:
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
|
||||
# Write to WAV bytes
|
||||
buffered = io.BytesIO()
|
||||
sf.write(buffered, audio_np, sample_rate, format="WAV")
|
||||
wav_bytes = buffered.getvalue()
|
||||
wav_b64 = base64.b64encode(wav_bytes).decode("utf-8")
|
||||
return f"data:audio/wav;base64,{wav_b64}"
|
||||
|
||||
|
||||
def video_to_base64_data_url(video_file: str) -> str:
|
||||
"""Convert video file to base64 data URL."""
|
||||
video_path = Path(video_file)
|
||||
if not video_path.exists():
|
||||
raise FileNotFoundError(f"Video file not found: {video_file}")
|
||||
|
||||
# Detect MIME type from extension
|
||||
video_path_lower = str(video_path).lower()
|
||||
if video_path_lower.endswith(".mp4"):
|
||||
mime_type = "video/mp4"
|
||||
elif video_path_lower.endswith(".webm"):
|
||||
mime_type = "video/webm"
|
||||
elif video_path_lower.endswith(".mov"):
|
||||
mime_type = "video/quicktime"
|
||||
elif video_path_lower.endswith(".avi"):
|
||||
mime_type = "video/x-msvideo"
|
||||
elif video_path_lower.endswith(".mkv"):
|
||||
mime_type = "video/x-matroska"
|
||||
else:
|
||||
mime_type = "video/mp4"
|
||||
|
||||
with open(video_path, "rb") as f:
|
||||
video_bytes = f.read()
|
||||
video_b64 = base64.b64encode(video_bytes).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{video_b64}"
|
||||
|
||||
|
||||
def process_audio_file(
|
||||
@@ -265,44 +234,10 @@ def process_image_file(image_file: Image.Image | None) -> Image.Image | None:
|
||||
return image_file
|
||||
|
||||
|
||||
def process_video_file(
|
||||
video_file: str | None,
|
||||
enable_audio_in_video: bool = False,
|
||||
max_frames: int = 32,
|
||||
) -> tuple[np.ndarray, dict[str, Any], tuple[np.ndarray, int] | None] | None:
|
||||
"""Process video file and optionally extract audio track."""
|
||||
if video_file is None:
|
||||
return None
|
||||
|
||||
video_path = Path(video_file)
|
||||
if not video_path.exists():
|
||||
print(f"Video file not found: {video_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
frames = video_to_ndarrays(str(video_path), num_frames=max_frames)
|
||||
metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0])
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Failed to decode video {video_path}: {exc}")
|
||||
return None
|
||||
|
||||
audio_tuple: tuple[np.ndarray, int] | None = None
|
||||
if enable_audio_in_video:
|
||||
try:
|
||||
import librosa # type: ignore import
|
||||
|
||||
audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000)
|
||||
audio_tuple = (audio_signal.astype(np.float32), sampling_rate)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Failed to extract audio from video {video_path}: {exc}")
|
||||
|
||||
return frames, metadata, audio_tuple
|
||||
|
||||
|
||||
async def run_inference_async_omni(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
def run_inference_api(
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
sampling_params_dict: list[dict],
|
||||
user_prompt: str,
|
||||
audio_file: tuple[str, tuple[int, np.ndarray]] | None = None,
|
||||
image_file: Image.Image | None = None,
|
||||
@@ -310,121 +245,117 @@ async def run_inference_async_omni(
|
||||
use_audio_in_video: bool = False,
|
||||
output_modalities: str | None = None,
|
||||
):
|
||||
"""Run inference using AsyncOmni directly with multimodal support."""
|
||||
"""Run inference using OpenAI API client with multimodal support."""
|
||||
if not user_prompt.strip() and not audio_file and not image_file and not video_file:
|
||||
return "Please provide at least a text prompt or multimodal input.", None
|
||||
|
||||
if output_modalities is not None:
|
||||
output_modalities = output_modalities.split(",")
|
||||
|
||||
try:
|
||||
# Build prompt with multimodal data
|
||||
prompt_args = SimpleNamespace(**prompt_args_template.__dict__)
|
||||
|
||||
# Process multimodal inputs
|
||||
multi_modal_data = {}
|
||||
mm_processor_kwargs = {}
|
||||
# Build message content list
|
||||
content_list = []
|
||||
|
||||
# Process audio
|
||||
audio_data = process_audio_file(audio_file)
|
||||
if audio_data is not None:
|
||||
multi_modal_data["audio"] = [audio_data] if not isinstance(audio_data, list) else audio_data
|
||||
audio_url = audio_to_base64_data_url(audio_data)
|
||||
content_list.append(
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_url},
|
||||
}
|
||||
)
|
||||
|
||||
# Process image
|
||||
image_data = process_image_file(image_file)
|
||||
if image_data is not None:
|
||||
multi_modal_data["image"] = image_data
|
||||
if image_file is not None:
|
||||
image_data = process_image_file(image_file)
|
||||
if image_data is not None:
|
||||
image_url = image_to_base64_data_url(image_data)
|
||||
content_list.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
}
|
||||
)
|
||||
|
||||
# Process video
|
||||
video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video)
|
||||
if video_payload is not None:
|
||||
video_frames, video_metadata, extracted_audio = video_payload
|
||||
video_entry: Any
|
||||
if video_metadata:
|
||||
video_entry = (video_frames, video_metadata)
|
||||
else:
|
||||
video_entry = video_frames
|
||||
multi_modal_data["video"] = video_entry
|
||||
if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data:
|
||||
multi_modal_data["audio"] = (
|
||||
[extracted_audio] if not isinstance(extracted_audio, list) else extracted_audio
|
||||
)
|
||||
mm_processor_kwargs["use_audio_in_video"] = True
|
||||
|
||||
# Build the prompt input
|
||||
if make_omni_prompt is not None:
|
||||
omni_prompt = make_omni_prompt(prompt_args, user_prompt)
|
||||
# Add multimodal data if present
|
||||
if multi_modal_data:
|
||||
if isinstance(omni_prompt, dict):
|
||||
omni_prompt["multi_modal_data"] = multi_modal_data
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
else:
|
||||
# If make_omni_prompt returns a string, we need to wrap it
|
||||
omni_prompt = {
|
||||
"prompt": omni_prompt,
|
||||
"multi_modal_data": multi_modal_data,
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
else:
|
||||
# Fallback: build prompt directly
|
||||
default_system = (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
)
|
||||
prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
|
||||
if audio_data:
|
||||
prompt += "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
if image_data:
|
||||
prompt += "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
if video_payload is not None:
|
||||
prompt += "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
if user_prompt.strip():
|
||||
prompt += f"{user_prompt}"
|
||||
prompt += "<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
omni_prompt = {
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": multi_modal_data,
|
||||
mm_processor_kwargs = {}
|
||||
if video_file is not None:
|
||||
video_url = video_to_base64_data_url(video_file)
|
||||
video_content = {
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
if use_audio_in_video:
|
||||
video_content["video_url"]["num_frames"] = 32 # Default max frames
|
||||
mm_processor_kwargs["use_audio_in_video"] = True
|
||||
content_list.append(video_content)
|
||||
|
||||
request_id = "0"
|
||||
# Add text prompt
|
||||
if user_prompt.strip():
|
||||
content_list.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": user_prompt,
|
||||
}
|
||||
)
|
||||
|
||||
# Build messages
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": content_list,
|
||||
},
|
||||
]
|
||||
|
||||
# Build extra_body
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_dict,
|
||||
}
|
||||
if mm_processor_kwargs:
|
||||
extra_body["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
# Parse output modalities
|
||||
if output_modalities is not None:
|
||||
output_modalities_list = [m.strip() for m in output_modalities.split(",")]
|
||||
else:
|
||||
output_modalities_list = None
|
||||
|
||||
# Call API
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=messages,
|
||||
model=model,
|
||||
modalities=output_modalities_list,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
|
||||
# Extract outputs
|
||||
text_outputs: list[str] = []
|
||||
audio_output = None
|
||||
|
||||
async for stage_outputs in omni.generate(
|
||||
prompt=omni_prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params,
|
||||
output_modalities=output_modalities,
|
||||
):
|
||||
# stage_outputs.request_output is a RequestOutput object, not a list
|
||||
request_output = stage_outputs.request_output
|
||||
if stage_outputs.final_output_type == "text":
|
||||
if request_output.outputs:
|
||||
for output in request_output.outputs:
|
||||
if output.text:
|
||||
text_outputs.append(output.text)
|
||||
elif stage_outputs.final_output_type == "audio":
|
||||
# multimodal_output is on the RequestOutput object
|
||||
# See vllm_omni/entrypoints/openai/serving_chat.py:733 for reference
|
||||
if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
|
||||
if "audio" in request_output.multimodal_output:
|
||||
audio_tensor = request_output.multimodal_output["audio"]
|
||||
# Ensure audio is 1D (flatten if needed)
|
||||
if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
|
||||
audio_tensor = audio_tensor.flatten()
|
||||
# Convert BFloat16 to Float32 before numpy conversion (numpy doesn't support BFloat16)
|
||||
audio_np = audio_tensor.float().detach().cpu().numpy()
|
||||
audio_output = (
|
||||
24000, # sampling rate in Hz
|
||||
audio_np,
|
||||
)
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.content:
|
||||
text_outputs.append(choice.message.content)
|
||||
if choice.message.audio:
|
||||
# Decode base64 audio
|
||||
audio_data = base64.b64decode(choice.message.audio.data)
|
||||
# Load audio from bytes
|
||||
audio_np, sample_rate = sf.read(io.BytesIO(audio_data))
|
||||
# Convert to mono if needed
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
audio_output = (int(sample_rate), audio_np.astype(np.float32))
|
||||
|
||||
text_response = "\n\n".join(text_outputs) if text_outputs else "No text output."
|
||||
return text_response, audio_output
|
||||
@@ -433,14 +364,13 @@ async def run_inference_async_omni(
|
||||
|
||||
|
||||
def build_interface(
|
||||
omni: AsyncOmni,
|
||||
sampling_params: list[SamplingParams],
|
||||
prompt_args_template: SimpleNamespace,
|
||||
client: OpenAI,
|
||||
model: str,
|
||||
sampling_params_dict: list[dict],
|
||||
):
|
||||
"""Build Gradio interface for AsyncOmni mode."""
|
||||
"""Build Gradio interface for API server mode."""
|
||||
|
||||
async def run_inference(
|
||||
def run_inference(
|
||||
user_prompt: str,
|
||||
audio_file: tuple[str, tuple[int, np.ndarray]] | None,
|
||||
image_file: Image.Image | None,
|
||||
@@ -448,10 +378,10 @@ def build_interface(
|
||||
use_audio_in_video: bool,
|
||||
output_modalities: str | None = None,
|
||||
):
|
||||
return await run_inference_async_omni(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
return run_inference_api(
|
||||
client,
|
||||
model,
|
||||
sampling_params_dict,
|
||||
user_prompt,
|
||||
audio_file,
|
||||
image_file,
|
||||
@@ -556,41 +486,27 @@ def build_interface(
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
omni = None
|
||||
|
||||
model_name = "/".join(args.model.split("/")[-2:])
|
||||
assert model_name in SUPPORTED_MODELS, (
|
||||
f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}"
|
||||
)
|
||||
|
||||
# Register signal handlers for graceful shutdown
|
||||
def signal_handler(sig, frame):
|
||||
print("\nReceived interrupt signal, shutting down...")
|
||||
if omni is not None:
|
||||
try:
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during shutdown: {e}")
|
||||
sys.exit(0)
|
||||
# Initialize OpenAI client
|
||||
print(f"Connecting to API server at: {args.api_base}")
|
||||
client = OpenAI(
|
||||
api_key="EMPTY",
|
||||
base_url=args.api_base,
|
||||
)
|
||||
print("✓ Connected to API server")
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
print(f"Initializing AsyncOmni with model: {args.model}")
|
||||
if args.stage_configs_path:
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
cli_args = build_async_omni_cli_args(args)
|
||||
omni = AsyncOmni(model=args.model, cli_args=cli_args)
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
# Build sampling params
|
||||
sampling_params_dict = build_sampling_params_dict(SEED, model_name)
|
||||
|
||||
demo = build_interface(
|
||||
omni,
|
||||
sampling_params,
|
||||
prompt_args_template,
|
||||
client,
|
||||
args.model,
|
||||
sampling_params_dict,
|
||||
)
|
||||
try:
|
||||
demo.launch(
|
||||
@@ -600,13 +516,6 @@ def main():
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
print("\nShutting down...")
|
||||
finally:
|
||||
# Cleanup
|
||||
if omni is not None:
|
||||
try:
|
||||
omni.shutdown()
|
||||
except Exception as e:
|
||||
print(f"Error during cleanup: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
#!/bin/bash
|
||||
# Convenience script to launch both vLLM server and Gradio demo for Qwen3-Omni
|
||||
#
|
||||
# Usage:
|
||||
# ./run_gradio_demo.sh [OPTIONS]
|
||||
#
|
||||
# Example:
|
||||
# ./run_gradio_demo.sh --model Qwen/Qwen3-Omni-30B-A3B-Instruct --server-port 8091 --gradio-port 7861
|
||||
|
||||
set -e
|
||||
|
||||
# Default values
|
||||
MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
SERVER_PORT=8091
|
||||
GRADIO_PORT=7861
|
||||
STAGE_CONFIGS_PATH=""
|
||||
SERVER_HOST="0.0.0.0"
|
||||
GRADIO_IP="127.0.0.1"
|
||||
GRADIO_SHARE=false
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--model)
|
||||
MODEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-port)
|
||||
SERVER_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--gradio-port)
|
||||
GRADIO_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--stage-configs-path)
|
||||
STAGE_CONFIGS_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-host)
|
||||
SERVER_HOST="$2"
|
||||
shift 2
|
||||
;;
|
||||
--gradio-ip)
|
||||
GRADIO_IP="$2"
|
||||
shift 2
|
||||
;;
|
||||
--share)
|
||||
GRADIO_SHARE=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --model MODEL Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)"
|
||||
echo " --server-port PORT Port for vLLM server (default: 8091)"
|
||||
echo " --gradio-port PORT Port for Gradio demo (default: 7861)"
|
||||
echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)"
|
||||
echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)"
|
||||
echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)"
|
||||
echo " --share Share Gradio demo publicly"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
API_BASE="http://localhost:${SERVER_PORT}/v1"
|
||||
HEALTH_URL="http://localhost:${SERVER_PORT}/health"
|
||||
|
||||
echo "=========================================="
|
||||
echo "Starting vLLM-Omni Gradio Demo"
|
||||
echo "=========================================="
|
||||
echo "Model: $MODEL"
|
||||
echo "Server: http://${SERVER_HOST}:${SERVER_PORT}"
|
||||
echo "Gradio: http://${GRADIO_IP}:${GRADIO_PORT}"
|
||||
echo "=========================================="
|
||||
|
||||
# Build vLLM server command
|
||||
SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
|
||||
if [ -n "$STAGE_CONFIGS_PATH" ]; then
|
||||
SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
|
||||
fi
|
||||
|
||||
# Function to cleanup on exit
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Shutting down..."
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
echo "Stopping vLLM server (PID: $SERVER_PID)..."
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "$GRADIO_PID" ]; then
|
||||
echo "Stopping Gradio demo (PID: $GRADIO_PID)..."
|
||||
kill "$GRADIO_PID" 2>/dev/null || true
|
||||
wait "$GRADIO_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Cleanup complete"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Set up signal handlers
|
||||
trap cleanup SIGINT SIGTERM
|
||||
|
||||
# Start vLLM server with output shown in real-time and saved to log
|
||||
echo ""
|
||||
echo "Starting vLLM server..."
|
||||
LOG_FILE="/tmp/vllm_server_${SERVER_PORT}.log"
|
||||
"${SERVER_CMD[@]}" 2>&1 | tee "$LOG_FILE" &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Start a background process to monitor the log for startup completion
|
||||
STARTUP_COMPLETE=false
|
||||
TAIL_PID=""
|
||||
|
||||
# Function to cleanup tail process
|
||||
cleanup_tail() {
|
||||
if [ -n "$TAIL_PID" ]; then
|
||||
kill "$TAIL_PID" 2>/dev/null || true
|
||||
wait "$TAIL_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Wait for server to be ready by checking log output
|
||||
echo ""
|
||||
echo "Waiting for vLLM server to be ready (checking for 'Application startup complete' message)..."
|
||||
echo ""
|
||||
|
||||
# Monitor log file for startup completion message
|
||||
MAX_WAIT=300 # 5 minutes timeout as fallback
|
||||
ELAPSED=0
|
||||
|
||||
# Use a temporary file to track startup completion
|
||||
STARTUP_FLAG="/tmp/vllm_startup_flag_${SERVER_PORT}.tmp"
|
||||
rm -f "$STARTUP_FLAG"
|
||||
|
||||
# Start monitoring in background
|
||||
(
|
||||
tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG"
|
||||
) &
|
||||
TAIL_PID=$!
|
||||
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
# Check if startup flag file exists (startup complete)
|
||||
if [ -f "$STARTUP_FLAG" ]; then
|
||||
cleanup_tail
|
||||
echo ""
|
||||
echo "✓ vLLM server is ready!"
|
||||
STARTUP_COMPLETE=true
|
||||
break
|
||||
fi
|
||||
|
||||
# Check if server process is still running
|
||||
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
cleanup_tail
|
||||
echo ""
|
||||
echo "Error: vLLM server failed to start (process terminated)"
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
ELAPSED=$((ELAPSED + 1))
|
||||
done
|
||||
|
||||
cleanup_tail
|
||||
rm -f "$STARTUP_FLAG"
|
||||
|
||||
if [ "$STARTUP_COMPLETE" != "true" ]; then
|
||||
echo ""
|
||||
echo "Error: vLLM server did not complete startup within ${MAX_WAIT} seconds"
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Start Gradio demo
|
||||
echo ""
|
||||
echo "Starting Gradio demo..."
|
||||
cd "$SCRIPT_DIR"
|
||||
GRADIO_CMD=("python" "gradio_demo.py" "--model" "$MODEL" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT")
|
||||
if [ "$GRADIO_SHARE" = true ]; then
|
||||
GRADIO_CMD+=("--share")
|
||||
fi
|
||||
|
||||
"${GRADIO_CMD[@]}" > /tmp/gradio_demo.log 2>&1 &
|
||||
GRADIO_PID=$!
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Both services are running!"
|
||||
echo "=========================================="
|
||||
echo "vLLM Server: http://${SERVER_HOST}:${SERVER_PORT}"
|
||||
echo "Gradio Demo: http://${GRADIO_IP}:${GRADIO_PORT}"
|
||||
echo ""
|
||||
echo "Press Ctrl+C to stop both services"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Wait for either process to exit
|
||||
wait $SERVER_PID $GRADIO_PID || true
|
||||
|
||||
cleanup
|
||||
Reference in new issue
Block a user