diff --git a/docs/mkdocs/hooks/generate_examples.py b/docs/mkdocs/hooks/generate_examples.py index bccaab6eb..37c2f8316 100644 --- a/docs/mkdocs/hooks/generate_examples.py +++ b/docs/mkdocs/hooks/generate_examples.py @@ -173,7 +173,7 @@ class Example: gh_file = gh_file.relative_to(ROOT_DIR) # Make GitHub URL - url = "https://github.com/vllm-project/vllm/" + url = "https://github.com/vllm-project/vllm-omni/" url += "tree/main" if self.path.is_dir() else "blob/main" gh_url = f"{url}/{gh_file}" @@ -183,7 +183,7 @@ class Example: def generate(self) -> str: content = f"# {self.title}\n\n" - url = "https://github.com/vllm-project/vllm/" + url = "https://github.com/vllm-project/vllm-omni/" url += "tree/main" if self.path.is_dir() else "blob/main" content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n" diff --git a/docs/user_guide/examples/offline_inference/qwen2_5_omni.md b/docs/user_guide/examples/offline_inference/qwen2_5_omni.md index e5effd7f6..e26a8fbe6 100644 --- a/docs/user_guide/examples/offline_inference/qwen2_5_omni.md +++ b/docs/user_guide/examples/offline_inference/qwen2_5_omni.md @@ -1,11 +1,11 @@ # Offline Example of vLLM-Omni for Qwen2.5-Omni -Source . +Source . ## 🛠️ Installation -Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md) +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) ## Run examples (Qwen2.5-Omni) ### Multiple Prompts diff --git a/docs/user_guide/examples/offline_inference/qwen3_omni.md b/docs/user_guide/examples/offline_inference/qwen3_omni.md index d32a3445f..aaeeb71b5 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_omni.md +++ b/docs/user_guide/examples/offline_inference/qwen3_omni.md @@ -1,15 +1,15 @@ # Offline Example of vLLM-Omni for Qwen3-Omni -Source . +Source . ## 🛠️ Installation -Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md) +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) ## Run examples (Qwen3-Omni) ### Multiple Prompts -Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen2_5_omni/README.md) +Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni/README.md) Get into the example folder ```bash cd examples/offline_inference/qwen3_omni diff --git a/docs/user_guide/examples/offline_inference/qwen_image.md b/docs/user_guide/examples/offline_inference/qwen_image.md index 662a23800..575da435a 100644 --- a/docs/user_guide/examples/offline_inference/qwen_image.md +++ b/docs/user_guide/examples/offline_inference/qwen_image.md @@ -1,6 +1,6 @@ # Qwen-Image Offline Inference -Source . +Source . This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni: diff --git a/docs/user_guide/examples/online_serving/qwen2_5_omni.md b/docs/user_guide/examples/online_serving/qwen2_5_omni.md index 56abc11f5..b9d9a2f19 100644 --- a/docs/user_guide/examples/online_serving/qwen2_5_omni.md +++ b/docs/user_guide/examples/online_serving/qwen2_5_omni.md @@ -1,15 +1,16 @@ # Online serving Example of vLLM-Omni for Qwen2.5-Omni -Source . +Source . ## 🛠️ Installation -Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md) +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) ## Run examples (Qwen2.5-Omni) -Launch the server +### Launch the Server + ```bash vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 ``` @@ -19,29 +20,59 @@ If you have custom stage configs file, launch the server with command below vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file ``` +### Send Multi-modal Request + Get into the example folder ```bash cd examples/online_serving/qwen2_5_omni ``` -Send request via python +#### Send request via python + ```bash python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities ``` -Send request via curl +The Python client supports the following command-line arguments: + +- `--query-type` (or `-q`): Query type (default: `mixed_modalities`) + - Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text` +- `--video-path` (or `-v`): Path to local video file or URL + - If not provided and query-type uses video, uses default video URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4` +- `--image-path` (or `-i`): Path to local image file or URL + - If not provided and query-type uses image, uses default image URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Supports common image formats: JPEG, PNG, GIF, WebP + - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png` +- `--audio-path` (or `-a`): Path to local audio file or URL + - If not provided and query-type uses audio, uses default audio URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A + - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3` +- `--prompt` (or `-p`): Custom text prompt/question + - If not provided, uses default prompt for the selected query type + - Example: `--prompt "What are the main activities shown in this video?"` + + +For example, to use mixed modalities with all local files: + +```bash +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type mixed_modalities \ + --video-path /path/to/your/video.mp4 \ + --image-path /path/to/your/image.jpg \ + --audio-path /path/to/your/audio.wav \ + --prompt "Analyze all the media content and provide a comprehensive summary." +``` + +#### Send request via curl + ```bash bash run_curl_multimodal_generation.sh mixed_modalities ``` -### FAQ - -If you encounter error about backend of librosa, try to install ffmpeg with command below. -``` -sudo apt update -sudo apt install ffmpeg -``` - ## Run Local Web UI Demo This Web UI demo allows users to interact with the model through a web browser. @@ -56,24 +87,21 @@ python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861 Then open `http://localhost:7861/` on your local browser to interact with the web UI. +The gradio script supports the following arguments: -### Options +- `--model`: Model name +- `--ip`: Host/IP for Gradio server (default: 127.0.0.1) +- `--port`: Port for Gradio server (default: 7861) +- `--stage-configs-path`: Path to custom stage configs YAML file (optional) +- `--share`: Share the Gradio demo publicly (creates a public link) -You can customize its basic launch parameters: +### FAQ -```bash -python gradio_demo.py \ - --model Qwen/Qwen2.5-Omni-7B \ - --ip 127.0.0.1 \ - --port 7861 \ - --stage-configs-path /path/to/stage_configs.yaml +If you encounter error about backend of librosa, try to install ffmpeg with command below. +``` +sudo apt update +sudo apt install ffmpeg ``` - -- `--model`: Local model checkpoint to load (default `Qwen/Qwen2.5-Omni-7B`). -- `--ip`: Host/IP for the Gradio server (default `127.0.0.1`). -- `--port`: Port for the Gradio server (default `7861`). -- `--stage-configs-path`: Optional path to custom stage configs YAML. -- `--share`: Set to expose a temporary public link via Gradio. ## Example materials diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index 975ed54da..6e805abbf 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -1,15 +1,16 @@ # Online serving Example of vLLM-Omni for Qwen3-omni -Source . +Source . ## 🛠️ Installation -Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md) +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) ## Run examples (Qwen3-Omni) -Launch the server +### Launch the Server + ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` @@ -19,21 +20,59 @@ If you have custom stage configs file, launch the server with command below vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file ``` +### Send Multi-modal Request + Get into the example folder ```bash cd examples/online_serving/qwen3_omni ``` -Send request via python +#### Send request via python + ```bash python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image ``` -Send request via curl +The Python client supports the following command-line arguments: + +- `--query-type` (or `-q`): Query type (default: `use_video`) + - Options: `text`, `use_audio`, `use_image`, `use_video` +- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`) +- `--video-path` (or `-v`): Path to local video file or URL + - If not provided and query-type is `use_video`, uses default video URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4` +- `--image-path` (or `-i`): Path to local image file or URL + - If not provided and query-type is `use_image`, uses default image URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Supports common image formats: JPEG, PNG, GIF, WebP + - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png` +- `--audio-path` (or `-a`): Path to local audio file or URL + - If not provided and query-type is `use_audio`, uses default audio URL + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs + - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A + - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3` +- `--prompt` (or `-p`): Custom text prompt/question + - If not provided, uses default prompt for the selected query type + - Example: `--prompt "What are the main activities shown in this video?"` + + +For example, to use a local video file with custom prompt: + +```bash +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type use_video \ + --video-path /path/to/your/video.mp4 \ + --prompt "What are the main activities shown in this video?" +``` + +#### Send request via curl + ```bash bash run_curl_multimodal_generation.sh use_image ``` + ### FAQ If you encounter error about backend of librosa, try to install ffmpeg with command below. @@ -57,23 +96,9 @@ python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861 Then open `http://localhost:7861/` on your local browser to interact with the web UI. -### Options - -The gradio demo also supports running with an existing API server and can be customized with the following arguments. - - -```bash -python gradio_demo.py \ - --model Qwen/Qwen3-Omni-30B-A3B-Instruct \ - --use-api-server \ - --api-base http://localhost:8091/v1 \ - --ip 127.0.0.1 \ - --port 7861 -``` +The gradio script supports the following arguments: - `--model`: Model name -- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmni locally. -- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1) - `--ip`: Host/IP for Gradio server (default: 127.0.0.1) - `--port`: Port for Gradio server (default: 7861) - `--stage-configs-path`: Path to custom stage configs YAML file (optional)