[Feature] Support online inference (#64)

Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
Gao Han authored and GitHub committed 2025-11-17 14:44:10 +08:00
1 parent bf82e18374
commit c5a89302f1
52 files changed
+3666 -685

No files matched your search

@@ -0,0 +1,34 @@
# Offline Example of vLLM-omni for Qwen2.5-omni
## 🛠️ Installation
Please refer to [README.md](../../../README.md)
## Run examples (Qwen2.5-omni)
### Multiple Prompts
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
```bash
tar -xf <Your Download Path>/seedtts_testset.tar
cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
--input examples/offline_inference/qwen2_5_omni/meta.lst \
--output examples/offline_inference/qwen2_5_omni/top100.txt \
--topk 100
```
Get into the example folder
```bash
cd examples/offline_inference/qwen2_5_omni
```
Then run the command below.
```bash
bash run_multiple_prompts.sh
```
### Single Prompts
Get into the example folder
```bash
cd examples/offline_inference/qwen2_5_omni
```
Then run the command below.
```bash
bash run_single_prompt.sh
```
@@ -7,8 +7,8 @@ import numpy as np
import soundfile as sf
import torch
from utils import make_omni_prompt
from vllm.sampling_params import SamplingParams
from vllm.sampling_params import SamplingParams
from vllm_omni.entrypoints.omni_llm import OmniLLM
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
@@ -109,10 +109,14 @@ def parse_args():
parser.add_argument("--use-torchvision", action="store_true")
parser.add_argument("--tokenize", action="store_true")
parser.add_argument(
"--output-wav", default="output.wav", help="[Deprecated] Output wav directory (use --output-dir)."
"--output-wav",
default="output.wav",
help="[Deprecated] Output wav directory (use --output-dir).",
)
parser.add_argument(
"--output-dir", default="outputs", help="Output directory to save text and wav files together."
"--output-dir",
default="outputs",
help="Output directory to save text and wav files together.",
)
parser.add_argument(
"--thinker-hidden-states-dir",
@@ -168,7 +172,9 @@ def main():
raise
if args.prompts is None:
raise ValueError("No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)")
raise ValueError(
"No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)"
)
omni_llm = OmniLLM(
model=model_name,
log_stats=args.enable_stats,
@@ -217,7 +223,9 @@ def main():
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
# Determine output directory: prefer --output-dir; fallback to --output-wav
output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
output_dir = (
args.output_dir if getattr(args, "output_dir", None) else args.output_wav
)
os.makedirs(output_dir, exist_ok=True)
for stage_outputs in omni_outputs:
if stage_outputs.final_output_type == "text":
@@ -1,26 +1,30 @@
#!/usr/bin/env python3
import argparse
from typing import Optional
import os
import torch
def extract_prompt(line: str) -> Optional[str]:
# 提取第一个 '|' 与第二个 '|' 之间的内容
i = line.find('|')
i = line.find("|")
if i == -1:
return None
j = line.find('|', i + 1)
j = line.find("|", i + 1)
if j == -1:
return None
return line[i + 1:j].strip()
return line[i + 1 : j].strip()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--input", "-i", required=True, help="Input .lst file path")
parser.add_argument("--output", "-o", required=True, help="Output file path")
parser.add_argument("--topk", "-k", type=int, default=100, help="Extract the top K prompts (default: 100)")
parser.add_argument(
"--topk",
"-k",
type=int,
default=100,
help="Extract the top K prompts (default: 100)",
)
args = parser.parse_args()
prompts = []
@@ -39,5 +43,3 @@ def main() -> None:
if __name__ == "__main__":
main()
@@ -1,4 +1,3 @@
export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
--voice-type "m02" \
--dit-ckpt none \
@@ -1,4 +1,3 @@
export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
--voice-type "m02" \
--dit-ckpt none \
@@ -9,6 +9,7 @@ import soundfile as sf
import torch
import torchvision.io
from processing_omni import fetch_image, fetch_video
from transformers import AutoConfig, AutoProcessor
from vllm.inputs import TextPrompt
from vllm_omni.inputs.data import OmniTokensPrompt
@@ -1,92 +0,0 @@
# Offline Example of vLLM-omni for Qwen2.5-omni
## 🛠️ Installation
### Environment setup
Use Docker to keep consistent basic environment (Optional, Recommended)
```bash
docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
```
Set up basic uv environment
```bash
pip install uv
uv venv --python 3.12 --seed
source .venv/bin/activate
```
### Installation of vLLM (for users)
Now we build it based on vLLM v0.11.0. Please install it with command below.
```bash
uv pip install vllm==0.11.0 --torch-backend=auto
```
### Installation of vLLM (for developers)
If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
```bash
git clone https://github.com/vllm-project/vllm.git
cd vllm
git checkout v0.11.0
```
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
```bash
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
```
Install vllm with command below (If you have no existing PyTorch).
```bash
uv pip install --editable .
```
Install vllm with command below (If you already have PyTorch).
```bash
python use_existing_torch.py
uv pip install -r requirements/build.txt
uv pip install --no-build-isolation --editable .
```
### Verification for successful installation of vLLM
Just run the command below. If no error, it demonstrates that the installation is successfull.
```bash
python -c "import vllm._C"
```
### Installation of vLLM-omni
Install additional requirements for vllm-omni
```bash
git clone https://github.com/vllm-project/vllm-omni.git
cd vllm_omni
uv pip install -e .
```
## Run examples (Qwen2.5-omni)
### Multiple Prompts
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
```bash
tar -xf <Your Download Path>/seedtts_testset.tar
cp seedtts_testset/en/meta.lst examples/offline_inference/qwen_2_5_omni/meta.lst
python3 examples/offline_inference/qwen_2_5_omni/extract_prompts.py \
--input examples/offline_inference/qwen_2_5_omni/meta.lst \
--output examples/offline_inference/qwen_2_5_omni/top100.txt \
--topk 100
```
Get into the example folder
```bash
cd examples/offline_inference/qwen_2_5_omni
```
Modify PYTHONPATH in run_multiple_prompts.sh as your path of vllm_omni. Then run.
```bash
bash run_multiple_prompts.sh
```
### Single Prompts
Get into the example folder
```bash
cd examples/offline_inference/qwen_2_5_omni
```
Modify PYTHONPATH in run_single_prompt.sh as your path of vllm_omni. Then run.
```bash
bash run_single_prompt.sh
```
+32
View File
@@ -0,0 +1,32 @@
# Online serving Example of vLLM-omni for Qwen2.5-omni
## 🛠️ Installation
Please refer to [README.md](../../README.md)
## Run examples (Qwen2.5-omni)
Launch the server
```bash
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
```
If you have custom stage configs file, launch the server with command below
```bash
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
```
Get into the example folder
```bash
cd examples/online_serving
```
Send request via python
```bash
python openai_chat_completion_client_for_multimodal_generation.py
```
Send request via curl
```bash
bash run_curl_multimodal_generation.sh
```
@@ -0,0 +1,104 @@
import base64
from openai import OpenAI
# Modify OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8091/v1"
client = OpenAI(
# defaults to os.environ.get("OPENAI_API_KEY")
api_key=openai_api_key,
base_url=openai_api_base,
)
SEED = 42
def get_system_prompt():
return {
"role": "system",
"content": [
{
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual inputs, "
"as well as generating text and speech."
),
}
],
}
def run_text_to_audio(model: str) -> None:
thinker_sampling_params = {
"temperature": 0.0, # Deterministic - no randomness
"top_p": 1.0, # Disable nucleus sampling
"top_k": -1, # Disable top-k sampling
"max_tokens": 2048,
"seed": SEED, # Fixed seed for sampling
"detokenize": True,
"repetition_penalty": 1.1,
}
talker_sampling_params = {
"temperature": 0.9,
"top_p": 0.8,
"top_k": 40,
"max_tokens": 2048,
"seed": SEED, # Fixed seed for sampling
"detokenize": True,
"repetition_penalty": 1.05,
"stop_token_ids": [8294],
}
code2wav_sampling_params = {
"temperature": 0.0, # Deterministic - no randomness
"top_p": 1.0, # Disable nucleus sampling
"top_k": -1, # Disable top-k sampling
"max_tokens": 2048,
"seed": SEED, # Fixed seed for sampling
"detokenize": True,
"repetition_penalty": 1.1,
}
sampling_params_list = [
thinker_sampling_params,
talker_sampling_params,
code2wav_sampling_params,
]
chat_completion = client.chat.completions.create(
messages=[
get_system_prompt(),
{
"role": "user",
"content": [
{
"type": "text",
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words.",
},
],
},
],
model=model,
extra_body={
"sampling_params_list": sampling_params_list
}, # Optional, it has a default setting in stage_configs of the corresponding model.
)
count = 0
for choice in chat_completion.choices:
if choice.message.audio:
audio_data = base64.b64decode(choice.message.audio.data)
audio_file_path = f"audio_{count}.wav"
with open(audio_file_path, "wb") as f:
f.write(audio_data)
print(f"Audio saved to {audio_file_path}")
count += 1
elif choice.message.content:
print("Chat completion output from text:", choice.message.content)
if __name__ == "__main__":
run_text_to_audio("Qwen/Qwen2.5-Omni-7B")
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -euo pipefail
SEED=42
thinker_sampling_params='{
"temperature": 0.0,
"top_p": 1.0,
"top_k": -1,
"max_tokens": 2048,
"seed": 42,
"detokenize": true,
"repetition_penalty": 1.1
}'
talker_sampling_params='{
"temperature": 0.9,
"top_p": 0.8,
"top_k": 40,
"max_tokens": 2048,
"seed": 42,
"detokenize": true,
"repetition_penalty": 1.05,
"stop_token_ids": [8294]
}'
code2wav_sampling_params='{
"temperature": 0.0,
"top_p": 1.0,
"top_k": -1,
"max_tokens": 2048,
"seed": 42,
"detokenize": true,
"repetition_penalty": 1.1
}'
# Above is optional, it has a default setting in stage_configs of the corresponding model.
output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
-H "Content-Type: application/json" \
-d @- <<EOF
{
"model": "Qwen/Qwen2.5-Omni-7B",
"extra_body": {
"sampling_params_list": [
$thinker_sampling_params,
$talker_sampling_params,
$code2wav_sampling_params
]
},
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
}
]
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
}
]
}
]
}
EOF
)
# Here it only shows the text content of the first choice. Audio content has many binaries, so it's not displayed here.
echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"