[Feature] Support online inference (#64)
Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
1 parent
bf82e18374
commit
c5a89302f1
52 files changed
+3666
-685
No files matched your search
@@ -0,0 +1,34 @@
|
||||
# Offline Example of vLLM-omni for Qwen2.5-omni
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](../../../README.md)
|
||||
|
||||
## Run examples (Qwen2.5-omni)
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
|
||||
```bash
|
||||
tar -xf <Your Download Path>/seedtts_testset.tar
|
||||
cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
|
||||
python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
|
||||
--input examples/offline_inference/qwen2_5_omni/meta.lst \
|
||||
--output examples/offline_inference/qwen2_5_omni/top100.txt \
|
||||
--topk 100
|
||||
```
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen2_5_omni
|
||||
```
|
||||
Then run the command below.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
### Single Prompts
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen2_5_omni
|
||||
```
|
||||
Then run the command below.
|
||||
```bash
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
+13
-5
@@ -7,8 +7,8 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from utils import make_omni_prompt
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
|
||||
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
|
||||
@@ -109,10 +109,14 @@ def parse_args():
|
||||
parser.add_argument("--use-torchvision", action="store_true")
|
||||
parser.add_argument("--tokenize", action="store_true")
|
||||
parser.add_argument(
|
||||
"--output-wav", default="output.wav", help="[Deprecated] Output wav directory (use --output-dir)."
|
||||
"--output-wav",
|
||||
default="output.wav",
|
||||
help="[Deprecated] Output wav directory (use --output-dir).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", default="outputs", help="Output directory to save text and wav files together."
|
||||
"--output-dir",
|
||||
default="outputs",
|
||||
help="Output directory to save text and wav files together.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--thinker-hidden-states-dir",
|
||||
@@ -168,7 +172,9 @@ def main():
|
||||
raise
|
||||
|
||||
if args.prompts is None:
|
||||
raise ValueError("No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)")
|
||||
raise ValueError(
|
||||
"No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)"
|
||||
)
|
||||
omni_llm = OmniLLM(
|
||||
model=model_name,
|
||||
log_stats=args.enable_stats,
|
||||
@@ -217,7 +223,9 @@ def main():
|
||||
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
|
||||
|
||||
# Determine output directory: prefer --output-dir; fallback to --output-wav
|
||||
output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
|
||||
output_dir = (
|
||||
args.output_dir if getattr(args, "output_dir", None) else args.output_wav
|
||||
)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
for stage_outputs in omni_outputs:
|
||||
if stage_outputs.final_output_type == "text":
|
||||
+10
-8
@@ -1,26 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from typing import Optional
|
||||
import os
|
||||
import torch
|
||||
|
||||
|
||||
def extract_prompt(line: str) -> Optional[str]:
|
||||
# 提取第一个 '|' 与第二个 '|' 之间的内容
|
||||
i = line.find('|')
|
||||
i = line.find("|")
|
||||
if i == -1:
|
||||
return None
|
||||
j = line.find('|', i + 1)
|
||||
j = line.find("|", i + 1)
|
||||
if j == -1:
|
||||
return None
|
||||
return line[i + 1:j].strip()
|
||||
return line[i + 1 : j].strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", "-i", required=True, help="Input .lst file path")
|
||||
parser.add_argument("--output", "-o", required=True, help="Output file path")
|
||||
parser.add_argument("--topk", "-k", type=int, default=100, help="Extract the top K prompts (default: 100)")
|
||||
parser.add_argument(
|
||||
"--topk",
|
||||
"-k",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Extract the top K prompts (default: 100)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
prompts = []
|
||||
@@ -39,5 +43,3 @@ def main() -> None:
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
File renamed without changes.
-1
@@ -1,4 +1,3 @@
|
||||
export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH
|
||||
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
|
||||
--voice-type "m02" \
|
||||
--dit-ckpt none \
|
||||
-1
@@ -1,4 +1,3 @@
|
||||
export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH
|
||||
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
|
||||
--voice-type "m02" \
|
||||
--dit-ckpt none \
|
||||
+1
@@ -9,6 +9,7 @@ import soundfile as sf
|
||||
import torch
|
||||
import torchvision.io
|
||||
from processing_omni import fetch_image, fetch_video
|
||||
from transformers import AutoConfig, AutoProcessor
|
||||
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
@@ -1,92 +0,0 @@
|
||||
# Offline Example of vLLM-omni for Qwen2.5-omni
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
### Environment setup
|
||||
|
||||
Use Docker to keep consistent basic environment (Optional, Recommended)
|
||||
```bash
|
||||
docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
|
||||
```
|
||||
|
||||
Set up basic uv environment
|
||||
```bash
|
||||
pip install uv
|
||||
uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
```
|
||||
|
||||
### Installation of vLLM (for users)
|
||||
|
||||
Now we build it based on vLLM v0.11.0. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.11.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
### Installation of vLLM (for developers)
|
||||
|
||||
If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.11.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
|
||||
```bash
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
uv pip install --editable .
|
||||
```
|
||||
|
||||
Install vllm with command below (If you already have PyTorch).
|
||||
```bash
|
||||
python use_existing_torch.py
|
||||
uv pip install -r requirements/build.txt
|
||||
uv pip install --no-build-isolation --editable .
|
||||
```
|
||||
|
||||
### Verification for successful installation of vLLM
|
||||
Just run the command below. If no error, it demonstrates that the installation is successfull.
|
||||
```bash
|
||||
python -c "import vllm._C"
|
||||
```
|
||||
|
||||
### Installation of vLLM-omni
|
||||
Install additional requirements for vllm-omni
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm_omni
|
||||
uv pip install -e .
|
||||
```
|
||||
|
||||
## Run examples (Qwen2.5-omni)
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
|
||||
```bash
|
||||
tar -xf <Your Download Path>/seedtts_testset.tar
|
||||
cp seedtts_testset/en/meta.lst examples/offline_inference/qwen_2_5_omni/meta.lst
|
||||
python3 examples/offline_inference/qwen_2_5_omni/extract_prompts.py \
|
||||
--input examples/offline_inference/qwen_2_5_omni/meta.lst \
|
||||
--output examples/offline_inference/qwen_2_5_omni/top100.txt \
|
||||
--topk 100
|
||||
```
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen_2_5_omni
|
||||
```
|
||||
Modify PYTHONPATH in run_multiple_prompts.sh as your path of vllm_omni. Then run.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
### Single Prompts
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen_2_5_omni
|
||||
```
|
||||
Modify PYTHONPATH in run_single_prompt.sh as your path of vllm_omni. Then run.
|
||||
```bash
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
@@ -0,0 +1,32 @@
|
||||
# Online serving Example of vLLM-omni for Qwen2.5-omni
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](../../README.md)
|
||||
|
||||
## Run examples (Qwen2.5-omni)
|
||||
|
||||
Launch the server
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file, launch the server with command below
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/online_serving
|
||||
```
|
||||
|
||||
Send request via python
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py
|
||||
```
|
||||
|
||||
Send request via curl
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh
|
||||
```
|
||||
@@ -0,0 +1,104 @@
|
||||
import base64
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
openai_api_base = "http://localhost:8091/v1"
|
||||
|
||||
client = OpenAI(
|
||||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||||
api_key=openai_api_key,
|
||||
base_url=openai_api_base,
|
||||
)
|
||||
|
||||
SEED = 42
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_text_to_audio(model: str) -> None:
|
||||
thinker_sampling_params = {
|
||||
"temperature": 0.0, # Deterministic - no randomness
|
||||
"top_p": 1.0, # Disable nucleus sampling
|
||||
"top_k": -1, # Disable top-k sampling
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.1,
|
||||
}
|
||||
talker_sampling_params = {
|
||||
"temperature": 0.9,
|
||||
"top_p": 0.8,
|
||||
"top_k": 40,
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.05,
|
||||
"stop_token_ids": [8294],
|
||||
}
|
||||
code2wav_sampling_params = {
|
||||
"temperature": 0.0, # Deterministic - no randomness
|
||||
"top_p": 1.0, # Disable nucleus sampling
|
||||
"top_k": -1, # Disable top-k sampling
|
||||
"max_tokens": 2048,
|
||||
"seed": SEED, # Fixed seed for sampling
|
||||
"detokenize": True,
|
||||
"repetition_penalty": 1.1,
|
||||
}
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
talker_sampling_params,
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=[
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
model=model,
|
||||
extra_body={
|
||||
"sampling_params_list": sampling_params_list
|
||||
}, # Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
)
|
||||
|
||||
count = 0
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.audio:
|
||||
audio_data = base64.b64decode(choice.message.audio.data)
|
||||
audio_file_path = f"audio_{count}.wav"
|
||||
with open(audio_file_path, "wb") as f:
|
||||
f.write(audio_data)
|
||||
print(f"Audio saved to {audio_file_path}")
|
||||
count += 1
|
||||
elif choice.message.content:
|
||||
print("Chat completion output from text:", choice.message.content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_text_to_audio("Qwen/Qwen2.5-Omni-7B")
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SEED=42
|
||||
|
||||
thinker_sampling_params='{
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"top_k": -1,
|
||||
"max_tokens": 2048,
|
||||
"seed": 42,
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.1
|
||||
}'
|
||||
|
||||
talker_sampling_params='{
|
||||
"temperature": 0.9,
|
||||
"top_p": 0.8,
|
||||
"top_k": 40,
|
||||
"max_tokens": 2048,
|
||||
"seed": 42,
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.05,
|
||||
"stop_token_ids": [8294]
|
||||
}'
|
||||
|
||||
code2wav_sampling_params='{
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"top_k": -1,
|
||||
"max_tokens": 2048,
|
||||
"seed": 42,
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.1
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
|
||||
output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- <<EOF
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-Omni-7B",
|
||||
"extra_body": {
|
||||
"sampling_params_list": [
|
||||
$thinker_sampling_params,
|
||||
$talker_sampling_params,
|
||||
$code2wav_sampling_params
|
||||
]
|
||||
},
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
# Here it only shows the text content of the first choice. Audio content has many binaries, so it's not displayed here.
|
||||
echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"
|
||||
Reference in new issue
Block a user