[Feature] support multimodal inputs with multiple requests (#76)
Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
1 parent
08faf16f81
commit
598b4ddb51
23 files changed
+1461
-985
No files matched your search
@@ -32,3 +32,11 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
@@ -1,166 +1,145 @@
|
||||
import argparse
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This example shows how to use vLLM-omni for running offline inference
|
||||
with the correct prompt format on Qwen2.5-Omni
|
||||
"""
|
||||
|
||||
import os
|
||||
import os as _os_env_toggle
|
||||
import random
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from utils import make_omni_prompt
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
|
||||
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
|
||||
from vllm_omni import OmniLLM
|
||||
|
||||
SEED = 42
|
||||
# Set all random seeds
|
||||
random.seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
torch.manual_seed(SEED)
|
||||
torch.cuda.manual_seed(SEED)
|
||||
torch.cuda.manual_seed_all(SEED)
|
||||
|
||||
# Make PyTorch deterministic
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
|
||||
# Set environment variables for deterministic behavior
|
||||
os.environ["PYTHONHASHSEED"] = str(SEED)
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
required=True,
|
||||
help="Path to merged model directory (will be created if downloading).",
|
||||
)
|
||||
parser.add_argument("--thinker-model", type=str, default=None)
|
||||
parser.add_argument("--talker-model", type=str, default=None)
|
||||
parser.add_argument("--code2wav-model", type=str, default=None)
|
||||
parser.add_argument(
|
||||
"--hf-hub-id",
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
help="Hugging Face repo id to download if needed.",
|
||||
)
|
||||
parser.add_argument("--hf-revision", default=None, help="Optional HF revision (branch/tag/commit).")
|
||||
parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
|
||||
parser.add_argument("--voice-type", default="default", help="Voice type, e.g., m02, f030, default.")
|
||||
parser.add_argument(
|
||||
"--code2wav-dir",
|
||||
default=None,
|
||||
help="Path to code2wav folder (contains spk_dict.pt).",
|
||||
)
|
||||
parser.add_argument("--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt).")
|
||||
parser.add_argument("--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file.")
|
||||
parser.add_argument("--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"])
|
||||
parser.add_argument("--max-model-len", type=int, default=32768)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Sleep seconds after starting each stage process to allow initialization (default: 20)",
|
||||
)
|
||||
|
||||
parser.add_argument("--thinker-only", action="store_true")
|
||||
parser.add_argument("--text-only", action="store_true")
|
||||
parser.add_argument("--do-wave", action="store_true")
|
||||
parser.add_argument(
|
||||
"--prompt_type",
|
||||
choices=[
|
||||
"text",
|
||||
"audio",
|
||||
"audio-long",
|
||||
"audio-long-chunks",
|
||||
"audio-long-expand-chunks",
|
||||
"image",
|
||||
"video",
|
||||
"video-frames",
|
||||
"audio-in-video",
|
||||
"audio-in-video-v2",
|
||||
"audio-multi-round",
|
||||
"badcase-vl",
|
||||
"badcase-text",
|
||||
"badcase-image-early-stop",
|
||||
"badcase-two-audios",
|
||||
"badcase-two-videos",
|
||||
"badcase-multi-round",
|
||||
"badcase-voice-type",
|
||||
"badcase-voice-type-v2",
|
||||
"badcase-audio-tower-1",
|
||||
"badcase-audio-only",
|
||||
],
|
||||
default="text",
|
||||
)
|
||||
parser.add_argument("--use-torchvision", action="store_true")
|
||||
parser.add_argument("--tokenize", action="store_true")
|
||||
parser.add_argument(
|
||||
"--output-wav",
|
||||
default="output.wav",
|
||||
help="[Deprecated] Output wav directory (use --output-dir).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="outputs",
|
||||
help="Output directory to save text and wav files together.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--thinker-hidden-states-dir",
|
||||
default="thinker_hidden_states",
|
||||
help="Path to thinker hidden states directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Timeout for batching in seconds (default: 5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=300,
|
||||
help="Timeout for initializing stages in seconds (default: 300)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold for using shared memory in bytes (default: 65536)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-stats",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable writing detailed statistics (default: disabled)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--txt-prompts",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to a .txt file with one prompt per line (preferred).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
class QueryResult(NamedTuple):
|
||||
inputs: dict
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
model_name = args.model
|
||||
try:
|
||||
# Preferred: load from txt file (one prompt per line)
|
||||
if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
|
||||
with open(args.txt_prompts, encoding="utf-8") as f:
|
||||
lines = [ln.strip() for ln in f.readlines()]
|
||||
args.prompts = [ln for ln in lines if ln != ""]
|
||||
print(f"[Info] Loaded {len(args.prompts)} prompts from {args.txt_prompts}")
|
||||
except Exception as e:
|
||||
print(f"[Error] Failed to load prompts: {e}")
|
||||
raise
|
||||
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
|
||||
# lower-end GPUs.
|
||||
# Unless specified, these settings have been tested to work on a single L4.
|
||||
|
||||
default_system = (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
|
||||
"Group, capable of perceiving auditory and visual inputs, as well as "
|
||||
"generating text and speech."
|
||||
)
|
||||
|
||||
|
||||
def get_text_query(question: str = None) -> QueryResult:
|
||||
if question is None:
|
||||
question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
},
|
||||
limit_mm_per_prompt={},
|
||||
)
|
||||
|
||||
|
||||
def get_mixed_modalities_query() -> QueryResult:
|
||||
question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
|
||||
"<|vision_bos|><|IMAGE|><|vision_eos|>"
|
||||
"<|vision_bos|><|VIDEO|><|vision_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
"image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
|
||||
"video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_use_audio_in_video_query() -> QueryResult:
|
||||
question = "Describe the content of the video, then convert what the baby say into text."
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
asset = VideoAsset(name="baby_reading", num_frames=16)
|
||||
audio = asset.get_audio(sampling_rate=16000)
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"video": asset.np_ndarrays,
|
||||
"audio": audio,
|
||||
},
|
||||
"mm_processor_kwargs": {
|
||||
"use_audio_in_video": True,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1, "video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_multi_audios_query() -> QueryResult:
|
||||
question = "Are these two audio clips the same?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
|
||||
"<|audio_bos|><|AUDIO|><|audio_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": [
|
||||
AudioAsset("winning_call").audio_and_sample_rate,
|
||||
AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
],
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={
|
||||
"audio": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
query_map = {
|
||||
"mixed_modalities": get_mixed_modalities_query,
|
||||
"use_audio_in_video": get_use_audio_in_video_query,
|
||||
"multi_audios": get_multi_audios_query,
|
||||
"text": get_text_query,
|
||||
}
|
||||
|
||||
|
||||
def main(args):
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
query_result = query_map[args.query_type]()
|
||||
|
||||
if args.prompts is None:
|
||||
raise ValueError("No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)")
|
||||
omni_llm = OmniLLM(
|
||||
model=model_name,
|
||||
log_stats=args.enable_stats,
|
||||
@@ -205,8 +184,16 @@ def main():
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
|
||||
prompt = [make_omni_prompt(args, prompt) for prompt in args.prompts]
|
||||
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
|
||||
if args.txt_prompts is None:
|
||||
prompts = [query_result.inputs for _ in range(args.num_prompts)]
|
||||
else:
|
||||
assert args.query_type == "text", "txt-prompts is only supported for text query type"
|
||||
with open(args.txt_prompts, encoding="utf-8") as f:
|
||||
lines = [ln.strip() for ln in f.readlines()]
|
||||
prompts = [get_text_query(ln).inputs for ln in lines if ln != ""]
|
||||
print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
|
||||
|
||||
omni_outputs = omni_llm.generate(prompts, sampling_params_list)
|
||||
|
||||
# Determine output directory: prefer --output-dir; fallback to --output-wav
|
||||
output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
|
||||
@@ -217,7 +204,7 @@ def main():
|
||||
request_id = int(output.request_id)
|
||||
text_output = output.outputs[0].text
|
||||
# Save aligned text file per request
|
||||
prompt_text = args.prompts[request_id]
|
||||
prompt_text = prompts[request_id]["prompt"]
|
||||
out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
|
||||
lines = []
|
||||
lines.append("Prompt:\n")
|
||||
@@ -239,5 +226,67 @@ def main():
|
||||
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="mixed_modalities",
|
||||
choices=query_map.keys(),
|
||||
help="Query type.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-stats",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable writing detailed statistics (default: disabled)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Sleep seconds after starting each stage process to allow initialization (default: 20)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Timeout for batching in seconds (default: 5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=300,
|
||||
help="Timeout for initializing stages in seconds (default: 300)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold for using shared memory in bytes (default: 65536)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-wav",
|
||||
default="output_audio",
|
||||
help="[Deprecated] Output wav directory (use --output-dir).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-prompts",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of prompts to generate.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--txt-prompts",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to a .txt file with one prompt per line (preferred).",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
args = parse_args()
|
||||
main(args)
|
||||
@@ -1,367 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
import warnings
|
||||
from functools import lru_cache
|
||||
from io import BytesIO
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import torchvision
|
||||
from packaging import version
|
||||
from PIL import Image
|
||||
from torchvision import io, transforms
|
||||
from torchvision.transforms import InterpolationMode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_FACTOR = 28
|
||||
MIN_PIXELS = 4 * 28 * 28
|
||||
MAX_PIXELS = 16384 * 28 * 28
|
||||
MAX_RATIO = 200
|
||||
|
||||
VIDEO_MIN_PIXELS = 128 * 28 * 28
|
||||
VIDEO_MAX_PIXELS = 768 * 28 * 28
|
||||
VIDEO_TOTAL_PIXELS = 24576 * 28 * 28
|
||||
FRAME_FACTOR = 2
|
||||
FPS = 2.0
|
||||
FPS_MIN_FRAMES = 4
|
||||
FPS_MAX_FRAMES = 768
|
||||
|
||||
temporal_patch_size = 2
|
||||
spatial_patch_size = 14
|
||||
spatial_merge_size = 2
|
||||
|
||||
|
||||
def round_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
||||
return round(number / factor) * factor
|
||||
|
||||
|
||||
def ceil_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the smallest integer greater than or equal to 'number' that is
|
||||
divisible by 'factor'."""
|
||||
return math.ceil(number / factor) * factor
|
||||
|
||||
|
||||
def floor_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the largest integer less than or equal to 'number' that is
|
||||
divisible by 'factor'."""
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
def smart_resize(
|
||||
height: int,
|
||||
width: int,
|
||||
factor: int = IMAGE_FACTOR,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Rescales the image so that the following conditions are met:
|
||||
|
||||
1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
|
||||
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
|
||||
|
||||
3. The aspect ratio of the image is maintained as closely as possible.
|
||||
"""
|
||||
if max(height, width) / min(height, width) > MAX_RATIO:
|
||||
raise ValueError(
|
||||
f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
|
||||
)
|
||||
h_bar = max(factor, round_by_factor(height, factor))
|
||||
w_bar = max(factor, round_by_factor(width, factor))
|
||||
if h_bar * w_bar > max_pixels:
|
||||
beta = math.sqrt((height * width) / max_pixels)
|
||||
h_bar = floor_by_factor(height / beta, factor)
|
||||
w_bar = floor_by_factor(width / beta, factor)
|
||||
elif h_bar * w_bar < min_pixels:
|
||||
beta = math.sqrt(min_pixels / (height * width))
|
||||
h_bar = ceil_by_factor(height * beta, factor)
|
||||
w_bar = ceil_by_factor(width * beta, factor)
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def fetch_image(ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR) -> Image.Image:
|
||||
if "image" in ele:
|
||||
image = ele["image"]
|
||||
else:
|
||||
image = ele["image_url"]
|
||||
image_obj = None
|
||||
if isinstance(image, Image.Image):
|
||||
image_obj = image
|
||||
elif image.startswith("http://") or image.startswith("https://"):
|
||||
image_obj = Image.open(requests.get(image, stream=True).raw)
|
||||
elif image.startswith("file://"):
|
||||
image_obj = Image.open(image[7:])
|
||||
elif image.startswith("data:image"):
|
||||
if "base64," in image:
|
||||
_, base64_data = image.split("base64,", 1)
|
||||
data = base64.b64decode(base64_data)
|
||||
image_obj = Image.open(BytesIO(data))
|
||||
else:
|
||||
image_obj = Image.open(image)
|
||||
if image_obj is None:
|
||||
raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
|
||||
image = image_obj.convert("RGB")
|
||||
# resize
|
||||
if "resized_height" in ele and "resized_width" in ele:
|
||||
resized_height, resized_width = smart_resize(
|
||||
ele["resized_height"],
|
||||
ele["resized_width"],
|
||||
factor=size_factor,
|
||||
)
|
||||
else:
|
||||
width, height = image.size
|
||||
min_pixels = ele.get("min_pixels", MIN_PIXELS)
|
||||
max_pixels = ele.get("max_pixels", MAX_PIXELS)
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=size_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
image = image.resize((resized_width, resized_height))
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def smart_nframes(
|
||||
ele: dict,
|
||||
total_frames: int,
|
||||
video_fps: int | float,
|
||||
) -> int:
|
||||
"""calculate the number of frames for video used for model inputs.
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support either `fps` or `nframes`:
|
||||
- nframes: the number of frames to extract for model inputs.
|
||||
- fps: the fps to extract frames for model inputs.
|
||||
- min_frames: the minimum number of frames of the video,
|
||||
only used when fps is provided.
|
||||
- max_frames: the maximum number of frames of the video,
|
||||
only used when fps is provided.
|
||||
total_frames (int): the original total number of frames of the video.
|
||||
video_fps (int | float): the original fps of the video.
|
||||
|
||||
Raises:
|
||||
ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
|
||||
|
||||
Returns:
|
||||
int: the number of frames for video used for model inputs.
|
||||
"""
|
||||
assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
|
||||
if "nframes" in ele:
|
||||
nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
|
||||
else:
|
||||
fps = ele.get("fps", FPS)
|
||||
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
|
||||
max_frames = floor_by_factor(
|
||||
ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)),
|
||||
FRAME_FACTOR,
|
||||
)
|
||||
nframes = total_frames / video_fps * fps
|
||||
nframes = min(max(nframes, min_frames), max_frames)
|
||||
nframes = round_by_factor(nframes, FRAME_FACTOR)
|
||||
if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
|
||||
raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
|
||||
return nframes
|
||||
|
||||
|
||||
def _read_video_torchvision(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using torchvision.io.read_video
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://",
|
||||
"https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
torch.Tensor: the video tensor with shape (T, C, H, W).
|
||||
"""
|
||||
video_path = ele["video"]
|
||||
if version.parse(torchvision.__version__) < version.parse("0.19.0"):
|
||||
if "http://" in video_path or "https://" in video_path:
|
||||
warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
|
||||
if "file://" in video_path:
|
||||
video_path = video_path[7:]
|
||||
st = time.time()
|
||||
video, audio, info = io.read_video(
|
||||
video_path,
|
||||
start_pts=ele.get("video_start", 0.0),
|
||||
end_pts=ele.get("video_end", None),
|
||||
pts_unit="sec",
|
||||
output_format="TCHW",
|
||||
)
|
||||
total_frames, video_fps = video.size(0), info["video_fps"]
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
logger.info(
|
||||
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
|
||||
)
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long()
|
||||
video = video[idx]
|
||||
return video, total_duration, nframes
|
||||
|
||||
|
||||
def is_decord_available() -> bool:
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec("decord") is not None
|
||||
|
||||
|
||||
def _read_video_decord(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using decord.VideoReader
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://",
|
||||
"https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
torch.Tensor: the video tensor with shape (T, C, H, W).
|
||||
"""
|
||||
import decord
|
||||
|
||||
video_path = ele["video"]
|
||||
st = time.time()
|
||||
vr = decord.VideoReader(video_path)
|
||||
# TODO: support start_pts and end_pts
|
||||
if "video_start" in ele or "video_end" in ele:
|
||||
raise NotImplementedError("not support start_pts and end_pts in decord for now.")
|
||||
total_frames, video_fps = len(vr), vr.get_avg_fps()
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
logger.info(f"decord: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
|
||||
video = vr.get_batch(idx).asnumpy()
|
||||
video = torch.tensor(video).permute(0, 3, 1, 2) # Convert to TCHW format
|
||||
return video, total_duration, nframes
|
||||
|
||||
|
||||
VIDEO_READER_BACKENDS = {
|
||||
"decord": _read_video_decord,
|
||||
"torchvision": _read_video_torchvision,
|
||||
}
|
||||
|
||||
FORCE_QWENVL_VIDEO_READER = os.getenv("FORCE_QWENVL_VIDEO_READER", None)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_video_reader_backend() -> str:
|
||||
if FORCE_QWENVL_VIDEO_READER is not None:
|
||||
video_reader_backend = FORCE_QWENVL_VIDEO_READER
|
||||
elif is_decord_available():
|
||||
video_reader_backend = "decord"
|
||||
else:
|
||||
video_reader_backend = "torchvision"
|
||||
# print(f"qwen-vl-utils using {video_reader_backend} to read video.",
|
||||
# file=sys.stderr)
|
||||
return video_reader_backend
|
||||
|
||||
|
||||
def fetch_video(ele: dict, image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
|
||||
if isinstance(ele["video"], str):
|
||||
video_reader_backend = get_video_reader_backend()
|
||||
video, total_dur, nframes = VIDEO_READER_BACKENDS[video_reader_backend](ele)
|
||||
frame_timestamps = total_dur * torch.arange(1, nframes + 1) / nframes
|
||||
grid_timestamps = frame_timestamps[::FRAME_FACTOR]
|
||||
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
|
||||
nframes, _, height, width = video.shape
|
||||
min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
|
||||
total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
|
||||
max_pixels = max(
|
||||
min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
|
||||
int(min_pixels * 1.05),
|
||||
)
|
||||
max_pixels = ele.get("max_pixels", max_pixels)
|
||||
# min_pixels = (factor ** 2) * 52
|
||||
# max_pixels = (factor ** 2) * min(768, (16384 / nframes * temporal_patch_size))
|
||||
if "resized_height" in ele and "resized_width" in ele:
|
||||
resized_height, resized_width = smart_resize(
|
||||
ele["resized_height"],
|
||||
ele["resized_width"],
|
||||
factor=image_factor,
|
||||
)
|
||||
else:
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=image_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
video = transforms.functional.resize(
|
||||
video,
|
||||
[resized_height, resized_width],
|
||||
interpolation=InterpolationMode.BICUBIC,
|
||||
antialias=True,
|
||||
).float()
|
||||
return video, total_dur, nframes, second_per_grid
|
||||
else:
|
||||
assert isinstance(ele["video"], (list, tuple))
|
||||
process_info = ele.copy()
|
||||
process_info.pop("type", None)
|
||||
process_info.pop("video", None)
|
||||
images = [
|
||||
fetch_image({"image": video_element, **process_info}, size_factor=image_factor)
|
||||
for video_element in ele["video"]
|
||||
]
|
||||
nframes = ceil_by_factor(len(images), FRAME_FACTOR)
|
||||
if len(images) < nframes:
|
||||
images.extend([images[-1]] * (nframes - len(images)))
|
||||
return images, None, None, None
|
||||
|
||||
|
||||
def extract_vision_info(conversations: list[dict] | list[list[dict]]) -> list[dict]:
|
||||
vision_infos = []
|
||||
if isinstance(conversations[0], dict):
|
||||
conversations = [conversations]
|
||||
for conversation in conversations:
|
||||
for message in conversation:
|
||||
if isinstance(message["content"], list):
|
||||
for ele in message["content"]:
|
||||
if (
|
||||
"image" in ele
|
||||
or "image_url" in ele
|
||||
or "video" in ele
|
||||
or ele["type"] in ("image", "image_url", "video")
|
||||
):
|
||||
vision_infos.append(ele)
|
||||
return vision_infos
|
||||
|
||||
|
||||
def process_vision_info(
|
||||
conversations: list[dict] | list[list[dict]],
|
||||
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None]:
|
||||
vision_infos = extract_vision_info(conversations)
|
||||
# Read images or videos
|
||||
image_inputs = []
|
||||
video_inputs = []
|
||||
for vision_info in vision_infos:
|
||||
if "image" in vision_info or "image_url" in vision_info:
|
||||
image_inputs.append(fetch_image(vision_info))
|
||||
elif "video" in vision_info:
|
||||
video_inputs.append(fetch_video(vision_info))
|
||||
else:
|
||||
raise ValueError("image, image_url or video should in content.")
|
||||
if len(image_inputs) == 0:
|
||||
image_inputs = None
|
||||
if len(video_inputs) == 0:
|
||||
video_inputs = None
|
||||
return image_inputs, video_inputs
|
||||
@@ -1,8 +1,3 @@
|
||||
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
|
||||
--voice-type "m02" \
|
||||
--dit-ckpt none \
|
||||
--bigvgan-ckpt none \
|
||||
--output-wav output_audio \
|
||||
--prompt_type text \
|
||||
--init-sleep-seconds 0 \
|
||||
--txt-prompts top100.txt
|
||||
python end2end.py --output-wav output_audio \
|
||||
--query-type text \
|
||||
--txt-prompts top10.txt
|
||||
@@ -1,8 +1,2 @@
|
||||
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
|
||||
--voice-type "m02" \
|
||||
--dit-ckpt none \
|
||||
--bigvgan-ckpt none \
|
||||
--output-wav output_audio \
|
||||
--prompt_type text \
|
||||
--init-sleep-seconds 0 \
|
||||
--prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
python end2end.py --output-wav output_audio \
|
||||
--query-type use_audio_in_video
|
||||
@@ -1,312 +0,0 @@
|
||||
import tempfile
|
||||
from typing import Optional, Union
|
||||
from urllib.request import urlopen
|
||||
|
||||
import librosa
|
||||
import requests
|
||||
import resampy
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torchvision.io
|
||||
from processing_omni import fetch_image, fetch_video
|
||||
from transformers import AutoConfig, AutoProcessor
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
# Simple caches to avoid repeated heavy HF loads per prompt
|
||||
_PROCESSOR_CACHE: dict[str, "AutoProcessor"] = {}
|
||||
_CONFIG_CACHE: dict[str, "AutoConfig"] = {}
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def resample_wav_to_16khz(input_filepath):
|
||||
data, original_sample_rate = sf.read(input_filepath)
|
||||
# Only use the first channel
|
||||
if len(data.shape) > 1:
|
||||
data = data[:, 0]
|
||||
# resample to 16kHz
|
||||
data_resampled = resampy.resample(data, sr_orig=original_sample_rate, sr_new=16000)
|
||||
return data_resampled
|
||||
|
||||
|
||||
def fetch_and_read_video(args, video_url: str, fps=2):
|
||||
def read_video_with_torchvision(video_file_name: str):
|
||||
video, audio, info = torchvision.io.read_video(
|
||||
video_file_name,
|
||||
start_pts=0.0,
|
||||
end_pts=None,
|
||||
pts_unit="sec",
|
||||
output_format="TCHW",
|
||||
)
|
||||
|
||||
total_frames, video_fps = video.size(0), info["video_fps"]
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
nframes = int(total_frames / video_fps * fps)
|
||||
|
||||
frame_timestamps = total_duration * torch.arange(1, nframes + 1) / nframes
|
||||
grid_timestamps = frame_timestamps[::2]
|
||||
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
|
||||
|
||||
idx = torch.linspace(0, video.size(0) - 1, nframes).round().long()
|
||||
video = video[idx]
|
||||
|
||||
if args.legacy_omni_video:
|
||||
return [video, total_duration, nframes, second_per_grid.item()]
|
||||
else:
|
||||
return video
|
||||
|
||||
def read_video_with_transformers(video_file_name: Union[str, list[str]]):
|
||||
video, total_duration, nframes, second_per_grid = fetch_video({"video": video_file_name})
|
||||
if total_duration is None and nframes is None:
|
||||
nframes = len(video)
|
||||
total_duration = 0.5 * nframes
|
||||
second_per_grid = 1.0
|
||||
if args.legacy_omni_video:
|
||||
return [video, total_duration, nframes, second_per_grid]
|
||||
else:
|
||||
return video
|
||||
|
||||
def read_video(video_file_name: str):
|
||||
if args.use_torchvision:
|
||||
return read_video_with_torchvision(video_file_name)
|
||||
else:
|
||||
return read_video_with_transformers(video_file_name)
|
||||
|
||||
if isinstance(video_url, str) and video_url.startswith("http"):
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
resp = requests.get(video_url)
|
||||
assert resp.status_code == requests.codes.ok, (
|
||||
f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
|
||||
)
|
||||
|
||||
temp_video_file.write(urlopen(video_url).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
video_file_name = temp_video_file_path
|
||||
return read_video(video_file_name)
|
||||
else:
|
||||
video_file_name = video_url
|
||||
return read_video(video_file_name)
|
||||
|
||||
|
||||
def make_inputs_qwen2_omni(
|
||||
args,
|
||||
messages: list[dict[str, Union[str, list[dict[str, str]]]]],
|
||||
use_audio_in_video: Optional[bool] = False,
|
||||
tokenize: bool = False,
|
||||
) -> Union[OmniTokensPrompt, TextPrompt]:
|
||||
from transformers import AutoConfig, AutoProcessor
|
||||
|
||||
# Cached processor/config to prevent per-prompt reloading and repeated warnings
|
||||
if args.model not in _PROCESSOR_CACHE:
|
||||
_PROCESSOR_CACHE[args.model] = AutoProcessor.from_pretrained(args.model)
|
||||
processor = _PROCESSOR_CACHE[args.model]
|
||||
|
||||
config = _CONFIG_CACHE.get(args.model)
|
||||
if config is None:
|
||||
try:
|
||||
config = AutoConfig.from_pretrained(args.model)
|
||||
except Exception:
|
||||
config = None
|
||||
_CONFIG_CACHE[args.model] = config # cache even if None to avoid retry storms
|
||||
|
||||
# Decide legacy flag only once based on config (default True if unknown)
|
||||
if getattr(args, "legacy_omni_video", None) is None:
|
||||
if config is not None and hasattr(config, "architectures"):
|
||||
args.legacy_omni_video = "Qwen2_5OmniModel" not in config.architectures
|
||||
else:
|
||||
args.legacy_omni_video = True
|
||||
|
||||
audios, images, videos = [], [], []
|
||||
for message in messages:
|
||||
if not isinstance(message["content"], list):
|
||||
message["content"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": message["content"],
|
||||
}
|
||||
]
|
||||
index, num_contents = 0, len(message["content"])
|
||||
while index < num_contents:
|
||||
ele = message["content"][index]
|
||||
if "type" not in ele:
|
||||
if "text" in ele:
|
||||
ele["type"] = "text"
|
||||
elif "audio" in ele:
|
||||
ele["type"] = "audio"
|
||||
elif "audio_url" in ele:
|
||||
ele["type"] = "audio_url"
|
||||
elif "image" in ele:
|
||||
ele["type"] = "image"
|
||||
elif "image_url" in ele:
|
||||
ele["type"] = "image_url"
|
||||
elif "video" in ele:
|
||||
ele["type"] = "video"
|
||||
elif "video_url" in ele:
|
||||
ele["type"] = "video_url"
|
||||
else:
|
||||
raise ValueError(f"Unknown ele: {ele}")
|
||||
|
||||
if ele["type"] == "audio" or ele["type"] == "audio_url":
|
||||
if "audio_url" in ele:
|
||||
audio_key = "audio_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_audio_file:
|
||||
temp_audio_file.write(urlopen(ele[audio_key]).read())
|
||||
temp_audio_file_path = temp_audio_file.name
|
||||
audios.append(resample_wav_to_16khz(temp_audio_file_path))
|
||||
ele["audio"] = temp_audio_file_path
|
||||
elif "audio" in ele:
|
||||
audio_key = "audio"
|
||||
audios.append(resample_wav_to_16khz(ele[audio_key]))
|
||||
else:
|
||||
raise ValueError(f"Unknown ele {ele}")
|
||||
elif use_audio_in_video and (ele["type"] == "video" or ele["type"] == "video_url"):
|
||||
# use video as audio as well
|
||||
if "video_url" in ele:
|
||||
audio_key = "video_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
temp_video_file.write(urlopen(ele[audio_key]).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
ele[audio_key] = temp_video_file_path
|
||||
audios.append(librosa.load(temp_video_file_path, sr=16000)[0])
|
||||
videos.append(fetch_and_read_video(args, temp_video_file_path))
|
||||
ele["video"] = temp_video_file_path
|
||||
elif "video" in ele:
|
||||
audio_key = "video"
|
||||
audios.append(librosa.load(ele[audio_key], sr=16000)[0])
|
||||
videos.append(fetch_and_read_video(args, audio_key))
|
||||
else:
|
||||
raise ValueError(f"Unknown ele {ele}")
|
||||
# insert a audio after the video
|
||||
message["content"].insert(
|
||||
index + 1,
|
||||
{
|
||||
"type": "audio",
|
||||
"audio": ele[audio_key],
|
||||
},
|
||||
)
|
||||
# no need to load the added audio again
|
||||
index += 1
|
||||
elif ele["type"] == "video" or ele["type"] == "video_url":
|
||||
if "video_url" in ele:
|
||||
video_key = "video_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
temp_video_file.write(urlopen(ele["video_url"]).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
videos.append(fetch_and_read_video(args, temp_video_file))
|
||||
ele["video"] = temp_video_file_path
|
||||
else:
|
||||
video_key = "video"
|
||||
videos.append(fetch_and_read_video(args, ele[video_key]))
|
||||
elif ele["type"] == "image" or ele["type"] == "image_url":
|
||||
images.append(fetch_image(ele))
|
||||
|
||||
# move to the next content
|
||||
index += 1
|
||||
|
||||
prompt = processor.apply_chat_template(
|
||||
messages,
|
||||
tokenize=tokenize,
|
||||
add_generation_prompt=True,
|
||||
add_vision_id=True,
|
||||
)
|
||||
|
||||
audios = audios if len(audios) > 0 else None
|
||||
images = images if len(images) > 0 else None
|
||||
videos = videos if len(videos) > 0 else None
|
||||
|
||||
multi_modal_data = {}
|
||||
if audios:
|
||||
multi_modal_data["audio"] = audios
|
||||
if images:
|
||||
multi_modal_data["image"] = images
|
||||
if videos:
|
||||
multi_modal_data["video"] = videos
|
||||
|
||||
if isinstance(prompt, list) and isinstance(prompt[0], (list, str)):
|
||||
prompt = prompt[0]
|
||||
|
||||
if tokenize:
|
||||
return OmniTokensPrompt(
|
||||
prompt_token_ids=prompt,
|
||||
multi_modal_data=multi_modal_data,
|
||||
)
|
||||
else:
|
||||
return TextPrompt(
|
||||
prompt=prompt,
|
||||
multi_modal_data=multi_modal_data,
|
||||
)
|
||||
|
||||
|
||||
def make_text_prompt(args, prompt):
|
||||
messages = [
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
prompt = make_inputs_qwen2_omni(args, messages, tokenize=args.tokenize)
|
||||
return prompt
|
||||
|
||||
|
||||
def make_audio_in_video_v2_prompt(args):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual "
|
||||
"inputs, as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": ("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"),
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
prompt = make_inputs_qwen2_omni(
|
||||
args,
|
||||
messages,
|
||||
use_audio_in_video=True,
|
||||
tokenize=args.tokenize,
|
||||
)
|
||||
return prompt
|
||||
|
||||
|
||||
def make_omni_prompt(args, prompt=None) -> Union[OmniTokensPrompt, list[OmniTokensPrompt]]:
|
||||
if args.prompt_type == "text":
|
||||
prompt = make_text_prompt(args, prompt)
|
||||
elif args.prompt_type == "audio-in-video-v2":
|
||||
prompt = make_audio_in_video_v2_prompt(args)
|
||||
else:
|
||||
raise ValueError(f"Unsupported prompt type: {args.prompt_type}")
|
||||
return prompt
|
||||
@@ -23,10 +23,18 @@ cd examples/online_serving
|
||||
|
||||
Send request via python
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py
|
||||
python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
|
||||
```
|
||||
|
||||
Send request via curl
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh
|
||||
bash run_curl_multimodal_generation.sh mixed_modalities
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
+140
-15
@@ -1,6 +1,9 @@
|
||||
import base64
|
||||
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
|
||||
# Modify OpenAI's API key and API base to use vLLM's API server.
|
||||
openai_api_key = "EMPTY"
|
||||
@@ -15,6 +18,16 @@ client = OpenAI(
|
||||
SEED = 42
|
||||
|
||||
|
||||
def encode_base64_content_from_url(content_url: str) -> str:
|
||||
"""Encode a content retrieved from a remote url to base64 format."""
|
||||
|
||||
with requests.get(content_url) as response:
|
||||
response.raise_for_status()
|
||||
result = base64.b64encode(response.content).decode("utf-8")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
@@ -31,7 +44,106 @@ def get_system_prompt():
|
||||
}
|
||||
|
||||
|
||||
def run_text_to_audio(model: str) -> None:
|
||||
def get_text_query():
|
||||
question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
}
|
||||
],
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
def get_mixed_modalities_query():
|
||||
question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": AudioAsset("mary_had_lamb").url},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def get_use_audio_in_video_query():
|
||||
question = "Describe the content of the video, then convert what the baby say into text."
|
||||
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4",
|
||||
"num_frames": 16,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def get_multi_audios_query():
|
||||
question = "Are these two audio clips the same?"
|
||||
prompt = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": AudioAsset("mary_had_lamb").url},
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": AudioAsset("winning_call").url},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"{question}",
|
||||
},
|
||||
],
|
||||
}
|
||||
return prompt
|
||||
|
||||
|
||||
query_map = {
|
||||
"mixed_modalities": get_mixed_modalities_query,
|
||||
"use_audio_in_video": get_use_audio_in_video_query,
|
||||
"multi_audios": get_multi_audios_query,
|
||||
"text": get_text_query,
|
||||
}
|
||||
|
||||
|
||||
def run_multimodal_generation(args) -> None:
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
thinker_sampling_params = {
|
||||
"temperature": 0.0, # Deterministic - no randomness
|
||||
"top_p": 1.0, # Disable nucleus sampling
|
||||
@@ -67,23 +179,21 @@ def run_text_to_audio(model: str) -> None:
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
|
||||
prompt = query_map[args.query_type]()
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_list # Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
}
|
||||
|
||||
if args.query_type == "use_audio_in_video":
|
||||
extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
|
||||
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=[
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words.",
|
||||
},
|
||||
],
|
||||
},
|
||||
prompt,
|
||||
],
|
||||
model=model,
|
||||
extra_body={
|
||||
"sampling_params_list": sampling_params_list
|
||||
}, # Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
model=model_name,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
|
||||
count = 0
|
||||
@@ -99,5 +209,20 @@ def run_text_to_audio(model: str) -> None:
|
||||
print("Chat completion output from text:", choice.message.content)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="mixed_modalities",
|
||||
choices=query_map.keys(),
|
||||
help="Query type.",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_text_to_audio("Qwen/Qwen2.5-Omni-7B")
|
||||
args = parse_args()
|
||||
run_multimodal_generation(args)
|
||||
@@ -1,6 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Default query type
|
||||
QUERY_TYPE="${1:-mixed_modalities}"
|
||||
|
||||
# Validate query type
|
||||
if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then
|
||||
echo "Error: Invalid query type '$QUERY_TYPE'"
|
||||
echo "Usage: $0 [mixed_modalities|use_audio_in_video|multi_audios|text]"
|
||||
echo " mixed_modalities: Audio + Image + Video + Text query"
|
||||
echo " use_audio_in_video: Video + Text query (with audio extraction from video)"
|
||||
echo " multi_audios: Two audio clips + Text query"
|
||||
echo " text: Text query"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SEED=42
|
||||
|
||||
thinker_sampling_params='{
|
||||
@@ -35,18 +49,121 @@ code2wav_sampling_params='{
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
|
||||
# Define URLs for assets
|
||||
MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
|
||||
WINNING_CALL_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/winning_call.ogg"
|
||||
CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
|
||||
SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
|
||||
|
||||
# Build user content and extra fields based on query type
|
||||
case "$QUERY_TYPE" in
|
||||
text)
|
||||
user_content='[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
}
|
||||
]'
|
||||
sampling_params_list='[
|
||||
'"$thinker_sampling_params"',
|
||||
'"$talker_sampling_params"',
|
||||
'"$code2wav_sampling_params"'
|
||||
]'
|
||||
mm_processor_kwargs="{}"
|
||||
;;
|
||||
mixed_modalities)
|
||||
user_content='[
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "'"$SAMPLE_VIDEO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
}
|
||||
]'
|
||||
sampling_params_list='[
|
||||
'"$thinker_sampling_params"',
|
||||
'"$talker_sampling_params"',
|
||||
'"$code2wav_sampling_params"'
|
||||
]'
|
||||
mm_processor_kwargs="{}"
|
||||
;;
|
||||
use_audio_in_video)
|
||||
user_content='[
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "'"$SAMPLE_VIDEO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe the content of the video, then convert what the baby say into text."
|
||||
}
|
||||
]'
|
||||
sampling_params_list='[
|
||||
'"$thinker_sampling_params"',
|
||||
'"$talker_sampling_params"',
|
||||
'"$code2wav_sampling_params"'
|
||||
]'
|
||||
mm_processor_kwargs='{
|
||||
"use_audio_in_video": true
|
||||
}'
|
||||
;;
|
||||
multi_audios)
|
||||
user_content='[
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "'"$WINNING_CALL_AUDIO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Are these two audio clips the same?"
|
||||
}
|
||||
]'
|
||||
sampling_params_list='[
|
||||
'"$thinker_sampling_params"',
|
||||
'"$talker_sampling_params"',
|
||||
'"$code2wav_sampling_params"'
|
||||
]'
|
||||
mm_processor_kwargs="{}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Running query type: $QUERY_TYPE"
|
||||
echo ""
|
||||
|
||||
|
||||
output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- <<EOF
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- <<EOF
|
||||
{
|
||||
"model": "Qwen/Qwen2.5-Omni-7B",
|
||||
"extra_body": {
|
||||
"sampling_params_list": [
|
||||
$thinker_sampling_params,
|
||||
$talker_sampling_params,
|
||||
$code2wav_sampling_params
|
||||
]
|
||||
},
|
||||
"sampling_params_list": $sampling_params_list,
|
||||
"mm_processor_kwargs": $mm_processor_kwargs,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
@@ -59,17 +176,12 @@ output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
|
||||
}
|
||||
]
|
||||
"content": $user_content
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
)
|
||||
|
||||
# Here it only shows the text content of the first choice. Audio content has many binaries, so it's not displayed here.
|
||||
echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"
|
||||
Reference in new issue
Block a user