added omni mooncake connectors to distributed directory, integrate omni connector to omni-llm, rebased with main (twice).

ray + mooncake connector work on 2 nodes, but needes several updates\n1.start ray inside vllm-omni \n2.placement group optimize \n3.fix close
This commit is contained in:
wzliu committed 2025-12-01 10:45:26 +08:00
1 parent 7391db62d1
commit 33e03b44c5
26 files changed
+2736 -78

No files matched your search

@@ -0,0 +1,34 @@
# Offline Example of vLLM-omni for Qwen2.5-omni
## 🛠️ Installation
Please refer to [README.md](../../../README.md)
## Run examples (Qwen2.5-omni)
### Multiple Prompts
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
```bash
tar -xf <Your Download Path>/seedtts_testset.tar
cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
--input examples/offline_inference/qwen2_5_omni/meta.lst \
--output examples/offline_inference/qwen2_5_omni/top100.txt \
--topk 100
```
Get into the example folder
```bash
cd examples/offline_inference/qwen2_5_omni
```
Then run the command below.
```bash
bash run_multiple_prompts.sh
```
### Single Prompts
Get into the example folder
```bash
cd examples/offline_inference/qwen2_5_omni
```
Then run the command below.
```bash
bash run_single_prompt.sh
```
@@ -0,0 +1,264 @@
import argparse
import os
import os as _os_env_toggle
import random
import numpy as np
import soundfile as sf
import torch
from utils import make_omni_prompt
from vllm.sampling_params import SamplingParams
from vllm_omni.entrypoints.omni_llm import OmniLLM
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
SEED = 42
# Set all random seeds
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
# Make PyTorch deterministic
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# Set environment variables for deterministic behavior
os.environ["PYTHONHASHSEED"] = str(SEED)
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
required=True,
help="Path to merged model directory (will be created if downloading).",
)
parser.add_argument("--thinker-model", type=str, default=None)
parser.add_argument("--talker-model", type=str, default=None)
parser.add_argument("--code2wav-model", type=str, default=None)
parser.add_argument(
"--hf-hub-id",
default="Qwen/Qwen2.5-Omni-7B",
help="Hugging Face repo id to download if needed.",
)
parser.add_argument("--hf-revision", default=None, help="Optional HF revision (branch/tag/commit).")
parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
parser.add_argument("--voice-type", default="default", help="Voice type, e.g., m02, f030, default.")
parser.add_argument(
"--code2wav-dir",
default=None,
help="Path to code2wav folder (contains spk_dict.pt).",
)
parser.add_argument("--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt).")
parser.add_argument("--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file.")
parser.add_argument("--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"])
parser.add_argument("--max-model-len", type=int, default=32768)
parser.add_argument(
"--init-sleep-seconds",
type=int,
default=20,
help="Sleep seconds after starting each stage process to allow initialization (default: 20)",
)
parser.add_argument("--thinker-only", action="store_true")
parser.add_argument("--text-only", action="store_true")
parser.add_argument("--do-wave", action="store_true")
parser.add_argument(
"--prompt_type",
choices=[
"text",
"audio",
"audio-long",
"audio-long-chunks",
"audio-long-expand-chunks",
"image",
"video",
"video-frames",
"audio-in-video",
"audio-in-video-v2",
"audio-multi-round",
"badcase-vl",
"badcase-text",
"badcase-image-early-stop",
"badcase-two-audios",
"badcase-two-videos",
"badcase-multi-round",
"badcase-voice-type",
"badcase-voice-type-v2",
"badcase-audio-tower-1",
"badcase-audio-only",
],
default="text",
)
parser.add_argument("--use-torchvision", action="store_true")
parser.add_argument("--tokenize", action="store_true")
parser.add_argument(
"--output-wav",
default="output.wav",
help="[Deprecated] Output wav directory (use --output-dir).",
)
parser.add_argument(
"--output-dir",
default="outputs",
help="Output directory to save text and wav files together.",
)
parser.add_argument(
"--thinker-hidden-states-dir",
default="thinker_hidden_states",
help="Path to thinker hidden states directory.",
)
parser.add_argument(
"--batch-timeout",
type=int,
default=5,
help="Timeout for batching in seconds (default: 5)",
)
parser.add_argument(
"--init-timeout",
type=int,
default=300,
help="Timeout for initializing stages in seconds (default: 300)",
)
parser.add_argument(
"--shm-threshold-bytes",
type=int,
default=65536,
help="Threshold for using shared memory in bytes (default: 65536)",
)
parser.add_argument(
"--enable-stats",
action="store_true",
default=False,
help="Enable writing detailed statistics (default: disabled)",
)
parser.add_argument(
"--txt-prompts",
type=str,
default=None,
help="Path to a .txt file with one prompt per line (preferred).",
)
parser.add_argument(
"--worker-backend",
type=str,
default="process",
choices=["process","ray"],
help="backend"
)
parser.add_argument(
"--ray-address",
type=str,
default=None,
help="Path to a .txt file with one prompt per line (preferred)."
)
args = parser.parse_args()
return args
def main():
args = parse_args()
model_name = args.model
try:
# Preferred: load from txt file (one prompt per line)
if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
with open(args.txt_prompts, encoding="utf-8") as f:
lines = [ln.strip() for ln in f.readlines()]
args.prompts = [ln for ln in lines if ln != ""]
print(f"[Info] Loaded {len(args.prompts)} prompts from {args.txt_prompts}")
except Exception as e:
print(f"[Error] Failed to load prompts: {e}")
raise
if args.prompts is None:
raise ValueError("No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)")
omni_llm = OmniLLM(
model=model_name,
log_stats=args.enable_stats,
log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
init_sleep_seconds=args.init_sleep_seconds,
batch_timeout=args.batch_timeout,
init_timeout=args.init_timeout,
shm_threshold_bytes=args.shm_threshold_bytes,
worker_backend=args.worker_backend,
ray_address=args.ray_address
)
thinker_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
talker_sampling_params = SamplingParams(
temperature=0.9,
top_p=0.8,
top_k=40,
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.05,
stop_token_ids=[8294],
)
code2wav_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
sampling_params_list = [
thinker_sampling_params,
talker_sampling_params,
code2wav_sampling_params,
]
import time
for i in range(1):
t1 = time.time()
prompt = [make_omni_prompt(args, prompt) for prompt in args.prompts]
print(f"prompt:{prompt}")
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
t2 = time.time()
print(f"==========> time:{t2-t1}")
# Determine output directory: prefer --output-dir; fallback to --output-wav
output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
os.makedirs(output_dir, exist_ok=True)
for stage_outputs in omni_outputs:
if stage_outputs.final_output_type == "text":
for output in stage_outputs.request_output:
request_id = int(output.request_id)
text_output = output.outputs[0].text
# Save aligned text file per request
prompt_text = args.prompts[request_id]
out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
lines = []
lines.append("Prompt:\n")
lines.append(str(prompt_text) + "\n")
lines.append("vllm_text_output:\n")
lines.append(str(text_output).strip() + "\n")
try:
with open(out_txt, "w", encoding="utf-8") as f:
f.writelines(lines)
except Exception as e:
print(f"[Warn] Failed writing text file {out_txt}: {e}")
print(f"Request ID: {request_id}, Text saved to {out_txt}")
elif stage_outputs.final_output_type == "audio":
for output in stage_outputs.request_output:
request_id = int(output.request_id)
audio_tensor = output.multimodal_output["audio"]
output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
if __name__ == "__main__":
main()
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
import argparse
from typing import Optional
def extract_prompt(line: str) -> Optional[str]:
# 提取第一个 '|' 与第二个 '|' 之间的内容
i = line.find("|")
if i == -1:
return None
j = line.find("|", i + 1)
if j == -1:
return None
return line[i + 1 : j].strip()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--input", "-i", required=True, help="Input .lst file path")
parser.add_argument("--output", "-o", required=True, help="Output file path")
parser.add_argument(
"--topk",
"-k",
type=int,
default=100,
help="Extract the top K prompts (default: 100)",
)
args = parser.parse_args()
prompts = []
with open(args.input, encoding="utf-8", errors="ignore") as f:
for line in f:
if len(prompts) >= args.topk:
break
p = extract_prompt(line.rstrip("\n"))
if p:
prompts.append(p)
with open(args.output, "w", encoding="utf-8") as f:
for p in prompts:
f.write(p + "\n")
if __name__ == "__main__":
main()
@@ -0,0 +1,367 @@
from __future__ import annotations
import base64
import logging
import math
import os
import time
import warnings
from functools import lru_cache
from io import BytesIO
import requests
import torch
import torchvision
from packaging import version
from PIL import Image
from torchvision import io, transforms
from torchvision.transforms import InterpolationMode
logger = logging.getLogger(__name__)
IMAGE_FACTOR = 28
MIN_PIXELS = 4 * 28 * 28
MAX_PIXELS = 16384 * 28 * 28
MAX_RATIO = 200
VIDEO_MIN_PIXELS = 128 * 28 * 28
VIDEO_MAX_PIXELS = 768 * 28 * 28
VIDEO_TOTAL_PIXELS = 24576 * 28 * 28
FRAME_FACTOR = 2
FPS = 2.0
FPS_MIN_FRAMES = 4
FPS_MAX_FRAMES = 768
temporal_patch_size = 2
spatial_patch_size = 14
spatial_merge_size = 2
def round_by_factor(number: int, factor: int) -> int:
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
return round(number / factor) * factor
def ceil_by_factor(number: int, factor: int) -> int:
"""Returns the smallest integer greater than or equal to 'number' that is
divisible by 'factor'."""
return math.ceil(number / factor) * factor
def floor_by_factor(number: int, factor: int) -> int:
"""Returns the largest integer less than or equal to 'number' that is
divisible by 'factor'."""
return math.floor(number / factor) * factor
def smart_resize(
height: int,
width: int,
factor: int = IMAGE_FACTOR,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS,
) -> tuple[int, int]:
"""
Rescales the image so that the following conditions are met:
1. Both dimensions (height and width) are divisible by 'factor'.
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
3. The aspect ratio of the image is maintained as closely as possible.
"""
if max(height, width) / min(height, width) > MAX_RATIO:
raise ValueError(
f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
)
h_bar = max(factor, round_by_factor(height, factor))
w_bar = max(factor, round_by_factor(width, factor))
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = floor_by_factor(height / beta, factor)
w_bar = floor_by_factor(width / beta, factor)
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = ceil_by_factor(height * beta, factor)
w_bar = ceil_by_factor(width * beta, factor)
return h_bar, w_bar
def fetch_image(ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR) -> Image.Image:
if "image" in ele:
image = ele["image"]
else:
image = ele["image_url"]
image_obj = None
if isinstance(image, Image.Image):
image_obj = image
elif image.startswith("http://") or image.startswith("https://"):
image_obj = Image.open(requests.get(image, stream=True).raw)
elif image.startswith("file://"):
image_obj = Image.open(image[7:])
elif image.startswith("data:image"):
if "base64," in image:
_, base64_data = image.split("base64,", 1)
data = base64.b64decode(base64_data)
image_obj = Image.open(BytesIO(data))
else:
image_obj = Image.open(image)
if image_obj is None:
raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
image = image_obj.convert("RGB")
# resize
if "resized_height" in ele and "resized_width" in ele:
resized_height, resized_width = smart_resize(
ele["resized_height"],
ele["resized_width"],
factor=size_factor,
)
else:
width, height = image.size
min_pixels = ele.get("min_pixels", MIN_PIXELS)
max_pixels = ele.get("max_pixels", MAX_PIXELS)
resized_height, resized_width = smart_resize(
height,
width,
factor=size_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
image = image.resize((resized_width, resized_height))
return image
def smart_nframes(
ele: dict,
total_frames: int,
video_fps: int | float,
) -> int:
"""calculate the number of frames for video used for model inputs.
Args:
ele (dict): a dict contains the configuration of video.
support either `fps` or `nframes`:
- nframes: the number of frames to extract for model inputs.
- fps: the fps to extract frames for model inputs.
- min_frames: the minimum number of frames of the video,
only used when fps is provided.
- max_frames: the maximum number of frames of the video,
only used when fps is provided.
total_frames (int): the original total number of frames of the video.
video_fps (int | float): the original fps of the video.
Raises:
ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
Returns:
int: the number of frames for video used for model inputs.
"""
assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
if "nframes" in ele:
nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
else:
fps = ele.get("fps", FPS)
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
max_frames = floor_by_factor(
ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)),
FRAME_FACTOR,
)
nframes = total_frames / video_fps * fps
nframes = min(max(nframes, min_frames), max_frames)
nframes = round_by_factor(nframes, FRAME_FACTOR)
if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
return nframes
def _read_video_torchvision(
ele: dict,
) -> torch.Tensor:
"""read video using torchvision.io.read_video
Args:
ele (dict): a dict contains the configuration of video.
support keys:
- video: the path of video. support "file://", "http://",
"https://" and local path.
- video_start: the start time of video.
- video_end: the end time of video.
Returns:
torch.Tensor: the video tensor with shape (T, C, H, W).
"""
video_path = ele["video"]
if version.parse(torchvision.__version__) < version.parse("0.19.0"):
if "http://" in video_path or "https://" in video_path:
warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
if "file://" in video_path:
video_path = video_path[7:]
st = time.time()
video, audio, info = io.read_video(
video_path,
start_pts=ele.get("video_start", 0.0),
end_pts=ele.get("video_end", None),
pts_unit="sec",
output_format="TCHW",
)
total_frames, video_fps = video.size(0), info["video_fps"]
total_duration = round(total_frames / video_fps, 3)
logger.info(
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
)
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
idx = torch.linspace(0, total_frames - 1, nframes).round().long()
video = video[idx]
return video, total_duration, nframes
def is_decord_available() -> bool:
import importlib.util
return importlib.util.find_spec("decord") is not None
def _read_video_decord(
ele: dict,
) -> torch.Tensor:
"""read video using decord.VideoReader
Args:
ele (dict): a dict contains the configuration of video.
support keys:
- video: the path of video. support "file://", "http://",
"https://" and local path.
- video_start: the start time of video.
- video_end: the end time of video.
Returns:
torch.Tensor: the video tensor with shape (T, C, H, W).
"""
import decord
video_path = ele["video"]
st = time.time()
vr = decord.VideoReader(video_path)
# TODO: support start_pts and end_pts
if "video_start" in ele or "video_end" in ele:
raise NotImplementedError("not support start_pts and end_pts in decord for now.")
total_frames, video_fps = len(vr), vr.get_avg_fps()
total_duration = round(total_frames / video_fps, 3)
logger.info(f"decord: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
video = vr.get_batch(idx).asnumpy()
video = torch.tensor(video).permute(0, 3, 1, 2) # Convert to TCHW format
return video, total_duration, nframes
VIDEO_READER_BACKENDS = {
"decord": _read_video_decord,
"torchvision": _read_video_torchvision,
}
FORCE_QWENVL_VIDEO_READER = os.getenv("FORCE_QWENVL_VIDEO_READER", None)
@lru_cache(maxsize=1)
def get_video_reader_backend() -> str:
if FORCE_QWENVL_VIDEO_READER is not None:
video_reader_backend = FORCE_QWENVL_VIDEO_READER
elif is_decord_available():
video_reader_backend = "decord"
else:
video_reader_backend = "torchvision"
# print(f"qwen-vl-utils using {video_reader_backend} to read video.",
# file=sys.stderr)
return video_reader_backend
def fetch_video(ele: dict, image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
if isinstance(ele["video"], str):
video_reader_backend = get_video_reader_backend()
video, total_dur, nframes = VIDEO_READER_BACKENDS[video_reader_backend](ele)
frame_timestamps = total_dur * torch.arange(1, nframes + 1) / nframes
grid_timestamps = frame_timestamps[::FRAME_FACTOR]
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
nframes, _, height, width = video.shape
min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
max_pixels = max(
min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
int(min_pixels * 1.05),
)
max_pixels = ele.get("max_pixels", max_pixels)
# min_pixels = (factor ** 2) * 52
# max_pixels = (factor ** 2) * min(768, (16384 / nframes * temporal_patch_size))
if "resized_height" in ele and "resized_width" in ele:
resized_height, resized_width = smart_resize(
ele["resized_height"],
ele["resized_width"],
factor=image_factor,
)
else:
resized_height, resized_width = smart_resize(
height,
width,
factor=image_factor,
min_pixels=min_pixels,
max_pixels=max_pixels,
)
video = transforms.functional.resize(
video,
[resized_height, resized_width],
interpolation=InterpolationMode.BICUBIC,
antialias=True,
).float()
return video, total_dur, nframes, second_per_grid
else:
assert isinstance(ele["video"], (list, tuple))
process_info = ele.copy()
process_info.pop("type", None)
process_info.pop("video", None)
images = [
fetch_image({"image": video_element, **process_info}, size_factor=image_factor)
for video_element in ele["video"]
]
nframes = ceil_by_factor(len(images), FRAME_FACTOR)
if len(images) < nframes:
images.extend([images[-1]] * (nframes - len(images)))
return images, None, None, None
def extract_vision_info(conversations: list[dict] | list[list[dict]]) -> list[dict]:
vision_infos = []
if isinstance(conversations[0], dict):
conversations = [conversations]
for conversation in conversations:
for message in conversation:
if isinstance(message["content"], list):
for ele in message["content"]:
if (
"image" in ele
or "image_url" in ele
or "video" in ele
or ele["type"] in ("image", "image_url", "video")
):
vision_infos.append(ele)
return vision_infos
def process_vision_info(
conversations: list[dict] | list[list[dict]],
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None]:
vision_infos = extract_vision_info(conversations)
# Read images or videos
image_inputs = []
video_inputs = []
for vision_info in vision_infos:
if "image" in vision_info or "image_url" in vision_info:
image_inputs.append(fetch_image(vision_info))
elif "video" in vision_info:
video_inputs.append(fetch_video(vision_info))
else:
raise ValueError("image, image_url or video should in content.")
if len(image_inputs) == 0:
image_inputs = None
if len(video_inputs) == 0:
video_inputs = None
return image_inputs, video_inputs
@@ -0,0 +1 @@
ray start --head --port=6399 --num-gpus=1
@@ -0,0 +1 @@
ray start --address=10.90.67.86:6399
@@ -0,0 +1,8 @@
python end2end.py --model Qwen/Qwen2.5-Omni-7B \
--voice-type "m02" \
--dit-ckpt none \
--bigvgan-ckpt none \
--output-wav output_audio \
--prompt_type text \
--init-sleep-seconds 0 \
--txt-prompts top100.txt
@@ -0,0 +1,11 @@
export PYTHONPATH=/workspace/omni/vllm-omni/:$PYTHONPATH
CUDA_VISIBLE_DEVICES=4 python end2end.py --model /workspace/Qwen2.5-Omni-7B/ \
--voice-type "m02" \
--dit-ckpt none \
--bigvgan-ckpt none \
--output-wav output_audio \
--prompt_type text \
--init-sleep-seconds 0 \
--worker-backend ray \
--ray-address auto \
--prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
@@ -0,0 +1,312 @@
import tempfile
from typing import Optional, Union
from urllib.request import urlopen
import librosa
import requests
import resampy
import soundfile as sf
import torch
import torchvision.io
from processing_omni import fetch_image, fetch_video
from transformers import AutoConfig, AutoProcessor
from vllm.inputs import TextPrompt
from vllm_omni.inputs.data import OmniTokensPrompt
# Simple caches to avoid repeated heavy HF loads per prompt
_PROCESSOR_CACHE: dict[str, "AutoProcessor"] = {}
_CONFIG_CACHE: dict[str, "AutoConfig"] = {}
def get_system_prompt():
return {
"role": "system",
"content": [
{
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual inputs, "
"as well as generating text and speech."
),
}
],
}
def resample_wav_to_16khz(input_filepath):
data, original_sample_rate = sf.read(input_filepath)
# Only use the first channel
if len(data.shape) > 1:
data = data[:, 0]
# resample to 16kHz
data_resampled = resampy.resample(data, sr_orig=original_sample_rate, sr_new=16000)
return data_resampled
def fetch_and_read_video(args, video_url: str, fps=2):
def read_video_with_torchvision(video_file_name: str):
video, audio, info = torchvision.io.read_video(
video_file_name,
start_pts=0.0,
end_pts=None,
pts_unit="sec",
output_format="TCHW",
)
total_frames, video_fps = video.size(0), info["video_fps"]
total_duration = round(total_frames / video_fps, 3)
nframes = int(total_frames / video_fps * fps)
frame_timestamps = total_duration * torch.arange(1, nframes + 1) / nframes
grid_timestamps = frame_timestamps[::2]
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
idx = torch.linspace(0, video.size(0) - 1, nframes).round().long()
video = video[idx]
if args.legacy_omni_video:
return [video, total_duration, nframes, second_per_grid.item()]
else:
return video
def read_video_with_transformers(video_file_name: Union[str, list[str]]):
video, total_duration, nframes, second_per_grid = fetch_video({"video": video_file_name})
if total_duration is None and nframes is None:
nframes = len(video)
total_duration = 0.5 * nframes
second_per_grid = 1.0
if args.legacy_omni_video:
return [video, total_duration, nframes, second_per_grid]
else:
return video
def read_video(video_file_name: str):
if args.use_torchvision:
return read_video_with_torchvision(video_file_name)
else:
return read_video_with_transformers(video_file_name)
if isinstance(video_url, str) and video_url.startswith("http"):
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
resp = requests.get(video_url)
assert resp.status_code == requests.codes.ok, (
f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
)
temp_video_file.write(urlopen(video_url).read())
temp_video_file_path = temp_video_file.name
video_file_name = temp_video_file_path
return read_video(video_file_name)
else:
video_file_name = video_url
return read_video(video_file_name)
def make_inputs_qwen2_omni(
args,
messages: list[dict[str, Union[str, list[dict[str, str]]]]],
use_audio_in_video: Optional[bool] = False,
tokenize: bool = False,
) -> Union[OmniTokensPrompt, TextPrompt]:
from transformers import AutoConfig, AutoProcessor
# Cached processor/config to prevent per-prompt reloading and repeated warnings
if args.model not in _PROCESSOR_CACHE:
_PROCESSOR_CACHE[args.model] = AutoProcessor.from_pretrained(args.model)
processor = _PROCESSOR_CACHE[args.model]
config = _CONFIG_CACHE.get(args.model)
if config is None:
try:
config = AutoConfig.from_pretrained(args.model)
except Exception:
config = None
_CONFIG_CACHE[args.model] = config # cache even if None to avoid retry storms
# Decide legacy flag only once based on config (default True if unknown)
if getattr(args, "legacy_omni_video", None) is None:
if config is not None and hasattr(config, "architectures"):
args.legacy_omni_video = "Qwen2_5OmniModel" not in config.architectures
else:
args.legacy_omni_video = True
audios, images, videos = [], [], []
for message in messages:
if not isinstance(message["content"], list):
message["content"] = [
{
"type": "text",
"text": message["content"],
}
]
index, num_contents = 0, len(message["content"])
while index < num_contents:
ele = message["content"][index]
if "type" not in ele:
if "text" in ele:
ele["type"] = "text"
elif "audio" in ele:
ele["type"] = "audio"
elif "audio_url" in ele:
ele["type"] = "audio_url"
elif "image" in ele:
ele["type"] = "image"
elif "image_url" in ele:
ele["type"] = "image_url"
elif "video" in ele:
ele["type"] = "video"
elif "video_url" in ele:
ele["type"] = "video_url"
else:
raise ValueError(f"Unknown ele: {ele}")
if ele["type"] == "audio" or ele["type"] == "audio_url":
if "audio_url" in ele:
audio_key = "audio_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_audio_file:
temp_audio_file.write(urlopen(ele[audio_key]).read())
temp_audio_file_path = temp_audio_file.name
audios.append(resample_wav_to_16khz(temp_audio_file_path))
ele["audio"] = temp_audio_file_path
elif "audio" in ele:
audio_key = "audio"
audios.append(resample_wav_to_16khz(ele[audio_key]))
else:
raise ValueError(f"Unknown ele {ele}")
elif use_audio_in_video and (ele["type"] == "video" or ele["type"] == "video_url"):
# use video as audio as well
if "video_url" in ele:
audio_key = "video_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
temp_video_file.write(urlopen(ele[audio_key]).read())
temp_video_file_path = temp_video_file.name
ele[audio_key] = temp_video_file_path
audios.append(librosa.load(temp_video_file_path, sr=16000)[0])
videos.append(fetch_and_read_video(args, temp_video_file_path))
ele["video"] = temp_video_file_path
elif "video" in ele:
audio_key = "video"
audios.append(librosa.load(ele[audio_key], sr=16000)[0])
videos.append(fetch_and_read_video(args, audio_key))
else:
raise ValueError(f"Unknown ele {ele}")
# insert a audio after the video
message["content"].insert(
index + 1,
{
"type": "audio",
"audio": ele[audio_key],
},
)
# no need to load the added audio again
index += 1
elif ele["type"] == "video" or ele["type"] == "video_url":
if "video_url" in ele:
video_key = "video_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
temp_video_file.write(urlopen(ele["video_url"]).read())
temp_video_file_path = temp_video_file.name
videos.append(fetch_and_read_video(args, temp_video_file))
ele["video"] = temp_video_file_path
else:
video_key = "video"
videos.append(fetch_and_read_video(args, ele[video_key]))
elif ele["type"] == "image" or ele["type"] == "image_url":
images.append(fetch_image(ele))
# move to the next content
index += 1
prompt = processor.apply_chat_template(
messages,
tokenize=tokenize,
add_generation_prompt=True,
add_vision_id=True,
)
audios = audios if len(audios) > 0 else None
images = images if len(images) > 0 else None
videos = videos if len(videos) > 0 else None
multi_modal_data = {}
if audios:
multi_modal_data["audio"] = audios
if images:
multi_modal_data["image"] = images
if videos:
multi_modal_data["video"] = videos
if isinstance(prompt, list) and isinstance(prompt[0], (list, str)):
prompt = prompt[0]
if tokenize:
return OmniTokensPrompt(
prompt_token_ids=prompt,
multi_modal_data=multi_modal_data,
)
else:
return TextPrompt(
prompt=prompt,
multi_modal_data=multi_modal_data,
)
def make_text_prompt(args, prompt):
messages = [
get_system_prompt(),
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
],
},
]
prompt = make_inputs_qwen2_omni(args, messages, tokenize=args.tokenize)
return prompt
def make_audio_in_video_v2_prompt(args):
messages = [
{
"role": "system",
"content": [
{
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual "
"inputs, as well as generating text and speech."
),
}
],
},
{
"role": "user",
"content": [
{
"type": "video_url",
"video_url": ("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"),
},
],
},
]
prompt = make_inputs_qwen2_omni(
args,
messages,
use_audio_in_video=True,
tokenize=args.tokenize,
)
return prompt
def make_omni_prompt(args, prompt=None) -> Union[OmniTokensPrompt, list[OmniTokensPrompt]]:
if args.prompt_type == "text":
prompt = make_text_prompt(args, prompt)
elif args.prompt_type == "audio-in-video-v2":
prompt = make_audio_in_video_v2_prompt(args)
else:
raise ValueError(f"Unsupported prompt type: {args.prompt_type}")
return prompt