fix conflicts

This commit is contained in:
Gaohan123 committed 2025-10-24 16:49:19 +08:00
1 parent 22f36de34e
commit 171f3c1ea7
42 files changed
+2930 -1842

No files matched your search

@@ -34,4 +34,4 @@ The output audio is saved in ./output_audio
- [ ] Adaptation from current vllm branch to stable vllm v0.11.0
- [ ] Offline inference example for Qwen2.5-omni with streaming multiple requests
- [ ] Online inference support
- [ ] Support for other models
- [ ] Support for other models
@@ -1,18 +1,17 @@
import argparse
import os
import soundfile as sf
import os as _os_env_toggle
import random
import numpy as np
import torch
import numpy as np
import soundfile as sf
import torch
from utils import make_omni_prompt
from vllm.sampling_params import SamplingParams
import os as _os_env_toggle
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
from vllm_omni.entrypoints.omni_llm import OmniLLM
from utils import make_omni_prompt
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
SEED = 42
# Set all random seeds
@@ -33,39 +32,84 @@ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True, help='Path to merged model directory (will be created if downloading).')
parser.add_argument('--thinker-model', type=str, default=None)
parser.add_argument('--talker-model', type=str, default=None)
parser.add_argument('--code2wav-model', type=str, default=None)
parser.add_argument('--hf-hub-id', default='Qwen/Qwen2.5-Omni-7B', help='Hugging Face repo id to download if needed.')
parser.add_argument('--hf-revision', default=None, help='Optional HF revision (branch/tag/commit).')
parser.add_argument('--prompts', required=True, nargs='+', help='Input text prompts.')
parser.add_argument('--voice-type', default='default', help='Voice type, e.g., m02, f030, default.')
parser.add_argument('--code2wav-dir', default=None, help='Path to code2wav folder (contains spk_dict.pt).')
parser.add_argument('--dit-ckpt', default=None, help='Path to DiT checkpoint file (e.g., dit.pt).')
parser.add_argument('--bigvgan-ckpt', default=None, help='Path to BigVGAN checkpoint file.')
parser.add_argument('--dtype', default='bfloat16', choices=['float16', 'bfloat16', 'float32'])
parser.add_argument('--max-model-len', type=int, default=32768)
parser.add_argument(
"--model",
required=True,
help="Path to merged model directory (will be created if downloading).",
)
parser.add_argument("--thinker-model", type=str, default=None)
parser.add_argument("--talker-model", type=str, default=None)
parser.add_argument("--code2wav-model", type=str, default=None)
parser.add_argument(
"--hf-hub-id",
default="Qwen/Qwen2.5-Omni-7B",
help="Hugging Face repo id to download if needed.",
)
parser.add_argument(
"--hf-revision", default=None, help="Optional HF revision (branch/tag/commit)."
)
parser.add_argument(
"--prompts", required=True, nargs="+", help="Input text prompts."
)
parser.add_argument(
"--voice-type", default="default", help="Voice type, e.g., m02, f030, default."
)
parser.add_argument(
"--code2wav-dir",
default=None,
help="Path to code2wav folder (contains spk_dict.pt).",
)
parser.add_argument(
"--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt)."
)
parser.add_argument(
"--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file."
)
parser.add_argument(
"--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"]
)
parser.add_argument("--max-model-len", type=int, default=32768)
parser.add_argument("--thinker-only", action="store_true")
parser.add_argument("--text-only", action="store_true")
parser.add_argument("--do-wave", action="store_true")
parser.add_argument('--prompt_type',
choices=[
'text', 'audio', 'audio-long', 'audio-long-chunks',
'audio-long-expand-chunks', 'image', 'video',
'video-frames', 'audio-in-video', 'audio-in-video-v2',
"audio-multi-round", "badcase-vl", "badcase-text",
"badcase-image-early-stop", "badcase-two-audios",
"badcase-two-videos", "badcase-multi-round",
"badcase-voice-type", "badcase-voice-type-v2",
"badcase-audio-tower-1", "badcase-audio-only"
],
default='text')
parser.add_argument('--use-torchvision', action='store_true')
parser.add_argument('--tokenize', action='store_true')
parser.add_argument('--output-wav', default="output.wav", help='Output wav file path.')
parser.add_argument('--thinker-hidden-states-dir', default="thinker_hidden_states", help='Path to thinker hidden states directory.')
parser.add_argument(
"--prompt_type",
choices=[
"text",
"audio",
"audio-long",
"audio-long-chunks",
"audio-long-expand-chunks",
"image",
"video",
"video-frames",
"audio-in-video",
"audio-in-video-v2",
"audio-multi-round",
"badcase-vl",
"badcase-text",
"badcase-image-early-stop",
"badcase-two-audios",
"badcase-two-videos",
"badcase-multi-round",
"badcase-voice-type",
"badcase-voice-type-v2",
"badcase-audio-tower-1",
"badcase-audio-only",
],
default="text",
)
parser.add_argument("--use-torchvision", action="store_true")
parser.add_argument("--tokenize", action="store_true")
parser.add_argument(
"--output-wav", default="output.wav", help="Output wav file path."
)
parser.add_argument(
"--thinker-hidden-states-dir",
default="thinker_hidden_states",
help="Path to thinker hidden states directory.",
)
args = parser.parse_args()
return args
@@ -75,38 +119,40 @@ def main():
model_name = args.model
omni_llm = OmniLLM(model=model_name)
thinker_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
talker_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
stop_token_ids=[8294]
)
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
stop_token_ids=[8294],
)
code2wav_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
top_k=-1, # Disable top-k sampling
max_tokens=2048,
seed=SEED, # Fixed seed for sampling
detokenize=True,
repetition_penalty=1.1,
)
sampling_params_list = [
thinker_sampling_params,
talker_sampling_params,
code2wav_sampling_params,
]
sampling_params_list = [thinker_sampling_params,
talker_sampling_params,
code2wav_sampling_params]
prompt = [make_omni_prompt(args, prompt) for prompt in args.prompts]
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
@@ -121,10 +167,14 @@ def main():
for output in stage_outputs.request_output:
request_id = output.request_id
audio_tensor = output.multimodal_output["audio"]
output_wav = os.path.join(args.output_wav, f"output_{output.request_id}.wav")
sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
output_wav = os.path.join(
args.output_wav, f"output_{output.request_id}.wav"
)
sf.write(
output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000
)
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
if __name__ == "__main__":
main()
main()
@@ -8,6 +8,7 @@ import time
import warnings
from functools import lru_cache
from io import BytesIO
import requests
import torch
import torchvision
@@ -16,7 +17,6 @@ from PIL import Image
from torchvision import io, transforms
from torchvision.transforms import InterpolationMode
logger = logging.getLogger(__name__)
IMAGE_FACTOR = 28
@@ -43,20 +43,24 @@ def round_by_factor(number: int, factor: int) -> int:
def ceil_by_factor(number: int, factor: int) -> int:
"""Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
"""Returns the smallest integer greater than or equal to 'number' that is
divisible by 'factor'."""
return math.ceil(number / factor) * factor
def floor_by_factor(number: int, factor: int) -> int:
"""Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
"""Returns the largest integer less than or equal to 'number' that is
divisible by 'factor'."""
return math.floor(number / factor) * factor
def smart_resize(height: int,
width: int,
factor: int = IMAGE_FACTOR,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS) -> tuple[int, int]:
def smart_resize(
height: int,
width: int,
factor: int = IMAGE_FACTOR,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS,
) -> tuple[int, int]:
"""
Rescales the image so that the following conditions are met:
@@ -68,7 +72,8 @@ def smart_resize(height: int,
"""
if max(height, width) / min(height, width) > MAX_RATIO:
raise ValueError(
f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
f"got {max(height, width) / min(height, width)}"
)
h_bar = max(factor, round_by_factor(height, factor))
w_bar = max(factor, round_by_factor(width, factor))
@@ -83,8 +88,9 @@ def smart_resize(height: int,
return h_bar, w_bar
def fetch_image(ele: dict[str, str | Image.Image],
size_factor: int = IMAGE_FACTOR) -> Image.Image:
def fetch_image(
ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR
) -> Image.Image:
if "image" in ele:
image = ele["image"]
else:
@@ -105,10 +111,11 @@ def fetch_image(ele: dict[str, str | Image.Image],
image_obj = Image.open(image)
if image_obj is None:
raise ValueError(
f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}"
f"Unrecognized image input, support local path, http url, "
f"base64 and PIL.Image, got {image}"
)
image = image_obj.convert("RGB")
## resize
# resize
if "resized_height" in ele and "resized_width" in ele:
resized_height, resized_width = smart_resize(
ele["resized_height"],
@@ -143,8 +150,10 @@ def smart_nframes(
support either `fps` or `nframes`:
- nframes: the number of frames to extract for model inputs.
- fps: the fps to extract frames for model inputs.
- min_frames: the minimum number of frames of the video, only used when fps is provided.
- max_frames: the maximum number of frames of the video, only used when fps is provided.
- min_frames: the minimum number of frames of the video,
only used when fps is provided.
- max_frames: the maximum number of frames of the video,
only used when fps is provided.
total_frames (int): the original total number of frames of the video.
video_fps (int | float): the original fps of the video.
@@ -154,34 +163,39 @@ def smart_nframes(
Returns:
int: the number of frames for video used for model inputs.
"""
assert not ("fps" in ele
and "nframes" in ele), "Only accept either `fps` or `nframes`"
assert not (
"fps" in ele and "nframes" in ele
), "Only accept either `fps` or `nframes`"
if "nframes" in ele:
nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
else:
fps = ele.get("fps", FPS)
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES),
FRAME_FACTOR)
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
max_frames = floor_by_factor(
ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)),
FRAME_FACTOR)
FRAME_FACTOR,
)
nframes = total_frames / video_fps * fps
nframes = min(max(nframes, min_frames), max_frames)
nframes = round_by_factor(nframes, FRAME_FACTOR)
if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
raise ValueError(
f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], "
f"but got {nframes}."
)
return nframes
def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
def _read_video_torchvision(
ele: dict,
) -> torch.Tensor:
"""read video using torchvision.io.read_video
Args:
ele (dict): a dict contains the configuration of video.
support keys:
- video: the path of video. support "file://", "http://", "https://" and local path.
- video: the path of video. support "file://", "http://",
"https://" and local path.
- video_start: the start time of video.
- video_end: the end time of video.
Returns:
@@ -191,7 +205,8 @@ def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
if version.parse(torchvision.__version__) < version.parse("0.19.0"):
if "http://" in video_path or "https://" in video_path:
warnings.warn(
"torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0."
"torchvision < 0.19.0 does not support http/https video path, "
"please upgrade to 0.19.0."
)
if "file://" in video_path:
video_path = video_path[7:]
@@ -206,11 +221,10 @@ def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
total_frames, video_fps = video.size(0), info["video_fps"]
total_duration = round(total_frames / video_fps, 3)
logger.info(
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, "
f"duration={total_duration}s, time={time.time() - st:.3f}s"
)
nframes = smart_nframes(ele,
total_frames=total_frames,
video_fps=video_fps)
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
idx = torch.linspace(0, total_frames - 1, nframes).round().long()
video = video[idx]
return video, total_duration, nframes
@@ -222,34 +236,38 @@ def is_decord_available() -> bool:
return importlib.util.find_spec("decord") is not None
def _read_video_decord(ele: dict, ) -> torch.Tensor:
def _read_video_decord(
ele: dict,
) -> torch.Tensor:
"""read video using decord.VideoReader
Args:
ele (dict): a dict contains the configuration of video.
support keys:
- video: the path of video. support "file://", "http://", "https://" and local path.
- video: the path of video. support "file://", "http://",
"https://" and local path.
- video_start: the start time of video.
- video_end: the end time of video.
Returns:
torch.Tensor: the video tensor with shape (T, C, H, W).
"""
import decord
video_path = ele["video"]
st = time.time()
vr = decord.VideoReader(video_path)
# TODO: support start_pts and end_pts
if 'video_start' in ele or 'video_end' in ele:
if "video_start" in ele or "video_end" in ele:
raise NotImplementedError(
"not support start_pts and end_pts in decord for now.")
"not support start_pts and end_pts in decord for now."
)
total_frames, video_fps = len(vr), vr.get_avg_fps()
total_duration = round(total_frames / video_fps, 3)
logger.info(
f"decord: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s"
f"decord: {video_path=}, {total_frames=}, {video_fps=}, "
f"time={time.time() - st:.3f}s"
)
nframes = smart_nframes(ele,
total_frames=total_frames,
video_fps=video_fps)
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
video = vr.get_batch(idx).asnumpy()
video = torch.tensor(video).permute(0, 3, 1, 2) # Convert to TCHW format
@@ -272,27 +290,27 @@ def get_video_reader_backend() -> str:
video_reader_backend = "decord"
else:
video_reader_backend = "torchvision"
# print(f"qwen-vl-utils using {video_reader_backend} to read video.", file=sys.stderr)
# print(f"qwen-vl-utils using {video_reader_backend} to read video.",
# file=sys.stderr)
return video_reader_backend
def fetch_video(
ele: dict,
image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
ele: dict, image_factor: int = IMAGE_FACTOR
) -> torch.Tensor | list[Image.Image]:
if isinstance(ele["video"], str):
video_reader_backend = get_video_reader_backend()
video, total_dur, nframes = VIDEO_READER_BACKENDS[
video_reader_backend](ele)
video, total_dur, nframes = VIDEO_READER_BACKENDS[video_reader_backend](ele)
frame_timestamps = total_dur * torch.arange(1, nframes + 1) / nframes
grid_timestamps = frame_timestamps[::FRAME_FACTOR]
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
nframes, _, height, width = video.shape
factor = spatial_patch_size * spatial_merge_size
min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
max_pixels = max(
min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
int(min_pixels * 1.05))
int(min_pixels * 1.05),
)
max_pixels = ele.get("max_pixels", max_pixels)
# min_pixels = (factor ** 2) * 52
# max_pixels = (factor ** 2) * min(768, (16384 / nframes * temporal_patch_size))
@@ -323,11 +341,9 @@ def fetch_video(
process_info.pop("type", None)
process_info.pop("video", None)
images = [
fetch_image({
"image": video_element,
**process_info
},
size_factor=image_factor)
fetch_image(
{"image": video_element, **process_info}, size_factor=image_factor
)
for video_element in ele["video"]
]
nframes = ceil_by_factor(len(images), FRAME_FACTOR)
@@ -336,8 +352,7 @@ def fetch_video(
return images, None, None, None
def extract_vision_info(
conversations: list[dict] | list[list[dict]]) -> list[dict]:
def extract_vision_info(conversations: list[dict] | list[list[dict]]) -> list[dict]:
vision_infos = []
if isinstance(conversations[0], dict):
conversations = [conversations]
@@ -345,18 +360,21 @@ def extract_vision_info(
for message in conversation:
if isinstance(message["content"], list):
for ele in message["content"]:
if ("image" in ele or "image_url" in ele or "video" in ele
or ele["type"] in ("image", "image_url", "video")):
if (
"image" in ele
or "image_url" in ele
or "video" in ele
or ele["type"] in ("image", "image_url", "video")
):
vision_infos.append(ele)
return vision_infos
def process_vision_info(
conversations: list[dict] | list[list[dict]],
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]]
| None]:
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None]:
vision_infos = extract_vision_info(conversations)
## Read images or videos
# Read images or videos
image_inputs = []
video_inputs = []
for vision_info in vision_infos:
@@ -370,4 +388,4 @@ def process_vision_info(
image_inputs = None
if len(video_inputs) == 0:
video_inputs = None
return image_inputs, video_inputs
return image_inputs, video_inputs
@@ -6,4 +6,4 @@ python end2end.py --model Qwen/Qwen2.5-Omni-7B \
--dit-ckpt none \
--bigvgan-ckpt none \
--output-wav output_audio \
--prompt_type text
--prompt_type text
+124 -118
View File
@@ -1,30 +1,33 @@
import tempfile
from typing import Dict, List, Optional, Union
from urllib.request import urlopen
import librosa
import soundfile as sf
import resampy
from typing import Dict, Optional
import torch
import requests
import torchvision.io
from typing import Union, List
from vllm.inputs import TextPrompt
from vllm_omni.inputs.data import OmniTokensPrompt
import librosa
import requests
import resampy
import soundfile as sf
import torch
import torchvision.io
from processing_omni import fetch_image, fetch_video
from vllm.inputs import TextPrompt
from vllm_omni.inputs.data import OmniTokensPrompt
def get_system_prompt():
return {
'role':
'system',
'content': [{
'type':
'text',
'text':
'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
}]
"role": "system",
"content": [
{
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual inputs, "
"as well as generating text and speech."
),
}
],
}
@@ -34,9 +37,7 @@ def resample_wav_to_16khz(input_filepath):
if len(data.shape) > 1:
data = data[:, 0]
# resample to 16kHz
data_resampled = resampy.resample(data,
sr_orig=original_sample_rate,
sr_new=16000)
data_resampled = resampy.resample(data, sr_orig=original_sample_rate, sr_new=16000)
return data_resampled
@@ -55,8 +56,7 @@ def fetch_and_read_video(args, video_url: str, fps=2):
total_duration = round(total_frames / video_fps, 3)
nframes = int(total_frames / video_fps * fps)
frame_timestamps = total_duration * torch.arange(1,
nframes + 1) / nframes
frame_timestamps = total_duration * torch.arange(1, nframes + 1) / nframes
grid_timestamps = frame_timestamps[::2]
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
@@ -70,7 +70,8 @@ def fetch_and_read_video(args, video_url: str, fps=2):
def read_video_with_transformers(video_file_name: Union[str, List[str]]):
video, total_duration, nframes, second_per_grid = fetch_video(
{'video': video_file_name})
{"video": video_file_name}
)
if total_duration is None and nframes is None:
nframes = len(video)
total_duration = 0.5 * nframes
@@ -89,7 +90,10 @@ def fetch_and_read_video(args, video_url: str, fps=2):
if isinstance(video_url, str) and video_url.startswith("http"):
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
resp = requests.get(video_url)
assert resp.status_code == requests.codes.ok, f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
assert resp.status_code == requests.codes.ok, (
f"Failed to fetch video from {video_url}, "
f"status_code:{resp.status_code}, resp:{resp}"
)
temp_video_file.write(urlopen(video_url).read())
temp_video_file_path = temp_video_file.name
@@ -106,104 +110,104 @@ def make_inputs_qwen2_omni(
use_audio_in_video: Optional[bool] = False,
tokenize: bool = False,
) -> Union[OmniTokensPrompt, TextPrompt]:
from transformers import AutoConfig, AutoProcessor, AutoTokenizer
from transformers import AutoConfig, AutoProcessor
processor = AutoProcessor.from_pretrained(args.model)
tokenizer = AutoTokenizer.from_pretrained(args.model)
try:
config = AutoConfig.from_pretrained(args.model)
if 'Qwen2_5OmniModel' in config.architectures:
if "Qwen2_5OmniModel" in config.architectures:
args.legacy_omni_video = False
else:
args.legacy_omni_video = True
except:
except Exception:
args.legacy_omni_video = True
audios, images, videos = [], [], []
for message in messages:
if not isinstance(message['content'], list):
message['content'] = [{
'type': 'text',
'text': message['content'],
}]
index, num_contents = 0, len(message['content'])
if not isinstance(message["content"], list):
message["content"] = [
{
"type": "text",
"text": message["content"],
}
]
index, num_contents = 0, len(message["content"])
while index < num_contents:
ele = message['content'][index]
if 'type' not in ele:
if 'text' in ele:
ele['type'] = 'text'
elif 'audio' in ele:
ele['type'] = 'audio'
elif 'audio_url' in ele:
ele['type'] = 'audio_url'
elif 'image' in ele:
ele['type'] = 'image'
elif 'image_url' in ele:
ele['type'] = 'image_url'
elif 'video' in ele:
ele['type'] = 'video'
elif 'video_url' in ele:
ele['type'] = 'video_url'
ele = message["content"][index]
if "type" not in ele:
if "text" in ele:
ele["type"] = "text"
elif "audio" in ele:
ele["type"] = "audio"
elif "audio_url" in ele:
ele["type"] = "audio_url"
elif "image" in ele:
ele["type"] = "image"
elif "image_url" in ele:
ele["type"] = "image_url"
elif "video" in ele:
ele["type"] = "video"
elif "video_url" in ele:
ele["type"] = "video_url"
else:
raise ValueError(f'Unknown ele: {ele}')
raise ValueError(f"Unknown ele: {ele}")
if ele['type'] == 'audio' or ele['type'] == 'audio_url':
if 'audio_url' in ele:
audio_key = 'audio_url'
with tempfile.NamedTemporaryFile(
delete=True) as temp_audio_file:
if ele["type"] == "audio" or ele["type"] == "audio_url":
if "audio_url" in ele:
audio_key = "audio_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_audio_file:
temp_audio_file.write(urlopen(ele[audio_key]).read())
temp_audio_file_path = temp_audio_file.name
audios.append(
resample_wav_to_16khz(temp_audio_file_path))
ele['audio'] = temp_audio_file_path
elif 'audio' in ele:
audio_key = 'audio'
audios.append(resample_wav_to_16khz(temp_audio_file_path))
ele["audio"] = temp_audio_file_path
elif "audio" in ele:
audio_key = "audio"
audios.append(resample_wav_to_16khz(ele[audio_key]))
else:
raise ValueError(f'Unknown ele {ele}')
elif use_audio_in_video and (ele['type'] == 'video'
or ele['type'] == 'video_url'):
raise ValueError(f"Unknown ele {ele}")
elif use_audio_in_video and (
ele["type"] == "video" or ele["type"] == "video_url"
):
# use video as audio as well
if 'video_url' in ele:
audio_key = 'video_url'
with tempfile.NamedTemporaryFile(
delete=True) as temp_video_file:
if "video_url" in ele:
audio_key = "video_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
temp_video_file.write(urlopen(ele[audio_key]).read())
temp_video_file_path = temp_video_file.name
ele[audio_key] = temp_video_file_path
audios.append(
librosa.load(temp_video_file_path, sr=16000)[0])
videos.append(
fetch_and_read_video(args, temp_video_file_path))
ele['video'] = temp_video_file_path
elif 'video' in ele:
audio_key = 'video'
audios.append(librosa.load(temp_video_file_path, sr=16000)[0])
videos.append(fetch_and_read_video(args, temp_video_file_path))
ele["video"] = temp_video_file_path
elif "video" in ele:
audio_key = "video"
audios.append(librosa.load(ele[audio_key], sr=16000)[0])
videos.append(fetch_and_read_video(args, audio_key))
else:
raise ValueError("Unknown ele {}".format(ele))
# insert a audio after the video
message['content'].insert(index + 1, {
"type": "audio",
"audio": ele[audio_key],
})
message["content"].insert(
index + 1,
{
"type": "audio",
"audio": ele[audio_key],
},
)
# no need to load the added audio again
index += 1
elif ele['type'] == 'video' or ele['type'] == 'video_url':
if 'video_url' in ele:
video_key = 'video_url'
with tempfile.NamedTemporaryFile(
delete=True) as temp_video_file:
temp_video_file.write(urlopen(ele['video_url']).read())
elif ele["type"] == "video" or ele["type"] == "video_url":
if "video_url" in ele:
video_key = "video_url"
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
temp_video_file.write(urlopen(ele["video_url"]).read())
temp_video_file_path = temp_video_file.name
videos.append(fetch_and_read_video(args, temp_video_file))
ele['video'] = temp_video_file_path
ele["video"] = temp_video_file_path
else:
video_key = 'video'
video_key = "video"
videos.append(fetch_and_read_video(args, ele[video_key]))
elif ele['type'] == 'image' or ele['type'] == 'image_url':
elif ele["type"] == "image" or ele["type"] == "image_url":
images.append(fetch_image(ele))
# move to the next content
@@ -249,11 +253,8 @@ def make_text_prompt(args, prompt):
{
"role": "user",
"content": [
{
"type": "text",
"text": prompt
},
]
{"type": "text", "text": prompt},
],
},
]
@@ -264,26 +265,29 @@ def make_text_prompt(args, prompt):
def make_audio_in_video_v2_prompt(args):
messages = [
{
'role':
'system',
'content': [{
'type':
'text',
'text':
'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
}]
},
{
"role":
"user",
"role": "system",
"content": [
{
"type":
"video_url",
"video_url":
"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual "
"inputs, as well as generating text and speech."
),
}
],
},
{
"role": "user",
"content": [
{
"type": "video_url",
"video_url": (
"https://qianwen-res.oss-cn-beijing.aliyuncs.com/"
"Qwen2.5-Omni/draw_small.mp4"
),
},
]
],
},
]
prompt = make_inputs_qwen2_omni(
@@ -295,11 +299,13 @@ def make_audio_in_video_v2_prompt(args):
return prompt
def make_omni_prompt(args, prompt = None) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
if args.prompt_type == 'text':
def make_omni_prompt(
args, prompt=None
) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
if args.prompt_type == "text":
prompt = make_text_prompt(args, prompt)
elif args.prompt_type == 'audio-in-video-v2':
elif args.prompt_type == "audio-in-video-v2":
prompt = make_audio_in_video_v2_prompt(args)
else:
raise ValueError(f'Unsupported prompt type: {args.prompt_type}')
return prompt
raise ValueError(f"Unsupported prompt type: {args.prompt_type}")
return prompt