fix conflicts
This commit is contained in:
1 parent
22f36de34e
commit
171f3c1ea7
42 files changed
+2930
-1842
No files matched your search
@@ -34,4 +34,4 @@ The output audio is saved in ./output_audio
|
||||
- [ ] Adaptation from current vllm branch to stable vllm v0.11.0
|
||||
- [ ] Offline inference example for Qwen2.5-omni with streaming multiple requests
|
||||
- [ ] Online inference support
|
||||
- [ ] Support for other models
|
||||
- [ ] Support for other models
|
||||
@@ -1,18 +1,17 @@
|
||||
import argparse
|
||||
import os
|
||||
import soundfile as sf
|
||||
import os as _os_env_toggle
|
||||
import random
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from utils import make_omni_prompt
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
import os as _os_env_toggle
|
||||
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
|
||||
|
||||
from vllm_omni.entrypoints.omni_llm import OmniLLM
|
||||
from utils import make_omni_prompt
|
||||
|
||||
_os_env_toggle.environ["VLLM_USE_V1"] = "1"
|
||||
|
||||
SEED = 42
|
||||
# Set all random seeds
|
||||
@@ -33,39 +32,84 @@ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--model', required=True, help='Path to merged model directory (will be created if downloading).')
|
||||
parser.add_argument('--thinker-model', type=str, default=None)
|
||||
parser.add_argument('--talker-model', type=str, default=None)
|
||||
parser.add_argument('--code2wav-model', type=str, default=None)
|
||||
parser.add_argument('--hf-hub-id', default='Qwen/Qwen2.5-Omni-7B', help='Hugging Face repo id to download if needed.')
|
||||
parser.add_argument('--hf-revision', default=None, help='Optional HF revision (branch/tag/commit).')
|
||||
parser.add_argument('--prompts', required=True, nargs='+', help='Input text prompts.')
|
||||
parser.add_argument('--voice-type', default='default', help='Voice type, e.g., m02, f030, default.')
|
||||
parser.add_argument('--code2wav-dir', default=None, help='Path to code2wav folder (contains spk_dict.pt).')
|
||||
parser.add_argument('--dit-ckpt', default=None, help='Path to DiT checkpoint file (e.g., dit.pt).')
|
||||
parser.add_argument('--bigvgan-ckpt', default=None, help='Path to BigVGAN checkpoint file.')
|
||||
parser.add_argument('--dtype', default='bfloat16', choices=['float16', 'bfloat16', 'float32'])
|
||||
parser.add_argument('--max-model-len', type=int, default=32768)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
required=True,
|
||||
help="Path to merged model directory (will be created if downloading).",
|
||||
)
|
||||
parser.add_argument("--thinker-model", type=str, default=None)
|
||||
parser.add_argument("--talker-model", type=str, default=None)
|
||||
parser.add_argument("--code2wav-model", type=str, default=None)
|
||||
parser.add_argument(
|
||||
"--hf-hub-id",
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
help="Hugging Face repo id to download if needed.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-revision", default=None, help="Optional HF revision (branch/tag/commit)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompts", required=True, nargs="+", help="Input text prompts."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--voice-type", default="default", help="Voice type, e.g., m02, f030, default."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--code2wav-dir",
|
||||
default=None,
|
||||
help="Path to code2wav folder (contains spk_dict.pt).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"]
|
||||
)
|
||||
parser.add_argument("--max-model-len", type=int, default=32768)
|
||||
|
||||
parser.add_argument("--thinker-only", action="store_true")
|
||||
parser.add_argument("--text-only", action="store_true")
|
||||
parser.add_argument("--do-wave", action="store_true")
|
||||
parser.add_argument('--prompt_type',
|
||||
choices=[
|
||||
'text', 'audio', 'audio-long', 'audio-long-chunks',
|
||||
'audio-long-expand-chunks', 'image', 'video',
|
||||
'video-frames', 'audio-in-video', 'audio-in-video-v2',
|
||||
"audio-multi-round", "badcase-vl", "badcase-text",
|
||||
"badcase-image-early-stop", "badcase-two-audios",
|
||||
"badcase-two-videos", "badcase-multi-round",
|
||||
"badcase-voice-type", "badcase-voice-type-v2",
|
||||
"badcase-audio-tower-1", "badcase-audio-only"
|
||||
],
|
||||
default='text')
|
||||
parser.add_argument('--use-torchvision', action='store_true')
|
||||
parser.add_argument('--tokenize', action='store_true')
|
||||
parser.add_argument('--output-wav', default="output.wav", help='Output wav file path.')
|
||||
parser.add_argument('--thinker-hidden-states-dir', default="thinker_hidden_states", help='Path to thinker hidden states directory.')
|
||||
parser.add_argument(
|
||||
"--prompt_type",
|
||||
choices=[
|
||||
"text",
|
||||
"audio",
|
||||
"audio-long",
|
||||
"audio-long-chunks",
|
||||
"audio-long-expand-chunks",
|
||||
"image",
|
||||
"video",
|
||||
"video-frames",
|
||||
"audio-in-video",
|
||||
"audio-in-video-v2",
|
||||
"audio-multi-round",
|
||||
"badcase-vl",
|
||||
"badcase-text",
|
||||
"badcase-image-early-stop",
|
||||
"badcase-two-audios",
|
||||
"badcase-two-videos",
|
||||
"badcase-multi-round",
|
||||
"badcase-voice-type",
|
||||
"badcase-voice-type-v2",
|
||||
"badcase-audio-tower-1",
|
||||
"badcase-audio-only",
|
||||
],
|
||||
default="text",
|
||||
)
|
||||
parser.add_argument("--use-torchvision", action="store_true")
|
||||
parser.add_argument("--tokenize", action="store_true")
|
||||
parser.add_argument(
|
||||
"--output-wav", default="output.wav", help="Output wav file path."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--thinker-hidden-states-dir",
|
||||
default="thinker_hidden_states",
|
||||
help="Path to thinker hidden states directory.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
@@ -75,38 +119,40 @@ def main():
|
||||
model_name = args.model
|
||||
omni_llm = OmniLLM(model=model_name)
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
talker_sampling_params = SamplingParams(
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
stop_token_ids=[8294]
|
||||
)
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
stop_token_ids=[8294],
|
||||
)
|
||||
code2wav_sampling_params = SamplingParams(
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
top_k=-1, # Disable top-k sampling
|
||||
max_tokens=2048,
|
||||
seed=SEED, # Fixed seed for sampling
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
talker_sampling_params,
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
|
||||
sampling_params_list = [thinker_sampling_params,
|
||||
talker_sampling_params,
|
||||
code2wav_sampling_params]
|
||||
|
||||
prompt = [make_omni_prompt(args, prompt) for prompt in args.prompts]
|
||||
omni_outputs = omni_llm.generate(prompt, sampling_params_list)
|
||||
|
||||
@@ -121,10 +167,14 @@ def main():
|
||||
for output in stage_outputs.request_output:
|
||||
request_id = output.request_id
|
||||
audio_tensor = output.multimodal_output["audio"]
|
||||
output_wav = os.path.join(args.output_wav, f"output_{output.request_id}.wav")
|
||||
sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
|
||||
output_wav = os.path.join(
|
||||
args.output_wav, f"output_{output.request_id}.wav"
|
||||
)
|
||||
sf.write(
|
||||
output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000
|
||||
)
|
||||
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
@@ -8,6 +8,7 @@ import time
|
||||
import warnings
|
||||
from functools import lru_cache
|
||||
from io import BytesIO
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import torchvision
|
||||
@@ -16,7 +17,6 @@ from PIL import Image
|
||||
from torchvision import io, transforms
|
||||
from torchvision.transforms import InterpolationMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_FACTOR = 28
|
||||
@@ -43,20 +43,24 @@ def round_by_factor(number: int, factor: int) -> int:
|
||||
|
||||
|
||||
def ceil_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
|
||||
"""Returns the smallest integer greater than or equal to 'number' that is
|
||||
divisible by 'factor'."""
|
||||
return math.ceil(number / factor) * factor
|
||||
|
||||
|
||||
def floor_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
|
||||
"""Returns the largest integer less than or equal to 'number' that is
|
||||
divisible by 'factor'."""
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
def smart_resize(height: int,
|
||||
width: int,
|
||||
factor: int = IMAGE_FACTOR,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS) -> tuple[int, int]:
|
||||
def smart_resize(
|
||||
height: int,
|
||||
width: int,
|
||||
factor: int = IMAGE_FACTOR,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Rescales the image so that the following conditions are met:
|
||||
|
||||
@@ -68,7 +72,8 @@ def smart_resize(height: int,
|
||||
"""
|
||||
if max(height, width) / min(height, width) > MAX_RATIO:
|
||||
raise ValueError(
|
||||
f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
|
||||
f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
|
||||
f"got {max(height, width) / min(height, width)}"
|
||||
)
|
||||
h_bar = max(factor, round_by_factor(height, factor))
|
||||
w_bar = max(factor, round_by_factor(width, factor))
|
||||
@@ -83,8 +88,9 @@ def smart_resize(height: int,
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def fetch_image(ele: dict[str, str | Image.Image],
|
||||
size_factor: int = IMAGE_FACTOR) -> Image.Image:
|
||||
def fetch_image(
|
||||
ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR
|
||||
) -> Image.Image:
|
||||
if "image" in ele:
|
||||
image = ele["image"]
|
||||
else:
|
||||
@@ -105,10 +111,11 @@ def fetch_image(ele: dict[str, str | Image.Image],
|
||||
image_obj = Image.open(image)
|
||||
if image_obj is None:
|
||||
raise ValueError(
|
||||
f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}"
|
||||
f"Unrecognized image input, support local path, http url, "
|
||||
f"base64 and PIL.Image, got {image}"
|
||||
)
|
||||
image = image_obj.convert("RGB")
|
||||
## resize
|
||||
# resize
|
||||
if "resized_height" in ele and "resized_width" in ele:
|
||||
resized_height, resized_width = smart_resize(
|
||||
ele["resized_height"],
|
||||
@@ -143,8 +150,10 @@ def smart_nframes(
|
||||
support either `fps` or `nframes`:
|
||||
- nframes: the number of frames to extract for model inputs.
|
||||
- fps: the fps to extract frames for model inputs.
|
||||
- min_frames: the minimum number of frames of the video, only used when fps is provided.
|
||||
- max_frames: the maximum number of frames of the video, only used when fps is provided.
|
||||
- min_frames: the minimum number of frames of the video,
|
||||
only used when fps is provided.
|
||||
- max_frames: the maximum number of frames of the video,
|
||||
only used when fps is provided.
|
||||
total_frames (int): the original total number of frames of the video.
|
||||
video_fps (int | float): the original fps of the video.
|
||||
|
||||
@@ -154,34 +163,39 @@ def smart_nframes(
|
||||
Returns:
|
||||
int: the number of frames for video used for model inputs.
|
||||
"""
|
||||
assert not ("fps" in ele
|
||||
and "nframes" in ele), "Only accept either `fps` or `nframes`"
|
||||
assert not (
|
||||
"fps" in ele and "nframes" in ele
|
||||
), "Only accept either `fps` or `nframes`"
|
||||
if "nframes" in ele:
|
||||
nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
|
||||
else:
|
||||
fps = ele.get("fps", FPS)
|
||||
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES),
|
||||
FRAME_FACTOR)
|
||||
min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR)
|
||||
max_frames = floor_by_factor(
|
||||
ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)),
|
||||
FRAME_FACTOR)
|
||||
FRAME_FACTOR,
|
||||
)
|
||||
nframes = total_frames / video_fps * fps
|
||||
nframes = min(max(nframes, min_frames), max_frames)
|
||||
nframes = round_by_factor(nframes, FRAME_FACTOR)
|
||||
if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
|
||||
raise ValueError(
|
||||
f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
|
||||
f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], "
|
||||
f"but got {nframes}."
|
||||
)
|
||||
return nframes
|
||||
|
||||
|
||||
def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
|
||||
def _read_video_torchvision(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using torchvision.io.read_video
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://", "https://" and local path.
|
||||
- video: the path of video. support "file://", "http://",
|
||||
"https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
@@ -191,7 +205,8 @@ def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
|
||||
if version.parse(torchvision.__version__) < version.parse("0.19.0"):
|
||||
if "http://" in video_path or "https://" in video_path:
|
||||
warnings.warn(
|
||||
"torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0."
|
||||
"torchvision < 0.19.0 does not support http/https video path, "
|
||||
"please upgrade to 0.19.0."
|
||||
)
|
||||
if "file://" in video_path:
|
||||
video_path = video_path[7:]
|
||||
@@ -206,11 +221,10 @@ def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
|
||||
total_frames, video_fps = video.size(0), info["video_fps"]
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
logger.info(
|
||||
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
|
||||
f"torchvision: {video_path=}, {total_frames=}, {video_fps=}, "
|
||||
f"duration={total_duration}s, time={time.time() - st:.3f}s"
|
||||
)
|
||||
nframes = smart_nframes(ele,
|
||||
total_frames=total_frames,
|
||||
video_fps=video_fps)
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long()
|
||||
video = video[idx]
|
||||
return video, total_duration, nframes
|
||||
@@ -222,34 +236,38 @@ def is_decord_available() -> bool:
|
||||
return importlib.util.find_spec("decord") is not None
|
||||
|
||||
|
||||
def _read_video_decord(ele: dict, ) -> torch.Tensor:
|
||||
def _read_video_decord(
|
||||
ele: dict,
|
||||
) -> torch.Tensor:
|
||||
"""read video using decord.VideoReader
|
||||
|
||||
Args:
|
||||
ele (dict): a dict contains the configuration of video.
|
||||
support keys:
|
||||
- video: the path of video. support "file://", "http://", "https://" and local path.
|
||||
- video: the path of video. support "file://", "http://",
|
||||
"https://" and local path.
|
||||
- video_start: the start time of video.
|
||||
- video_end: the end time of video.
|
||||
Returns:
|
||||
torch.Tensor: the video tensor with shape (T, C, H, W).
|
||||
"""
|
||||
import decord
|
||||
|
||||
video_path = ele["video"]
|
||||
st = time.time()
|
||||
vr = decord.VideoReader(video_path)
|
||||
# TODO: support start_pts and end_pts
|
||||
if 'video_start' in ele or 'video_end' in ele:
|
||||
if "video_start" in ele or "video_end" in ele:
|
||||
raise NotImplementedError(
|
||||
"not support start_pts and end_pts in decord for now.")
|
||||
"not support start_pts and end_pts in decord for now."
|
||||
)
|
||||
total_frames, video_fps = len(vr), vr.get_avg_fps()
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
logger.info(
|
||||
f"decord: {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s"
|
||||
f"decord: {video_path=}, {total_frames=}, {video_fps=}, "
|
||||
f"time={time.time() - st:.3f}s"
|
||||
)
|
||||
nframes = smart_nframes(ele,
|
||||
total_frames=total_frames,
|
||||
video_fps=video_fps)
|
||||
nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
|
||||
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
|
||||
video = vr.get_batch(idx).asnumpy()
|
||||
video = torch.tensor(video).permute(0, 3, 1, 2) # Convert to TCHW format
|
||||
@@ -272,27 +290,27 @@ def get_video_reader_backend() -> str:
|
||||
video_reader_backend = "decord"
|
||||
else:
|
||||
video_reader_backend = "torchvision"
|
||||
# print(f"qwen-vl-utils using {video_reader_backend} to read video.", file=sys.stderr)
|
||||
# print(f"qwen-vl-utils using {video_reader_backend} to read video.",
|
||||
# file=sys.stderr)
|
||||
return video_reader_backend
|
||||
|
||||
|
||||
def fetch_video(
|
||||
ele: dict,
|
||||
image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
|
||||
ele: dict, image_factor: int = IMAGE_FACTOR
|
||||
) -> torch.Tensor | list[Image.Image]:
|
||||
if isinstance(ele["video"], str):
|
||||
video_reader_backend = get_video_reader_backend()
|
||||
video, total_dur, nframes = VIDEO_READER_BACKENDS[
|
||||
video_reader_backend](ele)
|
||||
video, total_dur, nframes = VIDEO_READER_BACKENDS[video_reader_backend](ele)
|
||||
frame_timestamps = total_dur * torch.arange(1, nframes + 1) / nframes
|
||||
grid_timestamps = frame_timestamps[::FRAME_FACTOR]
|
||||
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
|
||||
nframes, _, height, width = video.shape
|
||||
factor = spatial_patch_size * spatial_merge_size
|
||||
min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
|
||||
total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
|
||||
max_pixels = max(
|
||||
min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
|
||||
int(min_pixels * 1.05))
|
||||
int(min_pixels * 1.05),
|
||||
)
|
||||
max_pixels = ele.get("max_pixels", max_pixels)
|
||||
# min_pixels = (factor ** 2) * 52
|
||||
# max_pixels = (factor ** 2) * min(768, (16384 / nframes * temporal_patch_size))
|
||||
@@ -323,11 +341,9 @@ def fetch_video(
|
||||
process_info.pop("type", None)
|
||||
process_info.pop("video", None)
|
||||
images = [
|
||||
fetch_image({
|
||||
"image": video_element,
|
||||
**process_info
|
||||
},
|
||||
size_factor=image_factor)
|
||||
fetch_image(
|
||||
{"image": video_element, **process_info}, size_factor=image_factor
|
||||
)
|
||||
for video_element in ele["video"]
|
||||
]
|
||||
nframes = ceil_by_factor(len(images), FRAME_FACTOR)
|
||||
@@ -336,8 +352,7 @@ def fetch_video(
|
||||
return images, None, None, None
|
||||
|
||||
|
||||
def extract_vision_info(
|
||||
conversations: list[dict] | list[list[dict]]) -> list[dict]:
|
||||
def extract_vision_info(conversations: list[dict] | list[list[dict]]) -> list[dict]:
|
||||
vision_infos = []
|
||||
if isinstance(conversations[0], dict):
|
||||
conversations = [conversations]
|
||||
@@ -345,18 +360,21 @@ def extract_vision_info(
|
||||
for message in conversation:
|
||||
if isinstance(message["content"], list):
|
||||
for ele in message["content"]:
|
||||
if ("image" in ele or "image_url" in ele or "video" in ele
|
||||
or ele["type"] in ("image", "image_url", "video")):
|
||||
if (
|
||||
"image" in ele
|
||||
or "image_url" in ele
|
||||
or "video" in ele
|
||||
or ele["type"] in ("image", "image_url", "video")
|
||||
):
|
||||
vision_infos.append(ele)
|
||||
return vision_infos
|
||||
|
||||
|
||||
def process_vision_info(
|
||||
conversations: list[dict] | list[list[dict]],
|
||||
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]]
|
||||
| None]:
|
||||
) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None]:
|
||||
vision_infos = extract_vision_info(conversations)
|
||||
## Read images or videos
|
||||
# Read images or videos
|
||||
image_inputs = []
|
||||
video_inputs = []
|
||||
for vision_info in vision_infos:
|
||||
@@ -370,4 +388,4 @@ def process_vision_info(
|
||||
image_inputs = None
|
||||
if len(video_inputs) == 0:
|
||||
video_inputs = None
|
||||
return image_inputs, video_inputs
|
||||
return image_inputs, video_inputs
|
||||
@@ -6,4 +6,4 @@ python end2end.py --model Qwen/Qwen2.5-Omni-7B \
|
||||
--dit-ckpt none \
|
||||
--bigvgan-ckpt none \
|
||||
--output-wav output_audio \
|
||||
--prompt_type text
|
||||
--prompt_type text
|
||||
@@ -1,30 +1,33 @@
|
||||
import tempfile
|
||||
from typing import Dict, List, Optional, Union
|
||||
from urllib.request import urlopen
|
||||
import librosa
|
||||
import soundfile as sf
|
||||
import resampy
|
||||
from typing import Dict, Optional
|
||||
import torch
|
||||
import requests
|
||||
import torchvision.io
|
||||
|
||||
from typing import Union, List
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
import librosa
|
||||
import requests
|
||||
import resampy
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torchvision.io
|
||||
from processing_omni import fetch_image, fetch_video
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
|
||||
|
||||
return {
|
||||
'role':
|
||||
'system',
|
||||
'content': [{
|
||||
'type':
|
||||
'text',
|
||||
'text':
|
||||
'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
|
||||
}]
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -34,9 +37,7 @@ def resample_wav_to_16khz(input_filepath):
|
||||
if len(data.shape) > 1:
|
||||
data = data[:, 0]
|
||||
# resample to 16kHz
|
||||
data_resampled = resampy.resample(data,
|
||||
sr_orig=original_sample_rate,
|
||||
sr_new=16000)
|
||||
data_resampled = resampy.resample(data, sr_orig=original_sample_rate, sr_new=16000)
|
||||
return data_resampled
|
||||
|
||||
|
||||
@@ -55,8 +56,7 @@ def fetch_and_read_video(args, video_url: str, fps=2):
|
||||
total_duration = round(total_frames / video_fps, 3)
|
||||
nframes = int(total_frames / video_fps * fps)
|
||||
|
||||
frame_timestamps = total_duration * torch.arange(1,
|
||||
nframes + 1) / nframes
|
||||
frame_timestamps = total_duration * torch.arange(1, nframes + 1) / nframes
|
||||
grid_timestamps = frame_timestamps[::2]
|
||||
second_per_grid = grid_timestamps[1] - grid_timestamps[0]
|
||||
|
||||
@@ -70,7 +70,8 @@ def fetch_and_read_video(args, video_url: str, fps=2):
|
||||
|
||||
def read_video_with_transformers(video_file_name: Union[str, List[str]]):
|
||||
video, total_duration, nframes, second_per_grid = fetch_video(
|
||||
{'video': video_file_name})
|
||||
{"video": video_file_name}
|
||||
)
|
||||
if total_duration is None and nframes is None:
|
||||
nframes = len(video)
|
||||
total_duration = 0.5 * nframes
|
||||
@@ -89,7 +90,10 @@ def fetch_and_read_video(args, video_url: str, fps=2):
|
||||
if isinstance(video_url, str) and video_url.startswith("http"):
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
resp = requests.get(video_url)
|
||||
assert resp.status_code == requests.codes.ok, f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
|
||||
assert resp.status_code == requests.codes.ok, (
|
||||
f"Failed to fetch video from {video_url}, "
|
||||
f"status_code:{resp.status_code}, resp:{resp}"
|
||||
)
|
||||
|
||||
temp_video_file.write(urlopen(video_url).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
@@ -106,104 +110,104 @@ def make_inputs_qwen2_omni(
|
||||
use_audio_in_video: Optional[bool] = False,
|
||||
tokenize: bool = False,
|
||||
) -> Union[OmniTokensPrompt, TextPrompt]:
|
||||
|
||||
from transformers import AutoConfig, AutoProcessor, AutoTokenizer
|
||||
|
||||
from transformers import AutoConfig, AutoProcessor
|
||||
|
||||
processor = AutoProcessor.from_pretrained(args.model)
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model)
|
||||
|
||||
try:
|
||||
config = AutoConfig.from_pretrained(args.model)
|
||||
if 'Qwen2_5OmniModel' in config.architectures:
|
||||
if "Qwen2_5OmniModel" in config.architectures:
|
||||
args.legacy_omni_video = False
|
||||
else:
|
||||
args.legacy_omni_video = True
|
||||
except:
|
||||
except Exception:
|
||||
args.legacy_omni_video = True
|
||||
|
||||
audios, images, videos = [], [], []
|
||||
for message in messages:
|
||||
if not isinstance(message['content'], list):
|
||||
message['content'] = [{
|
||||
'type': 'text',
|
||||
'text': message['content'],
|
||||
}]
|
||||
index, num_contents = 0, len(message['content'])
|
||||
if not isinstance(message["content"], list):
|
||||
message["content"] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": message["content"],
|
||||
}
|
||||
]
|
||||
index, num_contents = 0, len(message["content"])
|
||||
while index < num_contents:
|
||||
ele = message['content'][index]
|
||||
if 'type' not in ele:
|
||||
if 'text' in ele:
|
||||
ele['type'] = 'text'
|
||||
elif 'audio' in ele:
|
||||
ele['type'] = 'audio'
|
||||
elif 'audio_url' in ele:
|
||||
ele['type'] = 'audio_url'
|
||||
elif 'image' in ele:
|
||||
ele['type'] = 'image'
|
||||
elif 'image_url' in ele:
|
||||
ele['type'] = 'image_url'
|
||||
elif 'video' in ele:
|
||||
ele['type'] = 'video'
|
||||
elif 'video_url' in ele:
|
||||
ele['type'] = 'video_url'
|
||||
ele = message["content"][index]
|
||||
if "type" not in ele:
|
||||
if "text" in ele:
|
||||
ele["type"] = "text"
|
||||
elif "audio" in ele:
|
||||
ele["type"] = "audio"
|
||||
elif "audio_url" in ele:
|
||||
ele["type"] = "audio_url"
|
||||
elif "image" in ele:
|
||||
ele["type"] = "image"
|
||||
elif "image_url" in ele:
|
||||
ele["type"] = "image_url"
|
||||
elif "video" in ele:
|
||||
ele["type"] = "video"
|
||||
elif "video_url" in ele:
|
||||
ele["type"] = "video_url"
|
||||
else:
|
||||
raise ValueError(f'Unknown ele: {ele}')
|
||||
raise ValueError(f"Unknown ele: {ele}")
|
||||
|
||||
if ele['type'] == 'audio' or ele['type'] == 'audio_url':
|
||||
if 'audio_url' in ele:
|
||||
audio_key = 'audio_url'
|
||||
with tempfile.NamedTemporaryFile(
|
||||
delete=True) as temp_audio_file:
|
||||
if ele["type"] == "audio" or ele["type"] == "audio_url":
|
||||
if "audio_url" in ele:
|
||||
audio_key = "audio_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_audio_file:
|
||||
temp_audio_file.write(urlopen(ele[audio_key]).read())
|
||||
temp_audio_file_path = temp_audio_file.name
|
||||
audios.append(
|
||||
resample_wav_to_16khz(temp_audio_file_path))
|
||||
ele['audio'] = temp_audio_file_path
|
||||
elif 'audio' in ele:
|
||||
audio_key = 'audio'
|
||||
audios.append(resample_wav_to_16khz(temp_audio_file_path))
|
||||
ele["audio"] = temp_audio_file_path
|
||||
elif "audio" in ele:
|
||||
audio_key = "audio"
|
||||
audios.append(resample_wav_to_16khz(ele[audio_key]))
|
||||
else:
|
||||
raise ValueError(f'Unknown ele {ele}')
|
||||
elif use_audio_in_video and (ele['type'] == 'video'
|
||||
or ele['type'] == 'video_url'):
|
||||
raise ValueError(f"Unknown ele {ele}")
|
||||
elif use_audio_in_video and (
|
||||
ele["type"] == "video" or ele["type"] == "video_url"
|
||||
):
|
||||
# use video as audio as well
|
||||
if 'video_url' in ele:
|
||||
audio_key = 'video_url'
|
||||
with tempfile.NamedTemporaryFile(
|
||||
delete=True) as temp_video_file:
|
||||
if "video_url" in ele:
|
||||
audio_key = "video_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
temp_video_file.write(urlopen(ele[audio_key]).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
ele[audio_key] = temp_video_file_path
|
||||
audios.append(
|
||||
librosa.load(temp_video_file_path, sr=16000)[0])
|
||||
videos.append(
|
||||
fetch_and_read_video(args, temp_video_file_path))
|
||||
ele['video'] = temp_video_file_path
|
||||
elif 'video' in ele:
|
||||
audio_key = 'video'
|
||||
audios.append(librosa.load(temp_video_file_path, sr=16000)[0])
|
||||
videos.append(fetch_and_read_video(args, temp_video_file_path))
|
||||
ele["video"] = temp_video_file_path
|
||||
elif "video" in ele:
|
||||
audio_key = "video"
|
||||
audios.append(librosa.load(ele[audio_key], sr=16000)[0])
|
||||
videos.append(fetch_and_read_video(args, audio_key))
|
||||
else:
|
||||
raise ValueError("Unknown ele {}".format(ele))
|
||||
# insert a audio after the video
|
||||
message['content'].insert(index + 1, {
|
||||
"type": "audio",
|
||||
"audio": ele[audio_key],
|
||||
})
|
||||
message["content"].insert(
|
||||
index + 1,
|
||||
{
|
||||
"type": "audio",
|
||||
"audio": ele[audio_key],
|
||||
},
|
||||
)
|
||||
# no need to load the added audio again
|
||||
index += 1
|
||||
elif ele['type'] == 'video' or ele['type'] == 'video_url':
|
||||
if 'video_url' in ele:
|
||||
video_key = 'video_url'
|
||||
with tempfile.NamedTemporaryFile(
|
||||
delete=True) as temp_video_file:
|
||||
temp_video_file.write(urlopen(ele['video_url']).read())
|
||||
elif ele["type"] == "video" or ele["type"] == "video_url":
|
||||
if "video_url" in ele:
|
||||
video_key = "video_url"
|
||||
with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
|
||||
temp_video_file.write(urlopen(ele["video_url"]).read())
|
||||
temp_video_file_path = temp_video_file.name
|
||||
videos.append(fetch_and_read_video(args, temp_video_file))
|
||||
ele['video'] = temp_video_file_path
|
||||
ele["video"] = temp_video_file_path
|
||||
else:
|
||||
video_key = 'video'
|
||||
video_key = "video"
|
||||
videos.append(fetch_and_read_video(args, ele[video_key]))
|
||||
elif ele['type'] == 'image' or ele['type'] == 'image_url':
|
||||
elif ele["type"] == "image" or ele["type"] == "image_url":
|
||||
images.append(fetch_image(ele))
|
||||
|
||||
# move to the next content
|
||||
@@ -249,11 +253,8 @@ def make_text_prompt(args, prompt):
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt
|
||||
},
|
||||
]
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
@@ -264,26 +265,29 @@ def make_text_prompt(args, prompt):
|
||||
def make_audio_in_video_v2_prompt(args):
|
||||
messages = [
|
||||
{
|
||||
'role':
|
||||
'system',
|
||||
'content': [{
|
||||
'type':
|
||||
'text',
|
||||
'text':
|
||||
'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
|
||||
}]
|
||||
},
|
||||
{
|
||||
"role":
|
||||
"user",
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type":
|
||||
"video_url",
|
||||
"video_url":
|
||||
"https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual "
|
||||
"inputs, as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": (
|
||||
"https://qianwen-res.oss-cn-beijing.aliyuncs.com/"
|
||||
"Qwen2.5-Omni/draw_small.mp4"
|
||||
),
|
||||
},
|
||||
]
|
||||
],
|
||||
},
|
||||
]
|
||||
prompt = make_inputs_qwen2_omni(
|
||||
@@ -295,11 +299,13 @@ def make_audio_in_video_v2_prompt(args):
|
||||
return prompt
|
||||
|
||||
|
||||
def make_omni_prompt(args, prompt = None) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
|
||||
if args.prompt_type == 'text':
|
||||
def make_omni_prompt(
|
||||
args, prompt=None
|
||||
) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
|
||||
if args.prompt_type == "text":
|
||||
prompt = make_text_prompt(args, prompt)
|
||||
elif args.prompt_type == 'audio-in-video-v2':
|
||||
elif args.prompt_type == "audio-in-video-v2":
|
||||
prompt = make_audio_in_video_v2_prompt(args)
|
||||
else:
|
||||
raise ValueError(f'Unsupported prompt type: {args.prompt_type}')
|
||||
return prompt
|
||||
raise ValueError(f"Unsupported prompt type: {args.prompt_type}")
|
||||
return prompt
|
||||
Reference in new issue
Block a user