[CI] fix ci and format existing code (#71)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -7,8 +7,8 @@ import numpy as np
 import soundfile as sf
 import torch
 from utils import make_omni_prompt
-
 from vllm.sampling_params import SamplingParams
+
 from vllm_omni.entrypoints.omni_llm import OmniLLM
 
 _os_env_toggle.environ["VLLM_USE_V1"] = "1"
@@ -45,29 +45,17 @@ def parse_args():
         default="Qwen/Qwen2.5-Omni-7B",
         help="Hugging Face repo id to download if needed.",
     )
-    parser.add_argument(
-        "--hf-revision", default=None, help="Optional HF revision (branch/tag/commit)."
-    )
-    parser.add_argument(
-        "--prompts", nargs="+", default=None, help="Input text prompts."
-    )
-    parser.add_argument(
-        "--voice-type", default="default", help="Voice type, e.g., m02, f030, default."
-    )
+    parser.add_argument("--hf-revision", default=None, help="Optional HF revision (branch/tag/commit).")
+    parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
+    parser.add_argument("--voice-type", default="default", help="Voice type, e.g., m02, f030, default.")
     parser.add_argument(
         "--code2wav-dir",
         default=None,
         help="Path to code2wav folder (contains spk_dict.pt).",
     )
-    parser.add_argument(
-        "--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt)."
-    )
-    parser.add_argument(
-        "--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file."
-    )
-    parser.add_argument(
-        "--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"]
-    )
+    parser.add_argument("--dit-ckpt", default=None, help="Path to DiT checkpoint file (e.g., dit.pt).")
+    parser.add_argument("--bigvgan-ckpt", default=None, help="Path to BigVGAN checkpoint file.")
+    parser.add_argument("--dtype", default="bfloat16", choices=["float16", "bfloat16", "float32"])
     parser.add_argument("--max-model-len", type=int, default=32768)
     parser.add_argument(
         "--init-sleep-seconds",
@@ -163,7 +151,7 @@ def main():
     try:
         # Preferred: load from txt file (one prompt per line)
         if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
-            with open(args.txt_prompts, "r", encoding="utf-8") as f:
+            with open(args.txt_prompts, encoding="utf-8") as f:
                 lines = [ln.strip() for ln in f.readlines()]
             args.prompts = [ln for ln in lines if ln != ""]
             print(f"[Info] Loaded {len(args.prompts)} prompts from {args.txt_prompts}")
@@ -172,9 +160,7 @@ def main():
         raise
 
     if args.prompts is None:
-        raise ValueError(
-            "No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)"
-        )
+        raise ValueError("No prompts provided. Use --prompts ... or --txt-prompts <file.txt> (with --prompt_type text)")
     omni_llm = OmniLLM(
         model=model_name,
         log_stats=args.enable_stats,
@@ -223,9 +209,7 @@ def main():
     omni_outputs = omni_llm.generate(prompt, sampling_params_list)
 
     # Determine output directory: prefer --output-dir; fallback to --output-wav
-    output_dir = (
-        args.output_dir if getattr(args, "output_dir", None) else args.output_wav
-    )
+    output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
     os.makedirs(output_dir, exist_ok=True)
     for stage_outputs in omni_outputs:
         if stage_outputs.final_output_type == "text":
@@ -251,9 +235,7 @@ def main():
                 request_id = int(output.request_id)
                 audio_tensor = output.multimodal_output["audio"]
                 output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
-                sf.write(
-                    output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000
-                )
+                sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
                 print(f"Request ID: {request_id}, Saved audio to {output_wav}")
 
 

examples/offline_inference/qwen2_5_omni/extract_prompts.py

@@ -28,7 +28,7 @@ def main() -> None:
     args = parser.parse_args()
 
     prompts = []
-    with open(args.input, "r", encoding="utf-8", errors="ignore") as f:
+    with open(args.input, encoding="utf-8", errors="ignore") as f:
         for line in f:
             if len(prompts) >= args.topk:
                 break

examples/offline_inference/qwen2_5_omni/processing_omni.py

@@ -72,8 +72,7 @@ def smart_resize(
     """
     if max(height, width) / min(height, width) > MAX_RATIO:
         raise ValueError(
-            f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
-            f"got {max(height, width) / min(height, width)}"
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
         )
     h_bar = max(factor, round_by_factor(height, factor))
     w_bar = max(factor, round_by_factor(width, factor))
@@ -88,9 +87,7 @@ def smart_resize(
     return h_bar, w_bar
 
 
-def fetch_image(
-    ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR
-) -> Image.Image:
+def fetch_image(ele: dict[str, str | Image.Image], size_factor: int = IMAGE_FACTOR) -> Image.Image:
     if "image" in ele:
         image = ele["image"]
     else:
@@ -110,10 +107,7 @@ def fetch_image(
     else:
         image_obj = Image.open(image)
     if image_obj is None:
-        raise ValueError(
-            f"Unrecognized image input, support local path, http url, "
-            f"base64 and PIL.Image, got {image}"
-        )
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
     image = image_obj.convert("RGB")
     # resize
     if "resized_height" in ele and "resized_width" in ele:
@@ -163,9 +157,7 @@ def smart_nframes(
     Returns:
         int: the number of frames for video used for model inputs.
     """
-    assert not (
-        "fps" in ele and "nframes" in ele
-    ), "Only accept either `fps` or `nframes`"
+    assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
     if "nframes" in ele:
         nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
     else:
@@ -179,10 +171,7 @@ def smart_nframes(
         nframes = min(max(nframes, min_frames), max_frames)
         nframes = round_by_factor(nframes, FRAME_FACTOR)
     if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
-        raise ValueError(
-            f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], "
-            f"but got {nframes}."
-        )
+        raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
     return nframes
 
 
@@ -204,10 +193,7 @@ def _read_video_torchvision(
     video_path = ele["video"]
     if version.parse(torchvision.__version__) < version.parse("0.19.0"):
         if "http://" in video_path or "https://" in video_path:
-            warnings.warn(
-                "torchvision < 0.19.0 does not support http/https video path, "
-                "please upgrade to 0.19.0."
-            )
+            warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
         if "file://" in video_path:
             video_path = video_path[7:]
     st = time.time()
@@ -221,8 +207,7 @@ def _read_video_torchvision(
     total_frames, video_fps = video.size(0), info["video_fps"]
     total_duration = round(total_frames / video_fps, 3)
     logger.info(
-        f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, "
-        f"duration={total_duration}s, time={time.time() - st:.3f}s"
+        f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
     )
     nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
     idx = torch.linspace(0, total_frames - 1, nframes).round().long()
@@ -258,15 +243,10 @@ def _read_video_decord(
     vr = decord.VideoReader(video_path)
     # TODO: support start_pts and end_pts
     if "video_start" in ele or "video_end" in ele:
-        raise NotImplementedError(
-            "not support start_pts and end_pts in decord for now."
-        )
+        raise NotImplementedError("not support start_pts and end_pts in decord for now.")
     total_frames, video_fps = len(vr), vr.get_avg_fps()
     total_duration = round(total_frames / video_fps, 3)
-    logger.info(
-        f"decord:  {video_path=}, {total_frames=}, {video_fps=}, "
-        f"time={time.time() - st:.3f}s"
-    )
+    logger.info(f"decord:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
     nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
     idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
     video = vr.get_batch(idx).asnumpy()
@@ -295,9 +275,7 @@ def get_video_reader_backend() -> str:
     return video_reader_backend
 
 
-def fetch_video(
-    ele: dict, image_factor: int = IMAGE_FACTOR
-) -> torch.Tensor | list[Image.Image]:
+def fetch_video(ele: dict, image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
     if isinstance(ele["video"], str):
         video_reader_backend = get_video_reader_backend()
         video, total_dur, nframes = VIDEO_READER_BACKENDS[video_reader_backend](ele)
@@ -341,9 +319,7 @@ def fetch_video(
         process_info.pop("type", None)
         process_info.pop("video", None)
         images = [
-            fetch_image(
-                {"image": video_element, **process_info}, size_factor=image_factor
-            )
+            fetch_image({"image": video_element, **process_info}, size_factor=image_factor)
             for video_element in ele["video"]
         ]
         nframes = ceil_by_factor(len(images), FRAME_FACTOR)

examples/offline_inference/qwen2_5_omni/utils.py

@@ -1,5 +1,5 @@
 import tempfile
-from typing import Dict, List, Optional, Union
+from typing import Optional, Union
 from urllib.request import urlopen
 
 import librosa
@@ -10,17 +10,16 @@ import torch
 import torchvision.io
 from processing_omni import fetch_image, fetch_video
 from transformers import AutoConfig, AutoProcessor
-
 from vllm.inputs import TextPrompt
+
 from vllm_omni.inputs.data import OmniTokensPrompt
 
 # Simple caches to avoid repeated heavy HF loads per prompt
-_PROCESSOR_CACHE: Dict[str, "AutoProcessor"] = {}
-_CONFIG_CACHE: Dict[str, "AutoConfig"] = {}
+_PROCESSOR_CACHE: dict[str, "AutoProcessor"] = {}
+_CONFIG_CACHE: dict[str, "AutoConfig"] = {}
 
 
 def get_system_prompt():
-
     return {
         "role": "system",
         "content": [
@@ -47,7 +46,6 @@ def resample_wav_to_16khz(input_filepath):
 
 
 def fetch_and_read_video(args, video_url: str, fps=2):
-
     def read_video_with_torchvision(video_file_name: str):
         video, audio, info = torchvision.io.read_video(
             video_file_name,
@@ -73,10 +71,8 @@ def fetch_and_read_video(args, video_url: str, fps=2):
         else:
             return video
 
-    def read_video_with_transformers(video_file_name: Union[str, List[str]]):
-        video, total_duration, nframes, second_per_grid = fetch_video(
-            {"video": video_file_name}
-        )
+    def read_video_with_transformers(video_file_name: Union[str, list[str]]):
+        video, total_duration, nframes, second_per_grid = fetch_video({"video": video_file_name})
         if total_duration is None and nframes is None:
             nframes = len(video)
             total_duration = 0.5 * nframes
@@ -96,8 +92,7 @@ def fetch_and_read_video(args, video_url: str, fps=2):
         with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
             resp = requests.get(video_url)
             assert resp.status_code == requests.codes.ok, (
-                f"Failed to fetch video from {video_url}, "
-                f"status_code:{resp.status_code}, resp:{resp}"
+                f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
             )
 
             temp_video_file.write(urlopen(video_url).read())
@@ -111,11 +106,10 @@ def fetch_and_read_video(args, video_url: str, fps=2):
 
 def make_inputs_qwen2_omni(
     args,
-    messages: List[Dict[str, Union[str, List[Dict[str, str]]]]],
+    messages: list[dict[str, Union[str, list[dict[str, str]]]]],
     use_audio_in_video: Optional[bool] = False,
     tokenize: bool = False,
 ) -> Union[OmniTokensPrompt, TextPrompt]:
-
     from transformers import AutoConfig, AutoProcessor
 
     # Cached processor/config to prevent per-prompt reloading and repeated warnings
@@ -134,7 +128,7 @@ def make_inputs_qwen2_omni(
     # Decide legacy flag only once based on config (default True if unknown)
     if getattr(args, "legacy_omni_video", None) is None:
         if config is not None and hasattr(config, "architectures"):
-            args.legacy_omni_video = not ("Qwen2_5OmniModel" in config.architectures)
+            args.legacy_omni_video = "Qwen2_5OmniModel" not in config.architectures
         else:
             args.legacy_omni_video = True
 
@@ -181,9 +175,7 @@ def make_inputs_qwen2_omni(
                     audios.append(resample_wav_to_16khz(ele[audio_key]))
                 else:
                     raise ValueError(f"Unknown ele {ele}")
-            elif use_audio_in_video and (
-                ele["type"] == "video" or ele["type"] == "video_url"
-            ):
+            elif use_audio_in_video and (ele["type"] == "video" or ele["type"] == "video_url"):
                 # use video as audio as well
                 if "video_url" in ele:
                     audio_key = "video_url"
@@ -199,7 +191,7 @@ def make_inputs_qwen2_omni(
                     audios.append(librosa.load(ele[audio_key], sr=16000)[0])
                     videos.append(fetch_and_read_video(args, audio_key))
                 else:
-                    raise ValueError("Unknown ele {}".format(ele))
+                    raise ValueError(f"Unknown ele {ele}")
                 # insert a audio after the video
                 message["content"].insert(
                     index + 1,
@@ -296,10 +288,7 @@ def make_audio_in_video_v2_prompt(args):
             "content": [
                 {
                     "type": "video_url",
-                    "video_url": (
-                        "https://qianwen-res.oss-cn-beijing.aliyuncs.com/"
-                        "Qwen2.5-Omni/draw_small.mp4"
-                    ),
+                    "video_url": ("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"),
                 },
             ],
         },
@@ -313,9 +302,7 @@ def make_audio_in_video_v2_prompt(args):
     return prompt
 
 
-def make_omni_prompt(
-    args, prompt=None
-) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
+def make_omni_prompt(args, prompt=None) -> Union[OmniTokensPrompt, list[OmniTokensPrompt]]:
     if args.prompt_type == "text":
         prompt = make_text_prompt(args, prompt)
     elif args.prompt_type == "audio-in-video-v2":

examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py

@@ -16,7 +16,6 @@ SEED = 42
 
 
 def get_system_prompt():
-
     return {
         "role": "system",
         "content": [