[Model] Support Boogu/Boogu-Image-0.1-Base and Boogu/Boogu-Image-0.1-Edit (#4995)
Signed-off-by: zzehli <jaeli_ottawa@outlook.com> Signed-off-by: jl <jaeli_ottawa@outlook.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
d34b91722f
commit
ee33954dff
26 files changed
+4973
No files matched your search
@@ -0,0 +1,213 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Native port of the upstream ``BooguImageProcessor``.
|
||||
|
||||
Ported from the external ``boogu`` package
|
||||
(``boogu/pipelines/image_processor.py``) so the native vLLM-Omni pipeline can
|
||||
reproduce Boogu-Image's reference-image and VLM preprocessing without requiring
|
||||
``pip install boogu-image``.
|
||||
|
||||
Only the two Boogu-specific methods are ported: ``get_new_height_width`` (the
|
||||
``max_pixels`` / ``max_side_length`` aware downscale-only resize) and
|
||||
``preprocess`` (which routes through it). Everything else is inherited from the
|
||||
stock diffusers ``VaeImageProcessor``.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
from diffusers.image_processor import (
|
||||
PipelineImageInput,
|
||||
VaeImageProcessor,
|
||||
is_valid_image_imagelist,
|
||||
)
|
||||
|
||||
|
||||
class BooguImageProcessor(VaeImageProcessor):
|
||||
"""VaeImageProcessor variant with Boogu-Image pixel/side-length constraints.
|
||||
|
||||
Resizing never upscales (the ratio is clamped to ``<= 1``) and always aligns
|
||||
the target height/width to multiples of ``vae_scale_factor``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
do_resize: bool = True,
|
||||
vae_scale_factor: int = 16,
|
||||
resample: str = "lanczos",
|
||||
max_pixels: int | None = None,
|
||||
max_side_length: int | None = None,
|
||||
do_normalize: bool = True,
|
||||
do_binarize: bool = False,
|
||||
do_convert_rgb: bool = False,
|
||||
do_convert_grayscale: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
do_resize=do_resize,
|
||||
vae_scale_factor=vae_scale_factor,
|
||||
resample=resample,
|
||||
do_normalize=do_normalize,
|
||||
do_binarize=do_binarize,
|
||||
do_convert_rgb=do_convert_rgb,
|
||||
do_convert_grayscale=do_convert_grayscale,
|
||||
)
|
||||
self.max_pixels = max_pixels
|
||||
self.max_side_length = max_side_length
|
||||
|
||||
def get_new_height_width(
|
||||
self,
|
||||
image: PIL.Image.Image | np.ndarray | torch.Tensor,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
max_pixels: int | None = None,
|
||||
max_side_length: int | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return target ``(height, width)`` after downscale + alignment.
|
||||
|
||||
Faithful port of upstream ``BooguImageProcessor.get_new_height_width``.
|
||||
"""
|
||||
if height is None:
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
height = image.height
|
||||
elif isinstance(image, torch.Tensor):
|
||||
height = image.shape[2]
|
||||
else:
|
||||
height = image.shape[1]
|
||||
|
||||
if width is None:
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
width = image.width
|
||||
elif isinstance(image, torch.Tensor):
|
||||
width = image.shape[3]
|
||||
else:
|
||||
width = image.shape[2]
|
||||
|
||||
if max_side_length is None:
|
||||
max_side_length = self.max_side_length
|
||||
|
||||
if max_pixels is None:
|
||||
max_pixels = self.max_pixels
|
||||
|
||||
if height <= 0 or width <= 0:
|
||||
raise ValueError(f"Image height and width must be positive, got height={height}, width={width}")
|
||||
|
||||
max_side_length_ratio = 1.0
|
||||
if max_side_length is not None:
|
||||
if height > width:
|
||||
max_side_length_ratio = max_side_length / height
|
||||
else:
|
||||
max_side_length_ratio = max_side_length / width
|
||||
|
||||
cur_pixels = height * width
|
||||
max_pixels_ratio = (max_pixels / cur_pixels) ** 0.5 if max_pixels is not None else 1.0
|
||||
# Clamp ratio to <=1 to avoid upscaling input images in preprocessing.
|
||||
ratio = min(max_pixels_ratio, max_side_length_ratio, 1.0)
|
||||
|
||||
new_height, new_width = (
|
||||
int(height * ratio) // self.config.vae_scale_factor * self.config.vae_scale_factor,
|
||||
int(width * ratio) // self.config.vae_scale_factor * self.config.vae_scale_factor,
|
||||
)
|
||||
return new_height, new_width
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
image: PipelineImageInput,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
max_pixels: int | None = None,
|
||||
max_side_length: int | None = None,
|
||||
resize_mode: str = "default",
|
||||
crops_coords: tuple[int, int, int, int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Preprocess an image into a normalized ``[B, C, H, W]`` tensor.
|
||||
|
||||
Faithful port of upstream ``BooguImageProcessor.preprocess`` (PixArt-style
|
||||
downscale). Only the PIL branch is exercised by the native pipeline, but
|
||||
the numpy/tensor branches are kept for parity.
|
||||
"""
|
||||
supported_formats = (PIL.Image.Image, np.ndarray, torch.Tensor)
|
||||
|
||||
if self.config.do_convert_grayscale and isinstance(image, (torch.Tensor, np.ndarray)) and image.ndim == 3:
|
||||
if isinstance(image, torch.Tensor):
|
||||
image = image.unsqueeze(1)
|
||||
else:
|
||||
if image.shape[-1] == 1:
|
||||
image = np.expand_dims(image, axis=0)
|
||||
else:
|
||||
image = np.expand_dims(image, axis=-1)
|
||||
|
||||
if isinstance(image, list) and isinstance(image[0], np.ndarray) and image[0].ndim == 4:
|
||||
warnings.warn(
|
||||
"Passing `image` as a list of 4d np.ndarray is deprecated."
|
||||
"Please concatenate the list along the batch dimension and pass it as a single 4d np.ndarray",
|
||||
FutureWarning,
|
||||
)
|
||||
image = np.concatenate(image, axis=0)
|
||||
if isinstance(image, list) and isinstance(image[0], torch.Tensor) and image[0].ndim == 4:
|
||||
warnings.warn(
|
||||
"Passing `image` as a list of 4d torch.Tensor is deprecated."
|
||||
"Please concatenate the list along the batch dimension and pass it as a single 4d torch.Tensor",
|
||||
FutureWarning,
|
||||
)
|
||||
image = torch.cat(image, axis=0)
|
||||
|
||||
if not is_valid_image_imagelist(image):
|
||||
supported_formats_str = ", ".join(str(x) for x in supported_formats)
|
||||
raise ValueError(f"Input is in incorrect format. Currently, we only support {supported_formats_str}")
|
||||
|
||||
if not isinstance(image, list):
|
||||
image = [image]
|
||||
|
||||
if isinstance(image[0], PIL.Image.Image):
|
||||
if crops_coords is not None:
|
||||
image = [i.crop(crops_coords) for i in image]
|
||||
if self.config.do_resize:
|
||||
height, width = self.get_new_height_width(image[0], height, width, max_pixels, max_side_length)
|
||||
image = [self.resize(i, height, width, resize_mode=resize_mode) for i in image]
|
||||
if self.config.do_convert_rgb:
|
||||
image = [self.convert_to_rgb(i) for i in image]
|
||||
elif self.config.do_convert_grayscale:
|
||||
image = [self.convert_to_grayscale(i) for i in image]
|
||||
image = self.pil_to_numpy(image)
|
||||
image = self.numpy_to_pt(image)
|
||||
|
||||
elif isinstance(image[0], np.ndarray):
|
||||
image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0)
|
||||
image = self.numpy_to_pt(image)
|
||||
height, width = self.get_new_height_width(image, height, width, max_pixels, max_side_length)
|
||||
if self.config.do_resize:
|
||||
image = self.resize(image, height, width)
|
||||
|
||||
elif isinstance(image[0], torch.Tensor):
|
||||
image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0)
|
||||
|
||||
if self.config.do_convert_grayscale and image.ndim == 3:
|
||||
image = image.unsqueeze(1)
|
||||
|
||||
channel = image.shape[1]
|
||||
if channel == self.config.vae_latent_channels:
|
||||
return image
|
||||
|
||||
height, width = self.get_new_height_width(image, height, width, max_pixels, max_side_length)
|
||||
if self.config.do_resize:
|
||||
image = self.resize(image, height, width)
|
||||
|
||||
do_normalize = self.config.do_normalize
|
||||
if do_normalize and image.min() < 0:
|
||||
warnings.warn(
|
||||
"Passing `image` as torch tensor with value range in [-1,1] is deprecated. The expected value range "
|
||||
f"for image tensor is [0,1] when passing as pytorch tensor or numpy Array. You passed `image` with "
|
||||
f"value range [{image.min()},{image.max()}]",
|
||||
FutureWarning,
|
||||
)
|
||||
do_normalize = False
|
||||
if do_normalize:
|
||||
image = self.normalize(image)
|
||||
|
||||
if self.config.do_binarize:
|
||||
image = self.binarize(image)
|
||||
|
||||
return image
|
||||
Reference in new issue
Block a user