[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
9ece725fc5
commit
d64bbde561
64 files changed
+2995
-1414
No files matched your search
@@ -272,16 +272,22 @@ class BagelPipeline(nn.Module):
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
prompt = req.prompt or ""
|
||||
if isinstance(prompt, list):
|
||||
# vllm-omni request supports list; Bagel pipeline currently supports first prompt.
|
||||
prompt = prompt[0] if prompt else ""
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
# TODO: In online mode, sometimes it receives [{"prompts": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(req.prompts[0], str) else (req.prompts[0].get("prompt") or "")
|
||||
|
||||
max_hw = int(self.bagel.max_latent_size * self.bagel.latent_downsample)
|
||||
if req.height is None and req.width is None:
|
||||
if req.sampling_params.height is None and req.sampling_params.width is None:
|
||||
height = width = max_hw
|
||||
else:
|
||||
height = int(req.height) if req.height is not None else max_hw
|
||||
width = int(req.width) if req.width is not None else max_hw
|
||||
height = int(req.sampling_params.height) if req.sampling_params.height is not None else max_hw
|
||||
width = int(req.sampling_params.width) if req.sampling_params.width is not None else max_hw
|
||||
if height > max_hw or width > max_hw:
|
||||
raise ValueError(
|
||||
f"Requested resolution {height}x{width} exceeds Bagel checkpoint limit "
|
||||
@@ -292,7 +298,7 @@ class BagelPipeline(nn.Module):
|
||||
|
||||
# Map request params to Bagel gen params (defaults follow Bagel inferencer)
|
||||
gen_params = BagelGenParams(
|
||||
num_timesteps=int(req.num_inference_steps or 50),
|
||||
num_timesteps=int(req.sampling_params.num_inference_steps or 50),
|
||||
timestep_shift=3.0,
|
||||
)
|
||||
|
||||
@@ -304,7 +310,7 @@ class BagelPipeline(nn.Module):
|
||||
|
||||
# Add text prompt (prefill) on gen context.
|
||||
# [Omni] Check for injected KV Cache from remote transfer
|
||||
injected_kv = getattr(req, "past_key_values", None)
|
||||
injected_kv = req.sampling_params.past_key_values
|
||||
if injected_kv is not None:
|
||||
logger.info("Using injected KV Cache (direct)")
|
||||
gen_context["past_key_values"] = injected_kv
|
||||
@@ -316,9 +322,13 @@ class BagelPipeline(nn.Module):
|
||||
gen_context["ropes"] = [seq_len]
|
||||
|
||||
else:
|
||||
image_input = getattr(req, "pil_image", None)
|
||||
image_input = (
|
||||
None if isinstance(first_prompt, str) else (first_prompt.get("multi_modal_data") or {}).get("image")
|
||||
)
|
||||
if image_input and not isinstance(image_input, list):
|
||||
image_input = [image_input]
|
||||
if image_input:
|
||||
image_input = [Image.open(image) if isinstance(image, str) else image for image in image_input]
|
||||
|
||||
if image_input:
|
||||
# If we have an image, we prefill with it
|
||||
@@ -414,10 +424,10 @@ class BagelPipeline(nn.Module):
|
||||
gen_context["kv_lens"] = newlens
|
||||
gen_context["ropes"] = new_rope
|
||||
|
||||
if req.seed is not None:
|
||||
torch.manual_seed(req.seed)
|
||||
if req.sampling_params.seed is not None:
|
||||
torch.manual_seed(req.sampling_params.seed)
|
||||
if self.device.type == "cuda":
|
||||
torch.cuda.manual_seed(req.seed)
|
||||
torch.cuda.manual_seed(req.sampling_params.seed)
|
||||
|
||||
# Prepare latent query and run flow
|
||||
generation_input = self.bagel.prepare_vae_latent(
|
||||
|
||||
@@ -19,7 +19,7 @@ import json
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -742,27 +742,54 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput):
|
||||
`return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
|
||||
generated images.
|
||||
"""
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
image = req.pil_image if req.pil_image is not None else image
|
||||
height = req.height or height
|
||||
width = req.width or width
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
|
||||
generator = req.generator or generator
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
if (
|
||||
raw_image := None
|
||||
if isinstance(first_prompt, str)
|
||||
else first_prompt.get("multi_modal_data", {}).get("image")
|
||||
) is None:
|
||||
pass # use image from param list
|
||||
elif isinstance(raw_image, list):
|
||||
image = [PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image, im) for im in raw_image]
|
||||
else:
|
||||
image = PIL.Image.open(raw_image) if isinstance(raw_image, str) else cast(PIL.Image.Image, raw_image)
|
||||
|
||||
if isinstance(req.prompt_embeds, torch.Tensor):
|
||||
prompt_embeds = req.prompt_embeds
|
||||
if isinstance(req.negative_prompt_embeds, torch.Tensor):
|
||||
negative_prompt_embeds = req.negative_prompt_embeds
|
||||
height = req.sampling_params.height or height
|
||||
width = req.sampling_params.width or width
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
guidance_scale = (
|
||||
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
|
||||
)
|
||||
generator = req.sampling_params.generator or generator
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
text_encoder_out_layers = req.sampling_params.extra_args.get("text_encoder_out_layers", text_encoder_out_layers)
|
||||
|
||||
if req.max_sequence_length is not None:
|
||||
max_sequence_length = req.max_sequence_length
|
||||
if getattr(req, "text_encoder_out_layers", None) is not None:
|
||||
text_encoder_out_layers = req.text_encoder_out_layers
|
||||
req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts]
|
||||
if any(p is not None for p in req_prompt_embeds):
|
||||
# If at list one prompt is provided as an embedding,
|
||||
# Then assume that the user wants to provide embeddings for all prompts, and enter this if block
|
||||
# If the user in fact provides mixed input format, req_prompt_embeds will have some None's
|
||||
# And `torch.stack` automatically raises an exception for us
|
||||
prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError
|
||||
|
||||
req_negative_prompt_embeds = [
|
||||
p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts
|
||||
]
|
||||
if any(p is not None for p in req_negative_prompt_embeds):
|
||||
negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError
|
||||
|
||||
# 1. Check inputs. Raise error if not correct
|
||||
self.check_inputs(
|
||||
|
||||
@@ -17,6 +17,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -48,6 +49,7 @@ from vllm_omni.diffusion.models.glm_image.glm_image_transformer import (
|
||||
GlmImageTransformer2DModel,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
@@ -77,16 +79,26 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
# GLM-Image uses patch_size=2 for transformer
|
||||
patch_size = 2
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]):
|
||||
def pre_process_func(request: OmniDiffusionRequest):
|
||||
"""Pre-process condition images for Image Edit mode."""
|
||||
for req in requests:
|
||||
images = req.pil_image
|
||||
if images is None:
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if raw_image is None:
|
||||
# Text-to-image mode, no preprocessing needed
|
||||
continue
|
||||
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
if not isinstance(raw_image, list):
|
||||
raw_image = [raw_image]
|
||||
images = [
|
||||
PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im)
|
||||
for im in raw_image
|
||||
]
|
||||
|
||||
preprocessed = []
|
||||
height, width = None, None
|
||||
@@ -110,14 +122,19 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
height, width = img_h, img_w
|
||||
|
||||
# Store in request
|
||||
req.preprocessed_image = preprocessed
|
||||
req.prompt_image = images # Keep original PIL images
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt, additional_information={})
|
||||
elif "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
prompt["additional_information"]["preprocessed_image"] = processed # type: ignore
|
||||
prompt["additional_information"]["prompt_image"] = images # type: ignore
|
||||
request.prompts[i] = prompt
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
|
||||
return requests
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -821,27 +838,44 @@ class GlmImagePipeline(nn.Module):
|
||||
Returns:
|
||||
DiffusionOutput containing generated image
|
||||
"""
|
||||
prompt = req.prompt or ""
|
||||
if isinstance(prompt, list):
|
||||
prompt = prompt[0] if prompt else ""
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
|
||||
# Get pre-computed prompt embeddings if provided
|
||||
prompt_embeds = req.prompt_embeds if isinstance(req.prompt_embeds, torch.Tensor) else None
|
||||
if isinstance(first_prompt, str):
|
||||
prompt_embeds = None
|
||||
else:
|
||||
prompt_embeds = first_prompt.get("prompt_embeds")
|
||||
if not isinstance(prompt_embeds, torch.Tensor):
|
||||
prompt_embeds = None
|
||||
|
||||
# Get condition images for Image Edit mode
|
||||
# Use pre-processed images from pre_process_func
|
||||
preprocessed_images = req.preprocessed_image
|
||||
condition_images = getattr(req, "prompt_image", None)
|
||||
img_height = req.height
|
||||
img_width = req.width
|
||||
preprocessed_images = (
|
||||
None
|
||||
if isinstance(first_prompt, str)
|
||||
else first_prompt.get("additional_information", {}).get("preprocessed_image")
|
||||
)
|
||||
condition_images = (
|
||||
None
|
||||
if isinstance(first_prompt, str)
|
||||
else first_prompt.get("additional_information", {}).get("prompt_image")
|
||||
)
|
||||
img_height = req.sampling_params.height
|
||||
img_width = req.sampling_params.width
|
||||
|
||||
is_image_edit = preprocessed_images is not None
|
||||
|
||||
# Use image dimensions as default if available
|
||||
height = req.height or img_height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or img_width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.num_inference_steps or 50
|
||||
guidance_scale = req.guidance_scale or 1.5
|
||||
height = req.sampling_params.height or img_height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or img_width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or 50
|
||||
guidance_scale = req.sampling_params.guidance_scale or 1.5
|
||||
|
||||
# 0. Validate inputs
|
||||
self.check_inputs(prompt=prompt, height=height, width=width, prompt_embeds=prompt_embeds)
|
||||
@@ -851,13 +885,13 @@ class GlmImagePipeline(nn.Module):
|
||||
|
||||
# Set seed if provided
|
||||
generator = None
|
||||
if req.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(req.seed)
|
||||
if req.sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed)
|
||||
|
||||
# 1. Get prior tokens - either from external source (multistage) or generate internally
|
||||
# Check if prior_token_ids are provided externally (from AR stage in multistage mode)
|
||||
external_prior_tokens = req.extra.get("prior_token_ids") if req.extra else None
|
||||
external_prior_image_ids = req.extra.get("prior_token_image_ids") if req.extra else None
|
||||
external_prior_tokens = req.sampling_params.extra_args.get("prior_token_ids")
|
||||
external_prior_image_ids = req.sampling_params.extra_args.get("prior_token_image_ids")
|
||||
|
||||
if external_prior_tokens is not None:
|
||||
# Multistage mode: use externally provided prior tokens from vLLM AR stage
|
||||
|
||||
@@ -451,7 +451,7 @@ class LongCatImagePipeline(
|
||||
raise ValueError(
|
||||
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
|
||||
)
|
||||
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
|
||||
elif prompt is not None and not isinstance(prompt, (str, list)):
|
||||
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
|
||||
|
||||
if negative_prompt is not None and negative_prompt_embeds is not None:
|
||||
@@ -470,11 +470,11 @@ class LongCatImagePipeline(
|
||||
num_inference_steps: int = 50,
|
||||
sigmas: list[float] | None = None,
|
||||
guidance_scale: float = 4.5,
|
||||
num_images_per_prompt: int | None = 1,
|
||||
num_images_per_prompt: int = 1,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
latents: torch.FloatTensor | None = None,
|
||||
prompt_embeds: torch.FloatTensor | None = None,
|
||||
negative_prompt_embeds: torch.FloatTensor | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
output_type: str | None = "pil",
|
||||
return_dict: bool = True,
|
||||
joint_attention_kwargs: dict[str, Any] | None = None,
|
||||
@@ -482,20 +482,44 @@ class LongCatImagePipeline(
|
||||
cfg_renorm_min: float | None = 0.0,
|
||||
enable_prompt_rewrite: bool | None = True,
|
||||
) -> DiffusionOutput:
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
|
||||
negative_prompt = None
|
||||
elif req.prompts:
|
||||
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
|
||||
|
||||
height = req.height or height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
guidance_scale = req.guidance_scale if getattr(req, "guidance_scale", None) is not None else guidance_scale
|
||||
num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt
|
||||
enable_prompt_rewrite = getattr(req, "enable_prompt_rewrite", None) or enable_prompt_rewrite
|
||||
enable_cfg_renorm = getattr(req, "enable_cfg_renorm", None) or enable_cfg_renorm
|
||||
cfg_renorm_min = getattr(req, "cfg_renorm_min", None) or cfg_renorm_min
|
||||
prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds
|
||||
negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds
|
||||
height = req.sampling_params.height or height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
generator = req.sampling_params.generator or generator
|
||||
guidance_scale = (
|
||||
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
|
||||
)
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt is not None
|
||||
else num_images_per_prompt
|
||||
)
|
||||
enable_prompt_rewrite = req.sampling_params.extra_args.get("enable_prompt_rewrite", enable_prompt_rewrite)
|
||||
enable_cfg_renorm = req.sampling_params.extra_args.get("enable_cfg_renorm", enable_cfg_renorm)
|
||||
cfg_renorm_min = req.sampling_params.extra_args.get("cfg_renorm_min", cfg_renorm_min)
|
||||
|
||||
req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts]
|
||||
if any(p is not None for p in req_prompt_embeds):
|
||||
# If at list one prompt is provided as an embedding,
|
||||
# Then assume that the user wants to provide embeddings for all prompts, and enter this if block
|
||||
# If the user in fact provides mixed input format, req_prompt_embeds will have some None's
|
||||
# And `torch.stack` automatically raises an exception for us
|
||||
prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError
|
||||
|
||||
req_negative_prompt_embeds = [
|
||||
p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts
|
||||
]
|
||||
if any(p is not None for p in req_negative_prompt_embeds):
|
||||
negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError
|
||||
|
||||
self.check_inputs(
|
||||
prompt,
|
||||
|
||||
@@ -6,7 +6,7 @@ import math
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import (
|
||||
)
|
||||
from vllm_omni.diffusion.models.longcat_image.pipeline_longcat_image import calculate_shift
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
@@ -59,22 +60,38 @@ def get_longcat_image_edit_pre_process_func(
|
||||
latent_channels = vae_config.get("latent_channels", 16)
|
||||
|
||||
def pre_process_func(
|
||||
requests: list[OmniDiffusionRequest],
|
||||
request: OmniDiffusionRequest,
|
||||
):
|
||||
"""Pre-process requests for QwenImageEditPipeline."""
|
||||
for req in requests:
|
||||
image = req.pil_image
|
||||
"""Pre-process requests for LongCatImageEditPipeline."""
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
if raw_image is None or isinstance(raw_image, list):
|
||||
raise ValueError(
|
||||
"""Received no image or a list of image. Only a single image is supported by this model."""
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
|
||||
)
|
||||
|
||||
if isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
|
||||
|
||||
image_size = image.size
|
||||
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1])
|
||||
height = req.height or calculated_height
|
||||
width = req.width or calculated_width
|
||||
height = request.sampling_params.height or calculated_height
|
||||
width = request.sampling_params.width or calculated_width
|
||||
|
||||
# Store calculated dimensions in request
|
||||
req.calculated_height = calculated_height
|
||||
req.calculated_width = calculated_width
|
||||
req.height = height
|
||||
req.width = width
|
||||
prompt["additional_information"]["calculated_height"] = calculated_height
|
||||
prompt["additional_information"]["calculated_width"] = calculated_width
|
||||
request.sampling_params.height = height
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Preprocess image
|
||||
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels):
|
||||
@@ -83,9 +100,10 @@ def get_longcat_image_edit_pre_process_func(
|
||||
image = image_processor.preprocess(image, calculated_height, calculated_width)
|
||||
|
||||
# Store preprocessed image and prompt image in request
|
||||
req.preprocessed_image = image
|
||||
req.prompt_image = prompt_image
|
||||
return requests
|
||||
prompt["additional_information"]["preprocessed_image"] = image
|
||||
prompt["additional_information"]["prompt_image"] = prompt_image
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -500,42 +518,59 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput):
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
prompt: str | list[str] = None,
|
||||
negative_prompt: str | list[str] = None,
|
||||
prompt: str | list[str] | None = None,
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
num_inference_steps: int = 50,
|
||||
sigmas: list[float] | None = None,
|
||||
guidance_scale: float = 3.5,
|
||||
num_images_per_prompt: int | None = 1,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
latents: torch.FloatTensor | None = None,
|
||||
prompt_embeds: torch.FloatTensor | None = None,
|
||||
negative_prompt_embeds: torch.FloatTensor | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
output_type: str | None = "pil",
|
||||
return_dict: bool = True,
|
||||
joint_attention_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
negative_prompt = "" if negative_prompt is None else negative_prompt
|
||||
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt
|
||||
generator = req.generator or generator
|
||||
prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds
|
||||
negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds
|
||||
height = req.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or self.default_sample_size * self.vae_scale_factor
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
|
||||
prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("prompt_embeds")
|
||||
negative_prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt_embeds") # type: ignore # Why it is list[torch.Tensor] in OmniTokenInputs or OmniEmbedsPrompt? Doesn't make sense
|
||||
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
guidance_scale = (
|
||||
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
|
||||
)
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt is not None
|
||||
else num_images_per_prompt
|
||||
)
|
||||
generator = req.sampling_params.generator or generator
|
||||
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
|
||||
|
||||
if prompt is not None:
|
||||
batch_size = 1 if isinstance(prompt, str) else len(prompt)
|
||||
else:
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
if hasattr(req, "preprocessed_image"):
|
||||
prompt_image = req.prompt_image
|
||||
image = req.preprocessed_image
|
||||
calculated_height = req.calculated_height if hasattr(req, "calculated_height") else height
|
||||
calculated_width = req.calculated_width if hasattr(req, "calculated_width") else width
|
||||
if not isinstance(first_prompt, str) and "preprocessed_image" in (
|
||||
additional_information := first_prompt.get("additional_information", {})
|
||||
):
|
||||
prompt_image = additional_information.get("prompt_image")
|
||||
image = additional_information.get("preprocessed_image")
|
||||
calculated_height = additional_information.get("calculated_height", height)
|
||||
calculated_width = additional_information.get("calculated_width", width)
|
||||
else:
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1])
|
||||
|
||||
@@ -519,8 +519,8 @@ class OvisImagePipeline(
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
prompt: str | list[str] = None,
|
||||
negative_prompt: str | list[str] = None,
|
||||
prompt: str | list[str] | None = None,
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
guidance_scale: float = 5.0,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
@@ -606,16 +606,27 @@ class OvisImagePipeline(
|
||||
[`~pipelines.ovis_image.OvisImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When
|
||||
returning a tuple, the first element is a list with the generated images.
|
||||
"""
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
height = req.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
|
||||
generator = req.generator or generator
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
|
||||
negative_prompt = None
|
||||
elif req.prompts:
|
||||
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
|
||||
|
||||
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
guidance_scale = (
|
||||
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
|
||||
)
|
||||
generator = req.sampling_params.generator or generator
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
|
||||
# Steps:
|
||||
# 1. Check Inputs
|
||||
|
||||
@@ -662,21 +662,28 @@ class QwenImagePipeline(
|
||||
callback_on_step_end_tensor_inputs: list[str] = ["latents"],
|
||||
max_sequence_length: int = 512,
|
||||
) -> DiffusionOutput:
|
||||
# # TODO: only support single prompt now
|
||||
# if req.prompt is not None:
|
||||
# prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt
|
||||
prompt = req.prompt
|
||||
negative_prompt = req.negative_prompt
|
||||
height = req.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
|
||||
negative_prompt = None
|
||||
elif req.prompts:
|
||||
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
|
||||
|
||||
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
generator = req.sampling_params.generator or generator
|
||||
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
# 1. check inputs
|
||||
# 2. encode prompts
|
||||
# 3. prepare latents and timesteps
|
||||
|
||||
@@ -7,7 +7,7 @@ import logging
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -39,6 +39,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
@@ -64,16 +65,33 @@ def get_qwen_image_edit_pre_process_func(
|
||||
latent_channels = vae_config.get("z_dim", 16)
|
||||
|
||||
def pre_process_func(
|
||||
requests: list[OmniDiffusionRequest],
|
||||
request: OmniDiffusionRequest,
|
||||
):
|
||||
"""Pre-process requests for QwenImageEditPipeline."""
|
||||
for req in requests:
|
||||
image = req.pil_image
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
# Only handles single image
|
||||
if raw_image is None or isinstance(raw_image, list):
|
||||
raise ValueError(
|
||||
"""Received no image or a list of image. Only a single image is supported by this model."""
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
|
||||
)
|
||||
|
||||
if isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
|
||||
|
||||
image_size = image.size
|
||||
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1])
|
||||
height = req.height or calculated_height
|
||||
width = req.width or calculated_width
|
||||
height = request.sampling_params.height or calculated_height
|
||||
width = request.sampling_params.width or calculated_width
|
||||
|
||||
# Ensure dimensions are multiples of vae_scale_factor * 2
|
||||
multiple_of = vae_scale_factor * 2
|
||||
@@ -81,10 +99,10 @@ def get_qwen_image_edit_pre_process_func(
|
||||
width = width // multiple_of * multiple_of
|
||||
|
||||
# Store calculated dimensions in request
|
||||
req.calculated_height = calculated_height
|
||||
req.calculated_width = calculated_width
|
||||
req.height = height
|
||||
req.width = width
|
||||
prompt["additional_information"]["calculated_height"] = calculated_height
|
||||
prompt["additional_information"]["calculated_width"] = calculated_width
|
||||
request.sampling_params.height = height
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Preprocess image
|
||||
if image is not None and not (
|
||||
@@ -96,10 +114,10 @@ def get_qwen_image_edit_pre_process_func(
|
||||
image = image.unsqueeze(2)
|
||||
|
||||
# Store preprocessed image and prompt image in request
|
||||
req.preprocessed_image = image
|
||||
req.prompt_image = prompt_image
|
||||
|
||||
return requests
|
||||
prompt["additional_information"]["preprocessed_image"] = image
|
||||
prompt["additional_information"]["prompt_image"] = prompt_image
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -717,17 +735,27 @@ class QwenImageEditPipeline(
|
||||
max_sequence_length: int = 512,
|
||||
) -> DiffusionOutput:
|
||||
"""Forward pass for image editing."""
|
||||
prompt = req.prompt
|
||||
negative_prompt = req.negative_prompt
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
|
||||
|
||||
# Get preprocessed image from request (pre-processing is done in DiffusionEngine)
|
||||
if hasattr(req, "preprocessed_image"):
|
||||
prompt_image = req.prompt_image
|
||||
image = req.preprocessed_image
|
||||
calculated_height = req.calculated_height
|
||||
calculated_width = req.calculated_width
|
||||
height = req.height
|
||||
width = req.width
|
||||
if not isinstance(first_prompt, str) and "preprocessed_image" in (
|
||||
additional_information := first_prompt.get("additional_information", {})
|
||||
):
|
||||
prompt_image = additional_information.get("prompt_image")
|
||||
image = additional_information.get("preprocessed_image")
|
||||
calculated_height = additional_information.get("calculated_height")
|
||||
calculated_width = additional_information.get("calculated_width")
|
||||
height = req.sampling_params.height
|
||||
width = req.sampling_params.width
|
||||
else:
|
||||
# fallback to run pre-processing in pipeline (debug only)
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
@@ -745,14 +773,18 @@ class QwenImageEditPipeline(
|
||||
image = self.image_processor.preprocess(image, calculated_height, calculated_width)
|
||||
image = image.unsqueeze(2)
|
||||
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
generator = req.sampling_params.generator or generator
|
||||
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
|
||||
# 1. check inputs
|
||||
# 2. encode prompts
|
||||
|
||||
@@ -5,7 +5,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -42,6 +42,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
@@ -70,24 +71,33 @@ def get_qwen_image_edit_plus_pre_process_func(
|
||||
latent_channels = vae_config.get("z_dim", 16)
|
||||
|
||||
def pre_process_func(
|
||||
requests: list[OmniDiffusionRequest],
|
||||
request: OmniDiffusionRequest,
|
||||
):
|
||||
"""Pre-process requests for QwenImageEditPlusPipeline."""
|
||||
for req in requests:
|
||||
image = req.pil_image
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
# Handle single image or list of images
|
||||
if image is None:
|
||||
if raw_image is None:
|
||||
continue
|
||||
|
||||
if not isinstance(image, list):
|
||||
image = [image]
|
||||
if not isinstance(raw_image, list):
|
||||
raw_image = [raw_image]
|
||||
image = [
|
||||
PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im)
|
||||
for im in raw_image
|
||||
]
|
||||
|
||||
# Calculate dimensions based on first image
|
||||
image_size = image[0].size
|
||||
calculated_width, calculated_height = calculate_dimensions(VAE_IMAGE_SIZE, image_size[0] / image_size[1])
|
||||
height = req.height or calculated_height
|
||||
width = req.width or calculated_width
|
||||
height = request.sampling_params.height or calculated_height
|
||||
width = request.sampling_params.width or calculated_width
|
||||
|
||||
# Ensure dimensions are multiples of vae_scale_factor * 2
|
||||
multiple_of = vae_scale_factor * 2
|
||||
@@ -95,10 +105,10 @@ def get_qwen_image_edit_plus_pre_process_func(
|
||||
width = width // multiple_of * multiple_of
|
||||
|
||||
# Store calculated dimensions in request
|
||||
req.calculated_height = calculated_height
|
||||
req.calculated_width = calculated_width
|
||||
req.height = height
|
||||
req.width = width
|
||||
prompt["additional_information"]["calculated_height"] = calculated_height
|
||||
prompt["additional_information"]["calculated_width"] = calculated_width
|
||||
request.sampling_params.height = height
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Preprocess images into condition_images (for prompt encoding) and vae_images (for VAE encoding)
|
||||
condition_images = []
|
||||
@@ -124,12 +134,12 @@ def get_qwen_image_edit_plus_pre_process_func(
|
||||
vae_images.append(image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2))
|
||||
|
||||
# Store preprocessed images in request
|
||||
req.condition_images = condition_images
|
||||
req.vae_images = vae_images
|
||||
req.condition_image_sizes = condition_image_sizes
|
||||
req.vae_image_sizes = vae_image_sizes
|
||||
|
||||
return requests
|
||||
prompt["additional_information"]["condition_images"] = condition_images
|
||||
prompt["additional_information"]["vae_images"] = vae_images
|
||||
prompt["additional_information"]["condition_image_sizes"] = condition_image_sizes
|
||||
prompt["additional_information"]["vae_image_sizes"] = vae_image_sizes
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -281,7 +291,7 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
|
||||
def _get_qwen_prompt_embeds(
|
||||
self,
|
||||
prompt: str | list[str] = None,
|
||||
prompt: str | list[str],
|
||||
image: list[torch.Tensor] | torch.Tensor | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
@@ -655,19 +665,31 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
max_sequence_length: int = 512,
|
||||
) -> DiffusionOutput:
|
||||
"""Forward pass for image editing with support for multiple images."""
|
||||
prompt = req.prompt
|
||||
negative_prompt = req.negative_prompt
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
|
||||
|
||||
# Get preprocessed images from request (pre-processing is done in DiffusionEngine)
|
||||
if hasattr(req, "vae_images") and hasattr(req, "condition_images"):
|
||||
condition_images = req.condition_images
|
||||
vae_images = req.vae_images
|
||||
condition_image_sizes = req.condition_image_sizes
|
||||
vae_image_sizes = req.vae_image_sizes
|
||||
calculated_height = req.calculated_height
|
||||
calculated_width = req.calculated_width
|
||||
height = req.height
|
||||
width = req.width
|
||||
if (
|
||||
not isinstance(first_prompt, str)
|
||||
and "vae_images" in (additional_information := first_prompt.get("additional_information", {}))
|
||||
and "condition_images" in additional_information
|
||||
):
|
||||
condition_images = additional_information.get("condition_images")
|
||||
vae_images = additional_information.get("vae_images")
|
||||
condition_image_sizes = additional_information.get("condition_image_sizes")
|
||||
vae_image_sizes = additional_information.get("vae_image_sizes")
|
||||
calculated_height = additional_information.get("calculated_height")
|
||||
calculated_width = additional_information.get("calculated_width")
|
||||
height = req.sampling_params.height
|
||||
width = req.sampling_params.width
|
||||
else:
|
||||
# fallback to run pre-processing in pipeline (debug only)
|
||||
if image is None:
|
||||
@@ -701,14 +723,18 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
condition_images.append(self.image_processor.resize(img, condition_height, condition_width))
|
||||
vae_images.append(self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2))
|
||||
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
generator = req.sampling_params.generator or generator
|
||||
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
|
||||
# 1. check inputs
|
||||
# 2. encode prompts
|
||||
|
||||
@@ -7,7 +7,7 @@ import logging
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -38,6 +38,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
@@ -69,17 +70,35 @@ def get_qwen_image_layered_pre_process_func(
|
||||
image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2)
|
||||
|
||||
def pre_process_func(
|
||||
requests: list[OmniDiffusionRequest],
|
||||
request: OmniDiffusionRequest,
|
||||
):
|
||||
"""Pre-process requests for QwenImageLayeredPipeline."""
|
||||
for req in requests:
|
||||
image = req.pil_image
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if raw_image is None or isinstance(raw_image, list):
|
||||
raise ValueError(
|
||||
"""Received no image or a list of image. Only a single image is supported by this model."""
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
|
||||
)
|
||||
|
||||
if isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
|
||||
|
||||
# 1. calculate dimensions
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
assert req.resolution in [640, 1024], f"resolution must be either 640 or 1024, but got {req.resolution}"
|
||||
image_size = image.size
|
||||
assert request.sampling_params.resolution in [640, 1024], (
|
||||
f"resolution must be either 640 or 1024, but got {request.sampling_params.resolution}"
|
||||
)
|
||||
calculated_width, calculated_height = calculate_dimensions(
|
||||
req.resolution * req.resolution, image_size[0] / image_size[1]
|
||||
request.sampling_params.resolution * request.sampling_params.resolution, image_size[0] / image_size[1]
|
||||
)
|
||||
height = calculated_height
|
||||
width = calculated_width
|
||||
@@ -89,10 +108,10 @@ def get_qwen_image_layered_pre_process_func(
|
||||
height = height // multiple_of * multiple_of
|
||||
|
||||
# Store calculated dimensions in request
|
||||
req.calculated_height = calculated_height
|
||||
req.calculated_width = calculated_width
|
||||
req.height = height
|
||||
req.width = width
|
||||
prompt["additional_information"]["calculated_height"] = calculated_height
|
||||
prompt["additional_information"]["calculated_width"] = calculated_width
|
||||
request.sampling_params.height = height
|
||||
request.sampling_params.width = width
|
||||
|
||||
# 2. Preprocess image
|
||||
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels):
|
||||
@@ -103,10 +122,10 @@ def get_qwen_image_layered_pre_process_func(
|
||||
# image = image.to(dtype=self.text_encoder.dtype) # do it later
|
||||
|
||||
# Store preprocessed image and prompt image in request
|
||||
req.preprocessed_image = image
|
||||
req.prompt_image = prompt_image
|
||||
|
||||
return requests
|
||||
prompt["additional_information"]["preprocessed_image"] = image
|
||||
prompt["additional_information"]["prompt_image"] = prompt_image
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -689,7 +708,7 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
req: OmniDiffusionRequest,
|
||||
image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
prompt: str | list[str] | None = None,
|
||||
negative_prompt: str | list[str] = None,
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
true_cfg_scale: float = 4.0,
|
||||
layers: int | None = 4,
|
||||
num_inference_steps: int = 50,
|
||||
@@ -713,29 +732,48 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
|
||||
# 1. Get preprocessed image from request (pre-processing is done in DiffusionEngine)
|
||||
# Override parameters from request if provided
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
layers = req.layers if req.layers is not None else layers
|
||||
resolution = req.resolution if req.resolution is not None else resolution
|
||||
cfg_normalize = req.cfg_normalize if req.cfg_normalize is not None else cfg_normalize
|
||||
use_en_prompt = req.use_en_prompt if req.use_en_prompt is not None else use_en_prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
if len(req.prompts) > 1:
|
||||
logger.warning(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Taking only the first image for now.""",
|
||||
)
|
||||
first_prompt = req.prompts[0]
|
||||
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
|
||||
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
|
||||
|
||||
if hasattr(req, "preprocessed_image"):
|
||||
prompt_image = req.prompt_image
|
||||
image = req.preprocessed_image
|
||||
layers = req.sampling_params.layers if req.sampling_params.layers is not None else layers
|
||||
resolution = req.sampling_params.resolution if req.sampling_params.resolution is not None else resolution
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
cfg_normalize = (
|
||||
req.sampling_params.cfg_normalize if req.sampling_params.cfg_normalize is not None else cfg_normalize
|
||||
)
|
||||
use_en_prompt = (
|
||||
req.sampling_params.use_en_prompt if req.sampling_params.use_en_prompt is not None else use_en_prompt
|
||||
)
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
generator = req.sampling_params.generator or generator
|
||||
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
|
||||
if not isinstance(first_prompt, str) and "preprocessed_image" in (
|
||||
additional_information := first_prompt.get("additional_information", {})
|
||||
):
|
||||
prompt_image = additional_information.get("prompt_image")
|
||||
image = additional_information.get("preprocessed_image")
|
||||
image = image.to(dtype=self.text_encoder.dtype) # Now we get the type
|
||||
calculated_height = req.calculated_height
|
||||
calculated_width = req.calculated_width
|
||||
height = req.height
|
||||
width = req.width
|
||||
calculated_height = additional_information.get("calculated_height")
|
||||
calculated_width = additional_information.get("calculated_width")
|
||||
height = req.sampling_params.height
|
||||
width = req.sampling_params.width
|
||||
else:
|
||||
# fallback to run pre-processing in pipeline (debug only)
|
||||
image_size = image[0].size if isinstance(image, list) else image.size
|
||||
|
||||
@@ -562,19 +562,24 @@ class StableDiffusion3Pipeline(
|
||||
negative_pooled_prompt_embeds: torch.Tensor | None = None,
|
||||
max_sequence_length: int = 256,
|
||||
) -> DiffusionOutput:
|
||||
# # TODO: only support single prompt now
|
||||
# if req.prompt is not None:
|
||||
# prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
height = req.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.width or self.default_sample_size * self.vae_scale_factor
|
||||
sigmas = req.sigmas or sigmas
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
generator = req.generator or generator
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
negative_prompt = [
|
||||
"" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts
|
||||
] or negative_prompt
|
||||
|
||||
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
|
||||
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
generator = req.sampling_params.generator or generator
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
# 1. check inputs
|
||||
# 2. encode prompts
|
||||
# 3. prepare latents and timesteps
|
||||
@@ -595,7 +600,7 @@ class StableDiffusion3Pipeline(
|
||||
max_sequence_length=max_sequence_length,
|
||||
)
|
||||
|
||||
self._guidance_scale = req.guidance_scale
|
||||
self._guidance_scale = req.sampling_params.guidance_scale
|
||||
self._current_timestep = None
|
||||
self._interrupt = False
|
||||
|
||||
|
||||
@@ -332,7 +332,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
|
||||
sample_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
generator: torch.Generator | None,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Prepare initial latent noise."""
|
||||
@@ -357,7 +357,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
|
||||
num_inference_steps: int = 100,
|
||||
guidance_scale: float = 7.0,
|
||||
num_waveforms_per_prompt: int = 1,
|
||||
generator: torch.Generator | None = None,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
latents: torch.Tensor | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
@@ -385,20 +385,26 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
|
||||
DiffusionOutput containing generated audio
|
||||
"""
|
||||
# Extract from request
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
num_inference_steps = req.num_inference_steps or num_inference_steps
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
|
||||
negative_prompt = None
|
||||
elif req.prompts:
|
||||
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
|
||||
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(req.seed)
|
||||
generator = req.sampling_params.generator
|
||||
if generator is None and req.sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed)
|
||||
|
||||
# Get audio duration from request extra params or defaults
|
||||
audio_start_in_s = req.extra.get("audio_start_in_s", audio_start_in_s)
|
||||
audio_end_in_s = req.extra.get("audio_end_in_s", audio_end_in_s)
|
||||
audio_start_in_s = req.sampling_params.extra_args.get("audio_start_in_s", audio_start_in_s)
|
||||
audio_end_in_s = req.sampling_params.extra_args.get("audio_end_in_s", audio_end_in_s)
|
||||
|
||||
# Calculate audio length
|
||||
downsample_ratio = self.vae.hop_length
|
||||
|
||||
@@ -4,8 +4,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
@@ -21,6 +23,9 @@ from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineL
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def retrieve_latents(
|
||||
@@ -127,39 +132,54 @@ def get_wan22_pre_process_func(
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
if raw_image is None:
|
||||
continue
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 720P
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
if not isinstance(raw_image, (str, PIL.Image.Image)):
|
||||
raise TypeError(
|
||||
f"""Unsupported image format {raw_image.__class__}.""",
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
|
||||
)
|
||||
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if request.sampling_params.height is None or request.sampling_params.width is None:
|
||||
# Default max area for 720P
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
# Resize image to target dimensions
|
||||
image = image.resize(
|
||||
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
|
||||
PIL.Image.Resampling.LANCZOS,
|
||||
)
|
||||
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
|
||||
|
||||
return requests
|
||||
# Preprocess for VAE
|
||||
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
|
||||
image, height=request.sampling_params.height, width=request.sampling_params.width
|
||||
)
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -289,20 +309,27 @@ class Wan22Pipeline(nn.Module):
|
||||
guidance_scale: float | tuple[float, float] = 4.0,
|
||||
frame_num: int = 81,
|
||||
output_type: str | None = "np",
|
||||
generator: torch.Generator | None = None,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
attention_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> DiffusionOutput:
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
# Get parameters from request or arguments
|
||||
if len(req.prompts) > 1:
|
||||
raise ValueError(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Please pass in a single prompt object or string, or a single-item list.""",
|
||||
)
|
||||
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
|
||||
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
|
||||
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
|
||||
|
||||
height = req.height or height
|
||||
width = req.width or width
|
||||
num_frames = req.num_frames if req.num_frames else frame_num
|
||||
height = req.sampling_params.height or height
|
||||
width = req.sampling_params.width or width
|
||||
num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num
|
||||
|
||||
# Ensure dimensions are compatible with VAE and patch size
|
||||
# For expand_timesteps mode, we need latent dims to be even (divisible by patch_size)
|
||||
@@ -310,16 +337,16 @@ class Wan22Pipeline(nn.Module):
|
||||
mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V
|
||||
height = (height // mod_value) * mod_value
|
||||
width = (width // mod_value) * mod_value
|
||||
num_steps = req.num_inference_steps or num_inference_steps
|
||||
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
|
||||
# Respect per-request guidance_scale when explicitly provided.
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
|
||||
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
|
||||
guidance_high = (
|
||||
req.guidance_scale_2
|
||||
if req.guidance_scale_2 is not None
|
||||
req.sampling_params.guidance_scale_2
|
||||
if req.sampling_params.guidance_scale_2 is not None
|
||||
else (
|
||||
guidance_scale[1]
|
||||
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
|
||||
@@ -351,9 +378,9 @@ class Wan22Pipeline(nn.Module):
|
||||
|
||||
# Seed / generator
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.seed)
|
||||
generator = req.sampling_params.generator
|
||||
if generator is None and req.sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
|
||||
|
||||
# Encode prompts
|
||||
if prompt_embeds is None:
|
||||
@@ -361,8 +388,8 @@ class Wan22Pipeline(nn.Module):
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
|
||||
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.max_sequence_length or 512,
|
||||
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.sampling_params.max_sequence_length or 512,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -384,7 +411,22 @@ class Wan22Pipeline(nn.Module):
|
||||
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
|
||||
|
||||
# Handle I2V mode when expand_timesteps=True and image is provided
|
||||
image = req.pil_image
|
||||
multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(raw_image, list):
|
||||
if len(raw_image) > 1:
|
||||
logger.warning(
|
||||
"""Received a list of image. Only a single image is supported by this model."""
|
||||
"""Taking only the first image for now."""
|
||||
)
|
||||
raw_image = raw_image[0]
|
||||
if raw_image is None:
|
||||
image = None
|
||||
elif isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
|
||||
|
||||
latent_condition = None
|
||||
first_frame_mask = None
|
||||
|
||||
@@ -415,7 +457,7 @@ class Wan22Pipeline(nn.Module):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
latents=req.sampling_params.latents,
|
||||
)
|
||||
|
||||
# Encode image condition
|
||||
@@ -457,7 +499,7 @@ class Wan22Pipeline(nn.Module):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
latents=req.sampling_params.latents,
|
||||
)
|
||||
|
||||
if attention_kwargs is None:
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -26,6 +28,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
retrieve_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_model_index(model: str, local_files_only: bool) -> dict:
|
||||
@@ -77,39 +82,56 @@ def get_wan22_i2v_pre_process_func(
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
if raw_image is None:
|
||||
raise ValueError(
|
||||
"""No image is provided. This model requires an image to run.""",
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
|
||||
)
|
||||
if not isinstance(raw_image, (str, PIL.Image.Image)):
|
||||
raise TypeError(
|
||||
f"""Unsupported image format {raw_image.__class__}.""",
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
|
||||
)
|
||||
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 480P
|
||||
max_area = 480 * 832
|
||||
aspect_ratio = image.height / image.width
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if request.sampling_params.height is None or request.sampling_params.width is None:
|
||||
# Default max area for 480P
|
||||
max_area = 480 * 832
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
# Resize image to target dimensions
|
||||
image = image.resize(
|
||||
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
|
||||
PIL.Image.Resampling.LANCZOS,
|
||||
)
|
||||
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
|
||||
return requests
|
||||
# Preprocess for VAE
|
||||
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
|
||||
image, height=request.sampling_params.height, width=request.sampling_params.width
|
||||
)
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -266,7 +288,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
guidance_scale: float | tuple[float, float] = 5.0,
|
||||
frame_num: int = 81,
|
||||
output_type: str | None = "np",
|
||||
generator: torch.Generator | None = None,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
image_embeds: torch.Tensor | None = None,
|
||||
@@ -275,31 +297,51 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
**kwargs,
|
||||
) -> DiffusionOutput:
|
||||
# Get parameters from request or arguments
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
if len(req.prompts) > 1:
|
||||
raise ValueError(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Please pass in a single prompt object or string, or a single-item list.""",
|
||||
)
|
||||
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
|
||||
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
|
||||
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 I2V generation.")
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
|
||||
|
||||
# Get image from request
|
||||
if image is None:
|
||||
image = req.pil_image
|
||||
if image is None:
|
||||
raise ValueError("Image is required for I2V generation.")
|
||||
multi_modal_data = (
|
||||
req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
|
||||
)
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if raw_image is None:
|
||||
raise ValueError("Image is required for I2V generation.")
|
||||
if isinstance(raw_image, list):
|
||||
if len(raw_image) > 1:
|
||||
logger.warning(
|
||||
"""Received a list of image. Only a single image is supported by this model."""
|
||||
"""Taking only the first image for now."""
|
||||
)
|
||||
raw_image = raw_image[0]
|
||||
if isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
|
||||
|
||||
height = req.height or height
|
||||
width = req.width or width
|
||||
num_frames = req.num_frames if req.num_frames else frame_num
|
||||
num_steps = req.num_inference_steps or num_inference_steps
|
||||
height = req.sampling_params.height or height
|
||||
width = req.sampling_params.width or width
|
||||
num_frames = req.sampling_params.num_frames or frame_num
|
||||
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
|
||||
# Respect per-request guidance_scale when explicitly provided.
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
|
||||
# Handle guidance scales
|
||||
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
|
||||
guidance_high = (
|
||||
req.guidance_scale_2
|
||||
if req.guidance_scale_2 is not None
|
||||
req.sampling_params.guidance_scale_2
|
||||
if req.sampling_params.guidance_scale_2 is not None
|
||||
else (
|
||||
guidance_scale[1]
|
||||
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
|
||||
@@ -333,9 +375,9 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
|
||||
# Generator setup
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.seed)
|
||||
generator = req.sampling_params.generator
|
||||
if generator is None and req.sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
|
||||
|
||||
# Encode prompts
|
||||
if prompt_embeds is None:
|
||||
@@ -343,8 +385,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
|
||||
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.max_sequence_length or 512,
|
||||
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.sampling_params.max_sequence_length or 512,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -410,7 +452,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
latents=req.sampling_params.latents,
|
||||
last_image=last_image_tensor,
|
||||
)
|
||||
|
||||
|
||||
@@ -16,8 +16,10 @@ The key difference from the MoE-based I2V pipeline is:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -39,6 +41,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
retrieve_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_wan22_ti2v_post_process_func(
|
||||
@@ -67,39 +72,56 @@ def get_wan22_ti2v_pre_process_func(
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt)
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
if raw_image is None:
|
||||
raise ValueError(
|
||||
"""No image is provided. This model requires an image to run.""",
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
|
||||
)
|
||||
if not isinstance(raw_image, (str, PIL.Image.Image)):
|
||||
raise TypeError(
|
||||
f"""Unsupported image format {raw_image.__class__}.""",
|
||||
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
|
||||
)
|
||||
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 720P (TI2V-5B default)
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if request.sampling_params.height is None or request.sampling_params.width is None:
|
||||
# Default max area for 720P (TI2V-5B default)
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
# Resize image to target dimensions
|
||||
image = image.resize(
|
||||
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
|
||||
PIL.Image.Resampling.LANCZOS,
|
||||
)
|
||||
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
|
||||
return requests
|
||||
# Preprocess for VAE
|
||||
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
|
||||
image, height=request.sampling_params.height, width=request.sampling_params.width
|
||||
)
|
||||
request.prompts[i] = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
@@ -205,31 +227,53 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
guidance_scale: float = 5.0,
|
||||
frame_num: int = 81,
|
||||
output_type: str | None = "np",
|
||||
generator: torch.Generator | None = None,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
attention_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> DiffusionOutput:
|
||||
# Get parameters from request or arguments
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
if len(req.prompts) > 1:
|
||||
raise ValueError(
|
||||
"""This model only supports a single prompt, not a batched request.""",
|
||||
"""Please pass in a single prompt object or string, or a single-item list.""",
|
||||
)
|
||||
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
|
||||
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
|
||||
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 TI2V generation.")
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
|
||||
|
||||
# Get image from request (optional for TI2V)
|
||||
if image is None:
|
||||
image = req.pil_image
|
||||
multi_modal_data = (
|
||||
req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
|
||||
)
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
if isinstance(raw_image, list):
|
||||
if len(raw_image) > 1:
|
||||
logger.warning(
|
||||
"""Received a list of image. Only a single image is supported by this model."""
|
||||
"""Taking only the first image for now."""
|
||||
)
|
||||
raw_image = raw_image[0]
|
||||
if raw_image is None:
|
||||
image = None
|
||||
elif isinstance(raw_image, str):
|
||||
image = PIL.Image.open(raw_image)
|
||||
else:
|
||||
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
|
||||
|
||||
# Default dimensions for TI2V-5B (720P)
|
||||
height = req.height or height
|
||||
width = req.width or width
|
||||
num_frames = req.num_frames if req.num_frames else frame_num
|
||||
num_steps = req.num_inference_steps or num_inference_steps
|
||||
height = req.sampling_params.height or height
|
||||
width = req.sampling_params.width or width
|
||||
num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num
|
||||
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
|
||||
# Respect per-request guidance_scale when explicitly provided.
|
||||
if req.guidance_scale_provided:
|
||||
guidance_scale = req.guidance_scale
|
||||
if req.sampling_params.guidance_scale_provided:
|
||||
guidance_scale = req.sampling_params.guidance_scale
|
||||
|
||||
self._guidance_scale = guidance_scale
|
||||
|
||||
@@ -254,9 +298,9 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
|
||||
# Generator setup
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.seed)
|
||||
generator = req.sampling_params.generator
|
||||
if generator is None and req.sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
|
||||
|
||||
# Encode prompts
|
||||
if prompt_embeds is None:
|
||||
@@ -264,8 +308,8 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
do_classifier_free_guidance=guidance_scale > 1.0,
|
||||
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.max_sequence_length or 512,
|
||||
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.sampling_params.max_sequence_length or 512,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -310,7 +354,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
latents=req.sampling_params.latents,
|
||||
)
|
||||
else:
|
||||
# T2V mode: prepare random latents
|
||||
@@ -323,7 +367,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
latents=req.sampling_params.latents,
|
||||
)
|
||||
latent_condition = None
|
||||
first_frame_mask = torch.ones(
|
||||
|
||||
@@ -316,15 +316,15 @@ class ZImagePipeline(nn.Module):
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
prompt: str | list[str] | None = None,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
height: int = 1024,
|
||||
width: int = 1024,
|
||||
num_inference_steps: int = 50,
|
||||
sigmas: list[float] | None = None,
|
||||
guidance_scale: float = 5.0,
|
||||
cfg_normalization: bool = False,
|
||||
cfg_truncation: float = 1.0,
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
num_images_per_prompt: int | None = 1,
|
||||
num_images_per_prompt: int = 1,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
latents: torch.FloatTensor | None = None,
|
||||
prompt_embeds: list[torch.FloatTensor] | None = None,
|
||||
@@ -413,16 +413,28 @@ class ZImagePipeline(nn.Module):
|
||||
`return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
|
||||
generated images.
|
||||
"""
|
||||
prompt = req.prompt
|
||||
negative_prompt = req.negative_prompt
|
||||
height: int = req.height or 1024
|
||||
width: int = req.width or 1024
|
||||
num_inference_steps = req.num_inference_steps or 50
|
||||
generator = req.generator
|
||||
guidance_scale = req.guidance_scale if req.guidance_rescale is not None else guidance_scale
|
||||
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
|
||||
if req_num_outputs and req_num_outputs > 0:
|
||||
num_images_per_prompt = req_num_outputs
|
||||
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
|
||||
# TODO: May be some data formatting operations on the API side. Hack for now.
|
||||
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
|
||||
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
|
||||
negative_prompt = None
|
||||
elif req.prompts:
|
||||
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
|
||||
|
||||
height = req.sampling_params.height or height
|
||||
width = req.sampling_params.width or width
|
||||
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
generator = req.sampling_params.generator
|
||||
sigmas = req.sampling_params.sigmas or sigmas
|
||||
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
|
||||
guidance_scale = (
|
||||
req.sampling_params.guidance_scale if req.sampling_params.guidance_rescale is not None else guidance_scale
|
||||
)
|
||||
num_images_per_prompt = (
|
||||
req.sampling_params.num_outputs_per_prompt
|
||||
if req.sampling_params.num_outputs_per_prompt > 0
|
||||
else num_images_per_prompt
|
||||
)
|
||||
|
||||
vae_scale = self.vae_scale_factor * 2
|
||||
if height % vae_scale != 0:
|
||||
|
||||
Reference in new issue
Block a user