[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-01-28 16:07:42 +08:00
1 parent 9ece725fc5
commit d64bbde561
64 files changed
+2995 -1414

No files matched your search

@@ -272,16 +272,22 @@ class BagelPipeline(nn.Module):
@torch.inference_mode()
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
prompt = req.prompt or ""
if isinstance(prompt, list):
# vllm-omni request supports list; Bagel pipeline currently supports first prompt.
prompt = prompt[0] if prompt else ""
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
# TODO: In online mode, sometimes it receives [{"prompts": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(req.prompts[0], str) else (req.prompts[0].get("prompt") or "")
max_hw = int(self.bagel.max_latent_size * self.bagel.latent_downsample)
if req.height is None and req.width is None:
if req.sampling_params.height is None and req.sampling_params.width is None:
height = width = max_hw
else:
height = int(req.height) if req.height is not None else max_hw
width = int(req.width) if req.width is not None else max_hw
height = int(req.sampling_params.height) if req.sampling_params.height is not None else max_hw
width = int(req.sampling_params.width) if req.sampling_params.width is not None else max_hw
if height > max_hw or width > max_hw:
raise ValueError(
f"Requested resolution {height}x{width} exceeds Bagel checkpoint limit "
@@ -292,7 +298,7 @@ class BagelPipeline(nn.Module):
# Map request params to Bagel gen params (defaults follow Bagel inferencer)
gen_params = BagelGenParams(
num_timesteps=int(req.num_inference_steps or 50),
num_timesteps=int(req.sampling_params.num_inference_steps or 50),
timestep_shift=3.0,
)
@@ -304,7 +310,7 @@ class BagelPipeline(nn.Module):
# Add text prompt (prefill) on gen context.
# [Omni] Check for injected KV Cache from remote transfer
injected_kv = getattr(req, "past_key_values", None)
injected_kv = req.sampling_params.past_key_values
if injected_kv is not None:
logger.info("Using injected KV Cache (direct)")
gen_context["past_key_values"] = injected_kv
@@ -316,9 +322,13 @@ class BagelPipeline(nn.Module):
gen_context["ropes"] = [seq_len]
else:
image_input = getattr(req, "pil_image", None)
image_input = (
None if isinstance(first_prompt, str) else (first_prompt.get("multi_modal_data") or {}).get("image")
)
if image_input and not isinstance(image_input, list):
image_input = [image_input]
if image_input:
image_input = [Image.open(image) if isinstance(image, str) else image for image in image_input]
if image_input:
# If we have an image, we prefill with it
@@ -414,10 +424,10 @@ class BagelPipeline(nn.Module):
gen_context["kv_lens"] = newlens
gen_context["ropes"] = new_rope
if req.seed is not None:
torch.manual_seed(req.seed)
if req.sampling_params.seed is not None:
torch.manual_seed(req.sampling_params.seed)
if self.device.type == "cuda":
torch.cuda.manual_seed(req.seed)
torch.cuda.manual_seed(req.sampling_params.seed)
# Prepare latent query and run flow
generation_input = self.bagel.prepare_vae_latent(
@@ -19,7 +19,7 @@ import json
import math
import os
from collections.abc import Callable, Iterable
from typing import Any
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -742,27 +742,54 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput):
`return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
generated images.
"""
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
prompt = req.prompt if req.prompt is not None else prompt
image = req.pil_image if req.pil_image is not None else image
height = req.height or height
width = req.width or width
num_inference_steps = req.num_inference_steps or num_inference_steps
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
generator = req.generator or generator
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
if (
raw_image := None
if isinstance(first_prompt, str)
else first_prompt.get("multi_modal_data", {}).get("image")
) is None:
pass # use image from param list
elif isinstance(raw_image, list):
image = [PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image, im) for im in raw_image]
else:
image = PIL.Image.open(raw_image) if isinstance(raw_image, str) else cast(PIL.Image.Image, raw_image)
if isinstance(req.prompt_embeds, torch.Tensor):
prompt_embeds = req.prompt_embeds
if isinstance(req.negative_prompt_embeds, torch.Tensor):
negative_prompt_embeds = req.negative_prompt_embeds
height = req.sampling_params.height or height
width = req.sampling_params.width or width
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
guidance_scale = (
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
)
generator = req.sampling_params.generator or generator
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
text_encoder_out_layers = req.sampling_params.extra_args.get("text_encoder_out_layers", text_encoder_out_layers)
if req.max_sequence_length is not None:
max_sequence_length = req.max_sequence_length
if getattr(req, "text_encoder_out_layers", None) is not None:
text_encoder_out_layers = req.text_encoder_out_layers
req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts]
if any(p is not None for p in req_prompt_embeds):
# If at list one prompt is provided as an embedding,
# Then assume that the user wants to provide embeddings for all prompts, and enter this if block
# If the user in fact provides mixed input format, req_prompt_embeds will have some None's
# And `torch.stack` automatically raises an exception for us
prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError
req_negative_prompt_embeds = [
p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts
]
if any(p is not None for p in req_negative_prompt_embeds):
negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError
# 1. Check inputs. Raise error if not correct
self.check_inputs(
@@ -17,6 +17,7 @@ import logging
import os
import re
from collections.abc import Iterable
from typing import cast
import numpy as np
import PIL.Image
@@ -48,6 +49,7 @@ from vllm_omni.diffusion.models.glm_image.glm_image_transformer import (
GlmImageTransformer2DModel,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -77,16 +79,26 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig):
# GLM-Image uses patch_size=2 for transformer
patch_size = 2
def pre_process_func(requests: list[OmniDiffusionRequest]):
def pre_process_func(request: OmniDiffusionRequest):
"""Pre-process condition images for Image Edit mode."""
for req in requests:
images = req.pil_image
if images is None:
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if raw_image is None:
# Text-to-image mode, no preprocessing needed
continue
if not isinstance(images, list):
images = [images]
if not isinstance(raw_image, list):
raw_image = [raw_image]
images = [
PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im)
for im in raw_image
]
preprocessed = []
height, width = None, None
@@ -110,14 +122,19 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig):
height, width = img_h, img_w
# Store in request
req.preprocessed_image = preprocessed
req.prompt_image = images # Keep original PIL images
if req.height is None:
req.height = height
if req.width is None:
req.width = width
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt, additional_information={})
elif "additional_information" not in prompt:
prompt["additional_information"] = {}
prompt["additional_information"]["preprocessed_image"] = processed # type: ignore
prompt["additional_information"]["prompt_image"] = images # type: ignore
request.prompts[i] = prompt
if request.sampling_params.height is None:
request.sampling_params.height = height
if request.sampling_params.width is None:
request.sampling_params.width = width
return requests
return request
return pre_process_func
@@ -821,27 +838,44 @@ class GlmImagePipeline(nn.Module):
Returns:
DiffusionOutput containing generated image
"""
prompt = req.prompt or ""
if isinstance(prompt, list):
prompt = prompt[0] if prompt else ""
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
# Get pre-computed prompt embeddings if provided
prompt_embeds = req.prompt_embeds if isinstance(req.prompt_embeds, torch.Tensor) else None
if isinstance(first_prompt, str):
prompt_embeds = None
else:
prompt_embeds = first_prompt.get("prompt_embeds")
if not isinstance(prompt_embeds, torch.Tensor):
prompt_embeds = None
# Get condition images for Image Edit mode
# Use pre-processed images from pre_process_func
preprocessed_images = req.preprocessed_image
condition_images = getattr(req, "prompt_image", None)
img_height = req.height
img_width = req.width
preprocessed_images = (
None
if isinstance(first_prompt, str)
else first_prompt.get("additional_information", {}).get("preprocessed_image")
)
condition_images = (
None
if isinstance(first_prompt, str)
else first_prompt.get("additional_information", {}).get("prompt_image")
)
img_height = req.sampling_params.height
img_width = req.sampling_params.width
is_image_edit = preprocessed_images is not None
# Use image dimensions as default if available
height = req.height or img_height or self.default_sample_size * self.vae_scale_factor
width = req.width or img_width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.num_inference_steps or 50
guidance_scale = req.guidance_scale or 1.5
height = req.sampling_params.height or img_height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or img_width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.sampling_params.num_inference_steps or 50
guidance_scale = req.sampling_params.guidance_scale or 1.5
# 0. Validate inputs
self.check_inputs(prompt=prompt, height=height, width=width, prompt_embeds=prompt_embeds)
@@ -851,13 +885,13 @@ class GlmImagePipeline(nn.Module):
# Set seed if provided
generator = None
if req.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(req.seed)
if req.sampling_params.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed)
# 1. Get prior tokens - either from external source (multistage) or generate internally
# Check if prior_token_ids are provided externally (from AR stage in multistage mode)
external_prior_tokens = req.extra.get("prior_token_ids") if req.extra else None
external_prior_image_ids = req.extra.get("prior_token_image_ids") if req.extra else None
external_prior_tokens = req.sampling_params.extra_args.get("prior_token_ids")
external_prior_image_ids = req.sampling_params.extra_args.get("prior_token_image_ids")
if external_prior_tokens is not None:
# Multistage mode: use externally provided prior tokens from vLLM AR stage
@@ -451,7 +451,7 @@ class LongCatImagePipeline(
raise ValueError(
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
)
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
elif prompt is not None and not isinstance(prompt, (str, list)):
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
if negative_prompt is not None and negative_prompt_embeds is not None:
@@ -470,11 +470,11 @@ class LongCatImagePipeline(
num_inference_steps: int = 50,
sigmas: list[float] | None = None,
guidance_scale: float = 4.5,
num_images_per_prompt: int | None = 1,
num_images_per_prompt: int = 1,
generator: torch.Generator | list[torch.Generator] | None = None,
latents: torch.FloatTensor | None = None,
prompt_embeds: torch.FloatTensor | None = None,
negative_prompt_embeds: torch.FloatTensor | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
output_type: str | None = "pil",
return_dict: bool = True,
joint_attention_kwargs: dict[str, Any] | None = None,
@@ -482,20 +482,44 @@ class LongCatImagePipeline(
cfg_renorm_min: float | None = 0.0,
enable_prompt_rewrite: bool | None = True,
) -> DiffusionOutput:
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
negative_prompt = None
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
height = req.height or height or self.default_sample_size * self.vae_scale_factor
width = req.width or width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
guidance_scale = req.guidance_scale if getattr(req, "guidance_scale", None) is not None else guidance_scale
num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt
enable_prompt_rewrite = getattr(req, "enable_prompt_rewrite", None) or enable_prompt_rewrite
enable_cfg_renorm = getattr(req, "enable_cfg_renorm", None) or enable_cfg_renorm
cfg_renorm_min = getattr(req, "cfg_renorm_min", None) or cfg_renorm_min
prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds
negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds
height = req.sampling_params.height or height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
generator = req.sampling_params.generator or generator
guidance_scale = (
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
)
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt is not None
else num_images_per_prompt
)
enable_prompt_rewrite = req.sampling_params.extra_args.get("enable_prompt_rewrite", enable_prompt_rewrite)
enable_cfg_renorm = req.sampling_params.extra_args.get("enable_cfg_renorm", enable_cfg_renorm)
cfg_renorm_min = req.sampling_params.extra_args.get("cfg_renorm_min", cfg_renorm_min)
req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts]
if any(p is not None for p in req_prompt_embeds):
# If at list one prompt is provided as an embedding,
# Then assume that the user wants to provide embeddings for all prompts, and enter this if block
# If the user in fact provides mixed input format, req_prompt_embeds will have some None's
# And `torch.stack` automatically raises an exception for us
prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError
req_negative_prompt_embeds = [
p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts
]
if any(p is not None for p in req_negative_prompt_embeds):
negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError
self.check_inputs(
prompt,
@@ -6,7 +6,7 @@ import math
import os
import re
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import (
)
from vllm_omni.diffusion.models.longcat_image.pipeline_longcat_image import calculate_shift
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -59,22 +60,38 @@ def get_longcat_image_edit_pre_process_func(
latent_channels = vae_config.get("latent_channels", 16)
def pre_process_func(
requests: list[OmniDiffusionRequest],
request: OmniDiffusionRequest,
):
"""Pre-process requests for QwenImageEditPipeline."""
for req in requests:
image = req.pil_image
"""Pre-process requests for LongCatImageEditPipeline."""
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
image_size = image[0].size if isinstance(image, list) else image.size
if raw_image is None or isinstance(raw_image, list):
raise ValueError(
"""Received no image or a list of image. Only a single image is supported by this model."""
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
)
if isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
image_size = image.size
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1])
height = req.height or calculated_height
width = req.width or calculated_width
height = request.sampling_params.height or calculated_height
width = request.sampling_params.width or calculated_width
# Store calculated dimensions in request
req.calculated_height = calculated_height
req.calculated_width = calculated_width
req.height = height
req.width = width
prompt["additional_information"]["calculated_height"] = calculated_height
prompt["additional_information"]["calculated_width"] = calculated_width
request.sampling_params.height = height
request.sampling_params.width = width
# Preprocess image
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels):
@@ -83,9 +100,10 @@ def get_longcat_image_edit_pre_process_func(
image = image_processor.preprocess(image, calculated_height, calculated_width)
# Store preprocessed image and prompt image in request
req.preprocessed_image = image
req.prompt_image = prompt_image
return requests
prompt["additional_information"]["preprocessed_image"] = image
prompt["additional_information"]["prompt_image"] = prompt_image
request.prompts[i] = prompt
return request
return pre_process_func
@@ -500,42 +518,59 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput):
self,
req: OmniDiffusionRequest,
image: PIL.Image.Image | torch.Tensor | None = None,
prompt: str | list[str] = None,
negative_prompt: str | list[str] = None,
prompt: str | list[str] | None = None,
negative_prompt: str | list[str] | None = None,
num_inference_steps: int = 50,
sigmas: list[float] | None = None,
guidance_scale: float = 3.5,
num_images_per_prompt: int | None = 1,
generator: torch.Generator | list[torch.Generator] | None = None,
latents: torch.FloatTensor | None = None,
prompt_embeds: torch.FloatTensor | None = None,
negative_prompt_embeds: torch.FloatTensor | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
output_type: str | None = "pil",
return_dict: bool = True,
joint_attention_kwargs: dict[str, Any] | None = None,
):
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
negative_prompt = "" if negative_prompt is None else negative_prompt
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
num_inference_steps = req.num_inference_steps or num_inference_steps
num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt
generator = req.generator or generator
prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds
negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds
height = req.height or self.default_sample_size * self.vae_scale_factor
width = req.width or self.default_sample_size * self.vae_scale_factor
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("prompt_embeds")
negative_prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt_embeds") # type: ignore # Why it is list[torch.Tensor] in OmniTokenInputs or OmniEmbedsPrompt? Doesn't make sense
sigmas = req.sampling_params.sigmas or sigmas
guidance_scale = (
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
)
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt is not None
else num_images_per_prompt
)
generator = req.sampling_params.generator or generator
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
if prompt is not None:
batch_size = 1 if isinstance(prompt, str) else len(prompt)
else:
batch_size = prompt_embeds.shape[0]
if hasattr(req, "preprocessed_image"):
prompt_image = req.prompt_image
image = req.preprocessed_image
calculated_height = req.calculated_height if hasattr(req, "calculated_height") else height
calculated_width = req.calculated_width if hasattr(req, "calculated_width") else width
if not isinstance(first_prompt, str) and "preprocessed_image" in (
additional_information := first_prompt.get("additional_information", {})
):
prompt_image = additional_information.get("prompt_image")
image = additional_information.get("preprocessed_image")
calculated_height = additional_information.get("calculated_height", height)
calculated_width = additional_information.get("calculated_width", width)
else:
image_size = image[0].size if isinstance(image, list) else image.size
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1])
@@ -519,8 +519,8 @@ class OvisImagePipeline(
def forward(
self,
req: OmniDiffusionRequest,
prompt: str | list[str] = None,
negative_prompt: str | list[str] = None,
prompt: str | list[str] | None = None,
negative_prompt: str | list[str] | None = None,
guidance_scale: float = 5.0,
height: int | None = None,
width: int | None = None,
@@ -606,16 +606,27 @@ class OvisImagePipeline(
[`~pipelines.ovis_image.OvisImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When
returning a tuple, the first element is a list with the generated images.
"""
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
height = req.height or self.default_sample_size * self.vae_scale_factor
width = req.width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.num_inference_steps or num_inference_steps
guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale
generator = req.generator or generator
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
negative_prompt = None
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
guidance_scale = (
req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale
)
generator = req.sampling_params.generator or generator
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
# Steps:
# 1. Check Inputs
@@ -662,21 +662,28 @@ class QwenImagePipeline(
callback_on_step_end_tensor_inputs: list[str] = ["latents"],
max_sequence_length: int = 512,
) -> DiffusionOutput:
# # TODO: only support single prompt now
# if req.prompt is not None:
# prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt
prompt = req.prompt
negative_prompt = req.negative_prompt
height = req.height or self.default_sample_size * self.vae_scale_factor
width = req.width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
negative_prompt = None
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
generator = req.sampling_params.generator or generator
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
# 1. check inputs
# 2. encode prompts
# 3. prepare latents and timesteps
@@ -7,7 +7,7 @@ import logging
import math
import os
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -39,6 +39,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -64,16 +65,33 @@ def get_qwen_image_edit_pre_process_func(
latent_channels = vae_config.get("z_dim", 16)
def pre_process_func(
requests: list[OmniDiffusionRequest],
request: OmniDiffusionRequest,
):
"""Pre-process requests for QwenImageEditPipeline."""
for req in requests:
image = req.pil_image
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
image_size = image[0].size if isinstance(image, list) else image.size
# Only handles single image
if raw_image is None or isinstance(raw_image, list):
raise ValueError(
"""Received no image or a list of image. Only a single image is supported by this model."""
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
)
if isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
image_size = image.size
calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1])
height = req.height or calculated_height
width = req.width or calculated_width
height = request.sampling_params.height or calculated_height
width = request.sampling_params.width or calculated_width
# Ensure dimensions are multiples of vae_scale_factor * 2
multiple_of = vae_scale_factor * 2
@@ -81,10 +99,10 @@ def get_qwen_image_edit_pre_process_func(
width = width // multiple_of * multiple_of
# Store calculated dimensions in request
req.calculated_height = calculated_height
req.calculated_width = calculated_width
req.height = height
req.width = width
prompt["additional_information"]["calculated_height"] = calculated_height
prompt["additional_information"]["calculated_width"] = calculated_width
request.sampling_params.height = height
request.sampling_params.width = width
# Preprocess image
if image is not None and not (
@@ -96,10 +114,10 @@ def get_qwen_image_edit_pre_process_func(
image = image.unsqueeze(2)
# Store preprocessed image and prompt image in request
req.preprocessed_image = image
req.prompt_image = prompt_image
return requests
prompt["additional_information"]["preprocessed_image"] = image
prompt["additional_information"]["prompt_image"] = prompt_image
request.prompts[i] = prompt
return request
return pre_process_func
@@ -717,17 +735,27 @@ class QwenImageEditPipeline(
max_sequence_length: int = 512,
) -> DiffusionOutput:
"""Forward pass for image editing."""
prompt = req.prompt
negative_prompt = req.negative_prompt
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
# Get preprocessed image from request (pre-processing is done in DiffusionEngine)
if hasattr(req, "preprocessed_image"):
prompt_image = req.prompt_image
image = req.preprocessed_image
calculated_height = req.calculated_height
calculated_width = req.calculated_width
height = req.height
width = req.width
if not isinstance(first_prompt, str) and "preprocessed_image" in (
additional_information := first_prompt.get("additional_information", {})
):
prompt_image = additional_information.get("prompt_image")
image = additional_information.get("preprocessed_image")
calculated_height = additional_information.get("calculated_height")
calculated_width = additional_information.get("calculated_width")
height = req.sampling_params.height
width = req.sampling_params.width
else:
# fallback to run pre-processing in pipeline (debug only)
image_size = image[0].size if isinstance(image, list) else image.size
@@ -745,14 +773,18 @@ class QwenImageEditPipeline(
image = self.image_processor.preprocess(image, calculated_height, calculated_width)
image = image.unsqueeze(2)
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
generator = req.sampling_params.generator or generator
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
# 1. check inputs
# 2. encode prompts
@@ -5,7 +5,7 @@ import json
import logging
import os
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -42,6 +42,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -70,24 +71,33 @@ def get_qwen_image_edit_plus_pre_process_func(
latent_channels = vae_config.get("z_dim", 16)
def pre_process_func(
requests: list[OmniDiffusionRequest],
request: OmniDiffusionRequest,
):
"""Pre-process requests for QwenImageEditPlusPipeline."""
for req in requests:
image = req.pil_image
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
# Handle single image or list of images
if image is None:
if raw_image is None:
continue
if not isinstance(image, list):
image = [image]
if not isinstance(raw_image, list):
raw_image = [raw_image]
image = [
PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im)
for im in raw_image
]
# Calculate dimensions based on first image
image_size = image[0].size
calculated_width, calculated_height = calculate_dimensions(VAE_IMAGE_SIZE, image_size[0] / image_size[1])
height = req.height or calculated_height
width = req.width or calculated_width
height = request.sampling_params.height or calculated_height
width = request.sampling_params.width or calculated_width
# Ensure dimensions are multiples of vae_scale_factor * 2
multiple_of = vae_scale_factor * 2
@@ -95,10 +105,10 @@ def get_qwen_image_edit_plus_pre_process_func(
width = width // multiple_of * multiple_of
# Store calculated dimensions in request
req.calculated_height = calculated_height
req.calculated_width = calculated_width
req.height = height
req.width = width
prompt["additional_information"]["calculated_height"] = calculated_height
prompt["additional_information"]["calculated_width"] = calculated_width
request.sampling_params.height = height
request.sampling_params.width = width
# Preprocess images into condition_images (for prompt encoding) and vae_images (for VAE encoding)
condition_images = []
@@ -124,12 +134,12 @@ def get_qwen_image_edit_plus_pre_process_func(
vae_images.append(image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2))
# Store preprocessed images in request
req.condition_images = condition_images
req.vae_images = vae_images
req.condition_image_sizes = condition_image_sizes
req.vae_image_sizes = vae_image_sizes
return requests
prompt["additional_information"]["condition_images"] = condition_images
prompt["additional_information"]["vae_images"] = vae_images
prompt["additional_information"]["condition_image_sizes"] = condition_image_sizes
prompt["additional_information"]["vae_image_sizes"] = vae_image_sizes
request.prompts[i] = prompt
return request
return pre_process_func
@@ -281,7 +291,7 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
def _get_qwen_prompt_embeds(
self,
prompt: str | list[str] = None,
prompt: str | list[str],
image: list[torch.Tensor] | torch.Tensor | None = None,
dtype: torch.dtype | None = None,
):
@@ -655,19 +665,31 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
max_sequence_length: int = 512,
) -> DiffusionOutput:
"""Forward pass for image editing with support for multiple images."""
prompt = req.prompt
negative_prompt = req.negative_prompt
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
# Get preprocessed images from request (pre-processing is done in DiffusionEngine)
if hasattr(req, "vae_images") and hasattr(req, "condition_images"):
condition_images = req.condition_images
vae_images = req.vae_images
condition_image_sizes = req.condition_image_sizes
vae_image_sizes = req.vae_image_sizes
calculated_height = req.calculated_height
calculated_width = req.calculated_width
height = req.height
width = req.width
if (
not isinstance(first_prompt, str)
and "vae_images" in (additional_information := first_prompt.get("additional_information", {}))
and "condition_images" in additional_information
):
condition_images = additional_information.get("condition_images")
vae_images = additional_information.get("vae_images")
condition_image_sizes = additional_information.get("condition_image_sizes")
vae_image_sizes = additional_information.get("vae_image_sizes")
calculated_height = additional_information.get("calculated_height")
calculated_width = additional_information.get("calculated_width")
height = req.sampling_params.height
width = req.sampling_params.width
else:
# fallback to run pre-processing in pipeline (debug only)
if image is None:
@@ -701,14 +723,18 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
condition_images.append(self.image_processor.resize(img, condition_height, condition_width))
vae_images.append(self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2))
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
generator = req.sampling_params.generator or generator
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
# 1. check inputs
# 2. encode prompts
@@ -7,7 +7,7 @@ import logging
import math
import os
from collections.abc import Iterable
from typing import Any
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -38,6 +38,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -69,17 +70,35 @@ def get_qwen_image_layered_pre_process_func(
image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2)
def pre_process_func(
requests: list[OmniDiffusionRequest],
request: OmniDiffusionRequest,
):
"""Pre-process requests for QwenImageLayeredPipeline."""
for req in requests:
image = req.pil_image
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if raw_image is None or isinstance(raw_image, list):
raise ValueError(
"""Received no image or a list of image. Only a single image is supported by this model."""
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`"""
)
if isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image)
# 1. calculate dimensions
image_size = image[0].size if isinstance(image, list) else image.size
assert req.resolution in [640, 1024], f"resolution must be either 640 or 1024, but got {req.resolution}"
image_size = image.size
assert request.sampling_params.resolution in [640, 1024], (
f"resolution must be either 640 or 1024, but got {request.sampling_params.resolution}"
)
calculated_width, calculated_height = calculate_dimensions(
req.resolution * req.resolution, image_size[0] / image_size[1]
request.sampling_params.resolution * request.sampling_params.resolution, image_size[0] / image_size[1]
)
height = calculated_height
width = calculated_width
@@ -89,10 +108,10 @@ def get_qwen_image_layered_pre_process_func(
height = height // multiple_of * multiple_of
# Store calculated dimensions in request
req.calculated_height = calculated_height
req.calculated_width = calculated_width
req.height = height
req.width = width
prompt["additional_information"]["calculated_height"] = calculated_height
prompt["additional_information"]["calculated_width"] = calculated_width
request.sampling_params.height = height
request.sampling_params.width = width
# 2. Preprocess image
if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels):
@@ -103,10 +122,10 @@ def get_qwen_image_layered_pre_process_func(
# image = image.to(dtype=self.text_encoder.dtype) # do it later
# Store preprocessed image and prompt image in request
req.preprocessed_image = image
req.prompt_image = prompt_image
return requests
prompt["additional_information"]["preprocessed_image"] = image
prompt["additional_information"]["prompt_image"] = prompt_image
request.prompts[i] = prompt
return request
return pre_process_func
@@ -689,7 +708,7 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
req: OmniDiffusionRequest,
image: PIL.Image.Image | torch.Tensor | None = None,
prompt: str | list[str] | None = None,
negative_prompt: str | list[str] = None,
negative_prompt: str | list[str] | None = None,
true_cfg_scale: float = 4.0,
layers: int | None = 4,
num_inference_steps: int = 50,
@@ -713,29 +732,48 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
# 1. Get preprocessed image from request (pre-processing is done in DiffusionEngine)
# Override parameters from request if provided
prompt = req.prompt if req.prompt is not None else prompt
layers = req.layers if req.layers is not None else layers
resolution = req.resolution if req.resolution is not None else resolution
cfg_normalize = req.cfg_normalize if req.cfg_normalize is not None else cfg_normalize
use_en_prompt = req.use_en_prompt if req.use_en_prompt is not None else use_en_prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
true_cfg_scale = req.true_cfg_scale or true_cfg_scale
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
if len(req.prompts) > 1:
logger.warning(
"""This model only supports a single prompt, not a batched request.""",
"""Taking only the first image for now.""",
)
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt")
if hasattr(req, "preprocessed_image"):
prompt_image = req.prompt_image
image = req.preprocessed_image
layers = req.sampling_params.layers if req.sampling_params.layers is not None else layers
resolution = req.sampling_params.resolution if req.sampling_params.resolution is not None else resolution
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
cfg_normalize = (
req.sampling_params.cfg_normalize if req.sampling_params.cfg_normalize is not None else cfg_normalize
)
use_en_prompt = (
req.sampling_params.use_en_prompt if req.sampling_params.use_en_prompt is not None else use_en_prompt
)
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
sigmas = req.sampling_params.sigmas or sigmas
generator = req.sampling_params.generator or generator
true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
if not isinstance(first_prompt, str) and "preprocessed_image" in (
additional_information := first_prompt.get("additional_information", {})
):
prompt_image = additional_information.get("prompt_image")
image = additional_information.get("preprocessed_image")
image = image.to(dtype=self.text_encoder.dtype) # Now we get the type
calculated_height = req.calculated_height
calculated_width = req.calculated_width
height = req.height
width = req.width
calculated_height = additional_information.get("calculated_height")
calculated_width = additional_information.get("calculated_width")
height = req.sampling_params.height
width = req.sampling_params.width
else:
# fallback to run pre-processing in pipeline (debug only)
image_size = image[0].size if isinstance(image, list) else image.size
+19 -14
View File
@@ -562,19 +562,24 @@ class StableDiffusion3Pipeline(
negative_pooled_prompt_embeds: torch.Tensor | None = None,
max_sequence_length: int = 256,
) -> DiffusionOutput:
# # TODO: only support single prompt now
# if req.prompt is not None:
# prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
height = req.height or self.default_sample_size * self.vae_scale_factor
width = req.width or self.default_sample_size * self.vae_scale_factor
sigmas = req.sigmas or sigmas
num_inference_steps = req.num_inference_steps or num_inference_steps
generator = req.generator or generator
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
negative_prompt = [
"" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts
] or negative_prompt
height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor
width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor
sigmas = req.sampling_params.sigmas or sigmas
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
generator = req.sampling_params.generator or generator
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
# 1. check inputs
# 2. encode prompts
# 3. prepare latents and timesteps
@@ -595,7 +600,7 @@ class StableDiffusion3Pipeline(
max_sequence_length=max_sequence_length,
)
self._guidance_scale = req.guidance_scale
self._guidance_scale = req.sampling_params.guidance_scale
self._current_timestep = None
self._interrupt = False
@@ -332,7 +332,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
sample_size: int,
dtype: torch.dtype,
device: torch.device,
generator: torch.Generator | None,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None = None,
) -> torch.Tensor:
"""Prepare initial latent noise."""
@@ -357,7 +357,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
num_inference_steps: int = 100,
guidance_scale: float = 7.0,
num_waveforms_per_prompt: int = 1,
generator: torch.Generator | None = None,
generator: torch.Generator | list[torch.Generator] | None = None,
latents: torch.Tensor | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
@@ -385,20 +385,26 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
DiffusionOutput containing generated audio
"""
# Extract from request
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
num_inference_steps = req.num_inference_steps or num_inference_steps
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
negative_prompt = None
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(req.seed)
generator = req.sampling_params.generator
if generator is None and req.sampling_params.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed)
# Get audio duration from request extra params or defaults
audio_start_in_s = req.extra.get("audio_start_in_s", audio_start_in_s)
audio_end_in_s = req.extra.get("audio_end_in_s", audio_end_in_s)
audio_start_in_s = req.sampling_params.extra_args.get("audio_start_in_s", audio_start_in_s)
audio_end_in_s = req.sampling_params.extra_args.get("audio_end_in_s", audio_end_in_s)
# Calculate audio length
downsample_ratio = self.vae.hop_length
@@ -4,8 +4,10 @@
from __future__ import annotations
import json
import logging
import os
from collections.abc import Iterable
from typing import cast
import PIL.Image
import torch
@@ -21,6 +23,9 @@ from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineL
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
logger = logging.getLogger(__name__)
def retrieve_latents(
@@ -127,39 +132,54 @@ def get_wan22_pre_process_func(
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if req.pil_image is not None:
image = req.pil_image
if raw_image is None:
continue
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 720P
max_area = 720 * 1280
aspect_ratio = image.height / image.width
if not isinstance(raw_image, (str, PIL.Image.Image)):
raise TypeError(
f"""Unsupported image format {raw_image.__class__}.""",
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
)
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
# Calculate dimensions based on aspect ratio if not provided
if request.sampling_params.height is None or request.sampling_params.width is None:
# Default max area for 720P
max_area = 720 * 1280
aspect_ratio = image.height / image.width
if req.height is None:
req.height = height
if req.width is None:
req.width = width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
if request.sampling_params.height is None:
request.sampling_params.height = height
if request.sampling_params.width is None:
request.sampling_params.width = width
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
# Resize image to target dimensions
image = image.resize(
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
PIL.Image.Resampling.LANCZOS,
)
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
return requests
# Preprocess for VAE
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
image, height=request.sampling_params.height, width=request.sampling_params.width
)
request.prompts[i] = prompt
return request
return pre_process_func
@@ -289,20 +309,27 @@ class Wan22Pipeline(nn.Module):
guidance_scale: float | tuple[float, float] = 4.0,
frame_num: int = 81,
output_type: str | None = "np",
generator: torch.Generator | None = None,
generator: torch.Generator | list[torch.Generator] | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
attention_kwargs: dict | None = None,
**kwargs,
) -> DiffusionOutput:
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
# Get parameters from request or arguments
if len(req.prompts) > 1:
raise ValueError(
"""This model only supports a single prompt, not a batched request.""",
"""Please pass in a single prompt object or string, or a single-item list.""",
)
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
if prompt is None and prompt_embeds is None:
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
height = req.height or height
width = req.width or width
num_frames = req.num_frames if req.num_frames else frame_num
height = req.sampling_params.height or height
width = req.sampling_params.width or width
num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num
# Ensure dimensions are compatible with VAE and patch size
# For expand_timesteps mode, we need latent dims to be even (divisible by patch_size)
@@ -310,16 +337,16 @@ class Wan22Pipeline(nn.Module):
mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V
height = (height // mod_value) * mod_value
width = (width // mod_value) * mod_value
num_steps = req.num_inference_steps or num_inference_steps
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
# Respect per-request guidance_scale when explicitly provided.
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
guidance_high = (
req.guidance_scale_2
if req.guidance_scale_2 is not None
req.sampling_params.guidance_scale_2
if req.sampling_params.guidance_scale_2 is not None
else (
guidance_scale[1]
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
@@ -351,9 +378,9 @@ class Wan22Pipeline(nn.Module):
# Seed / generator
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.seed)
generator = req.sampling_params.generator
if generator is None and req.sampling_params.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
# Encode prompts
if prompt_embeds is None:
@@ -361,8 +388,8 @@ class Wan22Pipeline(nn.Module):
prompt=prompt,
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
max_sequence_length=req.max_sequence_length or 512,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -384,7 +411,22 @@ class Wan22Pipeline(nn.Module):
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
# Handle I2V mode when expand_timesteps=True and image is provided
image = req.pil_image
multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(raw_image, list):
if len(raw_image) > 1:
logger.warning(
"""Received a list of image. Only a single image is supported by this model."""
"""Taking only the first image for now."""
)
raw_image = raw_image[0]
if raw_image is None:
image = None
elif isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
latent_condition = None
first_frame_mask = None
@@ -415,7 +457,7 @@ class Wan22Pipeline(nn.Module):
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
latents=req.sampling_params.latents,
)
# Encode image condition
@@ -457,7 +499,7 @@ class Wan22Pipeline(nn.Module):
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
latents=req.sampling_params.latents,
)
if attention_kwargs is None:
@@ -3,8 +3,10 @@
from __future__ import annotations
import logging
import os
from collections.abc import Iterable
from typing import cast
import numpy as np
import PIL.Image
@@ -26,6 +28,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
retrieve_latents,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
logger = logging.getLogger(__name__)
def _load_model_index(model: str, local_files_only: bool) -> dict:
@@ -77,39 +82,56 @@ def get_wan22_i2v_pre_process_func(
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if req.pil_image is not None:
image = req.pil_image
if raw_image is None:
raise ValueError(
"""No image is provided. This model requires an image to run.""",
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
)
if not isinstance(raw_image, (str, PIL.Image.Image)):
raise TypeError(
f"""Unsupported image format {raw_image.__class__}.""",
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
)
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 480P
max_area = 480 * 832
aspect_ratio = image.height / image.width
# Calculate dimensions based on aspect ratio if not provided
if request.sampling_params.height is None or request.sampling_params.width is None:
# Default max area for 480P
max_area = 480 * 832
aspect_ratio = image.height / image.width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
if req.height is None:
req.height = height
if req.width is None:
req.width = width
if request.sampling_params.height is None:
request.sampling_params.height = height
if request.sampling_params.width is None:
request.sampling_params.width = width
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
# Resize image to target dimensions
image = image.resize(
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
PIL.Image.Resampling.LANCZOS,
)
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
return requests
# Preprocess for VAE
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
image, height=request.sampling_params.height, width=request.sampling_params.width
)
request.prompts[i] = prompt
return request
return pre_process_func
@@ -266,7 +288,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
guidance_scale: float | tuple[float, float] = 5.0,
frame_num: int = 81,
output_type: str | None = "np",
generator: torch.Generator | None = None,
generator: torch.Generator | list[torch.Generator] | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
image_embeds: torch.Tensor | None = None,
@@ -275,31 +297,51 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
**kwargs,
) -> DiffusionOutput:
# Get parameters from request or arguments
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
if len(req.prompts) > 1:
raise ValueError(
"""This model only supports a single prompt, not a batched request.""",
"""Please pass in a single prompt object or string, or a single-item list.""",
)
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
if prompt is None and prompt_embeds is None:
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 I2V generation.")
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
# Get image from request
if image is None:
image = req.pil_image
if image is None:
raise ValueError("Image is required for I2V generation.")
multi_modal_data = (
req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
)
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if raw_image is None:
raise ValueError("Image is required for I2V generation.")
if isinstance(raw_image, list):
if len(raw_image) > 1:
logger.warning(
"""Received a list of image. Only a single image is supported by this model."""
"""Taking only the first image for now."""
)
raw_image = raw_image[0]
if isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
height = req.height or height
width = req.width or width
num_frames = req.num_frames if req.num_frames else frame_num
num_steps = req.num_inference_steps or num_inference_steps
height = req.sampling_params.height or height
width = req.sampling_params.width or width
num_frames = req.sampling_params.num_frames or frame_num
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
# Respect per-request guidance_scale when explicitly provided.
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
# Handle guidance scales
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
guidance_high = (
req.guidance_scale_2
if req.guidance_scale_2 is not None
req.sampling_params.guidance_scale_2
if req.sampling_params.guidance_scale_2 is not None
else (
guidance_scale[1]
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
@@ -333,9 +375,9 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
# Generator setup
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.seed)
generator = req.sampling_params.generator
if generator is None and req.sampling_params.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
# Encode prompts
if prompt_embeds is None:
@@ -343,8 +385,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
prompt=prompt,
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
max_sequence_length=req.max_sequence_length or 512,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -410,7 +452,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
latents=req.sampling_params.latents,
last_image=last_image_tensor,
)
@@ -16,8 +16,10 @@ The key difference from the MoE-based I2V pipeline is:
from __future__ import annotations
import logging
import os
from collections.abc import Iterable
from typing import cast
import numpy as np
import PIL.Image
@@ -39,6 +41,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
retrieve_latents,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
logger = logging.getLogger(__name__)
def get_wan22_ti2v_post_process_func(
@@ -67,39 +72,56 @@ def get_wan22_ti2v_pre_process_func(
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
for i, prompt in enumerate(request.prompts):
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt)
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if req.pil_image is not None:
image = req.pil_image
if raw_image is None:
raise ValueError(
"""No image is provided. This model requires an image to run.""",
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
)
if not isinstance(raw_image, (str, PIL.Image.Image)):
raise TypeError(
f"""Unsupported image format {raw_image.__class__}.""",
"""Please correctly set `"multi_modal_data": {"image": <an image object or file path>, …}`""",
)
image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 720P (TI2V-5B default)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
# Calculate dimensions based on aspect ratio if not provided
if request.sampling_params.height is None or request.sampling_params.width is None:
# Default max area for 720P (TI2V-5B default)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
if req.height is None:
req.height = height
if req.width is None:
req.width = width
if request.sampling_params.height is None:
request.sampling_params.height = height
if request.sampling_params.width is None:
request.sampling_params.width = width
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
# Resize image to target dimensions
image = image.resize(
(request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None
PIL.Image.Resampling.LANCZOS,
)
prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
return requests
# Preprocess for VAE
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
image, height=request.sampling_params.height, width=request.sampling_params.width
)
request.prompts[i] = prompt
return request
return pre_process_func
@@ -205,31 +227,53 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
guidance_scale: float = 5.0,
frame_num: int = 81,
output_type: str | None = "np",
generator: torch.Generator | None = None,
generator: torch.Generator | list[torch.Generator] | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
attention_kwargs: dict | None = None,
**kwargs,
) -> DiffusionOutput:
# Get parameters from request or arguments
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
if len(req.prompts) > 1:
raise ValueError(
"""This model only supports a single prompt, not a batched request.""",
"""Please pass in a single prompt object or string, or a single-item list.""",
)
if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list
prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt")
negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt")
if prompt is None and prompt_embeds is None:
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 TI2V generation.")
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.")
# Get image from request (optional for TI2V)
if image is None:
image = req.pil_image
multi_modal_data = (
req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None
)
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
if isinstance(raw_image, list):
if len(raw_image) > 1:
logger.warning(
"""Received a list of image. Only a single image is supported by this model."""
"""Taking only the first image for now."""
)
raw_image = raw_image[0]
if raw_image is None:
image = None
elif isinstance(raw_image, str):
image = PIL.Image.open(raw_image)
else:
image = cast(PIL.Image.Image | torch.Tensor, raw_image)
# Default dimensions for TI2V-5B (720P)
height = req.height or height
width = req.width or width
num_frames = req.num_frames if req.num_frames else frame_num
num_steps = req.num_inference_steps or num_inference_steps
height = req.sampling_params.height or height
width = req.sampling_params.width or width
num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num
num_steps = req.sampling_params.num_inference_steps or num_inference_steps
# Respect per-request guidance_scale when explicitly provided.
if req.guidance_scale_provided:
guidance_scale = req.guidance_scale
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
self._guidance_scale = guidance_scale
@@ -254,9 +298,9 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
# Generator setup
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.seed)
generator = req.sampling_params.generator
if generator is None and req.sampling_params.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed)
# Encode prompts
if prompt_embeds is None:
@@ -264,8 +308,8 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
prompt=prompt,
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_scale > 1.0,
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
max_sequence_length=req.max_sequence_length or 512,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -310,7 +354,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
latents=req.sampling_params.latents,
)
else:
# T2V mode: prepare random latents
@@ -323,7 +367,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
latents=req.sampling_params.latents,
)
latent_condition = None
first_frame_mask = torch.ones(
@@ -316,15 +316,15 @@ class ZImagePipeline(nn.Module):
self,
req: OmniDiffusionRequest,
prompt: str | list[str] | None = None,
height: int | None = None,
width: int | None = None,
height: int = 1024,
width: int = 1024,
num_inference_steps: int = 50,
sigmas: list[float] | None = None,
guidance_scale: float = 5.0,
cfg_normalization: bool = False,
cfg_truncation: float = 1.0,
negative_prompt: str | list[str] | None = None,
num_images_per_prompt: int | None = 1,
num_images_per_prompt: int = 1,
generator: torch.Generator | list[torch.Generator] | None = None,
latents: torch.FloatTensor | None = None,
prompt_embeds: list[torch.FloatTensor] | None = None,
@@ -413,16 +413,28 @@ class ZImagePipeline(nn.Module):
`return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the
generated images.
"""
prompt = req.prompt
negative_prompt = req.negative_prompt
height: int = req.height or 1024
width: int = req.width or 1024
num_inference_steps = req.num_inference_steps or 50
generator = req.generator
guidance_scale = req.guidance_scale if req.guidance_rescale is not None else guidance_scale
req_num_outputs = getattr(req, "num_outputs_per_prompt", None)
if req_num_outputs and req_num_outputs > 0:
num_images_per_prompt = req_num_outputs
# TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "")
# TODO: May be some data formatting operations on the API side. Hack for now.
prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt
if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts):
negative_prompt = None
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
height = req.sampling_params.height or height
width = req.sampling_params.width or width
num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
generator = req.sampling_params.generator
sigmas = req.sampling_params.sigmas or sigmas
max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length
guidance_scale = (
req.sampling_params.guidance_scale if req.sampling_params.guidance_rescale is not None else guidance_scale
)
num_images_per_prompt = (
req.sampling_params.num_outputs_per_prompt
if req.sampling_params.num_outputs_per_prompt > 0
else num_images_per_prompt
)
vae_scale = self.vae_scale_factor * 2
if height % vae_scale != 0: