[Perf]: CFG parallel abstraction (#851)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
8ad39570b8
commit
23cf48d7bb
31 files changed
+1506
-814
No files matched your search
@@ -22,6 +22,7 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.flux import FluxTransformer2DModel
|
||||
@@ -554,6 +555,21 @@ class FluxPipeline(
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
return latents
|
||||
|
||||
def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool):
|
||||
if get_classifier_free_guidance_world_size() == 1:
|
||||
return True
|
||||
|
||||
if true_cfg_scale <= 1:
|
||||
logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.")
|
||||
return False
|
||||
|
||||
if not has_neg_prompt:
|
||||
logger.warning(
|
||||
"CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -637,6 +653,8 @@ class FluxPipeline(
|
||||
)
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
|
||||
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
|
||||
|
||||
(
|
||||
prompt_embeds,
|
||||
pooled_prompt_embeds,
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import (
|
||||
@@ -178,7 +179,7 @@ def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float:
|
||||
return float(mu)
|
||||
|
||||
|
||||
class Flux2KleinPipeline(nn.Module, SupportImageInput):
|
||||
class Flux2KleinPipeline(nn.Module, CFGParallelMixin, SupportImageInput):
|
||||
"""Flux2 klein pipeline for text-to-image generation."""
|
||||
|
||||
support_image_input = True
|
||||
@@ -923,38 +924,44 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput):
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype)
|
||||
latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1)
|
||||
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=None,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
txt_ids=text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
joint_attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
noise_pred = noise_pred[:, : latents.size(1) :]
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": None,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"txt_ids": text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"joint_attention_kwargs": self.attention_kwargs,
|
||||
"return_dict": False,
|
||||
}
|
||||
if self.do_classifier_free_guidance:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=None,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
txt_ids=negative_text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
joint_attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
neg_noise_pred = neg_noise_pred[:, : latents.size(1) :]
|
||||
noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred)
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": None,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"txt_ids": negative_text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"joint_attention_kwargs": self.attention_kwargs,
|
||||
"return_dict": False,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
latents_dtype = latents.dtype
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
# For editing pipelines, we need to slice the output to remove condition latents
|
||||
output_slice = latents.size(1) if image_latents is not None else None
|
||||
|
||||
if latents.dtype != latents_dtype and torch.backends.mps.is_available():
|
||||
latents = latents.to(latents_dtype)
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=self.do_classifier_free_guidance,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
output_slice=output_slice,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance)
|
||||
|
||||
if callback_on_step_end is not None:
|
||||
callback_kwargs = {}
|
||||
|
||||
@@ -8,6 +8,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -23,6 +24,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import LongCatImageTransformer2DModel
|
||||
@@ -197,9 +199,7 @@ def get_prompt_language(prompt):
|
||||
return "en"
|
||||
|
||||
|
||||
class LongCatImagePipeline(
|
||||
nn.Module,
|
||||
):
|
||||
class LongCatImagePipeline(nn.Module, CFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -460,6 +460,16 @@ class LongCatImagePipeline(
|
||||
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
||||
)
|
||||
|
||||
def cfg_normalize_function(self, noise_pred, comb_pred, cfg_renorm_min=0.0):
|
||||
"""
|
||||
Normalize the combined noise prediction.
|
||||
"""
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
|
||||
noise_pred = comb_pred * scale
|
||||
return noise_pred
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -604,6 +614,9 @@ class LongCatImagePipeline(
|
||||
if self.do_classifier_free_guidance:
|
||||
negative_prompt_embeds = negative_prompt_embeds.to(device)
|
||||
|
||||
# custom partial function with cfg_renorm_min
|
||||
self.cfg_normalize_function = partial(self.cfg_normalize_function, cfg_renorm_min=cfg_renorm_min)
|
||||
|
||||
# 6. Denoising loop
|
||||
for i, t in enumerate(timesteps):
|
||||
if self._interrupt:
|
||||
@@ -612,43 +625,37 @@ class LongCatImagePipeline(
|
||||
self._current_timestep = t
|
||||
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
||||
|
||||
noise_pred_text = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
txt_ids=text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": guidance,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"txt_ids": text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
if self.do_classifier_free_guidance:
|
||||
noise_pred_uncond = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
txt_ids=negative_text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_pred_uncond + self._guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
if enable_cfg_renorm:
|
||||
cond_norm = torch.norm(noise_pred_text, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
|
||||
noise_pred = noise_pred * scale
|
||||
negative_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"txt_ids": negative_text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
else:
|
||||
noise_pred = noise_pred_text
|
||||
negative_kwargs = None
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents_dtype = latents.dtype
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=self.do_classifier_free_guidance,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=enable_cfg_renorm,
|
||||
)
|
||||
|
||||
if latents.dtype != latents_dtype:
|
||||
if torch.backends.mps.is_available():
|
||||
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
||||
latents = latents.to(latents_dtype)
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
@@ -215,7 +216,7 @@ def split_quotation(prompt, quote_pairs=None):
|
||||
return result
|
||||
|
||||
|
||||
class LongCatImageEditPipeline(nn.Module, SupportImageInput):
|
||||
class LongCatImageEditPipeline(nn.Module, CFGParallelMixin, SupportImageInput):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -652,38 +653,39 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput):
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
timestep = t.expand(latent_model_input.shape[0]).to(latents.dtype)
|
||||
do_true_cfg = guidance_scale > 1
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": guidance,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"txt_ids": text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
|
||||
noise_pred_text = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
txt_ids=text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred_text = noise_pred_text[:, :image_seq_len]
|
||||
if guidance_scale > 1:
|
||||
noise_pred_uncond = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
txt_ids=negative_text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred_uncond = noise_pred_uncond[:, :image_seq_len]
|
||||
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"txt_ids": negative_text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
else:
|
||||
noise_pred = noise_pred_text
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents_dtype = latents.dtype
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
negative_kwargs = None
|
||||
|
||||
if latents.dtype != latents_dtype:
|
||||
if torch.backends.mps.is_available():
|
||||
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
||||
latents = latents.to(latents_dtype)
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
output_slice=image_seq_len,
|
||||
)
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
|
||||
@@ -23,18 +23,19 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from diffusers.image_processor import VaeImageProcessor
|
||||
from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
|
||||
from diffusers.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from transformers import Qwen2TokenizerFast, Qwen3Model
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel
|
||||
@@ -139,9 +140,7 @@ def retrieve_timesteps(
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
|
||||
class OvisImagePipeline(
|
||||
nn.Module,
|
||||
):
|
||||
class OvisImagePipeline(nn.Module, CFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -431,68 +430,77 @@ class OvisImagePipeline(
|
||||
)
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
def denoising(
|
||||
def diffuse(
|
||||
self,
|
||||
latents,
|
||||
timesteps,
|
||||
prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
guidance_scale,
|
||||
do_classifier_free_guidance,
|
||||
callback_on_step_end,
|
||||
callback_on_step_end_tensor_inputs,
|
||||
text_ids,
|
||||
negative_text_ids,
|
||||
latent_image_ids,
|
||||
):
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor,
|
||||
text_ids: torch.Tensor,
|
||||
negative_text_ids: torch.Tensor,
|
||||
latent_image_ids: torch.Tensor,
|
||||
do_true_cfg: bool,
|
||||
guidance_scale: float,
|
||||
cfg_normalize: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Diffusion loop with optional classifier-free guidance.
|
||||
|
||||
Args:
|
||||
latents: Noise latents to denoise
|
||||
timesteps: Diffusion timesteps
|
||||
prompt_embeds: Positive prompt embeddings
|
||||
negative_prompt_embeds: Negative prompt embeddings
|
||||
text_ids: Position IDs for positive text
|
||||
negative_text_ids: Position IDs for negative text
|
||||
latent_image_ids: Position IDs for image latents
|
||||
do_true_cfg: Whether to apply CFG
|
||||
guidance_scale: CFG scale factor
|
||||
cfg_normalize: Whether to normalize CFG output (default: False)
|
||||
|
||||
Returns:
|
||||
Denoised latents
|
||||
"""
|
||||
self.scheduler.set_begin_index(0)
|
||||
|
||||
for i, t in enumerate(timesteps):
|
||||
if self._interrupt:
|
||||
if self.interrupt:
|
||||
break
|
||||
|
||||
self._current_timestep = t
|
||||
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
||||
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
||||
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
txt_ids=text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
positive_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"txt_ids": text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"txt_ids": negative_text_ids,
|
||||
"img_ids": latent_image_ids,
|
||||
"return_dict": False,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
if do_classifier_free_guidance:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
txt_ids=negative_text_ids,
|
||||
img_ids=latent_image_ids,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg,
|
||||
guidance_scale,
|
||||
positive_kwargs,
|
||||
negative_kwargs,
|
||||
cfg_normalize,
|
||||
)
|
||||
|
||||
noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred)
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents_dtype = latents.dtype
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
|
||||
if latents.dtype != latents_dtype:
|
||||
if torch.backends.mps.is_available():
|
||||
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
|
||||
latents = latents.to(latents_dtype)
|
||||
|
||||
# Not used in this pipeline
|
||||
# if callback_on_step_end is not None:
|
||||
# callback_kwargs = {}
|
||||
# for k in callback_on_step_end_tensor_inputs:
|
||||
# callback_kwargs[k] = locals()[k]
|
||||
# callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
||||
|
||||
# latents = callback_outputs.pop("latents", latents)
|
||||
# prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
return latents
|
||||
|
||||
@@ -704,23 +712,18 @@ class OvisImagePipeline(
|
||||
if self.joint_attention_kwargs is None:
|
||||
self._joint_attention_kwargs = {}
|
||||
|
||||
# 6. Denoising loop
|
||||
|
||||
# We set the index here to remove DtoH sync, helpful especially during compilation
|
||||
# Check out more details here: https://github.com/huggingface/diffusers/pull/11696
|
||||
|
||||
latents = self.denoising(
|
||||
latents,
|
||||
timesteps,
|
||||
prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
guidance_scale,
|
||||
do_classifier_free_guidance,
|
||||
callback_on_step_end,
|
||||
callback_on_step_end_tensor_inputs,
|
||||
text_ids,
|
||||
negative_text_ids,
|
||||
latent_image_ids,
|
||||
# 6. Denoising loop using diffuse method
|
||||
latents = self.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds if do_classifier_free_guidance else None,
|
||||
text_ids=text_ids,
|
||||
negative_text_ids=negative_text_ids if do_classifier_free_guidance else None,
|
||||
latent_image_ids=latent_image_ids,
|
||||
do_true_cfg=do_classifier_free_guidance,
|
||||
guidance_scale=guidance_scale,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen Image diffusion model components."""
|
||||
|
||||
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
QwenImageCFGParallelMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import (
|
||||
QwenImagePipeline,
|
||||
get_qwen_image_post_process_func,
|
||||
@@ -11,6 +14,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"QwenImageCFGParallelMixin",
|
||||
"QwenImagePipeline",
|
||||
"QwenImageTransformer2DModel",
|
||||
"get_qwen_image_post_process_func",
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CFG Parallel Mixin for Qwen Image series
|
||||
Shared by
|
||||
- QwenImagePipeline
|
||||
- QwenImageEditPipeline
|
||||
- QwenImageEditPlusPipeline
|
||||
- QwenImageLayeredPipeline
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QwenImageCFGParallelMixin(CFGParallelMixin):
|
||||
"""
|
||||
Base Mixin class for Qwen Image pipelines providing shared CFG methods.
|
||||
"""
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds: torch.Tensor,
|
||||
prompt_embeds_mask: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds_mask: torch.Tensor,
|
||||
latents: torch.Tensor,
|
||||
img_shapes: torch.Tensor,
|
||||
txt_seq_lens: torch.Tensor,
|
||||
negative_txt_seq_lens: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
do_true_cfg: bool,
|
||||
guidance: torch.Tensor,
|
||||
true_cfg_scale: float,
|
||||
image_latents: torch.Tensor | None = None,
|
||||
cfg_normalize: bool = True,
|
||||
additional_transformer_kwargs: dict[str, Any] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Diffusion loop with optional classifier-free guidance.
|
||||
|
||||
Args:
|
||||
prompt_embeds: Positive prompt embeddings
|
||||
prompt_embeds_mask: Mask for positive prompt
|
||||
negative_prompt_embeds: Negative prompt embeddings
|
||||
negative_prompt_embeds_mask: Mask for negative prompt
|
||||
latents: Noise latents to denoise
|
||||
img_shapes: Image shape information
|
||||
txt_seq_lens: Text sequence lengths for positive prompts
|
||||
negative_txt_seq_lens: Text sequence lengths for negative prompts
|
||||
timesteps: Diffusion timesteps
|
||||
do_true_cfg: Whether to apply CFG
|
||||
guidance: Guidance scale tensor
|
||||
true_cfg_scale: CFG scale factor
|
||||
image_latents: Conditional image latents for editing (default: None)
|
||||
cfg_normalize: Whether to normalize CFG output (default: True)
|
||||
additional_transformer_kwargs: Extra kwargs to pass to transformer (default: None)
|
||||
|
||||
Returns:
|
||||
Denoised latents
|
||||
"""
|
||||
self.scheduler.set_begin_index(0)
|
||||
self.transformer.do_true_cfg = do_true_cfg
|
||||
additional_transformer_kwargs = additional_transformer_kwargs or {}
|
||||
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
|
||||
# Broadcast timestep to match batch size
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Concatenate image latents with noise latents if available (for editing pipelines)
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": guidance,
|
||||
"encoder_hidden_states_mask": prompt_embeds_mask,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"txt_seq_lens": txt_seq_lens,
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"guidance": guidance,
|
||||
"encoder_hidden_states_mask": negative_prompt_embeds_mask,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"txt_seq_lens": negative_txt_seq_lens,
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# For editing pipelines, we need to slice the output to remove condition latents
|
||||
output_slice = latents.size(1) if image_latents is not None else None
|
||||
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg,
|
||||
true_cfg_scale,
|
||||
positive_kwargs,
|
||||
negative_kwargs,
|
||||
cfg_normalize,
|
||||
output_slice,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
return latents
|
||||
|
||||
def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool):
|
||||
"""
|
||||
Validate whether CFG parallel is properly configured for the current generation request.
|
||||
|
||||
When CFG parallel is enabled (cfg_parallel_world_size > 1), this method verifies that the necessary
|
||||
conditions are met for correct parallel execution. If validation fails, a warning is
|
||||
logged to help identify configuration issues.
|
||||
|
||||
Args:
|
||||
true_cfg_scale: The classifier-free guidance scale value. Must be > 1 for CFG to
|
||||
have an effect.
|
||||
has_neg_prompt: Whether negative prompts or negative prompt embeddings are provided.
|
||||
Required for CFG to perform unconditional prediction.
|
||||
|
||||
Returns:
|
||||
True if CFG parallel is disabled or all validation checks pass, False otherwise.
|
||||
|
||||
Note:
|
||||
When CFG parallel is disabled (world_size == 1), this method always returns True
|
||||
as no parallel-specific validation is needed.
|
||||
"""
|
||||
if get_classifier_free_guidance_world_size() == 1:
|
||||
return True
|
||||
|
||||
if true_cfg_scale <= 1:
|
||||
logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.")
|
||||
return False
|
||||
|
||||
if not has_neg_prompt:
|
||||
logger.warning(
|
||||
"CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
@@ -25,13 +25,11 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
QwenImageCFGParallelMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
QwenImageTransformer2DModel,
|
||||
)
|
||||
@@ -239,9 +237,7 @@ def apply_rotary_emb_qwen(
|
||||
return x_out.type_as(x)
|
||||
|
||||
|
||||
class QwenImagePipeline(
|
||||
nn.Module,
|
||||
):
|
||||
class QwenImagePipeline(nn.Module, QwenImageCFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -536,109 +532,6 @@ class QwenImagePipeline(
|
||||
def interrupt(self):
|
||||
return self._interrupt
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds,
|
||||
prompt_embeds_mask,
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
timesteps,
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
):
|
||||
self.scheduler.set_begin_index(0)
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
|
||||
# Broadcast timestep to match batch size
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Enable CFG-parallel: rank0 computes positive, rank1 computes negative.
|
||||
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
|
||||
|
||||
self.transformer.do_true_cfg = do_true_cfg
|
||||
|
||||
if cfg_parallel_ready:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
|
||||
if cfg_rank == 0:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
else:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
|
||||
if cfg_rank == 0:
|
||||
noise_pred = gathered[0]
|
||||
neg_noise_pred = gathered[1]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
cfg_group.broadcast(latents, src=0)
|
||||
|
||||
else:
|
||||
# Forward pass for positive prompt (or unconditional if no CFG)
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
# Forward pass for negative prompt (CFG)
|
||||
if do_true_cfg:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latents,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -720,6 +613,8 @@ class QwenImagePipeline(
|
||||
)
|
||||
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
|
||||
|
||||
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
prompt_embeds=prompt_embeds,
|
||||
@@ -790,6 +685,12 @@ class QwenImagePipeline(
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
image_latents=None,
|
||||
cfg_normalize=True,
|
||||
additional_transformer_kwargs={
|
||||
"return_dict": False,
|
||||
"attention_kwargs": self.attention_kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -25,14 +25,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
QwenImageCFGParallelMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift
|
||||
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
QwenImageTransformer2DModel,
|
||||
@@ -213,10 +211,7 @@ def retrieve_latents(
|
||||
raise AttributeError("Could not access latents of provided encoder_output")
|
||||
|
||||
|
||||
class QwenImageEditPipeline(
|
||||
nn.Module,
|
||||
SupportImageInput,
|
||||
):
|
||||
class QwenImageEditPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -598,118 +593,6 @@ class QwenImageEditPipeline(
|
||||
def interrupt(self):
|
||||
return self._interrupt
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds,
|
||||
prompt_embeds_mask,
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
timesteps,
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
):
|
||||
"""Diffusion loop with optional image conditioning."""
|
||||
self.scheduler.set_begin_index(0)
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
# broadcast to batch dimension and place on same device/dtype as latents
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Concatenate image latents with noise latents if available
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
self.transformer.do_true_cfg = do_true_cfg # used in teacache hook
|
||||
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
|
||||
|
||||
if cfg_parallel_ready:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
|
||||
if cfg_rank == 0:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
else:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
local_pred = local_pred[:, : latents.size(1)]
|
||||
|
||||
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
|
||||
if cfg_rank == 0:
|
||||
noise_pred = gathered[0]
|
||||
neg_noise_pred = gathered[1]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
cfg_group.broadcast(latents, src=0)
|
||||
|
||||
else:
|
||||
# Forward pass for positive prompt (or unconditional if no CFG)
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_pred[:, : latents.size(1)]
|
||||
|
||||
# Forward pass for negative prompt (CFG)
|
||||
if do_true_cfg:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -823,6 +706,8 @@ class QwenImageEditPipeline(
|
||||
)
|
||||
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
|
||||
|
||||
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
image=prompt_image, # Use resized image for prompt encoding
|
||||
@@ -886,7 +771,6 @@ class QwenImageEditPipeline(
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
@@ -894,6 +778,12 @@ class QwenImageEditPipeline(
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
image_latents=image_latents,
|
||||
cfg_normalize=True,
|
||||
additional_transformer_kwargs={
|
||||
"return_dict": False,
|
||||
"attention_kwargs": self.attention_kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -23,14 +23,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
QwenImageCFGParallelMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import (
|
||||
calculate_dimensions,
|
||||
@@ -168,7 +166,7 @@ def get_qwen_image_edit_plus_post_process_func(
|
||||
return post_process_func
|
||||
|
||||
|
||||
class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
class QwenImageEditPlusPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -530,116 +528,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
def interrupt(self):
|
||||
return self._interrupt
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds,
|
||||
prompt_embeds_mask,
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
timesteps,
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
):
|
||||
"""Diffusion loop with optional image conditioning."""
|
||||
self.scheduler.set_begin_index(0)
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
# broadcast to batch dimension and place on same device/dtype as latents
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Concatenate image latents with noise latents if available
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
self.transformer.do_true_cfg = do_true_cfg
|
||||
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
|
||||
|
||||
if cfg_parallel_ready:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
|
||||
if cfg_rank == 0:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
else:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
local_pred = local_pred[:, : latents.size(1)]
|
||||
|
||||
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
|
||||
if cfg_rank == 0:
|
||||
noise_pred = gathered[0]
|
||||
neg_noise_pred = gathered[1]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
cfg_group.broadcast(latents, src=0)
|
||||
|
||||
else:
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_pred[:, : latents.size(1)]
|
||||
|
||||
if do_true_cfg:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -773,6 +661,8 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
)
|
||||
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
|
||||
|
||||
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
image=condition_images, # Use condition images for prompt encoding
|
||||
@@ -840,7 +730,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
@@ -848,6 +737,12 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
image_latents=image_latents,
|
||||
cfg_normalize=True,
|
||||
additional_transformer_kwargs={
|
||||
"return_dict": False,
|
||||
"attention_kwargs": self.attention_kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -22,17 +22,15 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.qwen_image.autoencoder_kl_qwenimage import (
|
||||
AutoencoderKLQwenImage,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
QwenImageCFGParallelMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
QwenImageTransformer2DModel,
|
||||
)
|
||||
@@ -191,7 +189,7 @@ def retrieve_latents(
|
||||
raise AttributeError("Could not access latents of provided encoder_output")
|
||||
|
||||
|
||||
class QwenImageLayeredPipeline(nn.Module, SupportImageInput):
|
||||
class QwenImageLayeredPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
|
||||
color_format = "RGBA"
|
||||
|
||||
def __init__(
|
||||
@@ -553,138 +551,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
|
||||
return latents
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds,
|
||||
prompt_embeds_mask,
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
timesteps,
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
cfg_normalize,
|
||||
is_rgb,
|
||||
):
|
||||
self.scheduler.set_begin_index(0)
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
|
||||
self._current_timestep = t
|
||||
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([latents, image_latents], dim=1)
|
||||
|
||||
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
||||
timestep = t.expand(latents.shape[0]).to(latents.dtype)
|
||||
|
||||
# Enable CFG-parallel: rank0 computes positive, rank1 computes negative.
|
||||
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
|
||||
|
||||
self.transformer.do_true_cfg = do_true_cfg # used in teacache hook
|
||||
|
||||
if cfg_parallel_ready:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
|
||||
if cfg_rank == 0:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
additional_t_cond=is_rgb,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
local_pred = local_pred[:, : latents.size(1)]
|
||||
else:
|
||||
local_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
additional_t_cond=is_rgb,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
local_pred = local_pred[:, : latents.size(1)]
|
||||
|
||||
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
|
||||
if cfg_rank == 0:
|
||||
noise_pred = gathered[0]
|
||||
neg_noise_pred = gathered[1]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
|
||||
if cfg_normalize:
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
else:
|
||||
noise_pred = comb_pred
|
||||
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
cfg_group.broadcast(latents, src=0)
|
||||
|
||||
else:
|
||||
# Forward pass for positive prompt (or unconditional if no CFG)
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=prompt_embeds_mask,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
additional_t_cond=is_rgb,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_pred[:, : latents.size(1)]
|
||||
|
||||
if do_true_cfg:
|
||||
neg_noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep / 1000,
|
||||
guidance=guidance,
|
||||
encoder_hidden_states_mask=negative_prompt_embeds_mask,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
img_shapes=img_shapes,
|
||||
txt_seq_lens=negative_txt_seq_lens,
|
||||
attention_kwargs=self.attention_kwargs,
|
||||
additional_t_cond=is_rgb,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
|
||||
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
|
||||
|
||||
if cfg_normalize:
|
||||
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
|
||||
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
|
||||
noise_pred = comb_pred * (cond_norm / noise_norm)
|
||||
else:
|
||||
noise_pred = comb_pred
|
||||
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
|
||||
# no need callback now
|
||||
return latents
|
||||
|
||||
@property
|
||||
def guidance_scale(self):
|
||||
return self._guidance_scale
|
||||
@@ -840,6 +706,8 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
)
|
||||
|
||||
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
|
||||
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
|
||||
|
||||
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
prompt_embeds=prompt_embeds,
|
||||
@@ -924,7 +792,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask,
|
||||
latents,
|
||||
image_latents,
|
||||
img_shapes,
|
||||
txt_seq_lens,
|
||||
negative_txt_seq_lens,
|
||||
@@ -932,8 +799,13 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
|
||||
do_true_cfg,
|
||||
guidance,
|
||||
true_cfg_scale,
|
||||
cfg_normalize,
|
||||
is_rgb,
|
||||
image_latents=image_latents,
|
||||
cfg_normalize=cfg_normalize,
|
||||
additional_transformer_kwargs={
|
||||
"return_dict": False,
|
||||
"additional_t_cond": is_rgb,
|
||||
"attention_kwargs": self.attention_kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -16,6 +16,7 @@ from transformers import CLIPTextModelWithProjection, CLIPTokenizer, T5EncoderMo
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.sd3.sd3_transformer import (
|
||||
@@ -126,9 +127,7 @@ def retrieve_timesteps(
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
|
||||
class StableDiffusion3Pipeline(
|
||||
nn.Module,
|
||||
):
|
||||
class StableDiffusion3Pipeline(nn.Module, CFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -498,15 +497,35 @@ class StableDiffusion3Pipeline(
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds,
|
||||
pooled_prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
negative_pooled_prompt_embeds,
|
||||
latents,
|
||||
timesteps,
|
||||
do_cfg,
|
||||
):
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
pooled_prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor,
|
||||
negative_pooled_prompt_embeds: torch.Tensor,
|
||||
do_true_cfg: bool,
|
||||
guidance_scale: float,
|
||||
cfg_normalize: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Diffusion loop with optional classifier-free guidance.
|
||||
|
||||
Args:
|
||||
latents: Noise latents to denoise
|
||||
timesteps: Diffusion timesteps
|
||||
prompt_embeds: Positive prompt embeddings
|
||||
pooled_prompt_embeds: Pooled positive prompt embeddings
|
||||
negative_prompt_embeds: Negative prompt embeddings
|
||||
negative_pooled_prompt_embeds: Pooled negative prompt embeddings
|
||||
do_true_cfg: Whether to apply CFG
|
||||
guidance_scale: CFG scale factor
|
||||
cfg_normalize: Whether to normalize CFG output (default: False)
|
||||
|
||||
Returns:
|
||||
Denoised latents
|
||||
"""
|
||||
self.scheduler.set_begin_index(0)
|
||||
|
||||
for _, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
@@ -515,29 +534,36 @@ class StableDiffusion3Pipeline(
|
||||
# Broadcast timestep to match batch size
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
transformer_kwargs = {
|
||||
positive_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"pooled_projections": pooled_prompt_embeds,
|
||||
"return_dict": False,
|
||||
}
|
||||
|
||||
noise_pred = self.transformer(**transformer_kwargs)[0]
|
||||
|
||||
if do_cfg:
|
||||
neg_transformer_kwargs = {
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latents,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"pooled_projections": negative_pooled_prompt_embeds,
|
||||
"return_dict": False,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg,
|
||||
guidance_scale,
|
||||
positive_kwargs,
|
||||
negative_kwargs,
|
||||
cfg_normalize,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
neg_noise_pred = self.transformer(**neg_transformer_kwargs)[0]
|
||||
noise_pred = neg_noise_pred + self.guidance_scale * (noise_pred - neg_noise_pred)
|
||||
# compute the previous noisy sample x_t -> x_t-1
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
@@ -644,14 +670,17 @@ class StableDiffusion3Pipeline(
|
||||
timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1])
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
# Denoising loop using diffuse method
|
||||
latents = self.diffuse(
|
||||
prompt_embeds,
|
||||
pooled_prompt_embeds,
|
||||
negative_prompt_embeds,
|
||||
negative_pooled_prompt_embeds,
|
||||
latents,
|
||||
timesteps,
|
||||
do_cfg,
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
pooled_prompt_embeds=pooled_prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds if do_cfg else None,
|
||||
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds if do_cfg else None,
|
||||
do_true_cfg=do_cfg,
|
||||
guidance_scale=self.guidance_scale,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
@@ -7,7 +7,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
@@ -18,12 +18,14 @@ from transformers import AutoTokenizer, UMT5EncoderModel
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -184,7 +186,7 @@ def get_wan22_pre_process_func(
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22Pipeline(nn.Module):
|
||||
class Wan22Pipeline(nn.Module, CFGParallelMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -585,26 +587,44 @@ class Wan22Pipeline(nn.Module):
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
noise_pred = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
|
||||
noise_uncond = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
self._current_timestep = None
|
||||
|
||||
# For I2V mode: blend final latents with condition
|
||||
@@ -629,6 +649,21 @@ class Wan22Pipeline(nn.Module):
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass through transformer to predict noise.
|
||||
|
||||
Args:
|
||||
current_model: The transformer model to use (transformer or transformer_2)
|
||||
**kwargs: Arguments to pass to the transformer
|
||||
|
||||
Returns:
|
||||
Predicted noise tensor
|
||||
"""
|
||||
if current_model is None:
|
||||
current_model = self.transformer
|
||||
return current_model(**kwargs)[0]
|
||||
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -18,6 +18,7 @@ from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
@@ -29,6 +30,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -136,7 +138,7 @@ def get_wan22_i2v_pre_process_func(
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin):
|
||||
"""
|
||||
Wan2.2 Image-to-Video Pipeline.
|
||||
|
||||
@@ -484,31 +486,46 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
# Forward pass
|
||||
noise_pred = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
encoder_hidden_states_image=image_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Classifier-free guidance
|
||||
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
|
||||
noise_uncond = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
encoder_hidden_states_image=image_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
# Scheduler step
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
self._current_timestep = None
|
||||
|
||||
# For expand_timesteps mode, blend final latents with condition
|
||||
@@ -533,6 +550,21 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass through transformer to predict noise.
|
||||
|
||||
Args:
|
||||
current_model: The transformer model to use (transformer or transformer_2)
|
||||
**kwargs: Arguments to pass to the transformer
|
||||
|
||||
Returns:
|
||||
Predicted noise tensor
|
||||
"""
|
||||
if current_model is None:
|
||||
current_model = self.transformer
|
||||
return current_model(**kwargs)[0]
|
||||
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
|
||||
@@ -19,7 +19,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -31,6 +31,7 @@ from transformers import AutoTokenizer, UMT5EncoderModel
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
@@ -42,6 +43,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -126,7 +128,7 @@ def get_wan22_ti2v_pre_process_func(
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin):
|
||||
"""
|
||||
Wan2.2 Text-Image-to-Video (TI2V) Pipeline.
|
||||
|
||||
@@ -399,29 +401,44 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
|
||||
# Forward pass
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Classifier-free guidance
|
||||
if guidance_scale > 1.0 and negative_prompt_embeds is not None:
|
||||
noise_uncond = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
# Scheduler step
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
self._current_timestep = None
|
||||
|
||||
# For I2V mode, blend final latents with condition
|
||||
@@ -446,6 +463,21 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass through transformer to predict noise.
|
||||
|
||||
Args:
|
||||
current_model: The transformer model to use
|
||||
**kwargs: Arguments to pass to the transformer
|
||||
|
||||
Returns:
|
||||
Predicted noise tensor
|
||||
"""
|
||||
if current_model is None:
|
||||
current_model = self.transformer
|
||||
return current_model(**kwargs)[0]
|
||||
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
|
||||
Reference in new issue
Block a user