[Perf]: CFG parallel abstraction (#851)

Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Didan DengandHongsheng Liu authored and GitHub committed 2026-01-30 16:23:44 +00:00
1 parent 8ad39570b8
commit 23cf48d7bb
31 files changed
+1506 -814

No files matched your search

@@ -22,6 +22,7 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.flux import FluxTransformer2DModel
@@ -554,6 +555,21 @@ class FluxPipeline(
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
return latents
def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool):
if get_classifier_free_guidance_world_size() == 1:
return True
if true_cfg_scale <= 1:
logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.")
return False
if not has_neg_prompt:
logger.warning(
"CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings."
)
return False
return True
def forward(
self,
req: OmniDiffusionRequest,
@@ -637,6 +653,8 @@ class FluxPipeline(
)
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
(
prompt_embeds,
pooled_prompt_embeds,
@@ -36,6 +36,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import (
@@ -178,7 +179,7 @@ def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float:
return float(mu)
class Flux2KleinPipeline(nn.Module, SupportImageInput):
class Flux2KleinPipeline(nn.Module, CFGParallelMixin, SupportImageInput):
"""Flux2 klein pipeline for text-to-image generation."""
support_image_input = True
@@ -923,38 +924,44 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput):
latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype)
latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1)
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=None,
encoder_hidden_states=prompt_embeds,
txt_ids=text_ids,
img_ids=latent_image_ids,
joint_attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_pred[:, : latents.size(1) :]
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"guidance": None,
"encoder_hidden_states": prompt_embeds,
"txt_ids": text_ids,
"img_ids": latent_image_ids,
"joint_attention_kwargs": self.attention_kwargs,
"return_dict": False,
}
if self.do_classifier_free_guidance:
neg_noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=None,
encoder_hidden_states=negative_prompt_embeds,
txt_ids=negative_text_ids,
img_ids=latent_image_ids,
joint_attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
neg_noise_pred = neg_noise_pred[:, : latents.size(1) :]
noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred)
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"guidance": None,
"encoder_hidden_states": negative_prompt_embeds,
"txt_ids": negative_text_ids,
"img_ids": latent_image_ids,
"joint_attention_kwargs": self.attention_kwargs,
"return_dict": False,
}
else:
negative_kwargs = None
latents_dtype = latents.dtype
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
# For editing pipelines, we need to slice the output to remove condition latents
output_slice = latents.size(1) if image_latents is not None else None
if latents.dtype != latents_dtype and torch.backends.mps.is_available():
latents = latents.to(latents_dtype)
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=self.do_classifier_free_guidance,
true_cfg_scale=guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=False,
output_slice=output_slice,
)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance)
if callback_on_step_end is not None:
callback_kwargs = {}
@@ -8,6 +8,7 @@ import json
import os
import re
from collections.abc import Iterable
from functools import partial
from typing import Any
import numpy as np
@@ -23,6 +24,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import LongCatImageTransformer2DModel
@@ -197,9 +199,7 @@ def get_prompt_language(prompt):
return "en"
class LongCatImagePipeline(
nn.Module,
):
class LongCatImagePipeline(nn.Module, CFGParallelMixin):
def __init__(
self,
*,
@@ -460,6 +460,16 @@ class LongCatImagePipeline(
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
)
def cfg_normalize_function(self, noise_pred, comb_pred, cfg_renorm_min=0.0):
"""
Normalize the combined noise prediction.
"""
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
noise_pred = comb_pred * scale
return noise_pred
def forward(
self,
req: OmniDiffusionRequest,
@@ -604,6 +614,9 @@ class LongCatImagePipeline(
if self.do_classifier_free_guidance:
negative_prompt_embeds = negative_prompt_embeds.to(device)
# custom partial function with cfg_renorm_min
self.cfg_normalize_function = partial(self.cfg_normalize_function, cfg_renorm_min=cfg_renorm_min)
# 6. Denoising loop
for i, t in enumerate(timesteps):
if self._interrupt:
@@ -612,43 +625,37 @@ class LongCatImagePipeline(
self._current_timestep = t
timestep = t.expand(latents.shape[0]).to(latents.dtype)
noise_pred_text = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states=prompt_embeds,
txt_ids=text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
positive_kwargs = {
"hidden_states": latents,
"timestep": timestep / 1000,
"guidance": guidance,
"encoder_hidden_states": prompt_embeds,
"txt_ids": text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
if self.do_classifier_free_guidance:
noise_pred_uncond = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
encoder_hidden_states=negative_prompt_embeds,
txt_ids=negative_text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
noise_pred = noise_pred_uncond + self._guidance_scale * (noise_pred_text - noise_pred_uncond)
if enable_cfg_renorm:
cond_norm = torch.norm(noise_pred_text, dim=-1, keepdim=True)
noise_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
noise_pred = noise_pred * scale
negative_kwargs = {
"hidden_states": latents,
"timestep": timestep / 1000,
"encoder_hidden_states": negative_prompt_embeds,
"txt_ids": negative_text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
else:
noise_pred = noise_pred_text
negative_kwargs = None
# compute the previous noisy sample x_t -> x_t-1
latents_dtype = latents.dtype
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=self.do_classifier_free_guidance,
true_cfg_scale=guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=enable_cfg_renorm,
)
if latents.dtype != latents_dtype:
if torch.backends.mps.is_available():
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
latents = latents.to(latents_dtype)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance)
self._current_timestep = None
@@ -26,6 +26,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
@@ -215,7 +216,7 @@ def split_quotation(prompt, quote_pairs=None):
return result
class LongCatImageEditPipeline(nn.Module, SupportImageInput):
class LongCatImageEditPipeline(nn.Module, CFGParallelMixin, SupportImageInput):
def __init__(
self,
*,
@@ -652,38 +653,39 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput):
latent_model_input = torch.cat([latents, image_latents], dim=1)
timestep = t.expand(latent_model_input.shape[0]).to(latents.dtype)
do_true_cfg = guidance_scale > 1
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"guidance": guidance,
"encoder_hidden_states": prompt_embeds,
"txt_ids": text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
noise_pred_text = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states=prompt_embeds,
txt_ids=text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
noise_pred_text = noise_pred_text[:, :image_seq_len]
if guidance_scale > 1:
noise_pred_uncond = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
encoder_hidden_states=negative_prompt_embeds,
txt_ids=negative_text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
noise_pred_uncond = noise_pred_uncond[:, :image_seq_len]
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"encoder_hidden_states": negative_prompt_embeds,
"txt_ids": negative_text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
else:
noise_pred = noise_pred_text
# compute the previous noisy sample x_t -> x_t-1
latents_dtype = latents.dtype
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
negative_kwargs = None
if latents.dtype != latents_dtype:
if torch.backends.mps.is_available():
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
latents = latents.to(latents_dtype)
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=do_true_cfg,
true_cfg_scale=guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=False,
output_slice=image_seq_len,
)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
self._current_timestep = None
@@ -23,18 +23,19 @@ from typing import Any
import numpy as np
import torch
import torch.nn as nn
from diffusers.image_processor import VaeImageProcessor
from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
from diffusers.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import Qwen2TokenizerFast, Qwen3Model
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel
@@ -139,9 +140,7 @@ def retrieve_timesteps(
return timesteps, num_inference_steps
class OvisImagePipeline(
nn.Module,
):
class OvisImagePipeline(nn.Module, CFGParallelMixin):
def __init__(
self,
*,
@@ -431,68 +430,77 @@ class OvisImagePipeline(
)
return timesteps, num_inference_steps
def denoising(
def diffuse(
self,
latents,
timesteps,
prompt_embeds,
negative_prompt_embeds,
guidance_scale,
do_classifier_free_guidance,
callback_on_step_end,
callback_on_step_end_tensor_inputs,
text_ids,
negative_text_ids,
latent_image_ids,
):
latents: torch.Tensor,
timesteps: torch.Tensor,
prompt_embeds: torch.Tensor,
negative_prompt_embeds: torch.Tensor,
text_ids: torch.Tensor,
negative_text_ids: torch.Tensor,
latent_image_ids: torch.Tensor,
do_true_cfg: bool,
guidance_scale: float,
cfg_normalize: bool = False,
) -> torch.Tensor:
"""
Diffusion loop with optional classifier-free guidance.
Args:
latents: Noise latents to denoise
timesteps: Diffusion timesteps
prompt_embeds: Positive prompt embeddings
negative_prompt_embeds: Negative prompt embeddings
text_ids: Position IDs for positive text
negative_text_ids: Position IDs for negative text
latent_image_ids: Position IDs for image latents
do_true_cfg: Whether to apply CFG
guidance_scale: CFG scale factor
cfg_normalize: Whether to normalize CFG output (default: False)
Returns:
Denoised latents
"""
self.scheduler.set_begin_index(0)
for i, t in enumerate(timesteps):
if self._interrupt:
if self.interrupt:
break
self._current_timestep = t
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
timestep = t.expand(latents.shape[0]).to(latents.dtype)
noise_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
encoder_hidden_states=prompt_embeds,
txt_ids=text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
positive_kwargs = {
"hidden_states": latents,
"timestep": timestep / 1000,
"encoder_hidden_states": prompt_embeds,
"txt_ids": text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latents,
"timestep": timestep / 1000,
"encoder_hidden_states": negative_prompt_embeds,
"txt_ids": negative_text_ids,
"img_ids": latent_image_ids,
"return_dict": False,
}
else:
negative_kwargs = None
if do_classifier_free_guidance:
neg_noise_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
encoder_hidden_states=negative_prompt_embeds,
txt_ids=negative_text_ids,
img_ids=latent_image_ids,
return_dict=False,
)[0]
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg,
guidance_scale,
positive_kwargs,
negative_kwargs,
cfg_normalize,
)
noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred)
# compute the previous noisy sample x_t -> x_t-1
latents_dtype = latents.dtype
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
if latents.dtype != latents_dtype:
if torch.backends.mps.is_available():
# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
latents = latents.to(latents_dtype)
# Not used in this pipeline
# if callback_on_step_end is not None:
# callback_kwargs = {}
# for k in callback_on_step_end_tensor_inputs:
# callback_kwargs[k] = locals()[k]
# callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
# latents = callback_outputs.pop("latents", latents)
# prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
return latents
@@ -704,23 +712,18 @@ class OvisImagePipeline(
if self.joint_attention_kwargs is None:
self._joint_attention_kwargs = {}
# 6. Denoising loop
# We set the index here to remove DtoH sync, helpful especially during compilation
# Check out more details here: https://github.com/huggingface/diffusers/pull/11696
latents = self.denoising(
latents,
timesteps,
prompt_embeds,
negative_prompt_embeds,
guidance_scale,
do_classifier_free_guidance,
callback_on_step_end,
callback_on_step_end_tensor_inputs,
text_ids,
negative_text_ids,
latent_image_ids,
# 6. Denoising loop using diffuse method
latents = self.diffuse(
latents=latents,
timesteps=timesteps,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds if do_classifier_free_guidance else None,
text_ids=text_ids,
negative_text_ids=negative_text_ids if do_classifier_free_guidance else None,
latent_image_ids=latent_image_ids,
do_true_cfg=do_classifier_free_guidance,
guidance_scale=guidance_scale,
cfg_normalize=False,
)
self._current_timestep = None
@@ -2,6 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen Image diffusion model components."""
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
QwenImageCFGParallelMixin,
)
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import (
QwenImagePipeline,
get_qwen_image_post_process_func,
@@ -11,6 +14,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
)
__all__ = [
"QwenImageCFGParallelMixin",
"QwenImagePipeline",
"QwenImageTransformer2DModel",
"get_qwen_image_post_process_func",
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CFG Parallel Mixin for Qwen Image series
Shared by
- QwenImagePipeline
- QwenImageEditPipeline
- QwenImageEditPlusPipeline
- QwenImageLayeredPipeline
"""
import logging
from typing import Any
import torch
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size
logger = logging.getLogger(__name__)
class QwenImageCFGParallelMixin(CFGParallelMixin):
"""
Base Mixin class for Qwen Image pipelines providing shared CFG methods.
"""
def diffuse(
self,
prompt_embeds: torch.Tensor,
prompt_embeds_mask: torch.Tensor,
negative_prompt_embeds: torch.Tensor,
negative_prompt_embeds_mask: torch.Tensor,
latents: torch.Tensor,
img_shapes: torch.Tensor,
txt_seq_lens: torch.Tensor,
negative_txt_seq_lens: torch.Tensor,
timesteps: torch.Tensor,
do_true_cfg: bool,
guidance: torch.Tensor,
true_cfg_scale: float,
image_latents: torch.Tensor | None = None,
cfg_normalize: bool = True,
additional_transformer_kwargs: dict[str, Any] | None = None,
) -> torch.Tensor:
"""
Diffusion loop with optional classifier-free guidance.
Args:
prompt_embeds: Positive prompt embeddings
prompt_embeds_mask: Mask for positive prompt
negative_prompt_embeds: Negative prompt embeddings
negative_prompt_embeds_mask: Mask for negative prompt
latents: Noise latents to denoise
img_shapes: Image shape information
txt_seq_lens: Text sequence lengths for positive prompts
negative_txt_seq_lens: Text sequence lengths for negative prompts
timesteps: Diffusion timesteps
do_true_cfg: Whether to apply CFG
guidance: Guidance scale tensor
true_cfg_scale: CFG scale factor
image_latents: Conditional image latents for editing (default: None)
cfg_normalize: Whether to normalize CFG output (default: True)
additional_transformer_kwargs: Extra kwargs to pass to transformer (default: None)
Returns:
Denoised latents
"""
self.scheduler.set_begin_index(0)
self.transformer.do_true_cfg = do_true_cfg
additional_transformer_kwargs = additional_transformer_kwargs or {}
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
# Broadcast timestep to match batch size
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
# Concatenate image latents with noise latents if available (for editing pipelines)
latent_model_input = latents
if image_latents is not None:
latent_model_input = torch.cat([latents, image_latents], dim=1)
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"guidance": guidance,
"encoder_hidden_states_mask": prompt_embeds_mask,
"encoder_hidden_states": prompt_embeds,
"img_shapes": img_shapes,
"txt_seq_lens": txt_seq_lens,
**additional_transformer_kwargs,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"guidance": guidance,
"encoder_hidden_states_mask": negative_prompt_embeds_mask,
"encoder_hidden_states": negative_prompt_embeds,
"img_shapes": img_shapes,
"txt_seq_lens": negative_txt_seq_lens,
**additional_transformer_kwargs,
}
else:
negative_kwargs = None
# For editing pipelines, we need to slice the output to remove condition latents
output_slice = latents.size(1) if image_latents is not None else None
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg,
true_cfg_scale,
positive_kwargs,
negative_kwargs,
cfg_normalize,
output_slice,
)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
return latents
def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool):
"""
Validate whether CFG parallel is properly configured for the current generation request.
When CFG parallel is enabled (cfg_parallel_world_size > 1), this method verifies that the necessary
conditions are met for correct parallel execution. If validation fails, a warning is
logged to help identify configuration issues.
Args:
true_cfg_scale: The classifier-free guidance scale value. Must be > 1 for CFG to
have an effect.
has_neg_prompt: Whether negative prompts or negative prompt embeddings are provided.
Required for CFG to perform unconditional prediction.
Returns:
True if CFG parallel is disabled or all validation checks pass, False otherwise.
Note:
When CFG parallel is disabled (world_size == 1), this method always returns True
as no parallel-specific validation is needed.
"""
if get_classifier_free_guidance_world_size() == 1:
return True
if true_cfg_scale <= 1:
logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.")
return False
if not has_neg_prompt:
logger.warning(
"CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings."
)
return False
return True
@@ -25,13 +25,11 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
)
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
QwenImageCFGParallelMixin,
)
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
@@ -239,9 +237,7 @@ def apply_rotary_emb_qwen(
return x_out.type_as(x)
class QwenImagePipeline(
nn.Module,
):
class QwenImagePipeline(nn.Module, QwenImageCFGParallelMixin):
def __init__(
self,
*,
@@ -536,109 +532,6 @@ class QwenImagePipeline(
def interrupt(self):
return self._interrupt
def diffuse(
self,
prompt_embeds,
prompt_embeds_mask,
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
timesteps,
do_true_cfg,
guidance,
true_cfg_scale,
):
self.scheduler.set_begin_index(0)
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
# Broadcast timestep to match batch size
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
# Enable CFG-parallel: rank0 computes positive, rank1 computes negative.
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
self.transformer.do_true_cfg = do_true_cfg
if cfg_parallel_ready:
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
if cfg_rank == 0:
local_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
else:
local_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
if cfg_rank == 0:
noise_pred = gathered[0]
neg_noise_pred = gathered[1]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
cfg_group.broadcast(latents, src=0)
else:
# Forward pass for positive prompt (or unconditional if no CFG)
noise_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
# Forward pass for negative prompt (CFG)
if do_true_cfg:
neg_noise_pred = self.transformer(
hidden_states=latents,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
return latents
def forward(
self,
req: OmniDiffusionRequest,
@@ -720,6 +613,8 @@ class QwenImagePipeline(
)
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
prompt=prompt,
prompt_embeds=prompt_embeds,
@@ -790,6 +685,12 @@ class QwenImagePipeline(
do_true_cfg,
guidance,
true_cfg_scale,
image_latents=None,
cfg_normalize=True,
additional_transformer_kwargs={
"return_dict": False,
"attention_kwargs": self.attention_kwargs,
},
)
self._current_timestep = None
@@ -25,14 +25,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
)
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
QwenImageCFGParallelMixin,
)
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
@@ -213,10 +211,7 @@ def retrieve_latents(
raise AttributeError("Could not access latents of provided encoder_output")
class QwenImageEditPipeline(
nn.Module,
SupportImageInput,
):
class QwenImageEditPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
def __init__(
self,
*,
@@ -598,118 +593,6 @@ class QwenImageEditPipeline(
def interrupt(self):
return self._interrupt
def diffuse(
self,
prompt_embeds,
prompt_embeds_mask,
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
timesteps,
do_true_cfg,
guidance,
true_cfg_scale,
):
"""Diffusion loop with optional image conditioning."""
self.scheduler.set_begin_index(0)
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
# broadcast to batch dimension and place on same device/dtype as latents
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
# Concatenate image latents with noise latents if available
latent_model_input = latents
if image_latents is not None:
latent_model_input = torch.cat([latents, image_latents], dim=1)
self.transformer.do_true_cfg = do_true_cfg # used in teacache hook
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
if cfg_parallel_ready:
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
if cfg_rank == 0:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
else:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
local_pred = local_pred[:, : latents.size(1)]
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
if cfg_rank == 0:
noise_pred = gathered[0]
neg_noise_pred = gathered[1]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
cfg_group.broadcast(latents, src=0)
else:
# Forward pass for positive prompt (or unconditional if no CFG)
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_pred[:, : latents.size(1)]
# Forward pass for negative prompt (CFG)
if do_true_cfg:
neg_noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
return latents
def forward(
self,
req: OmniDiffusionRequest,
@@ -823,6 +706,8 @@ class QwenImageEditPipeline(
)
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
prompt=prompt,
image=prompt_image, # Use resized image for prompt encoding
@@ -886,7 +771,6 @@ class QwenImageEditPipeline(
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
@@ -894,6 +778,12 @@ class QwenImageEditPipeline(
do_true_cfg,
guidance,
true_cfg_scale,
image_latents=image_latents,
cfg_normalize=True,
additional_transformer_kwargs={
"return_dict": False,
"attention_kwargs": self.attention_kwargs,
},
)
self._current_timestep = None
@@ -23,14 +23,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
)
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
QwenImageCFGParallelMixin,
)
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import (
calculate_dimensions,
@@ -168,7 +166,7 @@ def get_qwen_image_edit_plus_post_process_func(
return post_process_func
class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
class QwenImageEditPlusPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
def __init__(
self,
*,
@@ -530,116 +528,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
def interrupt(self):
return self._interrupt
def diffuse(
self,
prompt_embeds,
prompt_embeds_mask,
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
timesteps,
do_true_cfg,
guidance,
true_cfg_scale,
):
"""Diffusion loop with optional image conditioning."""
self.scheduler.set_begin_index(0)
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
# broadcast to batch dimension and place on same device/dtype as latents
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
# Concatenate image latents with noise latents if available
latent_model_input = latents
if image_latents is not None:
latent_model_input = torch.cat([latents, image_latents], dim=1)
self.transformer.do_true_cfg = do_true_cfg
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
if cfg_parallel_ready:
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
if cfg_rank == 0:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
else:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
local_pred = local_pred[:, : latents.size(1)]
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
if cfg_rank == 0:
noise_pred = gathered[0]
neg_noise_pred = gathered[1]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
cfg_group.broadcast(latents, src=0)
else:
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_pred[:, : latents.size(1)]
if do_true_cfg:
neg_noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
return_dict=False,
)[0]
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
return latents
def forward(
self,
req: OmniDiffusionRequest,
@@ -773,6 +661,8 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
)
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
prompt=prompt,
image=condition_images, # Use condition images for prompt encoding
@@ -840,7 +730,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
@@ -848,6 +737,12 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
do_true_cfg,
guidance,
true_cfg_scale,
image_latents=image_latents,
cfg_normalize=True,
additional_transformer_kwargs={
"return_dict": False,
"attention_kwargs": self.attention_kwargs,
},
)
self._current_timestep = None
@@ -22,17 +22,15 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
)
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.qwen_image.autoencoder_kl_qwenimage import (
AutoencoderKLQwenImage,
)
from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
QwenImageCFGParallelMixin,
)
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
@@ -191,7 +189,7 @@ def retrieve_latents(
raise AttributeError("Could not access latents of provided encoder_output")
class QwenImageLayeredPipeline(nn.Module, SupportImageInput):
class QwenImageLayeredPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin):
color_format = "RGBA"
def __init__(
@@ -553,138 +551,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
return latents
def diffuse(
self,
prompt_embeds,
prompt_embeds_mask,
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
timesteps,
do_true_cfg,
guidance,
true_cfg_scale,
cfg_normalize,
is_rgb,
):
self.scheduler.set_begin_index(0)
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
latent_model_input = latents
if image_latents is not None:
latent_model_input = torch.cat([latents, image_latents], dim=1)
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
timestep = t.expand(latents.shape[0]).to(latents.dtype)
# Enable CFG-parallel: rank0 computes positive, rank1 computes negative.
cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1
self.transformer.do_true_cfg = do_true_cfg # used in teacache hook
if cfg_parallel_ready:
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
if cfg_rank == 0:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
additional_t_cond=is_rgb,
return_dict=False,
)[0]
local_pred = local_pred[:, : latents.size(1)]
else:
local_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
additional_t_cond=is_rgb,
return_dict=False,
)[0]
local_pred = local_pred[:, : latents.size(1)]
gathered = cfg_group.all_gather(local_pred, separate_tensors=True)
if cfg_rank == 0:
noise_pred = gathered[0]
neg_noise_pred = gathered[1]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
if cfg_normalize:
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
else:
noise_pred = comb_pred
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
cfg_group.broadcast(latents, src=0)
else:
# Forward pass for positive prompt (or unconditional if no CFG)
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=prompt_embeds_mask,
encoder_hidden_states=prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=txt_seq_lens,
attention_kwargs=self.attention_kwargs,
additional_t_cond=is_rgb,
return_dict=False,
)[0]
noise_pred = noise_pred[:, : latents.size(1)]
if do_true_cfg:
neg_noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep / 1000,
guidance=guidance,
encoder_hidden_states_mask=negative_prompt_embeds_mask,
encoder_hidden_states=negative_prompt_embeds,
img_shapes=img_shapes,
txt_seq_lens=negative_txt_seq_lens,
attention_kwargs=self.attention_kwargs,
additional_t_cond=is_rgb,
return_dict=False,
)[0]
neg_noise_pred = neg_noise_pred[:, : latents.size(1)]
comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
if cfg_normalize:
cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
noise_pred = comb_pred * (cond_norm / noise_norm)
else:
noise_pred = comb_pred
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
# no need callback now
return latents
@property
def guidance_scale(self):
return self._guidance_scale
@@ -840,6 +706,8 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
)
do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt)
prompt_embeds, prompt_embeds_mask = self.encode_prompt(
prompt=prompt,
prompt_embeds=prompt_embeds,
@@ -924,7 +792,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
negative_prompt_embeds,
negative_prompt_embeds_mask,
latents,
image_latents,
img_shapes,
txt_seq_lens,
negative_txt_seq_lens,
@@ -932,8 +799,13 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as
do_true_cfg,
guidance,
true_cfg_scale,
cfg_normalize,
is_rgb,
image_latents=image_latents,
cfg_normalize=cfg_normalize,
additional_transformer_kwargs={
"return_dict": False,
"additional_t_cond": is_rgb,
"attention_kwargs": self.attention_kwargs,
},
)
self._current_timestep = None
+57 -28
View File
@@ -16,6 +16,7 @@ from transformers import CLIPTextModelWithProjection, CLIPTokenizer, T5EncoderMo
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.sd3.sd3_transformer import (
@@ -126,9 +127,7 @@ def retrieve_timesteps(
return timesteps, num_inference_steps
class StableDiffusion3Pipeline(
nn.Module,
):
class StableDiffusion3Pipeline(nn.Module, CFGParallelMixin):
def __init__(
self,
*,
@@ -498,15 +497,35 @@ class StableDiffusion3Pipeline(
def diffuse(
self,
prompt_embeds,
pooled_prompt_embeds,
negative_prompt_embeds,
negative_pooled_prompt_embeds,
latents,
timesteps,
do_cfg,
):
latents: torch.Tensor,
timesteps: torch.Tensor,
prompt_embeds: torch.Tensor,
pooled_prompt_embeds: torch.Tensor,
negative_prompt_embeds: torch.Tensor,
negative_pooled_prompt_embeds: torch.Tensor,
do_true_cfg: bool,
guidance_scale: float,
cfg_normalize: bool = False,
) -> torch.Tensor:
"""
Diffusion loop with optional classifier-free guidance.
Args:
latents: Noise latents to denoise
timesteps: Diffusion timesteps
prompt_embeds: Positive prompt embeddings
pooled_prompt_embeds: Pooled positive prompt embeddings
negative_prompt_embeds: Negative prompt embeddings
negative_pooled_prompt_embeds: Pooled negative prompt embeddings
do_true_cfg: Whether to apply CFG
guidance_scale: CFG scale factor
cfg_normalize: Whether to normalize CFG output (default: False)
Returns:
Denoised latents
"""
self.scheduler.set_begin_index(0)
for _, t in enumerate(timesteps):
if self.interrupt:
continue
@@ -515,29 +534,36 @@ class StableDiffusion3Pipeline(
# Broadcast timestep to match batch size
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
transformer_kwargs = {
positive_kwargs = {
"hidden_states": latents,
"timestep": timestep,
"encoder_hidden_states": prompt_embeds,
"pooled_projections": pooled_prompt_embeds,
"return_dict": False,
}
noise_pred = self.transformer(**transformer_kwargs)[0]
if do_cfg:
neg_transformer_kwargs = {
if do_true_cfg:
negative_kwargs = {
"hidden_states": latents,
"timestep": timestep,
"encoder_hidden_states": negative_prompt_embeds,
"pooled_projections": negative_pooled_prompt_embeds,
"return_dict": False,
}
else:
negative_kwargs = None
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg,
guidance_scale,
positive_kwargs,
negative_kwargs,
cfg_normalize,
)
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
neg_noise_pred = self.transformer(**neg_transformer_kwargs)[0]
noise_pred = neg_noise_pred + self.guidance_scale * (noise_pred - neg_noise_pred)
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
return latents
def forward(
@@ -644,14 +670,17 @@ class StableDiffusion3Pipeline(
timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1])
self._num_timesteps = len(timesteps)
# Denoising loop using diffuse method
latents = self.diffuse(
prompt_embeds,
pooled_prompt_embeds,
negative_prompt_embeds,
negative_pooled_prompt_embeds,
latents,
timesteps,
do_cfg,
latents=latents,
timesteps=timesteps,
prompt_embeds=prompt_embeds,
pooled_prompt_embeds=pooled_prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds if do_cfg else None,
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds if do_cfg else None,
do_true_cfg=do_cfg,
guidance_scale=self.guidance_scale,
cfg_normalize=False,
)
self._current_timestep = None
@@ -7,7 +7,7 @@ import json
import logging
import os
from collections.abc import Iterable
from typing import cast
from typing import Any, cast
import PIL.Image
import torch
@@ -18,12 +18,14 @@ from transformers import AutoTokenizer, UMT5EncoderModel
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
@@ -184,7 +186,7 @@ def get_wan22_pre_process_func(
return pre_process_func
class Wan22Pipeline(nn.Module):
class Wan22Pipeline(nn.Module, CFGParallelMixin):
def __init__(
self,
*,
@@ -585,26 +587,44 @@ class Wan22Pipeline(nn.Module):
latent_model_input = latents.to(dtype)
timestep = t.expand(latents.shape[0])
noise_pred = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
# Prepare kwargs for positive and negative predictions
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": prompt_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": current_model,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": negative_prompt_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": current_model,
}
else:
negative_kwargs = None
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
noise_uncond = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=negative_prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=do_true_cfg,
true_cfg_scale=current_guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=False,
)
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
# Wan2.2 is prone to out of memory errors when predicting large videos
# so we empty the cache here to avoid OOM before vae decoding.
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
self._current_timestep = None
# For I2V mode: blend final latents with condition
@@ -629,6 +649,21 @@ class Wan22Pipeline(nn.Module):
return DiffusionOutput(output=output)
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
"""
Forward pass through transformer to predict noise.
Args:
current_model: The transformer model to use (transformer or transformer_2)
**kwargs: Arguments to pass to the transformer
Returns:
Predicted noise tensor
"""
if current_model is None:
current_model = self.transformer
return current_model(**kwargs)[0]
def encode_prompt(
self,
prompt: str | list[str],
@@ -6,7 +6,7 @@ from __future__ import annotations
import logging
import os
from collections.abc import Iterable
from typing import cast
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -18,6 +18,7 @@ from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
@@ -29,6 +30,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
@@ -136,7 +138,7 @@ def get_wan22_i2v_pre_process_func(
return pre_process_func
class Wan22I2VPipeline(nn.Module, SupportImageInput):
class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin):
"""
Wan2.2 Image-to-Video Pipeline.
@@ -484,31 +486,46 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
timestep = t.expand(latents.shape[0])
# Forward pass
noise_pred = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
encoder_hidden_states_image=image_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
# Prepare kwargs for positive and negative predictions
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": prompt_embeds,
"encoder_hidden_states_image": image_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": current_model,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": negative_prompt_embeds,
"encoder_hidden_states_image": image_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": current_model,
}
else:
negative_kwargs = None
# Classifier-free guidance
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
noise_uncond = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=negative_prompt_embeds,
encoder_hidden_states_image=image_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=do_true_cfg,
true_cfg_scale=current_guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=False,
)
# Scheduler step
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
# Wan2.2 is prone to out of memory errors when predicting large videos
# so we empty the cache here to avoid OOM before vae decoding.
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
self._current_timestep = None
# For expand_timesteps mode, blend final latents with condition
@@ -533,6 +550,21 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput):
return DiffusionOutput(output=output)
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
"""
Forward pass through transformer to predict noise.
Args:
current_model: The transformer model to use (transformer or transformer_2)
**kwargs: Arguments to pass to the transformer
Returns:
Predicted noise tensor
"""
if current_model is None:
current_model = self.transformer
return current_model(**kwargs)[0]
def encode_prompt(
self,
prompt: str | list[str],
@@ -19,7 +19,7 @@ from __future__ import annotations
import logging
import os
from collections.abc import Iterable
from typing import cast
from typing import Any, cast
import numpy as np
import PIL.Image
@@ -31,6 +31,7 @@ from transformers import AutoTokenizer, UMT5EncoderModel
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
@@ -42,6 +43,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
@@ -126,7 +128,7 @@ def get_wan22_ti2v_pre_process_func(
return pre_process_func
class Wan22TI2VPipeline(nn.Module, SupportImageInput):
class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin):
"""
Wan2.2 Text-Image-to-Video (TI2V) Pipeline.
@@ -399,29 +401,44 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
# Forward pass
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
# Prepare kwargs for positive and negative predictions
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": prompt_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": self.transformer,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep,
"encoder_hidden_states": negative_prompt_embeds,
"attention_kwargs": attention_kwargs,
"return_dict": False,
"current_model": self.transformer,
}
else:
negative_kwargs = None
# Classifier-free guidance
if guidance_scale > 1.0 and negative_prompt_embeds is not None:
noise_uncond = self.transformer(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=negative_prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
# Predict noise with automatic CFG parallel handling
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg=do_true_cfg,
true_cfg_scale=guidance_scale,
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
cfg_normalize=False,
)
# Scheduler step
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
# Wan2.2 is prone to out of memory errors when predicting large videos
# so we empty the cache here to avoid OOM before vae decoding.
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
self._current_timestep = None
# For I2V mode, blend final latents with condition
@@ -446,6 +463,21 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput):
return DiffusionOutput(output=output)
def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
"""
Forward pass through transformer to predict noise.
Args:
current_model: The transformer model to use
**kwargs: Arguments to pass to the transformer
Returns:
Predicted noise tensor
"""
if current_model is None:
current_model = self.transformer
return current_model(**kwargs)[0]
def encode_prompt(
self,
prompt: str | list[str],