[Feature][Bagel] Add CFG parallel mode (#1578)

Signed-off-by: Ding Zuhao <e1583181@u.nus.edu>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
Ding Zuhaoand汪志鹏 authored and GitHub committed 2026-03-06 09:55:54 +08:00
1 parent 8536dce759
commit 128e6f4b97
6 files changed
+646 -36

No files matched your search

@@ -32,6 +32,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.transformers_utils.configs.bagel import BagelConfig
from vllm_omni.diffusion.attention.backends.utils.fa import flash_attn_varlen_func
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
)
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
@@ -1330,7 +1335,6 @@ class Bagel(nn.Module):
cfg_img_past_key_values: NaiveCache | None = None,
cfg_img_key_values_lens: torch.IntTensor | None = None,
cfg_img_packed_key_value_indexes: torch.LongTensor | None = None,
cfg_type: str = "parallel",
):
x_t = packed_init_noises
@@ -1339,9 +1343,45 @@ class Bagel(nn.Module):
dts = timesteps[:-1] - timesteps[1:]
timesteps = timesteps[:-1]
# ── Pre-compute batched CFG state (merged caches + indices) ──
use_cfg_text = cfg_text_scale > 1.0
use_cfg_img = cfg_img_scale > 1.0
# ── Detect CFG parallel mode ──
cfg_parallel_ready = use_cfg_text and get_classifier_free_guidance_world_size() > 1
if cfg_parallel_ready:
return self._generate_image_parallel(
x_t=x_t,
timesteps=timesteps,
dts=dts,
packed_text_ids=packed_text_ids,
packed_text_indexes=packed_text_indexes,
packed_vae_position_ids=packed_vae_position_ids,
packed_vae_token_indexes=packed_vae_token_indexes,
packed_seqlens=packed_seqlens,
packed_position_ids=packed_position_ids,
packed_indexes=packed_indexes,
past_key_values=past_key_values,
key_values_lens=key_values_lens,
packed_key_value_indexes=packed_key_value_indexes,
cfg_renorm_min=cfg_renorm_min,
cfg_renorm_type=cfg_renorm_type,
cfg_interval=cfg_interval,
cfg_text_scale=cfg_text_scale,
cfg_text_packed_query_indexes=cfg_text_packed_query_indexes,
cfg_text_packed_position_ids=cfg_text_packed_position_ids,
cfg_text_past_key_values=cfg_text_past_key_values,
cfg_text_key_values_lens=cfg_text_key_values_lens,
cfg_text_packed_key_value_indexes=cfg_text_packed_key_value_indexes,
cfg_img_scale=cfg_img_scale,
cfg_img_packed_query_indexes=cfg_img_packed_query_indexes,
cfg_img_packed_position_ids=cfg_img_packed_position_ids,
cfg_img_past_key_values=cfg_img_past_key_values,
cfg_img_key_values_lens=cfg_img_key_values_lens,
cfg_img_packed_key_value_indexes=cfg_img_packed_key_value_indexes,
)
# ── Batched CFG mode (cfg_parallel_size=1) ──
cfg_batched = None
if use_cfg_text:
@@ -1430,6 +1470,266 @@ class Bagel(nn.Module):
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent
def _generate_image_parallel(
self,
x_t: torch.Tensor,
timesteps: torch.Tensor,
dts: torch.Tensor,
packed_text_ids: torch.LongTensor,
packed_text_indexes: torch.LongTensor,
packed_vae_position_ids: torch.LongTensor,
packed_vae_token_indexes: torch.LongTensor,
packed_seqlens: torch.IntTensor,
packed_position_ids: torch.LongTensor,
packed_indexes: torch.LongTensor,
past_key_values: NaiveCache,
key_values_lens: torch.IntTensor,
packed_key_value_indexes: torch.LongTensor,
cfg_renorm_min: float,
cfg_renorm_type: str,
cfg_interval: tuple[float, float],
cfg_text_scale: float,
cfg_text_packed_query_indexes: torch.LongTensor | None,
cfg_text_packed_position_ids: torch.LongTensor | None,
cfg_text_past_key_values: NaiveCache | None,
cfg_text_key_values_lens: torch.IntTensor | None,
cfg_text_packed_key_value_indexes: torch.LongTensor | None,
cfg_img_scale: float,
cfg_img_packed_query_indexes: torch.LongTensor | None,
cfg_img_packed_position_ids: torch.LongTensor | None,
cfg_img_past_key_values: NaiveCache | None,
cfg_img_key_values_lens: torch.IntTensor | None,
cfg_img_packed_key_value_indexes: torch.LongTensor | None,
):
"""CFG parallel denoising loop: each rank computes one CFG branch.
Rank 0: gen branch (full conditioning)
Rank 1: text_cfg branch (unconditional text)
Rank 2: img_cfg branch (no image condition), only when cfg_img_scale > 1.0
"""
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
cfg_world_size = get_classifier_free_guidance_world_size()
use_cfg_img = cfg_img_scale > 1.0
# Validate cfg_parallel_size vs cfg_img_scale consistency
if cfg_world_size == 3 and not use_cfg_img:
raise ValueError(
f"cfg_parallel_size=3 requires cfg_img_scale > 1.0, "
f"but got cfg_img_scale={cfg_img_scale}. "
f"Use cfg_parallel_size=2 for text-only CFG parallel(text2img), or set cfg_img_scale > 1.0."
)
if cfg_world_size == 2 and use_cfg_img:
raise ValueError(
f"Image CFG (cfg_img_scale={cfg_img_scale}) requires cfg_parallel_size=3, "
f"but got cfg_parallel_size=2. "
f"Use cfg_parallel_size=3 to enable image CFG in parallel mode."
)
# Select this rank's branch inputs
if cfg_rank == 0:
# Gen branch: use main inputs directly
branch_position_ids = packed_position_ids
branch_indexes = packed_indexes
branch_past_key_values = past_key_values
branch_key_values_lens = key_values_lens
branch_key_value_indexes = packed_key_value_indexes
elif cfg_rank == 1:
# Text CFG branch
branch_position_ids = cfg_text_packed_position_ids
branch_indexes = cfg_text_packed_query_indexes
branch_past_key_values = cfg_text_past_key_values
branch_key_values_lens = cfg_text_key_values_lens
branch_key_value_indexes = cfg_text_packed_key_value_indexes
elif cfg_rank == 2:
# Image CFG branch
branch_position_ids = cfg_img_packed_position_ids
branch_indexes = cfg_img_packed_query_indexes
branch_past_key_values = cfg_img_past_key_values
branch_key_values_lens = cfg_img_key_values_lens
branch_key_value_indexes = cfg_img_packed_key_value_indexes
else:
raise RuntimeError(f"Unexpected cfg_rank={cfg_rank} for Bagel 3-branch CFG parallel")
for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
if t > cfg_interval[0] and t <= cfg_interval[1]:
cfg_text_scale_ = cfg_text_scale
cfg_img_scale_ = cfg_img_scale
else:
cfg_text_scale_ = 1.0
cfg_img_scale_ = 1.0
use_cfg_this_step = cfg_text_scale_ > 1.0
if use_cfg_this_step:
# Each rank computes its branch's velocity
local_v_t = self._forward_flow_single_branch(
x_t=x_t,
timestep=timestep,
packed_vae_token_indexes=packed_vae_token_indexes,
packed_vae_position_ids=packed_vae_position_ids,
packed_text_ids=packed_text_ids,
packed_text_indexes=packed_text_indexes,
packed_indexes=branch_indexes,
packed_position_ids=branch_position_ids,
packed_seqlens=packed_seqlens,
key_values_lens=branch_key_values_lens,
past_key_values=branch_past_key_values,
packed_key_value_indexes=branch_key_value_indexes,
)
# All-gather velocities from all CFG ranks
gathered = cfg_group.all_gather(local_v_t, separate_tensors=True)
# Rank 0 combines with CFG formula
if cfg_rank == 0:
v_t = gathered[0] # gen branch
cfg_text_v_t = gathered[1] # text_cfg branch
cfg_img_v_t = gathered[2] if (use_cfg_img and len(gathered) > 2) else None
v_t = self._combine_cfg(
v_t,
cfg_text_v_t,
cfg_img_v_t,
cfg_text_scale_,
cfg_img_scale_,
cfg_renorm_type,
cfg_renorm_min,
)
x_t = x_t - v_t.to(x_t.device) * dts[i]
else:
# Outside cfg_interval: only rank 0 computes (no CFG needed)
if cfg_rank == 0:
v_t = self._forward_flow_single_branch(
x_t=x_t,
timestep=timestep,
packed_vae_token_indexes=packed_vae_token_indexes,
packed_vae_position_ids=packed_vae_position_ids,
packed_text_ids=packed_text_ids,
packed_text_indexes=packed_text_indexes,
packed_indexes=packed_indexes,
packed_position_ids=packed_position_ids,
packed_seqlens=packed_seqlens,
key_values_lens=key_values_lens,
past_key_values=past_key_values,
packed_key_value_indexes=packed_key_value_indexes,
)
x_t = x_t - v_t.to(x_t.device) * dts[i]
# Broadcast updated x_t from rank 0 to all ranks
x_t = x_t.contiguous()
cfg_group.broadcast(x_t, src=0)
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent
@staticmethod
def _combine_cfg(
v_t: torch.Tensor,
cfg_text_v_t: torch.Tensor,
cfg_img_v_t: torch.Tensor | None,
cfg_text_scale: float,
cfg_img_scale: float,
cfg_renorm_type: str,
cfg_renorm_min: float,
) -> torch.Tensor:
"""Combine 3-branch CFG predictions with renormalization.
Args:
v_t: velocity from gen branch (full conditioning)
cfg_text_v_t: velocity from text_cfg branch (unconditional text)
cfg_img_v_t: velocity from img_cfg branch (no image), or None
cfg_text_scale: text guidance scale
cfg_img_scale: image guidance scale
cfg_renorm_type: "text_channel", "global", or "channel"
cfg_renorm_min: minimum renormalization scale
"""
if cfg_renorm_type == "text_channel":
v_t_text_ = cfg_text_v_t + cfg_text_scale * (v_t - cfg_text_v_t)
norm_v_t = torch.norm(v_t, dim=-1, keepdim=True)
norm_v_t_text_ = torch.norm(v_t_text_, dim=-1, keepdim=True)
scale = (norm_v_t / (norm_v_t_text_ + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
v_t_text = v_t_text_ * scale
if cfg_img_scale > 1.0 and cfg_img_v_t is not None:
v_t = cfg_img_v_t + cfg_img_scale * (v_t_text - cfg_img_v_t)
else:
v_t = v_t_text
else:
v_t_text_ = cfg_text_v_t + cfg_text_scale * (v_t - cfg_text_v_t)
if cfg_img_scale > 1.0 and cfg_img_v_t is not None:
v_t_ = cfg_img_v_t + cfg_img_scale * (v_t_text_ - cfg_img_v_t)
else:
v_t_ = v_t_text_
# NOTE norm is computed over all dimensions, thus currently only supports batch_size = 1 with navit
if cfg_renorm_type == "global":
norm_v_t = torch.norm(v_t)
norm_v_t_ = torch.norm(v_t_)
elif cfg_renorm_type == "channel":
norm_v_t = torch.norm(v_t, dim=-1, keepdim=True)
norm_v_t_ = torch.norm(v_t_, dim=-1, keepdim=True)
else:
raise NotImplementedError(f"{cfg_renorm_type} is not supported")
scale = (norm_v_t / (norm_v_t_ + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
v_t = v_t_ * scale
return v_t
def _forward_flow_single_branch(
self,
x_t: torch.Tensor,
timestep: torch.LongTensor,
packed_vae_token_indexes: torch.LongTensor,
packed_vae_position_ids: torch.LongTensor,
packed_text_ids: torch.LongTensor,
packed_text_indexes: torch.LongTensor,
packed_indexes: torch.LongTensor,
packed_position_ids: torch.LongTensor,
packed_seqlens: torch.IntTensor,
key_values_lens: torch.IntTensor,
past_key_values: NaiveCache,
packed_key_value_indexes: torch.LongTensor,
) -> torch.Tensor:
"""Run a single-branch forward pass (no CFG batching).
Used by CFG parallel mode where each rank computes one branch.
Returns the velocity v_t for the given branch.
"""
packed_text_embedding = self.language_model.model.embed_tokens(packed_text_ids)
packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size))
packed_sequence[packed_text_indexes] = packed_text_embedding
assert timestep.unique().shape[0] == 1
packed_pos_embed = self.latent_pos_embed(packed_vae_position_ids)
packed_timestep_embeds = self.time_embedder(timestep)
x_t_emb = self.vae2llm(x_t) + packed_timestep_embeds + packed_pos_embed
if x_t_emb.dtype != packed_sequence.dtype:
x_t_emb = x_t_emb.to(packed_sequence.dtype)
packed_sequence[packed_vae_token_indexes] = x_t_emb
extra_inputs = {}
if self.use_moe:
extra_inputs["mode"] = "gen"
extra_inputs["packed_vae_token_indexes"] = packed_vae_token_indexes
extra_inputs["packed_text_indexes"] = packed_text_indexes
output = self.language_model.forward(
packed_query_sequence=packed_sequence,
query_lens=packed_seqlens,
packed_query_position_ids=packed_position_ids,
packed_query_indexes=packed_indexes,
past_key_values=past_key_values,
key_values_lens=key_values_lens,
packed_key_value_indexes=packed_key_value_indexes,
update_past_key_values=False,
is_causal=False,
**extra_inputs,
)
v_t = self.llm2vae(output.packed_query_sequence)
v_t = v_t[packed_vae_token_indexes]
return v_t
def _forward_flow(
self,
x_t: torch.Tensor,
@@ -1535,34 +1835,14 @@ class Bagel(nn.Module):
# ── CFG combination ──
if use_cfg:
if cfg_renorm_type == "text_channel":
v_t_text_ = cfg_text_v_t + cfg_text_scale * (v_t - cfg_text_v_t)
norm_v_t = torch.norm(v_t, dim=-1, keepdim=True)
norm_v_t_text_ = torch.norm(v_t_text_, dim=-1, keepdim=True)
scale = (norm_v_t / (norm_v_t_text_ + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
v_t_text = v_t_text_ * scale
if cfg_img_scale > 1.0:
v_t = cfg_img_v_t + cfg_img_scale * (v_t_text - cfg_img_v_t)
else:
v_t = v_t_text
else:
v_t_text_ = cfg_text_v_t + cfg_text_scale * (v_t - cfg_text_v_t)
if cfg_img_scale > 1.0:
v_t_ = cfg_img_v_t + cfg_img_scale * (v_t_text_ - cfg_img_v_t)
else:
v_t_ = v_t_text_
# NOTE norm is computed over all dimensions, thus currently only supports batch_size = 1 with navit
if cfg_renorm_type == "global":
norm_v_t = torch.norm(v_t)
norm_v_t_ = torch.norm(v_t_)
elif cfg_renorm_type == "channel":
norm_v_t = torch.norm(v_t, dim=-1, keepdim=True)
norm_v_t_ = torch.norm(v_t_, dim=-1, keepdim=True)
else:
raise NotImplementedError(f"{cfg_renorm_type} is not supported")
scale = (norm_v_t / (norm_v_t_ + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
v_t = v_t_ * scale
v_t = self._combine_cfg(
v_t,
cfg_text_v_t,
cfg_img_v_t,
cfg_text_scale,
cfg_img_scale,
cfg_renorm_type,
cfg_renorm_min,
)
return v_t