[Model] Add Ming-flash-omni-2.0 Image Generation (Diffusion) Stage (#2875)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-05-27 23:21:20 +08:00
1 parent 36dc32ff81
commit a698269e47
25 files changed
+2614 -42

No files matched your search

@@ -0,0 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
from .byte5_encoder import MingByT5Encoder
from .condition_encoder import MingConditionEncoder
from .ming_zimage_transformer import MingZImageTransformer2DModel
from .pipeline_ming_imagegen import (
MingImagePipeline,
get_ming_image_post_process_func,
)
from .t5_block_mapper import T5EncoderBlockByT5Mapper
__all__ = [
"MingByT5Encoder",
"MingConditionEncoder",
"MingImagePipeline",
"MingZImageTransformer2DModel",
"T5EncoderBlockByT5Mapper",
"get_ming_image_post_process_func",
]
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
"""ByT5 glyph/text encoder for Ming-flash-omni-2.0 image generation.
Ported from Ming's ``load_byte5_and_byte5_tokenizer`` +
``T5EncoderBlockByT5Mapper`` + ``get_condition_embeds_for_image_gen`` byte5
branch. The released checkpoint's byte5 weights were trained with font/color
special tokens, so we replicate that vocabulary extension before loading —
otherwise ``byte5_model.pt`` would shape-mismatch at the embedding layer.
Typical forward: takes a list of user-supplied prompt strings (possibly with
``<cn-font-N>`` / ``<color-N>`` markers), returns ``[B, byte5_max_length,
diffusion_c_input_dim]`` features ready to be concatenated onto cap_feats
along the sequence dimension.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from types import SimpleNamespace
import torch
import torch.nn as nn
from transformers import AutoTokenizer, T5ForConditionalGeneration
from vllm_omni.diffusion.models.ming_flash_omni.t5_block_mapper import (
T5EncoderBlockByT5Mapper,
)
logger = logging.getLogger(__name__)
def _add_multilingual_special_tokens(
tokenizer,
text_encoder: nn.Module,
font_ann_path: Path,
color_ann_path: Path,
add_font: bool,
add_color: bool,
add_align: bool = False,
) -> None:
"""Extend the byte5 vocab with per-language font + color markers.
Mirrors ``add_special_token_multilingual`` in Ming's bizgen utils. The token
set must match the one the checkpoint was trained with, otherwise the
resized embedding table shape won't line up with the shipped weights.
"""
idx_font_dict = json.loads(font_ann_path.read_text())
idx_color_dict = json.loads(color_ann_path.read_text())
font_tokens: list[str] = []
for font_code in idx_font_dict:
prefix = font_code[:3]
if prefix in ("cn-", "en-", "jp-", "kr-"):
font_tokens.append(f"<{prefix}font-{idx_font_dict[font_code]}>")
else:
font_tokens.append(f"<font-{idx_font_dict[font_code]}>")
color_tokens = [f"<color-{i}>" for i in range(len(idx_color_dict))]
align_tokens = [f"<align-{i}>" for i in range(3)]
extra: list[str] = []
if add_color:
extra += color_tokens
if add_font:
extra += font_tokens
if add_align:
extra += align_tokens
tokenizer.add_tokens(extra, special_tokens=True)
text_encoder.resize_token_embeddings(len(tokenizer))
class MingByT5Encoder(nn.Module):
"""Bundles byte5 tokenizer + T5 encoder + T5EncoderBlockByT5Mapper.
Build with ``MingByT5Encoder.from_checkpoint(<model>/byte5)`` when the
checkpoint ships byte5 weights; otherwise callers can skip this and the
pipeline falls back to no-byte5 conditioning.
"""
def __init__(
self,
tokenizer,
text_encoder: nn.Module,
mapper: T5EncoderBlockByT5Mapper,
max_length: int,
) -> None:
super().__init__()
self.tokenizer = tokenizer
self.text_encoder = text_encoder
self.mapper = mapper
self.max_length = max_length
@classmethod
def from_checkpoint(
cls,
byte5_dir: Path,
*,
device: torch.device,
dtype: torch.dtype,
) -> MingByT5Encoder:
# Wrap the whole load in fork_rng so any nn.init.normal_ inside
# T5ForConditionalGeneration.from_pretrained / vocab-resize cannot
# advance the default CPU/CUDA generator state. Without this, the
# diffusion pipeline's seeded noise becomes order-dependent across
# requests (same-seed replays diverge).
cuda_devs = list(range(torch.accelerator.device_count())) if torch.cuda.is_available() else []
with torch.random.fork_rng(devices=cuda_devs, enabled=True):
return cls._from_checkpoint_impl(byte5_dir, device=device, dtype=dtype)
@classmethod
def _from_checkpoint_impl(
cls,
byte5_dir: Path,
*,
device: torch.device,
dtype: torch.dtype,
) -> MingByT5Encoder:
byte5_dir = Path(byte5_dir)
# Ming checkpoint uses ``byt5`` (no 'e') in filenames and JSON keys
# variable names below keep the ``byte5_`` spelling for readability.
cfg_path = byte5_dir / "byt5.json"
cfg_raw = json.loads(cfg_path.read_text())
cfg = SimpleNamespace(**cfg_raw)
byte5_config = cfg.byt5_config
mapper_config = cfg.byt5_mapper_config
max_length = int(cfg.byt5_max_length)
# ---- Tokenizer + T5 encoder (base).
ckpt_key = byte5_config.get("byt5_ckpt_path")
byte5_ckpt_path = byte5_dir / ckpt_key.lstrip("./")
tokenizer = AutoTokenizer.from_pretrained(byte5_ckpt_path, local_files_only=True)
text_encoder = T5ForConditionalGeneration.from_pretrained(byte5_ckpt_path, local_files_only=True).get_encoder()
# ---- Extend vocab with font/color markers so the shipped weights load.
if byte5_config.get("special_token"):
if not byte5_config.get("multilingual", True):
raise NotImplementedError(
"Non-multilingual byte5 vocab extension is not ported; "
"the released Ming checkpoint uses multilingual=True."
)
_add_multilingual_special_tokens(
tokenizer,
text_encoder,
font_ann_path=byte5_dir / byte5_config["font_ann_path"].lstrip("./"),
color_ann_path=byte5_dir / byte5_config["color_ann_path"].lstrip("./"),
add_font=bool(byte5_config.get("font_special_token")),
add_color=bool(byte5_config.get("color_special_token")),
)
# ---- Load byt5 text-encoder weights (base.pt wraps the backbone in a
# trainable-module container; byt5_model.pt has the top-level encoder
# state). Follow Ming's two-step load.
base_state = torch.load(byte5_dir / "byt5_model" / "base.pt", map_location="cpu", weights_only=False)
prefix = "module.text_tower.encoder."
base_filtered = {
name[len(prefix) :]: state for name, state in base_state["state_dict"].items() if name.startswith(prefix)
}
text_encoder.load_state_dict(base_filtered, strict=True)
del base_state, base_filtered
encoder_state = torch.load(byte5_dir / "byt5_model" / "byt5_model.pt", map_location="cpu", weights_only=False)
text_encoder.load_state_dict(encoder_state)
del encoder_state
text_encoder.to(device=device, dtype=dtype).eval()
# ---- Mapper.
mapper = T5EncoderBlockByT5Mapper(
byte5_config=text_encoder.config,
num_layers=int(mapper_config["num_layers"]),
sdxl_channels=int(mapper_config["sdxl_channels"]),
)
mapper_state = torch.load(byte5_dir / "byt5_mapper" / "byt5_mapper.pt", map_location="cpu", weights_only=False)
# mapper now uses vllm-omni TP-aware T5 layers (fused qkv_proj / wi);
mapper.load_weights(mapper_state.items())
del mapper_state
mapper.to(device=device, dtype=dtype).eval()
logger.info(
"[MingByT5Encoder] ready: d_model=%d mapper_layers=%d sdxl_channels=%d max_length=%d vocab=%d",
text_encoder.config.d_model,
mapper_config["num_layers"],
mapper_config["sdxl_channels"],
max_length,
len(tokenizer),
)
return cls(tokenizer, text_encoder, mapper, max_length)
@torch.inference_mode()
def forward(self, texts: list[str]) -> torch.Tensor:
"""Tokenize → T5 encode → mapper; masks out padded positions.
Returns ``[B, max_length, sdxl_channels]``. Padded positions are zeroed
so the downstream ``torch.cat`` with cap_feats doesn't inject garbage.
"""
device = next(self.text_encoder.parameters()).device
dtype = next(self.text_encoder.parameters()).dtype
tokens = self.tokenizer(
texts,
padding="max_length",
max_length=self.max_length,
truncation=True,
add_special_tokens=True,
return_tensors="pt",
)
input_ids = tokens.input_ids.to(device)
attention_mask = tokens.attention_mask.to(device)
encoder_out = self.text_encoder(
input_ids=input_ids,
attention_mask=attention_mask.float(),
)
hidden_states = encoder_out[0]
feats = self.mapper(hidden_states, attention_mask)
feats = feats * attention_mask.unsqueeze(-1).to(dtype=feats.dtype)
return feats.to(dtype=dtype)
__all__ = ["MingByT5Encoder"]
@@ -0,0 +1,373 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
#
# Adapted from Ming repository (inclusionAI/Ming) — the ``get_condition_embeds
# _for_image_gen`` path in modeling_bailingmm2.py.
"""Ming-flash-omni-2.0 condition encoder for image generation.
Pipeline (runs inside the imagegen stage):
thinker hidden states at query-token positions [B, N, 4096]
│
▼ proj_in (Linear, bias=True)
[B, N, 1536]
│
▼ Qwen2ForCausalLM connector (is_causal=False)
loaded from <checkpoint>/connector/
[B, N, 1536]
│
▼ proj_out (Linear, bias=True)
[B, N, 2560]
│
▼ F.normalize(dim=-1) × 1000 (text_encoder_norm)
[B, N, 2560]
│
▼
cap_feats consumed by ZImageTransformer2DModel
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
import torch
import torch.nn as nn
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingImageGenConfig
logger = logging.getLogger(__name__)
class MingConditionEncoder(nn.Module):
"""Wraps a Qwen2 connector + norm/projection, producing DiT condition embeds.
The connector is a ``Qwen2ForCausalLM`` loaded from the ``connector/``
subfolder of the Ming checkpoint. We run its base model in a non-causal
(bidirectional) mode, because the connector is used as an encoder over the
pre-baked query-token hidden states, not as an autoregressive decoder.
Args:
image_gen_config: ``MingImageGenConfig`` from ``MingFlashOmniConfig``.
thinker_hidden_size: Hidden size of the thinker (BailingMoeV2) model.
Used to build a ``proj_in`` layer when the connector embedding
dim differs. For the released checkpoint this is 4096.
device: Placement for the module.
dtype: Parameter dtype (typically bfloat16 / float16).
"""
def __init__(
self,
image_gen_config: MingImageGenConfig,
*,
thinker_hidden_size: int = 4096,
device: torch.device | str | None = None,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.config = image_gen_config
self.thinker_hidden_size = thinker_hidden_size
self._target_device = torch.device(device) if device is not None else None
self._target_dtype = dtype
# Populated lazily by ``load_from_checkpoint`` to keep this module
# cheap to construct (useful for dummy-init paths and unit tests).
self.connector: nn.Module | None = None
self.connector_hidden_size: int | None = None
self.proj_in: nn.Module = nn.Identity()
self.proj_out: nn.Module = nn.Identity()
self.norm: nn.Module = nn.Identity()
# ------------------------------------------------------------------
# Weight loading
# ------------------------------------------------------------------
def load_from_checkpoint(self, model_path: str | Path) -> None:
"""Load the Qwen2 connector + optional projection/norm weights.
This uses HF transformers directly (not vllm's weight loader) because
the connector is small (~1.5B params) and only runs once per request
as an encoder — vllm's distributed loading machinery is overkill.
"""
from transformers import AutoConfig, Qwen2ForCausalLM
model_path = Path(model_path)
connector_path = model_path / self.config.connector_subfolder
logger.info("[MingConditionEncoder] loading connector from %s", connector_path)
connector_cfg = AutoConfig.from_pretrained(connector_path, trust_remote_code=True, local_files_only=True)
# Disable causal masking: the connector is used as a bidirectional
# encoder over query-token hidden states in image-gen mode.
connector_cfg.is_decoder = False
self.connector_hidden_size = int(connector_cfg.hidden_size)
logger.info(
"[MingConditionEncoder] connector hidden_size=%d, layers=%d",
self.connector_hidden_size,
getattr(connector_cfg, "num_hidden_layers", -1),
)
connector = Qwen2ForCausalLM.from_pretrained(
connector_path,
config=connector_cfg,
torch_dtype=self._target_dtype,
local_files_only=True,
)
# Force bidirectional attention in every self-attn module. Qwen2 layers
# do not branch on ``is_causal`` directly, so we monkey-patch the flag
# on each attention block (defensive — some transformers versions read
# ``self_attn.is_causal`` in forward).
patched = 0
for module in connector.modules():
if hasattr(module, "is_causal"):
module.is_causal = False
patched += 1
logger.info("[MingConditionEncoder] disabled is_causal on %d sub-modules", patched)
# We only need the base encoder (no LM head).
base = getattr(connector, "model", connector)
self.connector = base
# proj_in: align thinker_hidden_size -> connector_hidden_size.
# Observed layout of mlp/ on the released checkpoint
# (inclusionAI/Ming-flash-omni-2.0):
# proj_in.{weight,bias} : Linear(4096 -> 1536)
# proj_out.{weight,bias} : Linear(1536 -> 2560)
# query_tokens_dict.16x16 : learnable tokens consumed on the thinker
# side (NOT loaded here).
logger.info(
"[MingConditionEncoder] adding proj_in: %d -> %d (bias=True)",
self.thinker_hidden_size,
self.connector_hidden_size,
)
self.proj_in = nn.Linear(self.thinker_hidden_size, self.connector_hidden_size, bias=True)
# NOTE: ``text_encoder_norm=True`` in Ming's mlp/config.json refers to
# applying ``F.normalize(..., dim=-1)`` (L2 normalization) on the
# FINAL cap_feats AFTER proj_out — NOT an intermediate RMSNorm. See
# modeling_bailingmm2.py::get_condition_embeds_for_image_gen. We keep
# ``self.norm = nn.Identity`` and apply the L2 normalize explicitly
# at the end of ``forward``.
self.norm = nn.Identity()
# proj_out: align connector_hidden_size -> diffusion_c_input_dim.
c_out = self.config.diffusion_c_input_dim
logger.info(
"[MingConditionEncoder] adding proj_out: %d -> %d (bias=True)",
self.connector_hidden_size,
c_out,
)
self.proj_out = nn.Linear(self.connector_hidden_size, c_out, bias=True)
mlp_path = model_path / self.config.mlp_subfolder
mlp_cfg_path = mlp_path / "config.json"
if mlp_cfg_path.exists() and not json.loads(mlp_cfg_path.read_text()).get("use_identity_mlp", False):
# MingImagePipeline skips ZImageModel_withMLP, only correct when its inner MLP is Identity.
raise NotImplementedError(f"{mlp_cfg_path} has use_identity_mlp=False; ToClipMLP path not implemented.")
self._load_optional_mlp_weights(mlp_path)
if self._target_device is not None:
self.to(self._target_device)
if self._target_dtype is not None:
self.to(dtype=self._target_dtype)
logger.info(
"[MingConditionEncoder] ready: in=%d -> conn=%d -> out=%d",
self.thinker_hidden_size,
self.connector_hidden_size,
c_out,
)
def _load_optional_mlp_weights(self, mlp_path: Path) -> None:
"""Load proj_in / proj_out (and optional norm) weights from mlp/.
Expected keys — observed on inclusionAI/Ming-flash-omni-2.0:
proj_in.weight [1536, 4096]
proj_in.bias [1536]
proj_out.weight [2560, 1536]
proj_out.bias [2560]
query_tokens_dict.16x16 [256, 4096] -> thinker-side, skipped here
(optional) norm.weight [1536]
Any extra keys are logged as warnings so we don't silently miss new
fields. Missing proj weights are fatal: they are required for the
condition path to be meaningful.
"""
if not mlp_path.exists():
logger.warning(
"[MingConditionEncoder] mlp/ subfolder missing at %s — proj/norm "
"will stay randomly initialized. EXPECT BAD IMAGES until this "
"is fixed on real hardware.",
mlp_path,
)
return
try:
from safetensors.torch import load_file # type: ignore
except ImportError:
logger.exception("[MingConditionEncoder] safetensors not installed")
return
candidates = sorted(mlp_path.glob("*.safetensors"))
if not candidates:
candidates = sorted(mlp_path.glob("*.bin"))
if not candidates:
logger.warning("[MingConditionEncoder] no weight files under %s", mlp_path)
return
state: dict[str, torch.Tensor] = {}
for p in candidates:
logger.info("[MingConditionEncoder] reading mlp weights: %s", p)
if p.suffix == ".safetensors":
state.update(load_file(str(p)))
else:
state.update(torch.load(str(p), map_location="cpu"))
logger.info(
"[MingConditionEncoder] mlp/ keys: %s",
sorted(state.keys()),
)
handled: set[str] = set()
def _copy(dst: nn.Parameter | torch.Tensor, src_key: str) -> bool:
src = state.get(src_key)
if src is None:
logger.error("[MingConditionEncoder] mlp/ missing key %r", src_key)
return False
if tuple(src.shape) != tuple(dst.shape):
logger.error(
"[MingConditionEncoder] mlp/%s shape mismatch: checkpoint=%s, module=%s",
src_key,
tuple(src.shape),
tuple(dst.shape),
)
return False
with torch.no_grad():
dst.copy_(src.to(dtype=dst.dtype, device=dst.device))
handled.add(src_key)
logger.info(
"[MingConditionEncoder] loaded mlp/%s -> %s",
src_key,
tuple(dst.shape),
)
return True
ok_w = _copy(self.proj_in.weight, "proj_in.weight")
ok_b = _copy(self.proj_in.bias, "proj_in.bias")
ok_ow = _copy(self.proj_out.weight, "proj_out.weight")
ok_ob = _copy(self.proj_out.bias, "proj_out.bias")
if not (ok_w and ok_b and ok_ow and ok_ob):
logger.error(
"[MingConditionEncoder] proj_in/proj_out NOT fully loaded; diffusion conditioning will be garbage."
)
# Optional norm weight (Ming uses plain RMSNorm; may or may not ship).
if "norm.weight" in state and hasattr(self.norm, "weight"):
_copy(self.norm.weight, "norm.weight")
# query_tokens_dict lives in mlp/ but is consumed on the thinker side
# — we neither need nor load it here. Just log the shape so the
# thinker-side patch knows what to expect.
for k, v in state.items():
if k.startswith("query_tokens_dict"):
logger.info(
"[MingConditionEncoder] thinker-side parameter %s shape=%s (NOT loaded here; thinker must own it)",
k,
tuple(v.shape),
)
handled.add(k)
leftover = set(state.keys()) - handled
if leftover:
logger.warning("[MingConditionEncoder] mlp/ unhandled keys: %s", sorted(leftover))
# ------------------------------------------------------------------
# Forward
# ------------------------------------------------------------------
def forward(
self,
thinker_hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Encode thinker hidden states into DiT condition embeddings.
Args:
thinker_hidden_states: ``[B, N, thinker_hidden_size]`` — sliced at
the learnable query-token positions by the stage input
processor before being passed here.
attention_mask: Optional ``[B, N]`` mask. Defaults to all-ones.
Returns:
``[B, N, diffusion_c_input_dim]`` condition tensor ready for the
ZImage transformer's ``cap_feats`` input.
"""
if self.connector is None:
raise RuntimeError("MingConditionEncoder.load_from_checkpoint() must be called before forward().")
if thinker_hidden_states.dim() != 3:
raise ValueError(f"expected [B, N, H], got shape {tuple(thinker_hidden_states.shape)}")
b, n, _ = thinker_hidden_states.shape
logger.debug(
"[MingCondEnc] input shape=%s dtype=%s", tuple(thinker_hidden_states.shape), thinker_hidden_states.dtype
)
x = self.proj_in(thinker_hidden_states) # [B, N, conn_hidden]
# Ming's ``get_condition_embeds_for_image_gen`` passes a 4D full-ones
# attention mask of shape [B, 1, N, N] to the Qwen2 connector,
# forcing full bidirectional self-attention over all query positions.
if attention_mask is None:
attention_mask = torch.ones(
(b, 1, n, n),
dtype=x.dtype,
device=x.device,
)
elif attention_mask.dim() == 2:
mask_2d = attention_mask.to(x.dtype)
attention_mask = mask_2d[:, None, None, :].expand(b, 1, n, n)
out = self.connector(
inputs_embeds=x,
attention_mask=attention_mask,
use_cache=False,
output_hidden_states=True,
return_dict=True,
)
hidden = out.hidden_states[-1] # [B, N, conn_hidden]
cap_feats = self.proj_out(hidden) # [B, N, diffusion_c_input_dim]
# L2 normalize + ×1000 rescale. Ming's ``get_condition_embeds_for_image_gen``
# applies F.normalize; ``ZImageModel_withMLP`` then rescales by 1000
# when ``text_encoder_norm=True``. We fold both into one place.
cap_feats = torch.nn.functional.normalize(cap_feats, dim=-1)
if self.config.text_encoder_norm:
cap_feats = cap_feats * 1000.0
logger.debug("[MingCondEnc] output shape=%s dtype=%s", tuple(cap_feats.shape), cap_feats.dtype)
return cap_feats
# ------------------------------------------------------------------
# Utility
# ------------------------------------------------------------------
@torch.no_grad()
def zero_negative(
self,
cap_feats: torch.Tensor,
) -> torch.Tensor:
"""Return a zero tensor shaped like ``cap_feats`` for CFG negatives."""
return torch.zeros_like(cap_feats)
def extra_repr(self) -> str:
return (
f"thinker_hidden_size={self.thinker_hidden_size}, "
f"connector_hidden_size={self.connector_hidden_size}, "
f"diffusion_c_input_dim={self.config.diffusion_c_input_dim}, "
f"text_encoder_norm={self.config.text_encoder_norm}"
)
__all__ = ["MingConditionEncoder"]
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
"""Ming-specific subclass of ZImageTransformer2DModel that supports ``ref_x``.
Ming's img2img path concatenates a VAE-encoded reference latent along the
frame axis before patchification, then drops the reference portion from the
unpatchified output. This is a surgical override — everything else (attention,
RoPE, final layer) stays on the parent implementation.
"""
from __future__ import annotations
import torch
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
from vllm_omni.diffusion.models.z_image.z_image_transformer import ZImageTransformer2DModel
class MingZImageTransformer2DModel(ZImageTransformer2DModel):
"""ZImage DiT with Ming's reference-latent support."""
def forward(
self,
x: list[torch.Tensor],
t,
cap_feats: list[torch.Tensor],
patch_size=2,
f_patch_size=1,
):
ref_latent = get_forward_context().ref_latent if is_forward_context_available() else None
if ref_latent is not None:
per_item = ref_latent[0].unsqueeze(1).to(dtype=x[0].dtype, device=x[0].device) # [C, 1, H, W]
x = [torch.cat([img, per_item], dim=1) for img in x]
return super().forward(x, t, cap_feats, patch_size=patch_size, f_patch_size=f_patch_size)
def unpatchify(
self,
x: list[torch.Tensor],
size: list[tuple],
patch_size,
f_patch_size,
) -> list[torch.Tensor]:
out = super().unpatchify(x, size, patch_size, f_patch_size)
# No-op when F==1 (pure t2i); drops the reference-frame prediction when F==2.
return [t[:, :1, :, :] for t in out]
__all__ = ["MingZImageTransformer2DModel"]
@@ -0,0 +1,520 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
"""Ming-flash-omni-2.0 imagegen (text-to-image / img2img) diffusion pipeline.
Cross-stage data flow:
Stage 0 (thinker, llm) Stage 1 (imagegen, diffusion)
──────────────────── ──────────────────────────────
forward returns thinker2imagegen hook slices
multimodal_output[ final_hidden_states at
"final_hidden_states"] <imagePatch> positions,
↓ returns list[dict] with
shared_memory_connector {"extra": {"thinker_hidden_states"}}
↓ ──── via OmniMsgpackEncoder ────>
MingImagePipeline.forward(req):
hidden = req.prompts[0]["extra"][...]
cond = condition_encoder(hidden)
img = ZImagePipeline-style loop
return DiffusionOutput(output=img)
"""
from __future__ import annotations
import logging
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import torch
import torch.nn as nn
from diffusers.image_processor import VaeImageProcessor
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl import DistributedAutoencoderKL
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.forward_context import set_forward_context_ref_latent
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders
from vllm_omni.diffusion.models.ming_flash_omni.byte5_encoder import (
MingByT5Encoder,
)
from vllm_omni.diffusion.models.ming_flash_omni.condition_encoder import (
MingConditionEncoder,
)
from vllm_omni.diffusion.models.ming_flash_omni.ming_zimage_transformer import (
MingZImageTransformer2DModel,
)
from vllm_omni.diffusion.models.z_image.pipeline_z_image import ZImagePipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingImageGenConfig
logger = logging.getLogger(__name__)
@dataclass
class _ZPipelineSamplingParams:
"""Typed shim satisfying the attributes ZImagePipeline.forward reads
from ``req.sampling_params``. Unlike SimpleNamespace this fails loudly
at construction if a required field is missing."""
height: int
width: int
num_inference_steps: int
guidance_scale: float
generator: torch.Generator | None = None
strength: float | None = None
sigmas: list[float] | None = None
max_sequence_length: int = 512
guidance_rescale: float | None = None
num_outputs_per_prompt: int = 1
@dataclass
class _ZPipelineRequest:
"""Typed shim satisfying the attributes ZImagePipeline.forward reads
from ``req``."""
request_id: str
sampling_params: _ZPipelineSamplingParams
prompts: list[dict[str, Any]] = field(default_factory=lambda: [{"prompt": "", "negative_prompt": ""}])
class MingImagePipeline(ZImagePipeline):
"""Ming-flash-omni-2.0 text-to-image diffusion pipeline.
Ming-specific components added on top of the inherited contract:
* ``condition_encoder`` — Qwen2 connector + proj_in/out + F.normalize×1000
* ``byte5`` — Optional ByT5 glyph encoder (loaded if checkpoint
ships ``byt5/``)
"""
def __init__(
self,
*,
od_config: OmniDiffusionConfig,
prefix: str = "", # noqa: ARG002
) -> None:
# Skip ZImagePipeline.__init__ (it would eagerly load the Z-Image text
# encoder/tokenizer that Ming replaces with its own condition_encoder).
nn.Module.__init__(self)
model_path = od_config.model
if not os.path.exists(model_path):
model_path = download_weights_from_hf_specific(model_path, od_config.revision, ["*"])
dtype = getattr(od_config, "dtype", torch.bfloat16)
local_files_only = os.path.exists(model_path)
self.od_config = od_config
self._execution_device = get_local_device()
self.device = self._execution_device # Ming convention alias
self._dtype = dtype
# Ming's per-checkpoint image-gen configuration. We cannot rely on
# ``od_config.hf_config.image_gen_config`` because the diffusion
# stage is started with ``hf_config_name: thinker_config`` (the
# BailingMM2Config), which does not carry a MingImageGenConfig.
# Fall back to defaults that match the released checkpoint.
self.image_gen_config = MingImageGenConfig()
logger.info(
"[MingImagePipeline] init: model=%s dtype=%s image_gen_config=%s",
model_path,
dtype,
self.image_gen_config,
)
# ----- weights_sources: DiT transformer + VAE.
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=model_path,
subfolder=self.image_gen_config.transformer_subfolder,
revision=od_config.revision,
prefix="transformer.",
fall_back_to_pt=True,
),
DiffusersPipelineLoader.ComponentSource(
model_or_path=model_path,
subfolder=self.image_gen_config.vae_subfolder,
revision=od_config.revision,
prefix="vae.",
),
]
prefetch_subfolders(
model_path,
[self.image_gen_config.scheduler_subfolder, self.image_gen_config.vae_subfolder],
local_files_only=local_files_only,
)
# ----- Scheduler: load config-only from disk + Ming-specific override.
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
model_path,
subfolder=self.image_gen_config.scheduler_subfolder,
local_files_only=local_files_only,
)
# Ming forces use_dynamic_shifting=True at runtime regardless of what
# the checkpoint scheduler_config.json ships.
self.scheduler.config["use_dynamic_shifting"] = True
logger.info(
"[MingImagePipeline] scheduler: %s (use_dynamic_shifting=True)",
type(self.scheduler).__name__,
)
# ----- VAE: DistributedAutoencoderKL.
vae_config = DistributedAutoencoderKL.load_config(
model_path, subfolder=self.image_gen_config.vae_subfolder, local_files_only=local_files_only
)
self.vae = DistributedAutoencoderKL.from_config(vae_config).to(self._execution_device, dtype=dtype)
self.vae.eval()
# ----- DiT transformer.
self.transformer = MingZImageTransformer2DModel(quant_config=None)
# Ming brings its own conditioning path — no Z-Image text_encoder /
# tokenizer.
self.text_encoder = None
self.tokenizer = None
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2, do_convert_rgb=True)
self.setup_diffusion_pipeline_profiler(
enable_diffusion_pipeline_profiler=getattr(od_config, "enable_diffusion_pipeline_profiler", False)
)
# ----- Condition encoder (Qwen2 connector + proj_in/out + norm×1000).
self.condition_encoder = MingConditionEncoder(
self.image_gen_config,
thinker_hidden_size=self.image_gen_config.thinker_hidden_size,
device=self.device,
dtype=dtype,
)
self.condition_encoder.load_from_checkpoint(model_path)
# Optional ByT5 glyph/text encoder. Only loaded when the checkpoint
# ``extra_args.image_gen.byte5_text`` requests will be ignored.
byte5_dir = Path(model_path) / "byt5"
if byte5_dir.exists():
self.byte5 = MingByT5Encoder.from_checkpoint(byte5_dir, device=self.device, dtype=dtype)
else:
self.byte5 = None
logger.info("[MingImagePipeline] no byt5/ subfolder at %s; ByT5 enhancement disabled", byte5_dir)
logger.info("[MingImagePipeline] ready — vae_scale_factor=%d", self.vae_scale_factor)
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
@staticmethod
def _resolve_byte5_texts(extra: dict, sampling_params) -> list[str]:
"""Resolve byte5 glyph texts.
Two sources, in order of priority:
1. ``extra["byte5_text"]`` — auto-extracted from the user prompt's
quoted spans by ``thinker2imagegen`` (already wrapped as
``'Text "<glyph>". '`` by Ming's ``get_text_from_prompt``).
2. ``sampling_params.extra_args["image_gen"]["byte5_text"]`` — an
explicit override for programmatic callers; raw strings without
the ``Text "..."`` wrapper are auto-wrapped here to match the
distribution ByT5 was trained on.
"""
# Source 1: auto-extracted, already wrapped. Return as-is if non-empty.
raw = extra.get("byte5_text")
if isinstance(raw, str):
raw = [raw]
if isinstance(raw, list):
cleaned = [t for t in raw if isinstance(t, str) and t.strip()]
if cleaned:
return cleaned
# Source 2: explicit override — wrap raw strings so the byte5 encoder
# sees the same ``Text "<glyph>". `` format Ming used during training.
raw = ((getattr(sampling_params, "extra_args", None) or {}).get("image_gen") or {}).get("byte5_text")
if isinstance(raw, str):
raw = [raw]
if isinstance(raw, list):
out: list[str] = []
for t in raw:
if not isinstance(t, str):
continue
s = t.strip()
if not s:
continue
# Don't double-wrap if the caller already supplied ``Text "...". ``.
out.append(s if s.startswith('Text "') else f'Text "{s}". ')
if out:
return out
return []
@torch.inference_mode()
def _encode_reference_image(self, ref, height: int, width: int) -> torch.Tensor | None:
"""Turn a PIL/tensor reference image into a VAE latent for ``ref_x``.
Applies the same shift/scale Ming uses (``(z - shift_factor) * scaling_factor``)
so the concatenated frame lives in the DiT's latent space.
"""
if ref is None:
return None
if not isinstance(ref, torch.Tensor):
ref = self.image_processor.preprocess(ref, height, width)
ref = ref.to(device=self.device, dtype=self.vae.dtype)
latent = self.vae.encode(ref).latent_dist.mode()
return (latent - self.vae.config.shift_factor) * self.vae.config.scaling_factor
# ------------------------------------------------------------------
# Forward
# ------------------------------------------------------------------
@torch.inference_mode()
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
"""Run one text-to-image generation request.
Args:
req: Diffusion request. The cross-stage thinker hidden states
must be present at
``req.prompts[0]["extra"]["thinker_hidden_states"]`` as a
``[N, H]`` (or ``[1, N, H]``) tensor, placed there by
``thinker2imagegen``.
Returns:
DiffusionOutput with ``.output`` set to a ``[B, 3, H, W]``
image tensor in ``[-1, 1]``. The vllm-omni diffusion engine's
output adapter converts this to PIL/base64 downstream.
"""
first_prompt = req.prompts[0] if req.prompts else None
if isinstance(first_prompt, str):
prompt_dict: dict[str, Any] = {}
elif isinstance(first_prompt, dict):
prompt_dict = first_prompt
elif first_prompt is not None and hasattr(first_prompt, "_asdict"):
prompt_dict = first_prompt._asdict()
elif first_prompt is not None and hasattr(first_prompt, "__dict__"):
prompt_dict = vars(first_prompt)
else:
prompt_dict = {}
extra = prompt_dict.get("extra") or {}
hidden = extra.get("thinker_hidden_states")
if hidden is None:
# Same dual-path convention as glm_image: also check
# ``sampling_params.extra_args``.
hidden = (req.sampling_params.extra_args or {}).get("thinker_hidden_states")
if hidden is None:
scale = self.image_gen_config.img_gen_scales[-1]
num_query_tokens = scale * scale
hidden = torch.zeros(
(num_query_tokens, self.image_gen_config.thinker_hidden_size),
dtype=self._dtype,
device=self.device,
)
logger.warning(
"[MingImagePipeline.forward] 'thinker_hidden_states' missing "
"from request; falling back to zero-conditioning %s. This is "
"expected during warmup; for real requests verify that "
"`custom_process_input_func: thinker2imagegen` is set on the "
"diffusion stage in the YAML.",
tuple(hidden.shape),
)
if not isinstance(hidden, torch.Tensor):
raise TypeError(
f"[MingImagePipeline] 'thinker_hidden_states' must be a Tensor, got {type(hidden).__name__}"
)
# Move to the pipeline's device+dtype.
target_device = next(self.parameters()).device
target_dtype = next(self.parameters()).dtype
hidden = hidden.to(device=target_device, dtype=target_dtype)
if hidden.dim() == 2:
hidden = hidden.unsqueeze(0) # [N, H] -> [1, N, H]
logger.debug(
"[MingImagePipeline.forward] thinker_hidden_states=%s on %s (%s)",
tuple(hidden.shape),
target_device,
target_dtype,
)
# ----- Condition encoder → cap_feats
cap_feats = self.condition_encoder(hidden)
logger.debug("[MingImagePipeline.forward] cap_feats=%s", tuple(cap_feats.shape))
# Real negative CFG conditioning (opt-in). See expand_cfg_prompts.
negative_hidden = extra.get("negative_thinker_hidden_states")
negative_cap_feats = None
if isinstance(negative_hidden, torch.Tensor):
negative_hidden = negative_hidden.to(device=target_device, dtype=target_dtype)
if negative_hidden.dim() == 2:
negative_hidden = negative_hidden.unsqueeze(0)
negative_cap_feats = self.condition_encoder(negative_hidden)
logger.debug("[MingImagePipeline.forward] negative_cap_feats=%s", tuple(negative_cap_feats.shape))
# ByT5 text enhancement (opt-in). Appends glyph-aware features along
# the sequence dim; negative side gets zeros for the byte5 portion so
# CFG doesn't push away from the rendered text.
byte5_texts = self._resolve_byte5_texts(extra, req.sampling_params)
if byte5_texts and self.byte5 is not None:
byte5_feats = self.byte5(byte5_texts).to(device=target_device, dtype=target_dtype)
cap_feats = torch.cat((cap_feats, byte5_feats), dim=1)
if negative_cap_feats is not None:
negative_cap_feats = torch.cat((negative_cap_feats, torch.zeros_like(byte5_feats)), dim=1)
logger.debug("[MingImagePipeline.forward] byte5 cat'd: cap_feats=%s", tuple(cap_feats.shape))
# Sampling knobs: extra_args.image_gen.* > sampling_params.* > MingImageGenConfig defaults.
sp = req.sampling_params
cfg = self.image_gen_config
ig = (sp.extra_args or {}).get("image_gen") or {}
resolved: dict[str, Any] = {}
for ig_key, sp_attr, default in (
("height", "height", cfg.default_height),
("width", "width", cfg.default_width),
("steps", "num_inference_steps", cfg.num_inference_steps),
("cfg", "guidance_scale", cfg.guidance_scale),
("seed", "seed", None),
):
for v in (ig.get(ig_key), getattr(sp, sp_attr), default):
if v is not None:
resolved[ig_key] = v
break
height = int(resolved["height"])
width = int(resolved["width"])
num_inference_steps = int(resolved["steps"])
guidance_scale = float(resolved["cfg"])
seed = resolved.get("seed")
# Always rebuild the generator from the resolved seed. Reusing
# ``sp.generator`` causes two problems:
# (1) if the caller pre-seeded it with sp.seed (e.g. the top-level
# ``seed`` key on OmniDiffusionSamplingParams), any override via
# ``extra_args.image_gen.seed`` would be silently ignored; and
# (2) a persistent generator instance accumulates state across
# requests → same-seed replays produce different outputs.
if seed is not None:
generator = torch.Generator(device=target_device).manual_seed(int(seed))
else:
generator = sp.generator
# Format prompt_embeds / negative_prompt_embeds as list[Tensor]
# (one entry per request) — matches ZImagePipeline's contract when
# prompt_embeds are pre-computed.
prompt_embeds = [cap_feats[i] for i in range(cap_feats.shape[0])]
if negative_cap_feats is not None:
negative_prompt_embeds = [negative_cap_feats[i] for i in range(negative_cap_feats.shape[0])]
else:
negative_prompt_embeds = [self.condition_encoder.zero_negative(e) for e in prompt_embeds]
z_sp = _ZPipelineSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
generator=generator,
)
z_req = _ZPipelineRequest(
request_id=req.request_id or "ming-imagegen",
sampling_params=z_sp,
)
# Reference image (img2img) → VAE-encoded latent published on the
# active ForwardContext so MingZImageTransformer2DModel can read it
# from request scope inside its forward().
ref_latent = self._encode_reference_image(extra.get("reference_image"), height, width)
set_forward_context_ref_latent(ref_latent)
logger.debug(
"[MingImagePipeline.forward] running z_pipeline hw=(%d,%d) steps=%d cfg=%.2f seed=%s overrides=%s ref=%s",
height,
width,
num_inference_steps,
guidance_scale,
seed,
ig,
None if ref_latent is None else tuple(ref_latent.shape),
)
try:
output = super().forward(
z_req,
prompt=None,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
generator=generator,
output_type="pt",
return_dict=True,
)
finally:
set_forward_context_ref_latent(None)
if hasattr(output, "output") and output.output is not None:
raw = output.output
elif hasattr(output, "images"):
raw = output.images
else:
raw = output
if not isinstance(raw, torch.Tensor):
raise RuntimeError(f"ZImagePipeline returned non-tensor output: {type(raw).__name__}")
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"[MingImagePipeline.forward] produced image tensor shape=%s range=[%.3f,%.3f]",
tuple(raw.shape),
raw.float().min().item(),
raw.float().max().item(),
)
return DiffusionOutput(output=raw)
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
def get_ming_image_post_process_func(od_config: OmniDiffusionConfig):
"""Return a post-process callable that converts the raw VAE tensor to PIL.
The diffusion engine calls ``post_process_func(output_data)`` where
``output_data`` is the ``DiffusionOutput.output`` tensor returned by
``MingImagePipeline.forward``. It has shape ``[B, 3, H, W]`` in ``[-1, 1]``
(Z-image VAE convention). We run the standard ``VaeImageProcessor``
postprocess to convert it to ``list[PIL.Image]`` which vllm-omni's
``OmniRequestOutput.from_diffusion`` then bubbles up as
``omni_outputs.images`` for serving_chat to base64-encode.
Registered via ``_DIFFUSION_POST_PROCESS_FUNCS["MingImagePipeline"]``
in vllm_omni/diffusion/registry.py.
"""
import json
model_path = od_config.model
vae_config_path = os.path.join(model_path, "vae", "config.json")
try:
with open(vae_config_path) as f:
vae_cfg = json.load(f)
block_out_channels = vae_cfg.get("block_out_channels", [128, 256, 512, 512])
vae_scale_factor = 2 ** (len(block_out_channels) - 1)
except Exception:
vae_scale_factor = 8 # Ming's Flux-format VAE default
image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True)
def post_process_func(images: torch.Tensor):
# VaeImageProcessor.postprocess with default output_type="pil"
# returns ``list[PIL.Image]``.
return image_processor.postprocess(images.float())
return post_process_func
__all__ = [
"MingImagePipeline",
"get_ming_image_post_process_func",
]
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
"""T5EncoderBlockByT5Mapper — Ming's per-block T5 stack mapping byte5 features
onto the DiT condition space.
"""
from __future__ import annotations
from collections.abc import Iterable
import torch
import torch.nn as nn
from diffusers.models.modeling_utils import ModelMixin
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.models.t5_encoder.t5_encoder import T5Block
class T5EncoderBlockByT5Mapper(ModelMixin):
"""Stacks ``num_layers`` T5 encoder blocks on top of byte5 features and
projects them to ``sdxl_channels`` (= Ming's ``diffusion_c_input_dim``).
"""
def __init__(self, byte5_config, num_layers: int, sdxl_channels: int | None = None) -> None:
super().__init__()
if num_layers > 0:
self.blocks = nn.ModuleList(
[
T5Block(
byte5_config,
has_relative_attention_bias=(i == 0),
prefix=f"blocks.{i}",
)
for i in range(num_layers)
]
)
else:
self.blocks = None
self.layer_norm = RMSNorm(byte5_config.d_model, eps=byte5_config.layer_norm_epsilon)
if sdxl_channels is not None:
self.channel_mapper = nn.Linear(byte5_config.d_model, sdxl_channels)
self.final_layer_norm = RMSNorm(sdxl_channels, eps=byte5_config.layer_norm_epsilon)
else:
self.channel_mapper = None
self.final_layer_norm = None
def get_extended_attention_mask(self, attention_mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
if attention_mask.dim() == 3:
extended = attention_mask[:, None, :, :]
elif attention_mask.dim() == 2:
extended = attention_mask[:, None, None, :]
else:
raise ValueError(f"Unexpected attention_mask shape {tuple(attention_mask.shape)}")
extended = extended.to(dtype=dtype)
return (1.0 - extended) * torch.finfo(dtype).min
def forward(self, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
extended_mask = self.get_extended_attention_mask(attention_mask, dtype=self.dtype)
hidden_states = inputs_embeds
position_bias = None
if self.blocks is not None:
for block in self.blocks:
hidden_states, position_bias = block(
hidden_states,
mask=extended_mask,
position_bias=position_bias,
)
hidden_states = self.layer_norm(hidden_states)
if self.channel_mapper is not None:
hidden_states = self.channel_mapper(hidden_states)
hidden_states = self.final_layer_norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load Ming's HF-format byte5_mapper checkpoint into the fused
TP-aware layers.
Source format (from ``byte5_mapper.pt``):
blocks.{i}.layer.0.SelfAttention.{q,k,v,o}.weight
blocks.{i}.layer.1.DenseReluDense.{wi_0,wi_1,wo}.weight
blocks.{i}.layer.{0,1}.layer_norm.weight
{layer_norm, channel_mapper, final_layer_norm}.{weight,bias}
Target format (after ``T5Block`` from t5_encoder.py):
blocks.{i}.layer.0.SelfAttention.qkv_proj.weight (fused q+k+v)
blocks.{i}.layer.1.DenseReluDense.wi.weight (fused wi_0+wi_1)
(others identical)
"""
stacked_params_mapping = [
(".qkv_proj.", ".q.", "q"),
(".qkv_proj.", ".k.", "k"),
(".qkv_proj.", ".v.", "v"),
(".DenseReluDense.wi.", ".DenseReluDense.wi_0.", 0),
(".DenseReluDense.wi.", ".DenseReluDense.wi_1.", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
matched = False
for target_substr, source_substr, shard_id in stacked_params_mapping:
if source_substr not in name:
continue
target_name = name.replace(source_substr, target_substr, 1)
if target_name in params_dict:
param = params_dict[target_name]
param.weight_loader(param, loaded_weight, shard_id)
loaded_params.add(target_name)
matched = True
break
if matched:
continue
if name in params_dict:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
__all__ = ["T5EncoderBlockByT5Mapper"]
@@ -832,5 +832,8 @@ class ZImagePipeline(nn.Module, DiffusionPipelineProfilerMixin):
loaded_weights = loader.load_weights(weights)
# Record components loaded by diffusers submodules to satisfy strict checks.
loaded_weights |= {f"vae.{name}" for name, _ in self.vae.named_parameters()}
loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
# downstream pipelines (e.g. MingImagePipeline) may set ``self.text_encoder = None`` when they
# bring their own conditioning path.
if self.text_encoder is not None:
loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
return loaded_weights