[Model] Add Ming-flash-omni-2.0 Image Generation (Diffusion) Stage (#2875)
Signed-off-by: ZhengWG <zwg0606@gmail.com> Signed-off-by: Zheng Wengang <zwg0606@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com>
This commit is contained in:
25 files changed
+2614
-42
No files matched your search
@@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
from .byte5_encoder import MingByT5Encoder
|
||||
from .condition_encoder import MingConditionEncoder
|
||||
from .ming_zimage_transformer import MingZImageTransformer2DModel
|
||||
from .pipeline_ming_imagegen import (
|
||||
MingImagePipeline,
|
||||
get_ming_image_post_process_func,
|
||||
)
|
||||
from .t5_block_mapper import T5EncoderBlockByT5Mapper
|
||||
|
||||
__all__ = [
|
||||
"MingByT5Encoder",
|
||||
"MingConditionEncoder",
|
||||
"MingImagePipeline",
|
||||
"MingZImageTransformer2DModel",
|
||||
"T5EncoderBlockByT5Mapper",
|
||||
"get_ming_image_post_process_func",
|
||||
]
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
"""ByT5 glyph/text encoder for Ming-flash-omni-2.0 image generation.
|
||||
|
||||
Ported from Ming's ``load_byte5_and_byte5_tokenizer`` +
|
||||
``T5EncoderBlockByT5Mapper`` + ``get_condition_embeds_for_image_gen`` byte5
|
||||
branch. The released checkpoint's byte5 weights were trained with font/color
|
||||
special tokens, so we replicate that vocabulary extension before loading —
|
||||
otherwise ``byte5_model.pt`` would shape-mismatch at the embedding layer.
|
||||
|
||||
Typical forward: takes a list of user-supplied prompt strings (possibly with
|
||||
``<cn-font-N>`` / ``<color-N>`` markers), returns ``[B, byte5_max_length,
|
||||
diffusion_c_input_dim]`` features ready to be concatenated onto cap_feats
|
||||
along the sequence dimension.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoTokenizer, T5ForConditionalGeneration
|
||||
|
||||
from vllm_omni.diffusion.models.ming_flash_omni.t5_block_mapper import (
|
||||
T5EncoderBlockByT5Mapper,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _add_multilingual_special_tokens(
|
||||
tokenizer,
|
||||
text_encoder: nn.Module,
|
||||
font_ann_path: Path,
|
||||
color_ann_path: Path,
|
||||
add_font: bool,
|
||||
add_color: bool,
|
||||
add_align: bool = False,
|
||||
) -> None:
|
||||
"""Extend the byte5 vocab with per-language font + color markers.
|
||||
|
||||
Mirrors ``add_special_token_multilingual`` in Ming's bizgen utils. The token
|
||||
set must match the one the checkpoint was trained with, otherwise the
|
||||
resized embedding table shape won't line up with the shipped weights.
|
||||
"""
|
||||
idx_font_dict = json.loads(font_ann_path.read_text())
|
||||
idx_color_dict = json.loads(color_ann_path.read_text())
|
||||
|
||||
font_tokens: list[str] = []
|
||||
for font_code in idx_font_dict:
|
||||
prefix = font_code[:3]
|
||||
if prefix in ("cn-", "en-", "jp-", "kr-"):
|
||||
font_tokens.append(f"<{prefix}font-{idx_font_dict[font_code]}>")
|
||||
else:
|
||||
font_tokens.append(f"<font-{idx_font_dict[font_code]}>")
|
||||
color_tokens = [f"<color-{i}>" for i in range(len(idx_color_dict))]
|
||||
align_tokens = [f"<align-{i}>" for i in range(3)]
|
||||
|
||||
extra: list[str] = []
|
||||
if add_color:
|
||||
extra += color_tokens
|
||||
if add_font:
|
||||
extra += font_tokens
|
||||
if add_align:
|
||||
extra += align_tokens
|
||||
tokenizer.add_tokens(extra, special_tokens=True)
|
||||
text_encoder.resize_token_embeddings(len(tokenizer))
|
||||
|
||||
|
||||
class MingByT5Encoder(nn.Module):
|
||||
"""Bundles byte5 tokenizer + T5 encoder + T5EncoderBlockByT5Mapper.
|
||||
|
||||
Build with ``MingByT5Encoder.from_checkpoint(<model>/byte5)`` when the
|
||||
checkpoint ships byte5 weights; otherwise callers can skip this and the
|
||||
pipeline falls back to no-byte5 conditioning.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer,
|
||||
text_encoder: nn.Module,
|
||||
mapper: T5EncoderBlockByT5Mapper,
|
||||
max_length: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tokenizer = tokenizer
|
||||
self.text_encoder = text_encoder
|
||||
self.mapper = mapper
|
||||
self.max_length = max_length
|
||||
|
||||
@classmethod
|
||||
def from_checkpoint(
|
||||
cls,
|
||||
byte5_dir: Path,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> MingByT5Encoder:
|
||||
# Wrap the whole load in fork_rng so any nn.init.normal_ inside
|
||||
# T5ForConditionalGeneration.from_pretrained / vocab-resize cannot
|
||||
# advance the default CPU/CUDA generator state. Without this, the
|
||||
# diffusion pipeline's seeded noise becomes order-dependent across
|
||||
# requests (same-seed replays diverge).
|
||||
cuda_devs = list(range(torch.accelerator.device_count())) if torch.cuda.is_available() else []
|
||||
with torch.random.fork_rng(devices=cuda_devs, enabled=True):
|
||||
return cls._from_checkpoint_impl(byte5_dir, device=device, dtype=dtype)
|
||||
|
||||
@classmethod
|
||||
def _from_checkpoint_impl(
|
||||
cls,
|
||||
byte5_dir: Path,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> MingByT5Encoder:
|
||||
byte5_dir = Path(byte5_dir)
|
||||
# Ming checkpoint uses ``byt5`` (no 'e') in filenames and JSON keys
|
||||
# variable names below keep the ``byte5_`` spelling for readability.
|
||||
cfg_path = byte5_dir / "byt5.json"
|
||||
cfg_raw = json.loads(cfg_path.read_text())
|
||||
cfg = SimpleNamespace(**cfg_raw)
|
||||
byte5_config = cfg.byt5_config
|
||||
mapper_config = cfg.byt5_mapper_config
|
||||
max_length = int(cfg.byt5_max_length)
|
||||
|
||||
# ---- Tokenizer + T5 encoder (base).
|
||||
ckpt_key = byte5_config.get("byt5_ckpt_path")
|
||||
byte5_ckpt_path = byte5_dir / ckpt_key.lstrip("./")
|
||||
tokenizer = AutoTokenizer.from_pretrained(byte5_ckpt_path, local_files_only=True)
|
||||
text_encoder = T5ForConditionalGeneration.from_pretrained(byte5_ckpt_path, local_files_only=True).get_encoder()
|
||||
|
||||
# ---- Extend vocab with font/color markers so the shipped weights load.
|
||||
if byte5_config.get("special_token"):
|
||||
if not byte5_config.get("multilingual", True):
|
||||
raise NotImplementedError(
|
||||
"Non-multilingual byte5 vocab extension is not ported; "
|
||||
"the released Ming checkpoint uses multilingual=True."
|
||||
)
|
||||
_add_multilingual_special_tokens(
|
||||
tokenizer,
|
||||
text_encoder,
|
||||
font_ann_path=byte5_dir / byte5_config["font_ann_path"].lstrip("./"),
|
||||
color_ann_path=byte5_dir / byte5_config["color_ann_path"].lstrip("./"),
|
||||
add_font=bool(byte5_config.get("font_special_token")),
|
||||
add_color=bool(byte5_config.get("color_special_token")),
|
||||
)
|
||||
|
||||
# ---- Load byt5 text-encoder weights (base.pt wraps the backbone in a
|
||||
# trainable-module container; byt5_model.pt has the top-level encoder
|
||||
# state). Follow Ming's two-step load.
|
||||
base_state = torch.load(byte5_dir / "byt5_model" / "base.pt", map_location="cpu", weights_only=False)
|
||||
prefix = "module.text_tower.encoder."
|
||||
base_filtered = {
|
||||
name[len(prefix) :]: state for name, state in base_state["state_dict"].items() if name.startswith(prefix)
|
||||
}
|
||||
text_encoder.load_state_dict(base_filtered, strict=True)
|
||||
del base_state, base_filtered
|
||||
|
||||
encoder_state = torch.load(byte5_dir / "byt5_model" / "byt5_model.pt", map_location="cpu", weights_only=False)
|
||||
text_encoder.load_state_dict(encoder_state)
|
||||
del encoder_state
|
||||
|
||||
text_encoder.to(device=device, dtype=dtype).eval()
|
||||
|
||||
# ---- Mapper.
|
||||
mapper = T5EncoderBlockByT5Mapper(
|
||||
byte5_config=text_encoder.config,
|
||||
num_layers=int(mapper_config["num_layers"]),
|
||||
sdxl_channels=int(mapper_config["sdxl_channels"]),
|
||||
)
|
||||
mapper_state = torch.load(byte5_dir / "byt5_mapper" / "byt5_mapper.pt", map_location="cpu", weights_only=False)
|
||||
# mapper now uses vllm-omni TP-aware T5 layers (fused qkv_proj / wi);
|
||||
mapper.load_weights(mapper_state.items())
|
||||
del mapper_state
|
||||
mapper.to(device=device, dtype=dtype).eval()
|
||||
|
||||
logger.info(
|
||||
"[MingByT5Encoder] ready: d_model=%d mapper_layers=%d sdxl_channels=%d max_length=%d vocab=%d",
|
||||
text_encoder.config.d_model,
|
||||
mapper_config["num_layers"],
|
||||
mapper_config["sdxl_channels"],
|
||||
max_length,
|
||||
len(tokenizer),
|
||||
)
|
||||
return cls(tokenizer, text_encoder, mapper, max_length)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, texts: list[str]) -> torch.Tensor:
|
||||
"""Tokenize → T5 encode → mapper; masks out padded positions.
|
||||
|
||||
Returns ``[B, max_length, sdxl_channels]``. Padded positions are zeroed
|
||||
so the downstream ``torch.cat`` with cap_feats doesn't inject garbage.
|
||||
"""
|
||||
device = next(self.text_encoder.parameters()).device
|
||||
dtype = next(self.text_encoder.parameters()).dtype
|
||||
|
||||
tokens = self.tokenizer(
|
||||
texts,
|
||||
padding="max_length",
|
||||
max_length=self.max_length,
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
input_ids = tokens.input_ids.to(device)
|
||||
attention_mask = tokens.attention_mask.to(device)
|
||||
|
||||
encoder_out = self.text_encoder(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask.float(),
|
||||
)
|
||||
hidden_states = encoder_out[0]
|
||||
feats = self.mapper(hidden_states, attention_mask)
|
||||
feats = feats * attention_mask.unsqueeze(-1).to(dtype=feats.dtype)
|
||||
return feats.to(dtype=dtype)
|
||||
|
||||
|
||||
__all__ = ["MingByT5Encoder"]
|
||||
@@ -0,0 +1,373 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
#
|
||||
# Adapted from Ming repository (inclusionAI/Ming) — the ``get_condition_embeds
|
||||
# _for_image_gen`` path in modeling_bailingmm2.py.
|
||||
|
||||
"""Ming-flash-omni-2.0 condition encoder for image generation.
|
||||
|
||||
Pipeline (runs inside the imagegen stage):
|
||||
|
||||
thinker hidden states at query-token positions [B, N, 4096]
|
||||
│
|
||||
▼ proj_in (Linear, bias=True)
|
||||
[B, N, 1536]
|
||||
│
|
||||
▼ Qwen2ForCausalLM connector (is_causal=False)
|
||||
loaded from <checkpoint>/connector/
|
||||
[B, N, 1536]
|
||||
│
|
||||
▼ proj_out (Linear, bias=True)
|
||||
[B, N, 2560]
|
||||
│
|
||||
▼ F.normalize(dim=-1) × 1000 (text_encoder_norm)
|
||||
[B, N, 2560]
|
||||
│
|
||||
▼
|
||||
cap_feats consumed by ZImageTransformer2DModel
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingImageGenConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MingConditionEncoder(nn.Module):
|
||||
"""Wraps a Qwen2 connector + norm/projection, producing DiT condition embeds.
|
||||
|
||||
The connector is a ``Qwen2ForCausalLM`` loaded from the ``connector/``
|
||||
subfolder of the Ming checkpoint. We run its base model in a non-causal
|
||||
(bidirectional) mode, because the connector is used as an encoder over the
|
||||
pre-baked query-token hidden states, not as an autoregressive decoder.
|
||||
|
||||
Args:
|
||||
image_gen_config: ``MingImageGenConfig`` from ``MingFlashOmniConfig``.
|
||||
thinker_hidden_size: Hidden size of the thinker (BailingMoeV2) model.
|
||||
Used to build a ``proj_in`` layer when the connector embedding
|
||||
dim differs. For the released checkpoint this is 4096.
|
||||
device: Placement for the module.
|
||||
dtype: Parameter dtype (typically bfloat16 / float16).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_gen_config: MingImageGenConfig,
|
||||
*,
|
||||
thinker_hidden_size: int = 4096,
|
||||
device: torch.device | str | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = image_gen_config
|
||||
self.thinker_hidden_size = thinker_hidden_size
|
||||
self._target_device = torch.device(device) if device is not None else None
|
||||
self._target_dtype = dtype
|
||||
|
||||
# Populated lazily by ``load_from_checkpoint`` to keep this module
|
||||
# cheap to construct (useful for dummy-init paths and unit tests).
|
||||
self.connector: nn.Module | None = None
|
||||
self.connector_hidden_size: int | None = None
|
||||
self.proj_in: nn.Module = nn.Identity()
|
||||
self.proj_out: nn.Module = nn.Identity()
|
||||
self.norm: nn.Module = nn.Identity()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_from_checkpoint(self, model_path: str | Path) -> None:
|
||||
"""Load the Qwen2 connector + optional projection/norm weights.
|
||||
|
||||
This uses HF transformers directly (not vllm's weight loader) because
|
||||
the connector is small (~1.5B params) and only runs once per request
|
||||
as an encoder — vllm's distributed loading machinery is overkill.
|
||||
"""
|
||||
from transformers import AutoConfig, Qwen2ForCausalLM
|
||||
|
||||
model_path = Path(model_path)
|
||||
connector_path = model_path / self.config.connector_subfolder
|
||||
logger.info("[MingConditionEncoder] loading connector from %s", connector_path)
|
||||
|
||||
connector_cfg = AutoConfig.from_pretrained(connector_path, trust_remote_code=True, local_files_only=True)
|
||||
# Disable causal masking: the connector is used as a bidirectional
|
||||
# encoder over query-token hidden states in image-gen mode.
|
||||
connector_cfg.is_decoder = False
|
||||
self.connector_hidden_size = int(connector_cfg.hidden_size)
|
||||
logger.info(
|
||||
"[MingConditionEncoder] connector hidden_size=%d, layers=%d",
|
||||
self.connector_hidden_size,
|
||||
getattr(connector_cfg, "num_hidden_layers", -1),
|
||||
)
|
||||
|
||||
connector = Qwen2ForCausalLM.from_pretrained(
|
||||
connector_path,
|
||||
config=connector_cfg,
|
||||
torch_dtype=self._target_dtype,
|
||||
local_files_only=True,
|
||||
)
|
||||
# Force bidirectional attention in every self-attn module. Qwen2 layers
|
||||
# do not branch on ``is_causal`` directly, so we monkey-patch the flag
|
||||
# on each attention block (defensive — some transformers versions read
|
||||
# ``self_attn.is_causal`` in forward).
|
||||
patched = 0
|
||||
for module in connector.modules():
|
||||
if hasattr(module, "is_causal"):
|
||||
module.is_causal = False
|
||||
patched += 1
|
||||
logger.info("[MingConditionEncoder] disabled is_causal on %d sub-modules", patched)
|
||||
|
||||
# We only need the base encoder (no LM head).
|
||||
base = getattr(connector, "model", connector)
|
||||
self.connector = base
|
||||
|
||||
# proj_in: align thinker_hidden_size -> connector_hidden_size.
|
||||
# Observed layout of mlp/ on the released checkpoint
|
||||
# (inclusionAI/Ming-flash-omni-2.0):
|
||||
# proj_in.{weight,bias} : Linear(4096 -> 1536)
|
||||
# proj_out.{weight,bias} : Linear(1536 -> 2560)
|
||||
# query_tokens_dict.16x16 : learnable tokens consumed on the thinker
|
||||
# side (NOT loaded here).
|
||||
logger.info(
|
||||
"[MingConditionEncoder] adding proj_in: %d -> %d (bias=True)",
|
||||
self.thinker_hidden_size,
|
||||
self.connector_hidden_size,
|
||||
)
|
||||
self.proj_in = nn.Linear(self.thinker_hidden_size, self.connector_hidden_size, bias=True)
|
||||
|
||||
# NOTE: ``text_encoder_norm=True`` in Ming's mlp/config.json refers to
|
||||
# applying ``F.normalize(..., dim=-1)`` (L2 normalization) on the
|
||||
# FINAL cap_feats AFTER proj_out — NOT an intermediate RMSNorm. See
|
||||
# modeling_bailingmm2.py::get_condition_embeds_for_image_gen. We keep
|
||||
# ``self.norm = nn.Identity`` and apply the L2 normalize explicitly
|
||||
# at the end of ``forward``.
|
||||
self.norm = nn.Identity()
|
||||
|
||||
# proj_out: align connector_hidden_size -> diffusion_c_input_dim.
|
||||
c_out = self.config.diffusion_c_input_dim
|
||||
logger.info(
|
||||
"[MingConditionEncoder] adding proj_out: %d -> %d (bias=True)",
|
||||
self.connector_hidden_size,
|
||||
c_out,
|
||||
)
|
||||
self.proj_out = nn.Linear(self.connector_hidden_size, c_out, bias=True)
|
||||
|
||||
mlp_path = model_path / self.config.mlp_subfolder
|
||||
mlp_cfg_path = mlp_path / "config.json"
|
||||
if mlp_cfg_path.exists() and not json.loads(mlp_cfg_path.read_text()).get("use_identity_mlp", False):
|
||||
# MingImagePipeline skips ZImageModel_withMLP, only correct when its inner MLP is Identity.
|
||||
raise NotImplementedError(f"{mlp_cfg_path} has use_identity_mlp=False; ToClipMLP path not implemented.")
|
||||
self._load_optional_mlp_weights(mlp_path)
|
||||
|
||||
if self._target_device is not None:
|
||||
self.to(self._target_device)
|
||||
if self._target_dtype is not None:
|
||||
self.to(dtype=self._target_dtype)
|
||||
|
||||
logger.info(
|
||||
"[MingConditionEncoder] ready: in=%d -> conn=%d -> out=%d",
|
||||
self.thinker_hidden_size,
|
||||
self.connector_hidden_size,
|
||||
c_out,
|
||||
)
|
||||
|
||||
def _load_optional_mlp_weights(self, mlp_path: Path) -> None:
|
||||
"""Load proj_in / proj_out (and optional norm) weights from mlp/.
|
||||
|
||||
Expected keys — observed on inclusionAI/Ming-flash-omni-2.0:
|
||||
proj_in.weight [1536, 4096]
|
||||
proj_in.bias [1536]
|
||||
proj_out.weight [2560, 1536]
|
||||
proj_out.bias [2560]
|
||||
query_tokens_dict.16x16 [256, 4096] -> thinker-side, skipped here
|
||||
(optional) norm.weight [1536]
|
||||
|
||||
Any extra keys are logged as warnings so we don't silently miss new
|
||||
fields. Missing proj weights are fatal: they are required for the
|
||||
condition path to be meaningful.
|
||||
"""
|
||||
if not mlp_path.exists():
|
||||
logger.warning(
|
||||
"[MingConditionEncoder] mlp/ subfolder missing at %s — proj/norm "
|
||||
"will stay randomly initialized. EXPECT BAD IMAGES until this "
|
||||
"is fixed on real hardware.",
|
||||
mlp_path,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
from safetensors.torch import load_file # type: ignore
|
||||
except ImportError:
|
||||
logger.exception("[MingConditionEncoder] safetensors not installed")
|
||||
return
|
||||
|
||||
candidates = sorted(mlp_path.glob("*.safetensors"))
|
||||
if not candidates:
|
||||
candidates = sorted(mlp_path.glob("*.bin"))
|
||||
if not candidates:
|
||||
logger.warning("[MingConditionEncoder] no weight files under %s", mlp_path)
|
||||
return
|
||||
|
||||
state: dict[str, torch.Tensor] = {}
|
||||
for p in candidates:
|
||||
logger.info("[MingConditionEncoder] reading mlp weights: %s", p)
|
||||
if p.suffix == ".safetensors":
|
||||
state.update(load_file(str(p)))
|
||||
else:
|
||||
state.update(torch.load(str(p), map_location="cpu"))
|
||||
logger.info(
|
||||
"[MingConditionEncoder] mlp/ keys: %s",
|
||||
sorted(state.keys()),
|
||||
)
|
||||
|
||||
handled: set[str] = set()
|
||||
|
||||
def _copy(dst: nn.Parameter | torch.Tensor, src_key: str) -> bool:
|
||||
src = state.get(src_key)
|
||||
if src is None:
|
||||
logger.error("[MingConditionEncoder] mlp/ missing key %r", src_key)
|
||||
return False
|
||||
if tuple(src.shape) != tuple(dst.shape):
|
||||
logger.error(
|
||||
"[MingConditionEncoder] mlp/%s shape mismatch: checkpoint=%s, module=%s",
|
||||
src_key,
|
||||
tuple(src.shape),
|
||||
tuple(dst.shape),
|
||||
)
|
||||
return False
|
||||
with torch.no_grad():
|
||||
dst.copy_(src.to(dtype=dst.dtype, device=dst.device))
|
||||
handled.add(src_key)
|
||||
logger.info(
|
||||
"[MingConditionEncoder] loaded mlp/%s -> %s",
|
||||
src_key,
|
||||
tuple(dst.shape),
|
||||
)
|
||||
return True
|
||||
|
||||
ok_w = _copy(self.proj_in.weight, "proj_in.weight")
|
||||
ok_b = _copy(self.proj_in.bias, "proj_in.bias")
|
||||
ok_ow = _copy(self.proj_out.weight, "proj_out.weight")
|
||||
ok_ob = _copy(self.proj_out.bias, "proj_out.bias")
|
||||
if not (ok_w and ok_b and ok_ow and ok_ob):
|
||||
logger.error(
|
||||
"[MingConditionEncoder] proj_in/proj_out NOT fully loaded; diffusion conditioning will be garbage."
|
||||
)
|
||||
|
||||
# Optional norm weight (Ming uses plain RMSNorm; may or may not ship).
|
||||
if "norm.weight" in state and hasattr(self.norm, "weight"):
|
||||
_copy(self.norm.weight, "norm.weight")
|
||||
|
||||
# query_tokens_dict lives in mlp/ but is consumed on the thinker side
|
||||
# — we neither need nor load it here. Just log the shape so the
|
||||
# thinker-side patch knows what to expect.
|
||||
for k, v in state.items():
|
||||
if k.startswith("query_tokens_dict"):
|
||||
logger.info(
|
||||
"[MingConditionEncoder] thinker-side parameter %s shape=%s (NOT loaded here; thinker must own it)",
|
||||
k,
|
||||
tuple(v.shape),
|
||||
)
|
||||
handled.add(k)
|
||||
|
||||
leftover = set(state.keys()) - handled
|
||||
if leftover:
|
||||
logger.warning("[MingConditionEncoder] mlp/ unhandled keys: %s", sorted(leftover))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Forward
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def forward(
|
||||
self,
|
||||
thinker_hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Encode thinker hidden states into DiT condition embeddings.
|
||||
|
||||
Args:
|
||||
thinker_hidden_states: ``[B, N, thinker_hidden_size]`` — sliced at
|
||||
the learnable query-token positions by the stage input
|
||||
processor before being passed here.
|
||||
attention_mask: Optional ``[B, N]`` mask. Defaults to all-ones.
|
||||
|
||||
Returns:
|
||||
``[B, N, diffusion_c_input_dim]`` condition tensor ready for the
|
||||
ZImage transformer's ``cap_feats`` input.
|
||||
"""
|
||||
if self.connector is None:
|
||||
raise RuntimeError("MingConditionEncoder.load_from_checkpoint() must be called before forward().")
|
||||
if thinker_hidden_states.dim() != 3:
|
||||
raise ValueError(f"expected [B, N, H], got shape {tuple(thinker_hidden_states.shape)}")
|
||||
|
||||
b, n, _ = thinker_hidden_states.shape
|
||||
logger.debug(
|
||||
"[MingCondEnc] input shape=%s dtype=%s", tuple(thinker_hidden_states.shape), thinker_hidden_states.dtype
|
||||
)
|
||||
|
||||
x = self.proj_in(thinker_hidden_states) # [B, N, conn_hidden]
|
||||
|
||||
# Ming's ``get_condition_embeds_for_image_gen`` passes a 4D full-ones
|
||||
# attention mask of shape [B, 1, N, N] to the Qwen2 connector,
|
||||
# forcing full bidirectional self-attention over all query positions.
|
||||
if attention_mask is None:
|
||||
attention_mask = torch.ones(
|
||||
(b, 1, n, n),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
elif attention_mask.dim() == 2:
|
||||
mask_2d = attention_mask.to(x.dtype)
|
||||
attention_mask = mask_2d[:, None, None, :].expand(b, 1, n, n)
|
||||
|
||||
out = self.connector(
|
||||
inputs_embeds=x,
|
||||
attention_mask=attention_mask,
|
||||
use_cache=False,
|
||||
output_hidden_states=True,
|
||||
return_dict=True,
|
||||
)
|
||||
hidden = out.hidden_states[-1] # [B, N, conn_hidden]
|
||||
|
||||
cap_feats = self.proj_out(hidden) # [B, N, diffusion_c_input_dim]
|
||||
|
||||
# L2 normalize + ×1000 rescale. Ming's ``get_condition_embeds_for_image_gen``
|
||||
# applies F.normalize; ``ZImageModel_withMLP`` then rescales by 1000
|
||||
# when ``text_encoder_norm=True``. We fold both into one place.
|
||||
cap_feats = torch.nn.functional.normalize(cap_feats, dim=-1)
|
||||
if self.config.text_encoder_norm:
|
||||
cap_feats = cap_feats * 1000.0
|
||||
|
||||
logger.debug("[MingCondEnc] output shape=%s dtype=%s", tuple(cap_feats.shape), cap_feats.dtype)
|
||||
return cap_feats
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Utility
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.no_grad()
|
||||
def zero_negative(
|
||||
self,
|
||||
cap_feats: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Return a zero tensor shaped like ``cap_feats`` for CFG negatives."""
|
||||
return torch.zeros_like(cap_feats)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return (
|
||||
f"thinker_hidden_size={self.thinker_hidden_size}, "
|
||||
f"connector_hidden_size={self.connector_hidden_size}, "
|
||||
f"diffusion_c_input_dim={self.config.diffusion_c_input_dim}, "
|
||||
f"text_encoder_norm={self.config.text_encoder_norm}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["MingConditionEncoder"]
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
"""Ming-specific subclass of ZImageTransformer2DModel that supports ``ref_x``.
|
||||
|
||||
Ming's img2img path concatenates a VAE-encoded reference latent along the
|
||||
frame axis before patchification, then drops the reference portion from the
|
||||
unpatchified output. This is a surgical override — everything else (attention,
|
||||
RoPE, final layer) stays on the parent implementation.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm_omni.diffusion.models.z_image.z_image_transformer import ZImageTransformer2DModel
|
||||
|
||||
|
||||
class MingZImageTransformer2DModel(ZImageTransformer2DModel):
|
||||
"""ZImage DiT with Ming's reference-latent support."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: list[torch.Tensor],
|
||||
t,
|
||||
cap_feats: list[torch.Tensor],
|
||||
patch_size=2,
|
||||
f_patch_size=1,
|
||||
):
|
||||
ref_latent = get_forward_context().ref_latent if is_forward_context_available() else None
|
||||
if ref_latent is not None:
|
||||
per_item = ref_latent[0].unsqueeze(1).to(dtype=x[0].dtype, device=x[0].device) # [C, 1, H, W]
|
||||
x = [torch.cat([img, per_item], dim=1) for img in x]
|
||||
return super().forward(x, t, cap_feats, patch_size=patch_size, f_patch_size=f_patch_size)
|
||||
|
||||
def unpatchify(
|
||||
self,
|
||||
x: list[torch.Tensor],
|
||||
size: list[tuple],
|
||||
patch_size,
|
||||
f_patch_size,
|
||||
) -> list[torch.Tensor]:
|
||||
out = super().unpatchify(x, size, patch_size, f_patch_size)
|
||||
# No-op when F==1 (pure t2i); drops the reference-frame prediction when F==2.
|
||||
return [t[:, :1, :, :] for t in out]
|
||||
|
||||
|
||||
__all__ = ["MingZImageTransformer2DModel"]
|
||||
@@ -0,0 +1,520 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
"""Ming-flash-omni-2.0 imagegen (text-to-image / img2img) diffusion pipeline.
|
||||
|
||||
Cross-stage data flow:
|
||||
|
||||
Stage 0 (thinker, llm) Stage 1 (imagegen, diffusion)
|
||||
──────────────────── ──────────────────────────────
|
||||
forward returns thinker2imagegen hook slices
|
||||
multimodal_output[ final_hidden_states at
|
||||
"final_hidden_states"] <imagePatch> positions,
|
||||
↓ returns list[dict] with
|
||||
shared_memory_connector {"extra": {"thinker_hidden_states"}}
|
||||
↓ ──── via OmniMsgpackEncoder ────>
|
||||
MingImagePipeline.forward(req):
|
||||
hidden = req.prompts[0]["extra"][...]
|
||||
cond = condition_encoder(hidden)
|
||||
img = ZImagePipeline-style loop
|
||||
return DiffusionOutput(output=img)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from diffusers.image_processor import VaeImageProcessor
|
||||
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl import DistributedAutoencoderKL
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context_ref_latent
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders
|
||||
from vllm_omni.diffusion.models.ming_flash_omni.byte5_encoder import (
|
||||
MingByT5Encoder,
|
||||
)
|
||||
from vllm_omni.diffusion.models.ming_flash_omni.condition_encoder import (
|
||||
MingConditionEncoder,
|
||||
)
|
||||
from vllm_omni.diffusion.models.ming_flash_omni.ming_zimage_transformer import (
|
||||
MingZImageTransformer2DModel,
|
||||
)
|
||||
from vllm_omni.diffusion.models.z_image.pipeline_z_image import ZImagePipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingImageGenConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ZPipelineSamplingParams:
|
||||
"""Typed shim satisfying the attributes ZImagePipeline.forward reads
|
||||
from ``req.sampling_params``. Unlike SimpleNamespace this fails loudly
|
||||
at construction if a required field is missing."""
|
||||
|
||||
height: int
|
||||
width: int
|
||||
num_inference_steps: int
|
||||
guidance_scale: float
|
||||
generator: torch.Generator | None = None
|
||||
strength: float | None = None
|
||||
sigmas: list[float] | None = None
|
||||
max_sequence_length: int = 512
|
||||
guidance_rescale: float | None = None
|
||||
num_outputs_per_prompt: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ZPipelineRequest:
|
||||
"""Typed shim satisfying the attributes ZImagePipeline.forward reads
|
||||
from ``req``."""
|
||||
|
||||
request_id: str
|
||||
sampling_params: _ZPipelineSamplingParams
|
||||
prompts: list[dict[str, Any]] = field(default_factory=lambda: [{"prompt": "", "negative_prompt": ""}])
|
||||
|
||||
|
||||
class MingImagePipeline(ZImagePipeline):
|
||||
"""Ming-flash-omni-2.0 text-to-image diffusion pipeline.
|
||||
|
||||
Ming-specific components added on top of the inherited contract:
|
||||
* ``condition_encoder`` — Qwen2 connector + proj_in/out + F.normalize×1000
|
||||
* ``byte5`` — Optional ByT5 glyph encoder (loaded if checkpoint
|
||||
ships ``byt5/``)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
od_config: OmniDiffusionConfig,
|
||||
prefix: str = "", # noqa: ARG002
|
||||
) -> None:
|
||||
# Skip ZImagePipeline.__init__ (it would eagerly load the Z-Image text
|
||||
# encoder/tokenizer that Ming replaces with its own condition_encoder).
|
||||
nn.Module.__init__(self)
|
||||
|
||||
model_path = od_config.model
|
||||
if not os.path.exists(model_path):
|
||||
model_path = download_weights_from_hf_specific(model_path, od_config.revision, ["*"])
|
||||
|
||||
dtype = getattr(od_config, "dtype", torch.bfloat16)
|
||||
local_files_only = os.path.exists(model_path)
|
||||
|
||||
self.od_config = od_config
|
||||
self._execution_device = get_local_device()
|
||||
self.device = self._execution_device # Ming convention alias
|
||||
self._dtype = dtype
|
||||
|
||||
# Ming's per-checkpoint image-gen configuration. We cannot rely on
|
||||
# ``od_config.hf_config.image_gen_config`` because the diffusion
|
||||
# stage is started with ``hf_config_name: thinker_config`` (the
|
||||
# BailingMM2Config), which does not carry a MingImageGenConfig.
|
||||
# Fall back to defaults that match the released checkpoint.
|
||||
self.image_gen_config = MingImageGenConfig()
|
||||
logger.info(
|
||||
"[MingImagePipeline] init: model=%s dtype=%s image_gen_config=%s",
|
||||
model_path,
|
||||
dtype,
|
||||
self.image_gen_config,
|
||||
)
|
||||
|
||||
# ----- weights_sources: DiT transformer + VAE.
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=model_path,
|
||||
subfolder=self.image_gen_config.transformer_subfolder,
|
||||
revision=od_config.revision,
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=True,
|
||||
),
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=model_path,
|
||||
subfolder=self.image_gen_config.vae_subfolder,
|
||||
revision=od_config.revision,
|
||||
prefix="vae.",
|
||||
),
|
||||
]
|
||||
|
||||
prefetch_subfolders(
|
||||
model_path,
|
||||
[self.image_gen_config.scheduler_subfolder, self.image_gen_config.vae_subfolder],
|
||||
local_files_only=local_files_only,
|
||||
)
|
||||
|
||||
# ----- Scheduler: load config-only from disk + Ming-specific override.
|
||||
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
model_path,
|
||||
subfolder=self.image_gen_config.scheduler_subfolder,
|
||||
local_files_only=local_files_only,
|
||||
)
|
||||
# Ming forces use_dynamic_shifting=True at runtime regardless of what
|
||||
# the checkpoint scheduler_config.json ships.
|
||||
self.scheduler.config["use_dynamic_shifting"] = True
|
||||
logger.info(
|
||||
"[MingImagePipeline] scheduler: %s (use_dynamic_shifting=True)",
|
||||
type(self.scheduler).__name__,
|
||||
)
|
||||
|
||||
# ----- VAE: DistributedAutoencoderKL.
|
||||
vae_config = DistributedAutoencoderKL.load_config(
|
||||
model_path, subfolder=self.image_gen_config.vae_subfolder, local_files_only=local_files_only
|
||||
)
|
||||
self.vae = DistributedAutoencoderKL.from_config(vae_config).to(self._execution_device, dtype=dtype)
|
||||
self.vae.eval()
|
||||
|
||||
# ----- DiT transformer.
|
||||
self.transformer = MingZImageTransformer2DModel(quant_config=None)
|
||||
|
||||
# Ming brings its own conditioning path — no Z-Image text_encoder /
|
||||
# tokenizer.
|
||||
self.text_encoder = None
|
||||
self.tokenizer = None
|
||||
|
||||
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
|
||||
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2, do_convert_rgb=True)
|
||||
self.setup_diffusion_pipeline_profiler(
|
||||
enable_diffusion_pipeline_profiler=getattr(od_config, "enable_diffusion_pipeline_profiler", False)
|
||||
)
|
||||
|
||||
# ----- Condition encoder (Qwen2 connector + proj_in/out + norm×1000).
|
||||
self.condition_encoder = MingConditionEncoder(
|
||||
self.image_gen_config,
|
||||
thinker_hidden_size=self.image_gen_config.thinker_hidden_size,
|
||||
device=self.device,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.condition_encoder.load_from_checkpoint(model_path)
|
||||
|
||||
# Optional ByT5 glyph/text encoder. Only loaded when the checkpoint
|
||||
# ``extra_args.image_gen.byte5_text`` requests will be ignored.
|
||||
byte5_dir = Path(model_path) / "byt5"
|
||||
if byte5_dir.exists():
|
||||
self.byte5 = MingByT5Encoder.from_checkpoint(byte5_dir, device=self.device, dtype=dtype)
|
||||
else:
|
||||
self.byte5 = None
|
||||
logger.info("[MingImagePipeline] no byt5/ subfolder at %s; ByT5 enhancement disabled", byte5_dir)
|
||||
|
||||
logger.info("[MingImagePipeline] ready — vae_scale_factor=%d", self.vae_scale_factor)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _resolve_byte5_texts(extra: dict, sampling_params) -> list[str]:
|
||||
"""Resolve byte5 glyph texts.
|
||||
|
||||
Two sources, in order of priority:
|
||||
1. ``extra["byte5_text"]`` — auto-extracted from the user prompt's
|
||||
quoted spans by ``thinker2imagegen`` (already wrapped as
|
||||
``'Text "<glyph>". '`` by Ming's ``get_text_from_prompt``).
|
||||
2. ``sampling_params.extra_args["image_gen"]["byte5_text"]`` — an
|
||||
explicit override for programmatic callers; raw strings without
|
||||
the ``Text "..."`` wrapper are auto-wrapped here to match the
|
||||
distribution ByT5 was trained on.
|
||||
"""
|
||||
# Source 1: auto-extracted, already wrapped. Return as-is if non-empty.
|
||||
raw = extra.get("byte5_text")
|
||||
if isinstance(raw, str):
|
||||
raw = [raw]
|
||||
if isinstance(raw, list):
|
||||
cleaned = [t for t in raw if isinstance(t, str) and t.strip()]
|
||||
if cleaned:
|
||||
return cleaned
|
||||
|
||||
# Source 2: explicit override — wrap raw strings so the byte5 encoder
|
||||
# sees the same ``Text "<glyph>". `` format Ming used during training.
|
||||
raw = ((getattr(sampling_params, "extra_args", None) or {}).get("image_gen") or {}).get("byte5_text")
|
||||
if isinstance(raw, str):
|
||||
raw = [raw]
|
||||
if isinstance(raw, list):
|
||||
out: list[str] = []
|
||||
for t in raw:
|
||||
if not isinstance(t, str):
|
||||
continue
|
||||
s = t.strip()
|
||||
if not s:
|
||||
continue
|
||||
# Don't double-wrap if the caller already supplied ``Text "...". ``.
|
||||
out.append(s if s.startswith('Text "') else f'Text "{s}". ')
|
||||
if out:
|
||||
return out
|
||||
return []
|
||||
|
||||
@torch.inference_mode()
|
||||
def _encode_reference_image(self, ref, height: int, width: int) -> torch.Tensor | None:
|
||||
"""Turn a PIL/tensor reference image into a VAE latent for ``ref_x``.
|
||||
|
||||
Applies the same shift/scale Ming uses (``(z - shift_factor) * scaling_factor``)
|
||||
so the concatenated frame lives in the DiT's latent space.
|
||||
"""
|
||||
if ref is None:
|
||||
return None
|
||||
if not isinstance(ref, torch.Tensor):
|
||||
ref = self.image_processor.preprocess(ref, height, width)
|
||||
ref = ref.to(device=self.device, dtype=self.vae.dtype)
|
||||
latent = self.vae.encode(ref).latent_dist.mode()
|
||||
return (latent - self.vae.config.shift_factor) * self.vae.config.scaling_factor
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Forward
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
"""Run one text-to-image generation request.
|
||||
|
||||
Args:
|
||||
req: Diffusion request. The cross-stage thinker hidden states
|
||||
must be present at
|
||||
``req.prompts[0]["extra"]["thinker_hidden_states"]`` as a
|
||||
``[N, H]`` (or ``[1, N, H]``) tensor, placed there by
|
||||
``thinker2imagegen``.
|
||||
|
||||
Returns:
|
||||
DiffusionOutput with ``.output`` set to a ``[B, 3, H, W]``
|
||||
image tensor in ``[-1, 1]``. The vllm-omni diffusion engine's
|
||||
output adapter converts this to PIL/base64 downstream.
|
||||
"""
|
||||
first_prompt = req.prompts[0] if req.prompts else None
|
||||
if isinstance(first_prompt, str):
|
||||
prompt_dict: dict[str, Any] = {}
|
||||
elif isinstance(first_prompt, dict):
|
||||
prompt_dict = first_prompt
|
||||
elif first_prompt is not None and hasattr(first_prompt, "_asdict"):
|
||||
prompt_dict = first_prompt._asdict()
|
||||
elif first_prompt is not None and hasattr(first_prompt, "__dict__"):
|
||||
prompt_dict = vars(first_prompt)
|
||||
else:
|
||||
prompt_dict = {}
|
||||
|
||||
extra = prompt_dict.get("extra") or {}
|
||||
hidden = extra.get("thinker_hidden_states")
|
||||
if hidden is None:
|
||||
# Same dual-path convention as glm_image: also check
|
||||
# ``sampling_params.extra_args``.
|
||||
hidden = (req.sampling_params.extra_args or {}).get("thinker_hidden_states")
|
||||
if hidden is None:
|
||||
scale = self.image_gen_config.img_gen_scales[-1]
|
||||
num_query_tokens = scale * scale
|
||||
hidden = torch.zeros(
|
||||
(num_query_tokens, self.image_gen_config.thinker_hidden_size),
|
||||
dtype=self._dtype,
|
||||
device=self.device,
|
||||
)
|
||||
logger.warning(
|
||||
"[MingImagePipeline.forward] 'thinker_hidden_states' missing "
|
||||
"from request; falling back to zero-conditioning %s. This is "
|
||||
"expected during warmup; for real requests verify that "
|
||||
"`custom_process_input_func: thinker2imagegen` is set on the "
|
||||
"diffusion stage in the YAML.",
|
||||
tuple(hidden.shape),
|
||||
)
|
||||
|
||||
if not isinstance(hidden, torch.Tensor):
|
||||
raise TypeError(
|
||||
f"[MingImagePipeline] 'thinker_hidden_states' must be a Tensor, got {type(hidden).__name__}"
|
||||
)
|
||||
|
||||
# Move to the pipeline's device+dtype.
|
||||
target_device = next(self.parameters()).device
|
||||
target_dtype = next(self.parameters()).dtype
|
||||
hidden = hidden.to(device=target_device, dtype=target_dtype)
|
||||
if hidden.dim() == 2:
|
||||
hidden = hidden.unsqueeze(0) # [N, H] -> [1, N, H]
|
||||
logger.debug(
|
||||
"[MingImagePipeline.forward] thinker_hidden_states=%s on %s (%s)",
|
||||
tuple(hidden.shape),
|
||||
target_device,
|
||||
target_dtype,
|
||||
)
|
||||
|
||||
# ----- Condition encoder → cap_feats
|
||||
cap_feats = self.condition_encoder(hidden)
|
||||
logger.debug("[MingImagePipeline.forward] cap_feats=%s", tuple(cap_feats.shape))
|
||||
|
||||
# Real negative CFG conditioning (opt-in). See expand_cfg_prompts.
|
||||
negative_hidden = extra.get("negative_thinker_hidden_states")
|
||||
negative_cap_feats = None
|
||||
if isinstance(negative_hidden, torch.Tensor):
|
||||
negative_hidden = negative_hidden.to(device=target_device, dtype=target_dtype)
|
||||
if negative_hidden.dim() == 2:
|
||||
negative_hidden = negative_hidden.unsqueeze(0)
|
||||
negative_cap_feats = self.condition_encoder(negative_hidden)
|
||||
logger.debug("[MingImagePipeline.forward] negative_cap_feats=%s", tuple(negative_cap_feats.shape))
|
||||
|
||||
# ByT5 text enhancement (opt-in). Appends glyph-aware features along
|
||||
# the sequence dim; negative side gets zeros for the byte5 portion so
|
||||
# CFG doesn't push away from the rendered text.
|
||||
byte5_texts = self._resolve_byte5_texts(extra, req.sampling_params)
|
||||
if byte5_texts and self.byte5 is not None:
|
||||
byte5_feats = self.byte5(byte5_texts).to(device=target_device, dtype=target_dtype)
|
||||
cap_feats = torch.cat((cap_feats, byte5_feats), dim=1)
|
||||
if negative_cap_feats is not None:
|
||||
negative_cap_feats = torch.cat((negative_cap_feats, torch.zeros_like(byte5_feats)), dim=1)
|
||||
logger.debug("[MingImagePipeline.forward] byte5 cat'd: cap_feats=%s", tuple(cap_feats.shape))
|
||||
|
||||
# Sampling knobs: extra_args.image_gen.* > sampling_params.* > MingImageGenConfig defaults.
|
||||
sp = req.sampling_params
|
||||
cfg = self.image_gen_config
|
||||
ig = (sp.extra_args or {}).get("image_gen") or {}
|
||||
resolved: dict[str, Any] = {}
|
||||
for ig_key, sp_attr, default in (
|
||||
("height", "height", cfg.default_height),
|
||||
("width", "width", cfg.default_width),
|
||||
("steps", "num_inference_steps", cfg.num_inference_steps),
|
||||
("cfg", "guidance_scale", cfg.guidance_scale),
|
||||
("seed", "seed", None),
|
||||
):
|
||||
for v in (ig.get(ig_key), getattr(sp, sp_attr), default):
|
||||
if v is not None:
|
||||
resolved[ig_key] = v
|
||||
break
|
||||
|
||||
height = int(resolved["height"])
|
||||
width = int(resolved["width"])
|
||||
num_inference_steps = int(resolved["steps"])
|
||||
guidance_scale = float(resolved["cfg"])
|
||||
seed = resolved.get("seed")
|
||||
|
||||
# Always rebuild the generator from the resolved seed. Reusing
|
||||
# ``sp.generator`` causes two problems:
|
||||
# (1) if the caller pre-seeded it with sp.seed (e.g. the top-level
|
||||
# ``seed`` key on OmniDiffusionSamplingParams), any override via
|
||||
# ``extra_args.image_gen.seed`` would be silently ignored; and
|
||||
# (2) a persistent generator instance accumulates state across
|
||||
# requests → same-seed replays produce different outputs.
|
||||
if seed is not None:
|
||||
generator = torch.Generator(device=target_device).manual_seed(int(seed))
|
||||
else:
|
||||
generator = sp.generator
|
||||
|
||||
# Format prompt_embeds / negative_prompt_embeds as list[Tensor]
|
||||
# (one entry per request) — matches ZImagePipeline's contract when
|
||||
# prompt_embeds are pre-computed.
|
||||
prompt_embeds = [cap_feats[i] for i in range(cap_feats.shape[0])]
|
||||
if negative_cap_feats is not None:
|
||||
negative_prompt_embeds = [negative_cap_feats[i] for i in range(negative_cap_feats.shape[0])]
|
||||
else:
|
||||
negative_prompt_embeds = [self.condition_encoder.zero_negative(e) for e in prompt_embeds]
|
||||
|
||||
z_sp = _ZPipelineSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
generator=generator,
|
||||
)
|
||||
z_req = _ZPipelineRequest(
|
||||
request_id=req.request_id or "ming-imagegen",
|
||||
sampling_params=z_sp,
|
||||
)
|
||||
|
||||
# Reference image (img2img) → VAE-encoded latent published on the
|
||||
# active ForwardContext so MingZImageTransformer2DModel can read it
|
||||
# from request scope inside its forward().
|
||||
ref_latent = self._encode_reference_image(extra.get("reference_image"), height, width)
|
||||
set_forward_context_ref_latent(ref_latent)
|
||||
|
||||
logger.debug(
|
||||
"[MingImagePipeline.forward] running z_pipeline hw=(%d,%d) steps=%d cfg=%.2f seed=%s overrides=%s ref=%s",
|
||||
height,
|
||||
width,
|
||||
num_inference_steps,
|
||||
guidance_scale,
|
||||
seed,
|
||||
ig,
|
||||
None if ref_latent is None else tuple(ref_latent.shape),
|
||||
)
|
||||
try:
|
||||
output = super().forward(
|
||||
z_req,
|
||||
prompt=None,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
generator=generator,
|
||||
output_type="pt",
|
||||
return_dict=True,
|
||||
)
|
||||
finally:
|
||||
set_forward_context_ref_latent(None)
|
||||
|
||||
if hasattr(output, "output") and output.output is not None:
|
||||
raw = output.output
|
||||
elif hasattr(output, "images"):
|
||||
raw = output.images
|
||||
else:
|
||||
raw = output
|
||||
if not isinstance(raw, torch.Tensor):
|
||||
raise RuntimeError(f"ZImagePipeline returned non-tensor output: {type(raw).__name__}")
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"[MingImagePipeline.forward] produced image tensor shape=%s range=[%.3f,%.3f]",
|
||||
tuple(raw.shape),
|
||||
raw.float().min().item(),
|
||||
raw.float().max().item(),
|
||||
)
|
||||
return DiffusionOutput(output=raw)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_ming_image_post_process_func(od_config: OmniDiffusionConfig):
|
||||
"""Return a post-process callable that converts the raw VAE tensor to PIL.
|
||||
|
||||
The diffusion engine calls ``post_process_func(output_data)`` where
|
||||
``output_data`` is the ``DiffusionOutput.output`` tensor returned by
|
||||
``MingImagePipeline.forward``. It has shape ``[B, 3, H, W]`` in ``[-1, 1]``
|
||||
(Z-image VAE convention). We run the standard ``VaeImageProcessor``
|
||||
postprocess to convert it to ``list[PIL.Image]`` which vllm-omni's
|
||||
``OmniRequestOutput.from_diffusion`` then bubbles up as
|
||||
``omni_outputs.images`` for serving_chat to base64-encode.
|
||||
|
||||
Registered via ``_DIFFUSION_POST_PROCESS_FUNCS["MingImagePipeline"]``
|
||||
in vllm_omni/diffusion/registry.py.
|
||||
"""
|
||||
import json
|
||||
|
||||
model_path = od_config.model
|
||||
vae_config_path = os.path.join(model_path, "vae", "config.json")
|
||||
try:
|
||||
with open(vae_config_path) as f:
|
||||
vae_cfg = json.load(f)
|
||||
block_out_channels = vae_cfg.get("block_out_channels", [128, 256, 512, 512])
|
||||
vae_scale_factor = 2 ** (len(block_out_channels) - 1)
|
||||
except Exception:
|
||||
vae_scale_factor = 8 # Ming's Flux-format VAE default
|
||||
|
||||
image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True)
|
||||
|
||||
def post_process_func(images: torch.Tensor):
|
||||
# VaeImageProcessor.postprocess with default output_type="pil"
|
||||
# returns ``list[PIL.Image]``.
|
||||
return image_processor.postprocess(images.float())
|
||||
|
||||
return post_process_func
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MingImagePipeline",
|
||||
"get_ming_image_post_process_func",
|
||||
]
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
"""T5EncoderBlockByT5Mapper — Ming's per-block T5 stack mapping byte5 features
|
||||
onto the DiT condition space.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.models.t5_encoder.t5_encoder import T5Block
|
||||
|
||||
|
||||
class T5EncoderBlockByT5Mapper(ModelMixin):
|
||||
"""Stacks ``num_layers`` T5 encoder blocks on top of byte5 features and
|
||||
projects them to ``sdxl_channels`` (= Ming's ``diffusion_c_input_dim``).
|
||||
"""
|
||||
|
||||
def __init__(self, byte5_config, num_layers: int, sdxl_channels: int | None = None) -> None:
|
||||
super().__init__()
|
||||
if num_layers > 0:
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
T5Block(
|
||||
byte5_config,
|
||||
has_relative_attention_bias=(i == 0),
|
||||
prefix=f"blocks.{i}",
|
||||
)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
else:
|
||||
self.blocks = None
|
||||
self.layer_norm = RMSNorm(byte5_config.d_model, eps=byte5_config.layer_norm_epsilon)
|
||||
if sdxl_channels is not None:
|
||||
self.channel_mapper = nn.Linear(byte5_config.d_model, sdxl_channels)
|
||||
self.final_layer_norm = RMSNorm(sdxl_channels, eps=byte5_config.layer_norm_epsilon)
|
||||
else:
|
||||
self.channel_mapper = None
|
||||
self.final_layer_norm = None
|
||||
|
||||
def get_extended_attention_mask(self, attention_mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
|
||||
if attention_mask.dim() == 3:
|
||||
extended = attention_mask[:, None, :, :]
|
||||
elif attention_mask.dim() == 2:
|
||||
extended = attention_mask[:, None, None, :]
|
||||
else:
|
||||
raise ValueError(f"Unexpected attention_mask shape {tuple(attention_mask.shape)}")
|
||||
extended = extended.to(dtype=dtype)
|
||||
return (1.0 - extended) * torch.finfo(dtype).min
|
||||
|
||||
def forward(self, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
|
||||
extended_mask = self.get_extended_attention_mask(attention_mask, dtype=self.dtype)
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
position_bias = None
|
||||
|
||||
if self.blocks is not None:
|
||||
for block in self.blocks:
|
||||
hidden_states, position_bias = block(
|
||||
hidden_states,
|
||||
mask=extended_mask,
|
||||
position_bias=position_bias,
|
||||
)
|
||||
|
||||
hidden_states = self.layer_norm(hidden_states)
|
||||
if self.channel_mapper is not None:
|
||||
hidden_states = self.channel_mapper(hidden_states)
|
||||
hidden_states = self.final_layer_norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load Ming's HF-format byte5_mapper checkpoint into the fused
|
||||
TP-aware layers.
|
||||
|
||||
Source format (from ``byte5_mapper.pt``):
|
||||
blocks.{i}.layer.0.SelfAttention.{q,k,v,o}.weight
|
||||
blocks.{i}.layer.1.DenseReluDense.{wi_0,wi_1,wo}.weight
|
||||
blocks.{i}.layer.{0,1}.layer_norm.weight
|
||||
{layer_norm, channel_mapper, final_layer_norm}.{weight,bias}
|
||||
|
||||
Target format (after ``T5Block`` from t5_encoder.py):
|
||||
blocks.{i}.layer.0.SelfAttention.qkv_proj.weight (fused q+k+v)
|
||||
blocks.{i}.layer.1.DenseReluDense.wi.weight (fused wi_0+wi_1)
|
||||
(others identical)
|
||||
"""
|
||||
stacked_params_mapping = [
|
||||
(".qkv_proj.", ".q.", "q"),
|
||||
(".qkv_proj.", ".k.", "k"),
|
||||
(".qkv_proj.", ".v.", "v"),
|
||||
(".DenseReluDense.wi.", ".DenseReluDense.wi_0.", 0),
|
||||
(".DenseReluDense.wi.", ".DenseReluDense.wi_1.", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
matched = False
|
||||
for target_substr, source_substr, shard_id in stacked_params_mapping:
|
||||
if source_substr not in name:
|
||||
continue
|
||||
target_name = name.replace(source_substr, target_substr, 1)
|
||||
if target_name in params_dict:
|
||||
param = params_dict[target_name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(target_name)
|
||||
matched = True
|
||||
break
|
||||
|
||||
if matched:
|
||||
continue
|
||||
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
__all__ = ["T5EncoderBlockByT5Mapper"]
|
||||
@@ -832,5 +832,8 @@ class ZImagePipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
loaded_weights = loader.load_weights(weights)
|
||||
# Record components loaded by diffusers submodules to satisfy strict checks.
|
||||
loaded_weights |= {f"vae.{name}" for name, _ in self.vae.named_parameters()}
|
||||
loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
|
||||
# downstream pipelines (e.g. MingImagePipeline) may set ``self.text_encoder = None`` when they
|
||||
# bring their own conditioning path.
|
||||
if self.text_encoder is not None:
|
||||
loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
|
||||
return loaded_weights
|
||||
Reference in new issue
Block a user