[Model] Migrate MammothModa2 DiT to the shared diffusion runtime (#7134)

Signed-off-by: levius <2114377220@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-18 13:29:18 -07:00
1 parent ca3ccfc9a3
commit 7b4fc7ce2e
16 files changed
+1046 -427

No files matched your search

@@ -3,19 +3,23 @@
from __future__ import annotations
from collections.abc import Iterable
from typing import Any, ClassVar
from dataclasses import dataclass
from typing import ClassVar
import torch
from diffusers.image_processor import VaeImageProcessor
from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.transformers_utils.configs.mammoth_moda2 import Mammothmoda2Config
from .mammothmoda2_dit_model import SimpleQFormerImageRefiner, Transformer2DModel
@@ -25,6 +29,59 @@ from .schedulers import FlowMatchEulerDiscreteScheduler
logger = init_logger(__name__)
def _first_request_value(value: object) -> object:
if isinstance(value, (list, tuple)):
return value[0] if value else None
return value
def get_mammoth_moda2_post_process_func(
_od_config: OmniDiffusionConfig,
):
image_processor = VaeImageProcessor()
def post_process_func(images: torch.Tensor):
return image_processor.postprocess(images)
return post_process_func
def _build_mammoth_config(od_config: OmniDiffusionConfig) -> Mammothmoda2Config:
raw_config = od_config.tf_model_config.to_dict()
if not raw_config:
raise ValueError("MammothModa2 diffusion stage requires the root checkpoint config")
return Mammothmoda2Config(**raw_config)
def _root_weight_source(
od_config: OmniDiffusionConfig,
) -> DiffusersPipelineLoader.ComponentSource:
if not od_config.model:
raise ValueError("MammothModa2 diffusion stage requires a model path")
return DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder=None,
revision=od_config.revision,
prefix="",
fall_back_to_pt=True,
)
@dataclass(frozen=True)
class _MammothRequest:
request_id: str
full_hidden_states: torch.Tensor
full_token_ids: list[int]
answer_start_index: int
height: int
width: int
text_guidance_scale: float
cfg_range: tuple[float, float]
num_inference_steps: int
seed: int | None
generator: torch.Generator | list[torch.Generator] | None
class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
"""
MammothModa2 DiT + VAE generation stage (non-autoregressive).
@@ -38,7 +95,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
_encoder_modules: ClassVar[list[str]] = ["gen_image_condition_refiner"]
_vae_modules: ClassVar[list[str]] = ["gen_vae"]
have_multimodal_outputs = True
supports_request_batch = False
supports_step_execution = False
# Load only gen_* weights; ignore llm_model.* to prevent loading the entire LLM backbone in the DiT stage.
hf_to_vllm_mapper = WeightsMapper(
@@ -48,15 +106,13 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
super().__init__()
del prefix
hf_config = vllm_config.model_config.hf_config
if not isinstance(hf_config, Mammothmoda2Config):
raise TypeError(f"Expected Mammothmoda2Config, got {type(hf_config)}")
self.config = hf_config
self.od_config = od_config
self.device = get_local_device()
self.config = _build_mammoth_config(od_config)
self.weights_sources = [_root_weight_source(od_config)]
# --- Build DiT / VAE modules (names must match checkpoint keys) ---
if self.config.gen_vae_config is None or self.config.gen_dit_config is None:
@@ -111,9 +167,6 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
theta=10000,
)
# vLLM PP interface compatibility
self.make_empty_intermediate_tensors = lambda: None
self._llm_hidden_size = llm_hidden_size
def _reinit_caption_embedder(self, in_features: int) -> None:
@@ -125,39 +178,118 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
nn.Linear(in_features, out_features, bias=True),
)
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict[str, object]]:
if num_reqs <= 0:
raise ValueError(f"num_reqs must be positive, got {num_reqs}")
if num_reqs > 1:
raise NotImplementedError(
f"get_dummy_runtime_additional_information does not support num_reqs > 1, got {num_reqs}"
)
text_prompt_embeds = torch.zeros((1, self._llm_hidden_size), dtype=torch.float32)
image_prompt_embeds = torch.zeros((1, self._llm_hidden_size), dtype=torch.float32)
negative_prompt_embeds = torch.zeros((0, self._llm_hidden_size), dtype=torch.float32)
info = {
"text_prompt_embeds": text_prompt_embeds,
"image_prompt_embeds": image_prompt_embeds,
"negative_prompt_embeds": negative_prompt_embeds,
"negative_prompt_attention_mask": [],
"image_height": [512],
"image_width": [512],
"text_guidance_scale": [1.0],
"cfg_range": [0.0, 1.0],
"num_inference_steps": [1],
}
return [info for _ in range(num_reqs)]
def _parse_request(self, req: DiffusionRequestBatch) -> _MammothRequest:
if req.num_reqs != 1:
raise ValueError("MammothModa2 diffusion requires exactly one request")
request_id = req.request_id
sampling = req.sampling_params
if sampling.num_outputs_per_prompt != 1:
raise ValueError(f"MammothModa2 requires num_outputs_per_prompt=1 for request {request_id}")
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
# DiT stage does not consume token embeddings; return a dummy tensor.
prompt = req.prompts[0]
prompt = prompt if isinstance(prompt, dict) else {}
info = prompt.get("additional_information")
if req.is_dummy_run():
full_hidden_states = torch.zeros((2, self._llm_hidden_size), dtype=torch.float32, device="cpu")
full_token_ids = [0, int(self.config.llm_config.gen_vocab_start_index)]
answer_start_index = 1
else:
if not isinstance(info, dict):
raise ValueError(f"Missing additional_information AR conditions for request {request_id}")
full_hidden_states = info.get("full_hidden_states")
full_token_ids = info.get("full_token_ids")
if not isinstance(full_hidden_states, torch.Tensor) or not isinstance(full_token_ids, list):
raise ValueError(f"Expected full_hidden_states tensor and full_token_ids list for request {request_id}")
try:
answer_start_index = int(info.get("answer_start_index"))
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"Invalid answer_start_index for request {request_id}") from exc
if full_hidden_states.ndim != 2:
raise ValueError(f"Expected 2D full_hidden_states for request {request_id}")
if full_hidden_states.shape[0] != len(full_token_ids):
raise ValueError(f"AR hidden-state/token-count mismatch for request {request_id}")
if not 0 <= answer_start_index <= len(full_token_ids):
raise ValueError(f"answer_start_index outside token range for request {request_id}")
try:
full_token_ids = [int(token_id) for token_id in full_token_ids]
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"Invalid full_token_ids for request {request_id}") from exc
dimensions = []
for name in ("height", "width"):
value = DiffusionRequestBatch.get_prompt_field(prompt, name)
if value is None:
value = getattr(sampling, name)
if value is None:
value = 1024
try:
dimensions.append(int(value))
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"Invalid image size {name}={value!r} for request {request_id}") from exc
height, width = dimensions
if height <= 0 or width <= 0:
raise ValueError(f"Invalid image size: {height}x{width} for request {request_id}")
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"Image size must be multiples of 16, got {height}x{width} for request {request_id}")
request_info = info if isinstance(info, dict) else {}
extra_args = sampling.extra_args or {}
guidance = extra_args.get("text_guidance_scale")
if guidance is None:
guidance = sampling.guidance_scale if sampling.guidance_scale_provided else None
if guidance is None:
guidance = _first_request_value(request_info.get("text_guidance_scale"))
if guidance is None:
guidance = 9.0
try:
dtype = next(self.parameters()).dtype
except StopIteration:
dtype = torch.float32
return torch.zeros(
(input_ids.numel(), self._llm_hidden_size),
device=input_ids.device,
dtype=dtype,
text_guidance_scale = float(guidance)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"Invalid text_guidance_scale for request {request_id}") from exc
raw_num_inference_steps = extra_args.get("num_inference_steps")
if raw_num_inference_steps is None:
raw_num_inference_steps = sampling.num_inference_steps
if raw_num_inference_steps is None:
raw_num_inference_steps = _first_request_value(request_info.get("num_inference_steps"))
if raw_num_inference_steps is None:
raw_num_inference_steps = 50
try:
num_inference_steps = int(raw_num_inference_steps)
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"Invalid num_inference_steps for request {request_id}") from exc
if num_inference_steps <= 0:
raise ValueError(f"num_inference_steps must be positive for request {request_id}")
cfg_range = extra_args.get("cfg_range")
if cfg_range is None:
cfg_range = request_info.get("cfg_range")
if cfg_range is None:
cfg_range = [0.0, 1.0]
if not isinstance(cfg_range, (list, tuple)) or len(cfg_range) != 2:
raise ValueError(f"cfg_range requires two values for request {request_id}")
try:
cfg_start, cfg_end = float(cfg_range[0]), float(cfg_range[1])
except (TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"cfg_range requires two values convertible to floats for request {request_id}") from exc
if not 0 <= cfg_start <= cfg_end <= 1:
raise ValueError(f"cfg_range must satisfy 0 <= start <= end <= 1 for request {request_id}")
generator = sampling.generator
if isinstance(generator, list) and len(generator) != 1:
raise ValueError(
f"MammothModa2 single-output request mode requires exactly one generator for request {request_id}"
)
return _MammothRequest(
request_id=request_id,
full_hidden_states=full_hidden_states,
full_token_ids=full_token_ids,
answer_start_index=answer_start_index,
height=height,
width=width,
text_guidance_scale=text_guidance_scale,
cfg_range=(cfg_start, cfg_end),
num_inference_steps=num_inference_steps,
seed=sampling.seed,
generator=generator,
)
def _split_ar_conditions(
@@ -202,42 +334,19 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
return text_cond, image_cond
@torch.inference_mode()
def forward(
self,
*,
inputs_embeds: torch.Tensor | None = None,
**kwargs: Any, # noqa: ARG002
) -> OmniOutput:
runtime_addi = kwargs.get("runtime_additional_information", None)
info = runtime_addi[0]
# Sampling knobs are declared in vllm_omni/model_extras/mammothmodal2_preview.py
# and routed via extra_body -> sampling_params.extra_args (surfaced here as
# ``sampling_extra_args``). Fall back to runtime_additional_information for the
# legacy bespoke-example path during the transition.
extra_args_list = kwargs.get("sampling_extra_args") or []
extra_args = extra_args_list[0] if extra_args_list else {}
text_guidance_scale = float(extra_args.get("text_guidance_scale", info["text_guidance_scale"][0]))
cfg_range_val = extra_args.get("cfg_range", info["cfg_range"])
cfg_range = float(cfg_range_val[0]), float(cfg_range_val[1])
num_inference_steps = int(extra_args.get("num_inference_steps", info["num_inference_steps"][0]))
negative_cond = info.get("negative_prompt_embeds")
negative_attention_mask = info.get("negative_prompt_attention_mask")
image_hw = info["image_height"][0], info["image_width"][0]
# Split the AR hidden states into text / image conditions. The token ids that
# drive the split are sourced from the model config (see _split_ar_conditions),
# formerly supplied by the bespoke example via additional_information. Legacy
# fallback: ar2dit may have already produced the split conditions.
if "text_prompt_embeds" in info:
text_cond = info["text_prompt_embeds"]
image_cond = info["image_prompt_embeds"]
else:
text_cond, image_cond = self._split_ar_conditions(
full_hidden_states=info["full_hidden_states"],
full_token_ids=info["full_token_ids"],
answer_start_index=int(info["answer_start_index"][0]),
def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:
request = self._parse_request(req)
text_cond, image_cond = self._split_ar_conditions(
full_hidden_states=request.full_hidden_states,
full_token_ids=request.full_token_ids,
answer_start_index=request.answer_start_index,
)
if image_cond.shape[0] == 0:
answer_token_ids = request.full_token_ids[request.answer_start_index :]
raise ValueError(
"MammothModa2 AR stage produced no visual-token hidden states for "
f"request {request.request_id}; the DiT stage requires at least one generated visual token. "
f"Generated token ids: {answer_token_ids[:32]}"
)
# Move to model device/dtype.
@@ -247,22 +356,6 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
else:
target_dtype = next(self.gen_transformer.parameters()).dtype
def _ensure_2d(x: torch.Tensor, name: str) -> torch.Tensor:
if x.ndim == 3 and x.shape[0] == 1:
x = x[0]
if x.ndim != 2:
raise ValueError(f"Expected {name} to be 2D [T,H], got shape={tuple(x.shape)}")
return x
text_cond = _ensure_2d(text_cond, "text_prompt_embeds")
image_cond = _ensure_2d(image_cond, "image_prompt_embeds")
if image_cond.shape[0] == 0:
answer_token_ids = info.get("full_token_ids", [])[int(info.get("answer_start_index", [0])[0]) :]
raise ValueError(
"MammothModa2 AR stage produced no visual-token hidden states; "
"the DiT stage requires at least one generated visual token. "
f"Generated token ids: {answer_token_ids[:32]}"
)
text_cond = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
image_cond = image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
@@ -301,63 +394,38 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
ar_image_embeds = image_embeds
ar_image_attention_mask = image_attention_mask
# Prepare negative prompt (for CFG). If none provided, fall back to unconditional.
# Empty unconditional prompt for classifier-free guidance.
negative_prompt_embeds = None
negative_prompt_attention_mask = None
if text_guidance_scale > 1.0:
if negative_cond is not None:
negative_cond = _ensure_2d(negative_cond, "negative_prompt_embeds")
negative_prompt_embeds = (
negative_cond.to(device=model_device, dtype=target_dtype, non_blocking=True)
.contiguous()
.unsqueeze(0)
)
if isinstance(negative_attention_mask, torch.Tensor):
neg_mask = negative_attention_mask
elif isinstance(negative_attention_mask, list):
neg_mask = torch.tensor(negative_attention_mask, dtype=torch.bool)
else:
neg_mask = None
if neg_mask is None:
negative_prompt_attention_mask = torch.ones(
(1, negative_prompt_embeds.shape[1]),
dtype=torch.bool,
device=negative_prompt_embeds.device,
)
else:
neg_mask = neg_mask.to(device=negative_prompt_embeds.device, dtype=torch.bool)
if neg_mask.ndim == 1:
neg_mask = neg_mask.unsqueeze(0)
negative_prompt_attention_mask = neg_mask
else:
hidden_size = int(prompt_embeds.shape[-1])
negative_prompt_embeds = torch.zeros(
(1, 0, hidden_size),
dtype=target_dtype,
device=prompt_embeds.device,
)
negative_prompt_attention_mask = torch.zeros(
(1, 0),
dtype=torch.bool,
device=prompt_embeds.device,
)
if request.text_guidance_scale > 1.0:
hidden_size = int(prompt_embeds.shape[-1])
negative_prompt_embeds = torch.zeros(
(1, 0, hidden_size),
dtype=target_dtype,
device=prompt_embeds.device,
)
negative_prompt_attention_mask = torch.zeros(
(1, 0),
dtype=torch.bool,
device=prompt_embeds.device,
)
generator = request.generator
if generator is None and request.seed is not None:
generator = torch.Generator(device=model_device).manual_seed(request.seed)
# Output image size (px), passed from stage input processor.
height, width = image_hw
if height <= 0 or width <= 0:
raise ValueError(f"Invalid image size: {height}x{width}")
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"Image size must be multiples of 16, got {height}x{width}")
height, width = request.height, request.width
vae_scale_factor = 16
latent_channels = int(self.gen_transformer.config.in_channels)
shape = (1, latent_channels, 2 * height // vae_scale_factor, 2 * width // vae_scale_factor)
latents = randn_tensor(shape, device=prompt_embeds.device, dtype=prompt_embeds.dtype)
latents = randn_tensor(shape, generator=generator, device=prompt_embeds.device, dtype=prompt_embeds.dtype)
scheduler = FlowMatchEulerDiscreteScheduler()
scheduler.set_timesteps(
num_inference_steps=num_inference_steps,
num_inference_steps=request.num_inference_steps,
device=prompt_embeds.device,
num_tokens=latents.shape[-2] * latents.shape[-1],
)
@@ -376,7 +444,9 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
ar_image_attention_mask=ar_image_attention_mask,
freqs_cis=self.gen_freqs_cis,
)
guidance_scale = text_guidance_scale if cfg_range[0] <= i / total_steps <= cfg_range[1] else 1.0
guidance_scale = (
request.text_guidance_scale if request.cfg_range[0] <= i / total_steps <= request.cfg_range[1] else 1.0
)
if guidance_scale > 1.0 and negative_prompt_embeds is not None:
model_pred_uncond = self.gen_transformer(
hidden_states=latents,
@@ -397,14 +467,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
latents = latents + self.gen_vae.config.shift_factor
image = self.gen_vae.decode(latents, return_dict=False)[0]
return OmniOutput(
text_hidden_states=inputs_embeds, # placeholder, not used by runner
multimodal_outputs=image,
intermediate_tensors=None,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: # noqa: ARG002
return None
return DiffusionOutput(output=image)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)