[Model] Support MammothModa2-Dev (#5411)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏 authored and GitHub committed 2026-07-27 17:01:54 +08:00
1 parent 5b230541c9
commit 88b7ca2ffa
13 files changed
+525 -64

No files matched your search

@@ -189,6 +189,7 @@ class SimpleQFormerImageRefiner(nn.Module):
def __init__(
self,
hidden_size: int,
output_hidden_size: int | None = None,
num_queries: int = 128,
num_layers: int = 2,
num_heads: int | None = None,
@@ -196,6 +197,8 @@ class SimpleQFormerImageRefiner(nn.Module):
norm_eps: float = 1e-5,
) -> None:
super().__init__()
input_hidden_size = hidden_size
hidden_size = output_hidden_size or hidden_size
self.hidden_size = hidden_size
self.num_queries = num_queries
# ensure num_heads divides hidden_size
@@ -203,8 +206,8 @@ class SimpleQFormerImageRefiner(nn.Module):
num_heads = max(1, hidden_size // 128)
self.num_heads = self._choose_valid_num_heads(hidden_size, num_heads)
self.input_proj = nn.Sequential(
Qwen2RMSNorm(hidden_size, eps=norm_eps),
nn.Linear(hidden_size, hidden_size, bias=True),
Qwen2RMSNorm(input_hidden_size, eps=norm_eps),
nn.Linear(input_hidden_size, hidden_size, bias=True),
)
# Learnable query embeddings
@@ -662,11 +665,32 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
batch_size: int,
height: int,
width: int,
ar_image_hidden_states: torch.Tensor | None = None,
ar_image_attention_mask: torch.Tensor | None = None,
):
device = hidden_states.device
p = self.config.patch_size
temb, text_hidden_states = self.time_caption_embed(timestep, text_hidden_states, hidden_states.dtype)
image_embedder = getattr(self.time_caption_embed, "image_embedder", None)
if image_embedder is not None and ar_image_hidden_states is not None:
if ar_image_attention_mask is None:
ar_image_attention_mask = torch.ones(
ar_image_hidden_states.shape[:2],
dtype=torch.bool,
device=ar_image_hidden_states.device,
)
image_hidden_states = image_embedder(
ar_image_hidden_states,
~ar_image_attention_mask.bool(),
)
image_attention_mask = torch.ones(
image_hidden_states.shape[:2],
dtype=torch.bool,
device=image_hidden_states.device,
)
text_hidden_states = torch.cat([text_hidden_states, image_hidden_states], dim=1)
text_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
img_tokens = rearrange(hidden_states, "b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=p, p2=p)
img_tokens = self.x_embedder(img_tokens)
@@ -699,6 +723,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
return (
temb,
text_hidden_states,
text_attention_mask,
img_tokens,
img_mask,
img_len,
@@ -740,6 +765,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
freqs_cis: torch.Tensor,
text_attention_mask: torch.Tensor,
ref_image_hidden_states: list[list[torch.Tensor]] | None = None,
ar_image_hidden_states: torch.Tensor | None = None,
ar_image_attention_mask: torch.Tensor | None = None,
return_dict: bool = False,
) -> torch.Tensor:
batch_size, height, width = self._validate_inputs(
@@ -749,6 +776,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
(
temb,
text_hidden_states,
text_attention_mask,
img_tokens,
img_mask,
img_len,
@@ -766,6 +794,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
batch_size,
height,
width,
ar_image_hidden_states,
ar_image_attention_mask,
)
text_hidden_states, img_tokens = self._apply_refiners(
@@ -42,6 +42,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"llm_model.": None,
"gen_tokenizer.": None,
}
)
@@ -77,11 +78,23 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
)
self._reinit_caption_embedder(llm_hidden_size)
# Optional: image condition refiner (Q-Former)
if self.config.gen_image_condition_refiner_config is not None:
# Optional image condition Q-Former. Preview stores it as a standalone
# module; Dev stores it under the DiT timestep/caption embedder.
llm_model_type = getattr(self.config.llm_config, "model_type", "")
refiner_config = self.config.gen_image_condition_refiner_config
if refiner_config is not None and llm_model_type == "mammothmoda2_qwen3_vl":
dit_hidden_size = int(self.gen_transformer.hidden_size)
self.gen_transformer.time_caption_embed.image_embedder = SimpleQFormerImageRefiner(
hidden_size=llm_hidden_size,
output_hidden_size=dit_hidden_size,
num_heads=max(1, dit_hidden_size // 128),
**refiner_config,
)
self.gen_image_condition_refiner = None
elif refiner_config is not None:
self.gen_image_condition_refiner = SimpleQFormerImageRefiner(
hidden_size=llm_hidden_size,
**self.config.gen_image_condition_refiner_config,
**refiner_config,
)
else:
self.gen_image_condition_refiner = None
@@ -237,6 +250,13 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
text_cond = _ensure_2d(text_cond, "text_prompt_embeds")
image_cond = _ensure_2d(image_cond, "image_prompt_embeds")
if image_cond.shape[0] == 0:
answer_token_ids = info.get("full_token_ids", [])[int(info.get("answer_start_index", [0])[0]) :]
raise ValueError(
"MammothModa2 AR stage produced no visual-token hidden states; "
"the DiT stage requires at least one generated visual token. "
f"Generated token ids: {answer_token_ids[:32]}"
)
text_cond = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
image_cond = image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
@@ -263,8 +283,17 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
device=image_embeds.device,
)
prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1)
prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
nested_image_embedder = getattr(self.gen_transformer.time_caption_embed, "image_embedder", None)
if nested_image_embedder is None:
prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1)
prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
ar_image_embeds = None
ar_image_attention_mask = None
else:
prompt_embeds = text_embeds
prompt_attention_mask = text_attention_mask
ar_image_embeds = image_embeds
ar_image_attention_mask = image_attention_mask
# Prepare negative prompt (for CFG). If none provided, fall back to unconditional.
negative_prompt_embeds = None
@@ -337,6 +366,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
text_hidden_states=prompt_embeds,
text_attention_mask=prompt_attention_mask,
ref_image_hidden_states=None,
ar_image_hidden_states=ar_image_embeds,
ar_image_attention_mask=ar_image_attention_mask,
freqs_cis=self.gen_freqs_cis,
)
guidance_scale = text_guidance_scale if cfg_range[0] <= i / total_steps <= cfg_range[1] else 1.0