[Model] Support MammothModa2-Dev (#5411)
Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
13 files changed
+525
-64
No files matched your search
@@ -189,6 +189,7 @@ class SimpleQFormerImageRefiner(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
output_hidden_size: int | None = None,
|
||||
num_queries: int = 128,
|
||||
num_layers: int = 2,
|
||||
num_heads: int | None = None,
|
||||
@@ -196,6 +197,8 @@ class SimpleQFormerImageRefiner(nn.Module):
|
||||
norm_eps: float = 1e-5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
input_hidden_size = hidden_size
|
||||
hidden_size = output_hidden_size or hidden_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_queries = num_queries
|
||||
# ensure num_heads divides hidden_size
|
||||
@@ -203,8 +206,8 @@ class SimpleQFormerImageRefiner(nn.Module):
|
||||
num_heads = max(1, hidden_size // 128)
|
||||
self.num_heads = self._choose_valid_num_heads(hidden_size, num_heads)
|
||||
self.input_proj = nn.Sequential(
|
||||
Qwen2RMSNorm(hidden_size, eps=norm_eps),
|
||||
nn.Linear(hidden_size, hidden_size, bias=True),
|
||||
Qwen2RMSNorm(input_hidden_size, eps=norm_eps),
|
||||
nn.Linear(input_hidden_size, hidden_size, bias=True),
|
||||
)
|
||||
|
||||
# Learnable query embeddings
|
||||
@@ -662,11 +665,32 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
batch_size: int,
|
||||
height: int,
|
||||
width: int,
|
||||
ar_image_hidden_states: torch.Tensor | None = None,
|
||||
ar_image_attention_mask: torch.Tensor | None = None,
|
||||
):
|
||||
device = hidden_states.device
|
||||
p = self.config.patch_size
|
||||
|
||||
temb, text_hidden_states = self.time_caption_embed(timestep, text_hidden_states, hidden_states.dtype)
|
||||
image_embedder = getattr(self.time_caption_embed, "image_embedder", None)
|
||||
if image_embedder is not None and ar_image_hidden_states is not None:
|
||||
if ar_image_attention_mask is None:
|
||||
ar_image_attention_mask = torch.ones(
|
||||
ar_image_hidden_states.shape[:2],
|
||||
dtype=torch.bool,
|
||||
device=ar_image_hidden_states.device,
|
||||
)
|
||||
image_hidden_states = image_embedder(
|
||||
ar_image_hidden_states,
|
||||
~ar_image_attention_mask.bool(),
|
||||
)
|
||||
image_attention_mask = torch.ones(
|
||||
image_hidden_states.shape[:2],
|
||||
dtype=torch.bool,
|
||||
device=image_hidden_states.device,
|
||||
)
|
||||
text_hidden_states = torch.cat([text_hidden_states, image_hidden_states], dim=1)
|
||||
text_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
|
||||
img_tokens = rearrange(hidden_states, "b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=p, p2=p)
|
||||
img_tokens = self.x_embedder(img_tokens)
|
||||
@@ -699,6 +723,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
return (
|
||||
temb,
|
||||
text_hidden_states,
|
||||
text_attention_mask,
|
||||
img_tokens,
|
||||
img_mask,
|
||||
img_len,
|
||||
@@ -740,6 +765,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
freqs_cis: torch.Tensor,
|
||||
text_attention_mask: torch.Tensor,
|
||||
ref_image_hidden_states: list[list[torch.Tensor]] | None = None,
|
||||
ar_image_hidden_states: torch.Tensor | None = None,
|
||||
ar_image_attention_mask: torch.Tensor | None = None,
|
||||
return_dict: bool = False,
|
||||
) -> torch.Tensor:
|
||||
batch_size, height, width = self._validate_inputs(
|
||||
@@ -749,6 +776,7 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
(
|
||||
temb,
|
||||
text_hidden_states,
|
||||
text_attention_mask,
|
||||
img_tokens,
|
||||
img_mask,
|
||||
img_len,
|
||||
@@ -766,6 +794,8 @@ class Transformer2DModel(ModelMixin, ConfigMixin):
|
||||
batch_size,
|
||||
height,
|
||||
width,
|
||||
ar_image_hidden_states,
|
||||
ar_image_attention_mask,
|
||||
)
|
||||
|
||||
text_hidden_states, img_tokens = self._apply_refiners(
|
||||
|
||||
@@ -42,6 +42,7 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"llm_model.": None,
|
||||
"gen_tokenizer.": None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -77,11 +78,23 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
)
|
||||
self._reinit_caption_embedder(llm_hidden_size)
|
||||
|
||||
# Optional: image condition refiner (Q-Former)
|
||||
if self.config.gen_image_condition_refiner_config is not None:
|
||||
# Optional image condition Q-Former. Preview stores it as a standalone
|
||||
# module; Dev stores it under the DiT timestep/caption embedder.
|
||||
llm_model_type = getattr(self.config.llm_config, "model_type", "")
|
||||
refiner_config = self.config.gen_image_condition_refiner_config
|
||||
if refiner_config is not None and llm_model_type == "mammothmoda2_qwen3_vl":
|
||||
dit_hidden_size = int(self.gen_transformer.hidden_size)
|
||||
self.gen_transformer.time_caption_embed.image_embedder = SimpleQFormerImageRefiner(
|
||||
hidden_size=llm_hidden_size,
|
||||
output_hidden_size=dit_hidden_size,
|
||||
num_heads=max(1, dit_hidden_size // 128),
|
||||
**refiner_config,
|
||||
)
|
||||
self.gen_image_condition_refiner = None
|
||||
elif refiner_config is not None:
|
||||
self.gen_image_condition_refiner = SimpleQFormerImageRefiner(
|
||||
hidden_size=llm_hidden_size,
|
||||
**self.config.gen_image_condition_refiner_config,
|
||||
**refiner_config,
|
||||
)
|
||||
else:
|
||||
self.gen_image_condition_refiner = None
|
||||
@@ -237,6 +250,13 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
|
||||
text_cond = _ensure_2d(text_cond, "text_prompt_embeds")
|
||||
image_cond = _ensure_2d(image_cond, "image_prompt_embeds")
|
||||
if image_cond.shape[0] == 0:
|
||||
answer_token_ids = info.get("full_token_ids", [])[int(info.get("answer_start_index", [0])[0]) :]
|
||||
raise ValueError(
|
||||
"MammothModa2 AR stage produced no visual-token hidden states; "
|
||||
"the DiT stage requires at least one generated visual token. "
|
||||
f"Generated token ids: {answer_token_ids[:32]}"
|
||||
)
|
||||
text_cond = text_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
|
||||
image_cond = image_cond.to(device=model_device, dtype=target_dtype, non_blocking=True).contiguous()
|
||||
|
||||
@@ -263,8 +283,17 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
device=image_embeds.device,
|
||||
)
|
||||
|
||||
prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1)
|
||||
prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
nested_image_embedder = getattr(self.gen_transformer.time_caption_embed, "image_embedder", None)
|
||||
if nested_image_embedder is None:
|
||||
prompt_embeds = torch.cat([text_embeds, image_embeds], dim=1)
|
||||
prompt_attention_mask = torch.cat([text_attention_mask, image_attention_mask], dim=1)
|
||||
ar_image_embeds = None
|
||||
ar_image_attention_mask = None
|
||||
else:
|
||||
prompt_embeds = text_embeds
|
||||
prompt_attention_mask = text_attention_mask
|
||||
ar_image_embeds = image_embeds
|
||||
ar_image_attention_mask = image_attention_mask
|
||||
|
||||
# Prepare negative prompt (for CFG). If none provided, fall back to unconditional.
|
||||
negative_prompt_embeds = None
|
||||
@@ -337,6 +366,8 @@ class MammothModa2DiTPipeline(nn.Module, SupportsComponentDiscovery):
|
||||
text_hidden_states=prompt_embeds,
|
||||
text_attention_mask=prompt_attention_mask,
|
||||
ref_image_hidden_states=None,
|
||||
ar_image_hidden_states=ar_image_embeds,
|
||||
ar_image_attention_mask=ar_image_attention_mask,
|
||||
freqs_cis=self.gen_freqs_cis,
|
||||
)
|
||||
guidance_scale = text_guidance_scale if cfg_range[0] <= i / total_steps <= cfg_range[1] else 1.0
|
||||
|
||||
Reference in new issue
Block a user