(Phase 2)Add ModelOpt mixed FP8/NVFP4 support for image generation (#3570)

Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
This commit is contained in:
baonudesifeizhai authored and GitHub committed 2026-05-22 10:39:10 +08:00
1 parent 7d8816b46b
commit 076c34abfb
16 files changed
+768 -68

No files matched your search

@@ -3,6 +3,7 @@
# Adapted from Helios (https://github.com/BestWishYsh/Helios)
import math
from collections import OrderedDict
from collections.abc import Iterable
from functools import lru_cache
from typing import TYPE_CHECKING, Any
@@ -447,21 +448,32 @@ class HeliosCrossAttention(nn.Module):
causal=False,
)
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
) -> torch.Tensor:
query = self.to_q(hidden_states)
query = self.norm_q(query)
def project_kv(self, encoder_hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
key = self.to_k(encoder_hidden_states)
value = self.to_v(encoder_hidden_states)
key = self.norm_k(key)
query = query.unflatten(2, (self.num_heads, self.head_dim))
key = key.unflatten(2, (self.num_heads, self.head_dim))
value = value.unflatten(2, (self.num_heads, self.head_dim))
return key, value
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None = None,
encoder_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
query = self.to_q(hidden_states)
query = self.norm_q(query)
if encoder_key_value is None:
if encoder_hidden_states is None:
raise ValueError("encoder_hidden_states is required when encoder_key_value is not provided.")
key, value = self.project_kv(encoder_hidden_states)
else:
key, value = encoder_key_value
query = query.unflatten(2, (self.num_heads, self.head_dim))
hidden_states = self.attn(query, key, value)
hidden_states = hidden_states.flatten(2, 3)
@@ -526,6 +538,7 @@ class HeliosTransformerBlock(nn.Module):
temb: torch.Tensor,
rotary_emb: torch.Tensor,
original_context_length: int | None = None,
cross_attn_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
if temb.ndim == 4:
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = (
@@ -556,12 +569,20 @@ class HeliosTransformerBlock(nn.Module):
hidden_states[:, history_seq_len:],
)
norm_hidden_states = self.norm2(current_hidden_states.float()).type_as(current_hidden_states)
attn_output = self.attn2(norm_hidden_states, encoder_hidden_states)
attn_output = self.attn2(
norm_hidden_states,
encoder_hidden_states,
encoder_key_value=cross_attn_key_value,
)
current_hidden_states = current_hidden_states + attn_output
hidden_states = torch.cat([history_hidden_states, current_hidden_states], dim=1)
else:
norm_hidden_states = self.norm2(hidden_states.float()).type_as(hidden_states)
attn_output = self.attn2(norm_hidden_states, encoder_hidden_states)
attn_output = self.attn2(
norm_hidden_states,
encoder_hidden_states,
encoder_key_value=cross_attn_key_value,
)
hidden_states = hidden_states + attn_output
# 3. Feed-forward
@@ -723,11 +744,80 @@ class HeliosTransformer3DModel(nn.Module):
# 5. Output norm & projection
self.norm_out = HeliosOutputNorm(inner_dim, eps)
self.proj_out = nn.Linear(inner_dim, out_channels * math.prod(patch_size))
self._projected_encoder_cache: OrderedDict[tuple, torch.Tensor] = OrderedDict()
self._cross_attn_kv_cache: OrderedDict[tuple, list[tuple[torch.Tensor, torch.Tensor]]] = OrderedDict()
self._cross_attn_cache_size = 2
@property
def dtype(self) -> torch.dtype:
return next(self.parameters()).dtype
@staticmethod
def _tensor_cache_key(tensor: torch.Tensor) -> tuple:
try:
version = tensor._version
except RuntimeError:
version = None
return (
tensor.data_ptr(),
tuple(tensor.shape),
tuple(tensor.stride()),
tensor.dtype,
tensor.device.type,
tensor.device.index,
version,
)
@staticmethod
def _get_from_lru(cache: OrderedDict, key: tuple):
value = cache.get(key)
if value is not None:
cache.move_to_end(key)
return value
def _put_lru(self, cache: OrderedDict, key: tuple, value) -> None:
cache[key] = value
cache.move_to_end(key)
while len(cache) > self._cross_attn_cache_size:
cache.popitem(last=False)
def clear_cross_attention_cache(self) -> None:
self._projected_encoder_cache.clear()
self._cross_attn_kv_cache.clear()
def _cache_enabled(self) -> bool:
return not self.training and not torch.is_grad_enabled() and not torch.compiler.is_compiling()
def _project_encoder_hidden_states(self, encoder_hidden_states: torch.Tensor) -> torch.Tensor:
if not self._cache_enabled():
return self.condition_embedder.text_embedder(encoder_hidden_states)
cache_key = self._tensor_cache_key(encoder_hidden_states)
cached = self._get_from_lru(self._projected_encoder_cache, cache_key)
if cached is not None:
return cached
projected = self.condition_embedder.text_embedder(encoder_hidden_states)
self._put_lru(self._projected_encoder_cache, cache_key, projected)
return projected
def _get_cross_attn_key_values(
self,
encoder_hidden_states: torch.Tensor,
) -> list[tuple[torch.Tensor, torch.Tensor]] | None:
if not self._cache_enabled():
return None
cache_key = self._tensor_cache_key(encoder_hidden_states)
cached = self._get_from_lru(self._cross_attn_kv_cache, cache_key)
if cached is not None:
return cached
key_values = [block.attn2.project_kv(encoder_hidden_states) for block in self.blocks]
self._put_lru(self._cross_attn_kv_cache, cache_key, key_values)
return key_values
def forward(
self,
hidden_states: torch.Tensor,
@@ -836,7 +926,12 @@ class HeliosTransformer3DModel(nn.Module):
.expand(batch_size, -1, history_context_length, -1)
)
temb, timestep_proj, encoder_hidden_states = self.condition_embedder(timestep, encoder_hidden_states)
temb, timestep_proj, _ = self.condition_embedder(
timestep,
encoder_hidden_states,
is_return_encoder_hidden_states=False,
)
encoder_hidden_states = self._project_encoder_hidden_states(encoder_hidden_states)
timestep_proj = timestep_proj.unflatten(-1, (6, -1))
if indices_hidden_states is not None and not self.zero_history_timestep:
@@ -857,14 +952,17 @@ class HeliosTransformer3DModel(nn.Module):
hidden_states = hidden_states.contiguous()
encoder_hidden_states = encoder_hidden_states.contiguous()
rotary_emb = rotary_emb.contiguous()
cross_attn_key_values = self._get_cross_attn_key_values(encoder_hidden_states)
for block in self.blocks:
for block_idx, block in enumerate(self.blocks):
cross_attn_key_value = None if cross_attn_key_values is None else cross_attn_key_values[block_idx]
hidden_states = block(
hidden_states,
encoder_hidden_states,
timestep_proj,
rotary_emb,
original_context_length,
cross_attn_key_value,
)
# 7. Output normalization
@@ -2073,7 +2073,7 @@ class HunyuanImage3Model(nn.Module):
return False
def is_scalar_quant_scale(name: str, tensor: torch.Tensor) -> bool:
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale"))
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale", ".weight_scale_2"))
def load_split_param(
name: str,
@@ -49,6 +49,10 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding
logger = init_logger(__name__)
def _join_prefix(prefix: str, suffix: str) -> str:
return f"{prefix}.{suffix}" if prefix else suffix
class ImageRopePrepare(nn.Module):
"""Prepares image hidden_states and RoPE embeddings for sequence parallel.
@@ -471,7 +475,12 @@ class FeedForward(nn.Module):
layers: list[nn.Module] = [
ColumnParallelApproxGELU(
dim, inner_dim, approximate="tanh", bias=bias, quant_config=quant_config, prefix=prefix
dim,
inner_dim,
approximate="tanh",
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "net.0.proj"),
),
nn.Identity(), # placeholder for weight loading
RowParallelLinear(
@@ -480,7 +489,7 @@ class FeedForward(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=prefix,
prefix=_join_prefix(prefix, "net.2"),
),
]
@@ -507,6 +516,7 @@ class QwenImageCrossAttention(nn.Module):
context_pre_only: bool = False,
out_dim: int | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
assert dim % num_heads == 0
@@ -523,7 +533,7 @@ class QwenImageCrossAttention(nn.Module):
head_size=self.head_dim,
total_num_heads=num_heads,
quant_config=quant_config,
prefix="to_qkv",
prefix=_join_prefix(prefix, "to_qkv"),
)
self.query_num_heads = self.to_qkv.num_heads
self.kv_num_heads = self.to_qkv.num_kv_heads
@@ -539,7 +549,7 @@ class QwenImageCrossAttention(nn.Module):
head_size=head_dim,
total_num_heads=num_heads,
quant_config=quant_config,
prefix="add_kv_proj",
prefix=_join_prefix(prefix, "add_kv_proj"),
)
self.add_query_num_heads = self.add_kv_proj.num_heads
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
@@ -552,7 +562,7 @@ class QwenImageCrossAttention(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix="to_add_out",
prefix=_join_prefix(prefix, "to_add_out"),
)
assert not pre_only
@@ -563,7 +573,7 @@ class QwenImageCrossAttention(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix="to_out",
prefix=_join_prefix(prefix, "to_out"),
)
self.norm_added_q = RMSNorm(head_dim, eps=eps)
@@ -697,6 +707,7 @@ class QwenImageTransformerBlock(nn.Module):
eps: float = 1e-6,
zero_cond_t: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -716,7 +727,7 @@ class QwenImageTransformerBlock(nn.Module):
bias=True,
return_bias=False,
quant_config=None,
prefix="img_mod.1",
prefix=_join_prefix(prefix, "img_mod.1"),
),
)
self.img_norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
@@ -727,9 +738,15 @@ class QwenImageTransformerBlock(nn.Module):
context_pre_only=False,
head_dim=attention_head_dim,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attn"),
)
self.img_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
self.img_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="img_mlp")
self.img_mlp = FeedForward(
dim=dim,
dim_out=dim,
quant_config=quant_config,
prefix=_join_prefix(prefix, "img_mlp"),
)
# Text processing modules.
self.txt_mod = nn.Sequential(
@@ -740,13 +757,18 @@ class QwenImageTransformerBlock(nn.Module):
bias=True,
return_bias=False,
quant_config=None,
prefix="txt_mod.1",
prefix=_join_prefix(prefix, "txt_mod.1"),
),
)
self.txt_norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
# Text doesn't need separate attention - it's handled by img_attn joint computation
self.txt_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="txt_mlp")
self.txt_mlp = FeedForward(
dim=dim,
dim_out=dim,
quant_config=quant_config,
prefix=_join_prefix(prefix, "txt_mlp"),
)
self.zero_cond_t = zero_cond_t
@@ -996,8 +1018,9 @@ class QwenImageTransformer2DModel(CachedTransformer):
attention_head_dim=attention_head_dim,
zero_cond_t=zero_cond_t,
quant_config=quant_config,
prefix=f"transformer_blocks.{i}",
)
for _ in range(num_layers)
for i in range(num_layers)
]
)