(Phase 2)Add ModelOpt mixed FP8/NVFP4 support for image generation (#3570)
Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
This commit is contained in:
1 parent
7d8816b46b
commit
076c34abfb
16 files changed
+768
-68
No files matched your search
@@ -3,6 +3,7 @@
|
||||
# Adapted from Helios (https://github.com/BestWishYsh/Helios)
|
||||
|
||||
import math
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -447,21 +448,32 @@ class HeliosCrossAttention(nn.Module):
|
||||
causal=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
query = self.to_q(hidden_states)
|
||||
query = self.norm_q(query)
|
||||
|
||||
def project_kv(self, encoder_hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
key = self.to_k(encoder_hidden_states)
|
||||
value = self.to_v(encoder_hidden_states)
|
||||
key = self.norm_k(key)
|
||||
|
||||
query = query.unflatten(2, (self.num_heads, self.head_dim))
|
||||
key = key.unflatten(2, (self.num_heads, self.head_dim))
|
||||
value = value.unflatten(2, (self.num_heads, self.head_dim))
|
||||
return key, value
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None = None,
|
||||
encoder_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
query = self.to_q(hidden_states)
|
||||
query = self.norm_q(query)
|
||||
|
||||
if encoder_key_value is None:
|
||||
if encoder_hidden_states is None:
|
||||
raise ValueError("encoder_hidden_states is required when encoder_key_value is not provided.")
|
||||
key, value = self.project_kv(encoder_hidden_states)
|
||||
else:
|
||||
key, value = encoder_key_value
|
||||
|
||||
query = query.unflatten(2, (self.num_heads, self.head_dim))
|
||||
|
||||
hidden_states = self.attn(query, key, value)
|
||||
hidden_states = hidden_states.flatten(2, 3)
|
||||
@@ -526,6 +538,7 @@ class HeliosTransformerBlock(nn.Module):
|
||||
temb: torch.Tensor,
|
||||
rotary_emb: torch.Tensor,
|
||||
original_context_length: int | None = None,
|
||||
cross_attn_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
if temb.ndim == 4:
|
||||
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = (
|
||||
@@ -556,12 +569,20 @@ class HeliosTransformerBlock(nn.Module):
|
||||
hidden_states[:, history_seq_len:],
|
||||
)
|
||||
norm_hidden_states = self.norm2(current_hidden_states.float()).type_as(current_hidden_states)
|
||||
attn_output = self.attn2(norm_hidden_states, encoder_hidden_states)
|
||||
attn_output = self.attn2(
|
||||
norm_hidden_states,
|
||||
encoder_hidden_states,
|
||||
encoder_key_value=cross_attn_key_value,
|
||||
)
|
||||
current_hidden_states = current_hidden_states + attn_output
|
||||
hidden_states = torch.cat([history_hidden_states, current_hidden_states], dim=1)
|
||||
else:
|
||||
norm_hidden_states = self.norm2(hidden_states.float()).type_as(hidden_states)
|
||||
attn_output = self.attn2(norm_hidden_states, encoder_hidden_states)
|
||||
attn_output = self.attn2(
|
||||
norm_hidden_states,
|
||||
encoder_hidden_states,
|
||||
encoder_key_value=cross_attn_key_value,
|
||||
)
|
||||
hidden_states = hidden_states + attn_output
|
||||
|
||||
# 3. Feed-forward
|
||||
@@ -723,11 +744,80 @@ class HeliosTransformer3DModel(nn.Module):
|
||||
# 5. Output norm & projection
|
||||
self.norm_out = HeliosOutputNorm(inner_dim, eps)
|
||||
self.proj_out = nn.Linear(inner_dim, out_channels * math.prod(patch_size))
|
||||
self._projected_encoder_cache: OrderedDict[tuple, torch.Tensor] = OrderedDict()
|
||||
self._cross_attn_kv_cache: OrderedDict[tuple, list[tuple[torch.Tensor, torch.Tensor]]] = OrderedDict()
|
||||
self._cross_attn_cache_size = 2
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return next(self.parameters()).dtype
|
||||
|
||||
@staticmethod
|
||||
def _tensor_cache_key(tensor: torch.Tensor) -> tuple:
|
||||
try:
|
||||
version = tensor._version
|
||||
except RuntimeError:
|
||||
version = None
|
||||
|
||||
return (
|
||||
tensor.data_ptr(),
|
||||
tuple(tensor.shape),
|
||||
tuple(tensor.stride()),
|
||||
tensor.dtype,
|
||||
tensor.device.type,
|
||||
tensor.device.index,
|
||||
version,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_from_lru(cache: OrderedDict, key: tuple):
|
||||
value = cache.get(key)
|
||||
if value is not None:
|
||||
cache.move_to_end(key)
|
||||
return value
|
||||
|
||||
def _put_lru(self, cache: OrderedDict, key: tuple, value) -> None:
|
||||
cache[key] = value
|
||||
cache.move_to_end(key)
|
||||
while len(cache) > self._cross_attn_cache_size:
|
||||
cache.popitem(last=False)
|
||||
|
||||
def clear_cross_attention_cache(self) -> None:
|
||||
self._projected_encoder_cache.clear()
|
||||
self._cross_attn_kv_cache.clear()
|
||||
|
||||
def _cache_enabled(self) -> bool:
|
||||
return not self.training and not torch.is_grad_enabled() and not torch.compiler.is_compiling()
|
||||
|
||||
def _project_encoder_hidden_states(self, encoder_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if not self._cache_enabled():
|
||||
return self.condition_embedder.text_embedder(encoder_hidden_states)
|
||||
|
||||
cache_key = self._tensor_cache_key(encoder_hidden_states)
|
||||
cached = self._get_from_lru(self._projected_encoder_cache, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
projected = self.condition_embedder.text_embedder(encoder_hidden_states)
|
||||
self._put_lru(self._projected_encoder_cache, cache_key, projected)
|
||||
return projected
|
||||
|
||||
def _get_cross_attn_key_values(
|
||||
self,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
) -> list[tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
if not self._cache_enabled():
|
||||
return None
|
||||
|
||||
cache_key = self._tensor_cache_key(encoder_hidden_states)
|
||||
cached = self._get_from_lru(self._cross_attn_kv_cache, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
key_values = [block.attn2.project_kv(encoder_hidden_states) for block in self.blocks]
|
||||
self._put_lru(self._cross_attn_kv_cache, cache_key, key_values)
|
||||
return key_values
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -836,7 +926,12 @@ class HeliosTransformer3DModel(nn.Module):
|
||||
.expand(batch_size, -1, history_context_length, -1)
|
||||
)
|
||||
|
||||
temb, timestep_proj, encoder_hidden_states = self.condition_embedder(timestep, encoder_hidden_states)
|
||||
temb, timestep_proj, _ = self.condition_embedder(
|
||||
timestep,
|
||||
encoder_hidden_states,
|
||||
is_return_encoder_hidden_states=False,
|
||||
)
|
||||
encoder_hidden_states = self._project_encoder_hidden_states(encoder_hidden_states)
|
||||
timestep_proj = timestep_proj.unflatten(-1, (6, -1))
|
||||
|
||||
if indices_hidden_states is not None and not self.zero_history_timestep:
|
||||
@@ -857,14 +952,17 @@ class HeliosTransformer3DModel(nn.Module):
|
||||
hidden_states = hidden_states.contiguous()
|
||||
encoder_hidden_states = encoder_hidden_states.contiguous()
|
||||
rotary_emb = rotary_emb.contiguous()
|
||||
cross_attn_key_values = self._get_cross_attn_key_values(encoder_hidden_states)
|
||||
|
||||
for block in self.blocks:
|
||||
for block_idx, block in enumerate(self.blocks):
|
||||
cross_attn_key_value = None if cross_attn_key_values is None else cross_attn_key_values[block_idx]
|
||||
hidden_states = block(
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
timestep_proj,
|
||||
rotary_emb,
|
||||
original_context_length,
|
||||
cross_attn_key_value,
|
||||
)
|
||||
|
||||
# 7. Output normalization
|
||||
|
||||
@@ -2073,7 +2073,7 @@ class HunyuanImage3Model(nn.Module):
|
||||
return False
|
||||
|
||||
def is_scalar_quant_scale(name: str, tensor: torch.Tensor) -> bool:
|
||||
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale"))
|
||||
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale", ".weight_scale_2"))
|
||||
|
||||
def load_split_param(
|
||||
name: str,
|
||||
|
||||
@@ -49,6 +49,10 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, suffix: str) -> str:
|
||||
return f"{prefix}.{suffix}" if prefix else suffix
|
||||
|
||||
|
||||
class ImageRopePrepare(nn.Module):
|
||||
"""Prepares image hidden_states and RoPE embeddings for sequence parallel.
|
||||
|
||||
@@ -471,7 +475,12 @@ class FeedForward(nn.Module):
|
||||
|
||||
layers: list[nn.Module] = [
|
||||
ColumnParallelApproxGELU(
|
||||
dim, inner_dim, approximate="tanh", bias=bias, quant_config=quant_config, prefix=prefix
|
||||
dim,
|
||||
inner_dim,
|
||||
approximate="tanh",
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "net.0.proj"),
|
||||
),
|
||||
nn.Identity(), # placeholder for weight loading
|
||||
RowParallelLinear(
|
||||
@@ -480,7 +489,7 @@ class FeedForward(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix=_join_prefix(prefix, "net.2"),
|
||||
),
|
||||
]
|
||||
|
||||
@@ -507,6 +516,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
context_pre_only: bool = False,
|
||||
out_dim: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert dim % num_heads == 0
|
||||
@@ -523,7 +533,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=num_heads,
|
||||
quant_config=quant_config,
|
||||
prefix="to_qkv",
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -539,7 +549,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
quant_config=quant_config,
|
||||
prefix="add_kv_proj",
|
||||
prefix=_join_prefix(prefix, "add_kv_proj"),
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -552,7 +562,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_add_out",
|
||||
prefix=_join_prefix(prefix, "to_add_out"),
|
||||
)
|
||||
|
||||
assert not pre_only
|
||||
@@ -563,7 +573,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_out",
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
|
||||
self.norm_added_q = RMSNorm(head_dim, eps=eps)
|
||||
@@ -697,6 +707,7 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
eps: float = 1e-6,
|
||||
zero_cond_t: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -716,7 +727,7 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
prefix="img_mod.1",
|
||||
prefix=_join_prefix(prefix, "img_mod.1"),
|
||||
),
|
||||
)
|
||||
self.img_norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -727,9 +738,15 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
context_pre_only=False,
|
||||
head_dim=attention_head_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
self.img_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.img_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="img_mlp")
|
||||
self.img_mlp = FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "img_mlp"),
|
||||
)
|
||||
|
||||
# Text processing modules.
|
||||
self.txt_mod = nn.Sequential(
|
||||
@@ -740,13 +757,18 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
prefix="txt_mod.1",
|
||||
prefix=_join_prefix(prefix, "txt_mod.1"),
|
||||
),
|
||||
)
|
||||
self.txt_norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
# Text doesn't need separate attention - it's handled by img_attn joint computation
|
||||
self.txt_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="txt_mlp")
|
||||
self.txt_mlp = FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "txt_mlp"),
|
||||
)
|
||||
|
||||
self.zero_cond_t = zero_cond_t
|
||||
|
||||
@@ -996,8 +1018,9 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
attention_head_dim=attention_head_dim,
|
||||
zero_cond_t=zero_cond_t,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user