(Phase 1)Add ModelOpt FP8 auto-detect support for diffusion checkpoints #2709 (#2913)

Signed-off-by: roG0d <rodgarcas98@gmail.com>
Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
Co-authored-by: roG0d <rodgarcas98@gmail.com>
This commit is contained in:
baonudesifeizhaiandroG0d authored and GitHub committed 2026-05-09 16:55:05 +08:00
1 parent 40a07e0d80
commit c4a0990044
26 files changed
+1214 -176

No files matched your search

@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from types import SimpleNamespace
from typing import Any
from typing import TYPE_CHECKING, Any
import torch
from diffusers.models.embeddings import (
@@ -27,6 +27,13 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def _join_prefix(prefix: str, name: str) -> str:
return f"{prefix}.{name}" if prefix else name
class Flux2SwiGLU(nn.Module):
"""SwiGLU activation used by Flux2."""
@@ -48,6 +55,8 @@ class Flux2FeedForward(nn.Module):
mult: float = 3.0,
inner_dim: int | None = None,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
if inner_dim is None:
@@ -59,6 +68,8 @@ class Flux2FeedForward(nn.Module):
[inner_dim, inner_dim],
bias=bias,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "linear_in"),
)
self.act_fn = Flux2SwiGLU()
self.linear_out = RowParallelLinear(
@@ -67,6 +78,8 @@ class Flux2FeedForward(nn.Module):
bias=bias,
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "linear_out"),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -89,6 +102,8 @@ class Flux2Attention(nn.Module):
eps: float = 1e-5,
out_dim: int = None,
elementwise_affine: bool = True,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.head_dim = dim_head
@@ -104,6 +119,8 @@ class Flux2Attention(nn.Module):
head_size=self.head_dim,
total_num_heads=self.heads,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_qkv"),
)
self.query_num_heads = self.to_qkv.num_heads
self.kv_num_heads = self.to_qkv.num_kv_heads
@@ -119,6 +136,8 @@ class Flux2Attention(nn.Module):
bias=out_bias,
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_out.0"),
),
nn.Dropout(dropout),
]
@@ -132,6 +151,8 @@ class Flux2Attention(nn.Module):
head_size=self.head_dim,
total_num_heads=self.heads,
bias=added_proj_bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "add_kv_proj"),
)
self.add_query_num_heads = self.add_kv_proj.num_heads
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
@@ -141,6 +162,8 @@ class Flux2Attention(nn.Module):
bias=out_bias,
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_add_out"),
)
self.rope = RotaryEmbedding(is_neox_style=False)
@@ -238,6 +261,8 @@ class Flux2ParallelSelfAttention(nn.Module):
elementwise_affine: bool = True,
mlp_ratio: float = 4.0,
mlp_mult_factor: int = 2,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.head_dim = dim_head
@@ -256,6 +281,8 @@ class Flux2ParallelSelfAttention(nn.Module):
self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor,
bias=bias,
gather_output=True,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
)
self.mlp_act_fn = Flux2SwiGLU()
@@ -267,6 +294,8 @@ class Flux2ParallelSelfAttention(nn.Module):
self.out_dim,
bias=out_bias,
gather_output=True,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_out"),
)
self.rope = RotaryEmbedding(is_neox_style=False)
self.attn = Attention(
@@ -329,6 +358,8 @@ class Flux2SingleTransformerBlock(nn.Module):
mlp_ratio: float = 3.0,
eps: float = 1e-6,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
@@ -342,6 +373,8 @@ class Flux2SingleTransformerBlock(nn.Module):
eps=eps,
mlp_ratio=mlp_ratio,
mlp_mult_factor=2,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attn"),
)
def forward(
@@ -389,6 +422,8 @@ class Flux2TransformerBlock(nn.Module):
mlp_ratio: float = 3.0,
eps: float = 1e-6,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.mlp_hidden_dim = int(dim * mlp_ratio)
@@ -406,13 +441,29 @@ class Flux2TransformerBlock(nn.Module):
added_proj_bias=bias,
out_bias=bias,
eps=eps,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attn"),
)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
self.ff = Flux2FeedForward(
dim=dim,
dim_out=dim,
mult=mlp_ratio,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "ff"),
)
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.ff_context = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
self.ff_context = Flux2FeedForward(
dim=dim,
dim_out=dim,
mult=mlp_ratio,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "ff_context"),
)
def forward(
self,
@@ -576,6 +627,7 @@ class Flux2Transformer2DModel(nn.Module):
rope_theta: int = 2000,
eps: float = 1e-6,
guidance_embeds: bool = True,
quant_config: "QuantizationConfig | None" = None,
):
super().__init__()
self.guidance_embeds = guidance_embeds
@@ -624,8 +676,10 @@ class Flux2Transformer2DModel(nn.Module):
mlp_ratio=mlp_ratio,
eps=eps,
bias=False,
quant_config=quant_config,
prefix=f"transformer_blocks.{i}",
)
for _ in range(num_layers)
for i in range(num_layers)
]
)
@@ -638,8 +692,10 @@ class Flux2Transformer2DModel(nn.Module):
mlp_ratio=mlp_ratio,
eps=eps,
bias=False,
quant_config=quant_config,
prefix=f"single_transformer_blocks.{i}",
)
for _ in range(num_single_layers)
for i in range(num_single_layers)
]
)
@@ -378,7 +378,7 @@ class Flux2Pipeline(nn.Module, CFGParallelMixin, SupportImageInput, ProgressBarM
self._execution_device
)
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, Flux2Transformer2DModel)
self.transformer = Flux2Transformer2DModel(**transformer_kwargs)
self.transformer = Flux2Transformer2DModel(quant_config=od_config.quantization_config, **transformer_kwargs)
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
@@ -53,6 +53,10 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def _join_prefix(prefix: str, name: str) -> str:
return f"{prefix}.{name}" if prefix else name
class Flux2SwiGLU(nn.Module):
"""SwiGLU activation used by Flux2."""
@@ -74,6 +78,7 @@ class Flux2FeedForward(nn.Module):
inner_dim: int | None = None,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
if inner_dim is None:
@@ -86,6 +91,7 @@ class Flux2FeedForward(nn.Module):
bias=bias,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "linear_in"),
)
self.act_fn = Flux2SwiGLU()
self.linear_out = RowParallelLinear(
@@ -95,6 +101,7 @@ class Flux2FeedForward(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "linear_out"),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -119,6 +126,7 @@ class Flux2Attention(nn.Module):
out_dim: int = None,
elementwise_affine: bool = True,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.parallel_config = parallel_config
@@ -136,6 +144,7 @@ class Flux2Attention(nn.Module):
total_num_heads=self.heads,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_qkv"),
)
self.query_num_heads = self.to_qkv.num_heads
self.kv_num_heads = self.to_qkv.num_kv_heads
@@ -152,6 +161,7 @@ class Flux2Attention(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_out.0"),
),
nn.Dropout(dropout),
]
@@ -166,6 +176,7 @@ class Flux2Attention(nn.Module):
total_num_heads=self.heads,
bias=added_proj_bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "add_kv_proj"),
)
self.add_query_num_heads = self.add_kv_proj.num_heads
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
@@ -176,6 +187,7 @@ class Flux2Attention(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_add_out"),
)
self.rope = RotaryEmbedding(is_neox_style=False)
@@ -330,6 +342,7 @@ class Flux2ParallelSelfAttention(nn.Module):
mlp_ratio: float = 4.0,
mlp_mult_factor: int = 2,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.parallel_config = parallel_config
@@ -350,6 +363,7 @@ class Flux2ParallelSelfAttention(nn.Module):
bias=bias,
gather_output=True,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
)
self.mlp_act_fn = Flux2SwiGLU()
@@ -362,6 +376,7 @@ class Flux2ParallelSelfAttention(nn.Module):
bias=out_bias,
gather_output=True,
quant_config=quant_config,
prefix=_join_prefix(prefix, "to_out"),
)
self.rope = RotaryEmbedding(is_neox_style=False)
self.attn = Attention(
@@ -463,6 +478,7 @@ class Flux2SingleTransformerBlock(nn.Module):
eps: float = 1e-6,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
@@ -478,6 +494,7 @@ class Flux2SingleTransformerBlock(nn.Module):
mlp_ratio=mlp_ratio,
mlp_mult_factor=2,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attn"),
)
def forward(
@@ -534,6 +551,7 @@ class Flux2TransformerBlock(nn.Module):
eps: float = 1e-6,
bias: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
@@ -551,10 +569,18 @@ class Flux2TransformerBlock(nn.Module):
out_bias=bias,
eps=eps,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attn"),
)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias, quant_config=quant_config)
self.ff = Flux2FeedForward(
dim=dim,
dim_out=dim,
mult=mlp_ratio,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "ff"),
)
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.ff_context = Flux2FeedForward(
@@ -563,6 +589,7 @@ class Flux2TransformerBlock(nn.Module):
mult=mlp_ratio,
bias=bias,
quant_config=quant_config,
prefix=_join_prefix(prefix, "ff_context"),
)
def forward(
@@ -835,8 +862,9 @@ class Flux2Transformer2DModel(nn.Module):
eps=eps,
bias=False,
quant_config=quant_config,
prefix=f"transformer_blocks.{i}",
)
for _ in range(num_layers)
for i in range(num_layers)
]
)
@@ -851,8 +879,9 @@ class Flux2Transformer2DModel(nn.Module):
eps=eps,
bias=False,
quant_config=quant_config,
prefix=f"single_transformer_blocks.{i}",
)
for _ in range(num_single_layers)
for i in range(num_single_layers)
]
)
@@ -27,8 +27,16 @@ def _set_forward_context_num_tokens(num_tokens: int) -> None:
class HunyuanFusedMoEDefault(FusedMoE):
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
# Current vLLM FusedMoE handles output reduction internally.
kwargs.pop("reduce_results", None)
super().__init__(prefix=prefix, **kwargs)
self._prefix = prefix
self._init_hook_handle = self.register_forward_pre_hook(self._initialize_kernel_hook, with_kwargs=True)
def _initialize_kernel_hook(self, module: Any, args: Any, kwargs: Any) -> None:
if self.quant_method and getattr(self.quant_method, "moe_kernel", None) is None:
self.quant_method.process_weights_after_loading(self)
self._init_hook_handle.remove()
def forward(self, hidden_states: Any, router_logits: Any) -> Any:
_set_forward_context_num_tokens(hidden_states.shape[0])
@@ -2035,6 +2035,44 @@ class HunyuanImage3Model(nn.Module):
return True
return False
def is_scalar_quant_scale(name: str, tensor: torch.Tensor) -> bool:
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale"))
def load_split_param(
name: str,
tensor: torch.Tensor,
den: int,
split_param: list[tuple[str | int, int]],
func: Callable[[torch.Tensor], torch.Tensor] | None,
) -> None:
param = params_dict[name]
weight_loader = param.weight_loader
if is_scalar_quant_scale(name, tensor):
for shard_id, _ in split_param:
weight_loader(param, tensor, shard_id)
return
assert tensor.shape[0] % den == 0
units = tensor.shape[0] // den
offset = 0
tensor = func(tensor) if func else tensor
for shard_id, num in split_param:
new_offset = offset + num * units
weight_loader(param, tensor[offset:new_offset], shard_id)
offset = new_offset
def get_loaded_weight_shard(
name: str,
tensor: torch.Tensor,
offset: int,
den: int,
) -> torch.Tensor:
if is_scalar_quant_scale(name, tensor):
return tensor
assert tensor.shape[0] % den == 0
units = tensor.shape[0] // den
return tensor[offset * units : offset * units + units]
for name, loaded_weight in weights:
# print(f"Loading weight name: {name}, tp_rank: {tp_rank}", flush=True)
if contains_unexpected_keyword(name, unexpected_keywords):
@@ -2112,19 +2150,7 @@ class HunyuanImage3Model(nn.Module):
if is_pp_missing_parameter(name, self):
continue
assert loaded_weight.shape[0] % den == 0
units = loaded_weight.shape[0] // den
param = params_dict[name]
weight_loader = param.weight_loader
offset = 0
for shard_id, num in split_param:
new_offset = offset + num * units
if func:
weight_loader(param, func(loaded_weight)[offset:new_offset], shard_id)
else:
weight_loader(param, loaded_weight[offset:new_offset], shard_id)
offset = new_offset
load_split_param(name, loaded_weight, den, split_param, func)
break
else:
# Skip loading extra bias for GPTQ models.
@@ -2154,12 +2180,11 @@ class HunyuanImage3Model(nn.Module):
continue
param = params_dict[name_mapped]
weight_loader = cast(Callable[..., bool], param.weight_loader)
assert loaded_weight.shape[0] % den == 0
units = loaded_weight.shape[0] // den
loaded_weight_shard = get_loaded_weight_shard(name, loaded_weight, offset, den)
success = weight_loader(
param,
loaded_weight[offset * units : offset * units + units],
loaded_weight_shard,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
@@ -209,6 +209,18 @@ def validate_zimage_tp_constraints(
return ffn_hidden_dim, final_out_dims, supported_tp_candidates
def _join_prefix(prefix: str, name: str) -> str:
return f"{prefix}.{name}" if prefix else name
def _restore_linear_output_shape(output: torch.Tensor, input_: torch.Tensor) -> torch.Tensor:
if input_.dim() > 2 and output.dim() == 2:
token_count = math.prod(input_.shape[:-1])
if output.shape[0] == token_count:
return output.reshape(*input_.shape[:-1], output.shape[-1])
return output
class TimestepEmbedder(nn.Module):
def __init__(
self, out_size, mid_size=None, frequency_embedding_size=256, quant_config: "QuantizationConfig | None" = None
@@ -256,6 +268,7 @@ class ZImageAttention(nn.Module):
qk_norm: bool = True,
eps: float = 1e-6,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
) -> None:
super().__init__()
self.dim = dim
@@ -271,7 +284,7 @@ class ZImageAttention(nn.Module):
total_num_kv_heads=num_kv_heads,
bias=False,
quant_config=quant_config,
prefix="to_qkv",
prefix=_join_prefix(prefix, "to_qkv"),
)
assert qk_norm is True
@@ -290,7 +303,7 @@ class ZImageAttention(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix="to_out",
prefix=_join_prefix(prefix, "to_out.0"),
)
]
)
@@ -312,6 +325,7 @@ class ZImageAttention(nn.Module):
sin: torch.Tensor,
):
qkv, _ = self.to_qkv(hidden_states)
qkv = _restore_linear_output_shape(qkv, hidden_states)
q_size = self.to_qkv.num_heads * self.head_dim
kv_size = self.to_qkv.num_kv_heads * self.head_dim
query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1)
@@ -344,7 +358,9 @@ class ZImageAttention(nn.Module):
hidden_states = hidden_states.flatten(2, 3)
hidden_states = hidden_states.to(dtype)
to_out_input = hidden_states
hidden_states = self.to_out[0](hidden_states)
hidden_states = _restore_linear_output_shape(hidden_states, to_out_input)
return hidden_states
@@ -364,7 +380,7 @@ class FeedForward(nn.Module):
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=prefix,
prefix=_join_prefix(prefix, "w13"),
)
self.act = SiluAndMul()
self.w2 = RowParallelLinear(
@@ -374,11 +390,15 @@ class FeedForward(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=prefix,
prefix=_join_prefix(prefix, "w2"),
)
def forward(self, x):
return self.w2(self.act(self.w13(x)))
hidden_states = self.w13(x)
hidden_states = _restore_linear_output_shape(hidden_states, x)
hidden_states = self.act(hidden_states)
hidden_states = self.w2(hidden_states)
return _restore_linear_output_shape(hidden_states, x)
class ZImageTransformerBlock(nn.Module):
@@ -392,6 +412,7 @@ class ZImageTransformerBlock(nn.Module):
qk_norm: bool,
modulation=True,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.dim = dim
@@ -403,10 +424,14 @@ class ZImageTransformerBlock(nn.Module):
qk_norm=qk_norm,
eps=1e-5,
quant_config=quant_config,
prefix=_join_prefix(prefix, "attention"),
)
self.feed_forward = FeedForward(
dim=dim, hidden_dim=int(dim / 3 * 8), quant_config=quant_config, prefix="feed_forward"
dim=dim,
hidden_dim=int(dim / 3 * 8),
quant_config=quant_config,
prefix=_join_prefix(prefix, "feed_forward"),
)
self.layer_id = layer_id
@@ -711,6 +736,7 @@ class ZImageTransformer2DModel(CachedTransformer):
qk_norm,
modulation=True,
quant_config=quant_config,
prefix=f"noise_refiner.{layer_id}",
)
for layer_id in range(n_refiner_layers)
]
@@ -726,6 +752,7 @@ class ZImageTransformer2DModel(CachedTransformer):
qk_norm,
modulation=False,
quant_config=quant_config,
prefix=f"context_refiner.{layer_id}",
)
for layer_id in range(n_refiner_layers)
]
@@ -757,6 +784,7 @@ class ZImageTransformer2DModel(CachedTransformer):
norm_eps,
qk_norm,
quant_config=quant_config,
prefix=f"layers.{layer_id}",
)
for layer_id in range(n_layers)
]