[Bugfix] Fix FLUX W4A16/AutoRound quant_config propagation (#3587)

Signed-off-by: yiliu30 <yi4.liu@intel.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Yi LiuandCopilot authored and GitHub committed 2026-06-01 04:06:02 +00:00
1 parent f92d84fcf9
commit ef71d74c81
2 files changed
+44 -13

No files matched your search

@@ -29,6 +29,7 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.layers.adalayernorm import (
@@ -41,6 +42,23 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
logger = init_logger(__name__)
def _safe_quant_config(quant_config: "QuantizationConfig | None") -> "QuantizationConfig | None":
"""Return quant_config only if it is safe to propagate here, else None.
Dual-stream transformer_blocks, norm modulation layers, and norm_out are
kept at full precision for FP8 (see #2728). Offline quantization (e.g.
INC/AutoRound W4A16) needs the config propagated so packed weights load
correctly.
"""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.inc import INCConfig
if isinstance(quant_config, INCConfig):
return quant_config
return None
class ColumnParallelApproxGELU(nn.Module):
def __init__(
self,
@@ -381,9 +399,7 @@ class FluxSingleTransformerBlock(nn.Module):
super().__init__()
self.mlp_hidden_dim = int(dim * mlp_ratio)
# Modulation linear kept full precision; shift/scale/gate outputs
# are multiplied into the residual stream every block (see #2728).
self.norm = AdaLayerNormZeroSingle(dim, quant_config=None, prefix=f"{prefix}.norm")
self.norm = AdaLayerNormZeroSingle(dim, quant_config=_safe_quant_config(quant_config), prefix=f"{prefix}.norm")
self.proj_mlp = ReplicatedLinear(
dim,
self.mlp_hidden_dim,
@@ -565,16 +581,13 @@ class FluxTransformer2DModel(nn.Module):
self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim)
self.x_embedder = nn.Linear(in_channels, self.inner_dim)
# Dual-stream blocks kept full precision — FP8 on their joint
# attention path causes noise on FLUX (#2728). Single-stream
# blocks (38 vs 19) still get FP8 for memory savings.
self.transformer_blocks = nn.ModuleList(
[
FluxTransformerBlock(
dim=self.inner_dim,
num_attention_heads=num_attention_heads,
attention_head_dim=attention_head_dim,
quant_config=None,
quant_config=_safe_quant_config(quant_config),
prefix=f"transformer_blocks.{i}",
)
for i in range(num_layers)
@@ -594,13 +607,12 @@ class FluxTransformer2DModel(nn.Module):
]
)
# Final modulation feeds proj_out; keep full precision (see #2728).
self.norm_out = AdaLayerNormContinuous(
self.inner_dim,
self.inner_dim,
elementwise_affine=False,
eps=1e-6,
quant_config=None,
quant_config=_safe_quant_config(quant_config),
prefix="norm_out",
)
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)