[Bugfix] Fix FLUX W4A16/AutoRound quant_config propagation (#3587)
Signed-off-by: yiliu30 <yi4.liu@intel.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
1 parent
f92d84fcf9
commit
ef71d74c81
2 files changed
+44
-13
No files matched your search
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.layers.adalayernorm import (
|
||||
@@ -41,6 +42,23 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _safe_quant_config(quant_config: "QuantizationConfig | None") -> "QuantizationConfig | None":
|
||||
"""Return quant_config only if it is safe to propagate here, else None.
|
||||
|
||||
Dual-stream transformer_blocks, norm modulation layers, and norm_out are
|
||||
kept at full precision for FP8 (see #2728). Offline quantization (e.g.
|
||||
INC/AutoRound W4A16) needs the config propagated so packed weights load
|
||||
correctly.
|
||||
"""
|
||||
if quant_config is None:
|
||||
return None
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
|
||||
if isinstance(quant_config, INCConfig):
|
||||
return quant_config
|
||||
return None
|
||||
|
||||
|
||||
class ColumnParallelApproxGELU(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -381,9 +399,7 @@ class FluxSingleTransformerBlock(nn.Module):
|
||||
super().__init__()
|
||||
self.mlp_hidden_dim = int(dim * mlp_ratio)
|
||||
|
||||
# Modulation linear kept full precision; shift/scale/gate outputs
|
||||
# are multiplied into the residual stream every block (see #2728).
|
||||
self.norm = AdaLayerNormZeroSingle(dim, quant_config=None, prefix=f"{prefix}.norm")
|
||||
self.norm = AdaLayerNormZeroSingle(dim, quant_config=_safe_quant_config(quant_config), prefix=f"{prefix}.norm")
|
||||
self.proj_mlp = ReplicatedLinear(
|
||||
dim,
|
||||
self.mlp_hidden_dim,
|
||||
@@ -565,16 +581,13 @@ class FluxTransformer2DModel(nn.Module):
|
||||
self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim)
|
||||
self.x_embedder = nn.Linear(in_channels, self.inner_dim)
|
||||
|
||||
# Dual-stream blocks kept full precision — FP8 on their joint
|
||||
# attention path causes noise on FLUX (#2728). Single-stream
|
||||
# blocks (38 vs 19) still get FP8 for memory savings.
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
FluxTransformerBlock(
|
||||
dim=self.inner_dim,
|
||||
num_attention_heads=num_attention_heads,
|
||||
attention_head_dim=attention_head_dim,
|
||||
quant_config=None,
|
||||
quant_config=_safe_quant_config(quant_config),
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for i in range(num_layers)
|
||||
@@ -594,13 +607,12 @@ class FluxTransformer2DModel(nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
# Final modulation feeds proj_out; keep full precision (see #2728).
|
||||
self.norm_out = AdaLayerNormContinuous(
|
||||
self.inner_dim,
|
||||
self.inner_dim,
|
||||
elementwise_affine=False,
|
||||
eps=1e-6,
|
||||
quant_config=None,
|
||||
quant_config=_safe_quant_config(quant_config),
|
||||
prefix="norm_out",
|
||||
)
|
||||
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
|
||||
|
||||
Reference in new issue
Block a user