[AutoRound] Add offline quantized W4A16 model support (#1777)

Signed-off-by: yiliu30 <yi4.liu@intel.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Yi LiuandHongsheng Liu authored and GitHub committed 2026-04-02 13:57:53 +08:00
1 parent 900f6aa837
commit c1d2dcc3ac
14 files changed
+1011 -19

No files matched your search

@@ -12,7 +12,6 @@ from diffusers.models.embeddings import (
get_1d_rotary_pos_embed,
)
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, AdaLayerNormZeroSingle
from diffusers.utils import is_torch_npu_available
from vllm.distributed import get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather
from vllm.logger import init_logger
@@ -32,6 +31,11 @@ if TYPE_CHECKING:
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.layers.adalayernorm import (
AdaLayerNormContinuous,
AdaLayerNormZero,
AdaLayerNormZeroSingle,
)
from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
logger = init_logger(__name__)
@@ -46,6 +50,7 @@ class ColumnParallelApproxGELU(nn.Module):
approximate: str,
bias: bool = True,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.proj = ColumnParallelLinear(
@@ -55,6 +60,7 @@ class ColumnParallelApproxGELU(nn.Module):
gather_output=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.proj",
)
self.approximate = approximate
@@ -73,6 +79,7 @@ class FeedForward(nn.Module):
inner_dim: int | None = None,
bias: bool = True,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -82,7 +89,9 @@ class FeedForward(nn.Module):
dim_out = dim_out or dim
layers: list[nn.Module] = [
ColumnParallelApproxGELU(dim, inner_dim, approximate="tanh", bias=bias, quant_config=quant_config),
ColumnParallelApproxGELU(
dim, inner_dim, approximate="tanh", bias=bias, quant_config=quant_config, prefix=f"{prefix}.net.0"
),
nn.Identity(), # placeholder for weight loading
RowParallelLinear(
inner_dim,
@@ -90,6 +99,7 @@ class FeedForward(nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.net.2",
),
]
@@ -117,6 +127,7 @@ class FluxAttention(torch.nn.Module):
context_pre_only: bool | None = None,
pre_only: bool = False,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
@@ -141,6 +152,7 @@ class FluxAttention(torch.nn.Module):
total_num_heads=self.heads,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.to_qkv",
)
if not self.pre_only:
@@ -153,6 +165,7 @@ class FluxAttention(torch.nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.to_out.0",
),
nn.Dropout(dropout),
]
@@ -168,8 +181,8 @@ class FluxAttention(torch.nn.Module):
total_num_heads=self.heads,
bias=added_proj_bias,
quant_config=quant_config,
prefix=f"{prefix}.add_kv_proj",
)
self.to_add_out = RowParallelLinear(
self.inner_dim,
query_dim,
@@ -177,6 +190,7 @@ class FluxAttention(torch.nn.Module):
input_is_parallel=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.to_add_out",
)
self.rope = RotaryEmbedding(is_neox_style=False)
@@ -272,11 +286,11 @@ class FluxTransformerBlock(nn.Module):
qk_norm: str = "rms_norm",
eps: float = 1e-6,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.norm1 = AdaLayerNormZero(dim)
self.norm1_context = AdaLayerNormZero(dim)
self.norm1 = AdaLayerNormZero(dim, quant_config=quant_config, prefix=f"{prefix}.norm1")
self.norm1_context = AdaLayerNormZero(dim, quant_config=quant_config, prefix=f"{prefix}.norm1_context")
self.attn = FluxAttention(
query_dim=dim,
@@ -288,13 +302,14 @@ class FluxTransformerBlock(nn.Module):
bias=True,
eps=eps,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config)
self.ff = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix=f"{prefix}.ff")
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_context = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config)
self.ff_context = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix=f"{prefix}.ff_context")
def forward(
self,
@@ -361,17 +376,28 @@ class FluxSingleTransformerBlock(nn.Module):
attention_head_dim: int,
mlp_ratio: float = 4.0,
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
):
super().__init__()
self.mlp_hidden_dim = int(dim * mlp_ratio)
self.norm = AdaLayerNormZeroSingle(dim)
self.norm = AdaLayerNormZeroSingle(dim, quant_config=quant_config, prefix=f"{prefix}.norm")
self.proj_mlp = ReplicatedLinear(
dim, self.mlp_hidden_dim, bias=True, return_bias=False, quant_config=quant_config
dim,
self.mlp_hidden_dim,
bias=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.proj_mlp",
)
self.act_mlp = nn.GELU(approximate="tanh")
self.proj_out = ReplicatedLinear(
dim + self.mlp_hidden_dim, dim, bias=True, return_bias=False, quant_config=quant_config
dim + self.mlp_hidden_dim,
dim,
bias=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.proj_out",
)
self.attn = FluxAttention(
@@ -383,6 +409,7 @@ class FluxSingleTransformerBlock(nn.Module):
eps=1e-6,
pre_only=True,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
@@ -542,8 +569,9 @@ class FluxTransformer2DModel(nn.Module):
num_attention_heads=num_attention_heads,
attention_head_dim=attention_head_dim,
quant_config=quant_config,
prefix=f"transformer_blocks.{i}",
)
for _ in range(num_layers)
for i in range(num_layers)
]
)
@@ -554,12 +582,20 @@ class FluxTransformer2DModel(nn.Module):
num_attention_heads=num_attention_heads,
attention_head_dim=attention_head_dim,
quant_config=quant_config,
prefix=f"single_transformer_blocks.{i}",
)
for _ in range(num_single_layers)
for i in range(num_single_layers)
]
)
self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
self.norm_out = AdaLayerNormContinuous(
self.inner_dim,
self.inner_dim,
elementwise_affine=False,
eps=1e-6,
quant_config=quant_config,
prefix="norm_out",
)
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
def forward(