[Feature]: FP8 Quantization Support for DiT (#1034)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
f0bca6f66b
commit
830d261f9d
17 files changed
+688
-13
No files matched your search
@@ -33,6 +33,7 @@ from vllm_omni.diffusion.models.qwen_image.cfg_parallel import (
|
||||
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
|
||||
QwenImageTransformer2DModel,
|
||||
)
|
||||
from vllm_omni.diffusion.quantization import get_vllm_quant_config_for_layers
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
@@ -272,7 +273,10 @@ class QwenImagePipeline(nn.Module, QwenImageCFGParallelMixin):
|
||||
self.device
|
||||
)
|
||||
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, QwenImageTransformer2DModel)
|
||||
self.transformer = QwenImageTransformer2DModel(od_config=od_config, **transformer_kwargs)
|
||||
quant_config = get_vllm_quant_config_for_layers(od_config.quantization_config)
|
||||
self.transformer = QwenImageTransformer2DModel(
|
||||
od_config=od_config, quant_config=quant_config, **transformer_kwargs
|
||||
)
|
||||
|
||||
self.tokenizer = Qwen2Tokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import lru_cache
|
||||
from math import prod
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -23,6 +25,11 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
@@ -390,7 +397,16 @@ class QwenEmbedRope(nn.Module):
|
||||
|
||||
|
||||
class ColumnParallelApproxGELU(nn.Module):
|
||||
def __init__(self, dim_in: int, dim_out: int, *, approximate: str, bias: bool = True):
|
||||
def __init__(
|
||||
self,
|
||||
dim_in: int,
|
||||
dim_out: int,
|
||||
*,
|
||||
approximate: str,
|
||||
bias: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.proj = ColumnParallelLinear(
|
||||
dim_in,
|
||||
@@ -398,6 +414,8 @@ class ColumnParallelApproxGELU(nn.Module):
|
||||
bias=bias,
|
||||
gather_output=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
self.approximate = approximate
|
||||
|
||||
@@ -415,6 +433,8 @@ class FeedForward(nn.Module):
|
||||
activation_fn: str = "gelu-approximate",
|
||||
inner_dim: int | None = None,
|
||||
bias: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -424,13 +444,17 @@ class FeedForward(nn.Module):
|
||||
dim_out = dim_out or dim
|
||||
|
||||
layers: list[nn.Module] = [
|
||||
ColumnParallelApproxGELU(dim, inner_dim, approximate="tanh", bias=bias),
|
||||
ColumnParallelApproxGELU(
|
||||
dim, inner_dim, approximate="tanh", bias=bias, quant_config=quant_config, prefix=prefix
|
||||
),
|
||||
nn.Identity(), # placeholder for weight loading
|
||||
RowParallelLinear(
|
||||
inner_dim,
|
||||
dim_out,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -456,6 +480,7 @@ class QwenImageCrossAttention(nn.Module):
|
||||
pre_only: bool = False,
|
||||
context_pre_only: bool = False,
|
||||
out_dim: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert dim % num_heads == 0
|
||||
@@ -471,6 +496,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
hidden_size=dim,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=num_heads,
|
||||
quant_config=quant_config,
|
||||
prefix="to_qkv",
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -485,6 +512,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
hidden_size=added_kv_proj_dim,
|
||||
head_size=head_dim,
|
||||
total_num_heads=num_heads,
|
||||
quant_config=quant_config,
|
||||
prefix="add_kv_proj",
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -496,6 +525,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_add_out",
|
||||
)
|
||||
|
||||
assert not pre_only
|
||||
@@ -505,6 +536,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_out",
|
||||
)
|
||||
|
||||
self.norm_added_q = RMSNorm(head_dim, eps=eps)
|
||||
@@ -637,6 +670,7 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
qk_norm: str = "rms_norm",
|
||||
eps: float = 1e-6,
|
||||
zero_cond_t: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -656,9 +690,10 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
added_kv_proj_dim=dim,
|
||||
context_pre_only=False,
|
||||
head_dim=attention_head_dim,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.img_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.img_mlp = FeedForward(dim=dim, dim_out=dim)
|
||||
self.img_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="img_mlp")
|
||||
|
||||
# Text processing modules
|
||||
self.txt_mod = nn.Sequential(
|
||||
@@ -668,7 +703,7 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
self.txt_norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
# Text doesn't need separate attention - it's handled by img_attn joint computation
|
||||
self.txt_norm2 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.txt_mlp = FeedForward(dim=dim, dim_out=dim)
|
||||
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, quant_config=quant_config, prefix="txt_mlp")
|
||||
|
||||
self.zero_cond_t = zero_cond_t
|
||||
|
||||
@@ -862,6 +897,7 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
zero_cond_t: bool = False,
|
||||
use_additional_t_cond: bool = False,
|
||||
use_layer3d_rope: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.parallel_config = od_config.parallel_config
|
||||
@@ -891,6 +927,7 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
num_attention_heads=num_attention_heads,
|
||||
attention_head_dim=attention_head_dim,
|
||||
zero_cond_t=zero_cond_t,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user