Signed-off-by: roG0d <rodgarcas98@gmail.com> Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: roG0d <rodgarcas98@gmail.com>
This commit is contained in:
1 parent
40a07e0d80
commit
c4a0990044
26 files changed
+1214
-176
No files matched your search
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from diffusers.models.embeddings import (
|
||||
@@ -27,6 +27,13 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
class Flux2SwiGLU(nn.Module):
|
||||
"""SwiGLU activation used by Flux2."""
|
||||
@@ -48,6 +55,8 @@ class Flux2FeedForward(nn.Module):
|
||||
mult: float = 3.0,
|
||||
inner_dim: int | None = None,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
if inner_dim is None:
|
||||
@@ -59,6 +68,8 @@ class Flux2FeedForward(nn.Module):
|
||||
[inner_dim, inner_dim],
|
||||
bias=bias,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_in"),
|
||||
)
|
||||
self.act_fn = Flux2SwiGLU()
|
||||
self.linear_out = RowParallelLinear(
|
||||
@@ -67,6 +78,8 @@ class Flux2FeedForward(nn.Module):
|
||||
bias=bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_out"),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -89,6 +102,8 @@ class Flux2Attention(nn.Module):
|
||||
eps: float = 1e-5,
|
||||
out_dim: int = None,
|
||||
elementwise_affine: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.head_dim = dim_head
|
||||
@@ -104,6 +119,8 @@ class Flux2Attention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -119,6 +136,8 @@ class Flux2Attention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
),
|
||||
nn.Dropout(dropout),
|
||||
]
|
||||
@@ -132,6 +151,8 @@ class Flux2Attention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.heads,
|
||||
bias=added_proj_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "add_kv_proj"),
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -141,6 +162,8 @@ class Flux2Attention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_add_out"),
|
||||
)
|
||||
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
@@ -238,6 +261,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
elementwise_affine: bool = True,
|
||||
mlp_ratio: float = 4.0,
|
||||
mlp_mult_factor: int = 2,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.head_dim = dim_head
|
||||
@@ -256,6 +281,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor,
|
||||
bias=bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
|
||||
)
|
||||
self.mlp_act_fn = Flux2SwiGLU()
|
||||
|
||||
@@ -267,6 +294,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
self.out_dim,
|
||||
bias=out_bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
self.attn = Attention(
|
||||
@@ -329,6 +358,8 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
mlp_ratio: float = 3.0,
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -342,6 +373,8 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
eps=eps,
|
||||
mlp_ratio=mlp_ratio,
|
||||
mlp_mult_factor=2,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -389,6 +422,8 @@ class Flux2TransformerBlock(nn.Module):
|
||||
mlp_ratio: float = 3.0,
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.mlp_hidden_dim = int(dim * mlp_ratio)
|
||||
@@ -406,13 +441,29 @@ class Flux2TransformerBlock(nn.Module):
|
||||
added_proj_bias=bias,
|
||||
out_bias=bias,
|
||||
eps=eps,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
|
||||
self.ff = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff"),
|
||||
)
|
||||
|
||||
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff_context = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
|
||||
self.ff_context = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff_context"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -576,6 +627,7 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
rope_theta: int = 2000,
|
||||
eps: float = 1e-6,
|
||||
guidance_embeds: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.guidance_embeds = guidance_embeds
|
||||
@@ -624,8 +676,10 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
@@ -638,8 +692,10 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"single_transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_single_layers)
|
||||
for i in range(num_single_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -378,7 +378,7 @@ class Flux2Pipeline(nn.Module, CFGParallelMixin, SupportImageInput, ProgressBarM
|
||||
self._execution_device
|
||||
)
|
||||
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, Flux2Transformer2DModel)
|
||||
self.transformer = Flux2Transformer2DModel(**transformer_kwargs)
|
||||
self.transformer = Flux2Transformer2DModel(quant_config=od_config.quantization_config, **transformer_kwargs)
|
||||
|
||||
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
|
||||
self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
|
||||
|
||||
@@ -53,6 +53,10 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
class Flux2SwiGLU(nn.Module):
|
||||
"""SwiGLU activation used by Flux2."""
|
||||
|
||||
@@ -74,6 +78,7 @@ class Flux2FeedForward(nn.Module):
|
||||
inner_dim: int | None = None,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
if inner_dim is None:
|
||||
@@ -86,6 +91,7 @@ class Flux2FeedForward(nn.Module):
|
||||
bias=bias,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_in"),
|
||||
)
|
||||
self.act_fn = Flux2SwiGLU()
|
||||
self.linear_out = RowParallelLinear(
|
||||
@@ -95,6 +101,7 @@ class Flux2FeedForward(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_out"),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -119,6 +126,7 @@ class Flux2Attention(nn.Module):
|
||||
out_dim: int = None,
|
||||
elementwise_affine: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.parallel_config = parallel_config
|
||||
@@ -136,6 +144,7 @@ class Flux2Attention(nn.Module):
|
||||
total_num_heads=self.heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -152,6 +161,7 @@ class Flux2Attention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
),
|
||||
nn.Dropout(dropout),
|
||||
]
|
||||
@@ -166,6 +176,7 @@ class Flux2Attention(nn.Module):
|
||||
total_num_heads=self.heads,
|
||||
bias=added_proj_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "add_kv_proj"),
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -176,6 +187,7 @@ class Flux2Attention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_add_out"),
|
||||
)
|
||||
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
@@ -330,6 +342,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
mlp_ratio: float = 4.0,
|
||||
mlp_mult_factor: int = 2,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.parallel_config = parallel_config
|
||||
@@ -350,6 +363,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
bias=bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
|
||||
)
|
||||
self.mlp_act_fn = Flux2SwiGLU()
|
||||
|
||||
@@ -362,6 +376,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
bias=out_bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
self.attn = Attention(
|
||||
@@ -463,6 +478,7 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -478,6 +494,7 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
mlp_mult_factor=2,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -534,6 +551,7 @@ class Flux2TransformerBlock(nn.Module):
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -551,10 +569,18 @@ class Flux2TransformerBlock(nn.Module):
|
||||
out_bias=bias,
|
||||
eps=eps,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias, quant_config=quant_config)
|
||||
self.ff = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff"),
|
||||
)
|
||||
|
||||
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff_context = Flux2FeedForward(
|
||||
@@ -563,6 +589,7 @@ class Flux2TransformerBlock(nn.Module):
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff_context"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -835,8 +862,9 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
@@ -851,8 +879,9 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"single_transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_single_layers)
|
||||
for i in range(num_single_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -27,8 +27,16 @@ def _set_forward_context_num_tokens(num_tokens: int) -> None:
|
||||
|
||||
class HunyuanFusedMoEDefault(FusedMoE):
|
||||
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
|
||||
# Current vLLM FusedMoE handles output reduction internally.
|
||||
kwargs.pop("reduce_results", None)
|
||||
super().__init__(prefix=prefix, **kwargs)
|
||||
self._prefix = prefix
|
||||
self._init_hook_handle = self.register_forward_pre_hook(self._initialize_kernel_hook, with_kwargs=True)
|
||||
|
||||
def _initialize_kernel_hook(self, module: Any, args: Any, kwargs: Any) -> None:
|
||||
if self.quant_method and getattr(self.quant_method, "moe_kernel", None) is None:
|
||||
self.quant_method.process_weights_after_loading(self)
|
||||
self._init_hook_handle.remove()
|
||||
|
||||
def forward(self, hidden_states: Any, router_logits: Any) -> Any:
|
||||
_set_forward_context_num_tokens(hidden_states.shape[0])
|
||||
|
||||
@@ -2035,6 +2035,44 @@ class HunyuanImage3Model(nn.Module):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_scalar_quant_scale(name: str, tensor: torch.Tensor) -> bool:
|
||||
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale"))
|
||||
|
||||
def load_split_param(
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
den: int,
|
||||
split_param: list[tuple[str | int, int]],
|
||||
func: Callable[[torch.Tensor], torch.Tensor] | None,
|
||||
) -> None:
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
if is_scalar_quant_scale(name, tensor):
|
||||
for shard_id, _ in split_param:
|
||||
weight_loader(param, tensor, shard_id)
|
||||
return
|
||||
|
||||
assert tensor.shape[0] % den == 0
|
||||
units = tensor.shape[0] // den
|
||||
offset = 0
|
||||
tensor = func(tensor) if func else tensor
|
||||
for shard_id, num in split_param:
|
||||
new_offset = offset + num * units
|
||||
weight_loader(param, tensor[offset:new_offset], shard_id)
|
||||
offset = new_offset
|
||||
|
||||
def get_loaded_weight_shard(
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
offset: int,
|
||||
den: int,
|
||||
) -> torch.Tensor:
|
||||
if is_scalar_quant_scale(name, tensor):
|
||||
return tensor
|
||||
assert tensor.shape[0] % den == 0
|
||||
units = tensor.shape[0] // den
|
||||
return tensor[offset * units : offset * units + units]
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# print(f"Loading weight name: {name}, tp_rank: {tp_rank}", flush=True)
|
||||
if contains_unexpected_keyword(name, unexpected_keywords):
|
||||
@@ -2112,19 +2150,7 @@ class HunyuanImage3Model(nn.Module):
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
assert loaded_weight.shape[0] % den == 0
|
||||
units = loaded_weight.shape[0] // den
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
offset = 0
|
||||
for shard_id, num in split_param:
|
||||
new_offset = offset + num * units
|
||||
if func:
|
||||
weight_loader(param, func(loaded_weight)[offset:new_offset], shard_id)
|
||||
else:
|
||||
weight_loader(param, loaded_weight[offset:new_offset], shard_id)
|
||||
offset = new_offset
|
||||
|
||||
load_split_param(name, loaded_weight, den, split_param, func)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
@@ -2154,12 +2180,11 @@ class HunyuanImage3Model(nn.Module):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = cast(Callable[..., bool], param.weight_loader)
|
||||
assert loaded_weight.shape[0] % den == 0
|
||||
units = loaded_weight.shape[0] // den
|
||||
loaded_weight_shard = get_loaded_weight_shard(name, loaded_weight, offset, den)
|
||||
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight[offset * units : offset * units + units],
|
||||
loaded_weight_shard,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
|
||||
@@ -209,6 +209,18 @@ def validate_zimage_tp_constraints(
|
||||
return ffn_hidden_dim, final_out_dims, supported_tp_candidates
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
def _restore_linear_output_shape(output: torch.Tensor, input_: torch.Tensor) -> torch.Tensor:
|
||||
if input_.dim() > 2 and output.dim() == 2:
|
||||
token_count = math.prod(input_.shape[:-1])
|
||||
if output.shape[0] == token_count:
|
||||
return output.reshape(*input_.shape[:-1], output.shape[-1])
|
||||
return output
|
||||
|
||||
|
||||
class TimestepEmbedder(nn.Module):
|
||||
def __init__(
|
||||
self, out_size, mid_size=None, frequency_embedding_size=256, quant_config: "QuantizationConfig | None" = None
|
||||
@@ -256,6 +268,7 @@ class ZImageAttention(nn.Module):
|
||||
qk_norm: bool = True,
|
||||
eps: float = 1e-6,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
@@ -271,7 +284,7 @@ class ZImageAttention(nn.Module):
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_qkv",
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
|
||||
assert qk_norm is True
|
||||
@@ -290,7 +303,7 @@ class ZImageAttention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_out",
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
)
|
||||
]
|
||||
)
|
||||
@@ -312,6 +325,7 @@ class ZImageAttention(nn.Module):
|
||||
sin: torch.Tensor,
|
||||
):
|
||||
qkv, _ = self.to_qkv(hidden_states)
|
||||
qkv = _restore_linear_output_shape(qkv, hidden_states)
|
||||
q_size = self.to_qkv.num_heads * self.head_dim
|
||||
kv_size = self.to_qkv.num_kv_heads * self.head_dim
|
||||
query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
@@ -344,7 +358,9 @@ class ZImageAttention(nn.Module):
|
||||
hidden_states = hidden_states.flatten(2, 3)
|
||||
hidden_states = hidden_states.to(dtype)
|
||||
|
||||
to_out_input = hidden_states
|
||||
hidden_states = self.to_out[0](hidden_states)
|
||||
hidden_states = _restore_linear_output_shape(hidden_states, to_out_input)
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -364,7 +380,7 @@ class FeedForward(nn.Module):
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix=_join_prefix(prefix, "w13"),
|
||||
)
|
||||
self.act = SiluAndMul()
|
||||
self.w2 = RowParallelLinear(
|
||||
@@ -374,11 +390,15 @@ class FeedForward(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix=_join_prefix(prefix, "w2"),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.w2(self.act(self.w13(x)))
|
||||
hidden_states = self.w13(x)
|
||||
hidden_states = _restore_linear_output_shape(hidden_states, x)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states = self.w2(hidden_states)
|
||||
return _restore_linear_output_shape(hidden_states, x)
|
||||
|
||||
|
||||
class ZImageTransformerBlock(nn.Module):
|
||||
@@ -392,6 +412,7 @@ class ZImageTransformerBlock(nn.Module):
|
||||
qk_norm: bool,
|
||||
modulation=True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
@@ -403,10 +424,14 @@ class ZImageTransformerBlock(nn.Module):
|
||||
qk_norm=qk_norm,
|
||||
eps=1e-5,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attention"),
|
||||
)
|
||||
|
||||
self.feed_forward = FeedForward(
|
||||
dim=dim, hidden_dim=int(dim / 3 * 8), quant_config=quant_config, prefix="feed_forward"
|
||||
dim=dim,
|
||||
hidden_dim=int(dim / 3 * 8),
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "feed_forward"),
|
||||
)
|
||||
self.layer_id = layer_id
|
||||
|
||||
@@ -711,6 +736,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
qk_norm,
|
||||
modulation=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"noise_refiner.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_refiner_layers)
|
||||
]
|
||||
@@ -726,6 +752,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
qk_norm,
|
||||
modulation=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"context_refiner.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_refiner_layers)
|
||||
]
|
||||
@@ -757,6 +784,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
norm_eps,
|
||||
qk_norm,
|
||||
quant_config=quant_config,
|
||||
prefix=f"layers.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_layers)
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user