Signed-off-by: roG0d <rodgarcas98@gmail.com> Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: roG0d <rodgarcas98@gmail.com>
This commit is contained in:
1 parent
40a07e0d80
commit
c4a0990044
26 files changed
+1214
-176
No files matched your search
+49
-56
@@ -208,10 +208,11 @@ class TransformerConfig:
|
||||
quant_method: str | None = None
|
||||
quant_config: QuantizationConfig | None = None
|
||||
disk_qc = params.get("quantization_config")
|
||||
if isinstance(disk_qc, dict) and "quant_method" in disk_qc:
|
||||
quant_method = disk_qc["quant_method"]
|
||||
kwargs = {k: v for k, v in disk_qc.items() if k != "quant_method"}
|
||||
quant_config = build_quant_config(quant_method, **kwargs)
|
||||
if isinstance(disk_qc, dict):
|
||||
raw_quant_method = disk_qc.get("quant_method", disk_qc.get("method"))
|
||||
quant_config = build_quant_config(disk_qc)
|
||||
if quant_config is not None:
|
||||
quant_method = raw_quant_method if raw_quant_method is not None else quant_config.get_name()
|
||||
|
||||
return cls(params=params, quant_method=quant_method, quant_config=quant_config)
|
||||
|
||||
@@ -513,6 +514,10 @@ class OmniDiffusionConfig:
|
||||
# str is resolved to {"method": <str>} internally.
|
||||
# Per-component: {"transformer": {"method": "fp8"}, "vae": None}
|
||||
quantization_config: str | QuantizationConfig | dict[str, Any] | None = None
|
||||
# Explicit runtime override for ModelOpt FP8 diffusion checkpoints. This
|
||||
# does not enable FP8 by itself; it only selects CUTLASS once the checkpoint
|
||||
# has already resolved to vLLM's ModelOpt FP8 linear method.
|
||||
force_cutlass_fp8: bool = False
|
||||
|
||||
# Diffusion pipeline Profiling config
|
||||
enable_diffusion_pipeline_profiler: bool = False
|
||||
@@ -630,14 +635,9 @@ class OmniDiffusionConfig:
|
||||
|
||||
# Auto-detect quantization from TransformerConfig if not explicitly set.
|
||||
# This covers the case where tf_model_config is passed at construction
|
||||
# time. For late (post-construction) assignment, callers should use
|
||||
# time. For late (post-construction) assignment, callers should use
|
||||
# set_tf_model_config() which propagates quant_config automatically.
|
||||
if self.quantization_config is None and self.tf_model_config.quant_config is not None:
|
||||
self.quantization_config = self.tf_model_config.quant_config
|
||||
logger.info(
|
||||
"Auto-detected quantization '%s' from model config",
|
||||
self.tf_model_config.quant_method,
|
||||
)
|
||||
self._propagate_quantization_from_tf_config(self.tf_model_config)
|
||||
|
||||
# Resolve quantization_config: str/dict -> QuantizationConfig via build_quant_config.
|
||||
if self.quantization_config is not None:
|
||||
@@ -664,6 +664,32 @@ class OmniDiffusionConfig:
|
||||
"valid together with diffusion_load_format=diffusers"
|
||||
)
|
||||
|
||||
def _propagate_quantization_from_tf_config(self, tf_config: "TransformerConfig") -> None:
|
||||
if tf_config.quant_config is None:
|
||||
return
|
||||
|
||||
is_checkpoint_fp8 = bool(getattr(tf_config.quant_config, "is_checkpoint_fp8_serialized", False))
|
||||
should_use_checkpoint_config = self.quantization_config is None or (
|
||||
is_checkpoint_fp8 and self._is_generic_fp8_quant_config(self.quantization_config)
|
||||
)
|
||||
if should_use_checkpoint_config:
|
||||
self.quantization_config = tf_config.quant_config
|
||||
logger.info(
|
||||
"Auto-detected quantization '%s' from model config",
|
||||
tf_config.quant_method,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_generic_fp8_quant_config(quant_config: object) -> bool:
|
||||
if isinstance(quant_config, str):
|
||||
return quant_config.lower() == "fp8"
|
||||
if isinstance(quant_config, Mapping):
|
||||
method = quant_config.get("method", quant_config.get("quant_method"))
|
||||
return isinstance(method, str) and method.lower() == "fp8"
|
||||
if hasattr(quant_config, "get_name"):
|
||||
return quant_config.get_name() == "fp8"
|
||||
return False
|
||||
|
||||
def set_tf_model_config(self, tf_config: "TransformerConfig") -> None:
|
||||
"""Assign `tf_model_config` and propagate quantization if detected.
|
||||
|
||||
@@ -679,12 +705,7 @@ class OmniDiffusionConfig:
|
||||
`TransformerConfig.from_dict`.
|
||||
"""
|
||||
self.tf_model_config = tf_config
|
||||
if self.quantization_config is None and tf_config.quant_config is not None:
|
||||
self.quantization_config = tf_config.quant_config
|
||||
logger.info(
|
||||
"Auto-detected quantization '%s' from model config",
|
||||
tf_config.quant_method,
|
||||
)
|
||||
self._propagate_quantization_from_tf_config(tf_config)
|
||||
|
||||
def update_multimodal_support(self) -> None:
|
||||
# Resolve serving-visible multimodal behavior from shared metadata
|
||||
@@ -716,50 +737,35 @@ class OmniDiffusionConfig:
|
||||
# Skip transformer config loading for diffusers adapter
|
||||
# (non-DiT models don't have a separate transformer folder/config)
|
||||
if self.diffusion_load_format == "diffusers":
|
||||
self.tf_model_config = TransformerConfig()
|
||||
try:
|
||||
diffusers_pipeline_cls_name = config_dict["_class_name"]
|
||||
self.diffusers_pipeline_cls = getattr(diffusers, diffusers_pipeline_cls_name)
|
||||
except (KeyError, AttributeError) as exc:
|
||||
logger.warning(
|
||||
f"Could not find valid _class_name for diffusers pipeline in model_index.json: {exc}. "
|
||||
"Without knowing the underlying diffusers pipeline class, the dummy run will input only "
|
||||
"text prompt, which may cause errors for pipelines that require additional inputs."
|
||||
)
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
else:
|
||||
tf_config_dict = get_hf_file_to_dict("transformer/config.json", self.model)
|
||||
self.tf_model_config = TransformerConfig.from_dict(tf_config_dict)
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(tf_config_dict))
|
||||
else:
|
||||
raise FileNotFoundError("model_index.json not found")
|
||||
except (AttributeError, OSError, ValueError, FileNotFoundError):
|
||||
# Skip transformer config loading for diffusers adapter
|
||||
# (non-DiT models don't have a separate transformer folder/config)
|
||||
if self.diffusion_load_format == "diffusers":
|
||||
self.tf_model_config = TransformerConfig()
|
||||
logger.warning(
|
||||
"Could not find valid model_index.json per diffusers format. "
|
||||
"This model is suspectedly unsupported by the diffusers backend. "
|
||||
"Also, without knowing the underlying diffusers pipeline class from model_index.json, "
|
||||
"the dummy run will input only text prompt, which may cause errors for pipelines "
|
||||
"that require additional inputs."
|
||||
)
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
else:
|
||||
cfg = get_hf_file_to_dict("config.json", self.model)
|
||||
if cfg is None:
|
||||
raise ValueError(f"Could not find config.json or model_index.json for model {self.model}")
|
||||
|
||||
self.tf_model_config = TransformerConfig.from_dict(cfg)
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(cfg))
|
||||
model_type = cfg.get("model_type")
|
||||
architectures = cfg.get("architectures") or []
|
||||
|
||||
if model_type == "bagel" or "BagelForConditionalGeneration" in architectures:
|
||||
self.model_class_name = "BagelPipeline"
|
||||
self.tf_model_config = TransformerConfig()
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
elif model_type == "nextstep":
|
||||
if self.model_class_name is None:
|
||||
self.model_class_name = "NextStep11Pipeline"
|
||||
self.tf_model_config = TransformerConfig()
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
elif model_type == "s2v":
|
||||
if self.model_class_name is None:
|
||||
@@ -794,23 +800,10 @@ class OmniDiffusionConfig:
|
||||
cache_backend = os.environ.get("DIFFUSION_CACHE_BACKEND") or os.environ.get("DIFFUSION_CACHE_ADAPTER")
|
||||
kwargs["cache_backend"] = cache_backend.lower() if cache_backend else "none"
|
||||
|
||||
# Falsy-value check for not-None fields (convert potential None values in YAML config to empty containers)
|
||||
if "diffusers_load_kwargs" in kwargs and kwargs["diffusers_load_kwargs"] is None:
|
||||
kwargs["diffusers_load_kwargs"] = {}
|
||||
if "diffusers_call_kwargs" in kwargs and kwargs["diffusers_call_kwargs"] is None:
|
||||
kwargs["diffusers_call_kwargs"] = {}
|
||||
|
||||
# Forward top-level parallel knobs (e.g. --tensor-parallel-size from CLI)
|
||||
# into parallel_config so the diffusion engine sees them.
|
||||
par = kwargs.get("parallel_config", {})
|
||||
if isinstance(par, Mapping):
|
||||
par = dict(par)
|
||||
if par.get("tensor_parallel_size") is None:
|
||||
par.pop("tensor_parallel_size", None)
|
||||
tensor_parallel_size = kwargs.get("tensor_parallel_size")
|
||||
if tensor_parallel_size is not None and "tensor_parallel_size" not in par:
|
||||
par["tensor_parallel_size"] = tensor_parallel_size
|
||||
kwargs["parallel_config"] = par
|
||||
# Convert optional YAML null values to empty containers.
|
||||
for key in ("diffusers_load_kwargs", "diffusers_call_kwargs"):
|
||||
if key in kwargs and kwargs[key] is None:
|
||||
kwargs[key] = {}
|
||||
|
||||
# Filter kwargs to only include valid fields
|
||||
valid_fields = {f.name for f in fields(cls)}
|
||||
|
||||
@@ -65,6 +65,22 @@ def apply_rotary_emb_mindiesd(
|
||||
return rotary_position_embedding(x, cos, sin, rotated_mode="rotated_half", head_first=False, fused=True)
|
||||
|
||||
|
||||
def _ensure_batch_dim(x: torch.Tensor) -> tuple[torch.Tensor, bool]:
|
||||
# Upstream fused rotary kernels expect ``x`` shaped as
|
||||
# ``[batch_size, seq_len, nheads, headdim]``. Some omni diffusion call
|
||||
# sites pass ``[seq_len, nheads, headdim]`` instead, so normalize to 4D
|
||||
# here before entering the fused path.
|
||||
if x.dim() == 3:
|
||||
return x.unsqueeze(0), True
|
||||
return x, False
|
||||
|
||||
|
||||
def _restore_batch_dim(x: torch.Tensor, squeezed: bool) -> torch.Tensor:
|
||||
if squeezed:
|
||||
return x.squeeze(0)
|
||||
return x
|
||||
|
||||
|
||||
class RotaryEmbedding(CustomOp):
|
||||
"""
|
||||
rotary positional embedding.
|
||||
@@ -98,12 +114,14 @@ class RotaryEmbedding(CustomOp):
|
||||
cos = cos[0]
|
||||
sin = sin[0]
|
||||
|
||||
return apply_rotary_emb(
|
||||
x, squeezed = _ensure_batch_dim(x)
|
||||
output = apply_rotary_emb(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
interleaved=self.interleaved,
|
||||
)
|
||||
return _restore_batch_dim(output, squeezed)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
@@ -119,12 +137,14 @@ class RotaryEmbedding(CustomOp):
|
||||
cos = cos[0]
|
||||
sin = sin[0]
|
||||
|
||||
return self.apply_rotary_emb_flash_attn(
|
||||
x, squeezed = _ensure_batch_dim(x)
|
||||
output = self.apply_rotary_emb_flash_attn(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
interleaved=self.interleaved,
|
||||
)
|
||||
return _restore_batch_dim(output, squeezed)
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from torch import nn
|
||||
|
||||
from .modelopt import ModelOptFp8CheckpointAdapter
|
||||
|
||||
|
||||
def get_checkpoint_adapter(
|
||||
model: nn.Module,
|
||||
source: object,
|
||||
quant_config: object | None,
|
||||
use_safetensors: bool,
|
||||
) -> ModelOptFp8CheckpointAdapter | None:
|
||||
if ModelOptFp8CheckpointAdapter.is_compatible(source, quant_config, use_safetensors):
|
||||
return ModelOptFp8CheckpointAdapter(model, source)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ModelOptFp8CheckpointAdapter",
|
||||
"get_checkpoint_adapter",
|
||||
]
|
||||
@@ -0,0 +1,251 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Generator, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.model_executor.utils import get_packed_modules_mapping
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
MODEL_OPT_SCALE_SUFFIXES = (".input_scale", ".weight_scale", ".weight_scale_inv")
|
||||
DEFAULT_PACKED_MODULES_MAPPING = {
|
||||
"to_qkv": ("to_q", "to_k", "to_v"),
|
||||
"add_kv_proj": ("add_q_proj", "add_k_proj", "add_v_proj"),
|
||||
"w13": ("w1", "w3"),
|
||||
}
|
||||
FP8_DTYPES = tuple(
|
||||
dtype
|
||||
for dtype in (
|
||||
getattr(torch, "float8_e4m3fn", None),
|
||||
getattr(torch, "float8_e5m2", None),
|
||||
getattr(torch, "float8_e4m3fnuz", None),
|
||||
getattr(torch, "float8_e5m2fnuz", None),
|
||||
)
|
||||
if dtype is not None
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _AdaptState:
|
||||
scale_tensors: dict[str, torch.Tensor] = field(default_factory=dict)
|
||||
pending_weights: dict[str, list[tuple[str, torch.Tensor, torch.dtype]]] = field(default_factory=dict)
|
||||
skipped_scales: int = 0
|
||||
dequantized_weights: int = 0
|
||||
|
||||
|
||||
class ModelOptFp8CheckpointAdapter:
|
||||
def __init__(self, model: nn.Module, source: object):
|
||||
self._loadable_tensors = self._get_model_loadable_tensors(model)
|
||||
self._weights_mapper = self._get_weights_mapper(model)
|
||||
self._source_label = getattr(source, "prefix", "") or getattr(source, "subfolder", None) or "model"
|
||||
|
||||
@classmethod
|
||||
def is_compatible(
|
||||
cls,
|
||||
source: object,
|
||||
quant_config: object | None,
|
||||
use_safetensors: bool,
|
||||
) -> bool:
|
||||
return use_safetensors and cls._is_transformer_source(source) and cls._is_checkpoint_quant_config(quant_config)
|
||||
|
||||
@staticmethod
|
||||
def _is_transformer_source(source: object) -> bool:
|
||||
if getattr(source, "subfolder", None) == "transformer":
|
||||
return True
|
||||
return str(getattr(source, "prefix", "")).startswith("transformer.")
|
||||
|
||||
@staticmethod
|
||||
def _is_checkpoint_quant_config(quant_config: object | None) -> bool:
|
||||
return (
|
||||
quant_config is not None
|
||||
and hasattr(quant_config, "get_name")
|
||||
and quant_config.get_name() == "modelopt"
|
||||
and bool(getattr(quant_config, "is_checkpoint_fp8_serialized", False))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_model_loadable_tensors(model: nn.Module) -> dict[str, torch.Tensor]:
|
||||
loadable_tensors: dict[str, torch.Tensor] = {name: param for name, param in model.named_parameters()}
|
||||
loadable_tensors.update({name: buffer for name, buffer in model.named_buffers()})
|
||||
return loadable_tensors
|
||||
|
||||
@staticmethod
|
||||
def _is_scale(name: str) -> bool:
|
||||
return name.endswith(MODEL_OPT_SCALE_SUFFIXES)
|
||||
|
||||
@staticmethod
|
||||
def _is_fp8_tensor(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dtype in FP8_DTYPES
|
||||
|
||||
@staticmethod
|
||||
def _get_weight_scale_name(weight_name: str) -> str | None:
|
||||
if weight_name.endswith(".weight"):
|
||||
return weight_name[: -len(".weight")] + ".weight_scale"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _get_weights_mapper(cls, model: nn.Module) -> WeightsMapper:
|
||||
mapping = {
|
||||
packed_name: tuple(shard_names) for packed_name, shard_names in DEFAULT_PACKED_MODULES_MAPPING.items()
|
||||
}
|
||||
mapping.update(
|
||||
{
|
||||
str(packed_name): tuple(str(shard_name) for shard_name in shard_names)
|
||||
for packed_name, shard_names in get_packed_modules_mapping(model).items()
|
||||
}
|
||||
)
|
||||
|
||||
orig_to_new_substr = {".to_out.0.": ".to_out."}
|
||||
orig_to_new_prefix: dict[str, str] = {}
|
||||
for packed_name, shard_names in mapping.items():
|
||||
for shard_name in shard_names:
|
||||
orig_to_new_substr[f".{shard_name}."] = f".{packed_name}."
|
||||
orig_to_new_prefix[f"{shard_name}."] = f"{packed_name}."
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr=orig_to_new_substr,
|
||||
orig_to_new_prefix=orig_to_new_prefix,
|
||||
)
|
||||
|
||||
def _resolve_target_name(self, name: str) -> str | None:
|
||||
if name in self._loadable_tensors:
|
||||
return name
|
||||
|
||||
for candidate in self._weights_mapper.apply_list([name]):
|
||||
if candidate != name and candidate in self._loadable_tensors:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _reshape_weight_scale(scale: torch.Tensor, weight_shape: torch.Size) -> torch.Tensor:
|
||||
if scale.numel() == 1:
|
||||
return scale.reshape(())
|
||||
if len(weight_shape) == 2 and scale.ndim == 1 and scale.shape[0] == weight_shape[0]:
|
||||
return scale.reshape(-1, 1)
|
||||
if tuple(scale.shape) == tuple(weight_shape):
|
||||
return scale
|
||||
if (
|
||||
len(weight_shape) == 2
|
||||
and scale.ndim == 4
|
||||
and scale.shape[1] == 1
|
||||
and scale.shape[3] == 1
|
||||
and weight_shape[0] % scale.shape[0] == 0
|
||||
and weight_shape[1] % scale.shape[2] == 0
|
||||
):
|
||||
block_n = weight_shape[0] // scale.shape[0]
|
||||
block_k = weight_shape[1] // scale.shape[2]
|
||||
return scale.expand(scale.shape[0], block_n, scale.shape[2], block_k).reshape(weight_shape)
|
||||
raise ValueError(f"Unsupported ModelOpt FP8 weight_scale shape {tuple(scale.shape)} for weight {weight_shape}")
|
||||
|
||||
def _dequantize_weight(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
state: _AdaptState,
|
||||
target_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
scale_name = self._get_weight_scale_name(name)
|
||||
if scale_name is None or scale_name not in state.scale_tensors:
|
||||
raise ValueError(f"Missing ModelOpt FP8 weight_scale for full-precision target weight {name!r}")
|
||||
|
||||
scale = state.scale_tensors[scale_name].to(dtype=target_dtype, device=loaded_weight.device)
|
||||
scale = self._reshape_weight_scale(scale, loaded_weight.shape)
|
||||
return loaded_weight.to(dtype=target_dtype) * scale
|
||||
|
||||
def _flush_pending_weights(
|
||||
self,
|
||||
scale_name: str,
|
||||
state: _AdaptState,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
for weight_name, weight_tensor, target_dtype in state.pending_weights.pop(scale_name, []):
|
||||
yield weight_name, self._dequantize_weight(weight_name, weight_tensor, state, target_dtype)
|
||||
state.dequantized_weights += 1
|
||||
|
||||
def _handle_scale_tensor(
|
||||
self,
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
target_name: str | None,
|
||||
state: _AdaptState,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
state.scale_tensors[name] = tensor
|
||||
if target_name is None:
|
||||
state.skipped_scales += 1
|
||||
else:
|
||||
yield name, tensor
|
||||
yield from self._flush_pending_weights(name, state)
|
||||
|
||||
def _target_dtype_for_dequantization(
|
||||
self,
|
||||
tensor: torch.Tensor,
|
||||
target_name: str | None,
|
||||
) -> torch.dtype | None:
|
||||
if target_name is None or not self._is_fp8_tensor(tensor):
|
||||
return None
|
||||
|
||||
target_dtype = self._loadable_tensors[target_name].dtype
|
||||
if target_dtype in FP8_DTYPES:
|
||||
return None
|
||||
return target_dtype
|
||||
|
||||
def _maybe_dequantize_or_defer_weight(
|
||||
self,
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
target_dtype: torch.dtype,
|
||||
state: _AdaptState,
|
||||
) -> torch.Tensor | None:
|
||||
scale_name = self._get_weight_scale_name(name)
|
||||
if scale_name is None:
|
||||
raise ValueError(f"Missing ModelOpt FP8 weight_scale name for weight {name!r}")
|
||||
|
||||
if scale_name not in state.scale_tensors:
|
||||
state.pending_weights.setdefault(scale_name, []).append((name, tensor, target_dtype))
|
||||
return None
|
||||
|
||||
state.dequantized_weights += 1
|
||||
return self._dequantize_weight(name, tensor, state, target_dtype)
|
||||
|
||||
@staticmethod
|
||||
def _check_pending_weights(state: _AdaptState) -> None:
|
||||
if not state.pending_weights:
|
||||
return
|
||||
|
||||
missing_scale_names = ", ".join(repr(name) for name in sorted(state.pending_weights))
|
||||
raise ValueError(f"Missing ModelOpt FP8 weight_scale for full-precision target weights: {missing_scale_names}")
|
||||
|
||||
def _log_adaptation_summary(self, state: _AdaptState) -> None:
|
||||
if not state.skipped_scales and not state.dequantized_weights:
|
||||
return
|
||||
|
||||
logger.info_once(
|
||||
"Adapted ModelOpt FP8 %s weights: dequantized %d full-precision weights, skipped %d scale tensors",
|
||||
self._source_label,
|
||||
state.dequantized_weights,
|
||||
state.skipped_scales,
|
||||
)
|
||||
|
||||
def adapt(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
state = _AdaptState()
|
||||
|
||||
for name, tensor in weights:
|
||||
target_name = self._resolve_target_name(name)
|
||||
if self._is_scale(name):
|
||||
yield from self._handle_scale_tensor(name, tensor, target_name, state)
|
||||
continue
|
||||
|
||||
target_dtype = self._target_dtype_for_dequantization(tensor, target_name)
|
||||
if target_dtype is not None:
|
||||
tensor = self._maybe_dequantize_or_defer_weight(name, tensor, target_dtype, state)
|
||||
if tensor is None:
|
||||
continue
|
||||
yield name, tensor
|
||||
|
||||
self._check_pending_weights(state)
|
||||
self._log_adaptation_summary(state)
|
||||
@@ -7,7 +7,7 @@ import re
|
||||
import time
|
||||
from collections.abc import Generator, Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download
|
||||
@@ -32,14 +32,14 @@ from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig
|
||||
from vllm_omni.diffusion.model_loader.checkpoint_adapters import (
|
||||
get_checkpoint_adapter,
|
||||
)
|
||||
from vllm_omni.diffusion.model_loader.gguf_adapters import get_gguf_adapter
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
|
||||
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
||||
from vllm_omni.diffusion.registry import initialize_model
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -177,7 +177,11 @@ class DiffusersPipelineLoader:
|
||||
|
||||
return hf_folder, hf_weights_files, use_safetensors
|
||||
|
||||
def _get_weights_iterator(self, source: "ComponentSource") -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
def _get_weights_iterator(
|
||||
self,
|
||||
source: "ComponentSource",
|
||||
model: nn.Module | None = None,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Get an iterator for the model weights based on the load format."""
|
||||
_, hf_weights_files, use_safetensors = self._prepare_weights(
|
||||
source.model_or_path,
|
||||
@@ -213,7 +217,34 @@ class DiffusersPipelineLoader:
|
||||
if self.counter_before_loading_weights == 0.0:
|
||||
self.counter_before_loading_weights = time.perf_counter()
|
||||
# Apply the prefix.
|
||||
return ((source.prefix + name, tensor) for (name, tensor) in weights_iterator)
|
||||
prefixed_weights_iterator = ((source.prefix + name, tensor) for (name, tensor) in weights_iterator)
|
||||
if model is not None:
|
||||
checkpoint_adapter = self._get_checkpoint_adapter(model, source, use_safetensors)
|
||||
if checkpoint_adapter is not None:
|
||||
return checkpoint_adapter.adapt(prefixed_weights_iterator)
|
||||
return prefixed_weights_iterator
|
||||
|
||||
def _get_source_quant_config(self, source: "ComponentSource") -> object | None:
|
||||
if self.od_config is None:
|
||||
return None
|
||||
|
||||
quant_config = self.od_config.quantization_config
|
||||
if hasattr(quant_config, "resolve"):
|
||||
return quant_config.resolve(source.prefix.rstrip("."))
|
||||
return quant_config
|
||||
|
||||
def _get_checkpoint_adapter(
|
||||
self,
|
||||
model: nn.Module,
|
||||
source: "ComponentSource",
|
||||
use_safetensors: bool,
|
||||
):
|
||||
return get_checkpoint_adapter(
|
||||
model=model,
|
||||
source=source,
|
||||
quant_config=self._get_source_quant_config(source),
|
||||
use_safetensors=use_safetensors,
|
||||
)
|
||||
|
||||
def get_all_weights(
|
||||
self,
|
||||
@@ -221,7 +252,7 @@ class DiffusersPipelineLoader:
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
sources = self._get_weight_sources(model)
|
||||
for source in sources:
|
||||
yield from self._get_weights_iterator(source)
|
||||
yield from self._get_weights_iterator(source, model=model)
|
||||
|
||||
def _get_weight_sources(self, model: nn.Module) -> tuple["ComponentSource", ...]:
|
||||
return tuple(
|
||||
@@ -269,7 +300,7 @@ class DiffusersPipelineLoader:
|
||||
"""Load a model with the given configurations."""
|
||||
if load_format is None:
|
||||
load_format = "default"
|
||||
|
||||
self.od_config = od_config
|
||||
# CPU offload + FP8: load weights on device for FP8 quantization
|
||||
if load_device == "cpu" and od_config.quantization_config is not None:
|
||||
load_device = device.type
|
||||
@@ -291,13 +322,12 @@ class DiffusersPipelineLoader:
|
||||
model_cls = resolve_obj_by_qualname(custom_pipeline_name)
|
||||
model = model_cls(od_config=od_config)
|
||||
else:
|
||||
# 'dummy' format should not call this function at all
|
||||
raise ValueError(f"Unknown load_format: {load_format}")
|
||||
logger.debug("Loading weights on %s ...", load_device)
|
||||
if load_format == "diffusers":
|
||||
# DiffusersAdapterPipeline.load_weights() calls
|
||||
# DiffusionPipeline.from_pretrained() internally — it does
|
||||
# NOT use our native (customized) pipeline classes.
|
||||
# DiffusionPipeline.from_pretrained() internally; it does
|
||||
# not use our native customized pipeline classes.
|
||||
cast(DiffusersAdapterPipeline, model).load_weights()
|
||||
elif self._is_gguf_quantization(od_config):
|
||||
self._load_weights_with_gguf(model, od_config)
|
||||
@@ -449,9 +479,7 @@ class DiffusersPipelineLoader:
|
||||
def _get_model_loadable_names(self, model: nn.Module) -> set[str]:
|
||||
# Avoid model.state_dict() here because GGUF uses UninitializedParameter
|
||||
# which raises during detach(). Collect names directly.
|
||||
names = {name for name, _ in model.named_parameters()}
|
||||
names.update(name for name, _ in model.named_buffers())
|
||||
return names
|
||||
return {name for name, _ in model.named_parameters()} | {name for name, _ in model.named_buffers()}
|
||||
|
||||
def _resolve_gguf_model_path(self, gguf_model: str, revision: str | None) -> str:
|
||||
if os.path.isfile(gguf_model):
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from diffusers.models.embeddings import (
|
||||
@@ -27,6 +27,13 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
class Flux2SwiGLU(nn.Module):
|
||||
"""SwiGLU activation used by Flux2."""
|
||||
@@ -48,6 +55,8 @@ class Flux2FeedForward(nn.Module):
|
||||
mult: float = 3.0,
|
||||
inner_dim: int | None = None,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
if inner_dim is None:
|
||||
@@ -59,6 +68,8 @@ class Flux2FeedForward(nn.Module):
|
||||
[inner_dim, inner_dim],
|
||||
bias=bias,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_in"),
|
||||
)
|
||||
self.act_fn = Flux2SwiGLU()
|
||||
self.linear_out = RowParallelLinear(
|
||||
@@ -67,6 +78,8 @@ class Flux2FeedForward(nn.Module):
|
||||
bias=bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_out"),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -89,6 +102,8 @@ class Flux2Attention(nn.Module):
|
||||
eps: float = 1e-5,
|
||||
out_dim: int = None,
|
||||
elementwise_affine: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.head_dim = dim_head
|
||||
@@ -104,6 +119,8 @@ class Flux2Attention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -119,6 +136,8 @@ class Flux2Attention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
),
|
||||
nn.Dropout(dropout),
|
||||
]
|
||||
@@ -132,6 +151,8 @@ class Flux2Attention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.heads,
|
||||
bias=added_proj_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "add_kv_proj"),
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -141,6 +162,8 @@ class Flux2Attention(nn.Module):
|
||||
bias=out_bias,
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_add_out"),
|
||||
)
|
||||
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
@@ -238,6 +261,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
elementwise_affine: bool = True,
|
||||
mlp_ratio: float = 4.0,
|
||||
mlp_mult_factor: int = 2,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.head_dim = dim_head
|
||||
@@ -256,6 +281,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor,
|
||||
bias=bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
|
||||
)
|
||||
self.mlp_act_fn = Flux2SwiGLU()
|
||||
|
||||
@@ -267,6 +294,8 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
self.out_dim,
|
||||
bias=out_bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
self.attn = Attention(
|
||||
@@ -329,6 +358,8 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
mlp_ratio: float = 3.0,
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -342,6 +373,8 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
eps=eps,
|
||||
mlp_ratio=mlp_ratio,
|
||||
mlp_mult_factor=2,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -389,6 +422,8 @@ class Flux2TransformerBlock(nn.Module):
|
||||
mlp_ratio: float = 3.0,
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.mlp_hidden_dim = int(dim * mlp_ratio)
|
||||
@@ -406,13 +441,29 @@ class Flux2TransformerBlock(nn.Module):
|
||||
added_proj_bias=bias,
|
||||
out_bias=bias,
|
||||
eps=eps,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
|
||||
self.ff = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff"),
|
||||
)
|
||||
|
||||
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff_context = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias)
|
||||
self.ff_context = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff_context"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -576,6 +627,7 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
rope_theta: int = 2000,
|
||||
eps: float = 1e-6,
|
||||
guidance_embeds: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.guidance_embeds = guidance_embeds
|
||||
@@ -624,8 +676,10 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
@@ -638,8 +692,10 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"single_transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_single_layers)
|
||||
for i in range(num_single_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -378,7 +378,7 @@ class Flux2Pipeline(nn.Module, CFGParallelMixin, SupportImageInput, ProgressBarM
|
||||
self._execution_device
|
||||
)
|
||||
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, Flux2Transformer2DModel)
|
||||
self.transformer = Flux2Transformer2DModel(**transformer_kwargs)
|
||||
self.transformer = Flux2Transformer2DModel(quant_config=od_config.quantization_config, **transformer_kwargs)
|
||||
|
||||
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
|
||||
self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2)
|
||||
|
||||
@@ -53,6 +53,10 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
class Flux2SwiGLU(nn.Module):
|
||||
"""SwiGLU activation used by Flux2."""
|
||||
|
||||
@@ -74,6 +78,7 @@ class Flux2FeedForward(nn.Module):
|
||||
inner_dim: int | None = None,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
if inner_dim is None:
|
||||
@@ -86,6 +91,7 @@ class Flux2FeedForward(nn.Module):
|
||||
bias=bias,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_in"),
|
||||
)
|
||||
self.act_fn = Flux2SwiGLU()
|
||||
self.linear_out = RowParallelLinear(
|
||||
@@ -95,6 +101,7 @@ class Flux2FeedForward(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "linear_out"),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -119,6 +126,7 @@ class Flux2Attention(nn.Module):
|
||||
out_dim: int = None,
|
||||
elementwise_affine: bool = True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.parallel_config = parallel_config
|
||||
@@ -136,6 +144,7 @@ class Flux2Attention(nn.Module):
|
||||
total_num_heads=self.heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
self.query_num_heads = self.to_qkv.num_heads
|
||||
self.kv_num_heads = self.to_qkv.num_kv_heads
|
||||
@@ -152,6 +161,7 @@ class Flux2Attention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
),
|
||||
nn.Dropout(dropout),
|
||||
]
|
||||
@@ -166,6 +176,7 @@ class Flux2Attention(nn.Module):
|
||||
total_num_heads=self.heads,
|
||||
bias=added_proj_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "add_kv_proj"),
|
||||
)
|
||||
self.add_query_num_heads = self.add_kv_proj.num_heads
|
||||
self.add_kv_num_heads = self.add_kv_proj.num_kv_heads
|
||||
@@ -176,6 +187,7 @@ class Flux2Attention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_add_out"),
|
||||
)
|
||||
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
@@ -330,6 +342,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
mlp_ratio: float = 4.0,
|
||||
mlp_mult_factor: int = 2,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.parallel_config = parallel_config
|
||||
@@ -350,6 +363,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
bias=bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_qkv_mlp_proj"),
|
||||
)
|
||||
self.mlp_act_fn = Flux2SwiGLU()
|
||||
|
||||
@@ -362,6 +376,7 @@ class Flux2ParallelSelfAttention(nn.Module):
|
||||
bias=out_bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "to_out"),
|
||||
)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
self.attn = Attention(
|
||||
@@ -463,6 +478,7 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -478,6 +494,7 @@ class Flux2SingleTransformerBlock(nn.Module):
|
||||
mlp_ratio=mlp_ratio,
|
||||
mlp_mult_factor=2,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -534,6 +551,7 @@ class Flux2TransformerBlock(nn.Module):
|
||||
eps: float = 1e-6,
|
||||
bias: bool = False,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
@@ -551,10 +569,18 @@ class Flux2TransformerBlock(nn.Module):
|
||||
out_bias=bias,
|
||||
eps=eps,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attn"),
|
||||
)
|
||||
|
||||
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias, quant_config=quant_config)
|
||||
self.ff = Flux2FeedForward(
|
||||
dim=dim,
|
||||
dim_out=dim,
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff"),
|
||||
)
|
||||
|
||||
self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.ff_context = Flux2FeedForward(
|
||||
@@ -563,6 +589,7 @@ class Flux2TransformerBlock(nn.Module):
|
||||
mult=mlp_ratio,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "ff_context"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -835,8 +862,9 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
@@ -851,8 +879,9 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
eps=eps,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"single_transformer_blocks.{i}",
|
||||
)
|
||||
for _ in range(num_single_layers)
|
||||
for i in range(num_single_layers)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -27,8 +27,16 @@ def _set_forward_context_num_tokens(num_tokens: int) -> None:
|
||||
|
||||
class HunyuanFusedMoEDefault(FusedMoE):
|
||||
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
|
||||
# Current vLLM FusedMoE handles output reduction internally.
|
||||
kwargs.pop("reduce_results", None)
|
||||
super().__init__(prefix=prefix, **kwargs)
|
||||
self._prefix = prefix
|
||||
self._init_hook_handle = self.register_forward_pre_hook(self._initialize_kernel_hook, with_kwargs=True)
|
||||
|
||||
def _initialize_kernel_hook(self, module: Any, args: Any, kwargs: Any) -> None:
|
||||
if self.quant_method and getattr(self.quant_method, "moe_kernel", None) is None:
|
||||
self.quant_method.process_weights_after_loading(self)
|
||||
self._init_hook_handle.remove()
|
||||
|
||||
def forward(self, hidden_states: Any, router_logits: Any) -> Any:
|
||||
_set_forward_context_num_tokens(hidden_states.shape[0])
|
||||
|
||||
@@ -2035,6 +2035,44 @@ class HunyuanImage3Model(nn.Module):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_scalar_quant_scale(name: str, tensor: torch.Tensor) -> bool:
|
||||
return tensor.numel() == 1 and name.endswith((".input_scale", ".weight_scale"))
|
||||
|
||||
def load_split_param(
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
den: int,
|
||||
split_param: list[tuple[str | int, int]],
|
||||
func: Callable[[torch.Tensor], torch.Tensor] | None,
|
||||
) -> None:
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
if is_scalar_quant_scale(name, tensor):
|
||||
for shard_id, _ in split_param:
|
||||
weight_loader(param, tensor, shard_id)
|
||||
return
|
||||
|
||||
assert tensor.shape[0] % den == 0
|
||||
units = tensor.shape[0] // den
|
||||
offset = 0
|
||||
tensor = func(tensor) if func else tensor
|
||||
for shard_id, num in split_param:
|
||||
new_offset = offset + num * units
|
||||
weight_loader(param, tensor[offset:new_offset], shard_id)
|
||||
offset = new_offset
|
||||
|
||||
def get_loaded_weight_shard(
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
offset: int,
|
||||
den: int,
|
||||
) -> torch.Tensor:
|
||||
if is_scalar_quant_scale(name, tensor):
|
||||
return tensor
|
||||
assert tensor.shape[0] % den == 0
|
||||
units = tensor.shape[0] // den
|
||||
return tensor[offset * units : offset * units + units]
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# print(f"Loading weight name: {name}, tp_rank: {tp_rank}", flush=True)
|
||||
if contains_unexpected_keyword(name, unexpected_keywords):
|
||||
@@ -2112,19 +2150,7 @@ class HunyuanImage3Model(nn.Module):
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
assert loaded_weight.shape[0] % den == 0
|
||||
units = loaded_weight.shape[0] // den
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
offset = 0
|
||||
for shard_id, num in split_param:
|
||||
new_offset = offset + num * units
|
||||
if func:
|
||||
weight_loader(param, func(loaded_weight)[offset:new_offset], shard_id)
|
||||
else:
|
||||
weight_loader(param, loaded_weight[offset:new_offset], shard_id)
|
||||
offset = new_offset
|
||||
|
||||
load_split_param(name, loaded_weight, den, split_param, func)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
@@ -2154,12 +2180,11 @@ class HunyuanImage3Model(nn.Module):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = cast(Callable[..., bool], param.weight_loader)
|
||||
assert loaded_weight.shape[0] % den == 0
|
||||
units = loaded_weight.shape[0] // den
|
||||
loaded_weight_shard = get_loaded_weight_shard(name, loaded_weight, offset, den)
|
||||
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight[offset * units : offset * units + units],
|
||||
loaded_weight_shard,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
|
||||
@@ -209,6 +209,18 @@ def validate_zimage_tp_constraints(
|
||||
return ffn_hidden_dim, final_out_dims, supported_tp_candidates
|
||||
|
||||
|
||||
def _join_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
def _restore_linear_output_shape(output: torch.Tensor, input_: torch.Tensor) -> torch.Tensor:
|
||||
if input_.dim() > 2 and output.dim() == 2:
|
||||
token_count = math.prod(input_.shape[:-1])
|
||||
if output.shape[0] == token_count:
|
||||
return output.reshape(*input_.shape[:-1], output.shape[-1])
|
||||
return output
|
||||
|
||||
|
||||
class TimestepEmbedder(nn.Module):
|
||||
def __init__(
|
||||
self, out_size, mid_size=None, frequency_embedding_size=256, quant_config: "QuantizationConfig | None" = None
|
||||
@@ -256,6 +268,7 @@ class ZImageAttention(nn.Module):
|
||||
qk_norm: bool = True,
|
||||
eps: float = 1e-6,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
@@ -271,7 +284,7 @@ class ZImageAttention(nn.Module):
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_qkv",
|
||||
prefix=_join_prefix(prefix, "to_qkv"),
|
||||
)
|
||||
|
||||
assert qk_norm is True
|
||||
@@ -290,7 +303,7 @@ class ZImageAttention(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix="to_out",
|
||||
prefix=_join_prefix(prefix, "to_out.0"),
|
||||
)
|
||||
]
|
||||
)
|
||||
@@ -312,6 +325,7 @@ class ZImageAttention(nn.Module):
|
||||
sin: torch.Tensor,
|
||||
):
|
||||
qkv, _ = self.to_qkv(hidden_states)
|
||||
qkv = _restore_linear_output_shape(qkv, hidden_states)
|
||||
q_size = self.to_qkv.num_heads * self.head_dim
|
||||
kv_size = self.to_qkv.num_kv_heads * self.head_dim
|
||||
query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
@@ -344,7 +358,9 @@ class ZImageAttention(nn.Module):
|
||||
hidden_states = hidden_states.flatten(2, 3)
|
||||
hidden_states = hidden_states.to(dtype)
|
||||
|
||||
to_out_input = hidden_states
|
||||
hidden_states = self.to_out[0](hidden_states)
|
||||
hidden_states = _restore_linear_output_shape(hidden_states, to_out_input)
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -364,7 +380,7 @@ class FeedForward(nn.Module):
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix=_join_prefix(prefix, "w13"),
|
||||
)
|
||||
self.act = SiluAndMul()
|
||||
self.w2 = RowParallelLinear(
|
||||
@@ -374,11 +390,15 @@ class FeedForward(nn.Module):
|
||||
input_is_parallel=True,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix=_join_prefix(prefix, "w2"),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.w2(self.act(self.w13(x)))
|
||||
hidden_states = self.w13(x)
|
||||
hidden_states = _restore_linear_output_shape(hidden_states, x)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states = self.w2(hidden_states)
|
||||
return _restore_linear_output_shape(hidden_states, x)
|
||||
|
||||
|
||||
class ZImageTransformerBlock(nn.Module):
|
||||
@@ -392,6 +412,7 @@ class ZImageTransformerBlock(nn.Module):
|
||||
qk_norm: bool,
|
||||
modulation=True,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
@@ -403,10 +424,14 @@ class ZImageTransformerBlock(nn.Module):
|
||||
qk_norm=qk_norm,
|
||||
eps=1e-5,
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "attention"),
|
||||
)
|
||||
|
||||
self.feed_forward = FeedForward(
|
||||
dim=dim, hidden_dim=int(dim / 3 * 8), quant_config=quant_config, prefix="feed_forward"
|
||||
dim=dim,
|
||||
hidden_dim=int(dim / 3 * 8),
|
||||
quant_config=quant_config,
|
||||
prefix=_join_prefix(prefix, "feed_forward"),
|
||||
)
|
||||
self.layer_id = layer_id
|
||||
|
||||
@@ -711,6 +736,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
qk_norm,
|
||||
modulation=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"noise_refiner.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_refiner_layers)
|
||||
]
|
||||
@@ -726,6 +752,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
qk_norm,
|
||||
modulation=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"context_refiner.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_refiner_layers)
|
||||
]
|
||||
@@ -757,6 +784,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
norm_eps,
|
||||
qk_norm,
|
||||
quant_config=quant_config,
|
||||
prefix=f"layers.{layer_id}",
|
||||
)
|
||||
for layer_id in range(n_layers)
|
||||
]
|
||||
|
||||
@@ -11,9 +11,9 @@ to DiffusionModelRunner.
|
||||
import gc
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from types import SimpleNamespace
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -22,7 +22,7 @@ from vllm.config import CompilationConfig, DeviceConfig, VllmConfig, set_current
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
from vllm.logger import init_logger
|
||||
from vllm.profiler.wrapper import CudaProfilerWrapper, WorkerProfiler
|
||||
from vllm.transformers_utils.config import get_config, get_hf_text_config
|
||||
from vllm.transformers_utils.config import get_hf_text_config
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from vllm.utils.mem_utils import GiB_bytes
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -54,6 +54,78 @@ from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DiffusionVllmModelConfig:
|
||||
model: str
|
||||
dtype: torch.dtype
|
||||
quantization: str | None = None
|
||||
quantization_config: Any | None = None
|
||||
hf_config: Any | None = None
|
||||
hf_text_config: Any | None = None
|
||||
multimodal_config: Any | None = None
|
||||
enforce_eager: bool = False
|
||||
disable_cascade_attn: bool = False
|
||||
enable_return_routed_experts: bool = False
|
||||
is_moe: bool = False
|
||||
|
||||
def is_quantized(self) -> bool:
|
||||
return self.quantization is not None
|
||||
|
||||
def is_model_moe(self) -> bool:
|
||||
return self.is_moe
|
||||
|
||||
def is_nvfp4_quantized(self) -> bool:
|
||||
return self.quantization == "modelopt_fp4"
|
||||
|
||||
|
||||
def _make_diffusion_vllm_model_config(od_config: OmniDiffusionConfig) -> _DiffusionVllmModelConfig:
|
||||
quant_config = getattr(od_config, "quantization_config", None)
|
||||
quantization = quant_config.get_name() if quant_config is not None and hasattr(quant_config, "get_name") else None
|
||||
hf_config = getattr(od_config, "tf_model_config", None)
|
||||
hf_text_config = get_hf_text_config(hf_config) if hasattr(hf_config, "get_text_config") else hf_config
|
||||
return _DiffusionVllmModelConfig(
|
||||
model=od_config.model,
|
||||
dtype=od_config.dtype,
|
||||
quantization=quantization,
|
||||
quantization_config=quant_config,
|
||||
hf_config=hf_config,
|
||||
hf_text_config=hf_text_config,
|
||||
enforce_eager=getattr(od_config, "enforce_eager", False),
|
||||
is_moe=bool(getattr(od_config, "is_moe", False)),
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _force_cutlass_fp8_linear_kernel(quant_config: object | None) -> Iterator[None]:
|
||||
from vllm.model_executor.layers.quantization import modelopt as vllm_modelopt
|
||||
|
||||
linear_method_cls = getattr(quant_config, "LinearMethodCls", None)
|
||||
if linear_method_cls in {
|
||||
vllm_modelopt.ModelOptFp8LinearMethod,
|
||||
vllm_modelopt.ModelOptFp8PcPtLinearMethod,
|
||||
}:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(89):
|
||||
from vllm.model_executor.kernels.linear import CutlassFP8ScaledMMLinearKernel
|
||||
|
||||
original_init_fp8_linear_kernel = vllm_modelopt.init_fp8_linear_kernel
|
||||
|
||||
def init_fp8_linear_kernel_with_cutlass(*args: Any, **kwargs: Any) -> Any:
|
||||
kwargs.setdefault("force_kernel", CutlassFP8ScaledMMLinearKernel)
|
||||
return original_init_fp8_linear_kernel(*args, **kwargs)
|
||||
|
||||
vllm_modelopt.init_fp8_linear_kernel = init_fp8_linear_kernel_with_cutlass
|
||||
logger.info("Using CUTLASS FP8 linear kernels for this ModelOpt FP8 diffusion stage.")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
vllm_modelopt.init_fp8_linear_kernel = original_init_fp8_linear_kernel
|
||||
return
|
||||
|
||||
yield
|
||||
|
||||
|
||||
class DiffusionWorker:
|
||||
"""
|
||||
A worker that manages GPU infrastructure and delegates to the model runner.
|
||||
@@ -124,19 +196,7 @@ class DiffusionWorker:
|
||||
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = self.od_config.parallel_config.enable_expert_parallel
|
||||
vllm_config.profiler_config = self.od_config.profiler_config
|
||||
try:
|
||||
hf_config = get_config(self.od_config.model, trust_remote_code=self.od_config.trust_remote_code)
|
||||
except ValueError:
|
||||
hf_config = None
|
||||
logger.info("Skipping hf_config loading for diffusion model %r", self.od_config.model_class_name)
|
||||
hf_text_config = get_hf_text_config(hf_config) if hf_config is not None else None
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=hf_text_config,
|
||||
enforce_eager=self.od_config.enforce_eager,
|
||||
dtype=self.od_config.dtype,
|
||||
enable_return_routed_experts=False,
|
||||
)
|
||||
vllm_config.model_config = _make_diffusion_vllm_model_config(self.od_config) # type: ignore[assignment]
|
||||
vllm_config.quant_config = self.od_config.quantization_config
|
||||
# Since vLLM v0.20.0, IR wraps GPU ops. Set IR op priority preference to enforce GPU op fusion during wrapping.
|
||||
# Also need to log, because vLLM internally logs another line in VllmConfig.__post_init__. Avoid confusion.
|
||||
@@ -191,9 +251,15 @@ class DiffusionWorker:
|
||||
"""Load the diffusion model using DiffusionModelRunner."""
|
||||
load_format = kwargs.get("load_format", load_format)
|
||||
custom_pipeline_name = kwargs.get("custom_pipeline_name", custom_pipeline_name)
|
||||
cutlass_fp8_context = (
|
||||
_force_cutlass_fp8_linear_kernel(self.od_config.quantization_config)
|
||||
if getattr(self.od_config, "force_cutlass_fp8", False)
|
||||
else nullcontext()
|
||||
)
|
||||
with (
|
||||
set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config),
|
||||
set_current_vllm_config(self.vllm_config),
|
||||
cutlass_fp8_context,
|
||||
):
|
||||
self.model_runner.load_model(
|
||||
memory_pool_context_fn=self._maybe_get_memory_pool_context,
|
||||
|
||||
Reference in new issue
Block a user