[Quantization] Migrate GGUF diffusion model support to OOT plugin (#4769)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py authored and GitHub committed 2026-07-11 18:48:43 +00:00
1 parent ff6f0da271
commit b95905deb1
35 files changed
+272 -1614

No files matched your search

@@ -11,7 +11,6 @@ from pathlib import Path
from typing import cast
import torch
from huggingface_hub import hf_hub_download
from torch import nn
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
@@ -36,7 +35,6 @@ from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig, apply_hsdp
from vllm_omni.diffusion.model_loader.checkpoint_adapters import (
get_checkpoint_adapter,
)
from vllm_omni.diffusion.model_loader.gguf_adapters import get_gguf_adapter
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
from vllm_omni.diffusion.registry import initialize_model
@@ -381,10 +379,7 @@ class DiffusersPipelineLoader:
# DiffusionPipeline.from_pretrained() internally; it does
# not use our native customized pipeline classes.
cast(DiffusersAdapterPipeline, model).load_weights()
elif self._is_gguf_quantization():
self._load_weights_with_gguf(model)
else:
# Quantization does not happen in `load_weights` but after it
self.load_weights(model)
# Process weights after loading for quantization (e.g., FP8 online quantization)
@@ -454,7 +449,7 @@ class DiffusersPipelineLoader:
def _is_expected_quantized_weight(name: str) -> bool:
"""Return True if *name* is a quantization-specific parameter.
Quantization methods (GPTQ, AWQ, FP8, GGUF, Autoround, etc.) create extra
Quantization methods (GPTQ, AWQ, FP8, Autoround, etc.) create extra
parameters that have no counterpart in an unquantized checkpoint.
These are expected to be absent and should not trigger a load error.
"""
@@ -467,8 +462,6 @@ class DiffusersPipelineLoader:
".weight_scale",
".weight_scale_inv",
".input_scale",
# GGUF
".qweight_type",
# INT8 (weight_scale already covered above)
)
return name.endswith(_QUANTIZED_WEIGHT_SUFFIXES)
@@ -500,116 +493,6 @@ class DiffusersPipelineLoader:
if unexpected_missing:
raise ValueError(f"Following weights were not initialized from checkpoint: {unexpected_missing}")
def _get_gguf_model_from_config(self) -> str | None:
"""Extract gguf_model from quant_config (handles both dict and object)."""
if self.quant_config is None:
return None
if isinstance(self.quant_config, dict):
return self.quant_config.get("gguf_model")
return getattr(self.quant_config, "gguf_model", None)
def _is_gguf_quantization(self) -> bool:
"""Check whether or not this pipeline loader is pointing at a GGUF config."""
from vllm_omni.quantization.gguf_config import DiffusionGGUFConfig
maybe_gguf_model = self._get_gguf_model_from_config()
is_gguf = (
isinstance(self.quant_config, DiffusionGGUFConfig)
or (isinstance(self.quant_config, dict) and self.quant_config.get("method") == "gguf")
or (hasattr(self.quant_config, "get_name") and self.quant_config.get_name() == "gguf")
)
if is_gguf:
if not maybe_gguf_model:
raise ValueError("GGUF quantization requires gguf_model")
return True
if isinstance(self.quant_config, dict):
return False
return bool(maybe_gguf_model)
def _is_transformer_source(self, source: "ComponentSource") -> bool:
if source.subfolder == "transformer":
return True
return source.prefix.startswith("transformer.")
def _get_model_loadable_names(self, model: nn.Module) -> set[str]:
# Avoid model.state_dict() here because GGUF uses UninitializedParameter
# which raises during detach(). Collect names directly.
return {name for name, _ in model.named_parameters()} | {name for name, _ in model.named_buffers()}
def _resolve_gguf_model_path(self, gguf_model: str, revision: str | None) -> str:
if os.path.isfile(gguf_model):
return gguf_model
# repo_id/filename.gguf
if "/" in gguf_model and gguf_model.endswith(".gguf"):
repo_id, filename = gguf_model.rsplit("/", 1)
return hf_hub_download(
repo_id=repo_id,
filename=filename,
revision=revision,
cache_dir=self.load_config.download_dir,
)
# repo_id:quant_type
if "/" in gguf_model and ":" in gguf_model:
repo_id, quant_type = gguf_model.rsplit(":", 1)
return download_gguf(
repo_id,
quant_type,
cache_dir=self.load_config.download_dir,
revision=revision,
ignore_patterns=self.load_config.ignore_patterns,
)
raise ValueError(
f"Unrecognized GGUF reference: {gguf_model!r} (expected local file, "
"<repo_id>/<filename>.gguf, or <repo_id>:<quant_type>)"
)
def _get_gguf_weights_iterator(
self,
source: "ComponentSource",
model: nn.Module,
) -> Generator[tuple[str, torch.Tensor], None, None]:
gguf_model = self._get_gguf_model_from_config()
if gguf_model is None:
raise ValueError("GGUF quantization requires quantization_config.gguf_model")
gguf_file = self._resolve_gguf_model_path(gguf_model, self.od_config.revision)
adapter = get_gguf_adapter(gguf_file, model, source, self.od_config)
weights_iter = adapter.weights_iterator()
return ((source.prefix + name, tensor) for (name, tensor) in weights_iter)
def _load_weights_with_gguf(self, model: nn.Module) -> set[str]:
sources = self._get_weight_sources(model)
loaded: set[str] = set()
loadable_names: set[str] | None = None
for source in sources:
if self._is_transformer_source(source):
loaded |= model.load_weights(self._get_gguf_weights_iterator(source, model))
# GGUF checkpoints can be transformer-only or partially quantized.
# Only fall back to HF if this source still has missing loadable weights.
loadable_names = loadable_names or self._get_model_loadable_names(model)
has_missing_for_source = any(
name.startswith(source.prefix) and name not in loaded for name in loadable_names
)
if not has_missing_for_source:
continue
hf_iter = self._get_weights_iterator(source)
hf_iter = (
(name, tensor) for (name, tensor) in hf_iter if name in loadable_names and name not in loaded
)
loaded |= model.load_weights(hf_iter)
else:
loaded |= model.load_weights(self._get_weights_iterator(source))
weights_to_load = self._get_expected_parameter_names(model)
weights_not_loaded = weights_to_load - loaded
if weights_not_loaded:
raise ValueError(f"Following weights were not initialized from checkpoint: {weights_not_loaded}")
return loaded
def _init_from_load_format(
self,
load_format: str,
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from .base import GGUFAdapter
from .flux2_klein import Flux2KleinGGUFAdapter
from .qwen_image import QwenImageGGUFAdapter
from .z_image import ZImageGGUFAdapter
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.model_loader.diffusers_loader import (
DiffusersPipelineLoader,
)
def get_gguf_adapter(
gguf_file: str,
model: torch.nn.Module,
source: DiffusersPipelineLoader.ComponentSource,
od_config: OmniDiffusionConfig,
) -> GGUFAdapter:
adapter_classes = (QwenImageGGUFAdapter, ZImageGGUFAdapter, Flux2KleinGGUFAdapter)
for adapter_cls in adapter_classes:
if adapter_cls.is_compatible(od_config, model, source):
return adapter_cls(gguf_file, model, source, od_config)
model_type = None
if od_config.tf_model_config is not None:
model_type = od_config.tf_model_config.get("model_type")
supported = ", ".join(cls.__name__ for cls in adapter_classes)
raise ValueError(
"No GGUF adapter matched diffusion model "
f"(model_class_name={od_config.model_class_name!r}, model_type={model_type!r}). "
f"Supported adapters: {supported}."
)
__all__ = [
"GGUFAdapter",
"Flux2KleinGGUFAdapter",
"QwenImageGGUFAdapter",
"ZImageGGUFAdapter",
"get_gguf_adapter",
]
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import Generator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
try:
import gguf
except ImportError:
# Optional dependency: only required for GGUF-quantized diffusion models.
# Keep this import lazy so a missing ``gguf`` does not break importing the
# diffusion model loader (and thus inspection/loading of every other
# diffusion model that imports through this package).
gguf = None
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.model_loader.diffusers_loader import (
DiffusersPipelineLoader,
)
@dataclass
class MappedTensor:
name: str
tensor: Any
tensor_type: Any
row_slice: slice | None = None
swap_scale_shift: bool = False
class GGUFAdapter(ABC):
"""Base class for model-specific GGUF adapters."""
def __init__(
self,
gguf_file: str,
model: torch.nn.Module,
source: DiffusersPipelineLoader.ComponentSource,
od_config: OmniDiffusionConfig,
) -> None:
self.gguf_file = gguf_file
self.model = model
self.source = source
self.od_config = od_config
@staticmethod
def is_compatible(
od_config: OmniDiffusionConfig,
model: torch.nn.Module,
source: DiffusersPipelineLoader.ComponentSource,
) -> bool:
return False
@abstractmethod
def weights_iterator(self) -> Generator[tuple[str, torch.Tensor], None, None]:
raise NotImplementedError
# FIXME(Isotr0py): Sync implemnentation with upstream vLLM?
def gguf_quant_weights_iterator(gguf_file: str) -> Generator[tuple[str, torch.Tensor]]:
"""
Iterate over the quant weights in the model gguf files and convert
them to torch tensors.
Be careful of the order of yielding weight types and weights data,
we have to yield all weight types first before yielding any weights.
Otherwise it would cause issue when loading weights with for packed
layer with different quant types.
"""
if gguf is None:
raise ImportError(
"The 'gguf' package is required to load GGUF-quantized diffusion models. "
"Install it with `pip install gguf`."
)
reader = gguf.GGUFReader(gguf_file)
for tensor in reader.tensors:
weight_type = tensor.tensor_type
name = tensor.name
if weight_type.name not in ("F32", "F16"):
weight_type_name = name.replace("weight", "qweight_type")
weight_type = torch.tensor(weight_type)
yield weight_type_name, weight_type
for tensor in reader.tensors:
weight = tensor.data
weight_type = tensor.tensor_type
name = tensor.name
if weight_type.name not in ("F32", "F16"):
name = name.replace("weight", "qweight")
if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
# BF16 is currently the only "quantization" type that isn't
# actually quantized but is read as a raw byte tensor.
# Reinterpret as `torch.bfloat16` tensor.
weight = weight.view(np.uint16)
if reader.byte_order == "S":
# GGUF endianness != system endianness
weight = weight.byteswap()
param = torch.tensor(weight).view(torch.bfloat16)
else:
param = torch.tensor(weight)
yield name, param
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from collections.abc import Iterable
import torch
from vllm.model_executor.models.utils import WeightsMapper
from .base import GGUFAdapter, gguf_quant_weights_iterator
FLUX2_TRANSFORMER_KEYS_RENAME_DICT = {
"single_blocks.": "single_transformer_blocks.",
# Image and text input projections
"img_in": "x_embedder",
"txt_in": "context_embedder",
# Timestep and guidance embeddings
"time_in.in_layer": "time_guidance_embed.timestep_embedder.linear_1",
"time_in.out_layer": "time_guidance_embed.timestep_embedder.linear_2",
"guidance_in.in_layer": "time_guidance_embed.guidance_embedder.linear_1",
"guidance_in.out_layer": "time_guidance_embed.guidance_embedder.linear_2",
# Modulation parameters
"double_stream_modulation_img.lin": "double_stream_modulation_img.linear",
"double_stream_modulation_txt.lin": "double_stream_modulation_txt.linear",
"single_stream_modulation.lin": "single_stream_modulation.linear",
# Final output layer
# "final_layer.adaLN_modulation.1": "norm_out.linear", # Handle separately since we need to swap mod params
"final_layer.linear": "proj_out",
}
FLUX2_TRANSFORMER_ADA_LAYER_NORM_KEY_MAP = {
"final_layer.adaLN_modulation.1": "norm_out.linear",
}
FLUX2_TRANSFORMER_DOUBLE_BLOCK_KEY_MAP = {
"double_blocks.": "transformer_blocks.",
# Handle fused QKV projections separately as we need to break into Q, K, V projections
"img_attn.norm.query_norm": "attn.norm_q",
"img_attn.norm.key_norm": "attn.norm_k",
"img_attn.proj": "attn.to_out.0",
"img_mlp.0": "ff.linear_in",
"img_mlp.2": "ff.linear_out",
"txt_attn.norm.query_norm": "attn.norm_added_q",
"txt_attn.norm.key_norm": "attn.norm_added_k",
"txt_attn.proj": "attn.to_add_out",
"txt_mlp.0": "ff_context.linear_in",
"txt_mlp.2": "ff_context.linear_out",
# Additional for fuse qkv
"img_attn.qkv": "attn.to_qkv",
"txt_attn.qkv": "attn.add_kv_proj",
}
FLUX2_TRANSFORMER_SINGLE_BLOCK_KEY_MAP = {
"linear1": "attn.to_qkv_mlp_proj",
"linear2": "attn.to_out",
"norm.query_norm": "attn.norm_q",
"norm.key_norm": "attn.norm_k",
}
class Flux2KleinGGUFAdapter(GGUFAdapter):
"""GGUF adapter for Flux2-Klein models with qkv splitting and adaLN swap."""
@staticmethod
def is_compatible(od_config, model: torch.nn.Module, source) -> bool:
model_class = od_config.model_class_name or ""
if model_class.startswith("Flux2"):
return True
cfg = od_config.tf_model_config
if cfg is not None:
model_type = str(cfg.get("model_type", "")).lower()
if model_type.startswith("flux"):
return True
return False
gguf_to_hf_mapper = WeightsMapper(
# double_stream_modulation
orig_to_new_prefix=FLUX2_TRANSFORMER_KEYS_RENAME_DICT | FLUX2_TRANSFORMER_ADA_LAYER_NORM_KEY_MAP,
orig_to_new_substr=FLUX2_TRANSFORMER_DOUBLE_BLOCK_KEY_MAP | FLUX2_TRANSFORMER_SINGLE_BLOCK_KEY_MAP,
)
def weights_iterator(self) -> Iterable[tuple[str, torch.Tensor]]:
def custom_weights_adapter(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
for name, weight in weights:
# Handle the special case for adaLN modulation parameters that require swapping shift and scale
if name.endswith(".scale"):
name = name.replace(".scale", ".weight")
if name == "norm_out.linear.weight":
shift, scale = weight.chunk(2, dim=0)
weight = torch.cat([scale, shift], dim=0)
yield name, weight
else:
yield name, weight
weights = gguf_quant_weights_iterator(self.gguf_file)
weights = self.gguf_to_hf_mapper.apply(weights)
yield from custom_weights_adapter(weights)
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from collections.abc import Iterable
import torch
from .base import GGUFAdapter, gguf_quant_weights_iterator
class QwenImageGGUFAdapter(GGUFAdapter):
"""GGUF adapter for the Qwen-Image transformer family."""
@staticmethod
def is_compatible(od_config, model: torch.nn.Module, source) -> bool:
model_class = od_config.model_class_name or ""
if model_class.startswith("QwenImage"):
return True
cfg = od_config.tf_model_config
if cfg is not None:
model_type = str(cfg.get("model_type", "")).lower()
if model_type.startswith("qwen_image"):
return True
return False
def weights_iterator(self) -> Iterable[tuple[str, torch.Tensor]]:
yield from gguf_quant_weights_iterator(self.gguf_file)
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from collections.abc import Iterable
import torch
from vllm.model_executor.models.utils import WeightsMapper
from .base import GGUFAdapter, gguf_quant_weights_iterator
Z_IMAGE_KEYS_RENAME_DICT = {
"final_layer.": "all_final_layer.2-1.",
"x_embedder.": "all_x_embedder.2-1.",
".attention.qkv": ".attention.to_qkv",
".attention.k_norm": ".attention.norm_k",
".attention.q_norm": ".attention.norm_q",
".attention.out": ".attention.to_out.0",
"model.diffusion_model.": "",
}
class ZImageGGUFAdapter(GGUFAdapter):
"""GGUF adapter for Z-Image models with QKV/FFN shard support."""
@staticmethod
def is_compatible(od_config, model: torch.nn.Module, source) -> bool:
model_class = od_config.model_class_name or ""
if model_class.startswith("ZImage"):
return True
cfg = od_config.tf_model_config
if cfg is not None:
model_type = str(cfg.get("model_type", "")).lower()
if model_type in {"z_image", "zimage", "z-image"}:
return True
return False
gguf_to_hf_mapper = WeightsMapper(
orig_to_new_substr=Z_IMAGE_KEYS_RENAME_DICT,
)
def weights_iterator(self) -> Iterable[tuple[str, torch.Tensor]]:
weights = gguf_quant_weights_iterator(self.gguf_file)
yield from self.gguf_to_hf_mapper.apply(weights)