[NPU][Quant] Add W4A4 MXFP4 online & MXFP4 dual-scale online/offline quantization support for Wan2.2 T2V / I2V inference on Ascend NPU (#3578)
Signed-off-by: hyh_hh <huyinghong1@huawei.com> Co-authored-by: hyh_hh <huyinghong1@huawei.com>
This commit is contained in:
1 parent
109bba8cdf
commit
9dd36e3afd
14 files changed
+3646
-61
No files matched your search
@@ -48,6 +48,29 @@ def _build_mxfp8(**kw: Any) -> QuantizationConfig:
|
||||
return DiffusionMXFP8Config(**kw)
|
||||
|
||||
|
||||
def _build_mxfp4(**kw: Any) -> QuantizationConfig:
|
||||
"""Lazy import for W4A4 MXFP4 diffusion config (NPU only)."""
|
||||
from .mxfp4_config import DiffusionMXFP4Config
|
||||
|
||||
return DiffusionMXFP4Config(**kw)
|
||||
|
||||
|
||||
def _build_mxfp4_dualscale(**kw: Any) -> QuantizationConfig:
|
||||
"""Lazy import for MXFP4 DualScale + BF16 mixed diffusion config (NPU only).
|
||||
|
||||
Offline mode (is_checkpoint_serialized=True):
|
||||
ignored_layers from config.json marks interleaved BF16 fallback layers.
|
||||
All other linear layers use W4A4 MXFP4 DualScale.
|
||||
|
||||
Online mode (is_checkpoint_serialized=False):
|
||||
num_bf16_fallback_layers leading transformer blocks use BF16 original weights
|
||||
(default 5 when not specified). Remaining blocks use W4A4 MXFP4 DualScale online.
|
||||
"""
|
||||
from .mxfp4_config import DiffusionMXFP4DualScaleMixedConfig
|
||||
|
||||
return DiffusionMXFP4DualScaleMixedConfig(**kw)
|
||||
|
||||
|
||||
def _build_inc(**kw: Any) -> QuantizationConfig:
|
||||
"""Lazy import for INC/AutoRound config with checkpoint kwarg normalization."""
|
||||
from .inc_config import OmniINCConfig
|
||||
@@ -66,6 +89,8 @@ _OVERRIDES: dict[str, Callable[..., QuantizationConfig]] = {
|
||||
"gguf": _build_gguf,
|
||||
"int8": _build_int8,
|
||||
"mxfp8": _build_mxfp8,
|
||||
"mxfp4": _build_mxfp4,
|
||||
"mxfp4_dualscale": _build_mxfp4_dualscale,
|
||||
"inc": _build_inc,
|
||||
"auto-round": _build_inc,
|
||||
"auto_round": _build_inc,
|
||||
@@ -255,3 +280,80 @@ def build_quant_config(
|
||||
return _build_from_method_and_config(method, merged)
|
||||
|
||||
raise TypeError(f"quantization config must be str, dict, QuantizationConfig, or None, got {type(spec).__name__}")
|
||||
|
||||
|
||||
def _disk_marks_serialized(qc_kwargs: dict[str, Any], quant_config: object) -> bool:
|
||||
"""Return True when config.json says serialized but the active quant_config does not.
|
||||
|
||||
Matches any flag following the is_checkpoint_*_serialized naming convention,
|
||||
so new quant methods don't require updating an explicit allowlist.
|
||||
"""
|
||||
for key, val in qc_kwargs.items():
|
||||
if key.startswith("is_checkpoint_") and key.endswith("_serialized"):
|
||||
if val and hasattr(quant_config, key) and not getattr(quant_config, key):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def resolve_quant_config_from_disk(
|
||||
quant_config: QuantizationConfig | None,
|
||||
disk_qc: dict[str, Any] | str | None,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Reconcile an active quant_config against quantization_config from a transformer's config.json.
|
||||
|
||||
Used when loading individual transformer blocks that each have their own config.json
|
||||
(e.g. cascade models with separate transformer and transformer_2 directories).
|
||||
|
||||
Rules:
|
||||
- disk_qc is None: return quant_config unchanged.
|
||||
- quant_config is None: auto-detect from disk_qc (full build).
|
||||
- Methods mismatch: raise ValueError — prevents silent weight corruption.
|
||||
- Disk marks serialized but quant_config is online: rebuild from disk.
|
||||
- ignored_layers differ: rebuild from disk (per-transformer BF16 routing).
|
||||
"""
|
||||
if disk_qc is None:
|
||||
return quant_config
|
||||
|
||||
if isinstance(disk_qc, str):
|
||||
if quant_config is None:
|
||||
logger.info("Auto-detected quantization from config.json: method=%s", disk_qc)
|
||||
return build_quant_config(disk_qc)
|
||||
return quant_config
|
||||
|
||||
if not isinstance(disk_qc, Mapping) or "quant_method" not in disk_qc:
|
||||
return quant_config
|
||||
|
||||
qc_method: str = disk_qc["quant_method"]
|
||||
qc_kwargs: dict[str, Any] = {k: v for k, v in disk_qc.items() if k != "quant_method"}
|
||||
|
||||
if quant_config is None:
|
||||
logger.info(
|
||||
"Auto-detected quantization from config.json: method=%s kwargs=%s",
|
||||
qc_method,
|
||||
qc_kwargs,
|
||||
)
|
||||
return build_quant_config(qc_method, **qc_kwargs)
|
||||
|
||||
if quant_config.get_name() != qc_method:
|
||||
raise ValueError(
|
||||
f"Checkpoint config.json declares quant_method={qc_method!r} but the "
|
||||
f"active quantization config is {quant_config.get_name()!r}. "
|
||||
"Pass a matching --quantization flag or omit it for auto-detection."
|
||||
)
|
||||
|
||||
if _disk_marks_serialized(qc_kwargs, quant_config):
|
||||
logger.info(
|
||||
"config.json marks checkpoint as serialized; switching to offline %s mode.",
|
||||
qc_method,
|
||||
)
|
||||
return build_quant_config(qc_method, **qc_kwargs)
|
||||
|
||||
if (
|
||||
"ignored_layers" in qc_kwargs
|
||||
and hasattr(quant_config, "ignored_layers")
|
||||
and set(qc_kwargs.get("ignored_layers") or []) != set(quant_config.ignored_layers or [])
|
||||
):
|
||||
logger.info("config.json ignored_layers differs from active config; rebuilding quant_config.")
|
||||
return build_quant_config(qc_method, **qc_kwargs)
|
||||
|
||||
return quant_config
|
||||
@@ -0,0 +1,709 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""W4A4 MXFP4 (Microscaling FP4) online/offline quantization for diffusion transformers.
|
||||
|
||||
Architecture mirrors mxfp8_config.py:
|
||||
|
||||
MXFPLinearMethodBase – platform-agnostic skeleton (imported from mxfp8_config)
|
||||
NPUMxfp4LinearMethod – NPU single-scale offline (W4A4 MXFP4)
|
||||
NPUMxfp4OnlineLinearMethod – NPU single-scale online (BF16 → FP4)
|
||||
NPUMxfp4DualScaleLinearMethod – NPU dual-scale offline (W4A4 MXFP4 DualScale)
|
||||
NPUMxfp4DualScaleOnlineLinearMethod – NPU dual-scale online (BF16 → FP4)
|
||||
|
||||
Quantization configs:
|
||||
|
||||
DiffusionMXFP4Config – single-scale online/offline (quant_method="mxfp4")
|
||||
DiffusionMXFP4DualScaleMixedConfig – dual-scale + per-layer BF16 fallback (quant_method="mxfp4_dualscale")
|
||||
Offline: ignored_layers from config.json routes interleaved BF16 layers
|
||||
Online: num_bf16_fallback_layers leading blocks stay in BF16 (default 5)
|
||||
|
||||
Key differences from MXFP8:
|
||||
|
||||
1. Precision: float4_e2m1fn_x2 (FP4 packed, 2 values per element).
|
||||
npu_dynamic_mx_quant(x) without dst_type defaults to float4_e2m1fn_x2.
|
||||
|
||||
2. Weight layout: stored as (N, K) — NOT pre-transposed.
|
||||
FP4 uses a packed format; transposing a packed tensor is not safely contiguous.
|
||||
Transpose is done inline in _quant_matmul via layer.weight.transpose(0, 1).
|
||||
|
||||
3. GEMM signature: npu_quant_matmul requires explicit
|
||||
x1_dtype=float4_e2m1fn_x2 and x2_dtype=float4_e2m1fn_x2.
|
||||
|
||||
Scale layout: (N, S/2, 2) — same reshape as MXFP8, also NOT pre-transposed;
|
||||
transposed inline in _quant_matmul.
|
||||
|
||||
Dual-scale (W4A4_MXFP4_DUALSCALE):
|
||||
|
||||
Two-level quantization: fine scale (per-32 K) + coarse scale (per-512 K) + per-channel
|
||||
activation pre-scale (mul_scale from calibration). Uses npu_dynamic_dual_level_mx_quant
|
||||
and npu_dual_level_quant_matmul. Weight stored in NZ hardware format via npu_format_cast.
|
||||
|
||||
Checkpoint tensor shapes for dual-scale:
|
||||
weight : (N, K) float8_e4m3fn – FP4 packed
|
||||
weight_scale : (N, K//32) uint8 – fine scale (float8_e8m0fnu bits)
|
||||
weight_dual_scale : (N, K//512, 1) float32 – coarse scale (extra dim avoids shape assert)
|
||||
mul_scale : (K,) float32 – per-input-channel activation pre-scale
|
||||
|
||||
Reference: MindIE-SD W4A4MXFP4QuantLinear / W4A4MXFP4DualQuantLinear (mindiesd/quantization/layer.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import _copy_missing_attrs
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.model_loader.weight_utils import initialize_single_dummy_weight
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.quantization.mxfp8_config import (
|
||||
MXFPLinearMethodBase,
|
||||
_LazyWeightMixin,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DiffusionMXFP4Config(QuantizationConfig):
|
||||
"""W4A4 MXFP4 quantization config for diffusion transformers.
|
||||
|
||||
Supports both online (BF16 checkpoint → quantize at load time) and offline
|
||||
(pre-quantized MXFP4 checkpoint) modes, mirroring DiffusionMXFP8Config.
|
||||
|
||||
MX (microscaling) format: groups of 32 K-dimension elements share one
|
||||
float8_e8m0fnu exponent scale. Weight and activation are float4_e2m1fn_x2.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_checkpoint_mxfp4_serialized: bool = False,
|
||||
ignored_layers: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.is_checkpoint_mxfp4_serialized = is_checkpoint_mxfp4_serialized
|
||||
self.ignored_layers = ignored_layers or []
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "mxfp4"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.bfloat16, torch.float16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 80
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
|
||||
if self.ignored_layers:
|
||||
self.ignored_layers = hf_to_vllm_mapper.apply_list(self.ignored_layers)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> DiffusionMXFP4Config:
|
||||
is_serialized = cls.get_from_keys_or(config, ["is_checkpoint_mxfp4_serialized"], False)
|
||||
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
|
||||
if not ignored_layers:
|
||||
ignored_layers = cls.get_from_keys_or(config, ["modules_to_not_convert"], None)
|
||||
return cls(
|
||||
is_checkpoint_mxfp4_serialized=is_serialized,
|
||||
ignored_layers=ignored_layers,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
prefix: str,
|
||||
) -> QuantizeMethodBase | None:
|
||||
if isinstance(layer, LinearBase):
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
if current_omni_platform.is_npu():
|
||||
if self.is_checkpoint_mxfp4_serialized:
|
||||
return NPUMxfp4LinearMethod(self)
|
||||
return NPUMxfp4OnlineLinearMethod(self)
|
||||
raise NotImplementedError(
|
||||
"DiffusionMXFP4Config (W4A4 MXFP4) is currently only supported "
|
||||
"on NPU (Ascend) platforms. CUDA support is not yet implemented."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NPU MXFP4 single-scale offline method (pre-quantized checkpoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NPUMxfp4LinearMethod(MXFPLinearMethodBase):
|
||||
"""NPU W4A4 MXFP4 offline linear method for pre-quantized checkpoints.
|
||||
|
||||
Weight canonical layout after process_weights_after_loading:
|
||||
weight : (N, K) in float4_e2m1fn_x2 — NOT pre-transposed (FP4 packed)
|
||||
weight_scale: (N, S/2, 2) in float8_e8m0fnu — NOT pre-transposed
|
||||
|
||||
Both are transposed inline in _quant_matmul, unlike MXFP8 which pre-transposes.
|
||||
NPUMxfp4OnlineLinearMethod normalizes to the same layout so apply() is shared.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: DiffusionMXFP4Config) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
# BF16 placeholder; cast to float4_e2m1fn_x2 in process_weights.
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
# Scale stored as uint8 in safetensors (float8_e8m0fnu is same bit width).
|
||||
# Using uint8 avoids a lossy float32 round-trip when loading the checkpoint.
|
||||
num_groups = (input_size_per_partition + 31) // 32
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, num_groups, dtype=torch.uint8),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
import torch_npu
|
||||
|
||||
# NPU: cast to float4_e2m1fn_x2. Weight stays (N, K) — no pre-transpose.
|
||||
w = layer.weight
|
||||
if w.dtype != torch_npu.float4_e2m1fn_x2:
|
||||
w = torch_npu.npu_dtype_cast(w.npu(), torch_npu.float4_e2m1fn_x2)
|
||||
|
||||
# Scale: checkpoint stores uint8 bytes that ARE float8_e8m0fnu bits.
|
||||
# Only convert if neither uint8 nor the target NPU dtype already.
|
||||
# (N, K_groups) → (N, K_groups/2, 2). Not pre-transposed; done inline in _quant_matmul.
|
||||
s = layer.weight_scale.data
|
||||
if s.dtype not in (torch.uint8, torch_npu.float8_e8m0fnu):
|
||||
s = s.to(torch_npu.float8_e8m0fnu)
|
||||
N, K_groups = s.shape
|
||||
if K_groups % 2 == 1:
|
||||
s = torch.cat([s, torch.zeros(N, 1, dtype=s.dtype, device=s.device)], dim=1)
|
||||
s = s.reshape(N, -1, 2).contiguous()
|
||||
|
||||
replace_parameter(layer, "weight", w)
|
||||
replace_parameter(layer, "weight_scale", s)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
# --- NPU MXFP4 ops — shared with online path via inheritance ---
|
||||
|
||||
def _quantize_activation(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import torch_npu
|
||||
|
||||
# No dst_type: npu_dynamic_mx_quant defaults to float4_e2m1fn_x2.
|
||||
return torch_npu.npu_dynamic_mx_quant(x)
|
||||
|
||||
def _quant_matmul(
|
||||
self,
|
||||
x_q: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
layer: torch.nn.Module,
|
||||
bias: torch.Tensor | None,
|
||||
ori_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
import torch_npu
|
||||
|
||||
if bias is not None and bias.dtype != torch.float32:
|
||||
bias = bias.to(torch.float32)
|
||||
# FP4 differences vs FP8:
|
||||
# weight (N,K) transposed inline → (K,N); scale (N,S/2,2) transposed inline → (S/2,N,2).
|
||||
# x1_dtype / x2_dtype required — FP4 dtype not inferred from tensor dtype.
|
||||
return torch_npu.npu_quant_matmul(
|
||||
x_q,
|
||||
layer.weight.transpose(0, 1), # (K, N) inline
|
||||
layer.weight_scale.transpose(0, 1), # (S/2, N, 2) inline
|
||||
scale_dtype=torch_npu.float8_e8m0fnu,
|
||||
x1_dtype=torch_npu.float4_e2m1fn_x2,
|
||||
x2_dtype=torch_npu.float4_e2m1fn_x2,
|
||||
pertoken_scale=x_scale,
|
||||
pertoken_scale_dtype=torch_npu.float8_e8m0fnu,
|
||||
bias=bias,
|
||||
output_dtype=ori_dtype,
|
||||
group_sizes=[1, 1, 32],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NPU MXFP4 single-scale online method (BF16 checkpoint → quantize at load time)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NPUMxfp4OnlineLinearMethod(_LazyWeightMixin, NPUMxfp4LinearMethod):
|
||||
"""NPU W4A4 MXFP4 online linear method.
|
||||
|
||||
MRO: NPUMxfp4OnlineLinearMethod → _LazyWeightMixin → NPUMxfp4LinearMethod
|
||||
→ MXFPLinearMethodBase → LinearMethodBase
|
||||
|
||||
create_weights : _LazyWeightMixin (meta device + patched loader)
|
||||
process_weights : NPUMxfp4OnlineLinearMethod (BF16 → FP4 + normalize)
|
||||
apply / ops : NPUMxfp4LinearMethod / MXFPLinearMethodBase (shared)
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
import torch_npu
|
||||
|
||||
if layer.weight.device == torch.device("meta"):
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty_like(layer.weight, device=layer._load_device),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=layer.weight.weight_loader,
|
||||
)
|
||||
_copy_missing_attrs(layer.weight, weight)
|
||||
layer.register_parameter("weight", weight)
|
||||
initialize_single_dummy_weight(layer.weight)
|
||||
|
||||
# NPU: quantize BF16/FP16 (N, K) → FP4. No dst_type → float4_e2m1fn_x2.
|
||||
weight_fp4, weight_scale_raw = torch_npu.npu_dynamic_mx_quant(layer.weight)
|
||||
|
||||
# Weight stays (N, K) — no pre-transpose for FP4 packed format.
|
||||
# Scale: (N, S) → (N, S/2, 2). Not pre-transposed; done inline in _quant_matmul.
|
||||
weight_scale = weight_scale_raw.reshape(weight_scale_raw.shape[0], -1, 2).contiguous()
|
||||
|
||||
replace_parameter(layer, "weight", weight_fp4)
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NPU MXFP4 dual-scale offline method (W4A4_MXFP4_DUALSCALE checkpoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NPUMxfp4DualScaleLinearMethod(MXFPLinearMethodBase):
|
||||
"""NPU W4A4 MXFP4 dual-scale offline method for pre-quantized checkpoints.
|
||||
|
||||
Checkpoint tensors and their canonical post-load shapes:
|
||||
weight : (N, K) float8_e4m3fn – FP4 packed (2 values per byte)
|
||||
weight_scale : (N, K//32) uint8 – fine scale (float8_e8m0fnu bits); reshaped to (N, K//64, 2)
|
||||
weight_dual_scale : (N, K//512, 1) float32 – coarse scale; transposed to (K//512, N)
|
||||
mul_scale : (K,) float32 – per-input-channel activation pre-scale (from calibration)
|
||||
|
||||
Forward pass:
|
||||
x_q, l0, l1 = npu_dynamic_dual_level_mx_quant(x, smooth_scale=mul_scale)
|
||||
out = npu_dual_level_quant_matmul(x_q, weight, l0, weight_dual_scale, l1, weight_scale)
|
||||
|
||||
Reference: MindIE-SD W4A4MXFP4DualQuantLinear.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: Any) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
# FP4 packed: 2 values per float8_e4m3fn byte → checkpoint stores as float8_e4m3fn; register as BF16
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
# Fine scale: one uint8 exponent (float8_e8m0fnu bit pattern) per group of 32 K elements.
|
||||
# input_dim=1: RowParallelLinear weight_loader shards along the K-group dimension (dim 1)
|
||||
# so each rank receives only its (K/TP)//32 groups; ColumnParallelLinear only uses
|
||||
# output_dim=0 for sharding and leaves dim 1 intact.
|
||||
num_groups_fine = (input_size_per_partition + 31) // 32
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, num_groups_fine, dtype=torch.uint8),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
# Coarse scale: one float32 per group of 512 K elements.
|
||||
# Shape (N, K_coarse, 1) matches checkpoint layout exactly, avoiding the
|
||||
# shape-mismatch assert in linear.py:1344.
|
||||
# input_dim=1: same TP sharding rationale as weight_scale above.
|
||||
num_groups_coarse = (input_size_per_partition + 511) // 512
|
||||
layer.register_parameter(
|
||||
"weight_dual_scale",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, num_groups_coarse, 1, dtype=torch.float32),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
# mul_scale is a float32 calibration tensor; register as float32
|
||||
# to avoid precision loss from an implicit BF16 cast during weight loading.
|
||||
# input_dim=0: RowParallelLinear shards the 1-D per-input-channel tensor along dim 0
|
||||
# so each rank receives only its K/TP channels; ColumnParallelLinear leaves it intact.
|
||||
layer.register_parameter(
|
||||
"mul_scale",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(input_size_per_partition, dtype=torch.float32),
|
||||
input_dim=0,
|
||||
output_dim=None,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
setattr(layer.mul_scale, "ignore_warning", True)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
import torch_npu
|
||||
|
||||
# float8_e4m3fn (FP4 packed) → float4_e2m1fn_x2 → NZ hardware format (format ID 29).
|
||||
# NZ layout is required by npu_dual_level_quant_matmul for FP4 weight matrices.
|
||||
w = torch_npu.npu_dtype_cast(layer.weight.data.npu(), torch_npu.float4_e2m1fn_x2)
|
||||
w = torch_npu.npu_format_cast(w.view(torch.int8), 29, customize_dtype=torch.int8)
|
||||
|
||||
# Fine scale: (N, K//32) uint8 → cast to float8_e8m0fnu → (N, K//64, 2).
|
||||
# The cast reinterprets the stored uint8 bit-patterns as float8_e8m0fnu exponents,
|
||||
# which is required for npu_dual_level_quant_matmul to compute correct scale values.
|
||||
s = layer.weight_scale.data.npu()
|
||||
s = s.reshape(s.shape[0], -1, 2).contiguous()
|
||||
|
||||
# Coarse scale: (N, K//512, 1) → squeeze → (N, K//512) → transpose → (K//512, N).
|
||||
ds = layer.weight_dual_scale.data.squeeze(-1).transpose(0, 1).npu().contiguous()
|
||||
|
||||
ms = layer.mul_scale.to(torch.bfloat16).data.view(-1).npu().contiguous()
|
||||
|
||||
replace_parameter(layer, "weight", w)
|
||||
replace_parameter(layer, "weight_scale", s)
|
||||
replace_parameter(layer, "weight_dual_scale", ds)
|
||||
replace_parameter(layer, "mul_scale", ms)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
def _apply_inner(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
ori_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dual-scale inner loop: dtype guard → 3-tuple quantize (mul_scale as smooth) → matmul.
|
||||
|
||||
Overrides the default single-scale _apply_inner from MXFPLinearMethodBase.
|
||||
apply() in the base class handles reshape/unreshape; this method is not
|
||||
responsible for that.
|
||||
"""
|
||||
if ori_dtype not in (torch.bfloat16, torch.float16):
|
||||
x = x.to(torch.bfloat16)
|
||||
x_q, l0_scale, l1_scale = self._quantize_activation(x, layer.mul_scale)
|
||||
return self._quant_matmul(x_q, l0_scale, l1_scale, layer, bias, ori_dtype)
|
||||
|
||||
def _quantize_activation( # type: ignore[override]
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
smooth_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
import torch_npu
|
||||
|
||||
return torch_npu.npu_dynamic_dual_level_mx_quant(x, smooth_scale=smooth_scale)
|
||||
|
||||
def _quant_matmul( # type: ignore[override]
|
||||
self,
|
||||
x_q: torch.Tensor,
|
||||
l0_scale: torch.Tensor,
|
||||
l1_scale: torch.Tensor,
|
||||
layer: torch.nn.Module,
|
||||
bias: torch.Tensor | None,
|
||||
ori_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
import torch_npu
|
||||
|
||||
if bias is not None and bias.dtype != torch.float32:
|
||||
bias = bias.to(torch.float32)
|
||||
# weight_scale is (N, K//64, 2) float8_e8m0fnu — operator expects output-major layout.
|
||||
# weight_dual_scale is (K//512, N) float32 — transposed to K-major in process_weights.
|
||||
return torch_npu.npu_dual_level_quant_matmul(
|
||||
x_q,
|
||||
layer.weight,
|
||||
l0_scale,
|
||||
layer.weight_dual_scale,
|
||||
l1_scale,
|
||||
layer.weight_scale,
|
||||
bias=bias,
|
||||
output_dtype=ori_dtype,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NPU MXFP4 dual-scale online method (BF16 checkpoint → quantize at load time)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NPUMxfp4DualScaleOnlineLinearMethod(_LazyWeightMixin, NPUMxfp4DualScaleLinearMethod):
|
||||
"""NPU W4A4 MXFP4 dual-scale online method: quantises BF16 weights at load time.
|
||||
|
||||
MRO: NPUMxfp4DualScaleOnlineLinearMethod → _LazyWeightMixin
|
||||
→ NPUMxfp4DualScaleLinearMethod → MXFPLinearMethodBase
|
||||
|
||||
create_weights : _LazyWeightMixin (meta device + patched loader)
|
||||
process_weights : NPUMxfp4DualScaleOnlineLinearMethod (BF16 → FP4 + dual scales)
|
||||
apply / _quant_matmul : NPUMxfp4DualScaleLinearMethod (shared with offline path)
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
import torch_npu
|
||||
|
||||
if layer.weight.device == torch.device("meta"):
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty_like(layer.weight, device=layer._load_device),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=layer.weight.weight_loader,
|
||||
)
|
||||
_copy_missing_attrs(layer.weight, weight)
|
||||
layer.register_parameter("weight", weight)
|
||||
initialize_single_dummy_weight(layer.weight)
|
||||
|
||||
# Quantize BF16 weight → FP4 + dual-level scales (no smooth pre-scale for online).
|
||||
# Returns: (weight_fp4, l0_scale[coarse per-512], l1_scale[fine per-32])
|
||||
weight_fp4, w_l0_scale, w_l1_scale = torch_npu.npu_dynamic_dual_level_mx_quant(
|
||||
layer.weight.data.npu(), smooth_scale=None
|
||||
)
|
||||
|
||||
# NZ hardware format for the FP4 weight (same as offline path).
|
||||
w = torch_npu.npu_format_cast(weight_fp4.view(torch.int8), 29, customize_dtype=torch.int8)
|
||||
|
||||
# Fine scale (l1): (N, K//32) → (N, K//64, 2). Dtype from op output (float8_e8m0fnu).
|
||||
s = w_l1_scale.reshape(w_l1_scale.shape[0], -1, 2).contiguous()
|
||||
|
||||
# Coarse scale (l0): (N, K_coarse) → (K_coarse, N). Dtype from op output.
|
||||
ds = w_l0_scale.reshape(w_l0_scale.shape[0], -1).transpose(0, 1).contiguous()
|
||||
|
||||
# No calibration available: identity pre-scale (no smooth quantization effect).
|
||||
ms = torch.ones(layer.input_size_per_partition, dtype=torch.bfloat16, device="npu")
|
||||
|
||||
replace_parameter(layer, "weight", w)
|
||||
replace_parameter(layer, "weight_scale", s)
|
||||
replace_parameter(layer, "weight_dual_scale", ds)
|
||||
replace_parameter(layer, "mul_scale", ms)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Block-index helper (shared by DiffusionMXFP4DualScaleMixedConfig)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BLOCK_IDX_RE = re.compile(r"^blocks\.(\d+)\.")
|
||||
|
||||
|
||||
def _parse_block_idx(prefix: str) -> int | None:
|
||||
"""Extract block index from prefix like 'blocks.5.attn1.to_q'."""
|
||||
m = _BLOCK_IDX_RE.match(prefix)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config: MXFP4 DualScale + per-layer BF16 fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DiffusionMXFP4DualScaleMixedConfig(QuantizationConfig):
|
||||
"""W4A4 MXFP4 DualScale with per-layer BF16 fallback for diffusion transformers.
|
||||
|
||||
Sensitive layers fall back to BF16 (original weights) while all other linear
|
||||
layers use W4A4 MXFP4 DualScale. BF16 fallback layers may be interleaved
|
||||
anywhere in the transformer.
|
||||
|
||||
Offline mode (is_checkpoint_serialized=True):
|
||||
Layers whose prefix appears in ignored_layers → UnquantizedLinearMethod (BF16)
|
||||
All other linear layers → NPUMxfp4DualScaleLinearMethod
|
||||
|
||||
ignored_layers is injected into transformer/config.json by the merge script
|
||||
and contains the prefixes of all non-MXFP4 linear layers.
|
||||
|
||||
Online mode (is_checkpoint_serialized=False):
|
||||
Layer routing applies two rules in priority order:
|
||||
1. ignored_layers (explicit per-layer BF16 override, user-supplied) → BF16
|
||||
2. Blocks 0 .. num_bf16_fallback_layers-1 (coarse leading-block rule) → BF16
|
||||
3. All other linear layers → NPUMxfp4DualScaleOnlineLinearMethod
|
||||
|
||||
num_bf16_fallback_layers defaults to 5 when not specified.
|
||||
Set ignored_layers to pin arbitrary interleaved layers to BF16 without
|
||||
needing an offline checkpoint (useful for accuracy debugging).
|
||||
Layers outside "blocks.N.*" (condition_embedder etc.) always use online MXFP4
|
||||
unless they appear in ignored_layers.
|
||||
|
||||
Config injected by merge_mxfp4_dualscale_checkpoint.py:
|
||||
{
|
||||
"quant_method": "mxfp4_dualscale",
|
||||
"is_checkpoint_serialized": true,
|
||||
"ignored_layers": ["blocks.0.attn1.to_q", ...]
|
||||
}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_checkpoint_serialized: bool = False,
|
||||
ignored_layers: list[str] | None = None,
|
||||
num_bf16_fallback_layers: int = 5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.is_checkpoint_serialized = is_checkpoint_serialized
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.num_bf16_fallback_layers = num_bf16_fallback_layers
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "mxfp4_dualscale"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.bfloat16, torch.float16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 80
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
|
||||
if self.ignored_layers:
|
||||
self.ignored_layers = hf_to_vllm_mapper.apply_list(self.ignored_layers)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> DiffusionMXFP4DualScaleMixedConfig:
|
||||
is_serialized = cls.get_from_keys_or(config, ["is_checkpoint_serialized"], False)
|
||||
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
|
||||
if not ignored_layers:
|
||||
ignored_layers = cls.get_from_keys_or(config, ["modules_to_not_convert"], None)
|
||||
num_bf16_fallback_layers = cls.get_from_keys_or(config, ["num_bf16_fallback_layers"], 5)
|
||||
return cls(
|
||||
is_checkpoint_serialized=is_serialized,
|
||||
ignored_layers=ignored_layers,
|
||||
num_bf16_fallback_layers=num_bf16_fallback_layers,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
prefix: str,
|
||||
) -> QuantizeMethodBase | None:
|
||||
if not isinstance(layer, LinearBase):
|
||||
return None
|
||||
|
||||
if self.is_checkpoint_serialized:
|
||||
# Offline: ignored_layers lists interleaved BF16 fallback layer prefixes.
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
if not current_omni_platform.is_npu():
|
||||
raise NotImplementedError(
|
||||
"DiffusionMXFP4DualScaleMixedConfig is currently only supported on NPU (Ascend) platforms."
|
||||
)
|
||||
return NPUMxfp4DualScaleLinearMethod(self)
|
||||
|
||||
# Online: explicit ignored_layers take priority (user-specified per-layer BF16 override),
|
||||
# then fall back to the coarse leading-block rule.
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
block_idx = _parse_block_idx(prefix)
|
||||
if block_idx is not None and block_idx < self.num_bf16_fallback_layers:
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
if not current_omni_platform.is_npu():
|
||||
raise NotImplementedError(
|
||||
"DiffusionMXFP4DualScaleMixedConfig is currently only supported on NPU (Ascend) platforms."
|
||||
)
|
||||
return NPUMxfp4DualScaleOnlineLinearMethod(self)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Merge W4A4_MXFP4_DUALSCALE quantized Wan2.2 weights into HF Diffusers format.
|
||||
|
||||
msModelSlim produces a checkpoint where each linear layer is either:
|
||||
- W4A4_MXFP4_DUALSCALE: quantized weights with fine/coarse scales and mul_scale
|
||||
- FLOAT (BF16 fallback): kept as original BF16 weights for precision-sensitive layers
|
||||
|
||||
BF16 fallback layers may be interleaved anywhere in the transformer (not just leading
|
||||
blocks). The merge script detects them from quant_model_description.json and writes
|
||||
their prefixes into config.json as ignored_layers so the runtime dynamically routes
|
||||
each layer to the correct quantization method at weight-loading time.
|
||||
|
||||
MXFP4_DUALSCALE key structure per linear layer (msModelSlim naming)
|
||||
---------------------------------------------------------------------
|
||||
blocks.N.X.linear.weight W4A4_MXFP4_DUALSCALE – int8 (FP4 packed)
|
||||
blocks.N.X.linear.weight_scale W4A4_MXFP4_DUALSCALE – uint8 (float8_e8m0fnu fine scale, per-32K)
|
||||
blocks.N.X.linear.weight_dual_scale W4A4_MXFP4_DUALSCALE – float32 (coarse scale, per-512K)
|
||||
blocks.N.X.linear.bias FLOAT – bias (if present)
|
||||
blocks.N.X.div.mul_scale FLOAT – float32 per-input-channel activation pre-scale
|
||||
|
||||
BF16 fallback key structure (no wrappers, plain linear weights):
|
||||
blocks.N.X.weight FLOAT
|
||||
condition_embedder.*.weight FLOAT (always BF16 — not quantized)
|
||||
|
||||
Self-attention QKV notes
|
||||
------------------------
|
||||
Self-attention Q/K/V weights are separate in the checkpoint (self_attn.q/k/v) but fused
|
||||
into a single to_qkv layer in vllm-omni. The transformer's load_weights() handles this via
|
||||
stacked_params_mapping. This script keeps Q/K/V keys separate — do NOT pre-fuse them here.
|
||||
|
||||
For mul_scale specifically: even though Q/K/V process the same input (same mul_scale value),
|
||||
they are kept separate. load_weights() routes all three to the same to_qkv.mul_scale parameter
|
||||
and each overwrites the previous. Since Q=K=V for mul_scale, the final value is correct.
|
||||
|
||||
NOTE: Pre-fusing as to_qkv.mul_scale would BREAK loading because ".attn1.to_q" is a
|
||||
substring of ".attn1.to_qkv", causing load_weights() stacked_params_mapping to produce a
|
||||
garbage key ("to_qkvkv") that is not in params_dict, triggering a break that skips the
|
||||
direct-load else branch entirely.
|
||||
|
||||
Supported model types:
|
||||
- Wan2.2-T2V-A14B (MoE cascade: transformer + transformer_2)
|
||||
- Wan2.2-I2V-A14B (MoE cascade: transformer + transformer_2)
|
||||
|
||||
Note: Wan2.2-TI2V-5B is NOT supported. Its smaller parameter count causes
|
||||
unacceptable accuracy loss under W4A4 quantization. Use MXFP8 for TI2V-5B.
|
||||
|
||||
Usage:
|
||||
python merge_mxfp4_dualscale_checkpoint.py \\
|
||||
--model-type Wan2.2-T2V-A14B \\
|
||||
--original-model /path/to/Wan2.2-T2V-A14B-Diffusers \\
|
||||
--quant-path /path/to/msmodelslim-output \\
|
||||
--output-path /path/to/merged-output
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key rename: msModelSlim naming → Diffusers / vllm-omni naming
|
||||
# Identical to merge_mxfp8_checkpoint.py; applied to all blocks uniformly.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TRANSFORMER_KEYS_RENAME_DICT: dict[str, str] = {
|
||||
"time_embedding.0": "condition_embedder.time_embedder.linear_1",
|
||||
"time_embedding.2": "condition_embedder.time_embedder.linear_2",
|
||||
"text_embedding.0": "condition_embedder.text_embedder.linear_1",
|
||||
"text_embedding.2": "condition_embedder.text_embedder.linear_2",
|
||||
"time_projection.1": "condition_embedder.time_proj",
|
||||
"head.modulation": "scale_shift_table",
|
||||
"head.head": "proj_out",
|
||||
"modulation": "scale_shift_table",
|
||||
"ffn.0": "ffn.net.0.proj",
|
||||
"ffn.2": "ffn.net.2",
|
||||
# Norm order swap (quant tool: norm1, norm3, norm2 → diffusers: norm1, norm2, norm3)
|
||||
"norm2": "norm__placeholder",
|
||||
"norm3": "norm2",
|
||||
"norm__placeholder": "norm3",
|
||||
# Self-attention
|
||||
"self_attn.q": "attn1.to_q",
|
||||
"self_attn.k": "attn1.to_k",
|
||||
"self_attn.v": "attn1.to_v",
|
||||
"self_attn.o": "attn1.to_out.0",
|
||||
"self_attn.norm_q": "attn1.norm_q",
|
||||
"self_attn.norm_k": "attn1.norm_k",
|
||||
# Cross-attention
|
||||
"cross_attn.q": "attn2.to_q",
|
||||
"cross_attn.k": "attn2.to_k",
|
||||
"cross_attn.v": "attn2.to_v",
|
||||
"cross_attn.o": "attn2.to_out.0",
|
||||
"cross_attn.norm_q": "attn2.norm_q",
|
||||
"cross_attn.norm_k": "attn2.norm_k",
|
||||
"attn2.to_k_img": "attn2.add_k_proj",
|
||||
"attn2.to_v_img": "attn2.add_v_proj",
|
||||
"attn2.norm_k_img": "attn2.norm_added_k",
|
||||
# I2V image embedder
|
||||
"img_emb.proj.0": "condition_embedder.image_embedder.norm1",
|
||||
"img_emb.proj.1": "condition_embedder.image_embedder.ff.net.0.proj",
|
||||
"img_emb.proj.3": "condition_embedder.image_embedder.ff.net.2",
|
||||
"img_emb.proj.4": "condition_embedder.image_embedder.norm2",
|
||||
"img_emb.emb_pos": "condition_embedder.image_embedder.pos_embed",
|
||||
}
|
||||
|
||||
SUPPORTED_MODEL_TYPES = ["Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B"]
|
||||
CASCADE_MODEL_TYPES = {"Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B"}
|
||||
|
||||
# Suffixes that appear inside .linear.* wrapper for MXFP4 tensors.
|
||||
_LINEAR_ATTRS = ("weight_dual_scale", "weight_scale", "weight", "bias")
|
||||
|
||||
_BLOCK_IDX_RE = re.compile(r"^blocks\.(\d+)\.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key transformation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_block_idx(key: str) -> int | None:
|
||||
"""Extract block index from a key like 'blocks.5.attn1.to_q.weight'."""
|
||||
m = _BLOCK_IDX_RE.match(key)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _apply_rename_dict(key: str) -> str:
|
||||
for src, dst in TRANSFORMER_KEYS_RENAME_DICT.items():
|
||||
key = key.replace(src, dst)
|
||||
return key
|
||||
|
||||
|
||||
def _strip_mxfp4_wrapper(key: str) -> str:
|
||||
"""Strip .linear.ATTR or .div.mul_scale wrappers added by msModelSlim.
|
||||
|
||||
MXFP4 tensors are wrapped in sub-modules:
|
||||
X.linear.weight / weight_scale / weight_dual_scale / bias
|
||||
X.div.mul_scale
|
||||
|
||||
MXFP8 and FLOAT tensors have no wrappers — this function is a no-op for them.
|
||||
|
||||
Examples after apply_rename_dict:
|
||||
attn1.to_q.linear.weight → attn1.to_q.weight
|
||||
attn1.to_q.linear.weight_scale → attn1.to_q.weight_scale
|
||||
attn1.to_q.div.mul_scale → attn1.to_q.mul_scale
|
||||
attn1.norm_q.weight → attn1.norm_q.weight (unchanged)
|
||||
"""
|
||||
# Check longest attribute names first to avoid partial suffix matches.
|
||||
for attr in _LINEAR_ATTRS:
|
||||
suffix = f".linear.{attr}"
|
||||
if key.endswith(suffix):
|
||||
return key[: -len(suffix)] + f".{attr}"
|
||||
if key.endswith(".div.mul_scale"):
|
||||
return key[: -len(".div.mul_scale")] + ".mul_scale"
|
||||
return key
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quantization metadata helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Known quantized weight types (FLOAT tensors don't determine block type).
|
||||
_MXFP8_TYPE = "mxfp8"
|
||||
_MXFP4_DUALSCALE_TYPE = "mxfp4_dualscale"
|
||||
|
||||
|
||||
def _classify_blocks(quant_meta: dict[str, str]) -> dict[int, str]:
|
||||
"""Classify each transformer block by quantization type from quant_meta.
|
||||
|
||||
Returns a dict mapping block_idx → 'mxfp8' | 'mxfp4_dualscale'.
|
||||
A block's type is determined by the first quantized (non-FLOAT) tensor found for it.
|
||||
"""
|
||||
block_types: dict[int, str] = {}
|
||||
for key, qtype in quant_meta.items():
|
||||
idx = _parse_block_idx(key)
|
||||
if idx is None or idx in block_types:
|
||||
continue
|
||||
if qtype.startswith("W8A8_MXFP8"):
|
||||
block_types[idx] = _MXFP8_TYPE
|
||||
elif qtype.startswith("W4A4_MXFP4_DUALSCALE"):
|
||||
block_types[idx] = _MXFP4_DUALSCALE_TYPE
|
||||
return block_types
|
||||
|
||||
|
||||
def _print_block_summary(block_types: dict[int, str]) -> None:
|
||||
"""Print a compact run-length summary of the block layout."""
|
||||
if not block_types:
|
||||
print(" Block layout: (empty)")
|
||||
return
|
||||
|
||||
sorted_indices = sorted(block_types)
|
||||
runs: list[tuple[int, int, str]] = []
|
||||
run_start = sorted_indices[0]
|
||||
run_type = block_types[run_start]
|
||||
for idx in sorted_indices[1:]:
|
||||
if block_types[idx] != run_type:
|
||||
runs.append((run_start, idx - 1, run_type))
|
||||
run_start = idx
|
||||
run_type = block_types[idx]
|
||||
runs.append((run_start, sorted_indices[-1], run_type))
|
||||
|
||||
print(f" Block layout ({len(sorted_indices)} blocks classified):")
|
||||
for start, end, btype in runs:
|
||||
count = end - start + 1
|
||||
range_str = f"{start}" if start == end else f"{start}–{end}"
|
||||
print(f" blocks {range_str:>8}: {btype} ({count} block{'s' if count > 1 else ''})")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safetensors I/O
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_safetensors_dir(directory: pathlib.Path, glob: str = "*.safetensors") -> dict[str, torch.Tensor]:
|
||||
candidates = sorted(directory.glob(glob))
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"No safetensors matching '{glob}' found in {directory}")
|
||||
state: dict[str, torch.Tensor] = {}
|
||||
for f in candidates:
|
||||
state.update(load_file(str(f)))
|
||||
return state
|
||||
|
||||
|
||||
def _load_quant_safetensors(directory: pathlib.Path) -> dict[str, torch.Tensor]:
|
||||
try:
|
||||
return _load_safetensors_dir(directory, "quant_model_weight*.safetensors")
|
||||
except FileNotFoundError:
|
||||
return _load_safetensors_dir(directory)
|
||||
|
||||
|
||||
def _load_quant_meta(directory: pathlib.Path) -> dict[str, str]:
|
||||
candidates = sorted(directory.glob("quant_model_description*.json"))
|
||||
if not candidates:
|
||||
print(f" WARNING: No quant_model_description*.json in {directory}; treating all tensors as FLOAT.")
|
||||
return {}
|
||||
with open(candidates[0]) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-transformer conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _is_mxfp4_tensor(key: str, qtype: str) -> bool:
|
||||
"""Return True for MXFP4 quantized tensors and their companion tensors.
|
||||
|
||||
MXFP4 DualScale layers produce four tensor types:
|
||||
.linear.weight / .linear.weight_scale / .linear.weight_dual_scale (W4A4_MXFP4_DUALSCALE)
|
||||
.linear.bias / .div.mul_scale (FLOAT but companion to a quantized layer)
|
||||
|
||||
BF16 fallback layers have plain .weight / .bias without .linear. / .div. wrappers.
|
||||
"""
|
||||
if qtype.startswith("W4A4_MXFP4_DUALSCALE"):
|
||||
return True
|
||||
# Companion tensors: bias and mul_scale that belong to an MXFP4 layer.
|
||||
return ".linear." in key or ".div.mul_scale" in key
|
||||
|
||||
|
||||
def _diffusers_to_vllm_ignored(diffusers_ignored: list[str]) -> list[str]:
|
||||
"""Translate diffusers checkpoint prefixes to vllm-omni model parameter names.
|
||||
|
||||
Three transformations align the naming conventions:
|
||||
|
||||
1. Self-attention QKV fusion (attn1 only, not attn2):
|
||||
attn1.to_q + attn1.to_k + attn1.to_v → attn1.to_qkv
|
||||
Cross-attention (attn2) to_q/k/v are not fused and remain separate.
|
||||
|
||||
2. FFN naming:
|
||||
ffn.net.0.proj → ffn.net_0.proj
|
||||
ffn.net.2 → ffn.net_2
|
||||
|
||||
3. to_out index:
|
||||
attn1.to_out.0 / attn2.to_out.0 → attn1.to_out / attn2.to_out
|
||||
"""
|
||||
ignored_set = set(diffusers_ignored)
|
||||
result: set[str] = set()
|
||||
|
||||
for name in diffusers_ignored:
|
||||
m = re.match(r"^(.*\.attn1)\.to_([qkv])$", name)
|
||||
if m:
|
||||
prefix = m.group(1)
|
||||
if all(f"{prefix}.to_{c}" in ignored_set for c in ("q", "k", "v")):
|
||||
result.add(f"{prefix}.to_qkv")
|
||||
else:
|
||||
present = [f"to_{c}" for c in ("q", "k", "v") if f"{prefix}.to_{c}" in ignored_set]
|
||||
missing = [f"to_{c}" for c in ("q", "k", "v") if f"{prefix}.to_{c}" not in ignored_set]
|
||||
raise ValueError(
|
||||
f"Partial BF16 fallback for '{prefix}': "
|
||||
f"{', '.join(present)} in ignored_layers but {', '.join(missing)} is not. "
|
||||
f"Self-attention Q/K/V are fused into a single to_qkv layer at runtime; "
|
||||
f"all three must share the same precision. "
|
||||
f"Either quantize all of to_q/to_k/to_v or keep all three in BF16."
|
||||
)
|
||||
continue
|
||||
|
||||
name = re.sub(r"\.ffn\.net\.0\.proj$", ".ffn.net_0.proj", name)
|
||||
name = re.sub(r"\.ffn\.net\.2$", ".ffn.net_2", name)
|
||||
name = re.sub(r"\.to_out\.0$", ".to_out", name)
|
||||
result.add(name)
|
||||
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def _collect_ignored_layers(merged: dict[str, Any], mxfp4_layer_prefixes: set[str]) -> list[str]:
|
||||
"""Collect vllm-omni parameter name prefixes for BF16 fallback layers.
|
||||
|
||||
A layer is BF16 if it has a .weight tensor but its prefix is not in
|
||||
mxfp4_layer_prefixes. Prefixes are returned in vllm-omni parameter naming
|
||||
(QKV-fused, FFN underscored, to_out unindexed) so the list can be written
|
||||
directly into ignored_layers in config.json without further translation.
|
||||
"""
|
||||
all_weight_prefixes = {key[: -len(".weight")] for key in merged if key.endswith(".weight")}
|
||||
return _diffusers_to_vllm_ignored(sorted(all_weight_prefixes - mxfp4_layer_prefixes))
|
||||
|
||||
|
||||
def _convert_transformer(
|
||||
quant_subdir: pathlib.Path,
|
||||
output_dir: pathlib.Path,
|
||||
original_transformer_dir: pathlib.Path,
|
||||
) -> None:
|
||||
"""Convert one transformer directory to the mxfp4_dualscale + BF16 mixed format."""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# BF16 base: provides the scaffold for non-MXFP4 tensors (norms, embeddings,
|
||||
# BF16 fallback linear layers that msModelSlim may omit or keep as FLOAT).
|
||||
print(f" Loading BF16 base from {original_transformer_dir} …")
|
||||
base_state = _load_safetensors_dir(original_transformer_dir)
|
||||
print(f" {len(base_state)} BF16 tensors loaded")
|
||||
|
||||
print(f" Loading quantized weights from {quant_subdir} …")
|
||||
quant_state = _load_quant_safetensors(quant_subdir)
|
||||
quant_meta = _load_quant_meta(quant_subdir)
|
||||
print(f" {len(quant_state)} quant tensors, {len(quant_meta)} meta entries")
|
||||
|
||||
# Classify blocks for a compact summary (informational only in the new scheme).
|
||||
block_types = _classify_blocks(quant_meta)
|
||||
_print_block_summary(block_types)
|
||||
|
||||
# Remap MXFP4 quantized tensors only.
|
||||
# BF16 fallback tensors (FLOAT in quant_meta, no .linear./.div. wrapper) are
|
||||
# skipped here; the base_state provides them unchanged.
|
||||
remapped: dict[str, torch.Tensor] = {}
|
||||
remapped_meta: dict[str, str] = {}
|
||||
mxfp4_layer_prefixes: set[str] = set()
|
||||
skipped: list[str] = []
|
||||
|
||||
for key, tensor in quant_state.items():
|
||||
qtype = quant_meta.get(key, "FLOAT")
|
||||
|
||||
if not _is_mxfp4_tensor(key, qtype):
|
||||
continue # BF16 fallback — base_state already covers this tensor
|
||||
|
||||
renamed = _apply_rename_dict(key)
|
||||
final_key = _strip_mxfp4_wrapper(renamed)
|
||||
|
||||
if final_key.endswith(".quant_type"):
|
||||
skipped.append(key)
|
||||
continue
|
||||
|
||||
remapped[final_key] = tensor
|
||||
remapped_meta[final_key] = qtype
|
||||
|
||||
# Track MXFP4 layer prefixes (via their .weight keys) for ignored_layers.
|
||||
if final_key.endswith(".weight") and qtype.startswith("W4A4_MXFP4_DUALSCALE"):
|
||||
mxfp4_layer_prefixes.add(final_key[: -len(".weight")])
|
||||
|
||||
if skipped:
|
||||
print(f" Skipped {len(skipped)} metadata keys (quant_type markers): {skipped[:5]}")
|
||||
print(f" {len(remapped)} MXFP4 tensors remapped, {len(mxfp4_layer_prefixes)} quantized layers")
|
||||
|
||||
# Merge: base_state provides BF16 scaffold; MXFP4 tensors override their BF16 counterparts
|
||||
# and add the new scale tensors (weight_scale, weight_dual_scale, mul_scale).
|
||||
merged = {**base_state, **remapped}
|
||||
|
||||
# Determine ignored_layers: BF16 fallback layer prefixes in vllm-omni parameter naming.
|
||||
ignored_layers = _collect_ignored_layers(merged, mxfp4_layer_prefixes)
|
||||
print(f" {len(ignored_layers)} BF16 fallback layers → ignored_layers in config.json")
|
||||
|
||||
# Save weights.
|
||||
out_weights = output_dir / "diffusion_pytorch_model.safetensors"
|
||||
save_file(merged, str(out_weights))
|
||||
print(f" Saved {len(merged)} tensors → {out_weights}")
|
||||
|
||||
# Save remapped quant metadata (for inspection / debugging).
|
||||
out_meta_path = output_dir / "quant_model_description.json"
|
||||
with open(out_meta_path, "w") as f:
|
||||
json.dump(remapped_meta, f, indent=2)
|
||||
|
||||
# Inject quantization_config into config.json.
|
||||
src_config = original_transformer_dir / "config.json"
|
||||
if src_config.is_file():
|
||||
with open(src_config) as f:
|
||||
config = json.load(f)
|
||||
config["quantization_config"] = _build_quant_config(ignored_layers)
|
||||
out_config = output_dir / "config.json"
|
||||
with open(out_config, "w") as f:
|
||||
json.dump(config, f, indent=2)
|
||||
print(f" Injected quantization_config (mxfp4_dualscale, {len(ignored_layers)} ignored_layers) → {out_config}")
|
||||
else:
|
||||
print(f" WARNING: No config.json at {src_config}; quantization_config not injected.")
|
||||
|
||||
|
||||
def _build_quant_config(ignored_layers: list[str]) -> dict[str, Any]:
|
||||
return {
|
||||
"quant_method": "mxfp4_dualscale",
|
||||
"is_checkpoint_serialized": True,
|
||||
"ignored_layers": ignored_layers,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model-type helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_transformer_dirs(model_type: str) -> list[str]:
|
||||
return ["transformer", "transformer_2"] if model_type in CASCADE_MODEL_TYPES else ["transformer"]
|
||||
|
||||
|
||||
def _get_quant_subdir(model_type: str, quant_path: pathlib.Path, transformer_dir: str) -> pathlib.Path:
|
||||
if model_type in CASCADE_MODEL_TYPES:
|
||||
sub = "high_noise_model" if transformer_dir == "transformer" else "low_noise_model"
|
||||
return quant_path / sub
|
||||
return quant_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main repack
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def repack(
|
||||
model_type: str,
|
||||
original_model_path: pathlib.Path,
|
||||
quant_path: pathlib.Path,
|
||||
output_path: pathlib.Path,
|
||||
) -> None:
|
||||
transformer_dirs = _get_transformer_dirs(model_type)
|
||||
|
||||
print(f"Copying original model to {output_path} (skipping {transformer_dirs}) …")
|
||||
shutil.copytree(
|
||||
str(original_model_path),
|
||||
str(output_path),
|
||||
ignore=shutil.ignore_patterns(*transformer_dirs),
|
||||
)
|
||||
|
||||
for tdir in transformer_dirs:
|
||||
q_subdir = _get_quant_subdir(model_type, quant_path, tdir)
|
||||
out_tdir = output_path / tdir
|
||||
orig_tdir = original_model_path / tdir
|
||||
print(f"\nConverting {tdir} (quant source: {q_subdir.name}) …")
|
||||
_convert_transformer(q_subdir, out_tdir, orig_tdir)
|
||||
|
||||
print(f"\nDone. Merged model → {output_path}")
|
||||
print("\nRun inference (quantization auto-detected from config.json):")
|
||||
print(f" python text_to_video.py --model {output_path}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--model-type", required=True, choices=SUPPORTED_MODEL_TYPES, help="Model variant.")
|
||||
parser.add_argument("--original-model", required=True, help="Original HF Diffusers model directory (BF16).")
|
||||
parser.add_argument("--quant-path", required=True, help="msModelSlim quantized weights directory.")
|
||||
parser.add_argument("--output-path", required=True, help="Output directory for merged model.")
|
||||
args = parser.parse_args()
|
||||
|
||||
repack(
|
||||
model_type=args.model_type,
|
||||
original_model_path=pathlib.Path(args.original_model),
|
||||
quant_path=pathlib.Path(args.quant_path),
|
||||
output_path=pathlib.Path(args.output_path),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user