[NPU][Quant] Add W4A4 MXFP4 online & MXFP4 dual-scale online/offline quantization support for Wan2.2 T2V / I2V inference on Ascend NPU (#3578)

Signed-off-by: hyh_hh <huyinghong1@huawei.com>
Co-authored-by: hyh_hh <huyinghong1@huawei.com>
This commit is contained in:
hxhhhlalalaandhyh_hh authored and GitHub committed 2026-05-20 06:21:29 +00:00
1 parent 109bba8cdf
commit 9dd36e3afd
14 files changed
+3646 -61

No files matched your search

+102
View File
@@ -48,6 +48,29 @@ def _build_mxfp8(**kw: Any) -> QuantizationConfig:
return DiffusionMXFP8Config(**kw)
def _build_mxfp4(**kw: Any) -> QuantizationConfig:
"""Lazy import for W4A4 MXFP4 diffusion config (NPU only)."""
from .mxfp4_config import DiffusionMXFP4Config
return DiffusionMXFP4Config(**kw)
def _build_mxfp4_dualscale(**kw: Any) -> QuantizationConfig:
"""Lazy import for MXFP4 DualScale + BF16 mixed diffusion config (NPU only).
Offline mode (is_checkpoint_serialized=True):
ignored_layers from config.json marks interleaved BF16 fallback layers.
All other linear layers use W4A4 MXFP4 DualScale.
Online mode (is_checkpoint_serialized=False):
num_bf16_fallback_layers leading transformer blocks use BF16 original weights
(default 5 when not specified). Remaining blocks use W4A4 MXFP4 DualScale online.
"""
from .mxfp4_config import DiffusionMXFP4DualScaleMixedConfig
return DiffusionMXFP4DualScaleMixedConfig(**kw)
def _build_inc(**kw: Any) -> QuantizationConfig:
"""Lazy import for INC/AutoRound config with checkpoint kwarg normalization."""
from .inc_config import OmniINCConfig
@@ -66,6 +89,8 @@ _OVERRIDES: dict[str, Callable[..., QuantizationConfig]] = {
"gguf": _build_gguf,
"int8": _build_int8,
"mxfp8": _build_mxfp8,
"mxfp4": _build_mxfp4,
"mxfp4_dualscale": _build_mxfp4_dualscale,
"inc": _build_inc,
"auto-round": _build_inc,
"auto_round": _build_inc,
@@ -255,3 +280,80 @@ def build_quant_config(
return _build_from_method_and_config(method, merged)
raise TypeError(f"quantization config must be str, dict, QuantizationConfig, or None, got {type(spec).__name__}")
def _disk_marks_serialized(qc_kwargs: dict[str, Any], quant_config: object) -> bool:
"""Return True when config.json says serialized but the active quant_config does not.
Matches any flag following the is_checkpoint_*_serialized naming convention,
so new quant methods don't require updating an explicit allowlist.
"""
for key, val in qc_kwargs.items():
if key.startswith("is_checkpoint_") and key.endswith("_serialized"):
if val and hasattr(quant_config, key) and not getattr(quant_config, key):
return True
return False
def resolve_quant_config_from_disk(
quant_config: QuantizationConfig | None,
disk_qc: dict[str, Any] | str | None,
) -> QuantizationConfig | None:
"""Reconcile an active quant_config against quantization_config from a transformer's config.json.
Used when loading individual transformer blocks that each have their own config.json
(e.g. cascade models with separate transformer and transformer_2 directories).
Rules:
- disk_qc is None: return quant_config unchanged.
- quant_config is None: auto-detect from disk_qc (full build).
- Methods mismatch: raise ValueError — prevents silent weight corruption.
- Disk marks serialized but quant_config is online: rebuild from disk.
- ignored_layers differ: rebuild from disk (per-transformer BF16 routing).
"""
if disk_qc is None:
return quant_config
if isinstance(disk_qc, str):
if quant_config is None:
logger.info("Auto-detected quantization from config.json: method=%s", disk_qc)
return build_quant_config(disk_qc)
return quant_config
if not isinstance(disk_qc, Mapping) or "quant_method" not in disk_qc:
return quant_config
qc_method: str = disk_qc["quant_method"]
qc_kwargs: dict[str, Any] = {k: v for k, v in disk_qc.items() if k != "quant_method"}
if quant_config is None:
logger.info(
"Auto-detected quantization from config.json: method=%s kwargs=%s",
qc_method,
qc_kwargs,
)
return build_quant_config(qc_method, **qc_kwargs)
if quant_config.get_name() != qc_method:
raise ValueError(
f"Checkpoint config.json declares quant_method={qc_method!r} but the "
f"active quantization config is {quant_config.get_name()!r}. "
"Pass a matching --quantization flag or omit it for auto-detection."
)
if _disk_marks_serialized(qc_kwargs, quant_config):
logger.info(
"config.json marks checkpoint as serialized; switching to offline %s mode.",
qc_method,
)
return build_quant_config(qc_method, **qc_kwargs)
if (
"ignored_layers" in qc_kwargs
and hasattr(quant_config, "ignored_layers")
and set(qc_kwargs.get("ignored_layers") or []) != set(quant_config.ignored_layers or [])
):
logger.info("config.json ignored_layers differs from active config; rebuilding quant_config.")
return build_quant_config(qc_method, **qc_kwargs)
return quant_config
+709
View File
@@ -0,0 +1,709 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""W4A4 MXFP4 (Microscaling FP4) online/offline quantization for diffusion transformers.
Architecture mirrors mxfp8_config.py:
MXFPLinearMethodBase – platform-agnostic skeleton (imported from mxfp8_config)
NPUMxfp4LinearMethod – NPU single-scale offline (W4A4 MXFP4)
NPUMxfp4OnlineLinearMethod – NPU single-scale online (BF16 → FP4)
NPUMxfp4DualScaleLinearMethod – NPU dual-scale offline (W4A4 MXFP4 DualScale)
NPUMxfp4DualScaleOnlineLinearMethod – NPU dual-scale online (BF16 → FP4)
Quantization configs:
DiffusionMXFP4Config – single-scale online/offline (quant_method="mxfp4")
DiffusionMXFP4DualScaleMixedConfig – dual-scale + per-layer BF16 fallback (quant_method="mxfp4_dualscale")
Offline: ignored_layers from config.json routes interleaved BF16 layers
Online: num_bf16_fallback_layers leading blocks stay in BF16 (default 5)
Key differences from MXFP8:
1. Precision: float4_e2m1fn_x2 (FP4 packed, 2 values per element).
npu_dynamic_mx_quant(x) without dst_type defaults to float4_e2m1fn_x2.
2. Weight layout: stored as (N, K) — NOT pre-transposed.
FP4 uses a packed format; transposing a packed tensor is not safely contiguous.
Transpose is done inline in _quant_matmul via layer.weight.transpose(0, 1).
3. GEMM signature: npu_quant_matmul requires explicit
x1_dtype=float4_e2m1fn_x2 and x2_dtype=float4_e2m1fn_x2.
Scale layout: (N, S/2, 2) — same reshape as MXFP8, also NOT pre-transposed;
transposed inline in _quant_matmul.
Dual-scale (W4A4_MXFP4_DUALSCALE):
Two-level quantization: fine scale (per-32 K) + coarse scale (per-512 K) + per-channel
activation pre-scale (mul_scale from calibration). Uses npu_dynamic_dual_level_mx_quant
and npu_dual_level_quant_matmul. Weight stored in NZ hardware format via npu_format_cast.
Checkpoint tensor shapes for dual-scale:
weight : (N, K) float8_e4m3fn – FP4 packed
weight_scale : (N, K//32) uint8 – fine scale (float8_e8m0fnu bits)
weight_dual_scale : (N, K//512, 1) float32 – coarse scale (extra dim avoids shape assert)
mul_scale : (K,) float32 – per-input-channel activation pre-scale
Reference: MindIE-SD W4A4MXFP4QuantLinear / W4A4MXFP4DualQuantLinear (mindiesd/quantization/layer.py).
"""
from __future__ import annotations
import re
from typing import TYPE_CHECKING, Any
import torch
from torch.nn import Module
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.fp8 import _copy_missing_attrs
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.model_loader.weight_utils import initialize_single_dummy_weight
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.model_executor.utils import replace_parameter
from vllm_omni.platforms import current_omni_platform
from vllm_omni.quantization.mxfp8_config import (
MXFPLinearMethodBase,
_LazyWeightMixin,
)
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
class DiffusionMXFP4Config(QuantizationConfig):
"""W4A4 MXFP4 quantization config for diffusion transformers.
Supports both online (BF16 checkpoint → quantize at load time) and offline
(pre-quantized MXFP4 checkpoint) modes, mirroring DiffusionMXFP8Config.
MX (microscaling) format: groups of 32 K-dimension elements share one
float8_e8m0fnu exponent scale. Weight and activation are float4_e2m1fn_x2.
"""
def __init__(
self,
is_checkpoint_mxfp4_serialized: bool = False,
ignored_layers: list[str] | None = None,
) -> None:
super().__init__()
self.is_checkpoint_mxfp4_serialized = is_checkpoint_mxfp4_serialized
self.ignored_layers = ignored_layers or []
@classmethod
def get_name(cls) -> QuantizationMethods:
return "mxfp4"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.bfloat16, torch.float16]
@classmethod
def get_min_capability(cls) -> int:
return 80
@classmethod
def get_config_filenames(cls) -> list[str]:
return []
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
if self.ignored_layers:
self.ignored_layers = hf_to_vllm_mapper.apply_list(self.ignored_layers)
@classmethod
def from_config(cls, config: dict[str, Any]) -> DiffusionMXFP4Config:
is_serialized = cls.get_from_keys_or(config, ["is_checkpoint_mxfp4_serialized"], False)
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
if not ignored_layers:
ignored_layers = cls.get_from_keys_or(config, ["modules_to_not_convert"], None)
return cls(
is_checkpoint_mxfp4_serialized=is_serialized,
ignored_layers=ignored_layers,
)
def get_quant_method(
self,
layer: torch.nn.Module,
prefix: str,
) -> QuantizeMethodBase | None:
if isinstance(layer, LinearBase):
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
if current_omni_platform.is_npu():
if self.is_checkpoint_mxfp4_serialized:
return NPUMxfp4LinearMethod(self)
return NPUMxfp4OnlineLinearMethod(self)
raise NotImplementedError(
"DiffusionMXFP4Config (W4A4 MXFP4) is currently only supported "
"on NPU (Ascend) platforms. CUDA support is not yet implemented."
)
return None
# ---------------------------------------------------------------------------
# NPU MXFP4 single-scale offline method (pre-quantized checkpoint)
# ---------------------------------------------------------------------------
class NPUMxfp4LinearMethod(MXFPLinearMethodBase):
"""NPU W4A4 MXFP4 offline linear method for pre-quantized checkpoints.
Weight canonical layout after process_weights_after_loading:
weight : (N, K) in float4_e2m1fn_x2 — NOT pre-transposed (FP4 packed)
weight_scale: (N, S/2, 2) in float8_e8m0fnu — NOT pre-transposed
Both are transposed inline in _quant_matmul, unlike MXFP8 which pre-transposes.
NPUMxfp4OnlineLinearMethod normalizes to the same layout so apply() is shared.
"""
def __init__(self, quant_config: DiffusionMXFP4Config) -> None:
self.quant_config = quant_config
self.out_dtype = torch.get_default_dtype()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = None
# BF16 placeholder; cast to float4_e2m1fn_x2 in process_weights.
layer.register_parameter(
"weight",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
# Scale stored as uint8 in safetensors (float8_e8m0fnu is same bit width).
# Using uint8 avoids a lossy float32 round-trip when loading the checkpoint.
num_groups = (input_size_per_partition + 31) // 32
layer.register_parameter(
"weight_scale",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, num_groups, dtype=torch.uint8),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
import torch_npu
# NPU: cast to float4_e2m1fn_x2. Weight stays (N, K) — no pre-transpose.
w = layer.weight
if w.dtype != torch_npu.float4_e2m1fn_x2:
w = torch_npu.npu_dtype_cast(w.npu(), torch_npu.float4_e2m1fn_x2)
# Scale: checkpoint stores uint8 bytes that ARE float8_e8m0fnu bits.
# Only convert if neither uint8 nor the target NPU dtype already.
# (N, K_groups) → (N, K_groups/2, 2). Not pre-transposed; done inline in _quant_matmul.
s = layer.weight_scale.data
if s.dtype not in (torch.uint8, torch_npu.float8_e8m0fnu):
s = s.to(torch_npu.float8_e8m0fnu)
N, K_groups = s.shape
if K_groups % 2 == 1:
s = torch.cat([s, torch.zeros(N, 1, dtype=s.dtype, device=s.device)], dim=1)
s = s.reshape(N, -1, 2).contiguous()
replace_parameter(layer, "weight", w)
replace_parameter(layer, "weight_scale", s)
layer._already_called_process_weights_after_loading = True
# --- NPU MXFP4 ops — shared with online path via inheritance ---
def _quantize_activation(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import torch_npu
# No dst_type: npu_dynamic_mx_quant defaults to float4_e2m1fn_x2.
return torch_npu.npu_dynamic_mx_quant(x)
def _quant_matmul(
self,
x_q: torch.Tensor,
x_scale: torch.Tensor,
layer: torch.nn.Module,
bias: torch.Tensor | None,
ori_dtype: torch.dtype,
) -> torch.Tensor:
import torch_npu
if bias is not None and bias.dtype != torch.float32:
bias = bias.to(torch.float32)
# FP4 differences vs FP8:
# weight (N,K) transposed inline → (K,N); scale (N,S/2,2) transposed inline → (S/2,N,2).
# x1_dtype / x2_dtype required — FP4 dtype not inferred from tensor dtype.
return torch_npu.npu_quant_matmul(
x_q,
layer.weight.transpose(0, 1), # (K, N) inline
layer.weight_scale.transpose(0, 1), # (S/2, N, 2) inline
scale_dtype=torch_npu.float8_e8m0fnu,
x1_dtype=torch_npu.float4_e2m1fn_x2,
x2_dtype=torch_npu.float4_e2m1fn_x2,
pertoken_scale=x_scale,
pertoken_scale_dtype=torch_npu.float8_e8m0fnu,
bias=bias,
output_dtype=ori_dtype,
group_sizes=[1, 1, 32],
)
# ---------------------------------------------------------------------------
# NPU MXFP4 single-scale online method (BF16 checkpoint → quantize at load time)
# ---------------------------------------------------------------------------
class NPUMxfp4OnlineLinearMethod(_LazyWeightMixin, NPUMxfp4LinearMethod):
"""NPU W4A4 MXFP4 online linear method.
MRO: NPUMxfp4OnlineLinearMethod → _LazyWeightMixin → NPUMxfp4LinearMethod
→ MXFPLinearMethodBase → LinearMethodBase
create_weights : _LazyWeightMixin (meta device + patched loader)
process_weights : NPUMxfp4OnlineLinearMethod (BF16 → FP4 + normalize)
apply / ops : NPUMxfp4LinearMethod / MXFPLinearMethodBase (shared)
"""
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
import torch_npu
if layer.weight.device == torch.device("meta"):
weight = ModelWeightParameter(
data=torch.empty_like(layer.weight, device=layer._load_device),
input_dim=1,
output_dim=0,
weight_loader=layer.weight.weight_loader,
)
_copy_missing_attrs(layer.weight, weight)
layer.register_parameter("weight", weight)
initialize_single_dummy_weight(layer.weight)
# NPU: quantize BF16/FP16 (N, K) → FP4. No dst_type → float4_e2m1fn_x2.
weight_fp4, weight_scale_raw = torch_npu.npu_dynamic_mx_quant(layer.weight)
# Weight stays (N, K) — no pre-transpose for FP4 packed format.
# Scale: (N, S) → (N, S/2, 2). Not pre-transposed; done inline in _quant_matmul.
weight_scale = weight_scale_raw.reshape(weight_scale_raw.shape[0], -1, 2).contiguous()
replace_parameter(layer, "weight", weight_fp4)
replace_parameter(layer, "weight_scale", weight_scale)
layer._already_called_process_weights_after_loading = True
# ---------------------------------------------------------------------------
# NPU MXFP4 dual-scale offline method (W4A4_MXFP4_DUALSCALE checkpoint)
# ---------------------------------------------------------------------------
class NPUMxfp4DualScaleLinearMethod(MXFPLinearMethodBase):
"""NPU W4A4 MXFP4 dual-scale offline method for pre-quantized checkpoints.
Checkpoint tensors and their canonical post-load shapes:
weight : (N, K) float8_e4m3fn – FP4 packed (2 values per byte)
weight_scale : (N, K//32) uint8 – fine scale (float8_e8m0fnu bits); reshaped to (N, K//64, 2)
weight_dual_scale : (N, K//512, 1) float32 – coarse scale; transposed to (K//512, N)
mul_scale : (K,) float32 – per-input-channel activation pre-scale (from calibration)
Forward pass:
x_q, l0, l1 = npu_dynamic_dual_level_mx_quant(x, smooth_scale=mul_scale)
out = npu_dual_level_quant_matmul(x_q, weight, l0, weight_dual_scale, l1, weight_scale)
Reference: MindIE-SD W4A4MXFP4DualQuantLinear.
"""
def __init__(self, quant_config: Any) -> None:
self.quant_config = quant_config
self.out_dtype = torch.get_default_dtype()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = None
# FP4 packed: 2 values per float8_e4m3fn byte → checkpoint stores as float8_e4m3fn; register as BF16
layer.register_parameter(
"weight",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
# Fine scale: one uint8 exponent (float8_e8m0fnu bit pattern) per group of 32 K elements.
# input_dim=1: RowParallelLinear weight_loader shards along the K-group dimension (dim 1)
# so each rank receives only its (K/TP)//32 groups; ColumnParallelLinear only uses
# output_dim=0 for sharding and leaves dim 1 intact.
num_groups_fine = (input_size_per_partition + 31) // 32
layer.register_parameter(
"weight_scale",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, num_groups_fine, dtype=torch.uint8),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
# Coarse scale: one float32 per group of 512 K elements.
# Shape (N, K_coarse, 1) matches checkpoint layout exactly, avoiding the
# shape-mismatch assert in linear.py:1344.
# input_dim=1: same TP sharding rationale as weight_scale above.
num_groups_coarse = (input_size_per_partition + 511) // 512
layer.register_parameter(
"weight_dual_scale",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, num_groups_coarse, 1, dtype=torch.float32),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
# mul_scale is a float32 calibration tensor; register as float32
# to avoid precision loss from an implicit BF16 cast during weight loading.
# input_dim=0: RowParallelLinear shards the 1-D per-input-channel tensor along dim 0
# so each rank receives only its K/TP channels; ColumnParallelLinear leaves it intact.
layer.register_parameter(
"mul_scale",
ModelWeightParameter(
data=torch.empty(input_size_per_partition, dtype=torch.float32),
input_dim=0,
output_dim=None,
weight_loader=weight_loader,
),
)
setattr(layer.mul_scale, "ignore_warning", True)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
import torch_npu
# float8_e4m3fn (FP4 packed) → float4_e2m1fn_x2 → NZ hardware format (format ID 29).
# NZ layout is required by npu_dual_level_quant_matmul for FP4 weight matrices.
w = torch_npu.npu_dtype_cast(layer.weight.data.npu(), torch_npu.float4_e2m1fn_x2)
w = torch_npu.npu_format_cast(w.view(torch.int8), 29, customize_dtype=torch.int8)
# Fine scale: (N, K//32) uint8 → cast to float8_e8m0fnu → (N, K//64, 2).
# The cast reinterprets the stored uint8 bit-patterns as float8_e8m0fnu exponents,
# which is required for npu_dual_level_quant_matmul to compute correct scale values.
s = layer.weight_scale.data.npu()
s = s.reshape(s.shape[0], -1, 2).contiguous()
# Coarse scale: (N, K//512, 1) → squeeze → (N, K//512) → transpose → (K//512, N).
ds = layer.weight_dual_scale.data.squeeze(-1).transpose(0, 1).npu().contiguous()
ms = layer.mul_scale.to(torch.bfloat16).data.view(-1).npu().contiguous()
replace_parameter(layer, "weight", w)
replace_parameter(layer, "weight_scale", s)
replace_parameter(layer, "weight_dual_scale", ds)
replace_parameter(layer, "mul_scale", ms)
layer._already_called_process_weights_after_loading = True
def _apply_inner(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None,
ori_dtype: torch.dtype,
) -> torch.Tensor:
"""Dual-scale inner loop: dtype guard → 3-tuple quantize (mul_scale as smooth) → matmul.
Overrides the default single-scale _apply_inner from MXFPLinearMethodBase.
apply() in the base class handles reshape/unreshape; this method is not
responsible for that.
"""
if ori_dtype not in (torch.bfloat16, torch.float16):
x = x.to(torch.bfloat16)
x_q, l0_scale, l1_scale = self._quantize_activation(x, layer.mul_scale)
return self._quant_matmul(x_q, l0_scale, l1_scale, layer, bias, ori_dtype)
def _quantize_activation( # type: ignore[override]
self,
x: torch.Tensor,
smooth_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
import torch_npu
return torch_npu.npu_dynamic_dual_level_mx_quant(x, smooth_scale=smooth_scale)
def _quant_matmul( # type: ignore[override]
self,
x_q: torch.Tensor,
l0_scale: torch.Tensor,
l1_scale: torch.Tensor,
layer: torch.nn.Module,
bias: torch.Tensor | None,
ori_dtype: torch.dtype,
) -> torch.Tensor:
import torch_npu
if bias is not None and bias.dtype != torch.float32:
bias = bias.to(torch.float32)
# weight_scale is (N, K//64, 2) float8_e8m0fnu — operator expects output-major layout.
# weight_dual_scale is (K//512, N) float32 — transposed to K-major in process_weights.
return torch_npu.npu_dual_level_quant_matmul(
x_q,
layer.weight,
l0_scale,
layer.weight_dual_scale,
l1_scale,
layer.weight_scale,
bias=bias,
output_dtype=ori_dtype,
)
# ---------------------------------------------------------------------------
# NPU MXFP4 dual-scale online method (BF16 checkpoint → quantize at load time)
# ---------------------------------------------------------------------------
class NPUMxfp4DualScaleOnlineLinearMethod(_LazyWeightMixin, NPUMxfp4DualScaleLinearMethod):
"""NPU W4A4 MXFP4 dual-scale online method: quantises BF16 weights at load time.
MRO: NPUMxfp4DualScaleOnlineLinearMethod → _LazyWeightMixin
→ NPUMxfp4DualScaleLinearMethod → MXFPLinearMethodBase
create_weights : _LazyWeightMixin (meta device + patched loader)
process_weights : NPUMxfp4DualScaleOnlineLinearMethod (BF16 → FP4 + dual scales)
apply / _quant_matmul : NPUMxfp4DualScaleLinearMethod (shared with offline path)
"""
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
import torch_npu
if layer.weight.device == torch.device("meta"):
weight = ModelWeightParameter(
data=torch.empty_like(layer.weight, device=layer._load_device),
input_dim=1,
output_dim=0,
weight_loader=layer.weight.weight_loader,
)
_copy_missing_attrs(layer.weight, weight)
layer.register_parameter("weight", weight)
initialize_single_dummy_weight(layer.weight)
# Quantize BF16 weight → FP4 + dual-level scales (no smooth pre-scale for online).
# Returns: (weight_fp4, l0_scale[coarse per-512], l1_scale[fine per-32])
weight_fp4, w_l0_scale, w_l1_scale = torch_npu.npu_dynamic_dual_level_mx_quant(
layer.weight.data.npu(), smooth_scale=None
)
# NZ hardware format for the FP4 weight (same as offline path).
w = torch_npu.npu_format_cast(weight_fp4.view(torch.int8), 29, customize_dtype=torch.int8)
# Fine scale (l1): (N, K//32) → (N, K//64, 2). Dtype from op output (float8_e8m0fnu).
s = w_l1_scale.reshape(w_l1_scale.shape[0], -1, 2).contiguous()
# Coarse scale (l0): (N, K_coarse) → (K_coarse, N). Dtype from op output.
ds = w_l0_scale.reshape(w_l0_scale.shape[0], -1).transpose(0, 1).contiguous()
# No calibration available: identity pre-scale (no smooth quantization effect).
ms = torch.ones(layer.input_size_per_partition, dtype=torch.bfloat16, device="npu")
replace_parameter(layer, "weight", w)
replace_parameter(layer, "weight_scale", s)
replace_parameter(layer, "weight_dual_scale", ds)
replace_parameter(layer, "mul_scale", ms)
layer._already_called_process_weights_after_loading = True
# ---------------------------------------------------------------------------
# Block-index helper (shared by DiffusionMXFP4DualScaleMixedConfig)
# ---------------------------------------------------------------------------
_BLOCK_IDX_RE = re.compile(r"^blocks\.(\d+)\.")
def _parse_block_idx(prefix: str) -> int | None:
"""Extract block index from prefix like 'blocks.5.attn1.to_q'."""
m = _BLOCK_IDX_RE.match(prefix)
return int(m.group(1)) if m else None
# ---------------------------------------------------------------------------
# Config: MXFP4 DualScale + per-layer BF16 fallback
# ---------------------------------------------------------------------------
class DiffusionMXFP4DualScaleMixedConfig(QuantizationConfig):
"""W4A4 MXFP4 DualScale with per-layer BF16 fallback for diffusion transformers.
Sensitive layers fall back to BF16 (original weights) while all other linear
layers use W4A4 MXFP4 DualScale. BF16 fallback layers may be interleaved
anywhere in the transformer.
Offline mode (is_checkpoint_serialized=True):
Layers whose prefix appears in ignored_layers → UnquantizedLinearMethod (BF16)
All other linear layers → NPUMxfp4DualScaleLinearMethod
ignored_layers is injected into transformer/config.json by the merge script
and contains the prefixes of all non-MXFP4 linear layers.
Online mode (is_checkpoint_serialized=False):
Layer routing applies two rules in priority order:
1. ignored_layers (explicit per-layer BF16 override, user-supplied) → BF16
2. Blocks 0 .. num_bf16_fallback_layers-1 (coarse leading-block rule) → BF16
3. All other linear layers → NPUMxfp4DualScaleOnlineLinearMethod
num_bf16_fallback_layers defaults to 5 when not specified.
Set ignored_layers to pin arbitrary interleaved layers to BF16 without
needing an offline checkpoint (useful for accuracy debugging).
Layers outside "blocks.N.*" (condition_embedder etc.) always use online MXFP4
unless they appear in ignored_layers.
Config injected by merge_mxfp4_dualscale_checkpoint.py:
{
"quant_method": "mxfp4_dualscale",
"is_checkpoint_serialized": true,
"ignored_layers": ["blocks.0.attn1.to_q", ...]
}
"""
def __init__(
self,
is_checkpoint_serialized: bool = False,
ignored_layers: list[str] | None = None,
num_bf16_fallback_layers: int = 5,
) -> None:
super().__init__()
self.is_checkpoint_serialized = is_checkpoint_serialized
self.ignored_layers = ignored_layers or []
self.num_bf16_fallback_layers = num_bf16_fallback_layers
@classmethod
def get_name(cls) -> QuantizationMethods:
return "mxfp4_dualscale"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.bfloat16, torch.float16]
@classmethod
def get_min_capability(cls) -> int:
return 80
@classmethod
def get_config_filenames(cls) -> list[str]:
return []
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
if self.ignored_layers:
self.ignored_layers = hf_to_vllm_mapper.apply_list(self.ignored_layers)
@classmethod
def from_config(cls, config: dict[str, Any]) -> DiffusionMXFP4DualScaleMixedConfig:
is_serialized = cls.get_from_keys_or(config, ["is_checkpoint_serialized"], False)
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
if not ignored_layers:
ignored_layers = cls.get_from_keys_or(config, ["modules_to_not_convert"], None)
num_bf16_fallback_layers = cls.get_from_keys_or(config, ["num_bf16_fallback_layers"], 5)
return cls(
is_checkpoint_serialized=is_serialized,
ignored_layers=ignored_layers,
num_bf16_fallback_layers=num_bf16_fallback_layers,
)
def get_quant_method(
self,
layer: torch.nn.Module,
prefix: str,
) -> QuantizeMethodBase | None:
if not isinstance(layer, LinearBase):
return None
if self.is_checkpoint_serialized:
# Offline: ignored_layers lists interleaved BF16 fallback layer prefixes.
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
if not current_omni_platform.is_npu():
raise NotImplementedError(
"DiffusionMXFP4DualScaleMixedConfig is currently only supported on NPU (Ascend) platforms."
)
return NPUMxfp4DualScaleLinearMethod(self)
# Online: explicit ignored_layers take priority (user-specified per-layer BF16 override),
# then fall back to the coarse leading-block rule.
if is_layer_skipped(
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
block_idx = _parse_block_idx(prefix)
if block_idx is not None and block_idx < self.num_bf16_fallback_layers:
return UnquantizedLinearMethod()
if not current_omni_platform.is_npu():
raise NotImplementedError(
"DiffusionMXFP4DualScaleMixedConfig is currently only supported on NPU (Ascend) platforms."
)
return NPUMxfp4DualScaleOnlineLinearMethod(self)
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,491 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Merge W4A4_MXFP4_DUALSCALE quantized Wan2.2 weights into HF Diffusers format.
msModelSlim produces a checkpoint where each linear layer is either:
- W4A4_MXFP4_DUALSCALE: quantized weights with fine/coarse scales and mul_scale
- FLOAT (BF16 fallback): kept as original BF16 weights for precision-sensitive layers
BF16 fallback layers may be interleaved anywhere in the transformer (not just leading
blocks). The merge script detects them from quant_model_description.json and writes
their prefixes into config.json as ignored_layers so the runtime dynamically routes
each layer to the correct quantization method at weight-loading time.
MXFP4_DUALSCALE key structure per linear layer (msModelSlim naming)
---------------------------------------------------------------------
blocks.N.X.linear.weight W4A4_MXFP4_DUALSCALE – int8 (FP4 packed)
blocks.N.X.linear.weight_scale W4A4_MXFP4_DUALSCALE – uint8 (float8_e8m0fnu fine scale, per-32K)
blocks.N.X.linear.weight_dual_scale W4A4_MXFP4_DUALSCALE – float32 (coarse scale, per-512K)
blocks.N.X.linear.bias FLOAT – bias (if present)
blocks.N.X.div.mul_scale FLOAT – float32 per-input-channel activation pre-scale
BF16 fallback key structure (no wrappers, plain linear weights):
blocks.N.X.weight FLOAT
condition_embedder.*.weight FLOAT (always BF16 — not quantized)
Self-attention QKV notes
------------------------
Self-attention Q/K/V weights are separate in the checkpoint (self_attn.q/k/v) but fused
into a single to_qkv layer in vllm-omni. The transformer's load_weights() handles this via
stacked_params_mapping. This script keeps Q/K/V keys separate — do NOT pre-fuse them here.
For mul_scale specifically: even though Q/K/V process the same input (same mul_scale value),
they are kept separate. load_weights() routes all three to the same to_qkv.mul_scale parameter
and each overwrites the previous. Since Q=K=V for mul_scale, the final value is correct.
NOTE: Pre-fusing as to_qkv.mul_scale would BREAK loading because ".attn1.to_q" is a
substring of ".attn1.to_qkv", causing load_weights() stacked_params_mapping to produce a
garbage key ("to_qkvkv") that is not in params_dict, triggering a break that skips the
direct-load else branch entirely.
Supported model types:
- Wan2.2-T2V-A14B (MoE cascade: transformer + transformer_2)
- Wan2.2-I2V-A14B (MoE cascade: transformer + transformer_2)
Note: Wan2.2-TI2V-5B is NOT supported. Its smaller parameter count causes
unacceptable accuracy loss under W4A4 quantization. Use MXFP8 for TI2V-5B.
Usage:
python merge_mxfp4_dualscale_checkpoint.py \\
--model-type Wan2.2-T2V-A14B \\
--original-model /path/to/Wan2.2-T2V-A14B-Diffusers \\
--quant-path /path/to/msmodelslim-output \\
--output-path /path/to/merged-output
"""
from __future__ import annotations
import argparse
import json
import pathlib
import re
import shutil
from typing import Any
import torch
from safetensors.torch import load_file, save_file
# ---------------------------------------------------------------------------
# Key rename: msModelSlim naming → Diffusers / vllm-omni naming
# Identical to merge_mxfp8_checkpoint.py; applied to all blocks uniformly.
# ---------------------------------------------------------------------------
TRANSFORMER_KEYS_RENAME_DICT: dict[str, str] = {
"time_embedding.0": "condition_embedder.time_embedder.linear_1",
"time_embedding.2": "condition_embedder.time_embedder.linear_2",
"text_embedding.0": "condition_embedder.text_embedder.linear_1",
"text_embedding.2": "condition_embedder.text_embedder.linear_2",
"time_projection.1": "condition_embedder.time_proj",
"head.modulation": "scale_shift_table",
"head.head": "proj_out",
"modulation": "scale_shift_table",
"ffn.0": "ffn.net.0.proj",
"ffn.2": "ffn.net.2",
# Norm order swap (quant tool: norm1, norm3, norm2 → diffusers: norm1, norm2, norm3)
"norm2": "norm__placeholder",
"norm3": "norm2",
"norm__placeholder": "norm3",
# Self-attention
"self_attn.q": "attn1.to_q",
"self_attn.k": "attn1.to_k",
"self_attn.v": "attn1.to_v",
"self_attn.o": "attn1.to_out.0",
"self_attn.norm_q": "attn1.norm_q",
"self_attn.norm_k": "attn1.norm_k",
# Cross-attention
"cross_attn.q": "attn2.to_q",
"cross_attn.k": "attn2.to_k",
"cross_attn.v": "attn2.to_v",
"cross_attn.o": "attn2.to_out.0",
"cross_attn.norm_q": "attn2.norm_q",
"cross_attn.norm_k": "attn2.norm_k",
"attn2.to_k_img": "attn2.add_k_proj",
"attn2.to_v_img": "attn2.add_v_proj",
"attn2.norm_k_img": "attn2.norm_added_k",
# I2V image embedder
"img_emb.proj.0": "condition_embedder.image_embedder.norm1",
"img_emb.proj.1": "condition_embedder.image_embedder.ff.net.0.proj",
"img_emb.proj.3": "condition_embedder.image_embedder.ff.net.2",
"img_emb.proj.4": "condition_embedder.image_embedder.norm2",
"img_emb.emb_pos": "condition_embedder.image_embedder.pos_embed",
}
SUPPORTED_MODEL_TYPES = ["Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B"]
CASCADE_MODEL_TYPES = {"Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B"}
# Suffixes that appear inside .linear.* wrapper for MXFP4 tensors.
_LINEAR_ATTRS = ("weight_dual_scale", "weight_scale", "weight", "bias")
_BLOCK_IDX_RE = re.compile(r"^blocks\.(\d+)\.")
# ---------------------------------------------------------------------------
# Key transformation helpers
# ---------------------------------------------------------------------------
def _parse_block_idx(key: str) -> int | None:
"""Extract block index from a key like 'blocks.5.attn1.to_q.weight'."""
m = _BLOCK_IDX_RE.match(key)
return int(m.group(1)) if m else None
def _apply_rename_dict(key: str) -> str:
for src, dst in TRANSFORMER_KEYS_RENAME_DICT.items():
key = key.replace(src, dst)
return key
def _strip_mxfp4_wrapper(key: str) -> str:
"""Strip .linear.ATTR or .div.mul_scale wrappers added by msModelSlim.
MXFP4 tensors are wrapped in sub-modules:
X.linear.weight / weight_scale / weight_dual_scale / bias
X.div.mul_scale
MXFP8 and FLOAT tensors have no wrappers — this function is a no-op for them.
Examples after apply_rename_dict:
attn1.to_q.linear.weight → attn1.to_q.weight
attn1.to_q.linear.weight_scale → attn1.to_q.weight_scale
attn1.to_q.div.mul_scale → attn1.to_q.mul_scale
attn1.norm_q.weight → attn1.norm_q.weight (unchanged)
"""
# Check longest attribute names first to avoid partial suffix matches.
for attr in _LINEAR_ATTRS:
suffix = f".linear.{attr}"
if key.endswith(suffix):
return key[: -len(suffix)] + f".{attr}"
if key.endswith(".div.mul_scale"):
return key[: -len(".div.mul_scale")] + ".mul_scale"
return key
# ---------------------------------------------------------------------------
# Quantization metadata helpers
# ---------------------------------------------------------------------------
# Known quantized weight types (FLOAT tensors don't determine block type).
_MXFP8_TYPE = "mxfp8"
_MXFP4_DUALSCALE_TYPE = "mxfp4_dualscale"
def _classify_blocks(quant_meta: dict[str, str]) -> dict[int, str]:
"""Classify each transformer block by quantization type from quant_meta.
Returns a dict mapping block_idx → 'mxfp8' | 'mxfp4_dualscale'.
A block's type is determined by the first quantized (non-FLOAT) tensor found for it.
"""
block_types: dict[int, str] = {}
for key, qtype in quant_meta.items():
idx = _parse_block_idx(key)
if idx is None or idx in block_types:
continue
if qtype.startswith("W8A8_MXFP8"):
block_types[idx] = _MXFP8_TYPE
elif qtype.startswith("W4A4_MXFP4_DUALSCALE"):
block_types[idx] = _MXFP4_DUALSCALE_TYPE
return block_types
def _print_block_summary(block_types: dict[int, str]) -> None:
"""Print a compact run-length summary of the block layout."""
if not block_types:
print(" Block layout: (empty)")
return
sorted_indices = sorted(block_types)
runs: list[tuple[int, int, str]] = []
run_start = sorted_indices[0]
run_type = block_types[run_start]
for idx in sorted_indices[1:]:
if block_types[idx] != run_type:
runs.append((run_start, idx - 1, run_type))
run_start = idx
run_type = block_types[idx]
runs.append((run_start, sorted_indices[-1], run_type))
print(f" Block layout ({len(sorted_indices)} blocks classified):")
for start, end, btype in runs:
count = end - start + 1
range_str = f"{start}" if start == end else f"{start}–{end}"
print(f" blocks {range_str:>8}: {btype} ({count} block{'s' if count > 1 else ''})")
# ---------------------------------------------------------------------------
# Safetensors I/O
# ---------------------------------------------------------------------------
def _load_safetensors_dir(directory: pathlib.Path, glob: str = "*.safetensors") -> dict[str, torch.Tensor]:
candidates = sorted(directory.glob(glob))
if not candidates:
raise FileNotFoundError(f"No safetensors matching '{glob}' found in {directory}")
state: dict[str, torch.Tensor] = {}
for f in candidates:
state.update(load_file(str(f)))
return state
def _load_quant_safetensors(directory: pathlib.Path) -> dict[str, torch.Tensor]:
try:
return _load_safetensors_dir(directory, "quant_model_weight*.safetensors")
except FileNotFoundError:
return _load_safetensors_dir(directory)
def _load_quant_meta(directory: pathlib.Path) -> dict[str, str]:
candidates = sorted(directory.glob("quant_model_description*.json"))
if not candidates:
print(f" WARNING: No quant_model_description*.json in {directory}; treating all tensors as FLOAT.")
return {}
with open(candidates[0]) as f:
return json.load(f)
# ---------------------------------------------------------------------------
# Per-transformer conversion
# ---------------------------------------------------------------------------
def _is_mxfp4_tensor(key: str, qtype: str) -> bool:
"""Return True for MXFP4 quantized tensors and their companion tensors.
MXFP4 DualScale layers produce four tensor types:
.linear.weight / .linear.weight_scale / .linear.weight_dual_scale (W4A4_MXFP4_DUALSCALE)
.linear.bias / .div.mul_scale (FLOAT but companion to a quantized layer)
BF16 fallback layers have plain .weight / .bias without .linear. / .div. wrappers.
"""
if qtype.startswith("W4A4_MXFP4_DUALSCALE"):
return True
# Companion tensors: bias and mul_scale that belong to an MXFP4 layer.
return ".linear." in key or ".div.mul_scale" in key
def _diffusers_to_vllm_ignored(diffusers_ignored: list[str]) -> list[str]:
"""Translate diffusers checkpoint prefixes to vllm-omni model parameter names.
Three transformations align the naming conventions:
1. Self-attention QKV fusion (attn1 only, not attn2):
attn1.to_q + attn1.to_k + attn1.to_v → attn1.to_qkv
Cross-attention (attn2) to_q/k/v are not fused and remain separate.
2. FFN naming:
ffn.net.0.proj → ffn.net_0.proj
ffn.net.2 → ffn.net_2
3. to_out index:
attn1.to_out.0 / attn2.to_out.0 → attn1.to_out / attn2.to_out
"""
ignored_set = set(diffusers_ignored)
result: set[str] = set()
for name in diffusers_ignored:
m = re.match(r"^(.*\.attn1)\.to_([qkv])$", name)
if m:
prefix = m.group(1)
if all(f"{prefix}.to_{c}" in ignored_set for c in ("q", "k", "v")):
result.add(f"{prefix}.to_qkv")
else:
present = [f"to_{c}" for c in ("q", "k", "v") if f"{prefix}.to_{c}" in ignored_set]
missing = [f"to_{c}" for c in ("q", "k", "v") if f"{prefix}.to_{c}" not in ignored_set]
raise ValueError(
f"Partial BF16 fallback for '{prefix}': "
f"{', '.join(present)} in ignored_layers but {', '.join(missing)} is not. "
f"Self-attention Q/K/V are fused into a single to_qkv layer at runtime; "
f"all three must share the same precision. "
f"Either quantize all of to_q/to_k/to_v or keep all three in BF16."
)
continue
name = re.sub(r"\.ffn\.net\.0\.proj$", ".ffn.net_0.proj", name)
name = re.sub(r"\.ffn\.net\.2$", ".ffn.net_2", name)
name = re.sub(r"\.to_out\.0$", ".to_out", name)
result.add(name)
return sorted(result)
def _collect_ignored_layers(merged: dict[str, Any], mxfp4_layer_prefixes: set[str]) -> list[str]:
"""Collect vllm-omni parameter name prefixes for BF16 fallback layers.
A layer is BF16 if it has a .weight tensor but its prefix is not in
mxfp4_layer_prefixes. Prefixes are returned in vllm-omni parameter naming
(QKV-fused, FFN underscored, to_out unindexed) so the list can be written
directly into ignored_layers in config.json without further translation.
"""
all_weight_prefixes = {key[: -len(".weight")] for key in merged if key.endswith(".weight")}
return _diffusers_to_vllm_ignored(sorted(all_weight_prefixes - mxfp4_layer_prefixes))
def _convert_transformer(
quant_subdir: pathlib.Path,
output_dir: pathlib.Path,
original_transformer_dir: pathlib.Path,
) -> None:
"""Convert one transformer directory to the mxfp4_dualscale + BF16 mixed format."""
output_dir.mkdir(parents=True, exist_ok=True)
# BF16 base: provides the scaffold for non-MXFP4 tensors (norms, embeddings,
# BF16 fallback linear layers that msModelSlim may omit or keep as FLOAT).
print(f" Loading BF16 base from {original_transformer_dir} …")
base_state = _load_safetensors_dir(original_transformer_dir)
print(f" {len(base_state)} BF16 tensors loaded")
print(f" Loading quantized weights from {quant_subdir} …")
quant_state = _load_quant_safetensors(quant_subdir)
quant_meta = _load_quant_meta(quant_subdir)
print(f" {len(quant_state)} quant tensors, {len(quant_meta)} meta entries")
# Classify blocks for a compact summary (informational only in the new scheme).
block_types = _classify_blocks(quant_meta)
_print_block_summary(block_types)
# Remap MXFP4 quantized tensors only.
# BF16 fallback tensors (FLOAT in quant_meta, no .linear./.div. wrapper) are
# skipped here; the base_state provides them unchanged.
remapped: dict[str, torch.Tensor] = {}
remapped_meta: dict[str, str] = {}
mxfp4_layer_prefixes: set[str] = set()
skipped: list[str] = []
for key, tensor in quant_state.items():
qtype = quant_meta.get(key, "FLOAT")
if not _is_mxfp4_tensor(key, qtype):
continue # BF16 fallback — base_state already covers this tensor
renamed = _apply_rename_dict(key)
final_key = _strip_mxfp4_wrapper(renamed)
if final_key.endswith(".quant_type"):
skipped.append(key)
continue
remapped[final_key] = tensor
remapped_meta[final_key] = qtype
# Track MXFP4 layer prefixes (via their .weight keys) for ignored_layers.
if final_key.endswith(".weight") and qtype.startswith("W4A4_MXFP4_DUALSCALE"):
mxfp4_layer_prefixes.add(final_key[: -len(".weight")])
if skipped:
print(f" Skipped {len(skipped)} metadata keys (quant_type markers): {skipped[:5]}")
print(f" {len(remapped)} MXFP4 tensors remapped, {len(mxfp4_layer_prefixes)} quantized layers")
# Merge: base_state provides BF16 scaffold; MXFP4 tensors override their BF16 counterparts
# and add the new scale tensors (weight_scale, weight_dual_scale, mul_scale).
merged = {**base_state, **remapped}
# Determine ignored_layers: BF16 fallback layer prefixes in vllm-omni parameter naming.
ignored_layers = _collect_ignored_layers(merged, mxfp4_layer_prefixes)
print(f" {len(ignored_layers)} BF16 fallback layers → ignored_layers in config.json")
# Save weights.
out_weights = output_dir / "diffusion_pytorch_model.safetensors"
save_file(merged, str(out_weights))
print(f" Saved {len(merged)} tensors → {out_weights}")
# Save remapped quant metadata (for inspection / debugging).
out_meta_path = output_dir / "quant_model_description.json"
with open(out_meta_path, "w") as f:
json.dump(remapped_meta, f, indent=2)
# Inject quantization_config into config.json.
src_config = original_transformer_dir / "config.json"
if src_config.is_file():
with open(src_config) as f:
config = json.load(f)
config["quantization_config"] = _build_quant_config(ignored_layers)
out_config = output_dir / "config.json"
with open(out_config, "w") as f:
json.dump(config, f, indent=2)
print(f" Injected quantization_config (mxfp4_dualscale, {len(ignored_layers)} ignored_layers) → {out_config}")
else:
print(f" WARNING: No config.json at {src_config}; quantization_config not injected.")
def _build_quant_config(ignored_layers: list[str]) -> dict[str, Any]:
return {
"quant_method": "mxfp4_dualscale",
"is_checkpoint_serialized": True,
"ignored_layers": ignored_layers,
}
# ---------------------------------------------------------------------------
# Model-type helpers
# ---------------------------------------------------------------------------
def _get_transformer_dirs(model_type: str) -> list[str]:
return ["transformer", "transformer_2"] if model_type in CASCADE_MODEL_TYPES else ["transformer"]
def _get_quant_subdir(model_type: str, quant_path: pathlib.Path, transformer_dir: str) -> pathlib.Path:
if model_type in CASCADE_MODEL_TYPES:
sub = "high_noise_model" if transformer_dir == "transformer" else "low_noise_model"
return quant_path / sub
return quant_path
# ---------------------------------------------------------------------------
# Main repack
# ---------------------------------------------------------------------------
def repack(
model_type: str,
original_model_path: pathlib.Path,
quant_path: pathlib.Path,
output_path: pathlib.Path,
) -> None:
transformer_dirs = _get_transformer_dirs(model_type)
print(f"Copying original model to {output_path} (skipping {transformer_dirs}) …")
shutil.copytree(
str(original_model_path),
str(output_path),
ignore=shutil.ignore_patterns(*transformer_dirs),
)
for tdir in transformer_dirs:
q_subdir = _get_quant_subdir(model_type, quant_path, tdir)
out_tdir = output_path / tdir
orig_tdir = original_model_path / tdir
print(f"\nConverting {tdir} (quant source: {q_subdir.name}) …")
_convert_transformer(q_subdir, out_tdir, orig_tdir)
print(f"\nDone. Merged model → {output_path}")
print("\nRun inference (quantization auto-detected from config.json):")
print(f" python text_to_video.py --model {output_path}")
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--model-type", required=True, choices=SUPPORTED_MODEL_TYPES, help="Model variant.")
parser.add_argument("--original-model", required=True, help="Original HF Diffusers model directory (BF16).")
parser.add_argument("--quant-path", required=True, help="msModelSlim quantized weights directory.")
parser.add_argument("--output-path", required=True, help="Output directory for merged model.")
args = parser.parse_args()
repack(
model_type=args.model_type,
original_model_path=pathlib.Path(args.original_model),
quant_path=pathlib.Path(args.quant_path),
output_path=pathlib.Path(args.output_path),
)
if __name__ == "__main__":
main()