[MXFP8][XPU] enable mxfp8 using vLLM main repo method (#3782)

Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
Chendi.Xue authored and GitHub committed 2026-05-28 02:47:22 +00:00
1 parent 39705f9ee8
commit 1903cee56a
8 files changed
+328 -35

No files matched your search

+6
View File
@@ -399,6 +399,12 @@ def resolve_quant_config_from_disk(
)
return build_quant_config(qc_method, **qc_kwargs)
# AutoRound MXFP8 checkpoints use data_type="mx_fp" instead of
# is_checkpoint_*_serialized; rebuild so the offline path is selected.
if qc_kwargs.get("data_type") == "mx_fp":
logger.info("config.json declares data_type='mx_fp'; rebuilding as offline AutoRound MXFP8.")
return build_quant_config(qc_method, **qc_kwargs)
if (
"ignored_layers" in qc_kwargs
and hasattr(quant_config, "ignored_layers")
+138 -1
View File
@@ -7,8 +7,13 @@ from __future__ import annotations
from os.path import commonprefix
from typing import TYPE_CHECKING, Any
import torch
from torch.nn import Module
from vllm.model_executor.layers.linear import LinearMethodBase
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.models.utils import WeightsMapper
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.model_executor.utils import replace_parameter
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import (
@@ -45,12 +50,39 @@ def _map_with_stage_prefix(
class OmniINCConfig(INCConfig):
"""INCConfig extended with multi-stage prefix remapping."""
"""INCConfig extended with multi-stage prefix remapping and MXFP8 support.
AutoRound MXFP8 checkpoints declare quant_method="auto-round" with data_type="mx_fp".
This config detects that case and dispatches to IncMxfp8OfflineLinearMethod instead
of the standard INT quantization path.
Architecture:
- AutoRound INT4/INT8 → standard INCConfig path (xpu_w4a16_quant_layer, etc.)
- AutoRound MXFP8 (data_type="mx_fp") → IncMxfp8OfflineLinearMethod
- Native MXFP8 (quant_method="mxfp8") → DiffusionMXFP8Config → NPUMxfp8LinearMethod (NPU only)
"""
# Extend supported data types and formats to include MXFP8
SUPPORTED_DTYPES = {"int", "mx_fp"}
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq", "auto_round:llm_compressor"}
# ------------------------------------------------------------------
# Core integration: called by vLLM's configure_quant_config()
# ------------------------------------------------------------------
def get_quant_method(self, layer, prefix: str):
"""Get quantization method, handling AutoRound MXFP8 as a special case."""
# Check if this is an AutoRound MXFP8 checkpoint (data_type="mx_fp")
if hasattr(self, "data_type") and self.data_type == "mx_fp" and self.weight_bits == 8:
from vllm.model_executor.layers.linear import LinearBase
if isinstance(layer, LinearBase):
return IncMxfp8OfflineLinearMethod()
return None
# Otherwise, use parent INCConfig logic
return super().get_quant_method(layer, prefix)
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
"""Remap HF checkpoint names to vLLM runtime prefixes."""
prefix_map = getattr(hf_to_vllm_mapper, "orig_to_new_prefix", None) or {}
@@ -138,3 +170,108 @@ class OmniINCConfig(INCConfig):
if isinstance(quant_config, INCConfig):
return cls.from_inc_config(quant_config)
return quant_config
# ---------------------------------------------------------------------------
# IncMxfp8OfflineLinearMethod - used by AutoRound MXFP8 checkpoints
# ---------------------------------------------------------------------------
class IncMxfp8OfflineLinearMethod(LinearMethodBase):
"""Offline MXFP8 linear method for AutoRound MXFP8 checkpoints.
For pre-quantized AutoRound MXFP8 checkpoints (data_type="mx_fp") where weights
are stored as BF16 (losslessly representing FP8 E4M3 values) with uint8 MX block scales.
Delegates to the platform-appropriate kernel (XPUMxFp8LinearKernel, etc.).
Used by OmniINCConfig when data_type="mx_fp".
"""
def __init__(self) -> None:
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
)
self.kernel = init_mxfp8_linear_kernel()
self.block_size = MXFP8_BLOCK_SIZE
self.scale_dtype = MXFP8_SCALE_DTYPE
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
if input_size_per_partition % self.block_size != 0:
raise ValueError(
f"MXFP8 requires input_size_per_partition "
f"({input_size_per_partition}) to be divisible by "
f"{self.block_size}."
)
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
# Allocate as params_dtype (BF16) to match checkpoint dtype.
# Will be cast to FP8 in process_weights_after_loading.
layer.register_parameter(
"weight",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
num_groups = input_size_per_partition // self.block_size
layer.register_parameter(
"weight_scale",
ModelWeightParameter(
data=torch.empty(output_size_per_partition, num_groups, dtype=self.scale_dtype),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
),
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
# Cast BF16 weight to FP8.
# Checkpoint stores weights as BF16 (losslessly representing FP8 E4M3 values).
w = layer.weight.data
if w.dtype != torch.float8_e4m3fn:
# Cast to FP8 E4M3. AutoRound stores FP8 values as BF16, so this is lossless.
w = w.to(torch.float8_e4m3fn).contiguous()
replace_parameter(layer, "weight", w)
# Delegate layout transforms (transpose, scale reinterpret) to the platform kernel.
self.kernel.process_weights_after_loading(layer)
layer._already_called_process_weights_after_loading = True
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
ori_shape = x.shape
if x.dim() > 2:
x = x.reshape(-1, ori_shape[-1])
output = self.kernel.apply_weights(layer, x, bias)
if len(ori_shape) > 2:
output = output.reshape(*ori_shape[:-1], -1)
return output
+70 -2
View File
@@ -133,10 +133,17 @@ class DiffusionMXFP8Config(QuantizationConfig):
if self.is_checkpoint_mxfp8_serialized:
return NPUMxfp8LinearMethod(self)
return NPUMxfp8OnlineLinearMethod(self)
# Placeholder for future platforms: add `elif is_cuda(): ...` here.
if current_omni_platform.is_xpu():
if self.is_checkpoint_mxfp8_serialized:
raise NotImplementedError(
"Native MXFP8 offline mode is not supported on XPU. "
"Use AutoRound MXFP8 checkpoints (quant_method='auto-round', data_type='mx_fp') instead, "
"or use online MXFP8 mode without is_checkpoint_mxfp8_serialized."
)
return VllmMxfp8OnlineLinearMethod()
raise NotImplementedError(
"DiffusionMXFP8Config (W8A8 MXFP8) is currently only supported "
"on NPU (Ascend) platforms. CUDA support is not yet implemented."
"on NPU (Ascend) and XPU (Intel) platforms."
)
return None
@@ -450,3 +457,64 @@ class NPUMxfp8OnlineLinearMethod(_LazyWeightMixin, NPUMxfp8LinearMethod):
replace_parameter(layer, "weight", weight_fp8)
replace_parameter(layer, "weight_scale", weight_scale)
layer._already_called_process_weights_after_loading = True
# ---------------------------------------------------------------------------
# vLLM kernel-based methods (XPU, CUDA, ROCm — any platform with Mxfp8LinearKernel)
# ---------------------------------------------------------------------------
# Note: VllmMxfp8OfflineLinearMethod removed - XPU only supports AutoRound MXFP8 offline
# (see IncMxfp8OfflineLinearMethod in inc_config.py)
try:
from vllm.model_executor.layers.quantization.online.mxfp8 import (
Mxfp8OnlineLinearMethod as _VllmMxfp8OnlineBase,
)
except ImportError as e:
raise ImportError(
"vLLM MXFP8 support is not available. "
"Online MXFP8 quantization requires vLLM with MXFP8 kernel support. "
"Please upgrade vLLM or use a compatible build."
) from e
class VllmMxfp8OnlineLinearMethod(_VllmMxfp8OnlineBase):
"""Online MXFP8 linear method extending vLLM's implementation with 3D tensor support.
Loads BF16 weights, quantizes to MXFP8 at load time using vLLM's kernel.
Adds reshape logic to handle 3D tensors from diffusion models.
Inherits from vLLM's Mxfp8OnlineLinearMethod and only overrides __init__ and apply().
__init__ directly initializes the kernel without calling super().__init__() to avoid
vLLM config dependency in unit tests. All other methods (create_weights,
process_weights_after_loading) are inherited directly from vLLM.
"""
def __init__(self) -> None:
"""Initialize kernel directly without calling parent __init__ to avoid config dependency.
TODO: skipping super().__init__() couples this to vLLM internals. If the parent
adds required initialization, consider a version-aware assertion or calling
super().__init__() with a fallback config.
"""
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
self.kernel = init_mxfp8_linear_kernel()
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply with 3D tensor support for diffusion models.
vLLM's kernel expects 2D tensors (M, K), but diffusion models pass
3D tensors (batch, seq, hidden). Flatten before kernel, reshape after.
"""
ori_shape = x.shape
if x.dim() > 2:
x = x.reshape(-1, ori_shape[-1])
output = super().apply(layer, x, bias)
if len(ori_shape) > 2:
output = output.reshape(*ori_shape[:-1], -1)
return output