[MXFP8][XPU] enable mxfp8 using vLLM main repo method (#3782)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
1 parent
39705f9ee8
commit
1903cee56a
8 files changed
+328
-35
No files matched your search
@@ -399,6 +399,12 @@ def resolve_quant_config_from_disk(
|
||||
)
|
||||
return build_quant_config(qc_method, **qc_kwargs)
|
||||
|
||||
# AutoRound MXFP8 checkpoints use data_type="mx_fp" instead of
|
||||
# is_checkpoint_*_serialized; rebuild so the offline path is selected.
|
||||
if qc_kwargs.get("data_type") == "mx_fp":
|
||||
logger.info("config.json declares data_type='mx_fp'; rebuilding as offline AutoRound MXFP8.")
|
||||
return build_quant_config(qc_method, **qc_kwargs)
|
||||
|
||||
if (
|
||||
"ignored_layers" in qc_kwargs
|
||||
and hasattr(quant_config, "ignored_layers")
|
||||
|
||||
@@ -7,8 +7,13 @@ from __future__ import annotations
|
||||
from os.path import commonprefix
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
@@ -45,12 +50,39 @@ def _map_with_stage_prefix(
|
||||
|
||||
|
||||
class OmniINCConfig(INCConfig):
|
||||
"""INCConfig extended with multi-stage prefix remapping."""
|
||||
"""INCConfig extended with multi-stage prefix remapping and MXFP8 support.
|
||||
|
||||
AutoRound MXFP8 checkpoints declare quant_method="auto-round" with data_type="mx_fp".
|
||||
This config detects that case and dispatches to IncMxfp8OfflineLinearMethod instead
|
||||
of the standard INT quantization path.
|
||||
|
||||
Architecture:
|
||||
- AutoRound INT4/INT8 → standard INCConfig path (xpu_w4a16_quant_layer, etc.)
|
||||
- AutoRound MXFP8 (data_type="mx_fp") → IncMxfp8OfflineLinearMethod
|
||||
- Native MXFP8 (quant_method="mxfp8") → DiffusionMXFP8Config → NPUMxfp8LinearMethod (NPU only)
|
||||
"""
|
||||
|
||||
# Extend supported data types and formats to include MXFP8
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp"}
|
||||
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq", "auto_round:llm_compressor"}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core integration: called by vLLM's configure_quant_config()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_quant_method(self, layer, prefix: str):
|
||||
"""Get quantization method, handling AutoRound MXFP8 as a special case."""
|
||||
# Check if this is an AutoRound MXFP8 checkpoint (data_type="mx_fp")
|
||||
if hasattr(self, "data_type") and self.data_type == "mx_fp" and self.weight_bits == 8:
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
return IncMxfp8OfflineLinearMethod()
|
||||
return None
|
||||
|
||||
# Otherwise, use parent INCConfig logic
|
||||
return super().get_quant_method(layer, prefix)
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None:
|
||||
"""Remap HF checkpoint names to vLLM runtime prefixes."""
|
||||
prefix_map = getattr(hf_to_vllm_mapper, "orig_to_new_prefix", None) or {}
|
||||
@@ -138,3 +170,108 @@ class OmniINCConfig(INCConfig):
|
||||
if isinstance(quant_config, INCConfig):
|
||||
return cls.from_inc_config(quant_config)
|
||||
return quant_config
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IncMxfp8OfflineLinearMethod - used by AutoRound MXFP8 checkpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IncMxfp8OfflineLinearMethod(LinearMethodBase):
|
||||
"""Offline MXFP8 linear method for AutoRound MXFP8 checkpoints.
|
||||
|
||||
For pre-quantized AutoRound MXFP8 checkpoints (data_type="mx_fp") where weights
|
||||
are stored as BF16 (losslessly representing FP8 E4M3 values) with uint8 MX block scales.
|
||||
Delegates to the platform-appropriate kernel (XPUMxFp8LinearKernel, etc.).
|
||||
|
||||
Used by OmniINCConfig when data_type="mx_fp".
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
)
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
self.block_size = MXFP8_BLOCK_SIZE
|
||||
self.scale_dtype = MXFP8_SCALE_DTYPE
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
if input_size_per_partition % self.block_size != 0:
|
||||
raise ValueError(
|
||||
f"MXFP8 requires input_size_per_partition "
|
||||
f"({input_size_per_partition}) to be divisible by "
|
||||
f"{self.block_size}."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
|
||||
# Allocate as params_dtype (BF16) to match checkpoint dtype.
|
||||
# Will be cast to FP8 in process_weights_after_loading.
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
num_groups = input_size_per_partition // self.block_size
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(output_size_per_partition, num_groups, dtype=self.scale_dtype),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
# Cast BF16 weight to FP8.
|
||||
# Checkpoint stores weights as BF16 (losslessly representing FP8 E4M3 values).
|
||||
w = layer.weight.data
|
||||
if w.dtype != torch.float8_e4m3fn:
|
||||
# Cast to FP8 E4M3. AutoRound stores FP8 values as BF16, so this is lossless.
|
||||
w = w.to(torch.float8_e4m3fn).contiguous()
|
||||
replace_parameter(layer, "weight", w)
|
||||
|
||||
# Delegate layout transforms (transpose, scale reinterpret) to the platform kernel.
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
ori_shape = x.shape
|
||||
if x.dim() > 2:
|
||||
x = x.reshape(-1, ori_shape[-1])
|
||||
output = self.kernel.apply_weights(layer, x, bias)
|
||||
if len(ori_shape) > 2:
|
||||
output = output.reshape(*ori_shape[:-1], -1)
|
||||
return output
|
||||
@@ -133,10 +133,17 @@ class DiffusionMXFP8Config(QuantizationConfig):
|
||||
if self.is_checkpoint_mxfp8_serialized:
|
||||
return NPUMxfp8LinearMethod(self)
|
||||
return NPUMxfp8OnlineLinearMethod(self)
|
||||
# Placeholder for future platforms: add `elif is_cuda(): ...` here.
|
||||
if current_omni_platform.is_xpu():
|
||||
if self.is_checkpoint_mxfp8_serialized:
|
||||
raise NotImplementedError(
|
||||
"Native MXFP8 offline mode is not supported on XPU. "
|
||||
"Use AutoRound MXFP8 checkpoints (quant_method='auto-round', data_type='mx_fp') instead, "
|
||||
"or use online MXFP8 mode without is_checkpoint_mxfp8_serialized."
|
||||
)
|
||||
return VllmMxfp8OnlineLinearMethod()
|
||||
raise NotImplementedError(
|
||||
"DiffusionMXFP8Config (W8A8 MXFP8) is currently only supported "
|
||||
"on NPU (Ascend) platforms. CUDA support is not yet implemented."
|
||||
"on NPU (Ascend) and XPU (Intel) platforms."
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -450,3 +457,64 @@ class NPUMxfp8OnlineLinearMethod(_LazyWeightMixin, NPUMxfp8LinearMethod):
|
||||
replace_parameter(layer, "weight", weight_fp8)
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# vLLM kernel-based methods (XPU, CUDA, ROCm — any platform with Mxfp8LinearKernel)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Note: VllmMxfp8OfflineLinearMethod removed - XPU only supports AutoRound MXFP8 offline
|
||||
# (see IncMxfp8OfflineLinearMethod in inc_config.py)
|
||||
|
||||
try:
|
||||
from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
Mxfp8OnlineLinearMethod as _VllmMxfp8OnlineBase,
|
||||
)
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"vLLM MXFP8 support is not available. "
|
||||
"Online MXFP8 quantization requires vLLM with MXFP8 kernel support. "
|
||||
"Please upgrade vLLM or use a compatible build."
|
||||
) from e
|
||||
|
||||
|
||||
class VllmMxfp8OnlineLinearMethod(_VllmMxfp8OnlineBase):
|
||||
"""Online MXFP8 linear method extending vLLM's implementation with 3D tensor support.
|
||||
|
||||
Loads BF16 weights, quantizes to MXFP8 at load time using vLLM's kernel.
|
||||
Adds reshape logic to handle 3D tensors from diffusion models.
|
||||
|
||||
Inherits from vLLM's Mxfp8OnlineLinearMethod and only overrides __init__ and apply().
|
||||
__init__ directly initializes the kernel without calling super().__init__() to avoid
|
||||
vLLM config dependency in unit tests. All other methods (create_weights,
|
||||
process_weights_after_loading) are inherited directly from vLLM.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize kernel directly without calling parent __init__ to avoid config dependency.
|
||||
|
||||
TODO: skipping super().__init__() couples this to vLLM internals. If the parent
|
||||
adds required initialization, consider a version-aware assertion or calling
|
||||
super().__init__() with a fallback config.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply with 3D tensor support for diffusion models.
|
||||
|
||||
vLLM's kernel expects 2D tensors (M, K), but diffusion models pass
|
||||
3D tensors (batch, seq, hidden). Flatten before kernel, reshape after.
|
||||
"""
|
||||
ori_shape = x.shape
|
||||
if x.dim() > 2:
|
||||
x = x.reshape(-1, ori_shape[-1])
|
||||
output = super().apply(layer, x, bias)
|
||||
if len(ori_shape) > 2:
|
||||
output = output.reshape(*ori_shape[:-1], -1)
|
||||
return output
|
||||
Reference in new issue
Block a user