[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
1f47fa2f29
commit
0b0d2d7009
141 files changed
+2885
-721
No files matched your search
@@ -81,6 +81,16 @@ from .component_config import ComponentQuantizationConfig # noqa: E402
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _build_fp8(**kw: Any) -> QuantizationConfig:
|
||||
if kw.get("is_checkpoint_fp8_serialized", False):
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
return Fp8Config(**kw)
|
||||
from .fp8_config import DiffusionFp8Config
|
||||
|
||||
return DiffusionFp8Config(**kw)
|
||||
|
||||
|
||||
def _build_int8(**kw: Any) -> QuantizationConfig:
|
||||
"""Lazy import for Int8 diffusion config (supports CUDA + NPU)."""
|
||||
from .int8_config import DiffusionInt8Config
|
||||
@@ -168,6 +178,7 @@ def _build_torchao_float8_weight_only(**kw: Any) -> QuantizationConfig:
|
||||
|
||||
|
||||
_OVERRIDES: dict[str, Callable[..., QuantizationConfig]] = {
|
||||
"fp8": _build_fp8,
|
||||
"int8": _build_int8,
|
||||
"bitsandbytes": _build_bitsandbytes,
|
||||
"mxfp8": _build_mxfp8,
|
||||
@@ -330,6 +341,17 @@ def _build_single(method: str, **kwargs: Any) -> QuantizationConfig:
|
||||
|
||||
config_cls = get_quantization_config(method)
|
||||
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
|
||||
if config_cls is OnlineQuantizationConfig:
|
||||
if "args" in kwargs:
|
||||
return config_cls(**kwargs)
|
||||
args = resolve_quantization_config(method, kwargs or None)
|
||||
if args is None:
|
||||
raise ValueError("Online quantization requires quantization config arguments")
|
||||
return config_cls(args)
|
||||
|
||||
try:
|
||||
return config_cls(**kwargs)
|
||||
except TypeError:
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Omni's online FP8 option backed by vLLM's online quantization API."""
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
|
||||
|
||||
class DiffusionFp8Config(OnlineQuantizationConfig):
|
||||
"""Keep the diffusion ``fp8`` option for quantizing BF16/FP16 weights.
|
||||
|
||||
Upstream Fp8Config now describes serialized checkpoints only. Keeping the
|
||||
Omni method name also lets checkpoint metadata replace this online config
|
||||
through the existing serialized-checkpoint reconciliation path.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_checkpoint_fp8_serialized: bool = False,
|
||||
activation_scheme: str = "dynamic",
|
||||
ignored_layers: list[str] | None = None,
|
||||
weight_block_size: list[int] | None = None,
|
||||
store_dtype: str | None = None,
|
||||
) -> None:
|
||||
if is_checkpoint_fp8_serialized:
|
||||
raise ValueError("Serialized FP8 checkpoints require upstream Fp8Config")
|
||||
if activation_scheme != "dynamic":
|
||||
raise ValueError("Online FP8 quantization requires activation_scheme='dynamic'")
|
||||
if weight_block_size is not None:
|
||||
raise ValueError("Block-wise FP8 quantization requires a serialized checkpoint or fp8_per_block")
|
||||
if store_dtype is not None:
|
||||
raise ValueError("Online FP8 quantization does not support store_dtype")
|
||||
super().__init__(
|
||||
QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight="fp8_per_tensor_static"),
|
||||
moe=QuantSpec(weight="fp8_per_tensor_static"),
|
||||
ignore=list(ignored_layers or []),
|
||||
)
|
||||
)
|
||||
self.is_checkpoint_fp8_serialized = False
|
||||
self.activation_scheme = activation_scheme
|
||||
self.weight_block_size = None
|
||||
self.store_dtype = None
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "fp8"
|
||||
Reference in new issue
Block a user