[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
1f47fa2f29
commit
0b0d2d7009
141 files changed
+2885
-721
No files matched your search
@@ -81,6 +81,16 @@ from .component_config import ComponentQuantizationConfig # noqa: E402
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _build_fp8(**kw: Any) -> QuantizationConfig:
|
||||
if kw.get("is_checkpoint_fp8_serialized", False):
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
return Fp8Config(**kw)
|
||||
from .fp8_config import DiffusionFp8Config
|
||||
|
||||
return DiffusionFp8Config(**kw)
|
||||
|
||||
|
||||
def _build_int8(**kw: Any) -> QuantizationConfig:
|
||||
"""Lazy import for Int8 diffusion config (supports CUDA + NPU)."""
|
||||
from .int8_config import DiffusionInt8Config
|
||||
@@ -168,6 +178,7 @@ def _build_torchao_float8_weight_only(**kw: Any) -> QuantizationConfig:
|
||||
|
||||
|
||||
_OVERRIDES: dict[str, Callable[..., QuantizationConfig]] = {
|
||||
"fp8": _build_fp8,
|
||||
"int8": _build_int8,
|
||||
"bitsandbytes": _build_bitsandbytes,
|
||||
"mxfp8": _build_mxfp8,
|
||||
@@ -330,6 +341,17 @@ def _build_single(method: str, **kwargs: Any) -> QuantizationConfig:
|
||||
|
||||
config_cls = get_quantization_config(method)
|
||||
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
|
||||
if config_cls is OnlineQuantizationConfig:
|
||||
if "args" in kwargs:
|
||||
return config_cls(**kwargs)
|
||||
args = resolve_quantization_config(method, kwargs or None)
|
||||
if args is None:
|
||||
raise ValueError("Online quantization requires quantization config arguments")
|
||||
return config_cls(args)
|
||||
|
||||
try:
|
||||
return config_cls(**kwargs)
|
||||
except TypeError:
|
||||
|
||||
Reference in new issue
Block a user