[bugfix] Fix FP8 Quack kernels when using Context Parallelism and reduce number of recompilations (#7458)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
This commit is contained in:
MaciejBalaNV authored and GitHub committed 2026-09-22 11:38:17 -07:00
1 parent c482626450
commit 7d6e2ade6c
3 files changed
+329 -13

No files matched your search

+80 -12
View File
@@ -5,6 +5,9 @@
from __future__ import annotations
import os
from contextlib import contextmanager
from importlib import import_module
from multiprocessing import current_process
import torch
from vllm.logger import init_logger
@@ -52,6 +55,41 @@ def _set_persistent_cache_dir() -> None:
os.environ["QUACK_CACHE_DIR"] = os.path.join(root, "vllm_omni", "quack")
def _configure_quack_compilation() -> None:
"""Keep autotuning in daemon workers without starting compiler children."""
try:
async_compile = import_module("quack.cache.async_compile")
except ModuleNotFoundError as exc:
# Older Quack releases compile synchronously and have no async pool.
if exc.name not in {"quack.cache", "quack.cache.async_compile"}:
raise
return
original_pool_scope = getattr(async_compile, "pool_scope", None)
suppress_pool = getattr(async_compile, "suppress_pool", None)
if not callable(original_pool_scope) or not callable(suppress_pool):
logger.warning(
"Quack async compilation API is unsupported: pool_scope and suppress_pool must be callable. "
"Skipping the compilation patch; autotuning in daemon workers may fail."
)
return
if getattr(original_pool_scope, "_omni_daemon_safe", False):
return
@contextmanager
def daemon_safe_pool_scope():
# spawn can import this module before installing the child's daemon flag.
# Check at tuning time; suppress_pool keeps compilation in-process.
if current_process().daemon:
with suppress_pool():
yield None
else:
with original_pool_scope() as pool:
yield pool
daemon_safe_pool_scope._omni_daemon_safe = True
async_compile.pool_scope = daemon_safe_pool_scope
def _load_quack():
global _gemm_interface
if _gemm_interface is not None:
@@ -72,6 +110,7 @@ def _load_quack():
torch2cute_dtype_map.setdefault(torch.float8_e4m3fn, cutlass.Float8E4M3FN)
torch2cute_dtype_map.setdefault(torch.float8_e5m2, cutlass.Float8E5M2)
_configure_quack_compilation()
_gemm_interface = gemm_interface
logger.info("Quack FP8 fused-bias GEMM enabled (CuteDSL).")
return gemm_interface
@@ -109,6 +148,9 @@ def _scales_valid(scale_a: torch.Tensor, scale_b: torch.Tensor) -> bool:
make ``alpha = scale_a * scale_b`` overflow to ``+inf`` and return an all-inf tile.
Cache only positive results: a buffer still holding the sentinel is re-checked and
picks up the fast path once the real scale is written.
Called inside the dispatch custom op, so pointer checks and cache updates
neither specialize Dynamo graphs on each layer's addresses nor break them.
"""
key = (scale_a.data_ptr(), scale_b.data_ptr())
if key in _valid_scale_ptrs:
@@ -121,6 +163,40 @@ def _scales_valid(scale_a: torch.Tensor, scale_b: torch.Tensor) -> bool:
return ok
@torch.library.custom_op("vllm_omni::quack_fp8_scaled_mm", mutates_args=())
def _quack_fp8_scaled_mm(
a: torch.Tensor,
b: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: torch.dtype,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Keep scale validation and runtime fallback inside one opaque graph node."""
if scale_a.numel() == 1 and scale_b.numel() == 1 and _scales_valid(scale_a, scale_b):
try:
out = quack_scaled_fp8_mm(a, b, scale_a, scale_b, out_dtype, bias)
if out is not None:
return out
except Exception as exc: # noqa: BLE001
logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)
# Use the same fallback as FlashInferFP8ScaledMMLinearKernel, without
# passing its Python instance through the custom-op schema. Unpopulated
# scales also take this path: FlashInfer tolerates the profiling sentinel.
from vllm.utils.flashinfer import flashinfer_scaled_fp8_mm
out = flashinfer_scaled_fp8_mm(a, b, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b, bias=bias)
# FlashInfer's separate bias add may promote the dtype. Both branches must
# match the fake kernel's output metadata, including with an FP32 bias.
return out.to(out_dtype)
@_quack_fp8_scaled_mm.register_fake
def _quack_fp8_scaled_mm_fake(a, b, scale_a, scale_b, out_dtype, bias=None):
return torch.empty(a.shape[0], b.shape[1], device=a.device, dtype=out_dtype)
def install_quack_fp8_patch() -> None:
if not quack_enabled():
return
@@ -138,34 +214,26 @@ def install_quack_fp8_patch() -> None:
return
def apply_scaled_mm(self, *, A, B, out_dtype, As, Bs, bias, output_shape): # noqa: N803
# An unpopulated scale makes alpha overflow to +inf; hand those calls to
# FlashInfer, whose bmm_fp8 tolerates them.
if As.numel() != 1 or Bs.numel() != 1 or not _scales_valid(As, Bs):
return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape)
try:
out = quack_scaled_fp8_mm(A, B, As, Bs, out_dtype, bias)
if out is not None:
return out.view(*output_shape)
except Exception as exc: # noqa: BLE001
logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)
return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape)
return _quack_fp8_scaled_mm(A, B, As, Bs, out_dtype, bias).view(*output_shape)
apply_scaled_mm._omni_quack_patched = True
FlashInferFP8ScaledMMLinearKernel.apply_scaled_mm = apply_scaled_mm
logger.info("Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.")
@torch.inference_mode()
def warmup_quack_fp8(
shapes: list[tuple[int, int, int]],
device: str = "cuda",
out_dtype: torch.dtype = torch.bfloat16,
) -> None:
"""Warm no-bias GEMMs with the transposed weight layout used by vLLM."""
if _load_quack() is None:
return
scale = torch.ones(1, device=device, dtype=torch.float32)
for m, k, n in shapes:
a = torch.zeros(m, k, device=device, dtype=torch.float8_e4m3fn)
b = torch.zeros(k, n, device=device, dtype=torch.float8_e4m3fn)
b = torch.zeros(n, k, device=device, dtype=torch.float8_e4m3fn).t()
quack_scaled_fp8_mm(a, b, scale, scale, out_dtype)
if torch.cuda.is_available():
torch.accelerator.synchronize()