[Quantization][Qwen3-Omni] Enable NVFP4 W4A4 serving on Blackwell (#4025)

Signed-off-by: Yihong Jin <yihongjin404@gmail.com>
This commit is contained in:
Yihong Jin authored and GitHub committed 2026-06-18 10:01:46 +08:00
1 parent 65bc737fd4
commit 2f64c712fd
4 files changed
+605

No files matched your search

+183
View File
@@ -1,7 +1,9 @@
import logging
import os
import sys
from functools import cached_property
import torch
from aenum import extend_enum
from vllm.config import ModelConfig as _OriginalModelConfig
from vllm.inputs import TokensPrompt as _OriginalTokensPrompt
@@ -21,6 +23,8 @@ from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.layers.rotary_embedding import OmniMRotaryEmbedding
from vllm_omni.request import OmniRequest, OmniStreamingUpdate
_PATCH_LOGGER = logging.getLogger("vllm_omni.patch")
# =============================================================================
# Patch ModelConfig.is_mm_prefix_lm to support omni-specific models
# =============================================================================
@@ -71,6 +75,185 @@ assert _installed is _patched_cp, (
"for HunyuanImage3 will not work. Check vLLM ModelConfig changes."
)
# =============================================================================
# Patch ModelOptNvFp4LinearMethod.process_weights_after_loading to clamp NaN
# bytes in per-block FP8 weight_scale tensors at load time.
# =============================================================================
# WHY: ModelOpt 0.44's float32 -> torch.float8_e4m3fn cast of per-block weight
# scales occasionally emits NaN bytes (E4M3 encoding 0x7F / 0xFF) when the
# pre-cast scale rounds above the FP8 E4M3 max of 448 after the global-scale
# division. Any single NaN byte in weight_scale propagates through the
# FlashInfer FP4 GEMM and the served model collapses to `!!!!`. NVFP4 W4A4
# Qwen3-Omni checkpoints published by the community — exported with stock
# ModelOpt 0.44 — currently fail this way the moment they're served.
#
# WHY here (vllm-omni, not upstream): the root cause sits in ModelOpt; the
# analogous fix in vLLM's own PWAL is also pending. Both are planned as
# follow-up PRs. Until those land in vllm-omni's pinned vLLM, this load-time
# clamp keeps the user-visible failure from looking like a vllm-omni
# inference bug. Newly calibrated, clean checkpoints pay no runtime cost
# (the clamp is a no-op when no NaN bytes are present).
#
# SCOPE: ModelOptNvFp4LinearMethod (W4A4 NVFP4 Linear) only. NvFp4FusedMoE /
# NvFp4W4A16 / CompressedTensors / Quark NVFP4 paths are not covered.
#
# SELF-EXTINGUISH: `_already_patched_upstream` heuristically detects when
# vLLM's own PWAL contains an in-place `masked_fill_` against `weight_scale`
# / `isnan` — the structure the upstream fix is expected to take when it is
# filed (planned as a follow-up PR after this one merges). Once vllm-omni's
# vllm pin moves to a release with that upstream fix, the override is
# skipped at import and this block can be deleted. NOTE: the heuristic only
# matches "vLLM PWAL clamps NaN in-place with `masked_fill_`"; if the
# upstream fix lands as `nan_to_num_` or as a clamp before the FP32→FP8
# cast, the check won't fire and this override stays active. The override
# is idempotent so the overlap is a warning log, not a correctness issue —
# but the heuristic should be revisited when the upstream PR is filed.
#
# ORDERING: the clamp must run BEFORE the original PWAL. The non-Blackwell
# Marlin fallback (sm_<100) casts weight_scale FP8 -> bf16/fp16 and permutes
# inside its kernel PWAL; an after-PWAL clamp would either trip the
# byte-view shape assertion (FP8 1B/elem -> bf16 2B/elem) or operate on
# already-transformed bytes where NaN has propagated through permute. On
# Blackwell (Cutlass/TRTLLM/cuDNN) the kernel PWAL only swizzles/shuffles
# FP8 bytes, so clamp-before is equivalent to clamp-after there.
def _clamp_nvfp4_weight_scale_nans(layer) -> int:
"""Scan ``layer.weight_scale`` for NaN bytes and clamp them to the FP8
E4M3 max byte (0x7E) in place. Returns the number of bytes clamped.
Scoped to NVFP4 ``float8_e4m3fn`` weight_scale only — a future MXFP4
weight-scale clamp would need its own helper (MXFP4 uses
``float8_e8m0fnu`` for scales, and the byte encoding of NaN differs).
Exposed at module scope so tests can exercise the clamp directly
without monkey-patching + reloading the parent ``vllm_omni.patch``
module.
"""
# Defensive: a stray subclass without weight_scale would crash inside
# the override; just bail out.
weight_scale = getattr(layer, "weight_scale", None)
if weight_scale is None:
return 0
# Re-entry safety: this clamp expects raw on-disk FP8 weight_scale (1
# byte/elem). If we are ever invoked after the original PWAL ran (e.g.
# a caller re-issues process_weights_after_loading on the same layer),
# weight_scale may have been cast to bf16/fp16 by the Marlin path, in
# which case the uint8 byte view below would have a different shape
# than the NaN mask. Bail out cleanly rather than trip the assert.
if weight_scale.dtype != torch.float8_e4m3fn:
return 0
# `.view(torch.uint8)` below requires a contiguous tensor and raises on a
# non-contiguous one. A checkpoint whose per-block scale tensor isn't
# contiguous would otherwise crash here. Materialize a contiguous copy and
# write it back to the Parameter so the in-place byte clamp lands on the
# tensor the kernel will actually read (a bare `.contiguous()` copy would
# be discarded and the original left un-clamped).
weight_scale_data = weight_scale.data
if not weight_scale_data.is_contiguous():
weight_scale_data = weight_scale_data.contiguous()
weight_scale.data = weight_scale_data
nan_mask = torch.isnan(weight_scale_data)
if not nan_mask.any():
return 0
# PyTorch does not implement masked_fill_ for float8_e4m3fn, so view
# the storage as uint8 and write the FP8 max byte (0x7E) directly.
# float8_e4m3fn is 1 byte per element so the byte view is shape-
# for-shape with the original tensor; pin that here so a future
# packed scale format would fail loudly rather than silently corrupt
# unrelated bytes.
byte_view = weight_scale_data.view(torch.uint8)
if byte_view.shape != nan_mask.shape:
raise RuntimeError(
f"NVFP4 weight_scale uint8 view shape {byte_view.shape} != NaN mask shape "
f"{nan_mask.shape}; per-block scale layout changed — recheck the clamp."
)
fp8_max_byte = (
torch.tensor(torch.finfo(torch.float8_e4m3fn).max, dtype=torch.float8_e4m3fn).view(torch.uint8).item()
)
byte_view.masked_fill_(nan_mask, fp8_max_byte)
n = int(nan_mask.sum().item())
_PATCH_LOGGER.warning("Clamped %d NaN entries in NVFP4 per-block weight_scale.", n)
return n
# Module-level defaults so downstream code (and tests) can import these names
# without guarding for the import-failure / escape-hatch branches below.
# `_already_patched_upstream` = upstream PWAL contains its own NaN clamp.
# `_clamp_installed` = our wrapper was installed on the upstream class.
# These are independent: the env-var escape hatch and the import-failure path
# both leave the wrapper uninstalled WITHOUT upstream being patched, so the
# right check for "we own NaN-clamp behavior" is `_clamp_installed`.
_already_patched_upstream = False
_clamp_installed = False
try:
# Escape hatch — set VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP=1 to skip installing
# the patch (e.g. to confirm a `!!!!` failure is the NaN-byte case).
# The escape-hatch deliberately raises ImportError so the not-installed
# warning below logs through the same path a real ImportError would.
# Use the repo-wide bool-env idiom so values like `0`, `false`, `no`,
# `off` correctly mean "do not skip" rather than tripping naive
# truthiness on the non-empty string.
if os.environ.get("VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP", "").lower() in ("1", "true", "yes", "on"):
raise ImportError("VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP is set; skipping NaN-clamp install")
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4LinearMethod as _OriginalModelOptNvFp4LinearMethod,
)
except ImportError as _nan_clamp_import_err:
_PATCH_LOGGER.warning(
"NVFP4 weight_scale NaN-clamp patch could NOT install: %s. NVFP4 W4A4 "
"checkpoints with NaN bytes in per-block weight_scale will serve `!!!!`.",
_nan_clamp_import_err,
)
else:
_current_nvfp4_pwal = _OriginalModelOptNvFp4LinearMethod.process_weights_after_loading
# Reload idempotency: on a module reload (importlib.reload in a test, or a
# second import path) the class attribute already holds OUR wrapper, so
# capturing it as the "original" and re-wrapping would nest the clamp and
# run it twice per load. Recover the genuine upstream method from the
# sentinel we stamp below; the heuristic + a fresh single-level wrapper are
# then computed against the real upstream, not against our own wrapper.
_original_nvfp4_pwal = getattr(_current_nvfp4_pwal, "_vllm_omni_wrapped_pwal", _current_nvfp4_pwal)
_upstream_pwal_names = set(_original_nvfp4_pwal.__code__.co_names or ())
# Require ALL three names — `weight_scale` alone is too loose because the
# current upstream PWAL already references `weight_scale_2` (close prefix
# collisions aside, future PWALs may legitimately reference `weight_scale`
# without a NaN clamp). `masked_fill_` + `isnan` + `weight_scale` together
# are the structural signature of an in-place NaN clamp on weight_scale.
# KNOWN BLIND SPOTS (false-negatives, safe direction — we install a
# redundant but idempotent clamp): upstream using
# `getattr(layer, "weight_scale")` (string lands in co_consts, not
# co_names), or factoring the clamp into a helper called from PWAL
# (none of the three names appear in the top-level co_names). Revisit
# this heuristic when the upstream PR is actually filed.
_already_patched_upstream = all(n in _upstream_pwal_names for n in ("masked_fill_", "isnan", "weight_scale"))
def _patched_nvfp4_pwal(self, layer, *args, **kwargs):
# Clamp BEFORE the original PWAL — see ORDERING note above.
_clamp_nvfp4_weight_scale_nans(layer)
_original_nvfp4_pwal(self, layer, *args, **kwargs)
# Sentinel so a later reload of this module recovers the genuine upstream
# method above instead of wrapping this wrapper (see reload-idempotency note).
_patched_nvfp4_pwal._vllm_omni_wrapped_pwal = _original_nvfp4_pwal
if not _already_patched_upstream:
_OriginalModelOptNvFp4LinearMethod.process_weights_after_loading = _patched_nvfp4_pwal
# Fail loudly if install dropped silently (e.g. another plugin
# patched the same class) rather than degrade to `!!!!` at decode.
# Use raise, not assert: asserts are compiled out under `python -O` /
# PYTHONOPTIMIZE, which would silently disable this guard in exactly
# the optimized runs where a conflicting plugin must still be caught.
if _OriginalModelOptNvFp4LinearMethod.process_weights_after_loading is not _patched_nvfp4_pwal:
raise RuntimeError("NVFP4 weight_scale NaN-clamp install failed — check for conflicting plugins.")
_clamp_installed = True
_PATCH_LOGGER.info(
"NVFP4 W4A4 weight_scale NaN-clamp: %s.",
"skipped (upstream already patched)" if _already_patched_upstream else "installed",
)
# =============================================================================
# Patch GlmImageTextConfig to expose mrope_section in rope_parameters
# =============================================================================