support online FP8 quantization for FA on NPU #2236 (#2640)

Signed-off-by: lyj-jjj <liuyingjun5@huawei.com>
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-05-13 11:46:00 +08:00
1 parent 83bbe39d39
commit 56ca7dd612
20 files changed
+823 -25

No files matched your search

+64
View File
@@ -32,6 +32,54 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def parse_kv_cache_skip_selector(
selector: str | list[int] | tuple[int, ...] | set[int] | None,
) -> set[int] | None:
"""Parse a non-negative index selector such as "0-9,20,25-30"."""
if selector is None:
return None
if isinstance(selector, set):
values = selector
elif isinstance(selector, (list, tuple)):
values = set(selector)
elif isinstance(selector, str):
text = selector.strip()
if not text:
return None
values: set[int] = set()
for chunk in text.split(","):
token = chunk.strip()
if not token:
continue
if "-" in token:
start_str, end_str = token.split("-", 1)
try:
start = int(start_str.strip())
end = int(end_str.strip())
except ValueError as exc:
raise ValueError(f"Invalid range token '{token}' in selector '{selector}'.") from exc
if start < 0 or end < 0 or start > end:
raise ValueError(f"Invalid range token '{token}' in selector '{selector}'.")
values.update(range(start, end + 1))
else:
try:
index = int(token)
except ValueError as exc:
raise ValueError(f"Invalid index token '{token}' in selector '{selector}'.") from exc
if index < 0:
raise ValueError(f"Negative index '{index}' is not allowed in selector '{selector}'.")
values.add(index)
else:
raise TypeError(f"Unsupported selector type: {type(selector)!r}")
for idx in values:
if not isinstance(idx, int):
raise TypeError(f"Selector index must be int, got {type(idx)!r}")
if idx < 0:
raise ValueError("Selector indices must be non-negative.")
return values
@config
@dataclass
class DiffusionParallelConfig:
@@ -519,6 +567,19 @@ class OmniDiffusionConfig:
# has already resolved to vLLM's ModelOpt FP8 linear method.
force_cutlass_fp8: bool = False
# KV cache dtype for attention. Aligned with upstream vLLM's --kv-cache-dtype.
# None = native dtype (no quantization).
# "fp8" = dynamic FP8 (float8_e4m3fn) quantization per forward pass.
# On Hopper+FA3: native FP8 attention (memory + compute savings).
# On other backends: no benefit, backends skip quantization.
kv_cache_dtype: str | None = None
# Optional skip selectors for KV-cache quantization. Format: "0-9,20,25-30".
# Listed steps/layers skip quantization; others keep quantized execution.
kv_cache_skip_steps: str | None = None
kv_cache_skip_layers: str | None = None
kv_cache_skip_step_indices: set[int] | None = None
kv_cache_skip_layer_indices: set[int] | None = None
# Diffusion pipeline Profiling config
enable_diffusion_pipeline_profiler: bool = False
@@ -527,6 +588,7 @@ class OmniDiffusionConfig:
# sleep mode
enable_sleep_mode: bool = False
# Maximum number of sequences to generate in a batch
max_num_seqs: int = 1
@@ -656,6 +718,8 @@ class OmniDiffusionConfig:
# Match vLLM's config flow: parse entrypoint shorthands before the
# config object is built, and keep a single runtime truth source.
self.diffusion_attention_config = build_attention_config(self.diffusion_attention_config)
self.kv_cache_skip_step_indices = parse_kv_cache_skip_selector(self.kv_cache_skip_steps)
self.kv_cache_skip_layer_indices = parse_kv_cache_skip_selector(self.kv_cache_skip_layers)
if self.max_cpu_loras is None:
self.max_cpu_loras = 1