Add normalized online FP8 cache for DLO

Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
Hongsheng Liu committed 2026-08-19 11:55:29 +08:00
1 parent 14bd8f8772
commit 1ae56cfcfb
14 files changed
+714 -11

No files matched your search

@@ -41,6 +41,8 @@ class _MiniMaxH3DirectMmapAdapter:
target: torch.Tensor,
) -> DirectMmapTensorPolicy:
del target
if getattr(self.pipeline, "_dlo_fp8_cache_normalized", False):
return DirectMmapTensorPolicy(allow_custom_loader=True)
transform = None
suffix = ".qkv_proj.weight"
if runtime_name.endswith(suffix):
@@ -8,7 +8,7 @@ import re
import time
from collections.abc import Generator, Iterable, Sequence
from pathlib import Path
from typing import cast
from typing import Any, cast
import torch
from torch import nn
@@ -34,6 +34,12 @@ from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig, apply_hsdp
from vllm_omni.diffusion.model_loader.checkpoint_adapters import (
get_checkpoint_adapter,
)
from vllm_omni.diffusion.model_loader.fp8_cache import (
_transformer_quant_config,
build_normalized_fp8_cache,
is_online_fp8_config,
load_fp8_cache_manifest,
)
from vllm_omni.diffusion.model_loader.host_weight_plan import (
HostWeightPlan,
build_checkpoint_mmap_plan,
@@ -141,6 +147,10 @@ class DiffusersPipelineLoader:
self.quant_config = od_config.quantization_config
self.parallel_config = od_config.parallel_config
self.host_weight_plan: HostWeightPlan | None = None
self._fp8_cache_dir: Path | None = None
self._fp8_cache_requested = False
self._fp8_cache_hit = False
self._fp8_cache_manifest: dict[str, Any] | None = None
def take_host_weight_plan(self) -> HostWeightPlan | None:
"""Transfer the loader-produced plan to the offload backend."""
@@ -367,12 +377,61 @@ class DiffusersPipelineLoader:
)
def _get_weight_sources(self, model: nn.Module) -> tuple["ComponentSource", ...]:
return tuple(
sources = tuple(
cast(
Iterable[DiffusersPipelineLoader.ComponentSource],
getattr(model, "weights_sources", ()),
)
)
if not self._fp8_cache_hit or self._fp8_cache_dir is None or self._fp8_cache_manifest is None:
return sources
cache_prefix = str(self._fp8_cache_manifest["source_prefix"])
cache_subfolder = str(self._fp8_cache_manifest.get("component_subfolder", "transformer"))
return tuple(
dataclasses.replace(
source,
model_or_path=str(self._fp8_cache_dir),
subfolder=cache_subfolder,
revision=None,
)
if source.prefix == cache_prefix
else source
for source in sources
)
def _prepare_fp8_cache(self) -> None:
"""Select the Phase-I cache path before constructing the pipeline."""
cache_dir = getattr(self.od_config, "dlo_fp8_cache_dir", None)
if not cache_dir or not is_online_fp8_config(self.quant_config):
return
if not getattr(self.od_config, "enable_distributed_layerwise_offload", False):
logger.warning("Ignoring --dlo-fp8-cache-dir because distributed layerwise offload is disabled")
return
if not getattr(self.od_config, "dlo_use_allgather", True):
logger.warning("Ignoring --dlo-fp8-cache-dir because DLO AllGather is disabled")
return
if int(getattr(self.parallel_config, "tensor_parallel_size", 1)) != 1:
logger.warning("Ignoring --dlo-fp8-cache-dir because Phase-I cache requires TP=1")
return
if bool(getattr(self.parallel_config, "use_hsdp", False)):
logger.warning("Ignoring --dlo-fp8-cache-dir because Phase-I cache does not support HSDP")
return
self._fp8_cache_dir = Path(cache_dir)
self._fp8_cache_requested = True
self._fp8_cache_manifest = load_fp8_cache_manifest(self._fp8_cache_dir)
if self._fp8_cache_manifest is None:
logger.info("DLO normalized FP8 cache miss: %s", self._fp8_cache_dir)
return
self._fp8_cache_hit = True
self.quant_config = _transformer_quant_config()
self.od_config.quantization_config = self.quant_config
logger.info(
"DLO normalized FP8 cache hit: %s; selecting serialized FP8 checkpoint path",
self._fp8_cache_dir,
)
def _get_expected_parameter_names(self, model: nn.Module) -> set[str]:
"""Return parameter names that should be covered by strict load checks."""
@@ -403,6 +462,7 @@ class DiffusersPipelineLoader:
self.host_weight_plan = None
if load_format is None:
load_format = "default"
self._prepare_fp8_cache()
# CPU offload + quantization: for offline-quantized models (e.g., AutoRound MXFP8),
# weights are already quantized in the checkpoint — load directly on CPU.
# For online quantization, load on device so quantization can run on accelerator,
@@ -413,7 +473,9 @@ class DiffusersPipelineLoader:
is_offline = getattr(quant_cfg, "data_type", None) == "mx_fp" or getattr(
quant_cfg, "is_checkpoint_quantized", False
)
if not is_offline:
if self._fp8_cache_hit:
logger.info("Normalized FP8 cache is already serialized; keeping model initialization on CPU")
elif not is_offline:
load_device = device.type
offload_after_quant = True
logger.info(
@@ -431,6 +493,11 @@ class DiffusersPipelineLoader:
)
else:
model = self._init_from_load_format(load_format, target_device, custom_pipeline_name, is_hsdp=False)
if self._fp8_cache_hit:
# The direct-mmap adapter must see this before plan
# construction so normalized QKV tensors bypass the raw
# checkpoint grouped-QKV transform.
setattr(model, "_dlo_fp8_cache_normalized", True)
_dist_offload = getattr(self.od_config, "enable_distributed_layerwise_offload", False)
_use_ag = getattr(self.od_config, "dlo_use_allgather", True)
@@ -520,6 +587,20 @@ class DiffusersPipelineLoader:
model.to("cpu")
logger.info("Quantization complete, offloaded model back to CPU")
if self._fp8_cache_requested and self._fp8_cache_dir is not None and not self._fp8_cache_hit:
try:
build_normalized_fp8_cache(
model,
sources=self._get_weight_sources(model),
cache_dir=self._fp8_cache_dir,
)
except Exception as exc:
logger.warning(
"DLO normalized FP8 cache build failed; retaining ordinary-loader path: %s",
exc,
exc_info=True,
)
self._apply_skip_softmax_calibration(model)
return model.eval()
@@ -0,0 +1,349 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Phase-I normalized FP8 cache support for DLO checkpoint mmap."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import uuid
from pathlib import Path
from typing import Any
import torch
from safetensors import safe_open
from safetensors.torch import save_file
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.model_loader.host_weight_plan import (
_build_source_map,
_collect_required_dit_tensors,
)
from vllm_omni.quantization.component_config import ComponentQuantizationConfig
logger = init_logger(__name__)
CACHE_SCHEMA_VERSION = 1
CACHE_KIND = "normalized_fp8_runtime"
CACHE_MANIFEST_NAME = "manifest.json"
CACHE_COMPLETE_NAME = "COMPLETE"
CACHE_COMPONENT_SUBFOLDER = "transformer"
def _transformer_quant_config() -> ComponentQuantizationConfig:
"""Build the serialized FP8 config used by a normalized cache."""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
return ComponentQuantizationConfig(
{
"transformer": Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
)
}
)
def is_online_fp8_config(quant_config: Any) -> bool:
"""Return whether the transformer component requests online FP8."""
if quant_config is None:
return False
if hasattr(quant_config, "resolve"):
quant_config = quant_config.resolve("transformer")
get_name = getattr(quant_config, "get_name", None)
if not callable(get_name) or str(get_name()).lower() != "fp8":
return False
return not bool(getattr(quant_config, "is_checkpoint_fp8_serialized", False))
def _source_fingerprint(files: set[str]) -> str:
entries: list[dict[str, int | str]] = []
for filename in sorted(files):
stat = os.stat(filename)
entries.append(
{
"path": os.path.abspath(filename),
"size": stat.st_size,
"mtime_ns": stat.st_mtime_ns,
}
)
payload = json.dumps(entries, sort_keys=True, separators=(",", ":")).encode()
return hashlib.sha256(payload).hexdigest()
def load_fp8_cache_manifest(cache_dir: str | os.PathLike[str] | None) -> dict[str, Any] | None:
"""Return a validated Phase-I manifest, or ``None`` for a cache miss."""
if cache_dir is None:
return None
root = Path(cache_dir)
manifest_path = root / CACHE_MANIFEST_NAME
if not (root / CACHE_COMPLETE_NAME).is_file() or not manifest_path.is_file():
return None
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
if not isinstance(manifest, dict):
return None
if manifest.get("schema_version") != CACHE_SCHEMA_VERSION:
return None
if manifest.get("cache_kind") != CACHE_KIND:
return None
if manifest.get("component") != CACHE_COMPONENT_SUBFOLDER:
return None
quantization = manifest.get("quantization")
if not isinstance(quantization, dict) or quantization != {
"method": "fp8",
"serialized": True,
"activation_scheme": "dynamic",
}:
return None
if manifest.get("component_subfolder") != CACHE_COMPONENT_SUBFOLDER:
return None
if manifest.get("source_prefix") != f"{CACHE_COMPONENT_SUBFOLDER}.":
return None
index_path = root / CACHE_COMPONENT_SUBFOLDER / "model.safetensors.index.json"
if not index_path.is_file():
return None
try:
index = json.loads(index_path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
if not isinstance(index, dict):
return None
weight_map = index.get("weight_map")
if not isinstance(weight_map, dict) or not weight_map or not all(isinstance(key, str) for key in weight_map):
return None
if manifest.get("tensor_count") != len(weight_map):
return None
for filename in weight_map.values():
if not isinstance(filename, str):
return None
relative_path = Path(filename)
if (
relative_path.is_absolute()
or relative_path.name != filename
or relative_path.suffix != ".safetensors"
or not (root / CACHE_COMPONENT_SUBFOLDER / relative_path).is_file()
):
return None
source_files = manifest.get("source_files")
source_fingerprint = manifest.get("source_fingerprint")
if not isinstance(source_files, list) or not source_files or not isinstance(source_fingerprint, str):
return None
try:
if _source_fingerprint({str(filename) for filename in source_files}) != source_fingerprint:
return None
except (OSError, TypeError):
return None
return manifest
def _cache_key_for_runtime_name(runtime_name: str, prefix: str) -> str:
if not runtime_name.startswith(prefix):
raise ValueError(f"runtime tensor {runtime_name!r} is outside cache source prefix {prefix!r}")
return runtime_name[len(prefix) :]
def build_normalized_fp8_cache(
model: nn.Module,
*,
sources: tuple[object, ...],
cache_dir: str | os.PathLike[str],
) -> dict[str, Any]:
"""Write a normalized FP8 cache from a completed online-FP8 model.
The builder writes tensors one at a time into individual safetensors shards,
so it does not create a second full-model state dict. The first cache build
still follows the existing online loader; warm launches consume the cache
through the direct DLO mmap path.
"""
cache_root = Path(cache_dir)
existing_manifest = load_fp8_cache_manifest(cache_root)
if existing_manifest is not None:
return existing_manifest
if cache_root.exists() and any(cache_root.iterdir()):
raise FileExistsError(f"FP8 cache directory is not empty and is not valid: {cache_root}")
cache_root.parent.mkdir(parents=True, exist_ok=True)
dit_names = tuple(getattr(model, "_dit_modules", ()))
dit_names = tuple(name for name in dit_names if hasattr(model, name))
if len(dit_names) != 1:
raise ValueError(f"Phase-I FP8 cache requires exactly one DiT module, got {dit_names}")
dit_name = dit_names[0]
if dit_name != CACHE_COMPONENT_SUBFOLDER:
raise ValueError(
f"Phase-I FP8 cache requires a top-level {CACHE_COMPONENT_SUBFOLDER!r} module, got {dit_name!r}"
)
source_prefix = f"{dit_name}."
dit_module = model.get_submodule(dit_name)
planned_sources = tuple(source for source in sources if getattr(source, "prefix", "") == source_prefix)
if len(planned_sources) != 1:
raise ValueError(f"Phase-I FP8 cache requires one source for {source_prefix!r}, got {len(planned_sources)}")
source_map = _build_source_map(
sources=planned_sources,
model_path=None,
remap_fn=getattr(type(model), "_remap_ckpt_key", None),
)
required = _collect_required_dit_tensors(((dit_name, dit_module),))
quantized_modules: dict[str, nn.Module] = {}
try:
from vllm.model_executor.layers.quantization.online.fp8 import Fp8PerTensorOnlineLinearMethod
except ImportError as exc: # pragma: no cover - vLLM 0.27 provides this class
raise RuntimeError("Phase-I FP8 cache requires vLLM's online FP8 implementation") from exc
for module_name, module in model.named_modules():
if not module_name.startswith(source_prefix):
continue
if isinstance(getattr(module, "quant_method", None), Fp8PerTensorOnlineLinearMethod):
if not hasattr(module, "weight_scale"):
raise ValueError(f"online FP8 module {module_name!r} has no generated weight_scale")
quantized_modules[module_name] = module
tmp_root = cache_root.parent / f".{cache_root.name}.tmp-{uuid.uuid4().hex}"
component_root = tmp_root / CACHE_COMPONENT_SUBFOLDER
component_root.mkdir(parents=True, exist_ok=False)
weight_map: dict[str, str] = {}
source_files: set[str] = set()
total_size = 0
part_index = 0
file_cache: dict[str, Any] = {}
def write_tensor(key: str, tensor: torch.Tensor) -> None:
nonlocal part_index, total_size
if key in weight_map:
raise ValueError(f"normalized FP8 cache has duplicate tensor key {key!r}")
tensor = tensor.detach().to("cpu").contiguous()
filename = f"part-{part_index:05d}.safetensors"
part_index += 1
save_file({key: tensor}, str(component_root / filename))
weight_map[key] = filename
total_size += tensor.numel() * tensor.element_size()
try:
for runtime_name, target in required.items():
binding_info = source_map.get(runtime_name)
module_name = runtime_name.removesuffix(".weight")
quant_module = quantized_modules.get(module_name) if runtime_name.endswith(".weight") else None
if runtime_name.endswith(".weight_scale"):
owner_name = runtime_name.removesuffix(".weight_scale")
if owner_name in quantized_modules:
# Generated online scales are published in the dedicated
# pass below; they have no raw checkpoint binding.
continue
if quant_module is not None:
if binding_info is None:
raise ValueError(f"no source binding for quantized weight {runtime_name!r}")
qweight = quant_module.weight.detach()
if qweight.ndim != 2:
raise ValueError(f"Phase-I FP8 cache only supports 2D weights, got {runtime_name!r}")
# Online FP8 stores the final runtime weight as qweight.T. The
# serialized FP8 method expects the pre-process qweight shape.
qweight = qweight.t().contiguous()
# Normalized caches contain the post-checkpoint-adapter QKV
# order. The cache-hit adapter bypasses the raw-checkpoint
# grouped-QKV transform, then the serialized FP8 finalizer
# performs only the standard weight transpose.
write_tensor(binding_info[0], qweight)
source_files.add(binding_info[1])
continue
if binding_info is not None:
checkpoint_key, file_path = binding_info
source_files.add(file_path)
handle = file_cache.get(file_path)
if handle is None:
handle = safe_open(file_path, framework="pt", device="cpu")
file_cache[file_path] = handle
write_tensor(checkpoint_key, handle.get_tensor(checkpoint_key))
else:
# Constructor-derived persistent buffers have no checkpoint
# entry; preserve them as canonical cache tensors.
write_tensor(_cache_key_for_runtime_name(runtime_name, source_prefix), target)
for module_name, module in quantized_modules.items():
scale = module.weight_scale.detach()
logical_widths = getattr(module, "logical_widths", None)
if scale.numel() == 1 and logical_widths is not None and len(logical_widths) > 1:
# Serialized Fp8LinearMethod represents a fused per-tensor
# weight with one scale slot per logical shard. Online FP8
# produces one scalar for the fused tensor, so replicate that
# scalar to the offline runtime shape without changing its
# value.
scale = scale.reshape(1).expand(len(logical_widths)).contiguous()
write_tensor(
_cache_key_for_runtime_name(f"{module_name}.weight_scale", source_prefix),
scale,
)
index = {
"metadata": {"total_size": total_size},
"weight_map": dict(sorted(weight_map.items())),
}
(component_root / "model.safetensors.index.json").write_text(
json.dumps(index, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
manifest = {
"schema_version": CACHE_SCHEMA_VERSION,
"cache_kind": CACHE_KIND,
"component": "transformer",
"component_subfolder": CACHE_COMPONENT_SUBFOLDER,
"source_prefix": source_prefix,
"quantization": {
"method": "fp8",
"serialized": True,
"activation_scheme": "dynamic",
},
"tensor_count": len(weight_map),
"source_fingerprint": _source_fingerprint(source_files),
"source_files": sorted(source_files),
}
(tmp_root / CACHE_MANIFEST_NAME).write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
(tmp_root / CACHE_COMPLETE_NAME).write_text("phase1\n", encoding="utf-8")
for handle in file_cache.values():
close = getattr(handle, "__exit__", None)
if callable(close):
close(None, None, None)
try:
os.replace(tmp_root, cache_root)
except FileExistsError:
# Multiple DLO workers may build the same cache concurrently. If
# another worker published a complete artifact first, treat this
# publisher as a successful cache race rather than a build error.
published = load_fp8_cache_manifest(cache_root)
if published is None:
raise
shutil.rmtree(tmp_root, ignore_errors=True)
logger.info("Normalized FP8 cache was published concurrently at %s", cache_root)
return published
logger.info("Published normalized FP8 cache at %s (%d tensors)", cache_root, len(weight_map))
return manifest
except BaseException:
for handle in file_cache.values():
close = getattr(handle, "__exit__", None)
if callable(close):
close(None, None, None)
shutil.rmtree(tmp_root, ignore_errors=True)
raise
__all__ = [
"CACHE_SCHEMA_VERSION",
"CACHE_KIND",
"CACHE_COMPONENT_SUBFOLDER",
"build_normalized_fp8_cache",
"is_online_fp8_config",
"load_fp8_cache_manifest",
"_transformer_quant_config",
]