[Diffusion] Rename runtime cache to host weight cache

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu committed 2026-08-19 11:23:36 +08:00
1 parent ee665fcfc3
commit 3bb7547a0f
20 files changed
+317 -302

No files matched your search

@@ -441,16 +441,16 @@ class DiffusersPipelineLoader:
_dp_size = int(getattr(self.parallel_config, "data_parallel_size", 1))
_sp_size = int(getattr(self.parallel_config, "sequence_parallel_size", 1))
_dlo_group_size = _dp_size if _dp_size > 1 else _sp_size
_prefer_registered_runtime_cache = (
_prefer_registered_host_weight_cache = (
_dist_offload
and not _use_ag
and bool(getattr(self.od_config, "dlo_enable_runtime_cache", False))
and float(getattr(self.od_config, "dlo_runtime_cache_pin_limit_gib", 0.0)) > 0
and bool(getattr(self.od_config, "dlo_enable_host_weight_cache", False))
and float(getattr(self.od_config, "dlo_host_weight_cache_pin_limit_gib", 0.0)) > 0
)
plan_result = None
weight_sources = self._get_weight_sources(model)
if _dist_offload and not _prefer_registered_runtime_cache:
if _dist_offload and not _prefer_registered_host_weight_cache:
modules = ModuleDiscovery.discover(model)
plan_result = build_checkpoint_mmap_plan(
model,
@@ -462,14 +462,14 @@ class DiffusersPipelineLoader:
online_quantization=_has_online_quant,
)
self.host_weight_plan = plan_result.plan
elif _prefer_registered_runtime_cache:
elif _prefer_registered_host_weight_cache:
# Direct checkpoint bindings can carry deferred transforms
# (for example grouped-QKV reordering), so registering the
# raw checkpoint mapping would not make every recurrent H2D
# source directly copyable. Materialize once, apply every
# post-load mutation, then publish/register the final layout.
logger.info(
"DLO runtime-cache registration requested; using the ordinary loader "
"DLO host weight cache registration requested; using the ordinary loader "
"once to publish a transform-complete final mmap layout"
)
@@ -540,19 +540,19 @@ class DiffusersPipelineLoader:
self._apply_skip_softmax_calibration(model)
model.eval()
self._maybe_build_runtime_weight_cache(model, load_format)
self._maybe_build_host_weight_cache(model, load_format)
return model
def _maybe_build_runtime_weight_cache(self, model: nn.Module, load_format: str) -> None:
def _maybe_build_host_weight_cache(self, model: nn.Module, load_format: str) -> None:
"""Publish final ordinary-loader DiT tensors for no-AllGather DLO."""
if self.host_weight_plan is not None:
return
if not getattr(self.od_config, "enable_distributed_layerwise_offload", False):
return
if not getattr(self.od_config, "dlo_enable_runtime_cache", False):
if not getattr(self.od_config, "dlo_enable_host_weight_cache", False):
return
if getattr(self.od_config, "dlo_use_allgather", True):
logger.info("DLO runtime cache is no-AllGather only; leaving AllGather storage unchanged")
logger.info("DLO host weight cache is no-AllGather only; leaving AllGather storage unchanged")
return
parallel_config = self.parallel_config
@@ -565,17 +565,18 @@ class DiffusersPipelineLoader:
tp_rank = int(get_tensor_model_parallel_rank())
except Exception as exc:
logger.warning(
"DLO runtime cache unavailable [parallel_rank_unavailable]: cannot resolve TP rank for TP=%d: %s",
"DLO host weight cache unavailable [parallel_rank_unavailable]: "
"cannot resolve TP rank for TP=%d: %s",
tp_size,
exc,
)
return
modules = ModuleDiscovery.discover(model)
from vllm_omni.diffusion.model_loader.runtime_weight_cache import (
from vllm_omni.diffusion.model_loader.host_weight_cache import (
DEFAULT_LOCK_TIMEOUT_SECONDS,
DEFAULT_SHARD_SIZE_BYTES,
build_runtime_weight_cache_plan,
build_host_weight_cache_plan,
)
loader_inputs = {
@@ -602,13 +603,13 @@ class DiffusersPipelineLoader:
for module in model.modules()
):
runtime_quantization = "module_quant_method"
result = build_runtime_weight_cache_plan(
result = build_host_weight_cache_plan(
model,
dit_modules=tuple(zip(modules.dit_names, modules.dits)),
loader_type=type(self),
cache_root=getattr(self.od_config, "dlo_runtime_cache_dir", None),
cache_root=getattr(self.od_config, "dlo_host_weight_cache_dir", None),
lock_timeout_seconds=float(
getattr(self.od_config, "dlo_runtime_cache_lock_timeout", DEFAULT_LOCK_TIMEOUT_SECONDS)
getattr(self.od_config, "dlo_host_weight_cache_lock_timeout", DEFAULT_LOCK_TIMEOUT_SECONDS)
),
max_shard_bytes=DEFAULT_SHARD_SIZE_BYTES,
model_identity=str(model_identity) if model_identity is not None else None,
@@ -628,7 +629,7 @@ class DiffusersPipelineLoader:
if result.plan is not None:
self.host_weight_plan = result.plan
logger.info(
"DLO runtime cache selected (layout=%s, TP=%d/%d, SP=%d)",
"DLO host weight cache selected (layout=%s, TP=%d/%d, SP=%d)",
result.plan.runtime_layout_key[:12] if result.plan.runtime_layout_key else "unknown",
tp_rank,
tp_size,
@@ -636,7 +637,7 @@ class DiffusersPipelineLoader:
)
else:
logger.warning(
"DLO runtime cache unavailable [%s]; retaining ordinary host tensors: %s",
"DLO host weight cache unavailable [%s]; retaining ordinary host tensors: %s",
result.fallback_code or "unknown",
result.fallback_reason or "unspecified reason",
)
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Build node-local runtime-weight cache plans for diffusion DLO.
"""Build node-local host weight cache plans for diffusion DLO.
``build_runtime_weight_cache_plan`` is the module's single entry point. It
``build_host_weight_cache_plan`` is the module's single entry point. It
validates final ordinary-loader DiT tensors, joins or atomically publishes an
immutable content-addressed entry, validates the mmap result, and returns a
fail-closed host-weight plan. Reusable identity, hashing, locking, and file
primitives live in :mod:`runtime_cache_utils`.
primitives live in :mod:`host_weight_cache_utils`.
"""
from __future__ import annotations
@@ -30,14 +30,9 @@ from safetensors.torch import save_file
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.model_loader.host_weight_plan import (
HostWeightPlan,
HostWeightPlanResult,
TensorBinding,
)
from vllm_omni.diffusion.model_loader.runtime_cache_utils import (
from vllm_omni.diffusion.model_loader.host_weight_cache_utils import (
HostWeightCacheError,
IdentityNormalizationError,
RuntimeCacheError,
RuntimeTensor,
canonical_json,
canonicalize_existing_local_path,
@@ -52,15 +47,20 @@ from vllm_omni.diffusion.model_loader.runtime_cache_utils import (
split_shards,
tensor_metadata,
)
from vllm_omni.diffusion.model_loader.host_weight_plan import (
HostWeightPlan,
HostWeightPlanResult,
TensorBinding,
)
logger = init_logger(__name__)
# Internal cache-format and I/O defaults, not environment variables. Public
# configuration may override the root and lock timeout; chunk sizes stay private.
# Bump the schema to isolate incompatible manifest layouts.
RUNTIME_CACHE_SCHEMA_VERSION = 1
HOST_WEIGHT_CACHE_SCHEMA_VERSION = 1
# Keep the default shared across local stage replicas.
DEFAULT_RUNTIME_CACHE_ROOT = "~/.cache/vllm-omni/dlo-runtime-weights"
DEFAULT_HOST_WEIGHT_CACHE_ROOT = "~/.cache/vllm-omni/dlo-host-weights"
# Bound waits for another publisher instead of hanging initialization forever.
DEFAULT_LOCK_TIMEOUT_SECONDS = 600.0
# Avoid one model-sized output file and bound hashing/read working sets.
@@ -83,9 +83,9 @@ for _dtype_name in ("float8_e4m3fn", "float8_e5m2"):
_SUPPORTED_DTYPES.add(_dtype)
def default_runtime_cache_root() -> str:
def default_host_weight_cache_root() -> str:
"""Return the default root, intentionally independent of VLLM_CACHE_ROOT."""
return os.path.expanduser(DEFAULT_RUNTIME_CACHE_ROOT)
return os.path.expanduser(DEFAULT_HOST_WEIGHT_CACHE_ROOT)
def _persistent_buffer(module: nn.Module, local_name: str) -> bool:
@@ -135,7 +135,7 @@ def _collect_runtime_tensors(
candidate = RuntimeTensor(runtime_name, tensor, "parameter")
existing = records.get(runtime_name)
if existing is not None and existing.tensor is not tensor:
raise RuntimeCacheError(
raise HostWeightCacheError(
"ambiguous_ownership",
f"multiple DiT tensors resolve to {runtime_name!r}",
)
@@ -147,14 +147,14 @@ def _collect_runtime_tensors(
candidate = RuntimeTensor(runtime_name, tensor, "buffer")
existing = records.get(runtime_name)
if existing is not None and existing.tensor is not tensor:
raise RuntimeCacheError(
raise HostWeightCacheError(
"ambiguous_ownership",
f"multiple DiT tensors resolve to {runtime_name!r}",
)
records[runtime_name] = candidate
if not records:
raise RuntimeCacheError(
raise HostWeightCacheError(
"ambiguous_ownership",
"no DiT parameters or persistent buffers were discovered",
)
@@ -165,37 +165,37 @@ def _collect_runtime_tensors(
try:
pipeline_tensor = _resolve_pipeline_tensor(pipeline, record.name)
except AttributeError as exc:
raise RuntimeCacheError(
raise HostWeightCacheError(
"ambiguous_ownership",
f"DiT tensor {record.name!r} is not owned by the pipeline",
) from exc
if pipeline_tensor is not tensor:
raise RuntimeCacheError(
raise HostWeightCacheError(
"ambiguous_ownership",
f"DiT tensor {record.name!r} does not resolve to the discovered object",
)
if tensor.device.type != "cpu" or tensor.is_meta:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} must be a materialized CPU tensor, got {tensor.device}",
)
if tensor.layout != torch.strided:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} uses unsupported layout {tensor.layout}",
)
if not tensor.is_contiguous():
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} is non-contiguous with stride {tensor.stride()}",
)
if tensor.dtype not in _SUPPORTED_DTYPES:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} uses unsupported dtype {tensor.dtype}",
)
if hasattr(tensor, "to_local"):
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} is a distributed tensor",
)
@@ -203,7 +203,7 @@ def _collect_runtime_tensors(
tensor_nbytes = tensor.numel() * tensor.element_size()
storage = tensor.untyped_storage()
if tensor.storage_offset() != 0 or storage.nbytes() != tensor_nbytes:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_tensor",
f"{record.name!r} is a view into a larger storage",
)
@@ -211,7 +211,7 @@ def _collect_runtime_tensors(
storage_id = (storage.data_ptr(), storage.nbytes())
owner = storage_owners.setdefault(storage_id, record.name)
if owner != record.name:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_alias",
f"{record.name!r} shares storage with {owner!r}",
)
@@ -228,7 +228,7 @@ def _collect_runtime_tensors(
storage = tensor.untyped_storage()
owner = storage_owners.get((storage.data_ptr(), storage.nbytes()))
if owner is not None and owner != pipeline_name:
raise RuntimeCacheError(
raise HostWeightCacheError(
"unsupported_alias",
f"cached tensor {owner!r} shares storage with pipeline tensor {pipeline_name!r}",
)
@@ -250,9 +250,9 @@ def _publish_entry(
free_bytes = shutil.disk_usage(cache_root).free
required_bytes = total_nbytes + max(64 * 1024**2, total_nbytes // 100)
if free_bytes < required_bytes:
raise RuntimeCacheError(
raise HostWeightCacheError(
"insufficient_disk",
f"runtime cache needs {required_bytes} bytes but only {free_bytes} are free under {cache_root}",
f"host weight cache needs {required_bytes} bytes but only {free_bytes} are free under {cache_root}",
)
tmp_root = cache_root / ".tmp"
@@ -270,7 +270,7 @@ def _publish_entry(
save_file(
{record.name: record.tensor.detach() for record in shard},
str(path),
metadata={"format": "pt", "dlo_runtime_cache_schema": str(RUNTIME_CACHE_SCHEMA_VERSION)},
metadata={"format": "pt", "dlo_host_weight_cache_schema": str(HOST_WEIGHT_CACHE_SCHEMA_VERSION)},
)
fsync_file(path)
file_manifest[filename] = {
@@ -285,7 +285,7 @@ def _publish_entry(
}
manifest = {
"schema_version": RUNTIME_CACHE_SCHEMA_VERSION,
"schema_version": HOST_WEIGHT_CACHE_SCHEMA_VERSION,
"cache_key": cache_key,
"layout_identity": layout_identity,
"runtime_content_sha256": content_digest,
@@ -328,35 +328,35 @@ def _validate_entry(
with manifest_path.open(encoding="utf-8") as handle:
manifest = json.load(handle)
except (OSError, ValueError) as exc:
raise RuntimeCacheError("entry_rejected", f"cannot read {manifest_path}: {exc}") from exc
raise HostWeightCacheError("entry_rejected", f"cannot read {manifest_path}: {exc}") from exc
if manifest.get("schema_version") != RUNTIME_CACHE_SCHEMA_VERSION:
raise RuntimeCacheError("entry_rejected", "runtime-cache schema mismatch")
if manifest.get("schema_version") != HOST_WEIGHT_CACHE_SCHEMA_VERSION:
raise HostWeightCacheError("entry_rejected", "host weight cache schema mismatch")
if manifest.get("cache_key") != cache_key:
raise RuntimeCacheError("entry_rejected", "runtime-cache key mismatch")
raise HostWeightCacheError("entry_rejected", "host weight cache key mismatch")
if manifest.get("layout_identity") != layout_identity:
raise RuntimeCacheError("entry_rejected", "runtime-cache layout identity mismatch")
raise HostWeightCacheError("entry_rejected", "host weight cache layout identity mismatch")
if manifest.get("runtime_content_sha256") != content_digest:
raise RuntimeCacheError("entry_rejected", "runtime-cache content identity mismatch")
raise HostWeightCacheError("entry_rejected", "host weight cache content identity mismatch")
expected_metadata = {record.name: tensor_metadata(record) for record in expected_records}
tensor_manifest = manifest.get("tensors")
file_manifest = manifest.get("files")
if not isinstance(tensor_manifest, dict) or not isinstance(file_manifest, dict):
raise RuntimeCacheError("entry_rejected", "runtime-cache manifest has invalid sections")
raise HostWeightCacheError("entry_rejected", "host weight cache manifest has invalid sections")
if set(tensor_manifest) != set(expected_metadata):
raise RuntimeCacheError("entry_rejected", "runtime-cache tensor names differ from the loaded DiT")
raise HostWeightCacheError("entry_rejected", "host weight cache tensor names differ from the loaded DiT")
for filename, metadata in file_manifest.items():
if Path(filename).name != filename or not isinstance(metadata, dict):
raise RuntimeCacheError("entry_rejected", f"invalid cache filename {filename!r}")
raise HostWeightCacheError("entry_rejected", f"invalid cache filename {filename!r}")
path = entry_dir / filename
try:
stat = path.stat()
except OSError as exc:
raise RuntimeCacheError("entry_rejected", f"missing cache shard {path}: {exc}") from exc
raise HostWeightCacheError("entry_rejected", f"missing cache shard {path}: {exc}") from exc
if stat.st_size != metadata.get("size") or file_digest(path) != metadata.get("sha256"):
raise RuntimeCacheError("entry_rejected", f"content digest mismatch for cache shard {path}")
raise HostWeightCacheError("entry_rejected", f"content digest mismatch for cache shard {path}")
bindings: dict[str, TensorBinding] = {}
mapped_records: list[RuntimeTensor] = []
@@ -369,43 +369,43 @@ def _validate_entry(
for record in expected_records:
stored = tensor_manifest.get(record.name)
if not isinstance(stored, dict):
raise RuntimeCacheError("entry_rejected", f"invalid tensor metadata for {record.name!r}")
raise HostWeightCacheError("entry_rejected", f"invalid tensor metadata for {record.name!r}")
filename = stored.get("file")
storage_key = stored.get("storage_key")
if filename not in handles or not isinstance(storage_key, str):
raise RuntimeCacheError("entry_rejected", f"invalid storage binding for {record.name!r}")
raise HostWeightCacheError("entry_rejected", f"invalid storage binding for {record.name!r}")
actual_metadata = {key: stored.get(key) for key in expected_metadata[record.name]}
if actual_metadata != expected_metadata[record.name]:
raise RuntimeCacheError("entry_rejected", f"metadata mismatch for {record.name!r}")
raise HostWeightCacheError("entry_rejected", f"metadata mismatch for {record.name!r}")
expected_keys_by_file[filename].add(storage_key)
try:
tensor = handles[filename].get_tensor(storage_key)
except Exception as exc:
raise RuntimeCacheError(
raise HostWeightCacheError(
"entry_rejected",
f"cannot map {record.name!r} from {filename}: {exc}",
) from exc
if tuple(tensor.shape) != tuple(record.tensor.shape) or tensor.dtype != record.tensor.dtype:
raise RuntimeCacheError("entry_rejected", f"mapped metadata mismatch for {record.name!r}")
raise HostWeightCacheError("entry_rejected", f"mapped metadata mismatch for {record.name!r}")
mapped_records.append(RuntimeTensor(record.name, tensor, record.kind))
bindings[record.name] = TensorBinding(storage_key=storage_key, file_path=str(entry_dir / filename))
for filename, expected_keys in expected_keys_by_file.items():
if set(handles[filename].keys()) != expected_keys:
raise RuntimeCacheError("entry_rejected", f"unexpected tensor keys in cache shard {filename}")
raise HostWeightCacheError("entry_rejected", f"unexpected tensor keys in cache shard {filename}")
if runtime_content_digest(mapped_records) != content_digest:
raise RuntimeCacheError("entry_rejected", "mapped runtime tensor content does not match the publisher")
raise HostWeightCacheError("entry_rejected", "mapped runtime tensor content does not match the publisher")
return HostWeightPlan(
backing_kind="runtime_cache",
backing_kind="host_weight_cache",
bindings=bindings,
runtime_layout_key=cache_key,
post_load_complete=True,
)
def build_runtime_weight_cache_plan(
def build_host_weight_cache_plan(
pipeline: nn.Module,
*,
dit_modules: Sequence[tuple[str, nn.Module]],
@@ -427,7 +427,7 @@ def build_runtime_weight_cache_plan(
cfg_parallel_size: int,
pipeline_parallel_size: int,
) -> HostWeightPlanResult:
"""Build the runtime-cache host-weight plan after ordinary model loading.
"""Build a host-weight plan from the host weight cache after ordinary loading.
This is the module's main entry point. It first rejects unsupported runtime
layouts, then discovers and validates the final CPU DiT tensors and derives
@@ -448,33 +448,37 @@ def build_runtime_weight_cache_plan(
"unsupported_load_format",
)
if use_hsdp:
return HostWeightPlanResult(None, "HSDP/DTensor layouts are not runtime-cache compatible", "unsupported_hsdp")
return HostWeightPlanResult(
None, "HSDP/DTensor layouts are not host weight cache compatible", "unsupported_hsdp"
)
if enable_expert_parallel:
return HostWeightPlanResult(
None,
"expert-parallel weight ownership is outside runtime-cache v1",
"expert-parallel weight ownership is outside host weight cache v1",
"unsupported_expert_parallel",
)
if quantization_config is not None:
return HostWeightPlanResult(
None,
"quantized runtime layouts are not supported by the first runtime-cache version",
"quantized runtime layouts are not supported by the first host weight cache version",
"unsupported_quantization",
)
if cfg_parallel_size != 1:
return HostWeightPlanResult(None, "CFG-parallel component ownership is not proven", "unsupported_cfg_parallel")
if pipeline_parallel_size != 1:
return HostWeightPlanResult(None, "pipeline-parallel ownership is outside runtime-cache v1", "unsupported_pp")
return HostWeightPlanResult(
None, "pipeline-parallel ownership is outside host weight cache v1", "unsupported_pp"
)
if not math.isfinite(lock_timeout_seconds) or lock_timeout_seconds <= 0:
return HostWeightPlanResult(None, "runtime-cache lock timeout must be positive", "invalid_configuration")
return HostWeightPlanResult(None, "host weight cache lock timeout must be positive", "invalid_configuration")
if max_shard_bytes <= 0:
return HostWeightPlanResult(None, "runtime-cache shard size must be positive", "invalid_configuration")
return HostWeightPlanResult(None, "host weight cache shard size must be positive", "invalid_configuration")
try:
records = _collect_runtime_tensors(pipeline, dit_modules)
content_digest = runtime_content_digest(records)
layout_identity = {
"schema_version": RUNTIME_CACHE_SCHEMA_VERSION,
"schema_version": HOST_WEIGHT_CACHE_SCHEMA_VERSION,
"model": (canonicalize_existing_local_path(model_identity) if model_identity is not None else None),
"revision": revision,
"runtime_dtype": str(runtime_dtype),
@@ -488,8 +492,8 @@ def build_runtime_weight_cache_plan(
"runtime_content_sha256": content_digest,
}
cache_key = hashlib.sha256(canonical_json(layout_identity)).hexdigest()
root = Path(cache_root or default_runtime_cache_root()).expanduser().resolve()
entries_root = root / f"v{RUNTIME_CACHE_SCHEMA_VERSION}"
root = Path(cache_root or default_host_weight_cache_root()).expanduser().resolve()
entries_root = root / f"v{HOST_WEIGHT_CACHE_SCHEMA_VERSION}"
entry_dir = entries_root / cache_key
entries_root.mkdir(parents=True, exist_ok=True)
@@ -502,10 +506,10 @@ def build_runtime_weight_cache_plan(
content_digest=content_digest,
expected_records=records,
)
logger.info("Reusing DLO runtime cache entry %s from %s", cache_key[:12], entry_dir)
logger.info("Reusing DLO host weight cache entry %s from %s", cache_key[:12], entry_dir)
return HostWeightPlanResult(plan)
except RuntimeCacheError as exc:
logger.warning("DLO runtime cache entry %s will be rebuilt: %s", cache_key[:12], exc)
except HostWeightCacheError as exc:
logger.warning("DLO host weight cache entry %s will be rebuilt: %s", cache_key[:12], exc)
lock_path = root / ".locks" / f"{cache_key}.lock"
with exclusive_lock(lock_path, lock_timeout_seconds):
@@ -519,9 +523,9 @@ def build_runtime_weight_cache_plan(
content_digest=content_digest,
expected_records=records,
)
logger.info("Joined DLO runtime cache entry %s from %s", cache_key[:12], entry_dir)
logger.info("Joined DLO host weight cache entry %s from %s", cache_key[:12], entry_dir)
return HostWeightPlanResult(plan)
except RuntimeCacheError:
except HostWeightCacheError:
pass
_publish_entry(
@@ -541,25 +545,25 @@ def build_runtime_weight_cache_plan(
expected_records=records,
)
logger.info(
"Published DLO runtime cache entry %s with %d tensors at %s",
"Published DLO host weight cache entry %s with %d tensors at %s",
cache_key[:12],
len(records),
entry_dir,
)
return HostWeightPlanResult(plan)
except RuntimeCacheError as exc:
except HostWeightCacheError as exc:
return HostWeightPlanResult(None, str(exc), exc.code)
except IdentityNormalizationError as exc:
return HostWeightPlanResult(None, str(exc), "unstable_identity")
except Exception as exc:
return HostWeightPlanResult(None, f"runtime-cache operation failed: {exc}", "cache_operation_failed")
return HostWeightPlanResult(None, f"host weight cache operation failed: {exc}", "cache_operation_failed")
__all__ = [
"DEFAULT_LOCK_TIMEOUT_SECONDS",
"DEFAULT_RUNTIME_CACHE_ROOT",
"DEFAULT_HOST_WEIGHT_CACHE_ROOT",
"DEFAULT_SHARD_SIZE_BYTES",
"RUNTIME_CACHE_SCHEMA_VERSION",
"build_runtime_weight_cache_plan",
"default_runtime_cache_root",
"HOST_WEIGHT_CACHE_SCHEMA_VERSION",
"build_host_weight_cache_plan",
"default_host_weight_cache_root",
]
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable identity, hashing, and file helpers for the DLO runtime cache."""
"""Reusable identity, hashing, and file helpers for the DLO host weight cache."""
from __future__ import annotations
@@ -43,7 +43,7 @@ class RuntimeTensor:
kind: str
class RuntimeCacheError(RuntimeError):
class HostWeightCacheError(RuntimeError):
"""Expected cache failure that should fall back with a stable code."""
def __init__(self, code: str, message: str):
@@ -102,7 +102,7 @@ def normalize_identity(value: object) -> JsonValue:
normalized: dict[str, JsonValue] = {}
for key, item in value.items():
if not isinstance(key, str):
raise IdentityNormalizationError("runtime-cache identity mappings require string keys")
raise IdentityNormalizationError("host weight cache identity mappings require string keys")
normalized[key] = normalize_identity(item)
return {key: normalized[key] for key in sorted(normalized)}
if isinstance(value, (set, frozenset)):
@@ -110,7 +110,9 @@ def normalize_identity(value: object) -> JsonValue:
if isinstance(value, (list, tuple)):
return [normalize_identity(item) for item in value]
raise IdentityNormalizationError(f"runtime-cache identity does not support values of type {_type_identity(value)}")
raise IdentityNormalizationError(
f"host weight cache identity does not support values of type {_type_identity(value)}"
)
def canonical_json(value: object) -> bytes:
@@ -203,12 +205,12 @@ def exclusive_lock(path: Path, timeout_seconds: float) -> Iterator[None]:
break
except OSError as exc:
if exc.errno not in (errno.EACCES, errno.EAGAIN):
raise RuntimeCacheError("lock_failed", f"failed to lock {path}: {exc}") from exc
raise HostWeightCacheError("lock_failed", f"failed to lock {path}: {exc}") from exc
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeCacheError(
raise HostWeightCacheError(
"lock_timeout",
f"timed out after {timeout_seconds:g}s waiting for runtime-cache writer {path.name}",
f"timed out after {timeout_seconds:g}s waiting for host weight cache writer {path.name}",
) from exc
time.sleep(min(0.2, remaining))
try:
@@ -268,7 +270,7 @@ def remove_stale_temps(cache_root: Path, cache_key: str) -> None:
__all__ = [
"IdentityNormalizationError",
"JsonValue",
"RuntimeCacheError",
"HostWeightCacheError",
"RuntimeTensor",
"canonical_json",
"canonicalize_existing_local_path",