[Diffusion] Prefer host-cache registration when pinned
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
859003fb97
commit
6bcd350301
14 files changed
+212
-101
No files matched your search
@@ -708,16 +708,15 @@ class OmniDiffusionConfig:
|
||||
# This avoids AllGather synchronization, while host memory follows the
|
||||
# loader's existing rank-local layout instead of adding a second DP shard.
|
||||
dlo_use_allgather: bool = True
|
||||
# Opt into publishing/joining the final-layout host weight cache when a
|
||||
# compatible direct-checkpoint mmap plan is unavailable. A positive pin
|
||||
# limit also selects the cache because registration requires its layout.
|
||||
# Opt into publishing/joining the transform-complete final-layout host
|
||||
# weight cache instead of direct-checkpoint mmap.
|
||||
dlo_use_host_weight_cache: bool = False
|
||||
# Shared local-disk root. None selects ~/.cache/vllm-omni/dlo-host-weights.
|
||||
dlo_host_weight_cache_dir: str | None = None
|
||||
# Maximum wait for the per-layout POSIX writer lock.
|
||||
dlo_host_weight_cache_lock_timeout: float = 600.0
|
||||
# Maximum file-backed memory a worker may register with CUDA. Zero keeps
|
||||
# the bounded mmap-to-pinned staging path.
|
||||
# Optional maximum file-backed memory a worker may register with CUDA.
|
||||
# Zero applies no additional ceiling; pin_cpu_memory controls registration.
|
||||
dlo_host_weight_cache_pin_limit_gib: float = 0.0
|
||||
# Leading main-DiT blocks kept resident by distributed layerwise offload.
|
||||
dlo_resident_layers: int = 0
|
||||
@@ -957,6 +956,8 @@ class OmniDiffusionConfig:
|
||||
raise ValueError("dlo_host_weight_cache_pin_limit_gib requires no-AllGather distributed layerwise offload")
|
||||
if self.dlo_use_host_weight_cache and (not self.enable_distributed_layerwise_offload or self.dlo_use_allgather):
|
||||
raise ValueError("dlo_use_host_weight_cache requires no-AllGather distributed layerwise offload")
|
||||
if self.dlo_host_weight_cache_pin_limit_gib and not self.dlo_use_host_weight_cache:
|
||||
raise ValueError("dlo_host_weight_cache_pin_limit_gib requires dlo_use_host_weight_cache=True")
|
||||
if self.diffusion_compile_granularity not in {"regional", "full"}:
|
||||
raise ValueError(
|
||||
"diffusion_compile_granularity must be 'regional' or 'full', "
|
||||
|
||||
@@ -441,15 +441,13 @@ class DiffusersPipelineLoader:
|
||||
_dp_size = int(getattr(self.parallel_config, "data_parallel_size", 1))
|
||||
_sp_size = int(getattr(self.parallel_config, "sequence_parallel_size", 1))
|
||||
_dlo_group_size = _dp_size if _dp_size > 1 else _sp_size
|
||||
_prefer_registered_host_weight_cache = (
|
||||
_dist_offload
|
||||
and not _use_ag
|
||||
and float(getattr(self.od_config, "dlo_host_weight_cache_pin_limit_gib", 0.0)) > 0
|
||||
_select_host_weight_cache = (
|
||||
_dist_offload and not _use_ag and bool(getattr(self.od_config, "dlo_use_host_weight_cache", False))
|
||||
)
|
||||
|
||||
plan_result = None
|
||||
weight_sources = self._get_weight_sources(model)
|
||||
if _dist_offload and not _prefer_registered_host_weight_cache:
|
||||
if _dist_offload and not _select_host_weight_cache:
|
||||
modules = ModuleDiscovery.discover(model)
|
||||
plan_result = build_checkpoint_mmap_plan(
|
||||
model,
|
||||
@@ -461,14 +459,14 @@ class DiffusersPipelineLoader:
|
||||
online_quantization=_has_online_quant,
|
||||
)
|
||||
self.host_weight_plan = plan_result.plan
|
||||
elif _prefer_registered_host_weight_cache:
|
||||
elif _select_host_weight_cache:
|
||||
# Direct checkpoint bindings can carry deferred transforms
|
||||
# (for example grouped-QKV reordering), so registering the
|
||||
# raw checkpoint mapping would not make every recurrent H2D
|
||||
# source directly copyable. Materialize once, apply every
|
||||
# post-load mutation, then publish/register the final layout.
|
||||
logger.info(
|
||||
"DLO host weight cache registration requested; using the ordinary loader "
|
||||
"DLO host weight cache selected; using the ordinary loader "
|
||||
"once to publish a transform-complete final mmap layout"
|
||||
)
|
||||
|
||||
@@ -550,10 +548,7 @@ class DiffusersPipelineLoader:
|
||||
return
|
||||
if getattr(self.od_config, "dlo_use_allgather", True):
|
||||
return
|
||||
if not (
|
||||
getattr(self.od_config, "dlo_use_host_weight_cache", False)
|
||||
or float(getattr(self.od_config, "dlo_host_weight_cache_pin_limit_gib", 0.0)) > 0
|
||||
):
|
||||
if not getattr(self.od_config, "dlo_use_host_weight_cache", False):
|
||||
logger.info(
|
||||
"DLO host weight cache not selected; retaining ordinary host tensors after checkpoint mmap fallback"
|
||||
)
|
||||
|
||||
@@ -31,8 +31,8 @@ class OffloadConfig:
|
||||
# blocks from the loader-selected host backing with H2D only.
|
||||
dlo_use_allgather: bool = True
|
||||
dlo_resident_layers: int = 0 # leading DiT layers kept on device
|
||||
# Per-worker budget for registering shared host weight cache mappings with CUDA.
|
||||
# Zero retains bounded host staging.
|
||||
# Optional per-worker ceiling for registering shared host weight cache mappings.
|
||||
# Zero applies no additional ceiling; pin_cpu_memory controls registration.
|
||||
dlo_host_weight_cache_pin_limit_gib: float = 0.0
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -182,13 +182,11 @@ class CudaHostRegistration:
|
||||
cls,
|
||||
sources_by_mapping: Mapping[str, Sequence[torch.Tensor]],
|
||||
*,
|
||||
max_bytes: int,
|
||||
max_bytes: int | None,
|
||||
) -> CudaHostRegistration:
|
||||
if max_bytes <= 0:
|
||||
raise HostRegistrationBudgetError("CUDA host-registration budget is disabled")
|
||||
regions = _tensor_regions(sources_by_mapping)
|
||||
total_bytes = sum(region.size for region in regions)
|
||||
if total_bytes > max_bytes:
|
||||
if max_bytes is not None and total_bytes > max_bytes:
|
||||
raise HostRegistrationBudgetError(
|
||||
f"mapped host ranges need {total_bytes} bytes, exceeding the {max_bytes}-byte registration budget"
|
||||
)
|
||||
|
||||
@@ -1269,20 +1269,16 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
|
||||
)
|
||||
|
||||
def _try_register_host_weight_cache_mmap(self) -> bool:
|
||||
"""Register complete cache mappings when the operator budget permits."""
|
||||
"""Register complete cache mappings under the pinned-memory policy."""
|
||||
pin_limit_gib = self.config.dlo_host_weight_cache_pin_limit_gib
|
||||
if pin_limit_gib <= 0:
|
||||
return False
|
||||
if not self.config.pin_cpu_memory:
|
||||
logger.warning(
|
||||
"DLO host weight cache registration requires pin_cpu_memory=True; using bounded host staging"
|
||||
)
|
||||
logger.info("DLO host weight cache registration disabled by pin_cpu_memory=False; using host staging")
|
||||
return False
|
||||
if not self._host_weight_cache_mapped_sources:
|
||||
logger.warning("DLO host weight cache registration found no mapped sources; using bounded host staging")
|
||||
return False
|
||||
|
||||
max_bytes = int(pin_limit_gib * 1024**3)
|
||||
max_bytes = int(pin_limit_gib * 1024**3) if pin_limit_gib > 0 else None
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
registration = register_host_mappings(
|
||||
|
||||
@@ -38,12 +38,13 @@ def register_host_mappings(
|
||||
sources_by_mapping: Mapping[str, Sequence[torch.Tensor]],
|
||||
*,
|
||||
device: torch.device,
|
||||
max_bytes: int,
|
||||
max_bytes: int | None,
|
||||
) -> HostRegistration:
|
||||
"""Register mappings with the active platform or report unsupported use.
|
||||
|
||||
CUDA is the first implementation. Other platforms retain DLO's bounded
|
||||
host-staging path until they provide an equivalent registration backend.
|
||||
``max_bytes=None`` applies no additional ceiling to the complete mapping.
|
||||
"""
|
||||
if device.type == "cuda":
|
||||
from .cuda_host_registration import CudaHostRegistration
|
||||
|
||||
Reference in new issue
Block a user