[Diffusion] Add loader-owned host-weight plans for DLO
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
d1e230c95b
commit
e2796b6168
17 files changed
+1717
-588
No files matched your search
@@ -5,6 +5,7 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.model_loader.host_weight_plan import HostWeightPlan
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, OffloadStrategy
|
||||
@@ -17,7 +18,7 @@ from .distributed_layerwise_backend import (
|
||||
)
|
||||
from .layerwise_backend import LayerWiseOffloadBackend
|
||||
from .module_residency import PinnedModuleStager
|
||||
from .offload_plan import OffloadPlan, get_offload_plan, supports_mmap_loading
|
||||
from .offload_plan import OffloadPlan, get_offload_plan
|
||||
from .sequential_backend import (
|
||||
ModelLevelOffloadBackend,
|
||||
apply_sequential_offload,
|
||||
@@ -49,7 +50,6 @@ __all__ = [
|
||||
"remove_distributed_block_hook",
|
||||
"get_offload_backend",
|
||||
"get_offload_plan",
|
||||
"supports_mmap_loading",
|
||||
"get_blocks_attr_names",
|
||||
"get_blocks_from_dit",
|
||||
"set_blocks_attr_names",
|
||||
@@ -64,12 +64,15 @@ __all__ = [
|
||||
def get_offload_backend(
|
||||
od_config: OmniDiffusionConfig,
|
||||
device: torch.device | None = None,
|
||||
host_weight_plan: HostWeightPlan | None = None,
|
||||
) -> OffloadBackend | None:
|
||||
"""Create appropriate offload backend based on configuration.
|
||||
|
||||
Args:
|
||||
od_config: OmniDiffusionConfig with offload settings
|
||||
device: Target device (auto-detected if None)
|
||||
host_weight_plan: Exact loader-produced backing plan, if ordinary
|
||||
weight materialization was skipped.
|
||||
|
||||
Returns:
|
||||
OffloadBackend instance or None if offloading disabled
|
||||
@@ -82,12 +85,22 @@ def get_offload_backend(
|
||||
# Extract and validate configuration
|
||||
config = OffloadConfig.from_od_config(od_config)
|
||||
|
||||
if host_weight_plan is not None and config.strategy != OffloadStrategy.DISTRIBUTED_LAYER_WISE:
|
||||
raise RuntimeError(
|
||||
"A loader-owned DLO host-weight plan was produced, but distributed layerwise offload is not selected"
|
||||
)
|
||||
|
||||
# Return None if no offloading requested
|
||||
if config.strategy == OffloadStrategy.NONE:
|
||||
return None
|
||||
|
||||
# Validate platform (CUDA required for now)
|
||||
if not current_omni_platform.supports_cpu_offload() or current_omni_platform.get_device_count() < 1:
|
||||
if host_weight_plan is not None:
|
||||
raise RuntimeError(
|
||||
"The loader skipped ordinary weight materialization for DLO, "
|
||||
"but this platform cannot create the required offload backend"
|
||||
)
|
||||
logger.warning(
|
||||
"Current device: %s does not support CPU offloading. Skipping offloading.",
|
||||
current_omni_platform.get_device_name(),
|
||||
@@ -99,6 +112,11 @@ def get_offload_backend(
|
||||
try:
|
||||
device = current_omni_platform.get_torch_device()
|
||||
except (NotImplementedError, AttributeError) as exc:
|
||||
if host_weight_plan is not None:
|
||||
raise RuntimeError(
|
||||
"The loader skipped ordinary weight materialization for DLO, "
|
||||
"but the target device could not be resolved"
|
||||
) from exc
|
||||
logger.error("Failed to detect device: %s. Skipping offloading.", exc)
|
||||
return None
|
||||
|
||||
@@ -108,7 +126,11 @@ def get_offload_backend(
|
||||
elif config.strategy == OffloadStrategy.LAYER_WISE:
|
||||
return LayerWiseOffloadBackend(config, device)
|
||||
elif config.strategy == OffloadStrategy.DISTRIBUTED_LAYER_WISE:
|
||||
return DistributedLayerwiseOffloadBackend(config, device)
|
||||
return DistributedLayerwiseOffloadBackend(
|
||||
config,
|
||||
device,
|
||||
host_weight_plan=host_weight_plan,
|
||||
)
|
||||
else:
|
||||
logger.error("Unknown offload strategy: %s", config.strategy)
|
||||
return None
|
||||
@@ -27,11 +27,10 @@ class OffloadConfig:
|
||||
pin_cpu_memory: bool = True
|
||||
use_hsdp: bool = False
|
||||
dp_size: int = 1 # derived from parallel_config, not user-configurable
|
||||
# True: add DP sharding + AllGather. False: stream the standard loader's
|
||||
# rank-local tensors (including TP-local shards) with H2D only.
|
||||
# True: add DP sharding + AllGather. False: stream complete rank-local
|
||||
# blocks from the loader-selected host backing with H2D only.
|
||||
dlo_use_allgather: bool = True
|
||||
dlo_resident_layers: int = 0 # leading DiT layers kept on device
|
||||
model_path: str | None = None # checkpoint path for mmap weight loading
|
||||
|
||||
@classmethod
|
||||
def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig":
|
||||
@@ -58,7 +57,6 @@ class OffloadConfig:
|
||||
|
||||
parallel_config = getattr(od_config, "parallel_config", None)
|
||||
use_hsdp = getattr(parallel_config, "use_hsdp", False) if parallel_config else False
|
||||
|
||||
# Derive dp_size from parallel_config — not user-configurable.
|
||||
# The offload adapts to whatever DP/SP is already configured.
|
||||
dp_size = 1
|
||||
@@ -116,7 +114,8 @@ class OffloadConfig:
|
||||
dp_size = 1
|
||||
logger.info(
|
||||
"Distributed layerwise offload: dlo_use_allgather=False, "
|
||||
"streaming standard-loader rank-local weights (no DP shard or AllGather)"
|
||||
"streaming complete rank-local blocks (no DLO shard or AllGather); "
|
||||
"the backend will select mmap or standard-loader host storage"
|
||||
)
|
||||
|
||||
# HSDP already shards parameters into DTensors. Running distributed
|
||||
@@ -137,7 +136,6 @@ class OffloadConfig:
|
||||
dp_size=dp_size,
|
||||
dlo_use_allgather=dlo_use_allgather,
|
||||
dlo_resident_layers=dlo_resident_layers,
|
||||
model_path=getattr(od_config, "model", None),
|
||||
)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -55,20 +55,3 @@ class OffloadPlan:
|
||||
def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None:
|
||||
"""Retrieve the OffloadPlan declared by the pipeline, if any."""
|
||||
return getattr(pipeline, "_offload_plan", None)
|
||||
|
||||
|
||||
def supports_mmap_loading(pipeline: nn.Module) -> bool:
|
||||
"""Whether the pipeline supports mmap weight loading.
|
||||
|
||||
A pipeline supports mmap if any module defines ``_remap_ckpt_key``,
|
||||
which maps checkpoint keys to model parameter names. Models without
|
||||
this method must use the regular ``load_weights()`` path.
|
||||
|
||||
This check is shared between ``diffusers_loader.py`` (to decide whether
|
||||
to skip ``load_weights``) and ``DistributedLayerwiseOffloadBackend.enable()``
|
||||
(to decide whether to call ``_load_weights_via_mmap``), ensuring both
|
||||
paths use the same gate condition.
|
||||
"""
|
||||
return bool(getattr(pipeline, "_supports_mmap_loading", True)) and any(
|
||||
callable(getattr(type(m), "_remap_ckpt_key", None)) for m in pipeline.modules()
|
||||
)
|
||||
Reference in new issue
Block a user