[diffusion][feature] Add component-selective offload policies (#5929)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
44d3ae100a
commit
ea3cf99894
74 files changed
+5121
-1584
No files matched your search
@@ -8,8 +8,9 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.model_loader.host_weight_plan import HostWeightPlan
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, OffloadStrategy, SupportsModelCpuOffload
|
||||
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
|
||||
from .block_discovery import get_blocks_attr_names, get_blocks_from_dit, set_blocks_attr_names
|
||||
from .config import OffloadStrategy
|
||||
from .distributed_layerwise_backend import (
|
||||
DistributedLayerwiseOffloadBackend,
|
||||
DistributedLayerwiseOffloadHook,
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import torch
|
||||
@@ -12,9 +12,33 @@ from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, validate_dlo_host_registration_options
|
||||
|
||||
from .component_utils import encoder_component_type
|
||||
from .config import (
|
||||
DIT_COMPONENT,
|
||||
TEXT_ENCODER_COMPONENT,
|
||||
DLOTransfer,
|
||||
OffloadStrategy,
|
||||
parse_dlo_transfer,
|
||||
parse_offload_components,
|
||||
resolve_offload,
|
||||
)
|
||||
from .offload_plan import OffloadPlan
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def run_cleanup_steps(steps: Iterable[tuple[str, Callable[[], None]]]) -> BaseException | None:
|
||||
"""Run every cleanup step and return the first failure, if any."""
|
||||
first_error: BaseException | None = None
|
||||
for description, step in steps:
|
||||
try:
|
||||
step()
|
||||
except BaseException as exc:
|
||||
logger.exception("Cleanup failed while %s", description)
|
||||
first_error = first_error or exc
|
||||
return first_error
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsModelCpuOffload(Protocol):
|
||||
"""Pipeline-owned lifecycle for model-level CPU offload.
|
||||
@@ -30,18 +54,12 @@ class SupportsModelCpuOffload(Protocol):
|
||||
device: torch.device,
|
||||
pin_memory: bool,
|
||||
use_hsdp: bool,
|
||||
offload_components: frozenset[str] | None = None,
|
||||
) -> None: ...
|
||||
|
||||
def disable_omni_model_cpu_offload(self) -> None: ...
|
||||
|
||||
|
||||
class OffloadStrategy(Enum):
|
||||
NONE = "none"
|
||||
MODEL_LEVEL = "model_level" # Sequential offloading between DiT and encoders
|
||||
LAYER_WISE = "layer_wise" # Block-level
|
||||
DISTRIBUTED_LAYER_WISE = "distributed_layer_wise" # Block-level with DP sharding + H2D/AllGather overlap
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadConfig:
|
||||
strategy: OffloadStrategy
|
||||
@@ -55,14 +73,59 @@ class OffloadConfig:
|
||||
# Optional per-worker ceiling for registering an HWR mmap. Zero means no
|
||||
# additional ceiling; pin_cpu_memory controls whether registration is tried.
|
||||
dlo_host_registration_limit_gib: float = 0.0
|
||||
# ``None`` preserves the model's legacy plan-driven component topology;
|
||||
# a frozenset is an explicit compact-API selection.
|
||||
components: frozenset[str] | None = None
|
||||
dlo_transfers: dict[str, DLOTransfer] | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.components is not None:
|
||||
self.components = parse_offload_components(self.components)
|
||||
if self.dlo_transfers is None:
|
||||
self.dlo_transfers = {
|
||||
DIT_COMPONENT: DLOTransfer.ALLGATHER if self.dlo_use_allgather else DLOTransfer.RANK_LOCAL,
|
||||
TEXT_ENCODER_COMPONENT: DLOTransfer.RANK_LOCAL,
|
||||
}
|
||||
self.dlo_transfers = parse_dlo_transfer(self.dlo_transfers)
|
||||
# Preserve the old field as the DiT transfer compatibility view.
|
||||
self.dlo_use_allgather = self.uses_allgather(DIT_COMPONENT)
|
||||
|
||||
def offloads(self, component: str) -> bool:
|
||||
# Historically the generic selector covered DiT directly while model
|
||||
# plans opted auxiliary components into their lifecycle.
|
||||
return component == DIT_COMPONENT if self.components is None else component in self.components
|
||||
|
||||
def transfer_for(self, component: str) -> DLOTransfer:
|
||||
if self.dlo_transfers is None:
|
||||
raise RuntimeError("DLO transfers were not initialized")
|
||||
try:
|
||||
return self.dlo_transfers[component]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unknown DLO component {component!r}") from exc
|
||||
|
||||
def uses_allgather(self, component: str) -> bool:
|
||||
return self.transfer_for(component) is DLOTransfer.ALLGATHER
|
||||
|
||||
def offloads_encoder(self, name: str, plan: OffloadPlan | None = None) -> bool:
|
||||
"""Return whether the selector covers a discovered encoder path.
|
||||
|
||||
Plans declare non-standard encoder names explicitly. The name-based
|
||||
fallback preserves compatibility with pipelines that predate OffloadPlan.
|
||||
"""
|
||||
component = encoder_component_type(name, plan)
|
||||
return component is not None and self.offloads(component)
|
||||
|
||||
def should_offload_encoder(self, name: str, plan: OffloadPlan | None = None) -> bool:
|
||||
"""Apply explicit selection while preserving the legacy encoder topology."""
|
||||
return self.components is None or self.offloads_encoder(name, plan)
|
||||
|
||||
@classmethod
|
||||
def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig":
|
||||
"""Extract and validate offload settings from OmniDiffusionConfig.
|
||||
|
||||
Enforces mutual exclusion among the three offload strategies.
|
||||
Distributed layer-wise takes the highest priority, then layer-wise,
|
||||
then model-level.
|
||||
``diffusion_offload_config`` is the canonical public selector. The
|
||||
historical ``enable_*_offload`` booleans remain compatibility aliases;
|
||||
ambiguous combinations fail instead of using silent precedence.
|
||||
|
||||
The ``dp_size`` is automatically derived from ``parallel_config`` —
|
||||
it is NOT a user-configurable parameter. The distributed layerwise
|
||||
@@ -74,10 +137,11 @@ class OffloadConfig:
|
||||
Returns:
|
||||
OffloadConfig with validated settings
|
||||
"""
|
||||
enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False)
|
||||
enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False)
|
||||
enable_distributed_layerwise_offload = getattr(od_config, "enable_distributed_layerwise_offload", False)
|
||||
pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True)
|
||||
resolved = resolve_offload(od_config)
|
||||
strategy = resolved.strategy
|
||||
public_config = resolved.public
|
||||
enable_distributed_layerwise_offload = strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE
|
||||
pin_cpu_memory = resolved.pin_memory
|
||||
|
||||
parallel_config = getattr(od_config, "parallel_config", None)
|
||||
use_hsdp = getattr(parallel_config, "use_hsdp", False) if parallel_config else False
|
||||
@@ -102,61 +166,40 @@ class OffloadConfig:
|
||||
if sp_size and sp_size > 1:
|
||||
dp_size = sp_size
|
||||
|
||||
# Determine strategy (mutual exclusion, distributed layer-wise takes priority)
|
||||
if enable_distributed_layerwise_offload:
|
||||
strategy = OffloadStrategy.DISTRIBUTED_LAYER_WISE
|
||||
if enable_layerwise_offload or enable_cpu_offload:
|
||||
logger.info("Distributed layer-wise offloading takes priority, disabling other offloading strategies.")
|
||||
elif enable_layerwise_offload:
|
||||
strategy = OffloadStrategy.LAYER_WISE
|
||||
if enable_cpu_offload:
|
||||
logger.info(
|
||||
"Both model-level and layer-wise offloading enabled. "
|
||||
"Layer-wise takes priority, disabling model-level offloading."
|
||||
)
|
||||
elif enable_cpu_offload:
|
||||
strategy = OffloadStrategy.MODEL_LEVEL
|
||||
if public_config is not None:
|
||||
components: frozenset[str] | None = resolved.components
|
||||
else:
|
||||
strategy = OffloadStrategy.NONE
|
||||
components = None
|
||||
|
||||
# With dlo_use_allgather=False, do not add another DP shard. Each rank
|
||||
# streams the tensors produced by the standard loader, which may
|
||||
# already be TP-local shards. This avoids AllGather synchronization
|
||||
# requirements (concurrent requests, dummy run skip).
|
||||
dlo_use_allgather = getattr(od_config, "dlo_use_allgather", True)
|
||||
dlo_resident_layers = int(getattr(od_config, "dlo_resident_layers", 0))
|
||||
dlo_transfers = dict(resolved.transfers)
|
||||
dlo_resident_layers = resolved.resident_layers
|
||||
dit_uses_allgather = resolved.uses_allgather(DIT_COMPONENT)
|
||||
dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
||||
limit_gib=getattr(od_config, "dlo_host_registration_limit_gib", 0.0),
|
||||
enable_dlo=enable_distributed_layerwise_offload,
|
||||
use_allgather=dlo_use_allgather,
|
||||
use_allgather=dit_uses_allgather,
|
||||
hwr_mode=getattr(od_config, "host_weight_runtime_mode", "disabled"),
|
||||
)
|
||||
if dlo_resident_layers < 0:
|
||||
raise ValueError(f"dlo_resident_layers must be >= 0, got {dlo_resident_layers}")
|
||||
if dlo_resident_layers and dlo_use_allgather:
|
||||
raise ValueError(
|
||||
"dlo_resident_layers currently requires --dlo-no-use-allgather so "
|
||||
"resident blocks use weights prepared by the standard TP-aware loader"
|
||||
)
|
||||
|
||||
# If dlo_use_allgather=False, force dp_size=1 (each rank independent)
|
||||
if enable_distributed_layerwise_offload and not dlo_use_allgather:
|
||||
dp_size = 1
|
||||
if enable_distributed_layerwise_offload and all(
|
||||
transfer is DLOTransfer.RANK_LOCAL
|
||||
for component, transfer in dlo_transfers.items()
|
||||
if component in resolved.components
|
||||
):
|
||||
logger.info(
|
||||
"Distributed layerwise offload: dlo_use_allgather=False, "
|
||||
"streaming complete rank-local blocks (no DLO shard or AllGather); "
|
||||
"the backend will select mmap or standard-loader host storage"
|
||||
"Distributed layerwise offload: all selected components use "
|
||||
"rank-local transfer (no DLO shard or AllGather)"
|
||||
)
|
||||
|
||||
# HSDP already shards parameters into DTensors. Running distributed
|
||||
# layerwise offload on top would shard each to_local() again, producing
|
||||
# incorrect reconstruction after AllGather. Reject this combination.
|
||||
if enable_distributed_layerwise_offload and use_hsdp and dlo_use_allgather:
|
||||
if enable_distributed_layerwise_offload and use_hsdp and resolved.any_allgather:
|
||||
raise ValueError(
|
||||
"Distributed layerwise offload with AllGather is incompatible with "
|
||||
"HSDP: HSDP parameters are already sharded DTensors, and the offloader "
|
||||
"would double-shard them. Use --dlo-no-use-allgather (standard-loader "
|
||||
"rank-local weights) or disable HSDP."
|
||||
"would double-shard them. Set weight_transfer='rank-local' for the affected "
|
||||
"component in diffusion_offload_config, or disable HSDP."
|
||||
)
|
||||
|
||||
return cls(
|
||||
@@ -164,9 +207,11 @@ class OffloadConfig:
|
||||
pin_cpu_memory=pin_cpu_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
dp_size=dp_size,
|
||||
dlo_use_allgather=dlo_use_allgather,
|
||||
dlo_use_allgather=dit_uses_allgather,
|
||||
dlo_resident_layers=dlo_resident_layers,
|
||||
dlo_host_registration_limit_gib=dlo_host_registration_limit_gib,
|
||||
components=components,
|
||||
dlo_transfers=dlo_transfers,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -35,9 +35,13 @@ def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
|
||||
|
||||
|
||||
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
|
||||
"""Retrieve blocks and attribute names from provided DiT model."""
|
||||
blocks_attr_names = get_blocks_attr_names(model)
|
||||
def get_blocks_from_dit(
|
||||
model: nn.Module,
|
||||
block_attrs: tuple[str, ...] | None = None,
|
||||
) -> tuple[list[str], list[nn.Module]]:
|
||||
"""Retrieve blocks from an explicit plan or the DiT class metadata."""
|
||||
explicit_attrs = block_attrs is not None
|
||||
blocks_attr_names = list(block_attrs) if explicit_attrs else get_blocks_attr_names(model)
|
||||
if not blocks_attr_names:
|
||||
logger.warning(
|
||||
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
|
||||
@@ -49,9 +53,9 @@ def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
|
||||
for name in blocks_attr_names:
|
||||
attr = getattr(model, name, None)
|
||||
if attr is None:
|
||||
source = "OffloadPlan.block_attrs" if explicit_attrs else "_layerwise_offload_blocks_attrs"
|
||||
raise AttributeError(
|
||||
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
|
||||
f"does not exist on model {model.__class__.__name__}"
|
||||
f"Attribute '{name}' declared in {source} does not exist on model {model.__class__.__name__}"
|
||||
)
|
||||
try:
|
||||
attr_iter = iter(attr)
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared component-plan helpers for diffusion offload backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.distributed.tensor import DTensor
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .block_discovery import get_blocks_from_dit
|
||||
from .config import DIT_COMPONENT, TEXT_ENCODER_COMPONENT
|
||||
from .offload_plan import OffloadPlan
|
||||
from .tensor_utils import set_tensor_storage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import OffloadConfig
|
||||
from .module_collector import PipelineModules
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def encoder_component_type(name: str, plan: OffloadPlan | None) -> str | None:
|
||||
"""Map a discovered encoder path to its public offload component."""
|
||||
declared = None if plan is None else plan.encoder_component_types.get(name)
|
||||
if declared is not None:
|
||||
if declared != TEXT_ENCODER_COMPONENT:
|
||||
raise ValueError(f"OffloadPlan maps encoder {name!r} to unknown component {declared!r}")
|
||||
return declared
|
||||
|
||||
leaf_name = name.rsplit(".", 1)[-1]
|
||||
if leaf_name.startswith(TEXT_ENCODER_COMPONENT) or leaf_name.endswith(TEXT_ENCODER_COMPONENT):
|
||||
return TEXT_ENCODER_COMPONENT
|
||||
return None
|
||||
|
||||
|
||||
def get_encoder_block_groups(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
plan: OffloadPlan | None,
|
||||
*,
|
||||
strict: bool = False,
|
||||
) -> list[nn.ModuleList]:
|
||||
"""Resolve the streamable block lists declared for one encoder."""
|
||||
if plan is None:
|
||||
return []
|
||||
|
||||
# Some distributed models expose an unloaded stub on ranks where the
|
||||
# component never executes. That is a valid local no-op even when the
|
||||
# component was selected explicitly.
|
||||
if getattr(module, "is_loaded", True) is False:
|
||||
return []
|
||||
|
||||
groups: list[nn.ModuleList] = []
|
||||
for block_path in plan.encoder_block_attrs.get(name, ()):
|
||||
try:
|
||||
blocks = attrgetter(block_path)(module)
|
||||
except AttributeError:
|
||||
if strict:
|
||||
raise ValueError(f"Encoder offload path {name}.{block_path} was not found") from None
|
||||
logger.warning("Encoder offload path %s.%s was not found", name, block_path)
|
||||
continue
|
||||
if not isinstance(blocks, nn.ModuleList) or len(blocks) <= 1:
|
||||
if strict:
|
||||
raise ValueError(f"Encoder offload path {name}.{block_path} is not a streamable block list")
|
||||
logger.warning("Encoder offload path %s.%s is not a streamable block list", name, block_path)
|
||||
continue
|
||||
groups.append(blocks)
|
||||
return groups
|
||||
|
||||
|
||||
def iter_streamable_dits(
|
||||
modules: PipelineModules,
|
||||
config: OffloadConfig,
|
||||
device: torch.device,
|
||||
plan: OffloadPlan | None,
|
||||
) -> Iterator[tuple[str, nn.Module, list[str], list[nn.Module]]]:
|
||||
"""Yield selected DiTs whose block metadata resolves successfully."""
|
||||
if not config.offloads(DIT_COMPONENT):
|
||||
return
|
||||
for name, module in zip(modules.dit_names, modules.dits):
|
||||
logger.info("Applying hooks on %s (%s)", name, module.__class__.__name__)
|
||||
planned_attrs = None if plan is None else plan.block_attrs.get(name)
|
||||
block_attrs, blocks = get_blocks_from_dit(module, planned_attrs)
|
||||
if blocks:
|
||||
yield name, module, block_attrs, blocks
|
||||
continue
|
||||
if config.components is not None:
|
||||
raise ValueError(f"Selected DiT {name!r} has no streamable layerwise-offload blocks")
|
||||
logger.warning("Target layers (blocks) not found. Skipping offloading on %s (%s)", name, type(module).__name__)
|
||||
module.to(device)
|
||||
|
||||
|
||||
def move_non_block_state_to_device(
|
||||
module: nn.Module,
|
||||
block_groups: list[nn.ModuleList],
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""Keep component state outside streamed block lists resident on device."""
|
||||
block_tensors = {
|
||||
id(tensor)
|
||||
for blocks in block_groups
|
||||
for block in blocks
|
||||
for tensor in chain(block.parameters(), block.buffers())
|
||||
}
|
||||
for tensor in chain(module.parameters(), module.buffers()):
|
||||
if id(tensor) in block_tensors:
|
||||
continue
|
||||
local = tensor.to_local() if isinstance(tensor, DTensor) else tensor
|
||||
if local.device != device:
|
||||
set_tensor_storage(tensor, local.to(device, non_blocking=True))
|
||||
|
||||
|
||||
def set_encoder_layerwise_state(
|
||||
module: nn.Module,
|
||||
hooks: list[Any],
|
||||
block_groups: list[nn.ModuleList],
|
||||
) -> None:
|
||||
"""Publish the backend-neutral state used by encoder stage lifecycles."""
|
||||
module._omni_layerwise_hooks = hooks
|
||||
module._omni_layerwise_block_groups = block_groups
|
||||
module._omni_layerwise_enabled = True
|
||||
|
||||
|
||||
def clear_encoder_layerwise_state(module: nn.Module) -> None:
|
||||
"""Clear encoder layerwise state after its backend hooks are removed."""
|
||||
module._omni_layerwise_hooks = []
|
||||
module._omni_layerwise_block_groups = []
|
||||
module._omni_layerwise_enabled = False
|
||||
|
||||
|
||||
def validate_on_demand_component(module: nn.Module, name: str) -> None:
|
||||
"""Require the explicit lifecycle used by pipeline-managed components."""
|
||||
if not callable(getattr(module, "load_to_device", None)) or not callable(getattr(module, "offload_to_cpu", None)):
|
||||
raise ValueError(
|
||||
f"Component {name!r} declares on-demand offload but must implement load_to_device() and offload_to_cpu()"
|
||||
)
|
||||
|
||||
|
||||
def prepare_component(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
stage_on_demand: bool,
|
||||
blockwise: bool,
|
||||
staged_components: list[nn.Module],
|
||||
) -> None:
|
||||
"""Stage a selected component or keep its non-streamed form resident."""
|
||||
if stage_on_demand:
|
||||
validate_on_demand_component(module, name)
|
||||
getattr(module, "offload_to_cpu")()
|
||||
staged_components.append(module)
|
||||
logger.info("Prepared %s for pipeline-managed staged offload", name)
|
||||
elif not blockwise:
|
||||
module.to(device)
|
||||
|
||||
|
||||
def prepare_pipeline_components(
|
||||
modules: PipelineModules,
|
||||
config: OffloadConfig,
|
||||
plan: OffloadPlan | None,
|
||||
*,
|
||||
device: torch.device,
|
||||
staged_components: list[nn.Module],
|
||||
enable_encoder_blocks: Callable[[nn.Module, str, OffloadPlan | None, bool], bool],
|
||||
) -> None:
|
||||
"""Apply the shared encoder/VAE/resident placement policy."""
|
||||
if config.components is not None and config.offloads(TEXT_ENCODER_COMPONENT):
|
||||
selected_encoder_names = [name for name in modules.encoder_names if config.offloads_encoder(name, plan)]
|
||||
if not selected_encoder_names:
|
||||
raise ValueError("No text encoder modules found for selected text_encoder offload")
|
||||
|
||||
if plan is not None:
|
||||
for encoder, name in zip(modules.encoders, modules.encoder_names):
|
||||
if config.should_offload_encoder(name, plan) and name in plan.on_demand_component_paths:
|
||||
validate_on_demand_component(encoder, name)
|
||||
|
||||
for encoder, name in zip(modules.encoders, modules.encoder_names):
|
||||
selected = config.should_offload_encoder(name, plan)
|
||||
stage_on_demand = bool(selected and plan is not None and name in plan.on_demand_component_paths)
|
||||
blockwise = selected and enable_encoder_blocks(encoder, name, plan, stage_on_demand)
|
||||
if stage_on_demand and not blockwise and config.uses_allgather(TEXT_ENCODER_COMPONENT):
|
||||
raise ValueError(
|
||||
f"Text encoder {name!r} cannot use AllGather without a model-declared streamable block plan"
|
||||
)
|
||||
if selected and config.components is not None and not (blockwise or stage_on_demand):
|
||||
raise ValueError(f"Selected text encoder {name!r} requires a model-declared streamable or on-demand plan")
|
||||
prepare_component(
|
||||
encoder,
|
||||
name,
|
||||
device=device,
|
||||
stage_on_demand=stage_on_demand,
|
||||
blockwise=blockwise,
|
||||
staged_components=staged_components,
|
||||
)
|
||||
|
||||
for vae, name in zip(modules.vaes, modules.vae_names):
|
||||
legacy_staged = config.components is None and plan is not None and name in plan.on_demand_component_paths
|
||||
prepare_component(
|
||||
vae,
|
||||
name,
|
||||
device=device,
|
||||
stage_on_demand=legacy_staged,
|
||||
blockwise=False,
|
||||
staged_components=staged_components,
|
||||
)
|
||||
|
||||
for name, module in zip(modules.resident_names, modules.resident_modules):
|
||||
module.to(device)
|
||||
logger.debug("Moved resident module %s to %s", name, device)
|
||||
|
||||
if not config.offloads(DIT_COMPONENT):
|
||||
for dit in modules.dits:
|
||||
dit.to(device)
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Public diffusion CPU-offload configuration helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Collection, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, TypeVar
|
||||
|
||||
DIT_COMPONENT = "dit"
|
||||
TEXT_ENCODER_COMPONENT = "text_encoder"
|
||||
OFFLOAD_COMPONENTS = frozenset({DIT_COMPONENT, TEXT_ENCODER_COMPONENT})
|
||||
DEFAULT_OFFLOAD_COMPONENTS = frozenset({DIT_COMPONENT})
|
||||
|
||||
_KeyT = TypeVar("_KeyT")
|
||||
_ValueT = TypeVar("_ValueT")
|
||||
|
||||
|
||||
class _FrozenMapping(Mapping[_KeyT, _ValueT]):
|
||||
"""Small immutable mapping that remains safe to pickle between workers."""
|
||||
|
||||
def __init__(self, value: Mapping[_KeyT, _ValueT]) -> None:
|
||||
self._items = tuple(value.items())
|
||||
|
||||
def __getitem__(self, key: _KeyT) -> _ValueT:
|
||||
for candidate, value in self._items:
|
||||
if candidate == key:
|
||||
return value
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self) -> Iterator[_KeyT]:
|
||||
return (key for key, _ in self._items)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._items)
|
||||
|
||||
def __reduce__(self):
|
||||
return type(self), (dict(self._items),)
|
||||
|
||||
|
||||
class OffloadMode(str, Enum):
|
||||
"""User-facing offload granularity."""
|
||||
|
||||
MODULE = "module"
|
||||
LAYER = "layer"
|
||||
|
||||
|
||||
class OffloadStrategy(str, Enum):
|
||||
"""Resolved internal backend strategy."""
|
||||
|
||||
NONE = "none"
|
||||
MODEL_LEVEL = "model_level"
|
||||
LAYER_WISE = "layer_wise"
|
||||
DISTRIBUTED_LAYER_WISE = "distributed_layer_wise"
|
||||
|
||||
|
||||
class DLOTransfer(str, Enum):
|
||||
"""How one component's next block reaches the device."""
|
||||
|
||||
ALLGATHER = "allgather"
|
||||
RANK_LOCAL = "rank-local"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LayerOffloadOptions:
|
||||
"""Layer-mode settings for one selected diffusion component."""
|
||||
|
||||
weight_transfer: DLOTransfer | None = None
|
||||
resident_layers: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedDiffusionOffloadConfig:
|
||||
"""Validated internal form of the raw public offload mapping."""
|
||||
|
||||
mode: OffloadMode
|
||||
components: frozenset[str]
|
||||
layer_options: Mapping[str, LayerOffloadOptions]
|
||||
pin_memory: bool | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "layer_options", _FrozenMapping(self.layer_options))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedOffload:
|
||||
"""Canonical offload policy derived once from public or legacy inputs."""
|
||||
|
||||
strategy: OffloadStrategy
|
||||
public: ParsedDiffusionOffloadConfig | None
|
||||
components: frozenset[str]
|
||||
transfers: Mapping[str, DLOTransfer]
|
||||
pin_memory: bool
|
||||
resident_layers: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "transfers", _FrozenMapping(self.transfers))
|
||||
|
||||
def offloads(self, component: str) -> bool:
|
||||
return component in self.components
|
||||
|
||||
def uses_allgather(self, component: str = DIT_COMPONENT) -> bool:
|
||||
try:
|
||||
return self.transfers[component] is DLOTransfer.ALLGATHER
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unknown offload component {component!r}") from exc
|
||||
|
||||
@property
|
||||
def any_allgather(self) -> bool:
|
||||
return self.strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE and any(
|
||||
self.uses_allgather(component) for component in self.components
|
||||
)
|
||||
|
||||
|
||||
_LEGACY_STRATEGY_FIELDS = {
|
||||
"enable_cpu_offload": OffloadStrategy.MODEL_LEVEL,
|
||||
"enable_layerwise_offload": OffloadStrategy.LAYER_WISE,
|
||||
"enable_distributed_layerwise_offload": OffloadStrategy.DISTRIBUTED_LAYER_WISE,
|
||||
}
|
||||
_LEGACY_STRATEGY_PRIORITY = (
|
||||
OffloadStrategy.DISTRIBUTED_LAYER_WISE,
|
||||
OffloadStrategy.LAYER_WISE,
|
||||
OffloadStrategy.MODEL_LEVEL,
|
||||
)
|
||||
|
||||
|
||||
_EnumT = TypeVar("_EnumT", bound=Enum)
|
||||
|
||||
|
||||
def _parse_enum(value: Any, enum_type: type[_EnumT], label: str) -> _EnumT:
|
||||
if isinstance(value, enum_type):
|
||||
return value
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{label} must be a string, got {type(value).__name__}")
|
||||
try:
|
||||
return enum_type(value)
|
||||
except ValueError as exc:
|
||||
choices = ", ".join(item.value for item in enum_type)
|
||||
raise ValueError(f"Unknown {label} {value!r}; choose from: {choices}") from exc
|
||||
|
||||
|
||||
def _validate_string_keys(value: Mapping[Any, Any], label: str) -> None:
|
||||
if any(not isinstance(key, str) for key in value):
|
||||
raise TypeError(f"{label} keys must be strings")
|
||||
|
||||
|
||||
def _parse_layer_options(component: str, value: Any) -> LayerOffloadOptions:
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(
|
||||
f"diffusion_offload_config.layer_options[{component!r}] must be a mapping, got {type(value).__name__}"
|
||||
)
|
||||
_validate_string_keys(value, f"diffusion_offload_config.layer_options[{component!r}]")
|
||||
unknown = sorted(set(value) - {"weight_transfer", "resident_layers"})
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Unknown diffusion offload setting(s) for {component}: {', '.join(unknown)}; "
|
||||
"choose from: weight_transfer, resident_layers"
|
||||
)
|
||||
weight_transfer = (
|
||||
_parse_enum(value["weight_transfer"], DLOTransfer, "offload transfer") if "weight_transfer" in value else None
|
||||
)
|
||||
resident_layers = value.get("resident_layers", 0)
|
||||
|
||||
if type(resident_layers) is not int or resident_layers < 0:
|
||||
raise ValueError(f"resident_layers for {component} must be a non-negative integer")
|
||||
parsed = LayerOffloadOptions(
|
||||
weight_transfer=weight_transfer,
|
||||
resident_layers=resident_layers,
|
||||
)
|
||||
if component != DIT_COMPONENT and parsed.resident_layers:
|
||||
raise ValueError("resident_layers currently supports only the 'dit' component")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_diffusion_offload_config(value: Any) -> ParsedDiffusionOffloadConfig | None:
|
||||
"""Validate the compact ``diffusion_offload_config`` public API."""
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(f"diffusion_offload_config must be a mapping, got {type(value).__name__}")
|
||||
|
||||
_validate_string_keys(value, "diffusion_offload_config")
|
||||
unknown = sorted(set(value) - {"mode", "components", "layer_options", "pin_memory"})
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Unknown diffusion_offload_config field(s): {', '.join(unknown)}; "
|
||||
"choose from: mode, components, layer_options, pin_memory"
|
||||
)
|
||||
if "mode" not in value:
|
||||
raise ValueError("diffusion_offload_config requires 'mode'")
|
||||
if "components" not in value:
|
||||
raise ValueError("diffusion_offload_config requires 'components'")
|
||||
|
||||
mode = _parse_enum(value["mode"], OffloadMode, "diffusion offload mode")
|
||||
components = parse_offload_components(value["components"])
|
||||
|
||||
raw_layer_options = value.get("layer_options", {})
|
||||
if not isinstance(raw_layer_options, Mapping):
|
||||
raise TypeError("diffusion_offload_config.layer_options must be a mapping")
|
||||
_validate_string_keys(raw_layer_options, "diffusion_offload_config.layer_options")
|
||||
unselected_options = sorted(set(raw_layer_options) - components)
|
||||
if unselected_options:
|
||||
raise ValueError(
|
||||
"diffusion_offload_config.layer_options requires selecting the same component(s): "
|
||||
+ ", ".join(unselected_options)
|
||||
)
|
||||
layer_options = {
|
||||
component: _parse_layer_options(component, raw_layer_options.get(component, {})) for component in components
|
||||
}
|
||||
|
||||
pin_memory = value.get("pin_memory")
|
||||
if pin_memory is not None and type(pin_memory) is not bool:
|
||||
raise TypeError("diffusion_offload_config.pin_memory must be a bool")
|
||||
|
||||
if mode is OffloadMode.MODULE and raw_layer_options:
|
||||
raise ValueError(
|
||||
"diffusion_offload_config.layer_options requires mode='layer'; "
|
||||
f"configured for: {', '.join(sorted(raw_layer_options))}"
|
||||
)
|
||||
dit = layer_options.get(DIT_COMPONENT)
|
||||
if dit is not None and dit.resident_layers and dit.weight_transfer is DLOTransfer.ALLGATHER:
|
||||
raise ValueError("resident_layers requires dit.weight_transfer='rank-local'")
|
||||
|
||||
return ParsedDiffusionOffloadConfig(
|
||||
mode=mode,
|
||||
components=components,
|
||||
layer_options=layer_options,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
|
||||
|
||||
def _legacy_strategy(config: Any) -> OffloadStrategy:
|
||||
enabled = {strategy for field, strategy in _LEGACY_STRATEGY_FIELDS.items() if bool(getattr(config, field, False))}
|
||||
return next((strategy for strategy in _LEGACY_STRATEGY_PRIORITY if strategy in enabled), OffloadStrategy.NONE)
|
||||
|
||||
|
||||
def _public_strategy(public: ParsedDiffusionOffloadConfig) -> OffloadStrategy:
|
||||
if public.mode is OffloadMode.MODULE:
|
||||
return OffloadStrategy.MODEL_LEVEL
|
||||
|
||||
needs_distributed_backend = any(
|
||||
settings.weight_transfer is DLOTransfer.ALLGATHER or settings.resident_layers
|
||||
for settings in public.layer_options.values()
|
||||
)
|
||||
return OffloadStrategy.DISTRIBUTED_LAYER_WISE if needs_distributed_backend else OffloadStrategy.LAYER_WISE
|
||||
|
||||
|
||||
def _validate_legacy_layer_options(config: Any, public: ParsedDiffusionOffloadConfig | None) -> None:
|
||||
"""Reject ambiguous or invalid compatibility DLO tuning before loading."""
|
||||
resident_layers = getattr(config, "dlo_resident_layers", 0)
|
||||
if type(resident_layers) is not int or resident_layers < 0:
|
||||
raise ValueError(f"dlo_resident_layers must be a non-negative integer, got {resident_layers!r}")
|
||||
|
||||
if public is not None:
|
||||
if getattr(config, "host_weight_runtime_mode", "disabled") != "disabled":
|
||||
raise ValueError("diffusion_offload_config cannot be combined with Host Weight Runtime")
|
||||
conflicting_fields = []
|
||||
if getattr(config, "dlo_use_allgather", True) is not True:
|
||||
conflicting_fields.append("dlo_use_allgather")
|
||||
if resident_layers:
|
||||
conflicting_fields.append("dlo_resident_layers")
|
||||
if float(getattr(config, "dlo_host_registration_limit_gib", 0.0)) > 0:
|
||||
conflicting_fields.append("dlo_host_registration_limit_gib")
|
||||
if conflicting_fields:
|
||||
raise ValueError(
|
||||
"diffusion_offload_config cannot be combined with legacy DLO option(s): "
|
||||
+ ", ".join(conflicting_fields)
|
||||
)
|
||||
return
|
||||
|
||||
if resident_layers and bool(getattr(config, "dlo_use_allgather", True)):
|
||||
raise ValueError(
|
||||
"dlo_resident_layers requires the DiT DLO transfer to be rank-local; set dlo_use_allgather=False"
|
||||
)
|
||||
|
||||
|
||||
def resolve_offload(config: Any) -> ResolvedOffload:
|
||||
"""Resolve and cache the complete offload policy on its owning config."""
|
||||
cached = getattr(config, "_resolved_diffusion_offload", None)
|
||||
if isinstance(cached, ResolvedOffload):
|
||||
return cached
|
||||
|
||||
public = parse_diffusion_offload_config(getattr(config, "diffusion_offload_config", None))
|
||||
_validate_legacy_layer_options(config, public)
|
||||
legacy = _legacy_strategy(config)
|
||||
if public is None:
|
||||
transfers = {
|
||||
DIT_COMPONENT: (
|
||||
DLOTransfer.ALLGATHER if bool(getattr(config, "dlo_use_allgather", True)) else DLOTransfer.RANK_LOCAL
|
||||
),
|
||||
TEXT_ENCODER_COMPONENT: DLOTransfer.RANK_LOCAL,
|
||||
}
|
||||
resolved = ResolvedOffload(
|
||||
strategy=legacy,
|
||||
public=None,
|
||||
components=DEFAULT_OFFLOAD_COMPONENTS,
|
||||
transfers=transfers,
|
||||
pin_memory=bool(getattr(config, "pin_cpu_memory", True)),
|
||||
resident_layers=int(getattr(config, "dlo_resident_layers", 0)),
|
||||
)
|
||||
else:
|
||||
strategy = _public_strategy(public)
|
||||
if legacy is not OffloadStrategy.NONE and legacy is not strategy:
|
||||
raise ValueError("diffusion_offload_config cannot be combined with legacy enable_*_offload flags")
|
||||
transfers = {component: DLOTransfer.RANK_LOCAL for component in OFFLOAD_COMPONENTS}
|
||||
for component, options in public.layer_options.items():
|
||||
transfers[component] = options.weight_transfer or DLOTransfer.RANK_LOCAL
|
||||
dit_options = public.layer_options.get(DIT_COMPONENT)
|
||||
resolved = ResolvedOffload(
|
||||
strategy=strategy,
|
||||
public=public,
|
||||
components=public.components,
|
||||
transfers=transfers,
|
||||
pin_memory=(
|
||||
public.pin_memory if public.pin_memory is not None else bool(getattr(config, "pin_cpu_memory", True))
|
||||
),
|
||||
resident_layers=0 if dit_options is None else dit_options.resident_layers,
|
||||
)
|
||||
|
||||
parallel_config = getattr(config, "parallel_config", None)
|
||||
data_parallel_size = int(getattr(parallel_config, "data_parallel_size", 1))
|
||||
cache_backend = getattr(config, "cache_backend", None)
|
||||
if data_parallel_size > 1 and resolved.any_allgather and cache_backend not in (None, "none"):
|
||||
raise ValueError(
|
||||
"Diffusion cache acceleration cannot be combined with DLO AllGather "
|
||||
"across data-parallel ranks because rank-local cache decisions can skip "
|
||||
"different weight collectives. Disable cache_backend or use rank-local transfer."
|
||||
)
|
||||
if (
|
||||
data_parallel_size > 1
|
||||
and bool(getattr(config, "enable_prompt_embed_cache", False))
|
||||
and resolved.offloads(TEXT_ENCODER_COMPONENT)
|
||||
and resolved.uses_allgather(TEXT_ENCODER_COMPONENT)
|
||||
):
|
||||
raise ValueError(
|
||||
"Prompt embedding cache cannot be combined with text_encoder AllGather "
|
||||
"across data-parallel ranks: rank-local cache hits would skip different "
|
||||
"encoder collectives. Use text_encoder weight_transfer='rank-local' or "
|
||||
"disable enable_prompt_embed_cache."
|
||||
)
|
||||
|
||||
# Actual runtime configs are immutable after construction. Keeping the
|
||||
# derived object off the public dataclass fields also keeps projections and
|
||||
# serialization transport-safe while making hot-path accessors constant-time.
|
||||
try:
|
||||
setattr(config, "_resolved_diffusion_offload", resolved)
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_offload_strategy(config: Any) -> OffloadStrategy:
|
||||
"""Resolve the compact config or compatibility boolean entry points."""
|
||||
return resolve_offload(config).strategy
|
||||
|
||||
|
||||
def materialize_legacy_offload_flags(config: Any) -> OffloadStrategy:
|
||||
"""Keep existing strategy readers working after resolving the compact API."""
|
||||
resolved = resolve_offload(config)
|
||||
for field, field_strategy in _LEGACY_STRATEGY_FIELDS.items():
|
||||
setattr(config, field, resolved.strategy is field_strategy)
|
||||
setattr(config, "pin_cpu_memory", resolved.pin_memory)
|
||||
return resolved.strategy
|
||||
|
||||
|
||||
def parse_offload_components(value: Collection[str]) -> frozenset[str]:
|
||||
"""Validate an internal component collection."""
|
||||
if isinstance(value, (str, Mapping)) or not isinstance(value, Collection):
|
||||
raise TypeError("diffusion_offload_config.components must be a non-empty list of component names")
|
||||
if any(not isinstance(item, str) for item in value):
|
||||
raise TypeError("offload component entries must be strings")
|
||||
components = list(value)
|
||||
if not components:
|
||||
raise ValueError("offload components must not be empty")
|
||||
unknown = sorted(set(components) - OFFLOAD_COMPONENTS)
|
||||
if unknown:
|
||||
choices = ", ".join(sorted(OFFLOAD_COMPONENTS))
|
||||
raise ValueError(f"Unknown diffusion offload component(s): {', '.join(unknown)}; choose from: {choices}")
|
||||
return frozenset(components)
|
||||
|
||||
|
||||
def parse_dlo_transfer(value: Mapping[str, str | DLOTransfer]) -> dict[str, DLOTransfer]:
|
||||
"""Resolve an internal per-component transfer mapping."""
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(f"offload transfers must be a mapping, got {type(value).__name__}")
|
||||
_validate_string_keys(value, "offload transfers")
|
||||
missing = sorted(OFFLOAD_COMPONENTS - set(value))
|
||||
if missing:
|
||||
raise ValueError("offload transfers require every component; missing: " + ", ".join(missing))
|
||||
resolved: dict[str, DLOTransfer] = {}
|
||||
for component, raw_transfer in value.items():
|
||||
if component not in OFFLOAD_COMPONENTS:
|
||||
choices = ", ".join(sorted(OFFLOAD_COMPONENTS))
|
||||
raise ValueError(f"Unknown offload transfer component {component!r}; choose from: {choices}")
|
||||
resolved[component] = _parse_enum(raw_transfer, DLOTransfer, "offload transfer")
|
||||
return resolved
|
||||
|
||||
|
||||
def component_uses_allgather(config: Any, component: str = DIT_COMPONENT) -> bool:
|
||||
"""Return whether one selected component uses AllGather transport."""
|
||||
resolved = resolve_offload(config)
|
||||
if not resolved.offloads(component):
|
||||
raise ValueError(f"Offload component {component!r} is not selected")
|
||||
return resolved.uses_allgather(component)
|
||||
|
||||
|
||||
def selected_offload_components(config: Any) -> frozenset[str]:
|
||||
"""Resolve selected components while preserving legacy topology defaults."""
|
||||
return resolve_offload(config).components
|
||||
|
||||
|
||||
def should_offload_component(config: Any, component: str) -> bool:
|
||||
"""Return whether an active layer policy selects ``component``."""
|
||||
if component not in OFFLOAD_COMPONENTS:
|
||||
raise ValueError(f"Unknown offload component: {component}")
|
||||
resolved = resolve_offload(config)
|
||||
if resolved.strategy not in {
|
||||
OffloadStrategy.LAYER_WISE,
|
||||
OffloadStrategy.DISTRIBUTED_LAYER_WISE,
|
||||
}:
|
||||
return False
|
||||
return resolved.offloads(component)
|
||||
|
||||
|
||||
def any_selected_component_uses_allgather(config: Any) -> bool:
|
||||
"""Return whether an enabled layer backend requires weight collectives."""
|
||||
return resolve_offload(config).any_allgather
|
||||
File diff suppressed because it is too large.
Load diff
@@ -7,14 +7,41 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.distributed.tensor import DTensor
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig
|
||||
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
|
||||
from .block_discovery import (
|
||||
get_blocks_attr_names,
|
||||
get_blocks_from_dit,
|
||||
set_blocks_attr_names,
|
||||
)
|
||||
from .component_utils import (
|
||||
clear_encoder_layerwise_state,
|
||||
get_encoder_block_groups,
|
||||
iter_streamable_dits,
|
||||
move_non_block_state_to_device,
|
||||
prepare_pipeline_components,
|
||||
set_encoder_layerwise_state,
|
||||
)
|
||||
from .config import DIT_COMPONENT
|
||||
from .module_collector import ModuleDiscovery
|
||||
from .offload_plan import OffloadPlan, get_offload_plan
|
||||
from .tensor_utils import (
|
||||
clear_block_storage,
|
||||
clear_tensor_storage,
|
||||
describe_tensor_storage,
|
||||
flatten_physical_storage,
|
||||
group_named_tensors_by_dtype,
|
||||
is_materialized_tensor,
|
||||
materialization_probe,
|
||||
module_materialization_probe,
|
||||
restore_tensor_storage,
|
||||
set_tensor_storage,
|
||||
tensor_storage_metadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -41,6 +68,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
materialization_probe_tensor: torch.Tensor | None = None,
|
||||
):
|
||||
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
|
||||
|
||||
@@ -59,31 +87,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = {}
|
||||
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_dtensor(t: torch.Tensor) -> bool:
|
||||
return isinstance(t, DTensor)
|
||||
|
||||
@staticmethod
|
||||
def _set_tensor_storage(target: torch.Tensor, value: torch.Tensor) -> None:
|
||||
if LayerwiseOffloadHook._is_dtensor(target):
|
||||
target._local_tensor = value
|
||||
else:
|
||||
target.data = value
|
||||
|
||||
@staticmethod
|
||||
def _make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
|
||||
if LayerwiseOffloadHook._is_dtensor(tensor):
|
||||
local_shape = tuple(tensor.to_local().shape)
|
||||
return torch.empty(local_shape, device="meta", dtype=tensor.dtype)
|
||||
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
|
||||
|
||||
@staticmethod
|
||||
def _is_materialized_tensor(t: torch.Tensor) -> bool:
|
||||
if LayerwiseOffloadHook._is_dtensor(t):
|
||||
local_t = t.to_local()
|
||||
return not local_t.is_meta
|
||||
return not t.is_meta and t.data.numel() > 0
|
||||
self._materialization_probe = materialization_probe_tensor
|
||||
|
||||
def initialize_hook(self, module: nn.Module) -> nn.Module:
|
||||
# This all happen during the hook instance being registered to hook registry;
|
||||
@@ -92,6 +96,8 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
|
||||
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
|
||||
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
|
||||
if self._materialization_probe is None:
|
||||
self._materialization_probe = materialization_probe(self.block_parameters, self.block_buffers)
|
||||
|
||||
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
|
||||
@@ -100,7 +106,6 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
|
||||
self.next_block_parameters,
|
||||
self.next_block_buffers,
|
||||
self.device,
|
||||
self.pin_memory,
|
||||
)
|
||||
|
||||
@@ -110,7 +115,6 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
def _to_cpu(
|
||||
params: dict[str, nn.Parameter],
|
||||
bufs: dict[str, torch.Tensor],
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
|
||||
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
|
||||
@@ -123,85 +127,39 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
flattened CPU tensors by dtype,
|
||||
metadata for reconstruction by dtype
|
||||
"""
|
||||
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
|
||||
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
# NOTE: order does matter
|
||||
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
targets_to_offload: list[torch.Tensor] = []
|
||||
|
||||
for name, param_or_buf in chain(params.items(), bufs.items()):
|
||||
dtype = param_or_buf.dtype
|
||||
if dtype not in dtype_grouped_weights:
|
||||
dtype_grouped_weights[dtype] = {}
|
||||
dtype_grouped_weights[dtype][name] = param_or_buf
|
||||
|
||||
for dtype, name2weights in dtype_grouped_weights.items():
|
||||
for dtype, named_weights in group_named_tensors_by_dtype(params, bufs).items():
|
||||
# total # of parameters + buffers
|
||||
weights_with_local = []
|
||||
for name, t in name2weights.items():
|
||||
local_t = t.to_local() if hasattr(t, "to_local") else t
|
||||
stride = local_t.stride()
|
||||
storage_numel = (
|
||||
0
|
||||
if local_t.numel() == 0
|
||||
else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride))
|
||||
)
|
||||
weights_with_local.append((name, t, local_t, storage_numel, stride))
|
||||
total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local)
|
||||
specs = describe_tensor_storage(named_weights)
|
||||
total_numel = sum(spec.storage_numel for spec in specs)
|
||||
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
|
||||
|
||||
current_offset = 0
|
||||
for (
|
||||
name,
|
||||
original_tensor,
|
||||
local_tensor,
|
||||
storage_numel,
|
||||
stride,
|
||||
) in weights_with_local:
|
||||
if local_tensor.is_contiguous():
|
||||
flat_storage = local_tensor.flatten()
|
||||
else:
|
||||
# Cutlass FP8 weights use a transposed physical layout.
|
||||
# Preserve it across the flattened CPU staging buffer.
|
||||
flat_storage = torch.zeros(
|
||||
storage_numel,
|
||||
dtype=dtype,
|
||||
device=local_tensor.device,
|
||||
)
|
||||
physical_view = torch.as_strided(
|
||||
flat_storage,
|
||||
size=local_tensor.shape,
|
||||
stride=stride,
|
||||
)
|
||||
physical_view.copy_(local_tensor)
|
||||
cpu_tensor[current_offset : current_offset + storage_numel].copy_(flat_storage)
|
||||
if dtype not in dtype_metadata:
|
||||
dtype_metadata[dtype] = []
|
||||
dtype_metadata[dtype].append(
|
||||
{
|
||||
"name": name,
|
||||
"offset": current_offset,
|
||||
"numel": storage_numel,
|
||||
"shape": local_tensor.shape,
|
||||
"stride": stride,
|
||||
}
|
||||
for spec in specs:
|
||||
flat_storage = flatten_physical_storage(spec.value, spec.storage_numel)
|
||||
cpu_tensor[current_offset : current_offset + spec.storage_numel].copy_(flat_storage)
|
||||
dtype_metadata.setdefault(dtype, []).append(
|
||||
tensor_storage_metadata(spec, current_offset, include_device=True)
|
||||
)
|
||||
|
||||
LayerwiseOffloadHook._set_tensor_storage(
|
||||
original_tensor, LayerwiseOffloadHook._make_offload_placeholder(original_tensor)
|
||||
)
|
||||
current_offset += storage_numel
|
||||
targets_to_offload.append(spec.target)
|
||||
current_offset += spec.storage_numel
|
||||
|
||||
dtype_cpu_flattened_weights[dtype] = cpu_tensor
|
||||
|
||||
# Do not mutate the module until every host master has been built.
|
||||
# Allocation/copy failures therefore leave the input module intact.
|
||||
clear_tensor_storage(targets_to_offload)
|
||||
|
||||
return dtype_cpu_flattened_weights, dtype_metadata
|
||||
|
||||
@property
|
||||
def is_materialized(self) -> bool:
|
||||
"""Check whether this block's parameters hold real data on device."""
|
||||
for param in self.block_parameters.values():
|
||||
return LayerwiseOffloadHook._is_materialized_tensor(param)
|
||||
|
||||
return True
|
||||
return self._materialization_probe is None or is_materialized_tensor(self._materialization_probe)
|
||||
|
||||
@torch.compiler.disable
|
||||
def prefetch_layer(self, non_blocking: bool = True) -> None:
|
||||
@@ -236,7 +194,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
|
||||
)
|
||||
|
||||
LayerwiseOffloadHook._set_tensor_storage(
|
||||
set_tensor_storage(
|
||||
target_param_or_buf,
|
||||
torch.as_strided(
|
||||
gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]],
|
||||
@@ -252,17 +210,27 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
"""Free GPU memory for layer by replacing tensors with empty placeholders.
|
||||
This function does not actually offload weights from GPU back to CPU.
|
||||
"""
|
||||
evt = self._prefetch_done
|
||||
if evt is not None:
|
||||
current_omni_platform.current_stream().wait_event(evt)
|
||||
|
||||
clear_block_storage(self.block_parameters, self.block_buffers, self._prefetch_done)
|
||||
self._prefetch_done = None
|
||||
|
||||
# free GPU residency
|
||||
for _, param in self.block_parameters.items():
|
||||
LayerwiseOffloadHook._set_tensor_storage(param, LayerwiseOffloadHook._make_offload_placeholder(param))
|
||||
for _, buf in self.block_buffers.items():
|
||||
LayerwiseOffloadHook._set_tensor_storage(buf, LayerwiseOffloadHook._make_offload_placeholder(buf))
|
||||
@torch.compiler.disable
|
||||
def restore_next_block(self) -> None:
|
||||
"""Detach the next block from this hook's host backing store."""
|
||||
for dtype, ordered_metadata in self.dtype_metadata.items():
|
||||
flat = self.dtype_cpu_flattened_weights[dtype]
|
||||
for metadata in ordered_metadata:
|
||||
value = torch.as_strided(
|
||||
flat[metadata["offset"] : metadata["offset"] + metadata["numel"]],
|
||||
size=metadata["shape"],
|
||||
stride=metadata["stride"],
|
||||
)
|
||||
target_name = metadata["name"]
|
||||
target = (
|
||||
self.next_block_parameters[target_name]
|
||||
if target_name in self.next_block_parameters
|
||||
else self.next_block_buffers[target_name]
|
||||
)
|
||||
restore_tensor_storage(target, value, device=metadata["device"])
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
|
||||
# if the previous hook was skipped and the weights are not on device,
|
||||
@@ -287,9 +255,17 @@ def apply_block_hook(
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
*,
|
||||
materialization_probe_tensor: torch.Tensor | None = None,
|
||||
) -> LayerwiseOffloadHook:
|
||||
registry = HookRegistry.get_or_create(module)
|
||||
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
|
||||
hook = LayerwiseOffloadHook(
|
||||
next_block,
|
||||
device,
|
||||
stream,
|
||||
pin_memory,
|
||||
materialization_probe_tensor,
|
||||
)
|
||||
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
|
||||
|
||||
return hook
|
||||
@@ -302,6 +278,135 @@ def remove_block_hook(module: nn.Module) -> None:
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
|
||||
|
||||
def _install_layerwise_hook_group(
|
||||
blocks: list[nn.Module] | nn.ModuleList,
|
||||
device: torch.device,
|
||||
stream: Any,
|
||||
pin_memory: bool,
|
||||
) -> list[LayerwiseOffloadHook]:
|
||||
"""Install one circular hook ring and roll it back transactionally."""
|
||||
block_list = list(blocks)
|
||||
if len(block_list) <= 1:
|
||||
raise ValueError("A layerwise hook group requires at least two blocks")
|
||||
|
||||
hooks: list[LayerwiseOffloadHook] = []
|
||||
hooked_blocks: list[nn.Module] = []
|
||||
probes = {id(block): module_materialization_probe(block) for block in block_list}
|
||||
try:
|
||||
for block, next_block in zip(
|
||||
chain((block_list[-1],), block_list[:-1]),
|
||||
block_list,
|
||||
strict=True,
|
||||
):
|
||||
hooks.append(
|
||||
apply_block_hook(
|
||||
block,
|
||||
next_block,
|
||||
device,
|
||||
stream,
|
||||
pin_memory,
|
||||
materialization_probe_tensor=probes[id(block)],
|
||||
)
|
||||
)
|
||||
hooked_blocks.append(block)
|
||||
except BaseException:
|
||||
run_cleanup_steps(
|
||||
[
|
||||
*(("restoring a partially installed layerwise block", hook.restore_next_block) for hook in hooks),
|
||||
*(
|
||||
("removing a partially installed layerwise hook", lambda block=block: remove_block_hook(block))
|
||||
for block in hooked_blocks
|
||||
),
|
||||
]
|
||||
)
|
||||
raise
|
||||
|
||||
for index, hook in enumerate(hooks):
|
||||
hook._prev_hook = hooks[index - 1]
|
||||
return hooks
|
||||
|
||||
|
||||
def enable_plan_encoder_layerwise_offload(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
plan: OffloadPlan | None,
|
||||
*,
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream,
|
||||
pin_memory: bool,
|
||||
stage_on_demand: bool = False,
|
||||
strict: bool = False,
|
||||
) -> bool:
|
||||
"""Apply rank-local layerwise hooks to plan-declared encoder stacks."""
|
||||
if getattr(module, "_omni_layerwise_enabled", False):
|
||||
return True
|
||||
|
||||
hooks: list[LayerwiseOffloadHook] = []
|
||||
hooked_blocks: list[nn.Module] = []
|
||||
block_groups = get_encoder_block_groups(
|
||||
module,
|
||||
name,
|
||||
plan,
|
||||
strict=strict,
|
||||
)
|
||||
if not block_groups:
|
||||
return False
|
||||
try:
|
||||
for blocks in block_groups:
|
||||
group_hooks = _install_layerwise_hook_group(blocks, device, stream, pin_memory)
|
||||
hooks.extend(group_hooks)
|
||||
hooked_blocks.extend(blocks)
|
||||
if not stage_on_demand:
|
||||
move_non_block_state_to_device(module, block_groups, device)
|
||||
except BaseException:
|
||||
run_cleanup_steps(
|
||||
[
|
||||
*(("restoring a partially installed encoder block", hook.restore_next_block) for hook in hooks),
|
||||
*(
|
||||
("removing a partially installed encoder hook", lambda block=block: remove_block_hook(block))
|
||||
for block in hooked_blocks
|
||||
),
|
||||
]
|
||||
)
|
||||
raise
|
||||
set_encoder_layerwise_state(
|
||||
module,
|
||||
hooks,
|
||||
block_groups,
|
||||
)
|
||||
logger.info(
|
||||
"Enabled rank-local layerwise offload for encoder %s (%d blocks across %d stacks)",
|
||||
name,
|
||||
sum(len(blocks) for blocks in block_groups),
|
||||
len(block_groups),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def disable_plan_encoder_layerwise_offload(
|
||||
module: nn.Module,
|
||||
*,
|
||||
restore_weights: bool = True,
|
||||
) -> None:
|
||||
"""Remove hooks installed by :func:`enable_plan_encoder_layerwise_offload`."""
|
||||
if not getattr(module, "_omni_layerwise_enabled", False):
|
||||
return
|
||||
hooks = getattr(module, "_omni_layerwise_hooks", [])
|
||||
block_groups = getattr(module, "_omni_layerwise_block_groups", [])
|
||||
steps = []
|
||||
if restore_weights:
|
||||
steps.extend(("restoring an encoder block", hook.restore_next_block) for hook in hooks)
|
||||
steps.extend(
|
||||
("removing an encoder hook", lambda block=block: remove_block_hook(block))
|
||||
for blocks in block_groups
|
||||
for block in blocks
|
||||
)
|
||||
cleanup_error = run_cleanup_steps(steps)
|
||||
if cleanup_error is not None:
|
||||
raise RuntimeError("Failed to fully disable encoder layerwise offload") from cleanup_error
|
||||
clear_encoder_layerwise_state(module)
|
||||
|
||||
|
||||
class LayerWiseOffloadBackend(OffloadBackend):
|
||||
"""Layer-wise (block-level) offloading backend.
|
||||
|
||||
@@ -315,56 +420,88 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
|
||||
self.copy_stream = current_omni_platform.Stream()
|
||||
self._blocks: list[list[nn.Module]] = []
|
||||
self._dit_hooks: list[LayerwiseOffloadHook] = []
|
||||
self._hooked_dit_blocks: list[nn.Module] = []
|
||||
self._encoder_modules: list[nn.Module] = []
|
||||
self._staged_components: list[nn.Module] = []
|
||||
|
||||
def enable(self, pipeline: nn.Module) -> None:
|
||||
try:
|
||||
self._enable(pipeline)
|
||||
except BaseException:
|
||||
try:
|
||||
self.disable()
|
||||
except BaseException:
|
||||
logger.exception("LayerWiseOffloadBackend cleanup failed while handling an enable failure")
|
||||
raise
|
||||
|
||||
def _enable(self, pipeline: nn.Module) -> None:
|
||||
if self.enabled:
|
||||
logger.warning("LayerWiseOffloadBackend already enabled")
|
||||
return
|
||||
|
||||
modules = ModuleDiscovery.discover(pipeline)
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
|
||||
plan = get_offload_plan(pipeline)
|
||||
if not modules.dits and self.config.offloads(DIT_COMPONENT):
|
||||
message = "No DiT/transformer modules found for selected DiT layerwise offload"
|
||||
if self.config.components is not None:
|
||||
raise ValueError(message)
|
||||
logger.warning(message)
|
||||
return
|
||||
|
||||
# Move encoders to GPU (they stay resident)
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
def enable_encoder_blocks(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
component_plan: OffloadPlan | None,
|
||||
stage_on_demand: bool,
|
||||
) -> bool:
|
||||
enabled = enable_plan_encoder_layerwise_offload(
|
||||
module,
|
||||
name,
|
||||
component_plan,
|
||||
device=self.device,
|
||||
stream=self.copy_stream,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
stage_on_demand=stage_on_demand,
|
||||
strict=self.config.components is not None,
|
||||
)
|
||||
if enabled:
|
||||
# Record each successful installation immediately so a later
|
||||
# component failure can remove these hooks transactionally.
|
||||
self._encoder_modules.append(module)
|
||||
return enabled
|
||||
|
||||
# Move VAE(s) to GPU if available
|
||||
for vae in modules.vaes:
|
||||
try:
|
||||
vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
prepare_pipeline_components(
|
||||
modules,
|
||||
self.config,
|
||||
plan,
|
||||
device=self.device,
|
||||
staged_components=self._staged_components,
|
||||
enable_encoder_blocks=enable_encoder_blocks,
|
||||
)
|
||||
|
||||
# Move resident modules to GPU (small modules needed every forward)
|
||||
for name, module in zip(modules.resident_names, modules.resident_modules):
|
||||
try:
|
||||
module.to(self.device)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
|
||||
if not self.config.offloads(DIT_COMPONENT):
|
||||
self.enabled = bool(self._encoder_modules or self._staged_components)
|
||||
if not self.enabled:
|
||||
raise ValueError(
|
||||
"None of the selected layerwise offload components have "
|
||||
"a model-declared streamable or on-demand plan"
|
||||
)
|
||||
return
|
||||
|
||||
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
|
||||
|
||||
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
|
||||
# Note that there might exist multiple DiT models in specific pipelines
|
||||
for i, dit_module in enumerate(modules.dits):
|
||||
dit_name = modules.dit_names[i]
|
||||
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
|
||||
|
||||
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
|
||||
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
dit_module.__class__.__name__,
|
||||
)
|
||||
dit_module.to(self.device)
|
||||
continue
|
||||
|
||||
for dit_name, dit_module, blocks_attr_names, blocks in iter_streamable_dits(
|
||||
modules, self.config, self.device, plan
|
||||
):
|
||||
num_blocks = len(blocks)
|
||||
if num_blocks <= 1:
|
||||
if self.config.components is not None:
|
||||
raise ValueError(
|
||||
f"Selected DiT {dit_name!r} requires at least two streamable layerwise-offload blocks"
|
||||
)
|
||||
logger.warning(
|
||||
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
@@ -390,137 +527,66 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
if buffer is not None:
|
||||
buffer.data = buffer.data.to(self.device, non_blocking=True)
|
||||
|
||||
# Pre-fetch the first layer by manually calling the hook function on the last layer;
|
||||
# For subsequent requests, the first layer/block will be pre-fetched
|
||||
# during the last layer compute of the previous request.
|
||||
last_block, first_block = blocks[-1], blocks[0]
|
||||
last_hook = apply_block_hook(
|
||||
last_block,
|
||||
first_block,
|
||||
block_hooks = _install_layerwise_hook_group(
|
||||
blocks,
|
||||
self.device,
|
||||
self.copy_stream,
|
||||
self.config.pin_cpu_memory,
|
||||
)
|
||||
last_hook.prefetch_layer(non_blocking=False)
|
||||
self._dit_hooks.extend(block_hooks)
|
||||
self._hooked_dit_blocks.extend(blocks)
|
||||
|
||||
block_hooks: list[LayerwiseOffloadHook] = [last_hook]
|
||||
# Register hook for each of blocks
|
||||
for i, block in enumerate(blocks[:-1]):
|
||||
next_block = blocks[(i + 1) % num_blocks]
|
||||
hook = apply_block_hook(
|
||||
block,
|
||||
next_block,
|
||||
self.device,
|
||||
self.copy_stream,
|
||||
self.config.pin_cpu_memory,
|
||||
)
|
||||
block_hooks.append(hook)
|
||||
|
||||
# NOTE(yuanheng-zhao): We make each hook gets a backward reference to the hook
|
||||
# that is responsible for prefetching its block's weights. This is specifically a
|
||||
# workaround for that arbitrary blocks are skipped by caching systems (e.g., cache-dit)
|
||||
for i in range(len(block_hooks)):
|
||||
block_hooks[i]._prev_hook = block_hooks[i - 1]
|
||||
# The last block owns block zero's host backing. Materialize block
|
||||
# zero once; later denoising iterations prefetch it from the ring.
|
||||
block_hooks[0].prefetch_layer(non_blocking=False)
|
||||
|
||||
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
|
||||
|
||||
# Track hooked blocks for cleanup
|
||||
self._blocks.append(blocks)
|
||||
|
||||
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
|
||||
self.enabled = True
|
||||
self.enabled = bool(self._blocks or self._encoder_modules or self._staged_components)
|
||||
|
||||
def disable(self) -> None:
|
||||
if not self.enabled:
|
||||
def _disable(self, *, restore_weights: bool) -> None:
|
||||
if not self.enabled and not (
|
||||
self._dit_hooks or self._hooked_dit_blocks or self._encoder_modules or self._staged_components
|
||||
):
|
||||
return
|
||||
|
||||
for blocks in self._blocks:
|
||||
for block in blocks:
|
||||
remove_block_hook(block)
|
||||
|
||||
steps = []
|
||||
if restore_weights:
|
||||
steps.extend(("restoring a DiT block", hook.restore_next_block) for hook in self._dit_hooks)
|
||||
steps.extend(
|
||||
("removing a DiT block hook", lambda block=block: remove_block_hook(block))
|
||||
for block in self._hooked_dit_blocks
|
||||
)
|
||||
steps.extend(
|
||||
(
|
||||
"disabling encoder layerwise offload",
|
||||
lambda module=module: disable_plan_encoder_layerwise_offload(
|
||||
module,
|
||||
restore_weights=restore_weights,
|
||||
),
|
||||
)
|
||||
for module in self._encoder_modules
|
||||
)
|
||||
cleanup_error = run_cleanup_steps(steps)
|
||||
if cleanup_error is not None:
|
||||
# Keep every host-master reference so a transient teardown error
|
||||
# can be retried without losing weights behind placeholders.
|
||||
raise RuntimeError("Failed to fully disable layerwise offload") from cleanup_error
|
||||
self._blocks.clear()
|
||||
self._dit_hooks.clear()
|
||||
self._hooked_dit_blocks.clear()
|
||||
self._encoder_modules.clear()
|
||||
self._staged_components.clear()
|
||||
self.enabled = False
|
||||
logger.info("Layer-wise offloading disabled")
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_attr_names(model: nn.Module) -> list[str]:
|
||||
"""Get block attribute names from model class."""
|
||||
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
|
||||
def disable(self) -> None:
|
||||
self._disable(restore_weights=True)
|
||||
|
||||
if not attrs:
|
||||
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
|
||||
if old_attr is not None:
|
||||
logger.warning(
|
||||
"'_layerwise_offload_blocks_attr' is deprecated, "
|
||||
"please use '_layerwise_offload_blocks_attrs' instead. "
|
||||
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
|
||||
)
|
||||
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
|
||||
|
||||
return attrs
|
||||
|
||||
@staticmethod
|
||||
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
|
||||
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
|
||||
"""
|
||||
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
|
||||
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
|
||||
|
||||
```
|
||||
class WanTransformer3DModel(nn.Module):
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
```
|
||||
|
||||
Returns:
|
||||
Tuple of (blocks_attr_names, blocks)
|
||||
"""
|
||||
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
|
||||
if not blocks_attr_names:
|
||||
logger.warning(
|
||||
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
)
|
||||
return [], []
|
||||
|
||||
blocks = []
|
||||
for name in blocks_attr_names:
|
||||
attr = getattr(model, name, None)
|
||||
if attr is None:
|
||||
raise AttributeError(
|
||||
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
|
||||
f"does not exist on model {model.__class__.__name__}"
|
||||
)
|
||||
try:
|
||||
attr_iter = iter(attr)
|
||||
except TypeError:
|
||||
if isinstance(attr, nn.Module):
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable; treating it as one block.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
blocks.append(attr)
|
||||
continue
|
||||
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
type(attr).__name__,
|
||||
)
|
||||
else:
|
||||
blocks.extend(attr_iter)
|
||||
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
"No blocks found in %s for %s, skipping layerwise offloading",
|
||||
blocks_attr_names,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
return [], []
|
||||
|
||||
return blocks_attr_names, blocks
|
||||
# Compatibility aliases for existing model integrations.
|
||||
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
|
||||
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
|
||||
get_blocks_from_dit = staticmethod(get_blocks_from_dit)
|
||||
@@ -1,10 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Declarative OffloadPlan for distributed layerwise offload.
|
||||
"""Declarative model capabilities for component layerwise offload.
|
||||
|
||||
Models declare this as a class attribute ``_offload_plan`` on the
|
||||
pipeline class. When present, the offloader uses it instead of
|
||||
heuristic block discovery.
|
||||
Models declare an ``_offload_plan`` class attribute on the pipeline. Both the
|
||||
ordinary and distributed layerwise backends consume the same plan instead of
|
||||
embedding model-specific block paths in backend code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,12 +16,11 @@ from torch import nn
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadPlan:
|
||||
"""Optional declarative metadata for distributed layerwise offload.
|
||||
"""Optional declarative metadata for component layerwise offload.
|
||||
|
||||
Models declare this as a class attribute ``_offload_plan`` on the
|
||||
pipeline class. When present, the offloader uses it instead of
|
||||
heuristic block discovery, making it easier to support new models
|
||||
without offload-specific logic in constructor or forward code.
|
||||
pipeline class. When present, both layerwise backends use it instead of
|
||||
model-specific backend branches, making new integrations data-driven.
|
||||
|
||||
If not declared, the offloader falls back to:
|
||||
1. ``_layerwise_offload_blocks_attrs`` on each DiT module class.
|
||||
@@ -39,9 +38,14 @@ class OffloadPlan:
|
||||
device when ``dlo_resident_layers`` is nonzero. Keeping this
|
||||
model-declared avoids applying a consumer-GPU tuning knob to
|
||||
auxiliary or dual DiTs unintentionally.
|
||||
encoder_block_attrs: Maps encoder paths to rank-local block-list paths.
|
||||
These blocks are streamed with ordinary layerwise hooks, never
|
||||
with the DiT AllGather group.
|
||||
encoder_component_types: Maps encoder paths to public selector types
|
||||
(currently text_encoder). This declaration is used before the
|
||||
compatibility name heuristic.
|
||||
encoder_block_attrs: Maps encoder paths to streamable block-list paths.
|
||||
encoder_dlo_weight_replication: Encoder paths whose loader-produced
|
||||
block tensors are identical across the DiT DLO group. Only these
|
||||
encoders may use multi-rank AllGather transfer; this must not be
|
||||
declared for encoder-TP shards.
|
||||
"""
|
||||
|
||||
on_demand_component_paths: frozenset[str] = field(default_factory=frozenset)
|
||||
@@ -49,7 +53,9 @@ class OffloadPlan:
|
||||
block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
offload_submodules: dict[str, str] = field(default_factory=dict)
|
||||
resident_dit_paths: frozenset[str] = field(default_factory=frozenset)
|
||||
encoder_component_types: dict[str, str] = field(default_factory=dict)
|
||||
encoder_block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
encoder_dlo_weight_replication: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
|
||||
def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Collection, Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
@@ -13,11 +13,38 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
|
||||
from .config import DIT_COMPONENT, TEXT_ENCODER_COMPONENT
|
||||
from .module_collector import ModuleDiscovery
|
||||
from .offload_plan import get_offload_plan
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _capture_tensor_devices(modules: Collection[nn.Module]) -> list[tuple[torch.Tensor, torch.device]]:
|
||||
seen: set[int] = set()
|
||||
devices: list[tuple[torch.Tensor, torch.device]] = []
|
||||
for module in modules:
|
||||
for tensor in (*module.parameters(), *module.buffers()):
|
||||
if id(tensor) in seen:
|
||||
continue
|
||||
seen.add(id(tensor))
|
||||
devices.append((tensor, tensor.device))
|
||||
return devices
|
||||
|
||||
|
||||
def _restore_tensor_devices(devices: list[tuple[torch.Tensor, torch.device]]) -> None:
|
||||
first_error: BaseException | None = None
|
||||
for tensor, device in devices:
|
||||
try:
|
||||
if tensor.device != device:
|
||||
tensor.data = tensor.data.to(device)
|
||||
except BaseException as exc:
|
||||
logger.exception("Failed to restore tensor placement to %s", device)
|
||||
first_error = first_error or exc
|
||||
if first_error is not None:
|
||||
raise RuntimeError("Failed to restore one or more tensor placements") from first_error
|
||||
|
||||
|
||||
class SequentialOffloadHook(ModelHook):
|
||||
"""Hook for sequential offloading with mutual exclusion on encoder and DiT modules.
|
||||
|
||||
@@ -34,12 +61,14 @@ class SequentialOffloadHook(ModelHook):
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
use_hsdp: bool = False,
|
||||
offload_after_context: bool = True,
|
||||
):
|
||||
# Modules to offload to CPU before this module runs
|
||||
self.offload_targets = offload_targets
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
self.use_hsdp = use_hsdp
|
||||
self.offload_after_context = offload_after_context
|
||||
|
||||
@staticmethod
|
||||
def _move_params(
|
||||
@@ -48,7 +77,7 @@ class SequentialOffloadHook(ModelHook):
|
||||
*,
|
||||
non_blocking: bool = False,
|
||||
pin_memory: bool = False,
|
||||
) -> None:
|
||||
) -> bool:
|
||||
"""Move module parameters and buffers to device.
|
||||
|
||||
This cls method specifically prevents recursion device movement,
|
||||
@@ -57,47 +86,38 @@ class SequentialOffloadHook(ModelHook):
|
||||
refer to
|
||||
https://github.com/vipshop/cache-dit/blob/v1.2.3/src/cache_dit/caching/cache_blocks/__init__.py#L83
|
||||
"""
|
||||
moved = False
|
||||
for p in module.parameters():
|
||||
if p.data.device != target_device:
|
||||
data = p.data.to(target_device, non_blocking=non_blocking)
|
||||
if pin_memory and target_device.type == "cpu" and not isinstance(data, DTensor):
|
||||
data = data.pin_memory()
|
||||
p.data = data
|
||||
moved = True
|
||||
for b in module.buffers():
|
||||
if b.device != target_device:
|
||||
data = b.data.to(target_device, non_blocking=non_blocking)
|
||||
if pin_memory and target_device.type == "cpu" and not isinstance(data, DTensor):
|
||||
data = data.pin_memory()
|
||||
b.data = data
|
||||
moved = True
|
||||
return moved
|
||||
|
||||
def _to_cpu(self, module: nn.Module) -> None:
|
||||
try:
|
||||
param = next(module.parameters())
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
if param.device.type == "cpu":
|
||||
return
|
||||
|
||||
# XPU's allocator doesn't respect stream dependencies in empty_cache,
|
||||
# so non-blocking copies can race with cache eviction. Use blocking
|
||||
# copies on XPU to avoid NULL pointer errors during DMA.
|
||||
non_blocking = not self.use_hsdp and not current_omni_platform.is_xpu()
|
||||
self._move_params(
|
||||
moved = self._move_params(
|
||||
module,
|
||||
torch.device("cpu"),
|
||||
non_blocking=non_blocking,
|
||||
pin_memory=self.pin_memory,
|
||||
)
|
||||
current_omni_platform.empty_cache()
|
||||
if moved:
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
def _to_gpu(self, module: nn.Module) -> None:
|
||||
try:
|
||||
if next(module.parameters()).device == self.device:
|
||||
return
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
self._move_params(module, self.device, non_blocking=False)
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
|
||||
@@ -127,6 +147,8 @@ def apply_sequential_offload(
|
||||
pin_memory: bool = True,
|
||||
use_hsdp: bool = False,
|
||||
offload_initial_dits: bool = False,
|
||||
offload_dit_modules: Collection[nn.Module] | None = None,
|
||||
offload_encoder_modules: Collection[nn.Module] | None = None,
|
||||
) -> None:
|
||||
"""Apply sequential offloading hooks to DiT and encoder modules.
|
||||
|
||||
@@ -141,6 +163,10 @@ def apply_sequential_offload(
|
||||
pin_memory: Whether to pin CPU memory for faster transfers
|
||||
use_hsdp: Whether HSDP is enabled (affects non_blocking behavior)
|
||||
offload_initial_dits: Whether to begin with all DiT modules on CPU.
|
||||
offload_dit_modules: DiT modules allowed to move to CPU. None selects
|
||||
every DiT for backward compatibility.
|
||||
offload_encoder_modules: Encoder/stage modules allowed to move to CPU.
|
||||
None selects every supplied module for backward compatibility.
|
||||
|
||||
Example:
|
||||
>>> apply_sequential_offload(
|
||||
@@ -150,38 +176,59 @@ def apply_sequential_offload(
|
||||
... )
|
||||
>>> # Modules of pipeline now automatically swap between CPU and GPU
|
||||
"""
|
||||
# Register hooks on DiT modules (offload encoders AND other DiTs when a DiT runs)
|
||||
for i, dit_mod in enumerate(dit_modules):
|
||||
other_dits = [d for j, d in enumerate(dit_modules) if j != i]
|
||||
registry = HookRegistry.get_or_create(dit_mod)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=encoder_modules + other_dits,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
|
||||
selected_dit_ids = {id(module) for module in (dit_modules if offload_dit_modules is None else offload_dit_modules)}
|
||||
selected_encoder_ids = {
|
||||
id(module) for module in (encoder_modules if offload_encoder_modules is None else offload_encoder_modules)
|
||||
}
|
||||
|
||||
# Register hooks on encoders (offload DiTs when encoder runs)
|
||||
for enc in encoder_modules:
|
||||
registry = HookRegistry.get_or_create(enc)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=dit_modules,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
|
||||
all_modules = [*dit_modules, *encoder_modules]
|
||||
initial_devices = _capture_tensor_devices(dit_modules) if offload_initial_dits else []
|
||||
try:
|
||||
selected_encoders = [encoder for encoder in encoder_modules if id(encoder) in selected_encoder_ids]
|
||||
# Register hooks on DiT modules (offload selected encoders and other selected DiTs).
|
||||
for i, dit_mod in enumerate(dit_modules):
|
||||
other_dits = [d for j, d in enumerate(dit_modules) if j != i and id(d) in selected_dit_ids]
|
||||
registry = HookRegistry.get_or_create(dit_mod)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=selected_encoders + other_dits,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
offload_after_context=id(dit_mod) in selected_dit_ids,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
|
||||
|
||||
if offload_initial_dits:
|
||||
try:
|
||||
# Register hooks on all execution stages so unselected resident modules can
|
||||
# still evict selected DiTs before they run.
|
||||
selected_dits = [dit for dit in dit_modules if id(dit) in selected_dit_ids]
|
||||
for enc in encoder_modules:
|
||||
registry = HookRegistry.get_or_create(enc)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=selected_dits,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
offload_after_context=id(enc) in selected_encoder_ids,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
|
||||
|
||||
if offload_initial_dits:
|
||||
for dit_mod in dit_modules:
|
||||
if id(dit_mod) not in selected_dit_ids:
|
||||
continue
|
||||
_get_sequential_offload_hook(dit_mod)._to_cpu(dit_mod)
|
||||
except Exception:
|
||||
remove_sequential_offload([*dit_modules, *encoder_modules])
|
||||
raise
|
||||
except BaseException:
|
||||
try:
|
||||
remove_sequential_offload(all_modules)
|
||||
except BaseException:
|
||||
logger.exception("Failed to remove every sequential hook during rollback")
|
||||
try:
|
||||
_restore_tensor_devices(initial_devices)
|
||||
except BaseException:
|
||||
logger.exception("Failed to restore initial DiT placement during rollback")
|
||||
raise
|
||||
|
||||
|
||||
def remove_sequential_offload(modules: list[nn.Module]) -> None:
|
||||
@@ -194,11 +241,18 @@ def remove_sequential_offload(modules: list[nn.Module]) -> None:
|
||||
>>> all_modules = [*dit_modules, *encoder_modules]
|
||||
>>> remove_sequential_offload(all_modules)
|
||||
"""
|
||||
first_error: BaseException | None = None
|
||||
for module in modules:
|
||||
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
|
||||
if registry is not None:
|
||||
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
try:
|
||||
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
|
||||
if registry is not None:
|
||||
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
except BaseException as exc:
|
||||
logger.exception("Failed to remove offload hook from %s", module.__class__.__name__)
|
||||
first_error = first_error or exc
|
||||
if first_error is not None:
|
||||
raise RuntimeError("Failed to remove one or more sequential offload hooks") from first_error
|
||||
|
||||
|
||||
def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
|
||||
@@ -218,12 +272,14 @@ def sequential_offload_component(module: nn.Module) -> Iterator[None]:
|
||||
yield
|
||||
except BaseException:
|
||||
try:
|
||||
hook._to_cpu(module)
|
||||
if hook.offload_after_context:
|
||||
hook._to_cpu(module)
|
||||
except Exception:
|
||||
logger.exception("Failed to release %s after component failure", module.__class__.__name__)
|
||||
raise
|
||||
else:
|
||||
hook._to_cpu(module)
|
||||
if hook.offload_after_context:
|
||||
hook._to_cpu(module)
|
||||
|
||||
|
||||
class ModelLevelOffloadBackend(OffloadBackend):
|
||||
@@ -245,11 +301,26 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
# Pipelines with non-forward component entry points own their complete
|
||||
# mutual-exclusion lifecycle. Delegate through the explicit protocol.
|
||||
if isinstance(pipeline, SupportsModelCpuOffload):
|
||||
pipeline.enable_omni_model_cpu_offload(
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
)
|
||||
try:
|
||||
if self.config.components is not None:
|
||||
pipeline.enable_omni_model_cpu_offload(
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
offload_components=self.config.components,
|
||||
)
|
||||
else:
|
||||
pipeline.enable_omni_model_cpu_offload(
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
)
|
||||
except BaseException:
|
||||
try:
|
||||
pipeline.disable_omni_model_cpu_offload()
|
||||
except BaseException:
|
||||
logger.exception("Model-level cleanup failed while handling an enable failure")
|
||||
raise
|
||||
self._custom_pipeline = pipeline
|
||||
self.enabled = True
|
||||
logger.info(
|
||||
@@ -259,44 +330,65 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
return
|
||||
|
||||
modules = ModuleDiscovery.discover(pipeline)
|
||||
plan = get_offload_plan(pipeline)
|
||||
selected_encoders = [
|
||||
encoder
|
||||
for encoder, name in zip(modules.encoders, modules.encoder_names)
|
||||
if self.config.should_offload_encoder(name, plan)
|
||||
]
|
||||
if self.config.components is not None:
|
||||
if not modules.dits:
|
||||
raise ValueError("Component-selective model offload requires a DiT/transformer module")
|
||||
if not modules.encoders:
|
||||
raise ValueError("Component-selective model offload requires an encoder execution stage")
|
||||
if self.config.offloads(TEXT_ENCODER_COMPONENT) and not selected_encoders:
|
||||
raise ValueError("No text encoder modules found for selected text_encoder module offload")
|
||||
|
||||
# Move encoders to GPU
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
|
||||
# Move VAE(s) to GPU if available
|
||||
for vae in modules.vaes:
|
||||
try:
|
||||
all_modules = [
|
||||
*modules.dits,
|
||||
*modules.encoders,
|
||||
*modules.vaes,
|
||||
*modules.resident_modules,
|
||||
]
|
||||
initial_devices = _capture_tensor_devices(all_modules)
|
||||
try:
|
||||
for encoder in modules.encoders:
|
||||
encoder.to(self.device)
|
||||
for vae in modules.vaes:
|
||||
vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
for resident in modules.resident_modules:
|
||||
resident.to(self.device)
|
||||
|
||||
# Pin resident modules on GPU (small hot submodules called inside the DiT loop).
|
||||
for res, name in zip(modules.resident_modules, modules.resident_names):
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
|
||||
return
|
||||
if not modules.encoders:
|
||||
for dit in modules.dits:
|
||||
dit.to(self.device)
|
||||
logger.warning("No encoder modules found, skipping model-level offloading")
|
||||
return
|
||||
|
||||
apply_sequential_offload(
|
||||
dit_modules=modules.dits,
|
||||
encoder_modules=modules.encoders,
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
offload_dit_modules=(
|
||||
modules.dits if self.config.components is None or self.config.offloads(DIT_COMPONENT) else ()
|
||||
),
|
||||
offload_encoder_modules=selected_encoders,
|
||||
)
|
||||
except BaseException:
|
||||
try:
|
||||
res.to(self.device)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to move resident module '%s' to GPU: %s", name, exc)
|
||||
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
|
||||
return
|
||||
|
||||
if not modules.encoders:
|
||||
# Nothing to swap against — move DiTs to GPU and skip hooks.
|
||||
for dit in modules.dits:
|
||||
dit.to(self.device)
|
||||
logger.warning("No encoder modules found, skipping model-level offloading")
|
||||
return
|
||||
|
||||
# Apply sequential offloading hooks
|
||||
apply_sequential_offload(
|
||||
dit_modules=modules.dits,
|
||||
encoder_modules=modules.encoders,
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
)
|
||||
remove_sequential_offload([*modules.dits, *modules.encoders])
|
||||
except BaseException:
|
||||
logger.exception("Failed to remove every model-level hook during rollback")
|
||||
try:
|
||||
_restore_tensor_devices(initial_devices)
|
||||
except BaseException:
|
||||
logger.exception("Failed to restore model placement during rollback")
|
||||
raise
|
||||
|
||||
# Track modules for cleanup
|
||||
self._offload_modules = [*modules.dits, *modules.encoders]
|
||||
|
||||
@@ -9,15 +9,100 @@ offload backends.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.distributed.tensor import DTensor
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TensorStorageSpec:
|
||||
"""One tensor's target and physical-storage layout."""
|
||||
|
||||
name: str
|
||||
target: torch.Tensor
|
||||
value: torch.Tensor
|
||||
storage_numel: int
|
||||
stride: tuple[int, ...]
|
||||
|
||||
|
||||
def dtype_size(dtype: torch.dtype) -> int:
|
||||
"""Return element size in bytes for a torch.dtype."""
|
||||
return torch.empty(1, dtype=dtype).element_size()
|
||||
|
||||
|
||||
def group_named_tensors_by_dtype(
|
||||
params: Mapping[str, torch.Tensor],
|
||||
buffers: Mapping[str, torch.Tensor],
|
||||
) -> dict[torch.dtype, list[tuple[str, torch.Tensor]]]:
|
||||
"""Group parameters and buffers by dtype while preserving their order."""
|
||||
grouped: dict[torch.dtype, list[tuple[str, torch.Tensor]]] = {}
|
||||
for name, tensor in chain(params.items(), buffers.items()):
|
||||
grouped.setdefault(tensor.dtype, []).append((name, tensor))
|
||||
return grouped
|
||||
|
||||
|
||||
def physical_storage_numel(tensor: torch.Tensor) -> int:
|
||||
"""Return the storage span needed to preserve ``tensor``'s stride."""
|
||||
if tensor.numel() == 0:
|
||||
return 0
|
||||
return 1 + sum((size - 1) * stride for size, stride in zip(tensor.shape, tensor.stride()))
|
||||
|
||||
|
||||
def flatten_physical_storage(
|
||||
tensor: torch.Tensor,
|
||||
storage_numel: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Pack a possibly strided tensor in physical storage order."""
|
||||
if tensor.is_contiguous():
|
||||
return tensor.flatten()
|
||||
storage_numel = physical_storage_numel(tensor) if storage_numel is None else storage_numel
|
||||
storage = torch.zeros(storage_numel, dtype=tensor.dtype, device=tensor.device)
|
||||
torch.as_strided(storage, size=tensor.shape, stride=tensor.stride()).copy_(tensor)
|
||||
return storage
|
||||
|
||||
|
||||
def describe_tensor_storage(
|
||||
named_tensors: Iterable[tuple[str, torch.Tensor]],
|
||||
transforms: Mapping[int, Callable[[torch.Tensor], torch.Tensor]] | None = None,
|
||||
) -> list[TensorStorageSpec]:
|
||||
"""Resolve local tensors and their transport-preserving layouts."""
|
||||
specs: list[TensorStorageSpec] = []
|
||||
for name, target in named_tensors:
|
||||
value = target.to_local() if hasattr(target, "to_local") else target
|
||||
transform = (transforms or {}).get(id(target))
|
||||
if callable(transform):
|
||||
value = transform(value)
|
||||
specs.append(
|
||||
TensorStorageSpec(
|
||||
name=name,
|
||||
target=target,
|
||||
value=value,
|
||||
storage_numel=physical_storage_numel(value),
|
||||
stride=value.stride(),
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def tensor_storage_metadata(spec: TensorStorageSpec, offset: int, *, include_device: bool = False) -> dict[str, Any]:
|
||||
"""Build reconstruction metadata shared by offload transports."""
|
||||
metadata: dict[str, Any] = {
|
||||
"name": spec.name,
|
||||
"offset": offset,
|
||||
"numel": spec.storage_numel,
|
||||
"shape": spec.value.shape,
|
||||
"stride": spec.stride,
|
||||
}
|
||||
if include_device:
|
||||
metadata["device"] = spec.value.device
|
||||
return metadata
|
||||
|
||||
|
||||
def is_dtensor(t: torch.Tensor) -> bool:
|
||||
"""Check if tensor is a DTensor."""
|
||||
return isinstance(t, DTensor)
|
||||
@@ -39,9 +124,74 @@ def make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
|
||||
|
||||
|
||||
def clear_tensor_storage(tensors: Iterable[torch.Tensor]) -> None:
|
||||
"""Atomically replace tensor storage with offload placeholders."""
|
||||
targets = list(tensors)
|
||||
originals = [tensor.to_local() if is_dtensor(tensor) else tensor.data for tensor in targets]
|
||||
placeholders = [make_offload_placeholder(tensor) for tensor in targets]
|
||||
committed = 0
|
||||
try:
|
||||
for target, placeholder in zip(targets, placeholders, strict=True):
|
||||
set_tensor_storage(target, placeholder)
|
||||
committed += 1
|
||||
except BaseException:
|
||||
rollback_error: BaseException | None = None
|
||||
for target, original in zip(targets[:committed], originals[:committed], strict=True):
|
||||
try:
|
||||
set_tensor_storage(target, original)
|
||||
except BaseException as exc:
|
||||
rollback_error = rollback_error or exc
|
||||
if rollback_error is not None:
|
||||
raise RuntimeError("Tensor storage clear failed and rollback was incomplete") from rollback_error
|
||||
raise
|
||||
|
||||
|
||||
def clear_block_storage(
|
||||
params: Mapping[str, torch.Tensor],
|
||||
buffers: Mapping[str, torch.Tensor],
|
||||
ready_event: Any | None,
|
||||
) -> None:
|
||||
"""Wait for an outstanding transfer, then release one block's residency."""
|
||||
if ready_event is not None:
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
current_omni_platform.current_stream().wait_event(ready_event)
|
||||
clear_tensor_storage(chain(params.values(), buffers.values()))
|
||||
|
||||
|
||||
def is_materialized_tensor(t: torch.Tensor) -> bool:
|
||||
"""Check if tensor holds real data (not meta or empty placeholder)."""
|
||||
if is_dtensor(t):
|
||||
local_t = t.to_local()
|
||||
return not local_t.is_meta
|
||||
return not t.is_meta and t.data.numel() > 0
|
||||
|
||||
|
||||
def materialization_probe(
|
||||
params: Mapping[str, torch.Tensor],
|
||||
buffers: Mapping[str, torch.Tensor],
|
||||
) -> torch.Tensor | None:
|
||||
"""Pick one non-empty tensor that tracks a block's atomic storage state."""
|
||||
return next((tensor for tensor in chain(params.values(), buffers.values()) if tensor.numel()), None)
|
||||
|
||||
|
||||
def module_materialization_probe(module: nn.Module) -> torch.Tensor | None:
|
||||
"""Capture a probe before another hook releases the module's storage."""
|
||||
return materialization_probe(dict(module.named_parameters()), dict(module.named_buffers()))
|
||||
|
||||
|
||||
def restore_tensor_storage(
|
||||
target: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
*,
|
||||
device: torch.device | str,
|
||||
) -> None:
|
||||
"""Detach ``target`` from an offloader-owned backing with a real copy."""
|
||||
restored = torch.empty_strided(
|
||||
value.shape,
|
||||
value.stride(),
|
||||
dtype=value.dtype,
|
||||
device=device,
|
||||
)
|
||||
restored.copy_(value)
|
||||
set_tensor_storage(target, restored)
|
||||
Reference in new issue
Block a user