[diffusion][feature] Add component-selective offload policies (#5929)

Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-09-06 05:40:09 +08:00
1 parent 44d3ae100a
commit ea3cf99894
74 files changed
+5121 -1584

No files matched your search

+2 -1
View File
@@ -8,8 +8,9 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.model_loader.host_weight_plan import HostWeightPlan
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, OffloadStrategy, SupportsModelCpuOffload
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .block_discovery import get_blocks_attr_names, get_blocks_from_dit, set_blocks_attr_names
from .config import OffloadStrategy
from .distributed_layerwise_backend import (
DistributedLayerwiseOffloadBackend,
DistributedLayerwiseOffloadHook,
+99 -54
View File
@@ -2,8 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from enum import Enum
from typing import Protocol, runtime_checkable
import torch
@@ -12,9 +12,33 @@ from vllm.logger import init_logger
from vllm_omni.diffusion.data import OmniDiffusionConfig, validate_dlo_host_registration_options
from .component_utils import encoder_component_type
from .config import (
DIT_COMPONENT,
TEXT_ENCODER_COMPONENT,
DLOTransfer,
OffloadStrategy,
parse_dlo_transfer,
parse_offload_components,
resolve_offload,
)
from .offload_plan import OffloadPlan
logger = init_logger(__name__)
def run_cleanup_steps(steps: Iterable[tuple[str, Callable[[], None]]]) -> BaseException | None:
"""Run every cleanup step and return the first failure, if any."""
first_error: BaseException | None = None
for description, step in steps:
try:
step()
except BaseException as exc:
logger.exception("Cleanup failed while %s", description)
first_error = first_error or exc
return first_error
@runtime_checkable
class SupportsModelCpuOffload(Protocol):
"""Pipeline-owned lifecycle for model-level CPU offload.
@@ -30,18 +54,12 @@ class SupportsModelCpuOffload(Protocol):
device: torch.device,
pin_memory: bool,
use_hsdp: bool,
offload_components: frozenset[str] | None = None,
) -> None: ...
def disable_omni_model_cpu_offload(self) -> None: ...
class OffloadStrategy(Enum):
NONE = "none"
MODEL_LEVEL = "model_level" # Sequential offloading between DiT and encoders
LAYER_WISE = "layer_wise" # Block-level
DISTRIBUTED_LAYER_WISE = "distributed_layer_wise" # Block-level with DP sharding + H2D/AllGather overlap
@dataclass
class OffloadConfig:
strategy: OffloadStrategy
@@ -55,14 +73,59 @@ class OffloadConfig:
# Optional per-worker ceiling for registering an HWR mmap. Zero means no
# additional ceiling; pin_cpu_memory controls whether registration is tried.
dlo_host_registration_limit_gib: float = 0.0
# ``None`` preserves the model's legacy plan-driven component topology;
# a frozenset is an explicit compact-API selection.
components: frozenset[str] | None = None
dlo_transfers: dict[str, DLOTransfer] | None = None
def __post_init__(self) -> None:
if self.components is not None:
self.components = parse_offload_components(self.components)
if self.dlo_transfers is None:
self.dlo_transfers = {
DIT_COMPONENT: DLOTransfer.ALLGATHER if self.dlo_use_allgather else DLOTransfer.RANK_LOCAL,
TEXT_ENCODER_COMPONENT: DLOTransfer.RANK_LOCAL,
}
self.dlo_transfers = parse_dlo_transfer(self.dlo_transfers)
# Preserve the old field as the DiT transfer compatibility view.
self.dlo_use_allgather = self.uses_allgather(DIT_COMPONENT)
def offloads(self, component: str) -> bool:
# Historically the generic selector covered DiT directly while model
# plans opted auxiliary components into their lifecycle.
return component == DIT_COMPONENT if self.components is None else component in self.components
def transfer_for(self, component: str) -> DLOTransfer:
if self.dlo_transfers is None:
raise RuntimeError("DLO transfers were not initialized")
try:
return self.dlo_transfers[component]
except KeyError as exc:
raise ValueError(f"Unknown DLO component {component!r}") from exc
def uses_allgather(self, component: str) -> bool:
return self.transfer_for(component) is DLOTransfer.ALLGATHER
def offloads_encoder(self, name: str, plan: OffloadPlan | None = None) -> bool:
"""Return whether the selector covers a discovered encoder path.
Plans declare non-standard encoder names explicitly. The name-based
fallback preserves compatibility with pipelines that predate OffloadPlan.
"""
component = encoder_component_type(name, plan)
return component is not None and self.offloads(component)
def should_offload_encoder(self, name: str, plan: OffloadPlan | None = None) -> bool:
"""Apply explicit selection while preserving the legacy encoder topology."""
return self.components is None or self.offloads_encoder(name, plan)
@classmethod
def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig":
"""Extract and validate offload settings from OmniDiffusionConfig.
Enforces mutual exclusion among the three offload strategies.
Distributed layer-wise takes the highest priority, then layer-wise,
then model-level.
``diffusion_offload_config`` is the canonical public selector. The
historical ``enable_*_offload`` booleans remain compatibility aliases;
ambiguous combinations fail instead of using silent precedence.
The ``dp_size`` is automatically derived from ``parallel_config`` —
it is NOT a user-configurable parameter. The distributed layerwise
@@ -74,10 +137,11 @@ class OffloadConfig:
Returns:
OffloadConfig with validated settings
"""
enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False)
enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False)
enable_distributed_layerwise_offload = getattr(od_config, "enable_distributed_layerwise_offload", False)
pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True)
resolved = resolve_offload(od_config)
strategy = resolved.strategy
public_config = resolved.public
enable_distributed_layerwise_offload = strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE
pin_cpu_memory = resolved.pin_memory
parallel_config = getattr(od_config, "parallel_config", None)
use_hsdp = getattr(parallel_config, "use_hsdp", False) if parallel_config else False
@@ -102,61 +166,40 @@ class OffloadConfig:
if sp_size and sp_size > 1:
dp_size = sp_size
# Determine strategy (mutual exclusion, distributed layer-wise takes priority)
if enable_distributed_layerwise_offload:
strategy = OffloadStrategy.DISTRIBUTED_LAYER_WISE
if enable_layerwise_offload or enable_cpu_offload:
logger.info("Distributed layer-wise offloading takes priority, disabling other offloading strategies.")
elif enable_layerwise_offload:
strategy = OffloadStrategy.LAYER_WISE
if enable_cpu_offload:
logger.info(
"Both model-level and layer-wise offloading enabled. "
"Layer-wise takes priority, disabling model-level offloading."
)
elif enable_cpu_offload:
strategy = OffloadStrategy.MODEL_LEVEL
if public_config is not None:
components: frozenset[str] | None = resolved.components
else:
strategy = OffloadStrategy.NONE
components = None
# With dlo_use_allgather=False, do not add another DP shard. Each rank
# streams the tensors produced by the standard loader, which may
# already be TP-local shards. This avoids AllGather synchronization
# requirements (concurrent requests, dummy run skip).
dlo_use_allgather = getattr(od_config, "dlo_use_allgather", True)
dlo_resident_layers = int(getattr(od_config, "dlo_resident_layers", 0))
dlo_transfers = dict(resolved.transfers)
dlo_resident_layers = resolved.resident_layers
dit_uses_allgather = resolved.uses_allgather(DIT_COMPONENT)
dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
limit_gib=getattr(od_config, "dlo_host_registration_limit_gib", 0.0),
enable_dlo=enable_distributed_layerwise_offload,
use_allgather=dlo_use_allgather,
use_allgather=dit_uses_allgather,
hwr_mode=getattr(od_config, "host_weight_runtime_mode", "disabled"),
)
if dlo_resident_layers < 0:
raise ValueError(f"dlo_resident_layers must be >= 0, got {dlo_resident_layers}")
if dlo_resident_layers and dlo_use_allgather:
raise ValueError(
"dlo_resident_layers currently requires --dlo-no-use-allgather so "
"resident blocks use weights prepared by the standard TP-aware loader"
)
# If dlo_use_allgather=False, force dp_size=1 (each rank independent)
if enable_distributed_layerwise_offload and not dlo_use_allgather:
dp_size = 1
if enable_distributed_layerwise_offload and all(
transfer is DLOTransfer.RANK_LOCAL
for component, transfer in dlo_transfers.items()
if component in resolved.components
):
logger.info(
"Distributed layerwise offload: dlo_use_allgather=False, "
"streaming complete rank-local blocks (no DLO shard or AllGather); "
"the backend will select mmap or standard-loader host storage"
"Distributed layerwise offload: all selected components use "
"rank-local transfer (no DLO shard or AllGather)"
)
# HSDP already shards parameters into DTensors. Running distributed
# layerwise offload on top would shard each to_local() again, producing
# incorrect reconstruction after AllGather. Reject this combination.
if enable_distributed_layerwise_offload and use_hsdp and dlo_use_allgather:
if enable_distributed_layerwise_offload and use_hsdp and resolved.any_allgather:
raise ValueError(
"Distributed layerwise offload with AllGather is incompatible with "
"HSDP: HSDP parameters are already sharded DTensors, and the offloader "
"would double-shard them. Use --dlo-no-use-allgather (standard-loader "
"rank-local weights) or disable HSDP."
"would double-shard them. Set weight_transfer='rank-local' for the affected "
"component in diffusion_offload_config, or disable HSDP."
)
return cls(
@@ -164,9 +207,11 @@ class OffloadConfig:
pin_cpu_memory=pin_cpu_memory,
use_hsdp=use_hsdp,
dp_size=dp_size,
dlo_use_allgather=dlo_use_allgather,
dlo_use_allgather=dit_uses_allgather,
dlo_resident_layers=dlo_resident_layers,
dlo_host_registration_limit_gib=dlo_host_registration_limit_gib,
components=components,
dlo_transfers=dlo_transfers,
)
@@ -35,9 +35,13 @@ def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
"""Retrieve blocks and attribute names from provided DiT model."""
blocks_attr_names = get_blocks_attr_names(model)
def get_blocks_from_dit(
model: nn.Module,
block_attrs: tuple[str, ...] | None = None,
) -> tuple[list[str], list[nn.Module]]:
"""Retrieve blocks from an explicit plan or the DiT class metadata."""
explicit_attrs = block_attrs is not None
blocks_attr_names = list(block_attrs) if explicit_attrs else get_blocks_attr_names(model)
if not blocks_attr_names:
logger.warning(
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
@@ -49,9 +53,9 @@ def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
for name in blocks_attr_names:
attr = getattr(model, name, None)
if attr is None:
source = "OffloadPlan.block_attrs" if explicit_attrs else "_layerwise_offload_blocks_attrs"
raise AttributeError(
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
f"does not exist on model {model.__class__.__name__}"
f"Attribute '{name}' declared in {source} does not exist on model {model.__class__.__name__}"
)
try:
attr_iter = iter(attr)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared component-plan helpers for diffusion offload backends."""
from __future__ import annotations
from collections.abc import Callable, Iterator
from itertools import chain
from operator import attrgetter
from typing import TYPE_CHECKING, Any
import torch
from torch import nn
from torch.distributed.tensor import DTensor
from vllm.logger import init_logger
from .block_discovery import get_blocks_from_dit
from .config import DIT_COMPONENT, TEXT_ENCODER_COMPONENT
from .offload_plan import OffloadPlan
from .tensor_utils import set_tensor_storage
if TYPE_CHECKING:
from .base import OffloadConfig
from .module_collector import PipelineModules
logger = init_logger(__name__)
def encoder_component_type(name: str, plan: OffloadPlan | None) -> str | None:
"""Map a discovered encoder path to its public offload component."""
declared = None if plan is None else plan.encoder_component_types.get(name)
if declared is not None:
if declared != TEXT_ENCODER_COMPONENT:
raise ValueError(f"OffloadPlan maps encoder {name!r} to unknown component {declared!r}")
return declared
leaf_name = name.rsplit(".", 1)[-1]
if leaf_name.startswith(TEXT_ENCODER_COMPONENT) or leaf_name.endswith(TEXT_ENCODER_COMPONENT):
return TEXT_ENCODER_COMPONENT
return None
def get_encoder_block_groups(
module: nn.Module,
name: str,
plan: OffloadPlan | None,
*,
strict: bool = False,
) -> list[nn.ModuleList]:
"""Resolve the streamable block lists declared for one encoder."""
if plan is None:
return []
# Some distributed models expose an unloaded stub on ranks where the
# component never executes. That is a valid local no-op even when the
# component was selected explicitly.
if getattr(module, "is_loaded", True) is False:
return []
groups: list[nn.ModuleList] = []
for block_path in plan.encoder_block_attrs.get(name, ()):
try:
blocks = attrgetter(block_path)(module)
except AttributeError:
if strict:
raise ValueError(f"Encoder offload path {name}.{block_path} was not found") from None
logger.warning("Encoder offload path %s.%s was not found", name, block_path)
continue
if not isinstance(blocks, nn.ModuleList) or len(blocks) <= 1:
if strict:
raise ValueError(f"Encoder offload path {name}.{block_path} is not a streamable block list")
logger.warning("Encoder offload path %s.%s is not a streamable block list", name, block_path)
continue
groups.append(blocks)
return groups
def iter_streamable_dits(
modules: PipelineModules,
config: OffloadConfig,
device: torch.device,
plan: OffloadPlan | None,
) -> Iterator[tuple[str, nn.Module, list[str], list[nn.Module]]]:
"""Yield selected DiTs whose block metadata resolves successfully."""
if not config.offloads(DIT_COMPONENT):
return
for name, module in zip(modules.dit_names, modules.dits):
logger.info("Applying hooks on %s (%s)", name, module.__class__.__name__)
planned_attrs = None if plan is None else plan.block_attrs.get(name)
block_attrs, blocks = get_blocks_from_dit(module, planned_attrs)
if blocks:
yield name, module, block_attrs, blocks
continue
if config.components is not None:
raise ValueError(f"Selected DiT {name!r} has no streamable layerwise-offload blocks")
logger.warning("Target layers (blocks) not found. Skipping offloading on %s (%s)", name, type(module).__name__)
module.to(device)
def move_non_block_state_to_device(
module: nn.Module,
block_groups: list[nn.ModuleList],
device: torch.device,
) -> None:
"""Keep component state outside streamed block lists resident on device."""
block_tensors = {
id(tensor)
for blocks in block_groups
for block in blocks
for tensor in chain(block.parameters(), block.buffers())
}
for tensor in chain(module.parameters(), module.buffers()):
if id(tensor) in block_tensors:
continue
local = tensor.to_local() if isinstance(tensor, DTensor) else tensor
if local.device != device:
set_tensor_storage(tensor, local.to(device, non_blocking=True))
def set_encoder_layerwise_state(
module: nn.Module,
hooks: list[Any],
block_groups: list[nn.ModuleList],
) -> None:
"""Publish the backend-neutral state used by encoder stage lifecycles."""
module._omni_layerwise_hooks = hooks
module._omni_layerwise_block_groups = block_groups
module._omni_layerwise_enabled = True
def clear_encoder_layerwise_state(module: nn.Module) -> None:
"""Clear encoder layerwise state after its backend hooks are removed."""
module._omni_layerwise_hooks = []
module._omni_layerwise_block_groups = []
module._omni_layerwise_enabled = False
def validate_on_demand_component(module: nn.Module, name: str) -> None:
"""Require the explicit lifecycle used by pipeline-managed components."""
if not callable(getattr(module, "load_to_device", None)) or not callable(getattr(module, "offload_to_cpu", None)):
raise ValueError(
f"Component {name!r} declares on-demand offload but must implement load_to_device() and offload_to_cpu()"
)
def prepare_component(
module: nn.Module,
name: str,
*,
device: torch.device,
stage_on_demand: bool,
blockwise: bool,
staged_components: list[nn.Module],
) -> None:
"""Stage a selected component or keep its non-streamed form resident."""
if stage_on_demand:
validate_on_demand_component(module, name)
getattr(module, "offload_to_cpu")()
staged_components.append(module)
logger.info("Prepared %s for pipeline-managed staged offload", name)
elif not blockwise:
module.to(device)
def prepare_pipeline_components(
modules: PipelineModules,
config: OffloadConfig,
plan: OffloadPlan | None,
*,
device: torch.device,
staged_components: list[nn.Module],
enable_encoder_blocks: Callable[[nn.Module, str, OffloadPlan | None, bool], bool],
) -> None:
"""Apply the shared encoder/VAE/resident placement policy."""
if config.components is not None and config.offloads(TEXT_ENCODER_COMPONENT):
selected_encoder_names = [name for name in modules.encoder_names if config.offloads_encoder(name, plan)]
if not selected_encoder_names:
raise ValueError("No text encoder modules found for selected text_encoder offload")
if plan is not None:
for encoder, name in zip(modules.encoders, modules.encoder_names):
if config.should_offload_encoder(name, plan) and name in plan.on_demand_component_paths:
validate_on_demand_component(encoder, name)
for encoder, name in zip(modules.encoders, modules.encoder_names):
selected = config.should_offload_encoder(name, plan)
stage_on_demand = bool(selected and plan is not None and name in plan.on_demand_component_paths)
blockwise = selected and enable_encoder_blocks(encoder, name, plan, stage_on_demand)
if stage_on_demand and not blockwise and config.uses_allgather(TEXT_ENCODER_COMPONENT):
raise ValueError(
f"Text encoder {name!r} cannot use AllGather without a model-declared streamable block plan"
)
if selected and config.components is not None and not (blockwise or stage_on_demand):
raise ValueError(f"Selected text encoder {name!r} requires a model-declared streamable or on-demand plan")
prepare_component(
encoder,
name,
device=device,
stage_on_demand=stage_on_demand,
blockwise=blockwise,
staged_components=staged_components,
)
for vae, name in zip(modules.vaes, modules.vae_names):
legacy_staged = config.components is None and plan is not None and name in plan.on_demand_component_paths
prepare_component(
vae,
name,
device=device,
stage_on_demand=legacy_staged,
blockwise=False,
staged_components=staged_components,
)
for name, module in zip(modules.resident_names, modules.resident_modules):
module.to(device)
logger.debug("Moved resident module %s to %s", name, device)
if not config.offloads(DIT_COMPONENT):
for dit in modules.dits:
dit.to(device)
+429
View File
@@ -0,0 +1,429 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Public diffusion CPU-offload configuration helpers."""
from __future__ import annotations
from collections.abc import Collection, Iterator, Mapping
from dataclasses import dataclass
from enum import Enum
from typing import Any, TypeVar
DIT_COMPONENT = "dit"
TEXT_ENCODER_COMPONENT = "text_encoder"
OFFLOAD_COMPONENTS = frozenset({DIT_COMPONENT, TEXT_ENCODER_COMPONENT})
DEFAULT_OFFLOAD_COMPONENTS = frozenset({DIT_COMPONENT})
_KeyT = TypeVar("_KeyT")
_ValueT = TypeVar("_ValueT")
class _FrozenMapping(Mapping[_KeyT, _ValueT]):
"""Small immutable mapping that remains safe to pickle between workers."""
def __init__(self, value: Mapping[_KeyT, _ValueT]) -> None:
self._items = tuple(value.items())
def __getitem__(self, key: _KeyT) -> _ValueT:
for candidate, value in self._items:
if candidate == key:
return value
raise KeyError(key)
def __iter__(self) -> Iterator[_KeyT]:
return (key for key, _ in self._items)
def __len__(self) -> int:
return len(self._items)
def __reduce__(self):
return type(self), (dict(self._items),)
class OffloadMode(str, Enum):
"""User-facing offload granularity."""
MODULE = "module"
LAYER = "layer"
class OffloadStrategy(str, Enum):
"""Resolved internal backend strategy."""
NONE = "none"
MODEL_LEVEL = "model_level"
LAYER_WISE = "layer_wise"
DISTRIBUTED_LAYER_WISE = "distributed_layer_wise"
class DLOTransfer(str, Enum):
"""How one component's next block reaches the device."""
ALLGATHER = "allgather"
RANK_LOCAL = "rank-local"
@dataclass(frozen=True)
class LayerOffloadOptions:
"""Layer-mode settings for one selected diffusion component."""
weight_transfer: DLOTransfer | None = None
resident_layers: int = 0
@dataclass(frozen=True)
class ParsedDiffusionOffloadConfig:
"""Validated internal form of the raw public offload mapping."""
mode: OffloadMode
components: frozenset[str]
layer_options: Mapping[str, LayerOffloadOptions]
pin_memory: bool | None = None
def __post_init__(self) -> None:
object.__setattr__(self, "layer_options", _FrozenMapping(self.layer_options))
@dataclass(frozen=True)
class ResolvedOffload:
"""Canonical offload policy derived once from public or legacy inputs."""
strategy: OffloadStrategy
public: ParsedDiffusionOffloadConfig | None
components: frozenset[str]
transfers: Mapping[str, DLOTransfer]
pin_memory: bool
resident_layers: int
def __post_init__(self) -> None:
object.__setattr__(self, "transfers", _FrozenMapping(self.transfers))
def offloads(self, component: str) -> bool:
return component in self.components
def uses_allgather(self, component: str = DIT_COMPONENT) -> bool:
try:
return self.transfers[component] is DLOTransfer.ALLGATHER
except KeyError as exc:
raise ValueError(f"Unknown offload component {component!r}") from exc
@property
def any_allgather(self) -> bool:
return self.strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE and any(
self.uses_allgather(component) for component in self.components
)
_LEGACY_STRATEGY_FIELDS = {
"enable_cpu_offload": OffloadStrategy.MODEL_LEVEL,
"enable_layerwise_offload": OffloadStrategy.LAYER_WISE,
"enable_distributed_layerwise_offload": OffloadStrategy.DISTRIBUTED_LAYER_WISE,
}
_LEGACY_STRATEGY_PRIORITY = (
OffloadStrategy.DISTRIBUTED_LAYER_WISE,
OffloadStrategy.LAYER_WISE,
OffloadStrategy.MODEL_LEVEL,
)
_EnumT = TypeVar("_EnumT", bound=Enum)
def _parse_enum(value: Any, enum_type: type[_EnumT], label: str) -> _EnumT:
if isinstance(value, enum_type):
return value
if not isinstance(value, str):
raise TypeError(f"{label} must be a string, got {type(value).__name__}")
try:
return enum_type(value)
except ValueError as exc:
choices = ", ".join(item.value for item in enum_type)
raise ValueError(f"Unknown {label} {value!r}; choose from: {choices}") from exc
def _validate_string_keys(value: Mapping[Any, Any], label: str) -> None:
if any(not isinstance(key, str) for key in value):
raise TypeError(f"{label} keys must be strings")
def _parse_layer_options(component: str, value: Any) -> LayerOffloadOptions:
if not isinstance(value, Mapping):
raise TypeError(
f"diffusion_offload_config.layer_options[{component!r}] must be a mapping, got {type(value).__name__}"
)
_validate_string_keys(value, f"diffusion_offload_config.layer_options[{component!r}]")
unknown = sorted(set(value) - {"weight_transfer", "resident_layers"})
if unknown:
raise ValueError(
f"Unknown diffusion offload setting(s) for {component}: {', '.join(unknown)}; "
"choose from: weight_transfer, resident_layers"
)
weight_transfer = (
_parse_enum(value["weight_transfer"], DLOTransfer, "offload transfer") if "weight_transfer" in value else None
)
resident_layers = value.get("resident_layers", 0)
if type(resident_layers) is not int or resident_layers < 0:
raise ValueError(f"resident_layers for {component} must be a non-negative integer")
parsed = LayerOffloadOptions(
weight_transfer=weight_transfer,
resident_layers=resident_layers,
)
if component != DIT_COMPONENT and parsed.resident_layers:
raise ValueError("resident_layers currently supports only the 'dit' component")
return parsed
def parse_diffusion_offload_config(value: Any) -> ParsedDiffusionOffloadConfig | None:
"""Validate the compact ``diffusion_offload_config`` public API."""
if value is None:
return None
if not isinstance(value, Mapping):
raise TypeError(f"diffusion_offload_config must be a mapping, got {type(value).__name__}")
_validate_string_keys(value, "diffusion_offload_config")
unknown = sorted(set(value) - {"mode", "components", "layer_options", "pin_memory"})
if unknown:
raise ValueError(
f"Unknown diffusion_offload_config field(s): {', '.join(unknown)}; "
"choose from: mode, components, layer_options, pin_memory"
)
if "mode" not in value:
raise ValueError("diffusion_offload_config requires 'mode'")
if "components" not in value:
raise ValueError("diffusion_offload_config requires 'components'")
mode = _parse_enum(value["mode"], OffloadMode, "diffusion offload mode")
components = parse_offload_components(value["components"])
raw_layer_options = value.get("layer_options", {})
if not isinstance(raw_layer_options, Mapping):
raise TypeError("diffusion_offload_config.layer_options must be a mapping")
_validate_string_keys(raw_layer_options, "diffusion_offload_config.layer_options")
unselected_options = sorted(set(raw_layer_options) - components)
if unselected_options:
raise ValueError(
"diffusion_offload_config.layer_options requires selecting the same component(s): "
+ ", ".join(unselected_options)
)
layer_options = {
component: _parse_layer_options(component, raw_layer_options.get(component, {})) for component in components
}
pin_memory = value.get("pin_memory")
if pin_memory is not None and type(pin_memory) is not bool:
raise TypeError("diffusion_offload_config.pin_memory must be a bool")
if mode is OffloadMode.MODULE and raw_layer_options:
raise ValueError(
"diffusion_offload_config.layer_options requires mode='layer'; "
f"configured for: {', '.join(sorted(raw_layer_options))}"
)
dit = layer_options.get(DIT_COMPONENT)
if dit is not None and dit.resident_layers and dit.weight_transfer is DLOTransfer.ALLGATHER:
raise ValueError("resident_layers requires dit.weight_transfer='rank-local'")
return ParsedDiffusionOffloadConfig(
mode=mode,
components=components,
layer_options=layer_options,
pin_memory=pin_memory,
)
def _legacy_strategy(config: Any) -> OffloadStrategy:
enabled = {strategy for field, strategy in _LEGACY_STRATEGY_FIELDS.items() if bool(getattr(config, field, False))}
return next((strategy for strategy in _LEGACY_STRATEGY_PRIORITY if strategy in enabled), OffloadStrategy.NONE)
def _public_strategy(public: ParsedDiffusionOffloadConfig) -> OffloadStrategy:
if public.mode is OffloadMode.MODULE:
return OffloadStrategy.MODEL_LEVEL
needs_distributed_backend = any(
settings.weight_transfer is DLOTransfer.ALLGATHER or settings.resident_layers
for settings in public.layer_options.values()
)
return OffloadStrategy.DISTRIBUTED_LAYER_WISE if needs_distributed_backend else OffloadStrategy.LAYER_WISE
def _validate_legacy_layer_options(config: Any, public: ParsedDiffusionOffloadConfig | None) -> None:
"""Reject ambiguous or invalid compatibility DLO tuning before loading."""
resident_layers = getattr(config, "dlo_resident_layers", 0)
if type(resident_layers) is not int or resident_layers < 0:
raise ValueError(f"dlo_resident_layers must be a non-negative integer, got {resident_layers!r}")
if public is not None:
if getattr(config, "host_weight_runtime_mode", "disabled") != "disabled":
raise ValueError("diffusion_offload_config cannot be combined with Host Weight Runtime")
conflicting_fields = []
if getattr(config, "dlo_use_allgather", True) is not True:
conflicting_fields.append("dlo_use_allgather")
if resident_layers:
conflicting_fields.append("dlo_resident_layers")
if float(getattr(config, "dlo_host_registration_limit_gib", 0.0)) > 0:
conflicting_fields.append("dlo_host_registration_limit_gib")
if conflicting_fields:
raise ValueError(
"diffusion_offload_config cannot be combined with legacy DLO option(s): "
+ ", ".join(conflicting_fields)
)
return
if resident_layers and bool(getattr(config, "dlo_use_allgather", True)):
raise ValueError(
"dlo_resident_layers requires the DiT DLO transfer to be rank-local; set dlo_use_allgather=False"
)
def resolve_offload(config: Any) -> ResolvedOffload:
"""Resolve and cache the complete offload policy on its owning config."""
cached = getattr(config, "_resolved_diffusion_offload", None)
if isinstance(cached, ResolvedOffload):
return cached
public = parse_diffusion_offload_config(getattr(config, "diffusion_offload_config", None))
_validate_legacy_layer_options(config, public)
legacy = _legacy_strategy(config)
if public is None:
transfers = {
DIT_COMPONENT: (
DLOTransfer.ALLGATHER if bool(getattr(config, "dlo_use_allgather", True)) else DLOTransfer.RANK_LOCAL
),
TEXT_ENCODER_COMPONENT: DLOTransfer.RANK_LOCAL,
}
resolved = ResolvedOffload(
strategy=legacy,
public=None,
components=DEFAULT_OFFLOAD_COMPONENTS,
transfers=transfers,
pin_memory=bool(getattr(config, "pin_cpu_memory", True)),
resident_layers=int(getattr(config, "dlo_resident_layers", 0)),
)
else:
strategy = _public_strategy(public)
if legacy is not OffloadStrategy.NONE and legacy is not strategy:
raise ValueError("diffusion_offload_config cannot be combined with legacy enable_*_offload flags")
transfers = {component: DLOTransfer.RANK_LOCAL for component in OFFLOAD_COMPONENTS}
for component, options in public.layer_options.items():
transfers[component] = options.weight_transfer or DLOTransfer.RANK_LOCAL
dit_options = public.layer_options.get(DIT_COMPONENT)
resolved = ResolvedOffload(
strategy=strategy,
public=public,
components=public.components,
transfers=transfers,
pin_memory=(
public.pin_memory if public.pin_memory is not None else bool(getattr(config, "pin_cpu_memory", True))
),
resident_layers=0 if dit_options is None else dit_options.resident_layers,
)
parallel_config = getattr(config, "parallel_config", None)
data_parallel_size = int(getattr(parallel_config, "data_parallel_size", 1))
cache_backend = getattr(config, "cache_backend", None)
if data_parallel_size > 1 and resolved.any_allgather and cache_backend not in (None, "none"):
raise ValueError(
"Diffusion cache acceleration cannot be combined with DLO AllGather "
"across data-parallel ranks because rank-local cache decisions can skip "
"different weight collectives. Disable cache_backend or use rank-local transfer."
)
if (
data_parallel_size > 1
and bool(getattr(config, "enable_prompt_embed_cache", False))
and resolved.offloads(TEXT_ENCODER_COMPONENT)
and resolved.uses_allgather(TEXT_ENCODER_COMPONENT)
):
raise ValueError(
"Prompt embedding cache cannot be combined with text_encoder AllGather "
"across data-parallel ranks: rank-local cache hits would skip different "
"encoder collectives. Use text_encoder weight_transfer='rank-local' or "
"disable enable_prompt_embed_cache."
)
# Actual runtime configs are immutable after construction. Keeping the
# derived object off the public dataclass fields also keeps projections and
# serialization transport-safe while making hot-path accessors constant-time.
try:
setattr(config, "_resolved_diffusion_offload", resolved)
except (AttributeError, TypeError):
pass
return resolved
def resolve_offload_strategy(config: Any) -> OffloadStrategy:
"""Resolve the compact config or compatibility boolean entry points."""
return resolve_offload(config).strategy
def materialize_legacy_offload_flags(config: Any) -> OffloadStrategy:
"""Keep existing strategy readers working after resolving the compact API."""
resolved = resolve_offload(config)
for field, field_strategy in _LEGACY_STRATEGY_FIELDS.items():
setattr(config, field, resolved.strategy is field_strategy)
setattr(config, "pin_cpu_memory", resolved.pin_memory)
return resolved.strategy
def parse_offload_components(value: Collection[str]) -> frozenset[str]:
"""Validate an internal component collection."""
if isinstance(value, (str, Mapping)) or not isinstance(value, Collection):
raise TypeError("diffusion_offload_config.components must be a non-empty list of component names")
if any(not isinstance(item, str) for item in value):
raise TypeError("offload component entries must be strings")
components = list(value)
if not components:
raise ValueError("offload components must not be empty")
unknown = sorted(set(components) - OFFLOAD_COMPONENTS)
if unknown:
choices = ", ".join(sorted(OFFLOAD_COMPONENTS))
raise ValueError(f"Unknown diffusion offload component(s): {', '.join(unknown)}; choose from: {choices}")
return frozenset(components)
def parse_dlo_transfer(value: Mapping[str, str | DLOTransfer]) -> dict[str, DLOTransfer]:
"""Resolve an internal per-component transfer mapping."""
if not isinstance(value, Mapping):
raise TypeError(f"offload transfers must be a mapping, got {type(value).__name__}")
_validate_string_keys(value, "offload transfers")
missing = sorted(OFFLOAD_COMPONENTS - set(value))
if missing:
raise ValueError("offload transfers require every component; missing: " + ", ".join(missing))
resolved: dict[str, DLOTransfer] = {}
for component, raw_transfer in value.items():
if component not in OFFLOAD_COMPONENTS:
choices = ", ".join(sorted(OFFLOAD_COMPONENTS))
raise ValueError(f"Unknown offload transfer component {component!r}; choose from: {choices}")
resolved[component] = _parse_enum(raw_transfer, DLOTransfer, "offload transfer")
return resolved
def component_uses_allgather(config: Any, component: str = DIT_COMPONENT) -> bool:
"""Return whether one selected component uses AllGather transport."""
resolved = resolve_offload(config)
if not resolved.offloads(component):
raise ValueError(f"Offload component {component!r} is not selected")
return resolved.uses_allgather(component)
def selected_offload_components(config: Any) -> frozenset[str]:
"""Resolve selected components while preserving legacy topology defaults."""
return resolve_offload(config).components
def should_offload_component(config: Any, component: str) -> bool:
"""Return whether an active layer policy selects ``component``."""
if component not in OFFLOAD_COMPONENTS:
raise ValueError(f"Unknown offload component: {component}")
resolved = resolve_offload(config)
if resolved.strategy not in {
OffloadStrategy.LAYER_WISE,
OffloadStrategy.DISTRIBUTED_LAYER_WISE,
}:
return False
return resolved.offloads(component)
def any_selected_component_uses_allgather(config: Any) -> bool:
"""Return whether an enabled layer backend requires weight collectives."""
return resolve_offload(config).any_allgather
File diff suppressed because it is too large. Load diff
+315 -249
View File
@@ -7,14 +7,41 @@ from typing import Any
import torch
from torch import nn
from torch.distributed.tensor import DTensor
from vllm.logger import init_logger
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
from .block_discovery import (
get_blocks_attr_names,
get_blocks_from_dit,
set_blocks_attr_names,
)
from .component_utils import (
clear_encoder_layerwise_state,
get_encoder_block_groups,
iter_streamable_dits,
move_non_block_state_to_device,
prepare_pipeline_components,
set_encoder_layerwise_state,
)
from .config import DIT_COMPONENT
from .module_collector import ModuleDiscovery
from .offload_plan import OffloadPlan, get_offload_plan
from .tensor_utils import (
clear_block_storage,
clear_tensor_storage,
describe_tensor_storage,
flatten_physical_storage,
group_named_tensors_by_dtype,
is_materialized_tensor,
materialization_probe,
module_materialization_probe,
restore_tensor_storage,
set_tensor_storage,
tensor_storage_metadata,
)
logger = init_logger(__name__)
@@ -41,6 +68,7 @@ class LayerwiseOffloadHook(ModelHook):
device: torch.device,
stream: current_omni_platform.Stream | None = None,
pin_memory: bool = True,
materialization_probe_tensor: torch.Tensor | None = None,
):
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
@@ -59,31 +87,7 @@ class LayerwiseOffloadHook(ModelHook):
self.next_block_buffers: dict[str, torch.Tensor] = {}
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
@staticmethod
def _is_dtensor(t: torch.Tensor) -> bool:
return isinstance(t, DTensor)
@staticmethod
def _set_tensor_storage(target: torch.Tensor, value: torch.Tensor) -> None:
if LayerwiseOffloadHook._is_dtensor(target):
target._local_tensor = value
else:
target.data = value
@staticmethod
def _make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
if LayerwiseOffloadHook._is_dtensor(tensor):
local_shape = tuple(tensor.to_local().shape)
return torch.empty(local_shape, device="meta", dtype=tensor.dtype)
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
@staticmethod
def _is_materialized_tensor(t: torch.Tensor) -> bool:
if LayerwiseOffloadHook._is_dtensor(t):
local_t = t.to_local()
return not local_t.is_meta
return not t.is_meta and t.data.numel() > 0
self._materialization_probe = materialization_probe_tensor
def initialize_hook(self, module: nn.Module) -> nn.Module:
# This all happen during the hook instance being registered to hook registry;
@@ -92,6 +96,8 @@ class LayerwiseOffloadHook(ModelHook):
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
if self._materialization_probe is None:
self._materialization_probe = materialization_probe(self.block_parameters, self.block_buffers)
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
@@ -100,7 +106,6 @@ class LayerwiseOffloadHook(ModelHook):
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
self.next_block_parameters,
self.next_block_buffers,
self.device,
self.pin_memory,
)
@@ -110,7 +115,6 @@ class LayerwiseOffloadHook(ModelHook):
def _to_cpu(
params: dict[str, nn.Parameter],
bufs: dict[str, torch.Tensor],
device: torch.device,
pin_memory: bool = True,
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
@@ -123,85 +127,39 @@ class LayerwiseOffloadHook(ModelHook):
flattened CPU tensors by dtype,
metadata for reconstruction by dtype
"""
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
# NOTE: order does matter
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
targets_to_offload: list[torch.Tensor] = []
for name, param_or_buf in chain(params.items(), bufs.items()):
dtype = param_or_buf.dtype
if dtype not in dtype_grouped_weights:
dtype_grouped_weights[dtype] = {}
dtype_grouped_weights[dtype][name] = param_or_buf
for dtype, name2weights in dtype_grouped_weights.items():
for dtype, named_weights in group_named_tensors_by_dtype(params, bufs).items():
# total # of parameters + buffers
weights_with_local = []
for name, t in name2weights.items():
local_t = t.to_local() if hasattr(t, "to_local") else t
stride = local_t.stride()
storage_numel = (
0
if local_t.numel() == 0
else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride))
)
weights_with_local.append((name, t, local_t, storage_numel, stride))
total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local)
specs = describe_tensor_storage(named_weights)
total_numel = sum(spec.storage_numel for spec in specs)
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
current_offset = 0
for (
name,
original_tensor,
local_tensor,
storage_numel,
stride,
) in weights_with_local:
if local_tensor.is_contiguous():
flat_storage = local_tensor.flatten()
else:
# Cutlass FP8 weights use a transposed physical layout.
# Preserve it across the flattened CPU staging buffer.
flat_storage = torch.zeros(
storage_numel,
dtype=dtype,
device=local_tensor.device,
)
physical_view = torch.as_strided(
flat_storage,
size=local_tensor.shape,
stride=stride,
)
physical_view.copy_(local_tensor)
cpu_tensor[current_offset : current_offset + storage_numel].copy_(flat_storage)
if dtype not in dtype_metadata:
dtype_metadata[dtype] = []
dtype_metadata[dtype].append(
{
"name": name,
"offset": current_offset,
"numel": storage_numel,
"shape": local_tensor.shape,
"stride": stride,
}
for spec in specs:
flat_storage = flatten_physical_storage(spec.value, spec.storage_numel)
cpu_tensor[current_offset : current_offset + spec.storage_numel].copy_(flat_storage)
dtype_metadata.setdefault(dtype, []).append(
tensor_storage_metadata(spec, current_offset, include_device=True)
)
LayerwiseOffloadHook._set_tensor_storage(
original_tensor, LayerwiseOffloadHook._make_offload_placeholder(original_tensor)
)
current_offset += storage_numel
targets_to_offload.append(spec.target)
current_offset += spec.storage_numel
dtype_cpu_flattened_weights[dtype] = cpu_tensor
# Do not mutate the module until every host master has been built.
# Allocation/copy failures therefore leave the input module intact.
clear_tensor_storage(targets_to_offload)
return dtype_cpu_flattened_weights, dtype_metadata
@property
def is_materialized(self) -> bool:
"""Check whether this block's parameters hold real data on device."""
for param in self.block_parameters.values():
return LayerwiseOffloadHook._is_materialized_tensor(param)
return True
return self._materialization_probe is None or is_materialized_tensor(self._materialization_probe)
@torch.compiler.disable
def prefetch_layer(self, non_blocking: bool = True) -> None:
@@ -236,7 +194,7 @@ class LayerwiseOffloadHook(ModelHook):
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
)
LayerwiseOffloadHook._set_tensor_storage(
set_tensor_storage(
target_param_or_buf,
torch.as_strided(
gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]],
@@ -252,17 +210,27 @@ class LayerwiseOffloadHook(ModelHook):
"""Free GPU memory for layer by replacing tensors with empty placeholders.
This function does not actually offload weights from GPU back to CPU.
"""
evt = self._prefetch_done
if evt is not None:
current_omni_platform.current_stream().wait_event(evt)
clear_block_storage(self.block_parameters, self.block_buffers, self._prefetch_done)
self._prefetch_done = None
# free GPU residency
for _, param in self.block_parameters.items():
LayerwiseOffloadHook._set_tensor_storage(param, LayerwiseOffloadHook._make_offload_placeholder(param))
for _, buf in self.block_buffers.items():
LayerwiseOffloadHook._set_tensor_storage(buf, LayerwiseOffloadHook._make_offload_placeholder(buf))
@torch.compiler.disable
def restore_next_block(self) -> None:
"""Detach the next block from this hook's host backing store."""
for dtype, ordered_metadata in self.dtype_metadata.items():
flat = self.dtype_cpu_flattened_weights[dtype]
for metadata in ordered_metadata:
value = torch.as_strided(
flat[metadata["offset"] : metadata["offset"] + metadata["numel"]],
size=metadata["shape"],
stride=metadata["stride"],
)
target_name = metadata["name"]
target = (
self.next_block_parameters[target_name]
if target_name in self.next_block_parameters
else self.next_block_buffers[target_name]
)
restore_tensor_storage(target, value, device=metadata["device"])
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
# if the previous hook was skipped and the weights are not on device,
@@ -287,9 +255,17 @@ def apply_block_hook(
device: torch.device,
stream: current_omni_platform.Stream | None = None,
pin_memory: bool = True,
*,
materialization_probe_tensor: torch.Tensor | None = None,
) -> LayerwiseOffloadHook:
registry = HookRegistry.get_or_create(module)
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
hook = LayerwiseOffloadHook(
next_block,
device,
stream,
pin_memory,
materialization_probe_tensor,
)
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
return hook
@@ -302,6 +278,135 @@ def remove_block_hook(module: nn.Module) -> None:
logger.debug("Removed offload hook from %s", module.__class__.__name__)
def _install_layerwise_hook_group(
blocks: list[nn.Module] | nn.ModuleList,
device: torch.device,
stream: Any,
pin_memory: bool,
) -> list[LayerwiseOffloadHook]:
"""Install one circular hook ring and roll it back transactionally."""
block_list = list(blocks)
if len(block_list) <= 1:
raise ValueError("A layerwise hook group requires at least two blocks")
hooks: list[LayerwiseOffloadHook] = []
hooked_blocks: list[nn.Module] = []
probes = {id(block): module_materialization_probe(block) for block in block_list}
try:
for block, next_block in zip(
chain((block_list[-1],), block_list[:-1]),
block_list,
strict=True,
):
hooks.append(
apply_block_hook(
block,
next_block,
device,
stream,
pin_memory,
materialization_probe_tensor=probes[id(block)],
)
)
hooked_blocks.append(block)
except BaseException:
run_cleanup_steps(
[
*(("restoring a partially installed layerwise block", hook.restore_next_block) for hook in hooks),
*(
("removing a partially installed layerwise hook", lambda block=block: remove_block_hook(block))
for block in hooked_blocks
),
]
)
raise
for index, hook in enumerate(hooks):
hook._prev_hook = hooks[index - 1]
return hooks
def enable_plan_encoder_layerwise_offload(
module: nn.Module,
name: str,
plan: OffloadPlan | None,
*,
device: torch.device,
stream: current_omni_platform.Stream,
pin_memory: bool,
stage_on_demand: bool = False,
strict: bool = False,
) -> bool:
"""Apply rank-local layerwise hooks to plan-declared encoder stacks."""
if getattr(module, "_omni_layerwise_enabled", False):
return True
hooks: list[LayerwiseOffloadHook] = []
hooked_blocks: list[nn.Module] = []
block_groups = get_encoder_block_groups(
module,
name,
plan,
strict=strict,
)
if not block_groups:
return False
try:
for blocks in block_groups:
group_hooks = _install_layerwise_hook_group(blocks, device, stream, pin_memory)
hooks.extend(group_hooks)
hooked_blocks.extend(blocks)
if not stage_on_demand:
move_non_block_state_to_device(module, block_groups, device)
except BaseException:
run_cleanup_steps(
[
*(("restoring a partially installed encoder block", hook.restore_next_block) for hook in hooks),
*(
("removing a partially installed encoder hook", lambda block=block: remove_block_hook(block))
for block in hooked_blocks
),
]
)
raise
set_encoder_layerwise_state(
module,
hooks,
block_groups,
)
logger.info(
"Enabled rank-local layerwise offload for encoder %s (%d blocks across %d stacks)",
name,
sum(len(blocks) for blocks in block_groups),
len(block_groups),
)
return True
def disable_plan_encoder_layerwise_offload(
module: nn.Module,
*,
restore_weights: bool = True,
) -> None:
"""Remove hooks installed by :func:`enable_plan_encoder_layerwise_offload`."""
if not getattr(module, "_omni_layerwise_enabled", False):
return
hooks = getattr(module, "_omni_layerwise_hooks", [])
block_groups = getattr(module, "_omni_layerwise_block_groups", [])
steps = []
if restore_weights:
steps.extend(("restoring an encoder block", hook.restore_next_block) for hook in hooks)
steps.extend(
("removing an encoder hook", lambda block=block: remove_block_hook(block))
for blocks in block_groups
for block in blocks
)
cleanup_error = run_cleanup_steps(steps)
if cleanup_error is not None:
raise RuntimeError("Failed to fully disable encoder layerwise offload") from cleanup_error
clear_encoder_layerwise_state(module)
class LayerWiseOffloadBackend(OffloadBackend):
"""Layer-wise (block-level) offloading backend.
@@ -315,56 +420,88 @@ class LayerWiseOffloadBackend(OffloadBackend):
self.copy_stream = current_omni_platform.Stream()
self._blocks: list[list[nn.Module]] = []
self._dit_hooks: list[LayerwiseOffloadHook] = []
self._hooked_dit_blocks: list[nn.Module] = []
self._encoder_modules: list[nn.Module] = []
self._staged_components: list[nn.Module] = []
def enable(self, pipeline: nn.Module) -> None:
try:
self._enable(pipeline)
except BaseException:
try:
self.disable()
except BaseException:
logger.exception("LayerWiseOffloadBackend cleanup failed while handling an enable failure")
raise
def _enable(self, pipeline: nn.Module) -> None:
if self.enabled:
logger.warning("LayerWiseOffloadBackend already enabled")
return
modules = ModuleDiscovery.discover(pipeline)
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
plan = get_offload_plan(pipeline)
if not modules.dits and self.config.offloads(DIT_COMPONENT):
message = "No DiT/transformer modules found for selected DiT layerwise offload"
if self.config.components is not None:
raise ValueError(message)
logger.warning(message)
return
# Move encoders to GPU (they stay resident)
for enc in modules.encoders:
enc.to(self.device)
def enable_encoder_blocks(
module: nn.Module,
name: str,
component_plan: OffloadPlan | None,
stage_on_demand: bool,
) -> bool:
enabled = enable_plan_encoder_layerwise_offload(
module,
name,
component_plan,
device=self.device,
stream=self.copy_stream,
pin_memory=self.config.pin_cpu_memory,
stage_on_demand=stage_on_demand,
strict=self.config.components is not None,
)
if enabled:
# Record each successful installation immediately so a later
# component failure can remove these hooks transactionally.
self._encoder_modules.append(module)
return enabled
# Move VAE(s) to GPU if available
for vae in modules.vaes:
try:
vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
prepare_pipeline_components(
modules,
self.config,
plan,
device=self.device,
staged_components=self._staged_components,
enable_encoder_blocks=enable_encoder_blocks,
)
# Move resident modules to GPU (small modules needed every forward)
for name, module in zip(modules.resident_names, modules.resident_modules):
try:
module.to(self.device)
except Exception as exc:
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
if not self.config.offloads(DIT_COMPONENT):
self.enabled = bool(self._encoder_modules or self._staged_components)
if not self.enabled:
raise ValueError(
"None of the selected layerwise offload components have "
"a model-declared streamable or on-demand plan"
)
return
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
# Note that there might exist multiple DiT models in specific pipelines
for i, dit_module in enumerate(modules.dits):
dit_name = modules.dit_names[i]
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
if not blocks:
logger.warning(
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
dit_name,
dit_module.__class__.__name__,
)
dit_module.to(self.device)
continue
for dit_name, dit_module, blocks_attr_names, blocks in iter_streamable_dits(
modules, self.config, self.device, plan
):
num_blocks = len(blocks)
if num_blocks <= 1:
if self.config.components is not None:
raise ValueError(
f"Selected DiT {dit_name!r} requires at least two streamable layerwise-offload blocks"
)
logger.warning(
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
dit_name,
@@ -390,137 +527,66 @@ class LayerWiseOffloadBackend(OffloadBackend):
if buffer is not None:
buffer.data = buffer.data.to(self.device, non_blocking=True)
# Pre-fetch the first layer by manually calling the hook function on the last layer;
# For subsequent requests, the first layer/block will be pre-fetched
# during the last layer compute of the previous request.
last_block, first_block = blocks[-1], blocks[0]
last_hook = apply_block_hook(
last_block,
first_block,
block_hooks = _install_layerwise_hook_group(
blocks,
self.device,
self.copy_stream,
self.config.pin_cpu_memory,
)
last_hook.prefetch_layer(non_blocking=False)
self._dit_hooks.extend(block_hooks)
self._hooked_dit_blocks.extend(blocks)
block_hooks: list[LayerwiseOffloadHook] = [last_hook]
# Register hook for each of blocks
for i, block in enumerate(blocks[:-1]):
next_block = blocks[(i + 1) % num_blocks]
hook = apply_block_hook(
block,
next_block,
self.device,
self.copy_stream,
self.config.pin_cpu_memory,
)
block_hooks.append(hook)
# NOTE(yuanheng-zhao): We make each hook gets a backward reference to the hook
# that is responsible for prefetching its block's weights. This is specifically a
# workaround for that arbitrary blocks are skipped by caching systems (e.g., cache-dit)
for i in range(len(block_hooks)):
block_hooks[i]._prev_hook = block_hooks[i - 1]
# The last block owns block zero's host backing. Materialize block
# zero once; later denoising iterations prefetch it from the ring.
block_hooks[0].prefetch_layer(non_blocking=False)
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
# Track hooked blocks for cleanup
self._blocks.append(blocks)
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
self.enabled = True
self.enabled = bool(self._blocks or self._encoder_modules or self._staged_components)
def disable(self) -> None:
if not self.enabled:
def _disable(self, *, restore_weights: bool) -> None:
if not self.enabled and not (
self._dit_hooks or self._hooked_dit_blocks or self._encoder_modules or self._staged_components
):
return
for blocks in self._blocks:
for block in blocks:
remove_block_hook(block)
steps = []
if restore_weights:
steps.extend(("restoring a DiT block", hook.restore_next_block) for hook in self._dit_hooks)
steps.extend(
("removing a DiT block hook", lambda block=block: remove_block_hook(block))
for block in self._hooked_dit_blocks
)
steps.extend(
(
"disabling encoder layerwise offload",
lambda module=module: disable_plan_encoder_layerwise_offload(
module,
restore_weights=restore_weights,
),
)
for module in self._encoder_modules
)
cleanup_error = run_cleanup_steps(steps)
if cleanup_error is not None:
# Keep every host-master reference so a transient teardown error
# can be retried without losing weights behind placeholders.
raise RuntimeError("Failed to fully disable layerwise offload") from cleanup_error
self._blocks.clear()
self._dit_hooks.clear()
self._hooked_dit_blocks.clear()
self._encoder_modules.clear()
self._staged_components.clear()
self.enabled = False
logger.info("Layer-wise offloading disabled")
@staticmethod
def get_blocks_attr_names(model: nn.Module) -> list[str]:
"""Get block attribute names from model class."""
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
def disable(self) -> None:
self._disable(restore_weights=True)
if not attrs:
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
if old_attr is not None:
logger.warning(
"'_layerwise_offload_blocks_attr' is deprecated, "
"please use '_layerwise_offload_blocks_attrs' instead. "
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
)
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
return attrs
@staticmethod
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
@staticmethod
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
"""
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
```
class WanTransformer3DModel(nn.Module):
_layerwise_offload_blocks_attrs = ["blocks"]
```
Returns:
Tuple of (blocks_attr_names, blocks)
"""
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
if not blocks_attr_names:
logger.warning(
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
"skipping layerwise offloading"
)
return [], []
blocks = []
for name in blocks_attr_names:
attr = getattr(model, name, None)
if attr is None:
raise AttributeError(
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
f"does not exist on model {model.__class__.__name__}"
)
try:
attr_iter = iter(attr)
except TypeError:
if isinstance(attr, nn.Module):
logger.warning(
"Attribute '%s' on %s is not iterable; treating it as one block.",
name,
model.__class__.__name__,
)
blocks.append(attr)
continue
logger.warning(
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
name,
model.__class__.__name__,
type(attr).__name__,
)
else:
blocks.extend(attr_iter)
if not blocks:
logger.warning(
"No blocks found in %s for %s, skipping layerwise offloading",
blocks_attr_names,
model.__class__.__name__,
)
return [], []
return blocks_attr_names, blocks
# Compatibility aliases for existing model integrations.
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
get_blocks_from_dit = staticmethod(get_blocks_from_dit)
+17 -11
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Declarative OffloadPlan for distributed layerwise offload.
"""Declarative model capabilities for component layerwise offload.
Models declare this as a class attribute ``_offload_plan`` on the
pipeline class. When present, the offloader uses it instead of
heuristic block discovery.
Models declare an ``_offload_plan`` class attribute on the pipeline. Both the
ordinary and distributed layerwise backends consume the same plan instead of
embedding model-specific block paths in backend code.
"""
from __future__ import annotations
@@ -16,12 +16,11 @@ from torch import nn
@dataclass(frozen=True)
class OffloadPlan:
"""Optional declarative metadata for distributed layerwise offload.
"""Optional declarative metadata for component layerwise offload.
Models declare this as a class attribute ``_offload_plan`` on the
pipeline class. When present, the offloader uses it instead of
heuristic block discovery, making it easier to support new models
without offload-specific logic in constructor or forward code.
pipeline class. When present, both layerwise backends use it instead of
model-specific backend branches, making new integrations data-driven.
If not declared, the offloader falls back to:
1. ``_layerwise_offload_blocks_attrs`` on each DiT module class.
@@ -39,9 +38,14 @@ class OffloadPlan:
device when ``dlo_resident_layers`` is nonzero. Keeping this
model-declared avoids applying a consumer-GPU tuning knob to
auxiliary or dual DiTs unintentionally.
encoder_block_attrs: Maps encoder paths to rank-local block-list paths.
These blocks are streamed with ordinary layerwise hooks, never
with the DiT AllGather group.
encoder_component_types: Maps encoder paths to public selector types
(currently text_encoder). This declaration is used before the
compatibility name heuristic.
encoder_block_attrs: Maps encoder paths to streamable block-list paths.
encoder_dlo_weight_replication: Encoder paths whose loader-produced
block tensors are identical across the DiT DLO group. Only these
encoders may use multi-rank AllGather transfer; this must not be
declared for encoder-TP shards.
"""
on_demand_component_paths: frozenset[str] = field(default_factory=frozenset)
@@ -49,7 +53,9 @@ class OffloadPlan:
block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
offload_submodules: dict[str, str] = field(default_factory=dict)
resident_dit_paths: frozenset[str] = field(default_factory=frozenset)
encoder_component_types: dict[str, str] = field(default_factory=dict)
encoder_block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
encoder_dlo_weight_replication: frozenset[str] = field(default_factory=frozenset)
def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None:
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from collections.abc import Collection, Iterator
from contextlib import contextmanager
import torch
@@ -13,11 +13,38 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .config import DIT_COMPONENT, TEXT_ENCODER_COMPONENT
from .module_collector import ModuleDiscovery
from .offload_plan import get_offload_plan
logger = init_logger(__name__)
def _capture_tensor_devices(modules: Collection[nn.Module]) -> list[tuple[torch.Tensor, torch.device]]:
seen: set[int] = set()
devices: list[tuple[torch.Tensor, torch.device]] = []
for module in modules:
for tensor in (*module.parameters(), *module.buffers()):
if id(tensor) in seen:
continue
seen.add(id(tensor))
devices.append((tensor, tensor.device))
return devices
def _restore_tensor_devices(devices: list[tuple[torch.Tensor, torch.device]]) -> None:
first_error: BaseException | None = None
for tensor, device in devices:
try:
if tensor.device != device:
tensor.data = tensor.data.to(device)
except BaseException as exc:
logger.exception("Failed to restore tensor placement to %s", device)
first_error = first_error or exc
if first_error is not None:
raise RuntimeError("Failed to restore one or more tensor placements") from first_error
class SequentialOffloadHook(ModelHook):
"""Hook for sequential offloading with mutual exclusion on encoder and DiT modules.
@@ -34,12 +61,14 @@ class SequentialOffloadHook(ModelHook):
device: torch.device,
pin_memory: bool = True,
use_hsdp: bool = False,
offload_after_context: bool = True,
):
# Modules to offload to CPU before this module runs
self.offload_targets = offload_targets
self.device = device
self.pin_memory = pin_memory
self.use_hsdp = use_hsdp
self.offload_after_context = offload_after_context
@staticmethod
def _move_params(
@@ -48,7 +77,7 @@ class SequentialOffloadHook(ModelHook):
*,
non_blocking: bool = False,
pin_memory: bool = False,
) -> None:
) -> bool:
"""Move module parameters and buffers to device.
This cls method specifically prevents recursion device movement,
@@ -57,47 +86,38 @@ class SequentialOffloadHook(ModelHook):
refer to
https://github.com/vipshop/cache-dit/blob/v1.2.3/src/cache_dit/caching/cache_blocks/__init__.py#L83
"""
moved = False
for p in module.parameters():
if p.data.device != target_device:
data = p.data.to(target_device, non_blocking=non_blocking)
if pin_memory and target_device.type == "cpu" and not isinstance(data, DTensor):
data = data.pin_memory()
p.data = data
moved = True
for b in module.buffers():
if b.device != target_device:
data = b.data.to(target_device, non_blocking=non_blocking)
if pin_memory and target_device.type == "cpu" and not isinstance(data, DTensor):
data = data.pin_memory()
b.data = data
moved = True
return moved
def _to_cpu(self, module: nn.Module) -> None:
try:
param = next(module.parameters())
except StopIteration:
return
if param.device.type == "cpu":
return
# XPU's allocator doesn't respect stream dependencies in empty_cache,
# so non-blocking copies can race with cache eviction. Use blocking
# copies on XPU to avoid NULL pointer errors during DMA.
non_blocking = not self.use_hsdp and not current_omni_platform.is_xpu()
self._move_params(
moved = self._move_params(
module,
torch.device("cpu"),
non_blocking=non_blocking,
pin_memory=self.pin_memory,
)
current_omni_platform.empty_cache()
if moved:
current_omni_platform.empty_cache()
def _to_gpu(self, module: nn.Module) -> None:
try:
if next(module.parameters()).device == self.device:
return
except StopIteration:
return
self._move_params(module, self.device, non_blocking=False)
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
@@ -127,6 +147,8 @@ def apply_sequential_offload(
pin_memory: bool = True,
use_hsdp: bool = False,
offload_initial_dits: bool = False,
offload_dit_modules: Collection[nn.Module] | None = None,
offload_encoder_modules: Collection[nn.Module] | None = None,
) -> None:
"""Apply sequential offloading hooks to DiT and encoder modules.
@@ -141,6 +163,10 @@ def apply_sequential_offload(
pin_memory: Whether to pin CPU memory for faster transfers
use_hsdp: Whether HSDP is enabled (affects non_blocking behavior)
offload_initial_dits: Whether to begin with all DiT modules on CPU.
offload_dit_modules: DiT modules allowed to move to CPU. None selects
every DiT for backward compatibility.
offload_encoder_modules: Encoder/stage modules allowed to move to CPU.
None selects every supplied module for backward compatibility.
Example:
>>> apply_sequential_offload(
@@ -150,38 +176,59 @@ def apply_sequential_offload(
... )
>>> # Modules of pipeline now automatically swap between CPU and GPU
"""
# Register hooks on DiT modules (offload encoders AND other DiTs when a DiT runs)
for i, dit_mod in enumerate(dit_modules):
other_dits = [d for j, d in enumerate(dit_modules) if j != i]
registry = HookRegistry.get_or_create(dit_mod)
hook = SequentialOffloadHook(
offload_targets=encoder_modules + other_dits,
device=device,
pin_memory=pin_memory,
use_hsdp=use_hsdp,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
selected_dit_ids = {id(module) for module in (dit_modules if offload_dit_modules is None else offload_dit_modules)}
selected_encoder_ids = {
id(module) for module in (encoder_modules if offload_encoder_modules is None else offload_encoder_modules)
}
# Register hooks on encoders (offload DiTs when encoder runs)
for enc in encoder_modules:
registry = HookRegistry.get_or_create(enc)
hook = SequentialOffloadHook(
offload_targets=dit_modules,
device=device,
pin_memory=pin_memory,
use_hsdp=use_hsdp,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
all_modules = [*dit_modules, *encoder_modules]
initial_devices = _capture_tensor_devices(dit_modules) if offload_initial_dits else []
try:
selected_encoders = [encoder for encoder in encoder_modules if id(encoder) in selected_encoder_ids]
# Register hooks on DiT modules (offload selected encoders and other selected DiTs).
for i, dit_mod in enumerate(dit_modules):
other_dits = [d for j, d in enumerate(dit_modules) if j != i and id(d) in selected_dit_ids]
registry = HookRegistry.get_or_create(dit_mod)
hook = SequentialOffloadHook(
offload_targets=selected_encoders + other_dits,
device=device,
pin_memory=pin_memory,
use_hsdp=use_hsdp,
offload_after_context=id(dit_mod) in selected_dit_ids,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
if offload_initial_dits:
try:
# Register hooks on all execution stages so unselected resident modules can
# still evict selected DiTs before they run.
selected_dits = [dit for dit in dit_modules if id(dit) in selected_dit_ids]
for enc in encoder_modules:
registry = HookRegistry.get_or_create(enc)
hook = SequentialOffloadHook(
offload_targets=selected_dits,
device=device,
pin_memory=pin_memory,
use_hsdp=use_hsdp,
offload_after_context=id(enc) in selected_encoder_ids,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
if offload_initial_dits:
for dit_mod in dit_modules:
if id(dit_mod) not in selected_dit_ids:
continue
_get_sequential_offload_hook(dit_mod)._to_cpu(dit_mod)
except Exception:
remove_sequential_offload([*dit_modules, *encoder_modules])
raise
except BaseException:
try:
remove_sequential_offload(all_modules)
except BaseException:
logger.exception("Failed to remove every sequential hook during rollback")
try:
_restore_tensor_devices(initial_devices)
except BaseException:
logger.exception("Failed to restore initial DiT placement during rollback")
raise
def remove_sequential_offload(modules: list[nn.Module]) -> None:
@@ -194,11 +241,18 @@ def remove_sequential_offload(modules: list[nn.Module]) -> None:
>>> all_modules = [*dit_modules, *encoder_modules]
>>> remove_sequential_offload(all_modules)
"""
first_error: BaseException | None = None
for module in modules:
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
if registry is not None:
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
logger.debug("Removed offload hook from %s", module.__class__.__name__)
try:
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
if registry is not None:
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
logger.debug("Removed offload hook from %s", module.__class__.__name__)
except BaseException as exc:
logger.exception("Failed to remove offload hook from %s", module.__class__.__name__)
first_error = first_error or exc
if first_error is not None:
raise RuntimeError("Failed to remove one or more sequential offload hooks") from first_error
def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
@@ -218,12 +272,14 @@ def sequential_offload_component(module: nn.Module) -> Iterator[None]:
yield
except BaseException:
try:
hook._to_cpu(module)
if hook.offload_after_context:
hook._to_cpu(module)
except Exception:
logger.exception("Failed to release %s after component failure", module.__class__.__name__)
raise
else:
hook._to_cpu(module)
if hook.offload_after_context:
hook._to_cpu(module)
class ModelLevelOffloadBackend(OffloadBackend):
@@ -245,11 +301,26 @@ class ModelLevelOffloadBackend(OffloadBackend):
# Pipelines with non-forward component entry points own their complete
# mutual-exclusion lifecycle. Delegate through the explicit protocol.
if isinstance(pipeline, SupportsModelCpuOffload):
pipeline.enable_omni_model_cpu_offload(
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
)
try:
if self.config.components is not None:
pipeline.enable_omni_model_cpu_offload(
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
offload_components=self.config.components,
)
else:
pipeline.enable_omni_model_cpu_offload(
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
)
except BaseException:
try:
pipeline.disable_omni_model_cpu_offload()
except BaseException:
logger.exception("Model-level cleanup failed while handling an enable failure")
raise
self._custom_pipeline = pipeline
self.enabled = True
logger.info(
@@ -259,44 +330,65 @@ class ModelLevelOffloadBackend(OffloadBackend):
return
modules = ModuleDiscovery.discover(pipeline)
plan = get_offload_plan(pipeline)
selected_encoders = [
encoder
for encoder, name in zip(modules.encoders, modules.encoder_names)
if self.config.should_offload_encoder(name, plan)
]
if self.config.components is not None:
if not modules.dits:
raise ValueError("Component-selective model offload requires a DiT/transformer module")
if not modules.encoders:
raise ValueError("Component-selective model offload requires an encoder execution stage")
if self.config.offloads(TEXT_ENCODER_COMPONENT) and not selected_encoders:
raise ValueError("No text encoder modules found for selected text_encoder module offload")
# Move encoders to GPU
for enc in modules.encoders:
enc.to(self.device)
# Move VAE(s) to GPU if available
for vae in modules.vaes:
try:
all_modules = [
*modules.dits,
*modules.encoders,
*modules.vaes,
*modules.resident_modules,
]
initial_devices = _capture_tensor_devices(all_modules)
try:
for encoder in modules.encoders:
encoder.to(self.device)
for vae in modules.vaes:
vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
for resident in modules.resident_modules:
resident.to(self.device)
# Pin resident modules on GPU (small hot submodules called inside the DiT loop).
for res, name in zip(modules.resident_modules, modules.resident_names):
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
return
if not modules.encoders:
for dit in modules.dits:
dit.to(self.device)
logger.warning("No encoder modules found, skipping model-level offloading")
return
apply_sequential_offload(
dit_modules=modules.dits,
encoder_modules=modules.encoders,
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
offload_dit_modules=(
modules.dits if self.config.components is None or self.config.offloads(DIT_COMPONENT) else ()
),
offload_encoder_modules=selected_encoders,
)
except BaseException:
try:
res.to(self.device)
except Exception as exc:
logger.warning("Failed to move resident module '%s' to GPU: %s", name, exc)
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
return
if not modules.encoders:
# Nothing to swap against — move DiTs to GPU and skip hooks.
for dit in modules.dits:
dit.to(self.device)
logger.warning("No encoder modules found, skipping model-level offloading")
return
# Apply sequential offloading hooks
apply_sequential_offload(
dit_modules=modules.dits,
encoder_modules=modules.encoders,
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
)
remove_sequential_offload([*modules.dits, *modules.encoders])
except BaseException:
logger.exception("Failed to remove every model-level hook during rollback")
try:
_restore_tensor_devices(initial_devices)
except BaseException:
logger.exception("Failed to restore model placement during rollback")
raise
# Track modules for cleanup
self._offload_modules = [*modules.dits, *modules.encoders]
@@ -9,15 +9,100 @@ offload backends.
from __future__ import annotations
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from itertools import chain
from typing import Any
import torch
from torch import nn
from torch.distributed.tensor import DTensor
@dataclass(frozen=True)
class TensorStorageSpec:
"""One tensor's target and physical-storage layout."""
name: str
target: torch.Tensor
value: torch.Tensor
storage_numel: int
stride: tuple[int, ...]
def dtype_size(dtype: torch.dtype) -> int:
"""Return element size in bytes for a torch.dtype."""
return torch.empty(1, dtype=dtype).element_size()
def group_named_tensors_by_dtype(
params: Mapping[str, torch.Tensor],
buffers: Mapping[str, torch.Tensor],
) -> dict[torch.dtype, list[tuple[str, torch.Tensor]]]:
"""Group parameters and buffers by dtype while preserving their order."""
grouped: dict[torch.dtype, list[tuple[str, torch.Tensor]]] = {}
for name, tensor in chain(params.items(), buffers.items()):
grouped.setdefault(tensor.dtype, []).append((name, tensor))
return grouped
def physical_storage_numel(tensor: torch.Tensor) -> int:
"""Return the storage span needed to preserve ``tensor``'s stride."""
if tensor.numel() == 0:
return 0
return 1 + sum((size - 1) * stride for size, stride in zip(tensor.shape, tensor.stride()))
def flatten_physical_storage(
tensor: torch.Tensor,
storage_numel: int | None = None,
) -> torch.Tensor:
"""Pack a possibly strided tensor in physical storage order."""
if tensor.is_contiguous():
return tensor.flatten()
storage_numel = physical_storage_numel(tensor) if storage_numel is None else storage_numel
storage = torch.zeros(storage_numel, dtype=tensor.dtype, device=tensor.device)
torch.as_strided(storage, size=tensor.shape, stride=tensor.stride()).copy_(tensor)
return storage
def describe_tensor_storage(
named_tensors: Iterable[tuple[str, torch.Tensor]],
transforms: Mapping[int, Callable[[torch.Tensor], torch.Tensor]] | None = None,
) -> list[TensorStorageSpec]:
"""Resolve local tensors and their transport-preserving layouts."""
specs: list[TensorStorageSpec] = []
for name, target in named_tensors:
value = target.to_local() if hasattr(target, "to_local") else target
transform = (transforms or {}).get(id(target))
if callable(transform):
value = transform(value)
specs.append(
TensorStorageSpec(
name=name,
target=target,
value=value,
storage_numel=physical_storage_numel(value),
stride=value.stride(),
)
)
return specs
def tensor_storage_metadata(spec: TensorStorageSpec, offset: int, *, include_device: bool = False) -> dict[str, Any]:
"""Build reconstruction metadata shared by offload transports."""
metadata: dict[str, Any] = {
"name": spec.name,
"offset": offset,
"numel": spec.storage_numel,
"shape": spec.value.shape,
"stride": spec.stride,
}
if include_device:
metadata["device"] = spec.value.device
return metadata
def is_dtensor(t: torch.Tensor) -> bool:
"""Check if tensor is a DTensor."""
return isinstance(t, DTensor)
@@ -39,9 +124,74 @@ def make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
def clear_tensor_storage(tensors: Iterable[torch.Tensor]) -> None:
"""Atomically replace tensor storage with offload placeholders."""
targets = list(tensors)
originals = [tensor.to_local() if is_dtensor(tensor) else tensor.data for tensor in targets]
placeholders = [make_offload_placeholder(tensor) for tensor in targets]
committed = 0
try:
for target, placeholder in zip(targets, placeholders, strict=True):
set_tensor_storage(target, placeholder)
committed += 1
except BaseException:
rollback_error: BaseException | None = None
for target, original in zip(targets[:committed], originals[:committed], strict=True):
try:
set_tensor_storage(target, original)
except BaseException as exc:
rollback_error = rollback_error or exc
if rollback_error is not None:
raise RuntimeError("Tensor storage clear failed and rollback was incomplete") from rollback_error
raise
def clear_block_storage(
params: Mapping[str, torch.Tensor],
buffers: Mapping[str, torch.Tensor],
ready_event: Any | None,
) -> None:
"""Wait for an outstanding transfer, then release one block's residency."""
if ready_event is not None:
from vllm_omni.platforms import current_omni_platform
current_omni_platform.current_stream().wait_event(ready_event)
clear_tensor_storage(chain(params.values(), buffers.values()))
def is_materialized_tensor(t: torch.Tensor) -> bool:
"""Check if tensor holds real data (not meta or empty placeholder)."""
if is_dtensor(t):
local_t = t.to_local()
return not local_t.is_meta
return not t.is_meta and t.data.numel() > 0
def materialization_probe(
params: Mapping[str, torch.Tensor],
buffers: Mapping[str, torch.Tensor],
) -> torch.Tensor | None:
"""Pick one non-empty tensor that tracks a block's atomic storage state."""
return next((tensor for tensor in chain(params.values(), buffers.values()) if tensor.numel()), None)
def module_materialization_probe(module: nn.Module) -> torch.Tensor | None:
"""Capture a probe before another hook releases the module's storage."""
return materialization_probe(dict(module.named_parameters()), dict(module.named_buffers()))
def restore_tensor_storage(
target: torch.Tensor,
value: torch.Tensor,
*,
device: torch.device | str,
) -> None:
"""Detach ``target`` from an offloader-owned backing with a real copy."""
restored = torch.empty_strided(
value.shape,
value.stride(),
dtype=value.dtype,
device=device,
)
restored.copy_(value)
set_tensor_storage(target, restored)