[diffusion][feature] Add component-selective offload policies (#5929)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
44d3ae100a
commit
ea3cf99894
74 files changed
+5121
-1584
No files matched your search
@@ -7,14 +7,41 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.distributed.tensor import DTensor
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig
|
||||
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
|
||||
from .block_discovery import (
|
||||
get_blocks_attr_names,
|
||||
get_blocks_from_dit,
|
||||
set_blocks_attr_names,
|
||||
)
|
||||
from .component_utils import (
|
||||
clear_encoder_layerwise_state,
|
||||
get_encoder_block_groups,
|
||||
iter_streamable_dits,
|
||||
move_non_block_state_to_device,
|
||||
prepare_pipeline_components,
|
||||
set_encoder_layerwise_state,
|
||||
)
|
||||
from .config import DIT_COMPONENT
|
||||
from .module_collector import ModuleDiscovery
|
||||
from .offload_plan import OffloadPlan, get_offload_plan
|
||||
from .tensor_utils import (
|
||||
clear_block_storage,
|
||||
clear_tensor_storage,
|
||||
describe_tensor_storage,
|
||||
flatten_physical_storage,
|
||||
group_named_tensors_by_dtype,
|
||||
is_materialized_tensor,
|
||||
materialization_probe,
|
||||
module_materialization_probe,
|
||||
restore_tensor_storage,
|
||||
set_tensor_storage,
|
||||
tensor_storage_metadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -41,6 +68,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
materialization_probe_tensor: torch.Tensor | None = None,
|
||||
):
|
||||
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
|
||||
|
||||
@@ -59,31 +87,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = {}
|
||||
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _is_dtensor(t: torch.Tensor) -> bool:
|
||||
return isinstance(t, DTensor)
|
||||
|
||||
@staticmethod
|
||||
def _set_tensor_storage(target: torch.Tensor, value: torch.Tensor) -> None:
|
||||
if LayerwiseOffloadHook._is_dtensor(target):
|
||||
target._local_tensor = value
|
||||
else:
|
||||
target.data = value
|
||||
|
||||
@staticmethod
|
||||
def _make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
|
||||
if LayerwiseOffloadHook._is_dtensor(tensor):
|
||||
local_shape = tuple(tensor.to_local().shape)
|
||||
return torch.empty(local_shape, device="meta", dtype=tensor.dtype)
|
||||
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
|
||||
|
||||
@staticmethod
|
||||
def _is_materialized_tensor(t: torch.Tensor) -> bool:
|
||||
if LayerwiseOffloadHook._is_dtensor(t):
|
||||
local_t = t.to_local()
|
||||
return not local_t.is_meta
|
||||
return not t.is_meta and t.data.numel() > 0
|
||||
self._materialization_probe = materialization_probe_tensor
|
||||
|
||||
def initialize_hook(self, module: nn.Module) -> nn.Module:
|
||||
# This all happen during the hook instance being registered to hook registry;
|
||||
@@ -92,6 +96,8 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
|
||||
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
|
||||
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
|
||||
if self._materialization_probe is None:
|
||||
self._materialization_probe = materialization_probe(self.block_parameters, self.block_buffers)
|
||||
|
||||
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
|
||||
@@ -100,7 +106,6 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
|
||||
self.next_block_parameters,
|
||||
self.next_block_buffers,
|
||||
self.device,
|
||||
self.pin_memory,
|
||||
)
|
||||
|
||||
@@ -110,7 +115,6 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
def _to_cpu(
|
||||
params: dict[str, nn.Parameter],
|
||||
bufs: dict[str, torch.Tensor],
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
|
||||
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
|
||||
@@ -123,85 +127,39 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
flattened CPU tensors by dtype,
|
||||
metadata for reconstruction by dtype
|
||||
"""
|
||||
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
|
||||
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
# NOTE: order does matter
|
||||
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
targets_to_offload: list[torch.Tensor] = []
|
||||
|
||||
for name, param_or_buf in chain(params.items(), bufs.items()):
|
||||
dtype = param_or_buf.dtype
|
||||
if dtype not in dtype_grouped_weights:
|
||||
dtype_grouped_weights[dtype] = {}
|
||||
dtype_grouped_weights[dtype][name] = param_or_buf
|
||||
|
||||
for dtype, name2weights in dtype_grouped_weights.items():
|
||||
for dtype, named_weights in group_named_tensors_by_dtype(params, bufs).items():
|
||||
# total # of parameters + buffers
|
||||
weights_with_local = []
|
||||
for name, t in name2weights.items():
|
||||
local_t = t.to_local() if hasattr(t, "to_local") else t
|
||||
stride = local_t.stride()
|
||||
storage_numel = (
|
||||
0
|
||||
if local_t.numel() == 0
|
||||
else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride))
|
||||
)
|
||||
weights_with_local.append((name, t, local_t, storage_numel, stride))
|
||||
total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local)
|
||||
specs = describe_tensor_storage(named_weights)
|
||||
total_numel = sum(spec.storage_numel for spec in specs)
|
||||
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
|
||||
|
||||
current_offset = 0
|
||||
for (
|
||||
name,
|
||||
original_tensor,
|
||||
local_tensor,
|
||||
storage_numel,
|
||||
stride,
|
||||
) in weights_with_local:
|
||||
if local_tensor.is_contiguous():
|
||||
flat_storage = local_tensor.flatten()
|
||||
else:
|
||||
# Cutlass FP8 weights use a transposed physical layout.
|
||||
# Preserve it across the flattened CPU staging buffer.
|
||||
flat_storage = torch.zeros(
|
||||
storage_numel,
|
||||
dtype=dtype,
|
||||
device=local_tensor.device,
|
||||
)
|
||||
physical_view = torch.as_strided(
|
||||
flat_storage,
|
||||
size=local_tensor.shape,
|
||||
stride=stride,
|
||||
)
|
||||
physical_view.copy_(local_tensor)
|
||||
cpu_tensor[current_offset : current_offset + storage_numel].copy_(flat_storage)
|
||||
if dtype not in dtype_metadata:
|
||||
dtype_metadata[dtype] = []
|
||||
dtype_metadata[dtype].append(
|
||||
{
|
||||
"name": name,
|
||||
"offset": current_offset,
|
||||
"numel": storage_numel,
|
||||
"shape": local_tensor.shape,
|
||||
"stride": stride,
|
||||
}
|
||||
for spec in specs:
|
||||
flat_storage = flatten_physical_storage(spec.value, spec.storage_numel)
|
||||
cpu_tensor[current_offset : current_offset + spec.storage_numel].copy_(flat_storage)
|
||||
dtype_metadata.setdefault(dtype, []).append(
|
||||
tensor_storage_metadata(spec, current_offset, include_device=True)
|
||||
)
|
||||
|
||||
LayerwiseOffloadHook._set_tensor_storage(
|
||||
original_tensor, LayerwiseOffloadHook._make_offload_placeholder(original_tensor)
|
||||
)
|
||||
current_offset += storage_numel
|
||||
targets_to_offload.append(spec.target)
|
||||
current_offset += spec.storage_numel
|
||||
|
||||
dtype_cpu_flattened_weights[dtype] = cpu_tensor
|
||||
|
||||
# Do not mutate the module until every host master has been built.
|
||||
# Allocation/copy failures therefore leave the input module intact.
|
||||
clear_tensor_storage(targets_to_offload)
|
||||
|
||||
return dtype_cpu_flattened_weights, dtype_metadata
|
||||
|
||||
@property
|
||||
def is_materialized(self) -> bool:
|
||||
"""Check whether this block's parameters hold real data on device."""
|
||||
for param in self.block_parameters.values():
|
||||
return LayerwiseOffloadHook._is_materialized_tensor(param)
|
||||
|
||||
return True
|
||||
return self._materialization_probe is None or is_materialized_tensor(self._materialization_probe)
|
||||
|
||||
@torch.compiler.disable
|
||||
def prefetch_layer(self, non_blocking: bool = True) -> None:
|
||||
@@ -236,7 +194,7 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
|
||||
)
|
||||
|
||||
LayerwiseOffloadHook._set_tensor_storage(
|
||||
set_tensor_storage(
|
||||
target_param_or_buf,
|
||||
torch.as_strided(
|
||||
gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]],
|
||||
@@ -252,17 +210,27 @@ class LayerwiseOffloadHook(ModelHook):
|
||||
"""Free GPU memory for layer by replacing tensors with empty placeholders.
|
||||
This function does not actually offload weights from GPU back to CPU.
|
||||
"""
|
||||
evt = self._prefetch_done
|
||||
if evt is not None:
|
||||
current_omni_platform.current_stream().wait_event(evt)
|
||||
|
||||
clear_block_storage(self.block_parameters, self.block_buffers, self._prefetch_done)
|
||||
self._prefetch_done = None
|
||||
|
||||
# free GPU residency
|
||||
for _, param in self.block_parameters.items():
|
||||
LayerwiseOffloadHook._set_tensor_storage(param, LayerwiseOffloadHook._make_offload_placeholder(param))
|
||||
for _, buf in self.block_buffers.items():
|
||||
LayerwiseOffloadHook._set_tensor_storage(buf, LayerwiseOffloadHook._make_offload_placeholder(buf))
|
||||
@torch.compiler.disable
|
||||
def restore_next_block(self) -> None:
|
||||
"""Detach the next block from this hook's host backing store."""
|
||||
for dtype, ordered_metadata in self.dtype_metadata.items():
|
||||
flat = self.dtype_cpu_flattened_weights[dtype]
|
||||
for metadata in ordered_metadata:
|
||||
value = torch.as_strided(
|
||||
flat[metadata["offset"] : metadata["offset"] + metadata["numel"]],
|
||||
size=metadata["shape"],
|
||||
stride=metadata["stride"],
|
||||
)
|
||||
target_name = metadata["name"]
|
||||
target = (
|
||||
self.next_block_parameters[target_name]
|
||||
if target_name in self.next_block_parameters
|
||||
else self.next_block_buffers[target_name]
|
||||
)
|
||||
restore_tensor_storage(target, value, device=metadata["device"])
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
|
||||
# if the previous hook was skipped and the weights are not on device,
|
||||
@@ -287,9 +255,17 @@ def apply_block_hook(
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
*,
|
||||
materialization_probe_tensor: torch.Tensor | None = None,
|
||||
) -> LayerwiseOffloadHook:
|
||||
registry = HookRegistry.get_or_create(module)
|
||||
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
|
||||
hook = LayerwiseOffloadHook(
|
||||
next_block,
|
||||
device,
|
||||
stream,
|
||||
pin_memory,
|
||||
materialization_probe_tensor,
|
||||
)
|
||||
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
|
||||
|
||||
return hook
|
||||
@@ -302,6 +278,135 @@ def remove_block_hook(module: nn.Module) -> None:
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
|
||||
|
||||
def _install_layerwise_hook_group(
|
||||
blocks: list[nn.Module] | nn.ModuleList,
|
||||
device: torch.device,
|
||||
stream: Any,
|
||||
pin_memory: bool,
|
||||
) -> list[LayerwiseOffloadHook]:
|
||||
"""Install one circular hook ring and roll it back transactionally."""
|
||||
block_list = list(blocks)
|
||||
if len(block_list) <= 1:
|
||||
raise ValueError("A layerwise hook group requires at least two blocks")
|
||||
|
||||
hooks: list[LayerwiseOffloadHook] = []
|
||||
hooked_blocks: list[nn.Module] = []
|
||||
probes = {id(block): module_materialization_probe(block) for block in block_list}
|
||||
try:
|
||||
for block, next_block in zip(
|
||||
chain((block_list[-1],), block_list[:-1]),
|
||||
block_list,
|
||||
strict=True,
|
||||
):
|
||||
hooks.append(
|
||||
apply_block_hook(
|
||||
block,
|
||||
next_block,
|
||||
device,
|
||||
stream,
|
||||
pin_memory,
|
||||
materialization_probe_tensor=probes[id(block)],
|
||||
)
|
||||
)
|
||||
hooked_blocks.append(block)
|
||||
except BaseException:
|
||||
run_cleanup_steps(
|
||||
[
|
||||
*(("restoring a partially installed layerwise block", hook.restore_next_block) for hook in hooks),
|
||||
*(
|
||||
("removing a partially installed layerwise hook", lambda block=block: remove_block_hook(block))
|
||||
for block in hooked_blocks
|
||||
),
|
||||
]
|
||||
)
|
||||
raise
|
||||
|
||||
for index, hook in enumerate(hooks):
|
||||
hook._prev_hook = hooks[index - 1]
|
||||
return hooks
|
||||
|
||||
|
||||
def enable_plan_encoder_layerwise_offload(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
plan: OffloadPlan | None,
|
||||
*,
|
||||
device: torch.device,
|
||||
stream: current_omni_platform.Stream,
|
||||
pin_memory: bool,
|
||||
stage_on_demand: bool = False,
|
||||
strict: bool = False,
|
||||
) -> bool:
|
||||
"""Apply rank-local layerwise hooks to plan-declared encoder stacks."""
|
||||
if getattr(module, "_omni_layerwise_enabled", False):
|
||||
return True
|
||||
|
||||
hooks: list[LayerwiseOffloadHook] = []
|
||||
hooked_blocks: list[nn.Module] = []
|
||||
block_groups = get_encoder_block_groups(
|
||||
module,
|
||||
name,
|
||||
plan,
|
||||
strict=strict,
|
||||
)
|
||||
if not block_groups:
|
||||
return False
|
||||
try:
|
||||
for blocks in block_groups:
|
||||
group_hooks = _install_layerwise_hook_group(blocks, device, stream, pin_memory)
|
||||
hooks.extend(group_hooks)
|
||||
hooked_blocks.extend(blocks)
|
||||
if not stage_on_demand:
|
||||
move_non_block_state_to_device(module, block_groups, device)
|
||||
except BaseException:
|
||||
run_cleanup_steps(
|
||||
[
|
||||
*(("restoring a partially installed encoder block", hook.restore_next_block) for hook in hooks),
|
||||
*(
|
||||
("removing a partially installed encoder hook", lambda block=block: remove_block_hook(block))
|
||||
for block in hooked_blocks
|
||||
),
|
||||
]
|
||||
)
|
||||
raise
|
||||
set_encoder_layerwise_state(
|
||||
module,
|
||||
hooks,
|
||||
block_groups,
|
||||
)
|
||||
logger.info(
|
||||
"Enabled rank-local layerwise offload for encoder %s (%d blocks across %d stacks)",
|
||||
name,
|
||||
sum(len(blocks) for blocks in block_groups),
|
||||
len(block_groups),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def disable_plan_encoder_layerwise_offload(
|
||||
module: nn.Module,
|
||||
*,
|
||||
restore_weights: bool = True,
|
||||
) -> None:
|
||||
"""Remove hooks installed by :func:`enable_plan_encoder_layerwise_offload`."""
|
||||
if not getattr(module, "_omni_layerwise_enabled", False):
|
||||
return
|
||||
hooks = getattr(module, "_omni_layerwise_hooks", [])
|
||||
block_groups = getattr(module, "_omni_layerwise_block_groups", [])
|
||||
steps = []
|
||||
if restore_weights:
|
||||
steps.extend(("restoring an encoder block", hook.restore_next_block) for hook in hooks)
|
||||
steps.extend(
|
||||
("removing an encoder hook", lambda block=block: remove_block_hook(block))
|
||||
for blocks in block_groups
|
||||
for block in blocks
|
||||
)
|
||||
cleanup_error = run_cleanup_steps(steps)
|
||||
if cleanup_error is not None:
|
||||
raise RuntimeError("Failed to fully disable encoder layerwise offload") from cleanup_error
|
||||
clear_encoder_layerwise_state(module)
|
||||
|
||||
|
||||
class LayerWiseOffloadBackend(OffloadBackend):
|
||||
"""Layer-wise (block-level) offloading backend.
|
||||
|
||||
@@ -315,56 +420,88 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
|
||||
self.copy_stream = current_omni_platform.Stream()
|
||||
self._blocks: list[list[nn.Module]] = []
|
||||
self._dit_hooks: list[LayerwiseOffloadHook] = []
|
||||
self._hooked_dit_blocks: list[nn.Module] = []
|
||||
self._encoder_modules: list[nn.Module] = []
|
||||
self._staged_components: list[nn.Module] = []
|
||||
|
||||
def enable(self, pipeline: nn.Module) -> None:
|
||||
try:
|
||||
self._enable(pipeline)
|
||||
except BaseException:
|
||||
try:
|
||||
self.disable()
|
||||
except BaseException:
|
||||
logger.exception("LayerWiseOffloadBackend cleanup failed while handling an enable failure")
|
||||
raise
|
||||
|
||||
def _enable(self, pipeline: nn.Module) -> None:
|
||||
if self.enabled:
|
||||
logger.warning("LayerWiseOffloadBackend already enabled")
|
||||
return
|
||||
|
||||
modules = ModuleDiscovery.discover(pipeline)
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
|
||||
plan = get_offload_plan(pipeline)
|
||||
if not modules.dits and self.config.offloads(DIT_COMPONENT):
|
||||
message = "No DiT/transformer modules found for selected DiT layerwise offload"
|
||||
if self.config.components is not None:
|
||||
raise ValueError(message)
|
||||
logger.warning(message)
|
||||
return
|
||||
|
||||
# Move encoders to GPU (they stay resident)
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
def enable_encoder_blocks(
|
||||
module: nn.Module,
|
||||
name: str,
|
||||
component_plan: OffloadPlan | None,
|
||||
stage_on_demand: bool,
|
||||
) -> bool:
|
||||
enabled = enable_plan_encoder_layerwise_offload(
|
||||
module,
|
||||
name,
|
||||
component_plan,
|
||||
device=self.device,
|
||||
stream=self.copy_stream,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
stage_on_demand=stage_on_demand,
|
||||
strict=self.config.components is not None,
|
||||
)
|
||||
if enabled:
|
||||
# Record each successful installation immediately so a later
|
||||
# component failure can remove these hooks transactionally.
|
||||
self._encoder_modules.append(module)
|
||||
return enabled
|
||||
|
||||
# Move VAE(s) to GPU if available
|
||||
for vae in modules.vaes:
|
||||
try:
|
||||
vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
prepare_pipeline_components(
|
||||
modules,
|
||||
self.config,
|
||||
plan,
|
||||
device=self.device,
|
||||
staged_components=self._staged_components,
|
||||
enable_encoder_blocks=enable_encoder_blocks,
|
||||
)
|
||||
|
||||
# Move resident modules to GPU (small modules needed every forward)
|
||||
for name, module in zip(modules.resident_names, modules.resident_modules):
|
||||
try:
|
||||
module.to(self.device)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
|
||||
if not self.config.offloads(DIT_COMPONENT):
|
||||
self.enabled = bool(self._encoder_modules or self._staged_components)
|
||||
if not self.enabled:
|
||||
raise ValueError(
|
||||
"None of the selected layerwise offload components have "
|
||||
"a model-declared streamable or on-demand plan"
|
||||
)
|
||||
return
|
||||
|
||||
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
|
||||
|
||||
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
|
||||
# Note that there might exist multiple DiT models in specific pipelines
|
||||
for i, dit_module in enumerate(modules.dits):
|
||||
dit_name = modules.dit_names[i]
|
||||
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
|
||||
|
||||
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
|
||||
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
dit_module.__class__.__name__,
|
||||
)
|
||||
dit_module.to(self.device)
|
||||
continue
|
||||
|
||||
for dit_name, dit_module, blocks_attr_names, blocks in iter_streamable_dits(
|
||||
modules, self.config, self.device, plan
|
||||
):
|
||||
num_blocks = len(blocks)
|
||||
if num_blocks <= 1:
|
||||
if self.config.components is not None:
|
||||
raise ValueError(
|
||||
f"Selected DiT {dit_name!r} requires at least two streamable layerwise-offload blocks"
|
||||
)
|
||||
logger.warning(
|
||||
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
@@ -390,137 +527,66 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
if buffer is not None:
|
||||
buffer.data = buffer.data.to(self.device, non_blocking=True)
|
||||
|
||||
# Pre-fetch the first layer by manually calling the hook function on the last layer;
|
||||
# For subsequent requests, the first layer/block will be pre-fetched
|
||||
# during the last layer compute of the previous request.
|
||||
last_block, first_block = blocks[-1], blocks[0]
|
||||
last_hook = apply_block_hook(
|
||||
last_block,
|
||||
first_block,
|
||||
block_hooks = _install_layerwise_hook_group(
|
||||
blocks,
|
||||
self.device,
|
||||
self.copy_stream,
|
||||
self.config.pin_cpu_memory,
|
||||
)
|
||||
last_hook.prefetch_layer(non_blocking=False)
|
||||
self._dit_hooks.extend(block_hooks)
|
||||
self._hooked_dit_blocks.extend(blocks)
|
||||
|
||||
block_hooks: list[LayerwiseOffloadHook] = [last_hook]
|
||||
# Register hook for each of blocks
|
||||
for i, block in enumerate(blocks[:-1]):
|
||||
next_block = blocks[(i + 1) % num_blocks]
|
||||
hook = apply_block_hook(
|
||||
block,
|
||||
next_block,
|
||||
self.device,
|
||||
self.copy_stream,
|
||||
self.config.pin_cpu_memory,
|
||||
)
|
||||
block_hooks.append(hook)
|
||||
|
||||
# NOTE(yuanheng-zhao): We make each hook gets a backward reference to the hook
|
||||
# that is responsible for prefetching its block's weights. This is specifically a
|
||||
# workaround for that arbitrary blocks are skipped by caching systems (e.g., cache-dit)
|
||||
for i in range(len(block_hooks)):
|
||||
block_hooks[i]._prev_hook = block_hooks[i - 1]
|
||||
# The last block owns block zero's host backing. Materialize block
|
||||
# zero once; later denoising iterations prefetch it from the ring.
|
||||
block_hooks[0].prefetch_layer(non_blocking=False)
|
||||
|
||||
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
|
||||
|
||||
# Track hooked blocks for cleanup
|
||||
self._blocks.append(blocks)
|
||||
|
||||
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
|
||||
self.enabled = True
|
||||
self.enabled = bool(self._blocks or self._encoder_modules or self._staged_components)
|
||||
|
||||
def disable(self) -> None:
|
||||
if not self.enabled:
|
||||
def _disable(self, *, restore_weights: bool) -> None:
|
||||
if not self.enabled and not (
|
||||
self._dit_hooks or self._hooked_dit_blocks or self._encoder_modules or self._staged_components
|
||||
):
|
||||
return
|
||||
|
||||
for blocks in self._blocks:
|
||||
for block in blocks:
|
||||
remove_block_hook(block)
|
||||
|
||||
steps = []
|
||||
if restore_weights:
|
||||
steps.extend(("restoring a DiT block", hook.restore_next_block) for hook in self._dit_hooks)
|
||||
steps.extend(
|
||||
("removing a DiT block hook", lambda block=block: remove_block_hook(block))
|
||||
for block in self._hooked_dit_blocks
|
||||
)
|
||||
steps.extend(
|
||||
(
|
||||
"disabling encoder layerwise offload",
|
||||
lambda module=module: disable_plan_encoder_layerwise_offload(
|
||||
module,
|
||||
restore_weights=restore_weights,
|
||||
),
|
||||
)
|
||||
for module in self._encoder_modules
|
||||
)
|
||||
cleanup_error = run_cleanup_steps(steps)
|
||||
if cleanup_error is not None:
|
||||
# Keep every host-master reference so a transient teardown error
|
||||
# can be retried without losing weights behind placeholders.
|
||||
raise RuntimeError("Failed to fully disable layerwise offload") from cleanup_error
|
||||
self._blocks.clear()
|
||||
self._dit_hooks.clear()
|
||||
self._hooked_dit_blocks.clear()
|
||||
self._encoder_modules.clear()
|
||||
self._staged_components.clear()
|
||||
self.enabled = False
|
||||
logger.info("Layer-wise offloading disabled")
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_attr_names(model: nn.Module) -> list[str]:
|
||||
"""Get block attribute names from model class."""
|
||||
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
|
||||
def disable(self) -> None:
|
||||
self._disable(restore_weights=True)
|
||||
|
||||
if not attrs:
|
||||
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
|
||||
if old_attr is not None:
|
||||
logger.warning(
|
||||
"'_layerwise_offload_blocks_attr' is deprecated, "
|
||||
"please use '_layerwise_offload_blocks_attrs' instead. "
|
||||
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
|
||||
)
|
||||
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
|
||||
|
||||
return attrs
|
||||
|
||||
@staticmethod
|
||||
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
|
||||
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
|
||||
"""
|
||||
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
|
||||
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
|
||||
|
||||
```
|
||||
class WanTransformer3DModel(nn.Module):
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
```
|
||||
|
||||
Returns:
|
||||
Tuple of (blocks_attr_names, blocks)
|
||||
"""
|
||||
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
|
||||
if not blocks_attr_names:
|
||||
logger.warning(
|
||||
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
)
|
||||
return [], []
|
||||
|
||||
blocks = []
|
||||
for name in blocks_attr_names:
|
||||
attr = getattr(model, name, None)
|
||||
if attr is None:
|
||||
raise AttributeError(
|
||||
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
|
||||
f"does not exist on model {model.__class__.__name__}"
|
||||
)
|
||||
try:
|
||||
attr_iter = iter(attr)
|
||||
except TypeError:
|
||||
if isinstance(attr, nn.Module):
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable; treating it as one block.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
blocks.append(attr)
|
||||
continue
|
||||
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
type(attr).__name__,
|
||||
)
|
||||
else:
|
||||
blocks.extend(attr_iter)
|
||||
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
"No blocks found in %s for %s, skipping layerwise offloading",
|
||||
blocks_attr_names,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
return [], []
|
||||
|
||||
return blocks_attr_names, blocks
|
||||
# Compatibility aliases for existing model integrations.
|
||||
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
|
||||
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
|
||||
get_blocks_from_dit = staticmethod(get_blocks_from_dit)
|
||||
Reference in new issue
Block a user