[diffusion][feature] Add component-selective offload policies (#5929)

Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-09-06 05:40:09 +08:00
1 parent 44d3ae100a
commit ea3cf99894
74 files changed
+5121 -1584

No files matched your search

+315 -249
View File
@@ -7,14 +7,41 @@ from typing import Any
import torch
from torch import nn
from torch.distributed.tensor import DTensor
from vllm.logger import init_logger
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
from .block_discovery import (
get_blocks_attr_names,
get_blocks_from_dit,
set_blocks_attr_names,
)
from .component_utils import (
clear_encoder_layerwise_state,
get_encoder_block_groups,
iter_streamable_dits,
move_non_block_state_to_device,
prepare_pipeline_components,
set_encoder_layerwise_state,
)
from .config import DIT_COMPONENT
from .module_collector import ModuleDiscovery
from .offload_plan import OffloadPlan, get_offload_plan
from .tensor_utils import (
clear_block_storage,
clear_tensor_storage,
describe_tensor_storage,
flatten_physical_storage,
group_named_tensors_by_dtype,
is_materialized_tensor,
materialization_probe,
module_materialization_probe,
restore_tensor_storage,
set_tensor_storage,
tensor_storage_metadata,
)
logger = init_logger(__name__)
@@ -41,6 +68,7 @@ class LayerwiseOffloadHook(ModelHook):
device: torch.device,
stream: current_omni_platform.Stream | None = None,
pin_memory: bool = True,
materialization_probe_tensor: torch.Tensor | None = None,
):
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
@@ -59,31 +87,7 @@ class LayerwiseOffloadHook(ModelHook):
self.next_block_buffers: dict[str, torch.Tensor] = {}
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
@staticmethod
def _is_dtensor(t: torch.Tensor) -> bool:
return isinstance(t, DTensor)
@staticmethod
def _set_tensor_storage(target: torch.Tensor, value: torch.Tensor) -> None:
if LayerwiseOffloadHook._is_dtensor(target):
target._local_tensor = value
else:
target.data = value
@staticmethod
def _make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor:
if LayerwiseOffloadHook._is_dtensor(tensor):
local_shape = tuple(tensor.to_local().shape)
return torch.empty(local_shape, device="meta", dtype=tensor.dtype)
return torch.empty((0,), device=tensor.device, dtype=tensor.dtype)
@staticmethod
def _is_materialized_tensor(t: torch.Tensor) -> bool:
if LayerwiseOffloadHook._is_dtensor(t):
local_t = t.to_local()
return not local_t.is_meta
return not t.is_meta and t.data.numel() > 0
self._materialization_probe = materialization_probe_tensor
def initialize_hook(self, module: nn.Module) -> nn.Module:
# This all happen during the hook instance being registered to hook registry;
@@ -92,6 +96,8 @@ class LayerwiseOffloadHook(ModelHook):
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
if self._materialization_probe is None:
self._materialization_probe = materialization_probe(self.block_parameters, self.block_buffers)
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
@@ -100,7 +106,6 @@ class LayerwiseOffloadHook(ModelHook):
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
self.next_block_parameters,
self.next_block_buffers,
self.device,
self.pin_memory,
)
@@ -110,7 +115,6 @@ class LayerwiseOffloadHook(ModelHook):
def _to_cpu(
params: dict[str, nn.Parameter],
bufs: dict[str, torch.Tensor],
device: torch.device,
pin_memory: bool = True,
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
@@ -123,85 +127,39 @@ class LayerwiseOffloadHook(ModelHook):
flattened CPU tensors by dtype,
metadata for reconstruction by dtype
"""
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
# NOTE: order does matter
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
targets_to_offload: list[torch.Tensor] = []
for name, param_or_buf in chain(params.items(), bufs.items()):
dtype = param_or_buf.dtype
if dtype not in dtype_grouped_weights:
dtype_grouped_weights[dtype] = {}
dtype_grouped_weights[dtype][name] = param_or_buf
for dtype, name2weights in dtype_grouped_weights.items():
for dtype, named_weights in group_named_tensors_by_dtype(params, bufs).items():
# total # of parameters + buffers
weights_with_local = []
for name, t in name2weights.items():
local_t = t.to_local() if hasattr(t, "to_local") else t
stride = local_t.stride()
storage_numel = (
0
if local_t.numel() == 0
else 1 + sum((size - 1) * axis_stride for size, axis_stride in zip(local_t.shape, stride))
)
weights_with_local.append((name, t, local_t, storage_numel, stride))
total_numel = sum(storage_numel for _, _, _, storage_numel, _ in weights_with_local)
specs = describe_tensor_storage(named_weights)
total_numel = sum(spec.storage_numel for spec in specs)
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
current_offset = 0
for (
name,
original_tensor,
local_tensor,
storage_numel,
stride,
) in weights_with_local:
if local_tensor.is_contiguous():
flat_storage = local_tensor.flatten()
else:
# Cutlass FP8 weights use a transposed physical layout.
# Preserve it across the flattened CPU staging buffer.
flat_storage = torch.zeros(
storage_numel,
dtype=dtype,
device=local_tensor.device,
)
physical_view = torch.as_strided(
flat_storage,
size=local_tensor.shape,
stride=stride,
)
physical_view.copy_(local_tensor)
cpu_tensor[current_offset : current_offset + storage_numel].copy_(flat_storage)
if dtype not in dtype_metadata:
dtype_metadata[dtype] = []
dtype_metadata[dtype].append(
{
"name": name,
"offset": current_offset,
"numel": storage_numel,
"shape": local_tensor.shape,
"stride": stride,
}
for spec in specs:
flat_storage = flatten_physical_storage(spec.value, spec.storage_numel)
cpu_tensor[current_offset : current_offset + spec.storage_numel].copy_(flat_storage)
dtype_metadata.setdefault(dtype, []).append(
tensor_storage_metadata(spec, current_offset, include_device=True)
)
LayerwiseOffloadHook._set_tensor_storage(
original_tensor, LayerwiseOffloadHook._make_offload_placeholder(original_tensor)
)
current_offset += storage_numel
targets_to_offload.append(spec.target)
current_offset += spec.storage_numel
dtype_cpu_flattened_weights[dtype] = cpu_tensor
# Do not mutate the module until every host master has been built.
# Allocation/copy failures therefore leave the input module intact.
clear_tensor_storage(targets_to_offload)
return dtype_cpu_flattened_weights, dtype_metadata
@property
def is_materialized(self) -> bool:
"""Check whether this block's parameters hold real data on device."""
for param in self.block_parameters.values():
return LayerwiseOffloadHook._is_materialized_tensor(param)
return True
return self._materialization_probe is None or is_materialized_tensor(self._materialization_probe)
@torch.compiler.disable
def prefetch_layer(self, non_blocking: bool = True) -> None:
@@ -236,7 +194,7 @@ class LayerwiseOffloadHook(ModelHook):
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
)
LayerwiseOffloadHook._set_tensor_storage(
set_tensor_storage(
target_param_or_buf,
torch.as_strided(
gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]],
@@ -252,17 +210,27 @@ class LayerwiseOffloadHook(ModelHook):
"""Free GPU memory for layer by replacing tensors with empty placeholders.
This function does not actually offload weights from GPU back to CPU.
"""
evt = self._prefetch_done
if evt is not None:
current_omni_platform.current_stream().wait_event(evt)
clear_block_storage(self.block_parameters, self.block_buffers, self._prefetch_done)
self._prefetch_done = None
# free GPU residency
for _, param in self.block_parameters.items():
LayerwiseOffloadHook._set_tensor_storage(param, LayerwiseOffloadHook._make_offload_placeholder(param))
for _, buf in self.block_buffers.items():
LayerwiseOffloadHook._set_tensor_storage(buf, LayerwiseOffloadHook._make_offload_placeholder(buf))
@torch.compiler.disable
def restore_next_block(self) -> None:
"""Detach the next block from this hook's host backing store."""
for dtype, ordered_metadata in self.dtype_metadata.items():
flat = self.dtype_cpu_flattened_weights[dtype]
for metadata in ordered_metadata:
value = torch.as_strided(
flat[metadata["offset"] : metadata["offset"] + metadata["numel"]],
size=metadata["shape"],
stride=metadata["stride"],
)
target_name = metadata["name"]
target = (
self.next_block_parameters[target_name]
if target_name in self.next_block_parameters
else self.next_block_buffers[target_name]
)
restore_tensor_storage(target, value, device=metadata["device"])
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
# if the previous hook was skipped and the weights are not on device,
@@ -287,9 +255,17 @@ def apply_block_hook(
device: torch.device,
stream: current_omni_platform.Stream | None = None,
pin_memory: bool = True,
*,
materialization_probe_tensor: torch.Tensor | None = None,
) -> LayerwiseOffloadHook:
registry = HookRegistry.get_or_create(module)
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
hook = LayerwiseOffloadHook(
next_block,
device,
stream,
pin_memory,
materialization_probe_tensor,
)
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
return hook
@@ -302,6 +278,135 @@ def remove_block_hook(module: nn.Module) -> None:
logger.debug("Removed offload hook from %s", module.__class__.__name__)
def _install_layerwise_hook_group(
blocks: list[nn.Module] | nn.ModuleList,
device: torch.device,
stream: Any,
pin_memory: bool,
) -> list[LayerwiseOffloadHook]:
"""Install one circular hook ring and roll it back transactionally."""
block_list = list(blocks)
if len(block_list) <= 1:
raise ValueError("A layerwise hook group requires at least two blocks")
hooks: list[LayerwiseOffloadHook] = []
hooked_blocks: list[nn.Module] = []
probes = {id(block): module_materialization_probe(block) for block in block_list}
try:
for block, next_block in zip(
chain((block_list[-1],), block_list[:-1]),
block_list,
strict=True,
):
hooks.append(
apply_block_hook(
block,
next_block,
device,
stream,
pin_memory,
materialization_probe_tensor=probes[id(block)],
)
)
hooked_blocks.append(block)
except BaseException:
run_cleanup_steps(
[
*(("restoring a partially installed layerwise block", hook.restore_next_block) for hook in hooks),
*(
("removing a partially installed layerwise hook", lambda block=block: remove_block_hook(block))
for block in hooked_blocks
),
]
)
raise
for index, hook in enumerate(hooks):
hook._prev_hook = hooks[index - 1]
return hooks
def enable_plan_encoder_layerwise_offload(
module: nn.Module,
name: str,
plan: OffloadPlan | None,
*,
device: torch.device,
stream: current_omni_platform.Stream,
pin_memory: bool,
stage_on_demand: bool = False,
strict: bool = False,
) -> bool:
"""Apply rank-local layerwise hooks to plan-declared encoder stacks."""
if getattr(module, "_omni_layerwise_enabled", False):
return True
hooks: list[LayerwiseOffloadHook] = []
hooked_blocks: list[nn.Module] = []
block_groups = get_encoder_block_groups(
module,
name,
plan,
strict=strict,
)
if not block_groups:
return False
try:
for blocks in block_groups:
group_hooks = _install_layerwise_hook_group(blocks, device, stream, pin_memory)
hooks.extend(group_hooks)
hooked_blocks.extend(blocks)
if not stage_on_demand:
move_non_block_state_to_device(module, block_groups, device)
except BaseException:
run_cleanup_steps(
[
*(("restoring a partially installed encoder block", hook.restore_next_block) for hook in hooks),
*(
("removing a partially installed encoder hook", lambda block=block: remove_block_hook(block))
for block in hooked_blocks
),
]
)
raise
set_encoder_layerwise_state(
module,
hooks,
block_groups,
)
logger.info(
"Enabled rank-local layerwise offload for encoder %s (%d blocks across %d stacks)",
name,
sum(len(blocks) for blocks in block_groups),
len(block_groups),
)
return True
def disable_plan_encoder_layerwise_offload(
module: nn.Module,
*,
restore_weights: bool = True,
) -> None:
"""Remove hooks installed by :func:`enable_plan_encoder_layerwise_offload`."""
if not getattr(module, "_omni_layerwise_enabled", False):
return
hooks = getattr(module, "_omni_layerwise_hooks", [])
block_groups = getattr(module, "_omni_layerwise_block_groups", [])
steps = []
if restore_weights:
steps.extend(("restoring an encoder block", hook.restore_next_block) for hook in hooks)
steps.extend(
("removing an encoder hook", lambda block=block: remove_block_hook(block))
for blocks in block_groups
for block in blocks
)
cleanup_error = run_cleanup_steps(steps)
if cleanup_error is not None:
raise RuntimeError("Failed to fully disable encoder layerwise offload") from cleanup_error
clear_encoder_layerwise_state(module)
class LayerWiseOffloadBackend(OffloadBackend):
"""Layer-wise (block-level) offloading backend.
@@ -315,56 +420,88 @@ class LayerWiseOffloadBackend(OffloadBackend):
self.copy_stream = current_omni_platform.Stream()
self._blocks: list[list[nn.Module]] = []
self._dit_hooks: list[LayerwiseOffloadHook] = []
self._hooked_dit_blocks: list[nn.Module] = []
self._encoder_modules: list[nn.Module] = []
self._staged_components: list[nn.Module] = []
def enable(self, pipeline: nn.Module) -> None:
try:
self._enable(pipeline)
except BaseException:
try:
self.disable()
except BaseException:
logger.exception("LayerWiseOffloadBackend cleanup failed while handling an enable failure")
raise
def _enable(self, pipeline: nn.Module) -> None:
if self.enabled:
logger.warning("LayerWiseOffloadBackend already enabled")
return
modules = ModuleDiscovery.discover(pipeline)
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
plan = get_offload_plan(pipeline)
if not modules.dits and self.config.offloads(DIT_COMPONENT):
message = "No DiT/transformer modules found for selected DiT layerwise offload"
if self.config.components is not None:
raise ValueError(message)
logger.warning(message)
return
# Move encoders to GPU (they stay resident)
for enc in modules.encoders:
enc.to(self.device)
def enable_encoder_blocks(
module: nn.Module,
name: str,
component_plan: OffloadPlan | None,
stage_on_demand: bool,
) -> bool:
enabled = enable_plan_encoder_layerwise_offload(
module,
name,
component_plan,
device=self.device,
stream=self.copy_stream,
pin_memory=self.config.pin_cpu_memory,
stage_on_demand=stage_on_demand,
strict=self.config.components is not None,
)
if enabled:
# Record each successful installation immediately so a later
# component failure can remove these hooks transactionally.
self._encoder_modules.append(module)
return enabled
# Move VAE(s) to GPU if available
for vae in modules.vaes:
try:
vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
prepare_pipeline_components(
modules,
self.config,
plan,
device=self.device,
staged_components=self._staged_components,
enable_encoder_blocks=enable_encoder_blocks,
)
# Move resident modules to GPU (small modules needed every forward)
for name, module in zip(modules.resident_names, modules.resident_modules):
try:
module.to(self.device)
except Exception as exc:
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
if not self.config.offloads(DIT_COMPONENT):
self.enabled = bool(self._encoder_modules or self._staged_components)
if not self.enabled:
raise ValueError(
"None of the selected layerwise offload components have "
"a model-declared streamable or on-demand plan"
)
return
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
# Note that there might exist multiple DiT models in specific pipelines
for i, dit_module in enumerate(modules.dits):
dit_name = modules.dit_names[i]
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
if not blocks:
logger.warning(
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
dit_name,
dit_module.__class__.__name__,
)
dit_module.to(self.device)
continue
for dit_name, dit_module, blocks_attr_names, blocks in iter_streamable_dits(
modules, self.config, self.device, plan
):
num_blocks = len(blocks)
if num_blocks <= 1:
if self.config.components is not None:
raise ValueError(
f"Selected DiT {dit_name!r} requires at least two streamable layerwise-offload blocks"
)
logger.warning(
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
dit_name,
@@ -390,137 +527,66 @@ class LayerWiseOffloadBackend(OffloadBackend):
if buffer is not None:
buffer.data = buffer.data.to(self.device, non_blocking=True)
# Pre-fetch the first layer by manually calling the hook function on the last layer;
# For subsequent requests, the first layer/block will be pre-fetched
# during the last layer compute of the previous request.
last_block, first_block = blocks[-1], blocks[0]
last_hook = apply_block_hook(
last_block,
first_block,
block_hooks = _install_layerwise_hook_group(
blocks,
self.device,
self.copy_stream,
self.config.pin_cpu_memory,
)
last_hook.prefetch_layer(non_blocking=False)
self._dit_hooks.extend(block_hooks)
self._hooked_dit_blocks.extend(blocks)
block_hooks: list[LayerwiseOffloadHook] = [last_hook]
# Register hook for each of blocks
for i, block in enumerate(blocks[:-1]):
next_block = blocks[(i + 1) % num_blocks]
hook = apply_block_hook(
block,
next_block,
self.device,
self.copy_stream,
self.config.pin_cpu_memory,
)
block_hooks.append(hook)
# NOTE(yuanheng-zhao): We make each hook gets a backward reference to the hook
# that is responsible for prefetching its block's weights. This is specifically a
# workaround for that arbitrary blocks are skipped by caching systems (e.g., cache-dit)
for i in range(len(block_hooks)):
block_hooks[i]._prev_hook = block_hooks[i - 1]
# The last block owns block zero's host backing. Materialize block
# zero once; later denoising iterations prefetch it from the ring.
block_hooks[0].prefetch_layer(non_blocking=False)
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
# Track hooked blocks for cleanup
self._blocks.append(blocks)
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
self.enabled = True
self.enabled = bool(self._blocks or self._encoder_modules or self._staged_components)
def disable(self) -> None:
if not self.enabled:
def _disable(self, *, restore_weights: bool) -> None:
if not self.enabled and not (
self._dit_hooks or self._hooked_dit_blocks or self._encoder_modules or self._staged_components
):
return
for blocks in self._blocks:
for block in blocks:
remove_block_hook(block)
steps = []
if restore_weights:
steps.extend(("restoring a DiT block", hook.restore_next_block) for hook in self._dit_hooks)
steps.extend(
("removing a DiT block hook", lambda block=block: remove_block_hook(block))
for block in self._hooked_dit_blocks
)
steps.extend(
(
"disabling encoder layerwise offload",
lambda module=module: disable_plan_encoder_layerwise_offload(
module,
restore_weights=restore_weights,
),
)
for module in self._encoder_modules
)
cleanup_error = run_cleanup_steps(steps)
if cleanup_error is not None:
# Keep every host-master reference so a transient teardown error
# can be retried without losing weights behind placeholders.
raise RuntimeError("Failed to fully disable layerwise offload") from cleanup_error
self._blocks.clear()
self._dit_hooks.clear()
self._hooked_dit_blocks.clear()
self._encoder_modules.clear()
self._staged_components.clear()
self.enabled = False
logger.info("Layer-wise offloading disabled")
@staticmethod
def get_blocks_attr_names(model: nn.Module) -> list[str]:
"""Get block attribute names from model class."""
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
def disable(self) -> None:
self._disable(restore_weights=True)
if not attrs:
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
if old_attr is not None:
logger.warning(
"'_layerwise_offload_blocks_attr' is deprecated, "
"please use '_layerwise_offload_blocks_attrs' instead. "
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
)
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
return attrs
@staticmethod
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
@staticmethod
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
"""
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
```
class WanTransformer3DModel(nn.Module):
_layerwise_offload_blocks_attrs = ["blocks"]
```
Returns:
Tuple of (blocks_attr_names, blocks)
"""
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
if not blocks_attr_names:
logger.warning(
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
"skipping layerwise offloading"
)
return [], []
blocks = []
for name in blocks_attr_names:
attr = getattr(model, name, None)
if attr is None:
raise AttributeError(
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
f"does not exist on model {model.__class__.__name__}"
)
try:
attr_iter = iter(attr)
except TypeError:
if isinstance(attr, nn.Module):
logger.warning(
"Attribute '%s' on %s is not iterable; treating it as one block.",
name,
model.__class__.__name__,
)
blocks.append(attr)
continue
logger.warning(
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
name,
model.__class__.__name__,
type(attr).__name__,
)
else:
blocks.extend(attr_iter)
if not blocks:
logger.warning(
"No blocks found in %s for %s, skipping layerwise offloading",
blocks_attr_names,
model.__class__.__name__,
)
return [], []
return blocks_attr_names, blocks
# Compatibility aliases for existing model integrations.
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
get_blocks_from_dit = staticmethod(get_blocks_from_dit)