[Feat] support for multi-block layerwise offloading, fix top-level parameters/buffers staying on CPU (#1486)

Signed-off-by: Lancer <maruixiang6688@gmail.com>
Signed-off-by: Lancer <402430575@qq.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
LancerandDidan Deng authored and GitHub committed 2026-04-06 11:49:35 +08:00
1 parent e23b2634d1
commit 328de586aa
13 files changed
+288 -38

No files matched your search

@@ -312,10 +312,9 @@ class LayerWiseOffloadBackend(OffloadBackend):
dit_name = modules.dit_names[i]
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(dit_module)
blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
if not blocks_attr_name or not blocks:
if not blocks:
logger.warning(
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
dit_name,
@@ -336,11 +335,20 @@ class LayerWiseOffloadBackend(OffloadBackend):
# Move non-block modules to GPU (they stay resident)
for name, m in dit_module.named_children():
if name == blocks_attr_name:
if name not in blocks_attr_names:
m.to(self.device)
logger.debug(f"Moved {name} to device {self.device}")
else:
logger.debug(f"Skipped blocks module {name}")
continue
m.to(self.device)
logger.debug(f"Moved {name} to device {self.device}")
# Move top-level params/buffers to GPU (dit_module's own, not sub-modules)
for param in dit_module._parameters.values():
if param is not None:
param.data = param.data.to(self.device, non_blocking=True)
for buffer in dit_module._buffers.values():
if buffer is not None:
buffer.data = buffer.data.to(self.device, non_blocking=True)
# Pre-fetch the first layer by manually calling the hook function on the last layer;
# For subsequent requests, the first layer/block will be pre-fetched
@@ -395,40 +403,84 @@ class LayerWiseOffloadBackend(OffloadBackend):
logger.info("Layer-wise offloading disabled")
@staticmethod
def get_blocks_attr_name(model: nn.Module) -> str | None:
"""Retrieve blocks attribute name from provided DiT model"""
return getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
def get_blocks_attr_names(model: nn.Module) -> list[str]:
"""Get block attribute names from model class."""
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
if not attrs:
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
if old_attr is not None:
logger.warning(
"'_layerwise_offload_blocks_attr' is deprecated, "
"please use '_layerwise_offload_blocks_attrs' instead. "
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
)
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
return attrs
@staticmethod
def set_blocks_attr_name(model: nn.Module, name: str) -> None:
if not hasattr(model.__class__, "_layerwise_offload_blocks_attr"):
setattr(model.__class__, "_layerwise_offload_blocks_attr", name)
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
@staticmethod
def get_blocks_from_dit(model: nn.Module) -> list[nn.Module]:
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
"""
Retrieve a list of blocks from provided DiT model. Blocks attribute name
are found by `_layerwise_offload_blocks_attr` set to DiT models. For example,
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
```
class WanTransformer3DModel(nn.Module):
_layerwise_offload_blocks_attr = "blocks"
_layerwise_offload_blocks_attrs = ["blocks"]
```
Returns:
Tuple of (blocks_attr_names, blocks)
"""
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(model)
if blocks_attr_name is None:
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
if not blocks_attr_names:
logger.warning(
f"No _layerwise_offload_blocks_attr defined for {model.__class__.__name__}, "
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
"skipping layerwise offloading"
)
return []
return [], []
_blocks = getattr(model, blocks_attr_name, None)
if _blocks is None:
blocks = []
for name in blocks_attr_names:
attr = getattr(model, name, None)
if attr is None:
raise AttributeError(
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
f"does not exist on model {model.__class__.__name__}"
)
try:
attr_iter = iter(attr)
except TypeError:
if isinstance(attr, nn.Module):
logger.warning(
"Attribute '%s' on %s is not iterable; treating it as one block.",
name,
model.__class__.__name__,
)
blocks.append(attr)
continue
logger.warning(
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
name,
model.__class__.__name__,
type(attr).__name__,
)
else:
blocks.extend(attr_iter)
if not blocks:
logger.warning(
f"Blocks (layers) '{blocks_attr_name}' not found on {model.__class__.__name__}, "
"skipping layerwise offloading"
"No blocks found in %s for %s, skipping layerwise offloading",
blocks_attr_names,
model.__class__.__name__,
)
return []
return [], []
return list(_blocks)
return blocks_attr_names, blocks