[Feat] support for multi-block layerwise offloading, fix top-level parameters/buffers staying on CPU (#1486)
Signed-off-by: Lancer <maruixiang6688@gmail.com> Signed-off-by: Lancer <402430575@qq.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
e23b2634d1
commit
328de586aa
13 files changed
+288
-38
No files matched your search
@@ -312,10 +312,9 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
dit_name = modules.dit_names[i]
|
||||
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
|
||||
|
||||
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(dit_module)
|
||||
blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
|
||||
blocks_attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
|
||||
|
||||
if not blocks_attr_name or not blocks:
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
@@ -336,11 +335,20 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
|
||||
# Move non-block modules to GPU (they stay resident)
|
||||
for name, m in dit_module.named_children():
|
||||
if name == blocks_attr_name:
|
||||
if name not in blocks_attr_names:
|
||||
m.to(self.device)
|
||||
logger.debug(f"Moved {name} to device {self.device}")
|
||||
else:
|
||||
logger.debug(f"Skipped blocks module {name}")
|
||||
continue
|
||||
m.to(self.device)
|
||||
logger.debug(f"Moved {name} to device {self.device}")
|
||||
|
||||
# Move top-level params/buffers to GPU (dit_module's own, not sub-modules)
|
||||
for param in dit_module._parameters.values():
|
||||
if param is not None:
|
||||
param.data = param.data.to(self.device, non_blocking=True)
|
||||
|
||||
for buffer in dit_module._buffers.values():
|
||||
if buffer is not None:
|
||||
buffer.data = buffer.data.to(self.device, non_blocking=True)
|
||||
|
||||
# Pre-fetch the first layer by manually calling the hook function on the last layer;
|
||||
# For subsequent requests, the first layer/block will be pre-fetched
|
||||
@@ -395,40 +403,84 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
logger.info("Layer-wise offloading disabled")
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_attr_name(model: nn.Module) -> str | None:
|
||||
"""Retrieve blocks attribute name from provided DiT model"""
|
||||
return getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
|
||||
def get_blocks_attr_names(model: nn.Module) -> list[str]:
|
||||
"""Get block attribute names from model class."""
|
||||
attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", [])
|
||||
|
||||
if not attrs:
|
||||
old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
|
||||
if old_attr is not None:
|
||||
logger.warning(
|
||||
"'_layerwise_offload_blocks_attr' is deprecated, "
|
||||
"please use '_layerwise_offload_blocks_attrs' instead. "
|
||||
"Example: _layerwise_offload_blocks_attrs = ['blocks']"
|
||||
)
|
||||
attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr)
|
||||
|
||||
return attrs
|
||||
|
||||
@staticmethod
|
||||
def set_blocks_attr_name(model: nn.Module, name: str) -> None:
|
||||
if not hasattr(model.__class__, "_layerwise_offload_blocks_attr"):
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attr", name)
|
||||
def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None:
|
||||
if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"):
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attrs", names)
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_from_dit(model: nn.Module) -> list[nn.Module]:
|
||||
def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]:
|
||||
"""
|
||||
Retrieve a list of blocks from provided DiT model. Blocks attribute name
|
||||
are found by `_layerwise_offload_blocks_attr` set to DiT models. For example,
|
||||
Retrieve blocks and attribute names from provided DiT model. Blocks attribute names
|
||||
are found by `_layerwise_offload_blocks_attrs` set to DiT models. For example,
|
||||
|
||||
```
|
||||
class WanTransformer3DModel(nn.Module):
|
||||
_layerwise_offload_blocks_attr = "blocks"
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
```
|
||||
|
||||
Returns:
|
||||
Tuple of (blocks_attr_names, blocks)
|
||||
"""
|
||||
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(model)
|
||||
if blocks_attr_name is None:
|
||||
blocks_attr_names = LayerWiseOffloadBackend.get_blocks_attr_names(model)
|
||||
if not blocks_attr_names:
|
||||
logger.warning(
|
||||
f"No _layerwise_offload_blocks_attr defined for {model.__class__.__name__}, "
|
||||
f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
)
|
||||
return []
|
||||
return [], []
|
||||
|
||||
_blocks = getattr(model, blocks_attr_name, None)
|
||||
if _blocks is None:
|
||||
blocks = []
|
||||
for name in blocks_attr_names:
|
||||
attr = getattr(model, name, None)
|
||||
if attr is None:
|
||||
raise AttributeError(
|
||||
f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs "
|
||||
f"does not exist on model {model.__class__.__name__}"
|
||||
)
|
||||
try:
|
||||
attr_iter = iter(attr)
|
||||
except TypeError:
|
||||
if isinstance(attr, nn.Module):
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable; treating it as one block.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
blocks.append(attr)
|
||||
continue
|
||||
|
||||
logger.warning(
|
||||
"Attribute '%s' on %s is not iterable (got %s); skipping it.",
|
||||
name,
|
||||
model.__class__.__name__,
|
||||
type(attr).__name__,
|
||||
)
|
||||
else:
|
||||
blocks.extend(attr_iter)
|
||||
|
||||
if not blocks:
|
||||
logger.warning(
|
||||
f"Blocks (layers) '{blocks_attr_name}' not found on {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
"No blocks found in %s for %s, skipping layerwise offloading",
|
||||
blocks_attr_names,
|
||||
model.__class__.__name__,
|
||||
)
|
||||
return []
|
||||
return [], []
|
||||
|
||||
return list(_blocks)
|
||||
return blocks_attr_names, blocks
|
||||
Reference in new issue
Block a user