[BugFix] Enable CPU offloading and Cache-DiT together on Diffusion Model (#1723)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
This commit is contained in:
Yuanheng Zhao authored and GitHub committed 2026-03-09 22:25:41 +08:00
1 parent 7271116973
commit 82f63a4403
5 files changed
+67 -28

No files matched your search

@@ -35,6 +35,23 @@ class SequentialOffloadHook(ModelHook):
self.device = device
self.pin_memory = pin_memory
@staticmethod
def _move_params(module: nn.Module, device: torch.device) -> None:
"""Move module parameters and buffers to device.
This cls method specifically prevents recursion device movement,
E.g., Cache-DiT CachedBlocks has attr `transformer` as a ref to original
transformer blocks, thus `module.to(device)` will fail for recursion calling,
refer to
https://github.com/vipshop/cache-dit/blob/v1.2.3/src/cache_dit/caching/cache_blocks/__init__.py#L83
"""
for p in module.parameters():
if p.data.device != device:
p.data = p.data.to(device, non_blocking=True)
for b in module.buffers():
if b.device != device:
b.data = b.data.to(device, non_blocking=True)
def _to_cpu(self, module: nn.Module) -> None:
"""Move module to CPU."""
try:
@@ -42,13 +59,12 @@ class SequentialOffloadHook(ModelHook):
except StopIteration:
return
previous_device = param.device
# Skip if already on CPU
if previous_device.type == "cpu":
if param.device.type == "cpu":
return
module.to("cpu")
torch.cuda.empty_cache()
self._move_params(module, torch.device("cpu"))
current_omni_platform.empty_cache()
if self.pin_memory:
for p in module.parameters():
@@ -64,7 +80,7 @@ class SequentialOffloadHook(ModelHook):
except StopIteration:
return
module.to(self.device)
self._move_params(module, self.device)
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
# Offload target modules to CPU
@@ -186,19 +202,6 @@ class ModelLevelOffloadBackend(OffloadBackend):
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
# Initial state: keep DiT modules on CPU (encoders typically run first)
# TODO: This part seems to be unnecessary, remove it after testing
# for dit_mod in modules.dits:
# dit_mod.to("cpu")
# torch.cuda.empty_cache()
# if self.config.pin_cpu_memory:
# for dit_mod in modules.dits:
# for p in dit_mod.parameters():
# if p.data.device.type == "cpu" and not p.data.is_pinned():
# p.data = p.data.pin_memory()
# Apply sequential offloading hooks
apply_sequential_offload(
dit_modules=modules.dits,