[BugFix] Enable CPU offloading and Cache-DiT together on Diffusion Model (#1723)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
This commit is contained in:
1 parent
7271116973
commit
82f63a4403
5 files changed
+67
-28
No files matched your search
@@ -35,6 +35,23 @@ class SequentialOffloadHook(ModelHook):
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
|
||||
@staticmethod
|
||||
def _move_params(module: nn.Module, device: torch.device) -> None:
|
||||
"""Move module parameters and buffers to device.
|
||||
|
||||
This cls method specifically prevents recursion device movement,
|
||||
E.g., Cache-DiT CachedBlocks has attr `transformer` as a ref to original
|
||||
transformer blocks, thus `module.to(device)` will fail for recursion calling,
|
||||
refer to
|
||||
https://github.com/vipshop/cache-dit/blob/v1.2.3/src/cache_dit/caching/cache_blocks/__init__.py#L83
|
||||
"""
|
||||
for p in module.parameters():
|
||||
if p.data.device != device:
|
||||
p.data = p.data.to(device, non_blocking=True)
|
||||
for b in module.buffers():
|
||||
if b.device != device:
|
||||
b.data = b.data.to(device, non_blocking=True)
|
||||
|
||||
def _to_cpu(self, module: nn.Module) -> None:
|
||||
"""Move module to CPU."""
|
||||
try:
|
||||
@@ -42,13 +59,12 @@ class SequentialOffloadHook(ModelHook):
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
previous_device = param.device
|
||||
# Skip if already on CPU
|
||||
if previous_device.type == "cpu":
|
||||
if param.device.type == "cpu":
|
||||
return
|
||||
|
||||
module.to("cpu")
|
||||
torch.cuda.empty_cache()
|
||||
self._move_params(module, torch.device("cpu"))
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
if self.pin_memory:
|
||||
for p in module.parameters():
|
||||
@@ -64,7 +80,7 @@ class SequentialOffloadHook(ModelHook):
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
module.to(self.device)
|
||||
self._move_params(module, self.device)
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
|
||||
# Offload target modules to CPU
|
||||
@@ -186,19 +202,6 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
|
||||
# Initial state: keep DiT modules on CPU (encoders typically run first)
|
||||
# TODO: This part seems to be unnecessary, remove it after testing
|
||||
# for dit_mod in modules.dits:
|
||||
# dit_mod.to("cpu")
|
||||
|
||||
# torch.cuda.empty_cache()
|
||||
|
||||
# if self.config.pin_cpu_memory:
|
||||
# for dit_mod in modules.dits:
|
||||
# for p in dit_mod.parameters():
|
||||
# if p.data.device.type == "cpu" and not p.data.is_pinned():
|
||||
# p.data = p.data.pin_memory()
|
||||
|
||||
# Apply sequential offloading hooks
|
||||
apply_sequential_offload(
|
||||
dit_modules=modules.dits,
|
||||
|
||||
Reference in new issue
Block a user