Refactor CPU Offloading Backend Pattern (#1223)

Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
Yuanheng ZhaoandSamit authored and GitHub committed 2026-02-12 17:29:55 +08:00
1 parent f117a07271
commit 8bf849c399
17 files changed
+980 -647

No files matched your search

+77
View File
@@ -0,0 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, OffloadStrategy
from .layerwise_backend import LayerWiseOffloadBackend
from .sequential_backend import (
ModelLevelOffloadBackend,
apply_sequential_offload,
remove_sequential_offload,
)
logger = init_logger(__name__)
__all__ = [
"OffloadBackend",
"OffloadConfig",
"OffloadStrategy",
"LayerWiseOffloadBackend",
"ModelLevelOffloadBackend",
"apply_sequential_offload",
"remove_sequential_offload",
"get_offload_backend",
]
def get_offload_backend(
od_config: OmniDiffusionConfig,
device: torch.device | None = None,
) -> OffloadBackend | None:
"""Create appropriate offload backend based on configuration.
Args:
od_config: OmniDiffusionConfig with offload settings
device: Target device (auto-detected if None)
Returns:
OffloadBackend instance or None if offloading disabled
Example:
>>> backend = get_offload_backend(od_config, device=torch.device("cuda:0"))
>>> if backend:
... backend.enable(pipeline)
"""
# Extract and validate configuration
config = OffloadConfig.from_od_config(od_config)
# Return None if no offloading requested
if config.strategy == OffloadStrategy.NONE:
return None
# Validate platform (CUDA required for now)
if not current_omni_platform.is_cuda() or current_omni_platform.get_device_count() < 1:
logger.warning("CPU offloading requires CUDA devices. Skipping offloading.")
return None
# Detect device if not provided
if device is None:
try:
device = current_omni_platform.get_torch_device()
except (NotImplementedError, AttributeError) as exc:
logger.error("Failed to detect device: %s. Skipping offloading.", exc)
return None
# Create appropriate backend
if config.strategy == OffloadStrategy.MODEL_LEVEL:
return ModelLevelOffloadBackend(config, device)
elif config.strategy == OffloadStrategy.LAYER_WISE:
return LayerWiseOffloadBackend(config, device)
else:
logger.error("Unknown offload strategy: %s", config.strategy)
return None
+94
View File
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
import torch
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.data import OmniDiffusionConfig
logger = init_logger(__name__)
class OffloadStrategy(Enum):
NONE = "none"
MODEL_LEVEL = "model_level" # Sequential offloading between DiT and encoders
LAYER_WISE = "layer_wise" # Block-level
@dataclass
class OffloadConfig:
strategy: OffloadStrategy
pin_cpu_memory: bool = True
@classmethod
def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig":
"""Extract and validate offload settings from OmniDiffusionConfig.
For now, enforces mutual exclusion between model-level and layer-wise offloading.
Layer-wise takes priority if both are enabled.
Args:
od_config: OmniDiffusionConfig with offload settings
Returns:
OffloadConfig with validated settings
"""
enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False)
enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False)
pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True)
# Determine strategy (mutual exclusion, layer-wise takes priority)
if enable_layerwise_offload:
strategy = OffloadStrategy.LAYER_WISE
if enable_cpu_offload:
logger.info(
"Both model-level and layer-wise offloading enabled. "
"Layer-wise takes priority, disabling model-level offloading."
)
elif enable_cpu_offload:
strategy = OffloadStrategy.MODEL_LEVEL
else:
strategy = OffloadStrategy.NONE
return cls(
strategy=strategy,
pin_cpu_memory=pin_cpu_memory,
)
class OffloadBackend(ABC):
"""Base class for CPU offload backends"""
def __init__(self, config: OffloadConfig, device: torch.device):
self.config = config
self.device = device
self.enabled = False
@abstractmethod
def enable(self, pipeline: nn.Module) -> None:
"""Enable offloading on the pipeline.
Discovers modules, moves them to appropriate devices, and
registers forward hooks for swapping/prefetching.
Args:
pipeline: Diffusion pipeline model (e.g., Wan22Pipeline)
"""
raise NotImplementedError
@abstractmethod
def disable(self) -> None:
"""Disable offloading and cleanup resources.
Removes all registered hooks. Does NOT move modules back to
original devices (caller responsible for that).
"""
raise NotImplementedError
def is_enabled(self) -> bool:
return self.enabled
@@ -0,0 +1,362 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import chain
from typing import Any
import torch
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig
from .module_collector import ModuleDiscovery
logger = init_logger(__name__)
class LayerwiseOffloadHook(ModelHook):
"""Hook for layerwise (transformer-block-wise) CPU offloading.
The hook instance retains parameters for both the current registered block
module and those for the next block, as well as flattened CPU tensors which
record the parameters of the current block module, so that these parameters
could be re-materialized on device in an overlapping way.
This hook should be registered to each of the transformer blocks in DiT
module(s) of the target pipeline.
Based on implementations from:
https://github.com/sgl-project/sglang/blob/v0.5.8/python/sglang/multimodal_gen/runtime/utils/layerwise_offload.py
"""
_HOOK_NAME = "layerwise_offload"
def __init__(
self,
next_block: nn.Module,
device: torch.device,
stream: torch.cuda.Stream | None = None,
pin_memory: bool = True,
):
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
assert current_omni_platform.is_cuda(), "Layerwise offloading is only supported on cuda devices for now"
self.next_block = next_block
self.device = device
self.copy_stream = stream or torch.cuda.current_stream()
self.pin_memory = pin_memory
# Per-block synchronization primitive: set after H2D copy completes.
self._prefetch_done: torch.cuda.Event | None = None
self.next_block_parameters: dict[str, nn.Parameter] = {}
self.next_block_buffers: dict[str, torch.Tensor] = {}
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
def initialize_hook(self, module: nn.Module) -> nn.Module:
# This all happen during the hook instance being registered to hook registry;
# the input module is kept intact
module = super().initialize_hook(module)
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
# Pre-allocate gpu tensors in a flattened way
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
self.next_block_parameters, self.next_block_buffers, self.device, self.pin_memory
)
return module
@staticmethod
def _to_cpu(
params: dict[str, nn.Parameter],
bufs: dict[str, torch.Tensor],
device: torch.device,
pin_memory: bool = True,
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
Consolidates parameters and buffers into contiguous CPU tensors grouped by dtype
for GPU transfers. Replaces original tensors with empty placeholders.
Returns:
Tuple of
flattened CPU tensors by dtype,
metadata for reconstruction by dtype
"""
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
# NOTE: order does matter
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
for name, param_or_buf in chain(params.items(), bufs.items()):
dtype = param_or_buf.dtype
if dtype not in dtype_grouped_weights:
dtype_grouped_weights[dtype] = {}
dtype_grouped_weights[dtype][name] = param_or_buf
for dtype, name2weights in dtype_grouped_weights.items():
# total # of parameters + buffers
total_numel = sum(t.numel() for _, t in name2weights.items())
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
current_offset = 0
for name, param_or_buf in name2weights.items():
numel = param_or_buf.numel()
cpu_tensor[current_offset : current_offset + numel].copy_(param_or_buf.flatten())
if dtype not in dtype_metadata:
dtype_metadata[dtype] = []
dtype_metadata[dtype].append(
{
"name": name,
"offset": current_offset,
"numel": numel,
"shape": param_or_buf.shape,
}
)
param_or_buf.data = torch.empty((), device=device, dtype=dtype)
current_offset += numel
dtype_cpu_flattened_weights[dtype] = cpu_tensor
return dtype_cpu_flattened_weights, dtype_metadata
@torch.compiler.disable
def prefetch_layer(self, non_blocking: bool = True) -> None:
"""Copy layer weights from CPU -> GPU.
Pre-fetch target block in an asynchronous way with compute - memory copy overlap,
with non_blocking set to True.
"""
self.copy_stream.wait_stream(torch.cuda.current_stream())
layer_params = self.next_block_parameters
layer_bufs = self.next_block_buffers
evt = torch.cuda.Event()
gpu_weights: dict[torch.dtype, torch.Tensor] = {}
with torch.cuda.stream(self.copy_stream):
for dtype, cpu_weight in self.dtype_cpu_flattened_weights.items():
gpu_weight = torch.empty(cpu_weight.shape, dtype=dtype, device=self.device)
gpu_weight.copy_(cpu_weight, non_blocking=non_blocking)
gpu_weights[dtype] = gpu_weight
evt.record(self.copy_stream)
for dtype, ordered_metadata in self.dtype_metadata.items():
# ordered_metadata: list[dict[str, Any]]
gpu_weight = gpu_weights[dtype]
for metadata in ordered_metadata:
target_name = metadata["name"]
target_param_or_buf = (
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
)
target_param_or_buf.data = gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]].view(
metadata["shape"]
)
self._prefetch_done = evt
@torch.compiler.disable
def offload_layer(self) -> None:
"""Free GPU memory for layer by replacing tensors with empty placeholders.
This function does not actually offload weights from GPU back to CPU.
"""
evt = self._prefetch_done
if evt is not None:
torch.cuda.current_stream().wait_event(evt)
self._prefetch_done = None
# free GPU residency
for _, param in self.block_parameters.items():
param.data = torch.empty((), device=self.device, dtype=param.dtype)
for _, buf in self.block_buffers.items():
buf.data = torch.empty((), device=self.device, dtype=buf.dtype)
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
self.prefetch_layer(non_blocking=True)
return args, kwargs
def post_forward(self, module: nn.Module, output: Any) -> Any:
self.offload_layer()
return output
def apply_block_hook(
module: nn.Module,
next_block: nn.Module,
device: torch.device,
stream: torch.cuda.Stream | None = None,
pin_memory: bool = True,
) -> LayerwiseOffloadHook:
registry = HookRegistry.get_or_create(module)
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
return hook
def remove_block_hook(module: nn.Module) -> None:
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
if registry is not None:
registry.remove_hook(LayerwiseOffloadHook._HOOK_NAME)
logger.debug("Removed offload hook from %s", module.__class__.__name__)
class LayerWiseOffloadBackend(OffloadBackend):
"""Layer-wise (block-level) offloading backend.
Implements sliding window offloading where only a small number of transformer
blocks reside on GPU at a time. Blocks are prefetched asynchronously while
previous blocks compute, and freed after use.
"""
def __init__(self, config: OffloadConfig, device: torch.device):
super().__init__(config, device)
self.copy_stream = torch.cuda.Stream()
self._blocks: list[list[nn.Module]] = []
def enable(self, pipeline: nn.Module) -> None:
if self.enabled:
logger.warning("LayerWiseOffloadBackend already enabled")
return
modules = ModuleDiscovery.discover(pipeline)
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
return
# Move encoders to GPU (they stay resident)
for enc in modules.encoders:
enc.to(self.device)
# Move VAE to GPU if available
if modules.vae is not None:
try:
modules.vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
# Note that there might exist multiple DiT models in specific pipelines
for i, dit_module in enumerate(modules.dits):
dit_name = modules.dit_names[i]
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(dit_module)
blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
if not blocks_attr_name or not blocks:
logger.warning(
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
dit_name,
dit_module.__class__.__name__,
)
dit_module.to(self.device)
continue
num_blocks = len(blocks)
if num_blocks <= 1:
logger.warning(
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
dit_name,
dit_module.__class__.__name__,
)
dit_module.to(self.device)
continue
# Move non-block modules to GPU (they stay resident)
for name, m in dit_module.named_children():
if name == blocks_attr_name:
logger.debug(f"Skipped blocks module {name}")
continue
m.to(self.device)
logger.debug(f"Moved {name} to device {self.device}")
# Pre-fetch the first layer by manually calling the hook function on the last layer;
# For subsequent requests, the first layer/block will be pre-fetched
# during the last layer compute of the previous request.
last_block, first_block = blocks[-1], blocks[0]
hook = apply_block_hook(last_block, first_block, self.device, self.copy_stream, self.config.pin_cpu_memory)
hook.prefetch_layer(non_blocking=False)
# Register hook for each of blocks
for i, block in enumerate(blocks[:-1]):
next_block = blocks[(i + 1) % num_blocks]
apply_block_hook(block, next_block, self.device, self.copy_stream, self.config.pin_cpu_memory)
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
# Track hooked blocks for cleanup
self._blocks.append(blocks)
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
self.enabled = True
def disable(self) -> None:
if not self.enabled:
return
for blocks in self._blocks:
for block in blocks:
remove_block_hook(block)
self._blocks.clear()
self.enabled = False
logger.info("Layer-wise offloading disabled")
@staticmethod
def get_blocks_attr_name(model: nn.Module) -> str | None:
"""Retrieve blocks attribute name from provided DiT model"""
return getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
@staticmethod
def set_blocks_attr_name(model: nn.Module, name: str) -> None:
if not hasattr(model.__class__, "_layerwise_offload_blocks_attr"):
setattr(model.__class__, "_layerwise_offload_blocks_attr", name)
@staticmethod
def get_blocks_from_dit(model: nn.Module) -> list[nn.Module]:
"""
Retrieve a list of blocks from provided DiT model. Blocks attribute name
are found by `_layerwise_offload_blocks_attr` set to DiT models. For example,
```
class WanTransformer3DModel(nn.Module):
_layerwise_offload_blocks_attr = "blocks"
```
"""
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(model)
if blocks_attr_name is None:
logger.warning(
f"No _layerwise_offload_blocks_attr defined for {model.__class__.__name__}, "
"skipping layerwise offloading"
)
return []
_blocks = getattr(model, blocks_attr_name, None)
if _blocks is None:
logger.warning(
f"Blocks (layers) '{blocks_attr_name}' not found on {model.__class__.__name__}, "
"skipping layerwise offloading"
)
return []
return list(_blocks)
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from torch import nn
from vllm.logger import init_logger
logger = init_logger(__name__)
@dataclass
class PipelineModules:
dits: list[nn.Module]
dit_names: list[str]
encoders: list[nn.Module]
encoder_names: list[str]
vae: nn.Module | None = None
class ModuleDiscovery:
"""Discovers pipeline components for offloading"""
DIT_ATTRS = ["transformer", "transformer_2", "dit", "language_model", "transformer_blocks"]
ENCODER_ATTRS = ["text_encoder", "text_encoder_2", "text_encoder_3", "image_encoder"]
VAE_ATTRS = ["vae"]
@staticmethod
def discover(pipeline: nn.Module) -> PipelineModules:
"""Discover DiT, encoder, and VAE modules from pipeline.
Args:
pipeline: Diffusion pipeline model
Returns:
PipelineModules with lists of discovered modules and names
"""
# Collect DiT/transformer modules
dit_modules: list[nn.Module] = []
dit_names: list[str] = []
for attr in ModuleDiscovery.DIT_ATTRS:
if not hasattr(pipeline, attr):
continue
module_obj = getattr(pipeline, attr)
if module_obj is None:
continue
if not isinstance(module_obj, nn.Module):
logger.warning(f"Expected {attr} to be nn.Module, got {type(module_obj)!r}")
continue
if module_obj in dit_modules:
continue
dit_modules.append(module_obj)
dit_names.append(attr)
# Collect all encoders
encoders: list[nn.Module] = []
encoder_names: list[str] = []
for attr in ModuleDiscovery.ENCODER_ATTRS:
if hasattr(pipeline, attr) and getattr(pipeline, attr) is not None:
encoders.append(getattr(pipeline, attr))
encoder_names.append(attr)
# Collect VAE
vae = None
for attr in ModuleDiscovery.VAE_ATTRS:
module = getattr(pipeline, attr, None)
if module is not None:
vae = module
break
return PipelineModules(
dits=dit_modules,
dit_names=dit_names,
encoders=encoders,
encoder_names=encoder_names,
vae=vae,
)
@@ -0,0 +1,227 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig
from .module_collector import ModuleDiscovery
logger = init_logger(__name__)
class SequentialOffloadHook(ModelHook):
"""Hook for sequential offloading with mutual exclusion on encoder and DiT modules.
To be used as a model-level (or "component-level") of CPU offloading method;
When a module's forward is called, this hook offloads target modules to CPU
and loads the current module to GPU.
"""
_HOOK_NAME = "sequential_offload"
def __init__(
self,
offload_targets: list[nn.Module],
device: torch.device,
pin_memory: bool = True,
):
# Modules to offload to CPU before this module runs
self.offload_targets = offload_targets
self.device = device
self.pin_memory = pin_memory
def _to_cpu(self, module: nn.Module) -> None:
"""Move module to CPU."""
try:
param = next(module.parameters())
except StopIteration:
return
previous_device = param.device
# Skip if already on CPU
if previous_device.type == "cpu":
return
module.to("cpu")
torch.cuda.empty_cache()
if self.pin_memory:
for p in module.parameters():
if p.data.device.type == "cpu" and not p.data.is_pinned():
p.data = p.data.pin_memory()
def _to_gpu(self, module: nn.Module) -> None:
"""Move module to GPU."""
try:
# Skip if already on target device
if next(module.parameters()).device == self.device:
return
except StopIteration:
return
module.to(self.device)
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
# Offload target modules to CPU
for target in self.offload_targets:
self._to_cpu(target)
# Load current module to GPU
self._to_gpu(module)
current_omni_platform.synchronize()
logger.debug(
"Swapped: %s -> CPU, %s -> GPU",
[t.__class__.__name__ for t in self.offload_targets],
module.__class__.__name__,
)
return args, kwargs
def apply_sequential_offload(
dit_modules: list[nn.Module],
encoder_modules: list[nn.Module],
device: torch.device,
pin_memory: bool = True,
) -> None:
"""Apply sequential offloading hooks to DiT and encoder modules.
Registers hooks on modules to implement mutual-exclusion GPU allocation.
- Before DiT runs, encoders are offloaded to CPU.
- Before encoders run, DiT is offloaded to CPU.
Args:
dit_modules: DiT/transformer modules to register hooks on
encoder_modules: Encoder modules to register hooks on
device: Target GPU device for loading
pin_memory: Whether to pin CPU memory for faster transfers
Example:
>>> apply_sequential_offload(
... dit_modules=[pipeline.transformer],
... encoder_modules=[pipeline.text_encoder, pipeline.vae],
... device=torch.device("cuda:0"),
... )
>>> # Modules of pipeline now automatically swap between CPU and GPU
"""
# Register hooks on DiT modules (offload encoders when DiT runs)
for dit_mod in dit_modules:
registry = HookRegistry.get_or_create(dit_mod)
hook = SequentialOffloadHook(
offload_targets=encoder_modules,
device=device,
pin_memory=pin_memory,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
# Register hooks on encoders (offload DiTs when encoder runs)
for enc in encoder_modules:
registry = HookRegistry.get_or_create(enc)
hook = SequentialOffloadHook(
offload_targets=dit_modules,
device=device,
pin_memory=pin_memory,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
def remove_sequential_offload(modules: list[nn.Module]) -> None:
"""Remove sequential offloading hooks from modules.
Args:
modules: Modules to remove hooks from
Example:
>>> all_modules = [*dit_modules, *encoder_modules]
>>> remove_sequential_offload(all_modules)
"""
for module in modules:
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
if registry is not None:
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
logger.debug("Removed offload hook from %s", module.__class__.__name__)
class ModelLevelOffloadBackend(OffloadBackend):
"""Model-level (sequential) offloading backend.
Uses SequentialOffloadHook registered via HookRegistry for automatic module swapping.
"""
def __init__(self, config: OffloadConfig, device: torch.device):
super().__init__(config, device)
self._offload_modules: list[nn.Module] = [] # Track modules with hooks
def enable(self, pipeline: nn.Module) -> None:
if self.enabled:
logger.warning("ModelLevelOffloadBackend already enabled")
return
modules = ModuleDiscovery.discover(pipeline)
if not modules.dits:
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
return
if not modules.encoders:
logger.warning("No encoder modules found, skipping model-level offloading")
return
# Move encoders to GPU
for enc in modules.encoders:
enc.to(self.device)
# Move VAE to GPU if available
if modules.vae is not None:
try:
modules.vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
# Initial state: keep DiT modules on CPU (encoders typically run first)
# TODO: This part seems to be unnecessary, remove it after testing
# for dit_mod in modules.dits:
# dit_mod.to("cpu")
# torch.cuda.empty_cache()
# if self.config.pin_cpu_memory:
# for dit_mod in modules.dits:
# for p in dit_mod.parameters():
# if p.data.device.type == "cpu" and not p.data.is_pinned():
# p.data = p.data.pin_memory()
# Apply sequential offloading hooks
apply_sequential_offload(
dit_modules=modules.dits,
encoder_modules=modules.encoders,
device=self.device,
pin_memory=self.config.pin_cpu_memory,
)
# Track modules for cleanup
self._offload_modules = [*modules.dits, *modules.encoders]
self.enabled = True
logger.info(
"Model-level offloading enabled: %s <-> %s (mutual exclusion)",
", ".join(modules.dit_names),
", ".join(modules.encoder_names),
)
def disable(self) -> None:
if not self.enabled:
return
remove_sequential_offload(self._offload_modules)
self._offload_modules.clear()
self.enabled = False
logger.info("Model-level offloading disabled")