Refactor CPU Offloading Backend Pattern (#1223)
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com> Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
1 parent
f117a07271
commit
8bf849c399
17 files changed
+980
-647
No files matched your search
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, OffloadStrategy
|
||||
from .layerwise_backend import LayerWiseOffloadBackend
|
||||
from .sequential_backend import (
|
||||
ModelLevelOffloadBackend,
|
||||
apply_sequential_offload,
|
||||
remove_sequential_offload,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"OffloadBackend",
|
||||
"OffloadConfig",
|
||||
"OffloadStrategy",
|
||||
"LayerWiseOffloadBackend",
|
||||
"ModelLevelOffloadBackend",
|
||||
"apply_sequential_offload",
|
||||
"remove_sequential_offload",
|
||||
"get_offload_backend",
|
||||
]
|
||||
|
||||
|
||||
def get_offload_backend(
|
||||
od_config: OmniDiffusionConfig,
|
||||
device: torch.device | None = None,
|
||||
) -> OffloadBackend | None:
|
||||
"""Create appropriate offload backend based on configuration.
|
||||
|
||||
Args:
|
||||
od_config: OmniDiffusionConfig with offload settings
|
||||
device: Target device (auto-detected if None)
|
||||
|
||||
Returns:
|
||||
OffloadBackend instance or None if offloading disabled
|
||||
|
||||
Example:
|
||||
>>> backend = get_offload_backend(od_config, device=torch.device("cuda:0"))
|
||||
>>> if backend:
|
||||
... backend.enable(pipeline)
|
||||
"""
|
||||
# Extract and validate configuration
|
||||
config = OffloadConfig.from_od_config(od_config)
|
||||
|
||||
# Return None if no offloading requested
|
||||
if config.strategy == OffloadStrategy.NONE:
|
||||
return None
|
||||
|
||||
# Validate platform (CUDA required for now)
|
||||
if not current_omni_platform.is_cuda() or current_omni_platform.get_device_count() < 1:
|
||||
logger.warning("CPU offloading requires CUDA devices. Skipping offloading.")
|
||||
return None
|
||||
|
||||
# Detect device if not provided
|
||||
if device is None:
|
||||
try:
|
||||
device = current_omni_platform.get_torch_device()
|
||||
except (NotImplementedError, AttributeError) as exc:
|
||||
logger.error("Failed to detect device: %s. Skipping offloading.", exc)
|
||||
return None
|
||||
|
||||
# Create appropriate backend
|
||||
if config.strategy == OffloadStrategy.MODEL_LEVEL:
|
||||
return ModelLevelOffloadBackend(config, device)
|
||||
elif config.strategy == OffloadStrategy.LAYER_WISE:
|
||||
return LayerWiseOffloadBackend(config, device)
|
||||
else:
|
||||
logger.error("Unknown offload strategy: %s", config.strategy)
|
||||
return None
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OffloadStrategy(Enum):
|
||||
NONE = "none"
|
||||
MODEL_LEVEL = "model_level" # Sequential offloading between DiT and encoders
|
||||
LAYER_WISE = "layer_wise" # Block-level
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadConfig:
|
||||
strategy: OffloadStrategy
|
||||
pin_cpu_memory: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig":
|
||||
"""Extract and validate offload settings from OmniDiffusionConfig.
|
||||
|
||||
For now, enforces mutual exclusion between model-level and layer-wise offloading.
|
||||
Layer-wise takes priority if both are enabled.
|
||||
|
||||
Args:
|
||||
od_config: OmniDiffusionConfig with offload settings
|
||||
|
||||
Returns:
|
||||
OffloadConfig with validated settings
|
||||
"""
|
||||
enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False)
|
||||
enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False)
|
||||
pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True)
|
||||
|
||||
# Determine strategy (mutual exclusion, layer-wise takes priority)
|
||||
if enable_layerwise_offload:
|
||||
strategy = OffloadStrategy.LAYER_WISE
|
||||
if enable_cpu_offload:
|
||||
logger.info(
|
||||
"Both model-level and layer-wise offloading enabled. "
|
||||
"Layer-wise takes priority, disabling model-level offloading."
|
||||
)
|
||||
elif enable_cpu_offload:
|
||||
strategy = OffloadStrategy.MODEL_LEVEL
|
||||
else:
|
||||
strategy = OffloadStrategy.NONE
|
||||
|
||||
return cls(
|
||||
strategy=strategy,
|
||||
pin_cpu_memory=pin_cpu_memory,
|
||||
)
|
||||
|
||||
|
||||
class OffloadBackend(ABC):
|
||||
"""Base class for CPU offload backends"""
|
||||
|
||||
def __init__(self, config: OffloadConfig, device: torch.device):
|
||||
self.config = config
|
||||
self.device = device
|
||||
self.enabled = False
|
||||
|
||||
@abstractmethod
|
||||
def enable(self, pipeline: nn.Module) -> None:
|
||||
"""Enable offloading on the pipeline.
|
||||
|
||||
Discovers modules, moves them to appropriate devices, and
|
||||
registers forward hooks for swapping/prefetching.
|
||||
|
||||
Args:
|
||||
pipeline: Diffusion pipeline model (e.g., Wan22Pipeline)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def disable(self) -> None:
|
||||
"""Disable offloading and cleanup resources.
|
||||
|
||||
Removes all registered hooks. Does NOT move modules back to
|
||||
original devices (caller responsible for that).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return self.enabled
|
||||
@@ -0,0 +1,362 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import chain
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig
|
||||
from .module_collector import ModuleDiscovery
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LayerwiseOffloadHook(ModelHook):
|
||||
"""Hook for layerwise (transformer-block-wise) CPU offloading.
|
||||
|
||||
The hook instance retains parameters for both the current registered block
|
||||
module and those for the next block, as well as flattened CPU tensors which
|
||||
record the parameters of the current block module, so that these parameters
|
||||
could be re-materialized on device in an overlapping way.
|
||||
This hook should be registered to each of the transformer blocks in DiT
|
||||
module(s) of the target pipeline.
|
||||
|
||||
Based on implementations from:
|
||||
https://github.com/sgl-project/sglang/blob/v0.5.8/python/sglang/multimodal_gen/runtime/utils/layerwise_offload.py
|
||||
"""
|
||||
|
||||
_HOOK_NAME = "layerwise_offload"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
next_block: nn.Module,
|
||||
device: torch.device,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
):
|
||||
assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`"
|
||||
assert current_omni_platform.is_cuda(), "Layerwise offloading is only supported on cuda devices for now"
|
||||
|
||||
self.next_block = next_block
|
||||
self.device = device
|
||||
self.copy_stream = stream or torch.cuda.current_stream()
|
||||
self.pin_memory = pin_memory
|
||||
|
||||
# Per-block synchronization primitive: set after H2D copy completes.
|
||||
self._prefetch_done: torch.cuda.Event | None = None
|
||||
|
||||
self.next_block_parameters: dict[str, nn.Parameter] = {}
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = {}
|
||||
self.dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
self.dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
|
||||
def initialize_hook(self, module: nn.Module) -> nn.Module:
|
||||
# This all happen during the hook instance being registered to hook registry;
|
||||
# the input module is kept intact
|
||||
module = super().initialize_hook(module)
|
||||
|
||||
self.block_parameters: dict[str, nn.Parameter] = dict(module.named_parameters())
|
||||
self.block_buffers: dict[str, torch.Tensor] = dict(module.named_buffers())
|
||||
|
||||
self.next_block_parameters: dict[str, nn.Parameter] = dict(self.next_block.named_parameters())
|
||||
self.next_block_buffers: dict[str, torch.Tensor] = dict(self.next_block.named_buffers())
|
||||
|
||||
# Pre-allocate gpu tensors in a flattened way
|
||||
self.dtype_cpu_flattened_weights, self.dtype_metadata = LayerwiseOffloadHook._to_cpu(
|
||||
self.next_block_parameters, self.next_block_buffers, self.device, self.pin_memory
|
||||
)
|
||||
|
||||
return module
|
||||
|
||||
@staticmethod
|
||||
def _to_cpu(
|
||||
params: dict[str, nn.Parameter],
|
||||
bufs: dict[str, torch.Tensor],
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]:
|
||||
"""Helper method to move block parameters and buffers to CPU, flattening by dtype.
|
||||
|
||||
Consolidates parameters and buffers into contiguous CPU tensors grouped by dtype
|
||||
for GPU transfers. Replaces original tensors with empty placeholders.
|
||||
|
||||
Returns:
|
||||
Tuple of
|
||||
flattened CPU tensors by dtype,
|
||||
metadata for reconstruction by dtype
|
||||
"""
|
||||
dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {}
|
||||
dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
# NOTE: order does matter
|
||||
dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {}
|
||||
|
||||
for name, param_or_buf in chain(params.items(), bufs.items()):
|
||||
dtype = param_or_buf.dtype
|
||||
if dtype not in dtype_grouped_weights:
|
||||
dtype_grouped_weights[dtype] = {}
|
||||
dtype_grouped_weights[dtype][name] = param_or_buf
|
||||
|
||||
for dtype, name2weights in dtype_grouped_weights.items():
|
||||
# total # of parameters + buffers
|
||||
total_numel = sum(t.numel() for _, t in name2weights.items())
|
||||
cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=pin_memory)
|
||||
|
||||
current_offset = 0
|
||||
for name, param_or_buf in name2weights.items():
|
||||
numel = param_or_buf.numel()
|
||||
cpu_tensor[current_offset : current_offset + numel].copy_(param_or_buf.flatten())
|
||||
if dtype not in dtype_metadata:
|
||||
dtype_metadata[dtype] = []
|
||||
dtype_metadata[dtype].append(
|
||||
{
|
||||
"name": name,
|
||||
"offset": current_offset,
|
||||
"numel": numel,
|
||||
"shape": param_or_buf.shape,
|
||||
}
|
||||
)
|
||||
|
||||
param_or_buf.data = torch.empty((), device=device, dtype=dtype)
|
||||
current_offset += numel
|
||||
|
||||
dtype_cpu_flattened_weights[dtype] = cpu_tensor
|
||||
|
||||
return dtype_cpu_flattened_weights, dtype_metadata
|
||||
|
||||
@torch.compiler.disable
|
||||
def prefetch_layer(self, non_blocking: bool = True) -> None:
|
||||
"""Copy layer weights from CPU -> GPU.
|
||||
|
||||
Pre-fetch target block in an asynchronous way with compute - memory copy overlap,
|
||||
with non_blocking set to True.
|
||||
"""
|
||||
self.copy_stream.wait_stream(torch.cuda.current_stream())
|
||||
|
||||
layer_params = self.next_block_parameters
|
||||
layer_bufs = self.next_block_buffers
|
||||
|
||||
evt = torch.cuda.Event()
|
||||
gpu_weights: dict[torch.dtype, torch.Tensor] = {}
|
||||
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
for dtype, cpu_weight in self.dtype_cpu_flattened_weights.items():
|
||||
gpu_weight = torch.empty(cpu_weight.shape, dtype=dtype, device=self.device)
|
||||
gpu_weight.copy_(cpu_weight, non_blocking=non_blocking)
|
||||
gpu_weights[dtype] = gpu_weight
|
||||
|
||||
evt.record(self.copy_stream)
|
||||
|
||||
for dtype, ordered_metadata in self.dtype_metadata.items():
|
||||
# ordered_metadata: list[dict[str, Any]]
|
||||
gpu_weight = gpu_weights[dtype]
|
||||
|
||||
for metadata in ordered_metadata:
|
||||
target_name = metadata["name"]
|
||||
target_param_or_buf = (
|
||||
layer_params[target_name] if target_name in layer_params else layer_bufs[target_name]
|
||||
)
|
||||
|
||||
target_param_or_buf.data = gpu_weight[metadata["offset"] : metadata["offset"] + metadata["numel"]].view(
|
||||
metadata["shape"]
|
||||
)
|
||||
|
||||
self._prefetch_done = evt
|
||||
|
||||
@torch.compiler.disable
|
||||
def offload_layer(self) -> None:
|
||||
"""Free GPU memory for layer by replacing tensors with empty placeholders.
|
||||
This function does not actually offload weights from GPU back to CPU.
|
||||
"""
|
||||
evt = self._prefetch_done
|
||||
if evt is not None:
|
||||
torch.cuda.current_stream().wait_event(evt)
|
||||
|
||||
self._prefetch_done = None
|
||||
|
||||
# free GPU residency
|
||||
for _, param in self.block_parameters.items():
|
||||
param.data = torch.empty((), device=self.device, dtype=param.dtype)
|
||||
for _, buf in self.block_buffers.items():
|
||||
buf.data = torch.empty((), device=self.device, dtype=buf.dtype)
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]:
|
||||
self.prefetch_layer(non_blocking=True)
|
||||
|
||||
return args, kwargs
|
||||
|
||||
def post_forward(self, module: nn.Module, output: Any) -> Any:
|
||||
self.offload_layer()
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def apply_block_hook(
|
||||
module: nn.Module,
|
||||
next_block: nn.Module,
|
||||
device: torch.device,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
pin_memory: bool = True,
|
||||
) -> LayerwiseOffloadHook:
|
||||
registry = HookRegistry.get_or_create(module)
|
||||
hook = LayerwiseOffloadHook(next_block, device, stream, pin_memory)
|
||||
registry.register_hook(LayerwiseOffloadHook._HOOK_NAME, hook)
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
def remove_block_hook(module: nn.Module) -> None:
|
||||
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
|
||||
if registry is not None:
|
||||
registry.remove_hook(LayerwiseOffloadHook._HOOK_NAME)
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
|
||||
|
||||
class LayerWiseOffloadBackend(OffloadBackend):
|
||||
"""Layer-wise (block-level) offloading backend.
|
||||
|
||||
Implements sliding window offloading where only a small number of transformer
|
||||
blocks reside on GPU at a time. Blocks are prefetched asynchronously while
|
||||
previous blocks compute, and freed after use.
|
||||
"""
|
||||
|
||||
def __init__(self, config: OffloadConfig, device: torch.device):
|
||||
super().__init__(config, device)
|
||||
|
||||
self.copy_stream = torch.cuda.Stream()
|
||||
self._blocks: list[list[nn.Module]] = []
|
||||
|
||||
def enable(self, pipeline: nn.Module) -> None:
|
||||
if self.enabled:
|
||||
logger.warning("LayerWiseOffloadBackend already enabled")
|
||||
return
|
||||
|
||||
modules = ModuleDiscovery.discover(pipeline)
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping layer-wise offloading")
|
||||
return
|
||||
|
||||
# Move encoders to GPU (they stay resident)
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
|
||||
# Move VAE to GPU if available
|
||||
if modules.vae is not None:
|
||||
try:
|
||||
modules.vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
|
||||
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
|
||||
|
||||
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
|
||||
# Note that there might exist multiple DiT models in specific pipelines
|
||||
for i, dit_module in enumerate(modules.dits):
|
||||
dit_name = modules.dit_names[i]
|
||||
logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})")
|
||||
|
||||
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(dit_module)
|
||||
blocks = LayerWiseOffloadBackend.get_blocks_from_dit(dit_module)
|
||||
|
||||
if not blocks_attr_name or not blocks:
|
||||
logger.warning(
|
||||
"Target layers (blocks) not found. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
dit_module.__class__.__name__,
|
||||
)
|
||||
dit_module.to(self.device)
|
||||
continue
|
||||
|
||||
num_blocks = len(blocks)
|
||||
if num_blocks <= 1:
|
||||
logger.warning(
|
||||
"#Target layers (blocks) <= 1. Skipping offloading on %s (%s)",
|
||||
dit_name,
|
||||
dit_module.__class__.__name__,
|
||||
)
|
||||
dit_module.to(self.device)
|
||||
continue
|
||||
|
||||
# Move non-block modules to GPU (they stay resident)
|
||||
for name, m in dit_module.named_children():
|
||||
if name == blocks_attr_name:
|
||||
logger.debug(f"Skipped blocks module {name}")
|
||||
continue
|
||||
m.to(self.device)
|
||||
logger.debug(f"Moved {name} to device {self.device}")
|
||||
|
||||
# Pre-fetch the first layer by manually calling the hook function on the last layer;
|
||||
# For subsequent requests, the first layer/block will be pre-fetched
|
||||
# during the last layer compute of the previous request.
|
||||
last_block, first_block = blocks[-1], blocks[0]
|
||||
hook = apply_block_hook(last_block, first_block, self.device, self.copy_stream, self.config.pin_cpu_memory)
|
||||
hook.prefetch_layer(non_blocking=False)
|
||||
|
||||
# Register hook for each of blocks
|
||||
for i, block in enumerate(blocks[:-1]):
|
||||
next_block = blocks[(i + 1) % num_blocks]
|
||||
apply_block_hook(block, next_block, self.device, self.copy_stream, self.config.pin_cpu_memory)
|
||||
|
||||
logger.info(f"Layer-wise offloading enabled on {num_blocks} layers (blocks)")
|
||||
|
||||
# Track hooked blocks for cleanup
|
||||
self._blocks.append(blocks)
|
||||
|
||||
if len(self._blocks) > 0 and len(self._blocks[0]) > 0:
|
||||
self.enabled = True
|
||||
|
||||
def disable(self) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
for blocks in self._blocks:
|
||||
for block in blocks:
|
||||
remove_block_hook(block)
|
||||
|
||||
self._blocks.clear()
|
||||
self.enabled = False
|
||||
logger.info("Layer-wise offloading disabled")
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_attr_name(model: nn.Module) -> str | None:
|
||||
"""Retrieve blocks attribute name from provided DiT model"""
|
||||
return getattr(model.__class__, "_layerwise_offload_blocks_attr", None)
|
||||
|
||||
@staticmethod
|
||||
def set_blocks_attr_name(model: nn.Module, name: str) -> None:
|
||||
if not hasattr(model.__class__, "_layerwise_offload_blocks_attr"):
|
||||
setattr(model.__class__, "_layerwise_offload_blocks_attr", name)
|
||||
|
||||
@staticmethod
|
||||
def get_blocks_from_dit(model: nn.Module) -> list[nn.Module]:
|
||||
"""
|
||||
Retrieve a list of blocks from provided DiT model. Blocks attribute name
|
||||
are found by `_layerwise_offload_blocks_attr` set to DiT models. For example,
|
||||
|
||||
```
|
||||
class WanTransformer3DModel(nn.Module):
|
||||
_layerwise_offload_blocks_attr = "blocks"
|
||||
```
|
||||
"""
|
||||
blocks_attr_name = LayerWiseOffloadBackend.get_blocks_attr_name(model)
|
||||
if blocks_attr_name is None:
|
||||
logger.warning(
|
||||
f"No _layerwise_offload_blocks_attr defined for {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
)
|
||||
return []
|
||||
|
||||
_blocks = getattr(model, blocks_attr_name, None)
|
||||
if _blocks is None:
|
||||
logger.warning(
|
||||
f"Blocks (layers) '{blocks_attr_name}' not found on {model.__class__.__name__}, "
|
||||
"skipping layerwise offloading"
|
||||
)
|
||||
return []
|
||||
|
||||
return list(_blocks)
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineModules:
|
||||
dits: list[nn.Module]
|
||||
dit_names: list[str]
|
||||
encoders: list[nn.Module]
|
||||
encoder_names: list[str]
|
||||
vae: nn.Module | None = None
|
||||
|
||||
|
||||
class ModuleDiscovery:
|
||||
"""Discovers pipeline components for offloading"""
|
||||
|
||||
DIT_ATTRS = ["transformer", "transformer_2", "dit", "language_model", "transformer_blocks"]
|
||||
ENCODER_ATTRS = ["text_encoder", "text_encoder_2", "text_encoder_3", "image_encoder"]
|
||||
VAE_ATTRS = ["vae"]
|
||||
|
||||
@staticmethod
|
||||
def discover(pipeline: nn.Module) -> PipelineModules:
|
||||
"""Discover DiT, encoder, and VAE modules from pipeline.
|
||||
|
||||
Args:
|
||||
pipeline: Diffusion pipeline model
|
||||
|
||||
Returns:
|
||||
PipelineModules with lists of discovered modules and names
|
||||
"""
|
||||
# Collect DiT/transformer modules
|
||||
dit_modules: list[nn.Module] = []
|
||||
dit_names: list[str] = []
|
||||
for attr in ModuleDiscovery.DIT_ATTRS:
|
||||
if not hasattr(pipeline, attr):
|
||||
continue
|
||||
module_obj = getattr(pipeline, attr)
|
||||
if module_obj is None:
|
||||
continue
|
||||
|
||||
if not isinstance(module_obj, nn.Module):
|
||||
logger.warning(f"Expected {attr} to be nn.Module, got {type(module_obj)!r}")
|
||||
continue
|
||||
|
||||
if module_obj in dit_modules:
|
||||
continue
|
||||
|
||||
dit_modules.append(module_obj)
|
||||
dit_names.append(attr)
|
||||
|
||||
# Collect all encoders
|
||||
encoders: list[nn.Module] = []
|
||||
encoder_names: list[str] = []
|
||||
for attr in ModuleDiscovery.ENCODER_ATTRS:
|
||||
if hasattr(pipeline, attr) and getattr(pipeline, attr) is not None:
|
||||
encoders.append(getattr(pipeline, attr))
|
||||
encoder_names.append(attr)
|
||||
|
||||
# Collect VAE
|
||||
vae = None
|
||||
for attr in ModuleDiscovery.VAE_ATTRS:
|
||||
module = getattr(pipeline, attr, None)
|
||||
if module is not None:
|
||||
vae = module
|
||||
break
|
||||
|
||||
return PipelineModules(
|
||||
dits=dit_modules,
|
||||
dit_names=dit_names,
|
||||
encoders=encoders,
|
||||
encoder_names=encoder_names,
|
||||
vae=vae,
|
||||
)
|
||||
@@ -0,0 +1,227 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig
|
||||
from .module_collector import ModuleDiscovery
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SequentialOffloadHook(ModelHook):
|
||||
"""Hook for sequential offloading with mutual exclusion on encoder and DiT modules.
|
||||
|
||||
To be used as a model-level (or "component-level") of CPU offloading method;
|
||||
When a module's forward is called, this hook offloads target modules to CPU
|
||||
and loads the current module to GPU.
|
||||
"""
|
||||
|
||||
_HOOK_NAME = "sequential_offload"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offload_targets: list[nn.Module],
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
):
|
||||
# Modules to offload to CPU before this module runs
|
||||
self.offload_targets = offload_targets
|
||||
self.device = device
|
||||
self.pin_memory = pin_memory
|
||||
|
||||
def _to_cpu(self, module: nn.Module) -> None:
|
||||
"""Move module to CPU."""
|
||||
try:
|
||||
param = next(module.parameters())
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
previous_device = param.device
|
||||
# Skip if already on CPU
|
||||
if previous_device.type == "cpu":
|
||||
return
|
||||
|
||||
module.to("cpu")
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if self.pin_memory:
|
||||
for p in module.parameters():
|
||||
if p.data.device.type == "cpu" and not p.data.is_pinned():
|
||||
p.data = p.data.pin_memory()
|
||||
|
||||
def _to_gpu(self, module: nn.Module) -> None:
|
||||
"""Move module to GPU."""
|
||||
try:
|
||||
# Skip if already on target device
|
||||
if next(module.parameters()).device == self.device:
|
||||
return
|
||||
except StopIteration:
|
||||
return
|
||||
|
||||
module.to(self.device)
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
|
||||
# Offload target modules to CPU
|
||||
for target in self.offload_targets:
|
||||
self._to_cpu(target)
|
||||
|
||||
# Load current module to GPU
|
||||
self._to_gpu(module)
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
logger.debug(
|
||||
"Swapped: %s -> CPU, %s -> GPU",
|
||||
[t.__class__.__name__ for t in self.offload_targets],
|
||||
module.__class__.__name__,
|
||||
)
|
||||
|
||||
return args, kwargs
|
||||
|
||||
|
||||
def apply_sequential_offload(
|
||||
dit_modules: list[nn.Module],
|
||||
encoder_modules: list[nn.Module],
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
) -> None:
|
||||
"""Apply sequential offloading hooks to DiT and encoder modules.
|
||||
|
||||
Registers hooks on modules to implement mutual-exclusion GPU allocation.
|
||||
- Before DiT runs, encoders are offloaded to CPU.
|
||||
- Before encoders run, DiT is offloaded to CPU.
|
||||
|
||||
Args:
|
||||
dit_modules: DiT/transformer modules to register hooks on
|
||||
encoder_modules: Encoder modules to register hooks on
|
||||
device: Target GPU device for loading
|
||||
pin_memory: Whether to pin CPU memory for faster transfers
|
||||
|
||||
Example:
|
||||
>>> apply_sequential_offload(
|
||||
... dit_modules=[pipeline.transformer],
|
||||
... encoder_modules=[pipeline.text_encoder, pipeline.vae],
|
||||
... device=torch.device("cuda:0"),
|
||||
... )
|
||||
>>> # Modules of pipeline now automatically swap between CPU and GPU
|
||||
"""
|
||||
# Register hooks on DiT modules (offload encoders when DiT runs)
|
||||
for dit_mod in dit_modules:
|
||||
registry = HookRegistry.get_or_create(dit_mod)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=encoder_modules,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
|
||||
|
||||
# Register hooks on encoders (offload DiTs when encoder runs)
|
||||
for enc in encoder_modules:
|
||||
registry = HookRegistry.get_or_create(enc)
|
||||
hook = SequentialOffloadHook(
|
||||
offload_targets=dit_modules,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
|
||||
|
||||
|
||||
def remove_sequential_offload(modules: list[nn.Module]) -> None:
|
||||
"""Remove sequential offloading hooks from modules.
|
||||
|
||||
Args:
|
||||
modules: Modules to remove hooks from
|
||||
|
||||
Example:
|
||||
>>> all_modules = [*dit_modules, *encoder_modules]
|
||||
>>> remove_sequential_offload(all_modules)
|
||||
"""
|
||||
for module in modules:
|
||||
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
|
||||
if registry is not None:
|
||||
registry.remove_hook(SequentialOffloadHook._HOOK_NAME)
|
||||
logger.debug("Removed offload hook from %s", module.__class__.__name__)
|
||||
|
||||
|
||||
class ModelLevelOffloadBackend(OffloadBackend):
|
||||
"""Model-level (sequential) offloading backend.
|
||||
|
||||
Uses SequentialOffloadHook registered via HookRegistry for automatic module swapping.
|
||||
"""
|
||||
|
||||
def __init__(self, config: OffloadConfig, device: torch.device):
|
||||
super().__init__(config, device)
|
||||
self._offload_modules: list[nn.Module] = [] # Track modules with hooks
|
||||
|
||||
def enable(self, pipeline: nn.Module) -> None:
|
||||
if self.enabled:
|
||||
logger.warning("ModelLevelOffloadBackend already enabled")
|
||||
return
|
||||
|
||||
modules = ModuleDiscovery.discover(pipeline)
|
||||
if not modules.dits:
|
||||
logger.warning("No DiT/transformer modules found, skipping model-level offloading")
|
||||
return
|
||||
if not modules.encoders:
|
||||
logger.warning("No encoder modules found, skipping model-level offloading")
|
||||
return
|
||||
|
||||
# Move encoders to GPU
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
|
||||
# Move VAE to GPU if available
|
||||
if modules.vae is not None:
|
||||
try:
|
||||
modules.vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
|
||||
# Initial state: keep DiT modules on CPU (encoders typically run first)
|
||||
# TODO: This part seems to be unnecessary, remove it after testing
|
||||
# for dit_mod in modules.dits:
|
||||
# dit_mod.to("cpu")
|
||||
|
||||
# torch.cuda.empty_cache()
|
||||
|
||||
# if self.config.pin_cpu_memory:
|
||||
# for dit_mod in modules.dits:
|
||||
# for p in dit_mod.parameters():
|
||||
# if p.data.device.type == "cpu" and not p.data.is_pinned():
|
||||
# p.data = p.data.pin_memory()
|
||||
|
||||
# Apply sequential offloading hooks
|
||||
apply_sequential_offload(
|
||||
dit_modules=modules.dits,
|
||||
encoder_modules=modules.encoders,
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
)
|
||||
|
||||
# Track modules for cleanup
|
||||
self._offload_modules = [*modules.dits, *modules.encoders]
|
||||
|
||||
self.enabled = True
|
||||
|
||||
logger.info(
|
||||
"Model-level offloading enabled: %s <-> %s (mutual exclusion)",
|
||||
", ".join(modules.dit_names),
|
||||
", ".join(modules.encoder_names),
|
||||
)
|
||||
|
||||
def disable(self) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
remove_sequential_offload(self._offload_modules)
|
||||
|
||||
self._offload_modules.clear()
|
||||
self.enabled = False
|
||||
logger.info("Model-level offloading disabled")
|
||||
Reference in new issue
Block a user