[Refactor] Let diffusion pipelines declare offloadable modules via SupportsModuleOffload (#2427)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick CaoandClaude authored and GitHub committed 2026-04-23 04:23:00 +08:00
1 parent ead87aa521
commit e3b0afbb97
11 files changed
+491 -62

No files matched your search

@@ -298,13 +298,20 @@ class LayerWiseOffloadBackend(OffloadBackend):
for enc in modules.encoders:
enc.to(self.device)
# Move VAE to GPU if available
if modules.vae is not None:
# Move VAE(s) to GPU if available
for vae in modules.vaes:
try:
modules.vae.to(self.device, non_blocking=True)
vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)
# Move resident modules to GPU (small modules needed every forward)
for name, module in zip(modules.resident_names, modules.resident_modules):
try:
module.to(self.device)
except Exception as exc:
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, field
from operator import attrgetter
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
logger = init_logger(__name__)
@@ -15,15 +18,81 @@ class PipelineModules:
dit_names: list[str]
encoders: list[nn.Module]
encoder_names: list[str]
vae: nn.Module | None = None
vaes: list[nn.Module]
resident_modules: list[nn.Module] = field(default_factory=list)
resident_names: list[str] = field(default_factory=list)
class ModuleDiscovery:
"""Discovers pipeline components for offloading"""
"""Discovers pipeline components for offloading.
DIT_ATTRS = ["transformer", "transformer_2", "dit", "sr_dit", "language_model", "transformer_blocks", "model"]
ENCODER_ATTRS = ["text_encoder", "text_encoder_2", "text_encoder_3", "image_encoder"]
VAE_ATTRS = ["vae", "audio_vae"]
If the pipeline implements :class:`SupportsModuleOffload`,
its ``_dit_modules``, ``_encoder_modules``, and ``_vae_modules``
class variables are used directly. Otherwise, falls back to
scanning well-known attribute names.
"""
# Fallback attribute names for pipelines that do not implement
# SupportsModuleOffload.
_FALLBACK_DIT_ATTRS = [
"transformer",
"transformer_2",
"dit",
"sr_dit",
"language_model",
"transformer_blocks",
"model",
]
_FALLBACK_ENCODER_ATTRS = [
"text_encoder",
"text_encoder_2",
"text_encoder_3",
"image_encoder",
]
_FALLBACK_VAE_ATTRS = [
"vae",
"audio_vae",
]
@staticmethod
def _collect_modules(
pipeline: nn.Module,
attr_names: list[str],
*,
warn_missing: bool = False,
) -> tuple[list[nn.Module], list[str]]:
"""Resolve attribute names to (module, name) pairs, skipping missing.
Supports dotted paths via :func:`operator.attrgetter`.
Warns on missing attributes when *warn_missing* is True.
"""
modules: list[nn.Module] = []
names: list[str] = []
seen: set[int] = set()
for attr in attr_names:
try:
module = attrgetter(attr)(pipeline)
except AttributeError:
module = None
if module is None:
if warn_missing:
logger.warning(
"Pipeline declares '%s' as offloadable but the attribute does not exist or is None",
attr,
)
continue
if not isinstance(module, nn.Module):
logger.warning(
"Expected '%s' to be nn.Module, got %r",
attr,
type(module),
)
continue
if id(module) not in seen:
seen.add(id(module))
modules.append(module)
names.append(attr)
return modules, names
@staticmethod
def discover(pipeline: nn.Module) -> PipelineModules:
@@ -35,46 +104,29 @@ class ModuleDiscovery:
Returns:
PipelineModules with lists of discovered modules and names
"""
# Collect DiT/transformer modules
dit_modules: list[nn.Module] = []
dit_names: list[str] = []
for attr in ModuleDiscovery.DIT_ATTRS:
if not hasattr(pipeline, attr):
continue
module_obj = getattr(pipeline, attr)
if module_obj is None:
continue
declared = isinstance(pipeline, SupportsModuleOffload)
if declared:
dit_attrs = pipeline._dit_modules
enc_attrs = pipeline._encoder_modules
vae_attrs = pipeline._vae_modules
res_attrs = pipeline._resident_modules
else:
dit_attrs = ModuleDiscovery._FALLBACK_DIT_ATTRS
enc_attrs = ModuleDiscovery._FALLBACK_ENCODER_ATTRS
vae_attrs = ModuleDiscovery._FALLBACK_VAE_ATTRS
res_attrs = []
if not isinstance(module_obj, nn.Module):
logger.warning(f"Expected {attr} to be nn.Module, got {type(module_obj)!r}")
continue
if module_obj in dit_modules:
continue
dit_modules.append(module_obj)
dit_names.append(attr)
# Collect all encoders
encoders: list[nn.Module] = []
encoder_names: list[str] = []
for attr in ModuleDiscovery.ENCODER_ATTRS:
if hasattr(pipeline, attr) and getattr(pipeline, attr) is not None:
encoders.append(getattr(pipeline, attr))
encoder_names.append(attr)
# Collect VAE
vae = None
for attr in ModuleDiscovery.VAE_ATTRS:
module = getattr(pipeline, attr, None)
if module is not None:
vae = module
break
dit_modules, dit_names = ModuleDiscovery._collect_modules(pipeline, dit_attrs, warn_missing=declared)
encoders, encoder_names = ModuleDiscovery._collect_modules(pipeline, enc_attrs, warn_missing=declared)
vaes, _ = ModuleDiscovery._collect_modules(pipeline, vae_attrs, warn_missing=declared)
residents, resident_names = ModuleDiscovery._collect_modules(pipeline, res_attrs, warn_missing=declared)
return PipelineModules(
dits=dit_modules,
dit_names=dit_names,
encoders=encoders,
encoder_names=encoder_names,
vae=vae,
vaes=vaes,
resident_modules=residents,
resident_names=resident_names,
)
@@ -210,10 +210,10 @@ class ModelLevelOffloadBackend(OffloadBackend):
for enc in modules.encoders:
enc.to(self.device)
# Move VAE to GPU if available
if modules.vae is not None:
# Move VAE(s) to GPU if available
for vae in modules.vaes:
try:
modules.vae.to(self.device, non_blocking=True)
vae.to(self.device, non_blocking=True)
except Exception as exc:
logger.debug("Failed to move VAE to GPU: %s", exc)