[Refactor] Let diffusion pipelines declare offloadable modules via SupportsModuleOffload (#2427)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
ead87aa521
commit
e3b0afbb97
11 files changed
+491
-62
No files matched your search
@@ -298,13 +298,20 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
|
||||
# Move VAE to GPU if available
|
||||
if modules.vae is not None:
|
||||
# Move VAE(s) to GPU if available
|
||||
for vae in modules.vaes:
|
||||
try:
|
||||
modules.vae.to(self.device, non_blocking=True)
|
||||
vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
|
||||
# Move resident modules to GPU (small modules needed every forward)
|
||||
for name, module in zip(modules.resident_names, modules.resident_modules):
|
||||
try:
|
||||
module.to(self.device)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move resident module %s to GPU: %s", name, exc)
|
||||
|
||||
logger.info("Applying layer-wise offloading on %s", modules.dit_names)
|
||||
|
||||
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from operator import attrgetter
|
||||
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -15,15 +18,81 @@ class PipelineModules:
|
||||
dit_names: list[str]
|
||||
encoders: list[nn.Module]
|
||||
encoder_names: list[str]
|
||||
vae: nn.Module | None = None
|
||||
vaes: list[nn.Module]
|
||||
resident_modules: list[nn.Module] = field(default_factory=list)
|
||||
resident_names: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class ModuleDiscovery:
|
||||
"""Discovers pipeline components for offloading"""
|
||||
"""Discovers pipeline components for offloading.
|
||||
|
||||
DIT_ATTRS = ["transformer", "transformer_2", "dit", "sr_dit", "language_model", "transformer_blocks", "model"]
|
||||
ENCODER_ATTRS = ["text_encoder", "text_encoder_2", "text_encoder_3", "image_encoder"]
|
||||
VAE_ATTRS = ["vae", "audio_vae"]
|
||||
If the pipeline implements :class:`SupportsModuleOffload`,
|
||||
its ``_dit_modules``, ``_encoder_modules``, and ``_vae_modules``
|
||||
class variables are used directly. Otherwise, falls back to
|
||||
scanning well-known attribute names.
|
||||
"""
|
||||
|
||||
# Fallback attribute names for pipelines that do not implement
|
||||
# SupportsModuleOffload.
|
||||
_FALLBACK_DIT_ATTRS = [
|
||||
"transformer",
|
||||
"transformer_2",
|
||||
"dit",
|
||||
"sr_dit",
|
||||
"language_model",
|
||||
"transformer_blocks",
|
||||
"model",
|
||||
]
|
||||
_FALLBACK_ENCODER_ATTRS = [
|
||||
"text_encoder",
|
||||
"text_encoder_2",
|
||||
"text_encoder_3",
|
||||
"image_encoder",
|
||||
]
|
||||
_FALLBACK_VAE_ATTRS = [
|
||||
"vae",
|
||||
"audio_vae",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _collect_modules(
|
||||
pipeline: nn.Module,
|
||||
attr_names: list[str],
|
||||
*,
|
||||
warn_missing: bool = False,
|
||||
) -> tuple[list[nn.Module], list[str]]:
|
||||
"""Resolve attribute names to (module, name) pairs, skipping missing.
|
||||
|
||||
Supports dotted paths via :func:`operator.attrgetter`.
|
||||
Warns on missing attributes when *warn_missing* is True.
|
||||
"""
|
||||
modules: list[nn.Module] = []
|
||||
names: list[str] = []
|
||||
seen: set[int] = set()
|
||||
for attr in attr_names:
|
||||
try:
|
||||
module = attrgetter(attr)(pipeline)
|
||||
except AttributeError:
|
||||
module = None
|
||||
if module is None:
|
||||
if warn_missing:
|
||||
logger.warning(
|
||||
"Pipeline declares '%s' as offloadable but the attribute does not exist or is None",
|
||||
attr,
|
||||
)
|
||||
continue
|
||||
if not isinstance(module, nn.Module):
|
||||
logger.warning(
|
||||
"Expected '%s' to be nn.Module, got %r",
|
||||
attr,
|
||||
type(module),
|
||||
)
|
||||
continue
|
||||
if id(module) not in seen:
|
||||
seen.add(id(module))
|
||||
modules.append(module)
|
||||
names.append(attr)
|
||||
return modules, names
|
||||
|
||||
@staticmethod
|
||||
def discover(pipeline: nn.Module) -> PipelineModules:
|
||||
@@ -35,46 +104,29 @@ class ModuleDiscovery:
|
||||
Returns:
|
||||
PipelineModules with lists of discovered modules and names
|
||||
"""
|
||||
# Collect DiT/transformer modules
|
||||
dit_modules: list[nn.Module] = []
|
||||
dit_names: list[str] = []
|
||||
for attr in ModuleDiscovery.DIT_ATTRS:
|
||||
if not hasattr(pipeline, attr):
|
||||
continue
|
||||
module_obj = getattr(pipeline, attr)
|
||||
if module_obj is None:
|
||||
continue
|
||||
declared = isinstance(pipeline, SupportsModuleOffload)
|
||||
if declared:
|
||||
dit_attrs = pipeline._dit_modules
|
||||
enc_attrs = pipeline._encoder_modules
|
||||
vae_attrs = pipeline._vae_modules
|
||||
res_attrs = pipeline._resident_modules
|
||||
else:
|
||||
dit_attrs = ModuleDiscovery._FALLBACK_DIT_ATTRS
|
||||
enc_attrs = ModuleDiscovery._FALLBACK_ENCODER_ATTRS
|
||||
vae_attrs = ModuleDiscovery._FALLBACK_VAE_ATTRS
|
||||
res_attrs = []
|
||||
|
||||
if not isinstance(module_obj, nn.Module):
|
||||
logger.warning(f"Expected {attr} to be nn.Module, got {type(module_obj)!r}")
|
||||
continue
|
||||
|
||||
if module_obj in dit_modules:
|
||||
continue
|
||||
|
||||
dit_modules.append(module_obj)
|
||||
dit_names.append(attr)
|
||||
|
||||
# Collect all encoders
|
||||
encoders: list[nn.Module] = []
|
||||
encoder_names: list[str] = []
|
||||
for attr in ModuleDiscovery.ENCODER_ATTRS:
|
||||
if hasattr(pipeline, attr) and getattr(pipeline, attr) is not None:
|
||||
encoders.append(getattr(pipeline, attr))
|
||||
encoder_names.append(attr)
|
||||
|
||||
# Collect VAE
|
||||
vae = None
|
||||
for attr in ModuleDiscovery.VAE_ATTRS:
|
||||
module = getattr(pipeline, attr, None)
|
||||
if module is not None:
|
||||
vae = module
|
||||
break
|
||||
dit_modules, dit_names = ModuleDiscovery._collect_modules(pipeline, dit_attrs, warn_missing=declared)
|
||||
encoders, encoder_names = ModuleDiscovery._collect_modules(pipeline, enc_attrs, warn_missing=declared)
|
||||
vaes, _ = ModuleDiscovery._collect_modules(pipeline, vae_attrs, warn_missing=declared)
|
||||
residents, resident_names = ModuleDiscovery._collect_modules(pipeline, res_attrs, warn_missing=declared)
|
||||
|
||||
return PipelineModules(
|
||||
dits=dit_modules,
|
||||
dit_names=dit_names,
|
||||
encoders=encoders,
|
||||
encoder_names=encoder_names,
|
||||
vae=vae,
|
||||
vaes=vaes,
|
||||
resident_modules=residents,
|
||||
resident_names=resident_names,
|
||||
)
|
||||
@@ -210,10 +210,10 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
for enc in modules.encoders:
|
||||
enc.to(self.device)
|
||||
|
||||
# Move VAE to GPU if available
|
||||
if modules.vae is not None:
|
||||
# Move VAE(s) to GPU if available
|
||||
for vae in modules.vaes:
|
||||
try:
|
||||
modules.vae.to(self.device, non_blocking=True)
|
||||
vae.to(self.device, non_blocking=True)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to move VAE to GPU: %s", exc)
|
||||
|
||||
|
||||
Reference in new issue
Block a user