[Refactor][Diffusion] Complete generic backend plan consumption (#7313)

Signed-off-by: specture724 <specture724@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Anjie HouandClaude Opus 5 authored and GitHub committed 2026-09-15 08:40:04 +08:00
1 parent 5385dda4fe
commit 85da943391
7 files changed
+228 -56

No files matched your search

@@ -14,11 +14,6 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
from .block_discovery import (
get_blocks_attr_names,
get_blocks_from_dit,
set_blocks_attr_names,
)
from .component_utils import (
clear_encoder_layerwise_state,
iter_streamable_dits,
@@ -26,7 +21,6 @@ from .component_utils import (
prepare_pipeline_components,
set_encoder_layerwise_state,
)
from .config import DIT_COMPONENT
from .plan_resolver import ResolvedComponent, resolve_offload_plan
from .tensor_utils import (
clear_block_storage,
@@ -433,9 +427,8 @@ class LayerWiseOffloadBackend(OffloadBackend):
return
resolved = resolve_offload_plan(pipeline, self.config)
if not resolved.dits and self.config.offloads(DIT_COMPONENT):
# An explicit selection already failed while resolving.
logger.warning("No DiT/transformer modules found for selected DiT layerwise offload")
if resolved.skip_reason is not None:
logger.warning("%s", resolved.skip_reason)
return
def enable_encoder_blocks(component: ResolvedComponent) -> bool:
@@ -458,15 +451,6 @@ class LayerWiseOffloadBackend(OffloadBackend):
enable_encoder_blocks=enable_encoder_blocks,
)
if not self.config.offloads(DIT_COMPONENT):
self.enabled = bool(self._encoder_modules or self._staged_components)
if not self.enabled:
raise ValueError(
"None of the selected layerwise offload components have "
"a model-declared streamable or on-demand plan"
)
return
logger.info("Applying layer-wise offloading on %s", [component.path for component in resolved.dits])
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
@@ -475,22 +459,9 @@ class LayerWiseOffloadBackend(OffloadBackend):
dit_module = component.module
blocks = list(stack.blocks)
# Move non-block modules to GPU (they stay resident)
for name, m in dit_module.named_children():
if name not in stack.attrs:
m.to(self.device)
logger.debug(f"Moved {name} to device {self.device}")
else:
logger.debug(f"Skipped blocks module {name}")
# Move top-level params/buffers to GPU (dit_module's own, not sub-modules)
for param in dit_module._parameters.values():
if param is not None:
param.data = param.data.to(self.device, non_blocking=True)
for buffer in dit_module._buffers.values():
if buffer is not None:
buffer.data = buffer.data.to(self.device, non_blocking=True)
# Place the remainder by resolved block tensor identity, just as
# for encoders. Attribute aliases must not move streamed weights.
move_non_block_state_to_device(dit_module, (stack.blocks,), self.device)
block_hooks = _install_layerwise_hook_group(
blocks,
@@ -550,8 +521,3 @@ class LayerWiseOffloadBackend(OffloadBackend):
def disable(self) -> None:
self._disable(restore_weights=True)
# Compatibility aliases for existing model integrations.
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
get_blocks_from_dit = staticmethod(get_blocks_from_dit)
@@ -91,6 +91,11 @@ class ResolvedOffloadPlan:
# still interpret topology themselves.
modules: PipelineModules
declaration: OffloadPlan | None = None
# Ordinary layerwise offload turns a legacy selection it cannot serve into
# a complete no-op, placement included. Distributed layerwise offload warns
# and keeps preparing its other components instead, so this stays scoped to
# the strategy that honors it.
skip_reason: str | None = None
@property
def components(self) -> tuple[ResolvedComponent, ...]:
@@ -304,6 +309,11 @@ def resolve_offload_plan(pipeline: nn.Module, config: OffloadConfig) -> Resolved
residents=tuple(residents),
modules=modules,
declaration=declaration,
skip_reason=(
"No DiT/transformer modules found for selected DiT layerwise offload"
if config.strategy is OffloadStrategy.LAYER_WISE and dit_selected and not dits
else None
),
)
_validate_unique_ownership(resolved)
return resolved
@@ -13,7 +13,6 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .config import DIT_COMPONENT
from .plan_resolver import resolve_offload_plan
logger = init_logger(__name__)
@@ -333,6 +332,7 @@ class ModelLevelOffloadBackend(OffloadBackend):
encoders = [component.module for component in resolved.encoders]
vaes = [component.module for component in resolved.vaes]
residents = [component.module for component in resolved.residents]
selected_dits = [component.module for component in resolved.dits if component.selected]
selected_encoders = [component.module for component in resolved.encoders if component.selected]
all_modules = [*dits, *encoders, *vaes, *residents]
@@ -360,7 +360,7 @@ class ModelLevelOffloadBackend(OffloadBackend):
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
offload_dit_modules=(dits if self.config.offloads(DIT_COMPONENT) else ()),
offload_dit_modules=selected_dits,
offload_encoder_modules=selected_encoders,
)
except BaseException: