[Refactor][Diffusion] Complete generic backend plan consumption (#7313)
Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5385dda4fe
commit
85da943391
7 files changed
+228
-56
No files matched your search
@@ -14,11 +14,6 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, run_cleanup_steps
|
||||
from .block_discovery import (
|
||||
get_blocks_attr_names,
|
||||
get_blocks_from_dit,
|
||||
set_blocks_attr_names,
|
||||
)
|
||||
from .component_utils import (
|
||||
clear_encoder_layerwise_state,
|
||||
iter_streamable_dits,
|
||||
@@ -26,7 +21,6 @@ from .component_utils import (
|
||||
prepare_pipeline_components,
|
||||
set_encoder_layerwise_state,
|
||||
)
|
||||
from .config import DIT_COMPONENT
|
||||
from .plan_resolver import ResolvedComponent, resolve_offload_plan
|
||||
from .tensor_utils import (
|
||||
clear_block_storage,
|
||||
@@ -433,9 +427,8 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
return
|
||||
|
||||
resolved = resolve_offload_plan(pipeline, self.config)
|
||||
if not resolved.dits and self.config.offloads(DIT_COMPONENT):
|
||||
# An explicit selection already failed while resolving.
|
||||
logger.warning("No DiT/transformer modules found for selected DiT layerwise offload")
|
||||
if resolved.skip_reason is not None:
|
||||
logger.warning("%s", resolved.skip_reason)
|
||||
return
|
||||
|
||||
def enable_encoder_blocks(component: ResolvedComponent) -> bool:
|
||||
@@ -458,15 +451,6 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
enable_encoder_blocks=enable_encoder_blocks,
|
||||
)
|
||||
|
||||
if not self.config.offloads(DIT_COMPONENT):
|
||||
self.enabled = bool(self._encoder_modules or self._staged_components)
|
||||
if not self.enabled:
|
||||
raise ValueError(
|
||||
"None of the selected layerwise offload components have "
|
||||
"a model-declared streamable or on-demand plan"
|
||||
)
|
||||
return
|
||||
|
||||
logger.info("Applying layer-wise offloading on %s", [component.path for component in resolved.dits])
|
||||
|
||||
# Apply block-wise offloading hook for each of the blocks in DiT model(s)
|
||||
@@ -475,22 +459,9 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
dit_module = component.module
|
||||
blocks = list(stack.blocks)
|
||||
|
||||
# Move non-block modules to GPU (they stay resident)
|
||||
for name, m in dit_module.named_children():
|
||||
if name not in stack.attrs:
|
||||
m.to(self.device)
|
||||
logger.debug(f"Moved {name} to device {self.device}")
|
||||
else:
|
||||
logger.debug(f"Skipped blocks module {name}")
|
||||
|
||||
# Move top-level params/buffers to GPU (dit_module's own, not sub-modules)
|
||||
for param in dit_module._parameters.values():
|
||||
if param is not None:
|
||||
param.data = param.data.to(self.device, non_blocking=True)
|
||||
|
||||
for buffer in dit_module._buffers.values():
|
||||
if buffer is not None:
|
||||
buffer.data = buffer.data.to(self.device, non_blocking=True)
|
||||
# Place the remainder by resolved block tensor identity, just as
|
||||
# for encoders. Attribute aliases must not move streamed weights.
|
||||
move_non_block_state_to_device(dit_module, (stack.blocks,), self.device)
|
||||
|
||||
block_hooks = _install_layerwise_hook_group(
|
||||
blocks,
|
||||
@@ -550,8 +521,3 @@ class LayerWiseOffloadBackend(OffloadBackend):
|
||||
|
||||
def disable(self) -> None:
|
||||
self._disable(restore_weights=True)
|
||||
|
||||
# Compatibility aliases for existing model integrations.
|
||||
get_blocks_attr_names = staticmethod(get_blocks_attr_names)
|
||||
set_blocks_attr_names = staticmethod(set_blocks_attr_names)
|
||||
get_blocks_from_dit = staticmethod(get_blocks_from_dit)
|
||||
@@ -91,6 +91,11 @@ class ResolvedOffloadPlan:
|
||||
# still interpret topology themselves.
|
||||
modules: PipelineModules
|
||||
declaration: OffloadPlan | None = None
|
||||
# Ordinary layerwise offload turns a legacy selection it cannot serve into
|
||||
# a complete no-op, placement included. Distributed layerwise offload warns
|
||||
# and keeps preparing its other components instead, so this stays scoped to
|
||||
# the strategy that honors it.
|
||||
skip_reason: str | None = None
|
||||
|
||||
@property
|
||||
def components(self) -> tuple[ResolvedComponent, ...]:
|
||||
@@ -304,6 +309,11 @@ def resolve_offload_plan(pipeline: nn.Module, config: OffloadConfig) -> Resolved
|
||||
residents=tuple(residents),
|
||||
modules=modules,
|
||||
declaration=declaration,
|
||||
skip_reason=(
|
||||
"No DiT/transformer modules found for selected DiT layerwise offload"
|
||||
if config.strategy is OffloadStrategy.LAYER_WISE and dit_selected and not dits
|
||||
else None
|
||||
),
|
||||
)
|
||||
_validate_unique_ownership(resolved)
|
||||
return resolved
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
|
||||
from .config import DIT_COMPONENT
|
||||
from .plan_resolver import resolve_offload_plan
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -333,6 +332,7 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
encoders = [component.module for component in resolved.encoders]
|
||||
vaes = [component.module for component in resolved.vaes]
|
||||
residents = [component.module for component in resolved.residents]
|
||||
selected_dits = [component.module for component in resolved.dits if component.selected]
|
||||
selected_encoders = [component.module for component in resolved.encoders if component.selected]
|
||||
|
||||
all_modules = [*dits, *encoders, *vaes, *residents]
|
||||
@@ -360,7 +360,7 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
device=self.device,
|
||||
pin_memory=self.config.pin_cpu_memory,
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
offload_dit_modules=(dits if self.config.offloads(DIT_COMPONENT) else ()),
|
||||
offload_dit_modules=selected_dits,
|
||||
offload_encoder_modules=selected_encoders,
|
||||
)
|
||||
except BaseException:
|
||||
|
||||
Reference in new issue
Block a user