[diffusion][feature] Add component-selective offload policies (#5929)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
44d3ae100a
commit
ea3cf99894
74 files changed
+5121
-1584
No files matched your search
@@ -1,10 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Declarative OffloadPlan for distributed layerwise offload.
|
||||
"""Declarative model capabilities for component layerwise offload.
|
||||
|
||||
Models declare this as a class attribute ``_offload_plan`` on the
|
||||
pipeline class. When present, the offloader uses it instead of
|
||||
heuristic block discovery.
|
||||
Models declare an ``_offload_plan`` class attribute on the pipeline. Both the
|
||||
ordinary and distributed layerwise backends consume the same plan instead of
|
||||
embedding model-specific block paths in backend code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,12 +16,11 @@ from torch import nn
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OffloadPlan:
|
||||
"""Optional declarative metadata for distributed layerwise offload.
|
||||
"""Optional declarative metadata for component layerwise offload.
|
||||
|
||||
Models declare this as a class attribute ``_offload_plan`` on the
|
||||
pipeline class. When present, the offloader uses it instead of
|
||||
heuristic block discovery, making it easier to support new models
|
||||
without offload-specific logic in constructor or forward code.
|
||||
pipeline class. When present, both layerwise backends use it instead of
|
||||
model-specific backend branches, making new integrations data-driven.
|
||||
|
||||
If not declared, the offloader falls back to:
|
||||
1. ``_layerwise_offload_blocks_attrs`` on each DiT module class.
|
||||
@@ -39,9 +38,14 @@ class OffloadPlan:
|
||||
device when ``dlo_resident_layers`` is nonzero. Keeping this
|
||||
model-declared avoids applying a consumer-GPU tuning knob to
|
||||
auxiliary or dual DiTs unintentionally.
|
||||
encoder_block_attrs: Maps encoder paths to rank-local block-list paths.
|
||||
These blocks are streamed with ordinary layerwise hooks, never
|
||||
with the DiT AllGather group.
|
||||
encoder_component_types: Maps encoder paths to public selector types
|
||||
(currently text_encoder). This declaration is used before the
|
||||
compatibility name heuristic.
|
||||
encoder_block_attrs: Maps encoder paths to streamable block-list paths.
|
||||
encoder_dlo_weight_replication: Encoder paths whose loader-produced
|
||||
block tensors are identical across the DiT DLO group. Only these
|
||||
encoders may use multi-rank AllGather transfer; this must not be
|
||||
declared for encoder-TP shards.
|
||||
"""
|
||||
|
||||
on_demand_component_paths: frozenset[str] = field(default_factory=frozenset)
|
||||
@@ -49,7 +53,9 @@ class OffloadPlan:
|
||||
block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
offload_submodules: dict[str, str] = field(default_factory=dict)
|
||||
resident_dit_paths: frozenset[str] = field(default_factory=frozenset)
|
||||
encoder_component_types: dict[str, str] = field(default_factory=dict)
|
||||
encoder_block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
encoder_dlo_weight_replication: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
|
||||
def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None:
|
||||
|
||||
Reference in new issue
Block a user