[Performance] Add per-component DLO offload policy
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
780e8d40aa
commit
7db3a0cd5e
15 files changed
+324
-21
No files matched your search
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
@@ -52,6 +53,9 @@ class OffloadConfig:
|
||||
# blocks from the loader-selected host backing with H2D only.
|
||||
dlo_use_allgather: bool = True
|
||||
dlo_resident_layers: int = 0 # leading DiT layers kept on device
|
||||
# Per auxiliary component: True permits model-declared DLO offload;
|
||||
# False keeps the whole component resident. "default" is the fallback.
|
||||
dlo_offload_components: dict[str, bool] = field(default_factory=dict)
|
||||
# Optional per-worker ceiling for registering an HWR mmap. Zero means no
|
||||
# additional ceiling; pin_cpu_memory controls whether registration is tried.
|
||||
dlo_host_registration_limit_gib: float = 0.0
|
||||
@@ -125,6 +129,25 @@ class OffloadConfig:
|
||||
# requirements (concurrent requests, dummy run skip).
|
||||
dlo_use_allgather = getattr(od_config, "dlo_use_allgather", True)
|
||||
dlo_resident_layers = int(getattr(od_config, "dlo_resident_layers", 0))
|
||||
raw_component_policy = getattr(od_config, "dlo_offload_components", {})
|
||||
if raw_component_policy is None:
|
||||
raw_component_policy = {}
|
||||
if not isinstance(raw_component_policy, Mapping):
|
||||
raise TypeError(
|
||||
"dlo_offload_components must be a mapping of component names to booleans, "
|
||||
f"got {type(raw_component_policy).__name__}"
|
||||
)
|
||||
dlo_offload_components: dict[str, bool] = {}
|
||||
for component, enabled in raw_component_policy.items():
|
||||
if not isinstance(component, str) or not component:
|
||||
raise TypeError("dlo_offload_components keys must be non-empty strings")
|
||||
if not isinstance(enabled, bool):
|
||||
raise TypeError(
|
||||
f"dlo_offload_components[{component!r}] must be a boolean, got {type(enabled).__name__}"
|
||||
)
|
||||
dlo_offload_components[component] = enabled
|
||||
if dlo_offload_components and not enable_distributed_layerwise_offload:
|
||||
raise ValueError("dlo_offload_components requires distributed layerwise offload")
|
||||
dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
||||
limit_gib=getattr(od_config, "dlo_host_registration_limit_gib", 0.0),
|
||||
enable_dlo=enable_distributed_layerwise_offload,
|
||||
@@ -166,6 +189,7 @@ class OffloadConfig:
|
||||
dp_size=dp_size,
|
||||
dlo_use_allgather=dlo_use_allgather,
|
||||
dlo_resident_layers=dlo_resident_layers,
|
||||
dlo_offload_components=dlo_offload_components,
|
||||
dlo_host_registration_limit_gib=dlo_host_registration_limit_gib,
|
||||
)
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ from .host_registration import (
|
||||
HostRegistrationError,
|
||||
register_host_mappings,
|
||||
)
|
||||
from .module_collector import ModuleDiscovery
|
||||
from .module_collector import ModuleDiscovery, PipelineModules
|
||||
from .offload_plan import (
|
||||
OffloadPlan,
|
||||
get_offload_plan,
|
||||
@@ -1400,6 +1400,39 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
|
||||
module.to(self.device)
|
||||
logger.info("Moved %s (%s) to GPU (resident)", label, module.__class__.__name__)
|
||||
|
||||
def _component_offload_enabled(self, name: str) -> bool:
|
||||
policy = self.config.dlo_offload_components
|
||||
return policy.get(name, policy.get("default", True))
|
||||
|
||||
def _validate_component_offload_policy(self, modules: PipelineModules) -> None:
|
||||
known_components = {"default", *modules.encoder_names, *modules.vae_names}
|
||||
unknown = sorted(self.config.dlo_offload_components.keys() - known_components)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"dlo_offload_components contains unknown auxiliary components: {unknown}. DLO always owns DiT offload."
|
||||
)
|
||||
|
||||
def _prepare_auxiliary_components(self, modules: PipelineModules, plan: OffloadPlan | None) -> None:
|
||||
for encoder, name in zip(modules.encoders, modules.encoder_names, strict=True):
|
||||
offload = self._component_offload_enabled(name)
|
||||
encoder._omni_dlo_offload_enabled = offload
|
||||
if offload:
|
||||
self._try_layerwise_offload_encoder(encoder, name, plan)
|
||||
self._register_on_demand_hook(
|
||||
encoder,
|
||||
name,
|
||||
stage_on_demand=(offload and plan is not None and name in plan.on_demand_component_paths),
|
||||
)
|
||||
|
||||
for vae, name in zip(modules.vaes, modules.vae_names, strict=True):
|
||||
offload = self._component_offload_enabled(name)
|
||||
vae._omni_dlo_offload_enabled = offload
|
||||
self._register_on_demand_hook(
|
||||
vae,
|
||||
name,
|
||||
stage_on_demand=(offload and plan is not None and name in plan.on_demand_component_paths),
|
||||
)
|
||||
|
||||
def _try_layerwise_offload_encoder(self, module: nn.Module, name: str, plan: OffloadPlan | None) -> bool:
|
||||
"""Stream plan-declared encoder blocks on each rank without AllGather."""
|
||||
if plan is None or name not in plan.encoder_block_attrs:
|
||||
@@ -1668,6 +1701,7 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
|
||||
# Retrieve optional declarative OffloadPlan from the pipeline.
|
||||
# When present, replaces heuristic block discovery.
|
||||
plan = get_offload_plan(pipeline)
|
||||
self._validate_component_offload_policy(modules)
|
||||
|
||||
if self.config.dlo_resident_layers and (plan is None or not plan.resident_dit_paths):
|
||||
logger.warning(
|
||||
@@ -1728,21 +1762,10 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
|
||||
)
|
||||
logger.info("DLO is using host tensors materialized by the ordinary loader")
|
||||
|
||||
# Keep VAE/encoders on CPU; move to GPU on-demand via hooks.
|
||||
# This saves ~4.3 GB HBM per card (VAE 1.3 + encoder 1.1 + sound 1.9)
|
||||
# during the DiT forward pass. They are only needed briefly for
|
||||
# text-encoding (before DiT) and VAE-decode (after DiT).
|
||||
for enc, enc_name in zip(modules.encoders, modules.encoder_names):
|
||||
self._try_layerwise_offload_encoder(enc, enc_name, plan)
|
||||
self._register_on_demand_hook(
|
||||
enc, "encoder", stage_on_demand=plan is not None and enc_name in plan.on_demand_component_paths
|
||||
)
|
||||
for vae, vae_name in zip(modules.vaes, modules.vae_names):
|
||||
self._register_on_demand_hook(
|
||||
vae,
|
||||
"vae",
|
||||
stage_on_demand=(plan is not None and vae_name in plan.on_demand_component_paths),
|
||||
)
|
||||
# Apply the user policy to discovered auxiliary components. The default
|
||||
# keeps the model-declared staged lifecycle; False leaves a component
|
||||
# resident for repeated requests.
|
||||
self._prepare_auxiliary_components(modules, plan)
|
||||
|
||||
# Move resident modules to GPU (small modules needed every forward)
|
||||
for name, module in zip(modules.resident_names, modules.resident_modules):
|
||||
|
||||
Reference in new issue
Block a user