[diffusion][feature] Add component-selective offload policies (#5929)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
44d3ae100a
commit
ea3cf99894
74 files changed
+5121
-1584
No files matched your search
@@ -68,6 +68,7 @@ from vllm_omni.diffusion.models.schedulers.scheduling_flow_match_euler_discrete
|
||||
from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.config import OffloadStrategy, resolve_offload
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
@@ -934,6 +935,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.od_config = od_config
|
||||
resolved_offload = resolve_offload(od_config)
|
||||
if resolved_offload.public is not None and resolved_offload.strategy is OffloadStrategy.MODEL_LEVEL:
|
||||
raise ValueError(
|
||||
"Cosmos3 model offload uses reasoner/generator topology and does "
|
||||
"not support the dit/text_encoder component selector"
|
||||
)
|
||||
self.device = get_local_device()
|
||||
self.dtype = od_config.dtype
|
||||
|
||||
@@ -1086,6 +1093,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
device: torch.device,
|
||||
pin_memory: bool = True,
|
||||
use_hsdp: bool = False,
|
||||
offload_components: frozenset[str] | None = None,
|
||||
) -> None:
|
||||
"""Enable Cosmos3 component-level model offload.
|
||||
|
||||
@@ -1094,6 +1102,11 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
mutual-exclusion swaps. The VAE stays resident on GPU like the generic
|
||||
model-level offloader.
|
||||
"""
|
||||
if offload_components is not None:
|
||||
raise ValueError(
|
||||
"Cosmos3 model offload uses reasoner/generator topology and does not support "
|
||||
"the dit/text_encoder offload_components selector"
|
||||
)
|
||||
self.vae.to(device, non_blocking=True)
|
||||
if isinstance(self._sound_tokenizer, nn.Module):
|
||||
self._sound_tokenizer.to(device)
|
||||
|
||||
@@ -237,6 +237,13 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
|
||||
def _raise_unsupported_features(self) -> None:
|
||||
"""Raise an error for incompatible feature switches."""
|
||||
if self.od_config.diffusion_offload_config is not None:
|
||||
raise NotImplementedError(
|
||||
"diffusion_offload_config is not supported with the diffusers backend. "
|
||||
"Its component selectors cannot be represented by Diffusers' pipeline-wide "
|
||||
"CPU-offload hooks. Use the legacy enable_cpu_offload or "
|
||||
"enable_layerwise_offload option, or use a native pipeline."
|
||||
)
|
||||
pc = self.od_config.parallel_config
|
||||
if pc.tensor_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -1275,55 +1275,6 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
|
||||
def enable_omni_layerwise_offload(self, *, pin_memory: bool = True) -> None:
|
||||
"""Stream the TP-local Qwen vision/text blocks for low-HBM serving.
|
||||
|
||||
The encoder has its own TP process group, so these blocks must remain
|
||||
rank-local. Reusing the DiT AllGather group would concatenate
|
||||
different TP shards and corrupt the encoder weights.
|
||||
"""
|
||||
if not self.is_loaded or getattr(self, "_omni_layerwise_enabled", False):
|
||||
return
|
||||
|
||||
from vllm_omni.diffusion.offloader.layerwise_backend import apply_block_hook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
self._omni_layerwise_hooks = []
|
||||
self._omni_layerwise_block_groups = []
|
||||
copy_stream = current_omni_platform.Stream()
|
||||
for blocks in (self.vision.blocks, self.text_model.layers):
|
||||
if len(blocks) <= 1:
|
||||
continue
|
||||
last_hook = apply_block_hook(
|
||||
blocks[-1],
|
||||
blocks[0],
|
||||
self.device_target,
|
||||
copy_stream,
|
||||
pin_memory,
|
||||
)
|
||||
hooks = [last_hook]
|
||||
for index, block in enumerate(blocks[:-1]):
|
||||
hooks.append(
|
||||
apply_block_hook(
|
||||
block,
|
||||
blocks[index + 1],
|
||||
self.device_target,
|
||||
copy_stream,
|
||||
pin_memory,
|
||||
)
|
||||
)
|
||||
for index, hook in enumerate(hooks):
|
||||
hook._prev_hook = hooks[index - 1]
|
||||
self._omni_layerwise_hooks.extend(hooks)
|
||||
self._omni_layerwise_block_groups.append(blocks)
|
||||
|
||||
self._omni_layerwise_enabled = bool(self._omni_layerwise_hooks)
|
||||
logger.info(
|
||||
"MiniMax H3 encoder layerwise offload enabled on %d blocks across %d stacks",
|
||||
sum(len(blocks) for blocks in self._omni_layerwise_block_groups),
|
||||
len(self._omni_layerwise_block_groups),
|
||||
)
|
||||
|
||||
def _encode(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
|
||||
@@ -52,6 +52,13 @@ from vllm_omni.diffusion.offloader import (
|
||||
remove_sequential_offload,
|
||||
sequential_offload_component,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.config import (
|
||||
DIT_COMPONENT,
|
||||
TEXT_ENCODER_COMPONENT,
|
||||
OffloadStrategy,
|
||||
resolve_offload,
|
||||
should_offload_component,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import (
|
||||
DiffusionPipelineProfilerMixin,
|
||||
@@ -668,6 +675,7 @@ class MiniMaxH3Pipeline(
|
||||
_offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
|
||||
offload_submodules={"token_refiner": "blocks"},
|
||||
resident_dit_paths=frozenset({"transformer"}),
|
||||
encoder_component_types={"text_encoder": TEXT_ENCODER_COMPONENT},
|
||||
encoder_block_attrs={"text_encoder": ("vision.blocks", "text_model.layers")},
|
||||
on_demand_component_paths=frozenset({"text_encoder", "video_vae", "audio_vae"}),
|
||||
)
|
||||
@@ -703,10 +711,13 @@ class MiniMaxH3Pipeline(
|
||||
self._lora_sigma_schedules.pop(lora_request.lora_int_id, None)
|
||||
od_config = getattr(self, "od_config", None)
|
||||
offload_modes = []
|
||||
if getattr(od_config, "enable_cpu_offload", False):
|
||||
offload_modes.append("model-level CPU offload (--enable-cpu-offload)")
|
||||
if getattr(od_config, "enable_layerwise_offload", False):
|
||||
offload_modes.append("layerwise offload (--enable-layerwise-offload)")
|
||||
if od_config is not None:
|
||||
resolved_offload = resolve_offload(od_config)
|
||||
if resolved_offload.offloads(DIT_COMPONENT):
|
||||
if resolved_offload.strategy is OffloadStrategy.MODEL_LEVEL:
|
||||
offload_modes.append("model-level CPU offload")
|
||||
elif resolved_offload.strategy is OffloadStrategy.LAYER_WISE:
|
||||
offload_modes.append("layerwise offload")
|
||||
loaded = load_minimax_h3_turbo_lora(
|
||||
partition=self.partition,
|
||||
lora_request=lora_request,
|
||||
@@ -1030,10 +1041,13 @@ class MiniMaxH3Pipeline(
|
||||
self.text_encoder_group = None
|
||||
self.text_encoder = None
|
||||
self._encoder_modules = []
|
||||
stage_components = bool(
|
||||
legacy_manual_components = getattr(od_config, "diffusion_offload_config", None) is None and bool(
|
||||
od_config.enable_layerwise_offload or getattr(od_config, "enable_distributed_layerwise_offload", False)
|
||||
)
|
||||
component_load_device = torch.device("cpu") if stage_components else self.device
|
||||
# Preserve the legacy MiniMax-H3 low-residency path. The compact API
|
||||
# deliberately limits explicit component selection to dit/text_encoder,
|
||||
# so VAEs stay resident for new configurations.
|
||||
component_load_device = torch.device("cpu") if legacy_manual_components else self.device
|
||||
self.video_vae = MiniMaxH3VideoVAE(
|
||||
os.path.join(model_path, "video_vae"),
|
||||
device=self.device,
|
||||
@@ -1048,14 +1062,19 @@ class MiniMaxH3Pipeline(
|
||||
self.vae = self.video_vae
|
||||
|
||||
self._dlo_component_cache = None
|
||||
if getattr(od_config, "enable_distributed_layerwise_offload", False):
|
||||
offloads_text_encoder = should_offload_component(od_config, TEXT_ENCODER_COMPONENT)
|
||||
needs_component_cache = legacy_manual_components or offloads_text_encoder
|
||||
if getattr(od_config, "enable_distributed_layerwise_offload", False) and needs_component_cache:
|
||||
self._dlo_component_cache = BoundedAllocatorCache(self.device)
|
||||
_register_dlo_component_cache(
|
||||
self._dlo_component_cache,
|
||||
self.text_encoder,
|
||||
self.video_vae,
|
||||
self.audio_vae,
|
||||
)
|
||||
if legacy_manual_components:
|
||||
_register_dlo_component_cache(
|
||||
self._dlo_component_cache,
|
||||
self.text_encoder,
|
||||
self.video_vae,
|
||||
self.audio_vae,
|
||||
)
|
||||
elif offloads_text_encoder:
|
||||
_register_dlo_component_cache(self._dlo_component_cache, self.text_encoder)
|
||||
|
||||
self._quality_policy = MiniMaxH3QualityPolicy(od_config)
|
||||
self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
|
||||
@@ -1519,25 +1538,25 @@ class MiniMaxH3Pipeline(
|
||||
# swaps the resident DiT and encoder.
|
||||
return self.text_encoder(input_ids, **vision_kwargs)
|
||||
|
||||
if self.od_config.enable_layerwise_offload or getattr(
|
||||
self.od_config, "enable_distributed_layerwise_offload", False
|
||||
):
|
||||
# Layerwise DiT offload already provides the low-residency encoder
|
||||
# phase used by the checkpoint reference.
|
||||
if self._uses_manual_component_offload(self.text_encoder):
|
||||
with self._component_on_device(self.text_encoder):
|
||||
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
|
||||
|
||||
# Keep both Qwen and DiT resident across requests. Moving either model
|
||||
# here makes encoder latency include a tens-of-gigabytes PCIe transfer,
|
||||
# which defeats the no-offload contract.
|
||||
# Keep Qwen resident when it is not selected for layerwise offload.
|
||||
self.text_encoder.load_to_device()
|
||||
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
|
||||
|
||||
def _uses_manual_component_offload(self) -> bool:
|
||||
def _uses_manual_component_offload(self, component: nn.Module) -> bool:
|
||||
od_config = getattr(self, "od_config", None)
|
||||
return bool(
|
||||
getattr(od_config, "enable_layerwise_offload", False)
|
||||
or getattr(od_config, "enable_distributed_layerwise_offload", False)
|
||||
if od_config is None:
|
||||
return False
|
||||
if getattr(od_config, "diffusion_offload_config", None) is None:
|
||||
return bool(
|
||||
getattr(od_config, "enable_layerwise_offload", False)
|
||||
or getattr(od_config, "enable_distributed_layerwise_offload", False)
|
||||
)
|
||||
return component is getattr(self, "text_encoder", None) and should_offload_component(
|
||||
od_config, TEXT_ENCODER_COMPONENT
|
||||
)
|
||||
|
||||
def enable_omni_model_cpu_offload(
|
||||
@@ -1546,6 +1565,7 @@ class MiniMaxH3Pipeline(
|
||||
device: torch.device,
|
||||
pin_memory: bool,
|
||||
use_hsdp: bool,
|
||||
offload_components: frozenset[str] | None = None,
|
||||
) -> None:
|
||||
if getattr(self, "_model_cpu_offload_modules", None):
|
||||
return
|
||||
@@ -1554,19 +1574,32 @@ class MiniMaxH3Pipeline(
|
||||
dits = components.dits
|
||||
stages = [*components.encoders, *components.vaes]
|
||||
modules = [*dits, *stages]
|
||||
selection_options: dict[str, Any] = {}
|
||||
if offload_components is not None:
|
||||
if DIT_COMPONENT in offload_components and not dits:
|
||||
raise ValueError("MiniMax-H3 has no loaded DiT for selected module offload")
|
||||
if TEXT_ENCODER_COMPONENT in offload_components and not components.encoders:
|
||||
raise ValueError("MiniMax-H3 has no loaded text encoder for selected module offload")
|
||||
selection_options = {
|
||||
"offload_dit_modules": dits if DIT_COMPONENT in offload_components else (),
|
||||
"offload_encoder_modules": (
|
||||
components.encoders if TEXT_ENCODER_COMPONENT in offload_components else ()
|
||||
),
|
||||
}
|
||||
apply_sequential_offload(
|
||||
dit_modules=dits,
|
||||
encoder_modules=stages,
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
offload_initial_dits=True,
|
||||
offload_initial_dits=offload_components is None or DIT_COMPONENT in offload_components,
|
||||
**selection_options,
|
||||
)
|
||||
|
||||
self._model_cpu_offload_modules = modules
|
||||
logger.info(
|
||||
"MiniMax-H3 model-level CPU offload enabled for %d DiT(s), text encoder, video VAE, and audio VAE",
|
||||
len(dits),
|
||||
"MiniMax-H3 model-level CPU offload enabled for selected components: %s",
|
||||
sorted(offload_components) if offload_components is not None else "legacy full topology",
|
||||
)
|
||||
|
||||
def disable_omni_model_cpu_offload(self) -> None:
|
||||
@@ -1582,7 +1615,7 @@ class MiniMaxH3Pipeline(
|
||||
with sequential_offload_component(component):
|
||||
yield
|
||||
return
|
||||
staged = self._uses_manual_component_offload()
|
||||
staged = self._uses_manual_component_offload(component)
|
||||
try:
|
||||
if staged:
|
||||
component.load_to_device()
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_z
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler import WanEulerScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanSelfAttention, WanTransformer3DModel
|
||||
from vllm_omni.diffusion.offloader import OffloadPlan
|
||||
from vllm_omni.diffusion.postprocess import interpolate_video_tensor
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
@@ -308,6 +309,13 @@ def get_wan22_pre_process_func(
|
||||
return pre_process_func
|
||||
|
||||
|
||||
_WAN_TEXT_ENCODER_OFFLOAD_PLAN = OffloadPlan(
|
||||
encoder_component_types={"text_encoder": "text_encoder"},
|
||||
encoder_block_attrs={"text_encoder": ("encoder.block",)},
|
||||
encoder_dlo_weight_replication=frozenset({"text_encoder"}),
|
||||
)
|
||||
|
||||
|
||||
class Wan22Pipeline(
|
||||
nn.Module,
|
||||
PipelineParallelMixin,
|
||||
@@ -322,6 +330,7 @@ class Wan22Pipeline(
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_offload_plan = _WAN_TEXT_ENCODER_OFFLOAD_PLAN
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComp
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
|
||||
from vllm_omni.diffusion.models.utils import _load_json
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
_WAN_TEXT_ENCODER_OFFLOAD_PLAN,
|
||||
build_wan_scheduler,
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
@@ -192,6 +193,7 @@ class Wan22I2VPipeline(
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["text_encoder", "image_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_offload_plan = _WAN_TEXT_ENCODER_OFFLOAD_PLAN
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -41,6 +41,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_s2v_transformer import (
|
||||
create_s2v_transformer_from_config,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.config import DIT_COMPONENT, selected_offload_components
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request
|
||||
@@ -1090,6 +1091,11 @@ class Wan22S2VPipeline(
|
||||
# Main forward
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _should_release_dit_before_decode(self) -> bool:
|
||||
if getattr(self.od_config.parallel_config, "use_hsdp", False):
|
||||
return False
|
||||
return self.od_config.enable_cpu_offload and DIT_COMPONENT in selected_offload_components(self.od_config)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: DiffusionRequestBatch,
|
||||
@@ -1401,7 +1407,7 @@ class Wan22S2VPipeline(
|
||||
)
|
||||
|
||||
# ---- Decode this clip ----
|
||||
if self.od_config.enable_cpu_offload and not getattr(self.od_config.parallel_config, "use_hsdp", False):
|
||||
if self._should_release_dit_before_decode():
|
||||
self.transformer.to("cpu")
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ determined by which inputs are provided (no explicit mode flag):
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import replace
|
||||
from copy import copy
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
@@ -198,7 +198,10 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput):
|
||||
):
|
||||
# VACE defaults to flow_shift=3.0 for 480p (base WAN T2V uses 5.0 for 720p)
|
||||
if od_config.flow_shift is None:
|
||||
od_config = replace(od_config, flow_shift=3.0)
|
||||
# Preserve runtime-only config provenance (including compact
|
||||
# offload aliases) instead of rerunning dataclass __post_init__.
|
||||
od_config = copy(od_config)
|
||||
od_config.flow_shift = 3.0
|
||||
|
||||
super().__init__(od_config=od_config, prefix=prefix)
|
||||
|
||||
|
||||
Reference in new issue
Block a user