[diffusion][feature] Add component-selective offload policies (#5929)

Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-09-06 05:40:09 +08:00
1 parent 44d3ae100a
commit ea3cf99894
74 files changed
+5121 -1584

No files matched your search

@@ -68,6 +68,7 @@ from vllm_omni.diffusion.models.schedulers.scheduling_flow_match_euler_discrete
from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import (
FlowUniPCMultistepScheduler,
)
from vllm_omni.diffusion.offloader.config import OffloadStrategy, resolve_offload
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
@@ -934,6 +935,12 @@ class Cosmos3OmniDiffusersPipeline(
) -> None:
super().__init__()
self.od_config = od_config
resolved_offload = resolve_offload(od_config)
if resolved_offload.public is not None and resolved_offload.strategy is OffloadStrategy.MODEL_LEVEL:
raise ValueError(
"Cosmos3 model offload uses reasoner/generator topology and does "
"not support the dit/text_encoder component selector"
)
self.device = get_local_device()
self.dtype = od_config.dtype
@@ -1086,6 +1093,7 @@ class Cosmos3OmniDiffusersPipeline(
device: torch.device,
pin_memory: bool = True,
use_hsdp: bool = False,
offload_components: frozenset[str] | None = None,
) -> None:
"""Enable Cosmos3 component-level model offload.
@@ -1094,6 +1102,11 @@ class Cosmos3OmniDiffusersPipeline(
mutual-exclusion swaps. The VAE stays resident on GPU like the generic
model-level offloader.
"""
if offload_components is not None:
raise ValueError(
"Cosmos3 model offload uses reasoner/generator topology and does not support "
"the dit/text_encoder offload_components selector"
)
self.vae.to(device, non_blocking=True)
if isinstance(self._sound_tokenizer, nn.Module):
self._sound_tokenizer.to(device)
@@ -237,6 +237,13 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
def _raise_unsupported_features(self) -> None:
"""Raise an error for incompatible feature switches."""
if self.od_config.diffusion_offload_config is not None:
raise NotImplementedError(
"diffusion_offload_config is not supported with the diffusers backend. "
"Its component selectors cannot be represented by Diffusers' pipeline-wide "
"CPU-offload hooks. Use the legacy enable_cpu_offload or "
"enable_layerwise_offload option, or use a native pipeline."
)
pc = self.od_config.parallel_config
if pc.tensor_parallel_size > 1:
raise NotImplementedError(
@@ -1275,55 +1275,6 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
else:
cache.release_if_needed()
def enable_omni_layerwise_offload(self, *, pin_memory: bool = True) -> None:
"""Stream the TP-local Qwen vision/text blocks for low-HBM serving.
The encoder has its own TP process group, so these blocks must remain
rank-local. Reusing the DiT AllGather group would concatenate
different TP shards and corrupt the encoder weights.
"""
if not self.is_loaded or getattr(self, "_omni_layerwise_enabled", False):
return
from vllm_omni.diffusion.offloader.layerwise_backend import apply_block_hook
from vllm_omni.platforms import current_omni_platform
self._omni_layerwise_hooks = []
self._omni_layerwise_block_groups = []
copy_stream = current_omni_platform.Stream()
for blocks in (self.vision.blocks, self.text_model.layers):
if len(blocks) <= 1:
continue
last_hook = apply_block_hook(
blocks[-1],
blocks[0],
self.device_target,
copy_stream,
pin_memory,
)
hooks = [last_hook]
for index, block in enumerate(blocks[:-1]):
hooks.append(
apply_block_hook(
block,
blocks[index + 1],
self.device_target,
copy_stream,
pin_memory,
)
)
for index, hook in enumerate(hooks):
hook._prev_hook = hooks[index - 1]
self._omni_layerwise_hooks.extend(hooks)
self._omni_layerwise_block_groups.append(blocks)
self._omni_layerwise_enabled = bool(self._omni_layerwise_hooks)
logger.info(
"MiniMax H3 encoder layerwise offload enabled on %d blocks across %d stacks",
sum(len(blocks) for blocks in self._omni_layerwise_block_groups),
len(self._omni_layerwise_block_groups),
)
def _encode(
self,
input_ids: torch.Tensor,
@@ -52,6 +52,13 @@ from vllm_omni.diffusion.offloader import (
remove_sequential_offload,
sequential_offload_component,
)
from vllm_omni.diffusion.offloader.config import (
DIT_COMPONENT,
TEXT_ENCODER_COMPONENT,
OffloadStrategy,
resolve_offload,
should_offload_component,
)
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import (
DiffusionPipelineProfilerMixin,
@@ -668,6 +675,7 @@ class MiniMaxH3Pipeline(
_offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
offload_submodules={"token_refiner": "blocks"},
resident_dit_paths=frozenset({"transformer"}),
encoder_component_types={"text_encoder": TEXT_ENCODER_COMPONENT},
encoder_block_attrs={"text_encoder": ("vision.blocks", "text_model.layers")},
on_demand_component_paths=frozenset({"text_encoder", "video_vae", "audio_vae"}),
)
@@ -703,10 +711,13 @@ class MiniMaxH3Pipeline(
self._lora_sigma_schedules.pop(lora_request.lora_int_id, None)
od_config = getattr(self, "od_config", None)
offload_modes = []
if getattr(od_config, "enable_cpu_offload", False):
offload_modes.append("model-level CPU offload (--enable-cpu-offload)")
if getattr(od_config, "enable_layerwise_offload", False):
offload_modes.append("layerwise offload (--enable-layerwise-offload)")
if od_config is not None:
resolved_offload = resolve_offload(od_config)
if resolved_offload.offloads(DIT_COMPONENT):
if resolved_offload.strategy is OffloadStrategy.MODEL_LEVEL:
offload_modes.append("model-level CPU offload")
elif resolved_offload.strategy is OffloadStrategy.LAYER_WISE:
offload_modes.append("layerwise offload")
loaded = load_minimax_h3_turbo_lora(
partition=self.partition,
lora_request=lora_request,
@@ -1030,10 +1041,13 @@ class MiniMaxH3Pipeline(
self.text_encoder_group = None
self.text_encoder = None
self._encoder_modules = []
stage_components = bool(
legacy_manual_components = getattr(od_config, "diffusion_offload_config", None) is None and bool(
od_config.enable_layerwise_offload or getattr(od_config, "enable_distributed_layerwise_offload", False)
)
component_load_device = torch.device("cpu") if stage_components else self.device
# Preserve the legacy MiniMax-H3 low-residency path. The compact API
# deliberately limits explicit component selection to dit/text_encoder,
# so VAEs stay resident for new configurations.
component_load_device = torch.device("cpu") if legacy_manual_components else self.device
self.video_vae = MiniMaxH3VideoVAE(
os.path.join(model_path, "video_vae"),
device=self.device,
@@ -1048,14 +1062,19 @@ class MiniMaxH3Pipeline(
self.vae = self.video_vae
self._dlo_component_cache = None
if getattr(od_config, "enable_distributed_layerwise_offload", False):
offloads_text_encoder = should_offload_component(od_config, TEXT_ENCODER_COMPONENT)
needs_component_cache = legacy_manual_components or offloads_text_encoder
if getattr(od_config, "enable_distributed_layerwise_offload", False) and needs_component_cache:
self._dlo_component_cache = BoundedAllocatorCache(self.device)
_register_dlo_component_cache(
self._dlo_component_cache,
self.text_encoder,
self.video_vae,
self.audio_vae,
)
if legacy_manual_components:
_register_dlo_component_cache(
self._dlo_component_cache,
self.text_encoder,
self.video_vae,
self.audio_vae,
)
elif offloads_text_encoder:
_register_dlo_component_cache(self._dlo_component_cache, self.text_encoder)
self._quality_policy = MiniMaxH3QualityPolicy(od_config)
self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
@@ -1519,25 +1538,25 @@ class MiniMaxH3Pipeline(
# swaps the resident DiT and encoder.
return self.text_encoder(input_ids, **vision_kwargs)
if self.od_config.enable_layerwise_offload or getattr(
self.od_config, "enable_distributed_layerwise_offload", False
):
# Layerwise DiT offload already provides the low-residency encoder
# phase used by the checkpoint reference.
if self._uses_manual_component_offload(self.text_encoder):
with self._component_on_device(self.text_encoder):
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
# Keep both Qwen and DiT resident across requests. Moving either model
# here makes encoder latency include a tens-of-gigabytes PCIe transfer,
# which defeats the no-offload contract.
# Keep Qwen resident when it is not selected for layerwise offload.
self.text_encoder.load_to_device()
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
def _uses_manual_component_offload(self) -> bool:
def _uses_manual_component_offload(self, component: nn.Module) -> bool:
od_config = getattr(self, "od_config", None)
return bool(
getattr(od_config, "enable_layerwise_offload", False)
or getattr(od_config, "enable_distributed_layerwise_offload", False)
if od_config is None:
return False
if getattr(od_config, "diffusion_offload_config", None) is None:
return bool(
getattr(od_config, "enable_layerwise_offload", False)
or getattr(od_config, "enable_distributed_layerwise_offload", False)
)
return component is getattr(self, "text_encoder", None) and should_offload_component(
od_config, TEXT_ENCODER_COMPONENT
)
def enable_omni_model_cpu_offload(
@@ -1546,6 +1565,7 @@ class MiniMaxH3Pipeline(
device: torch.device,
pin_memory: bool,
use_hsdp: bool,
offload_components: frozenset[str] | None = None,
) -> None:
if getattr(self, "_model_cpu_offload_modules", None):
return
@@ -1554,19 +1574,32 @@ class MiniMaxH3Pipeline(
dits = components.dits
stages = [*components.encoders, *components.vaes]
modules = [*dits, *stages]
selection_options: dict[str, Any] = {}
if offload_components is not None:
if DIT_COMPONENT in offload_components and not dits:
raise ValueError("MiniMax-H3 has no loaded DiT for selected module offload")
if TEXT_ENCODER_COMPONENT in offload_components and not components.encoders:
raise ValueError("MiniMax-H3 has no loaded text encoder for selected module offload")
selection_options = {
"offload_dit_modules": dits if DIT_COMPONENT in offload_components else (),
"offload_encoder_modules": (
components.encoders if TEXT_ENCODER_COMPONENT in offload_components else ()
),
}
apply_sequential_offload(
dit_modules=dits,
encoder_modules=stages,
device=device,
pin_memory=pin_memory,
use_hsdp=use_hsdp,
offload_initial_dits=True,
offload_initial_dits=offload_components is None or DIT_COMPONENT in offload_components,
**selection_options,
)
self._model_cpu_offload_modules = modules
logger.info(
"MiniMax-H3 model-level CPU offload enabled for %d DiT(s), text encoder, video VAE, and audio VAE",
len(dits),
"MiniMax-H3 model-level CPU offload enabled for selected components: %s",
sorted(offload_components) if offload_components is not None else "legacy full topology",
)
def disable_omni_model_cpu_offload(self) -> None:
@@ -1582,7 +1615,7 @@ class MiniMaxH3Pipeline(
with sequential_offload_component(component):
yield
return
staged = self._uses_manual_component_offload()
staged = self._uses_manual_component_offload(component)
try:
if staged:
component.load_to_device()
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_z
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler import WanEulerScheduler
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanSelfAttention, WanTransformer3DModel
from vllm_omni.diffusion.offloader import OffloadPlan
from vllm_omni.diffusion.postprocess import interpolate_video_tensor
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -308,6 +309,13 @@ def get_wan22_pre_process_func(
return pre_process_func
_WAN_TEXT_ENCODER_OFFLOAD_PLAN = OffloadPlan(
encoder_component_types={"text_encoder": "text_encoder"},
encoder_block_attrs={"text_encoder": ("encoder.block",)},
encoder_dlo_weight_replication=frozenset({"text_encoder"}),
)
class Wan22Pipeline(
nn.Module,
PipelineParallelMixin,
@@ -322,6 +330,7 @@ class Wan22Pipeline(
_dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"]
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
_vae_modules: ClassVar[list[str]] = ["vae"]
_offload_plan = _WAN_TEXT_ENCODER_OFFLOAD_PLAN
def __init__(
self,
@@ -32,6 +32,7 @@ from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComp
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.models.utils import _load_json
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
_WAN_TEXT_ENCODER_OFFLOAD_PLAN,
build_wan_scheduler,
create_transformer_from_config,
load_transformer_config,
@@ -192,6 +193,7 @@ class Wan22I2VPipeline(
_dit_modules: ClassVar[list[str]] = ["transformer", "transformer_2"]
_encoder_modules: ClassVar[list[str]] = ["text_encoder", "image_encoder"]
_vae_modules: ClassVar[list[str]] = ["vae"]
_offload_plan = _WAN_TEXT_ENCODER_OFFLOAD_PLAN
def __init__(
self,
@@ -41,6 +41,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
from vllm_omni.diffusion.models.wan2_2.wan2_2_s2v_transformer import (
create_s2v_transformer_from_config,
)
from vllm_omni.diffusion.offloader.config import DIT_COMPONENT, selected_offload_components
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request
@@ -1090,6 +1091,11 @@ class Wan22S2VPipeline(
# Main forward
# ------------------------------------------------------------------
def _should_release_dit_before_decode(self) -> bool:
if getattr(self.od_config.parallel_config, "use_hsdp", False):
return False
return self.od_config.enable_cpu_offload and DIT_COMPONENT in selected_offload_components(self.od_config)
def forward(
self,
req: DiffusionRequestBatch,
@@ -1401,7 +1407,7 @@ class Wan22S2VPipeline(
)
# ---- Decode this clip ----
if self.od_config.enable_cpu_offload and not getattr(self.od_config.parallel_config, "use_hsdp", False):
if self._should_release_dit_before_decode():
self.transformer.to("cpu")
current_omni_platform.empty_cache()
@@ -16,7 +16,7 @@ determined by which inputs are provided (no explicit mode flag):
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import replace
from copy import copy
import PIL.Image
import torch
@@ -198,7 +198,10 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput):
):
# VACE defaults to flow_shift=3.0 for 480p (base WAN T2V uses 5.0 for 720p)
if od_config.flow_shift is None:
od_config = replace(od_config, flow_shift=3.0)
# Preserve runtime-only config provenance (including compact
# offload aliases) instead of rerunning dataclass __post_init__.
od_config = copy(od_config)
od_config.flow_shift = 3.0
super().__init__(od_config=od_config, prefix=prefix)