[Bugfix][Diffusion] Fix MiniMax-H3 model-level CPU offload residency (#6072)

Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>
This commit is contained in:
R0CKSTAR authored and GitHub committed 2026-08-21 10:30:53 +00:00
1 parent 4f7c08c0eb
commit 260db8bab5
7 files changed
+553 -102

No files matched your search

@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
import torch
from torch import nn
from torch.distributed._tensor import DTensor # type: ignore[attr-defined]
@@ -9,7 +12,7 @@ from vllm.logger import init_logger
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .module_collector import ModuleDiscovery
logger = init_logger(__name__)
@@ -123,6 +126,7 @@ def apply_sequential_offload(
device: torch.device,
pin_memory: bool = True,
use_hsdp: bool = False,
offload_initial_dits: bool = False,
) -> None:
"""Apply sequential offloading hooks to DiT and encoder modules.
@@ -136,6 +140,7 @@ def apply_sequential_offload(
device: Target GPU device for loading
pin_memory: Whether to pin CPU memory for faster transfers
use_hsdp: Whether HSDP is enabled (affects non_blocking behavior)
offload_initial_dits: Whether to begin with all DiT modules on CPU.
Example:
>>> apply_sequential_offload(
@@ -170,6 +175,14 @@ def apply_sequential_offload(
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", enc.__class__.__name__)
if offload_initial_dits:
try:
for dit_mod in dit_modules:
_get_sequential_offload_hook(dit_mod)._to_cpu(dit_mod)
except Exception:
remove_sequential_offload([*dit_modules, *encoder_modules])
raise
def remove_sequential_offload(modules: list[nn.Module]) -> None:
"""Remove sequential offloading hooks from modules.
@@ -188,6 +201,31 @@ def remove_sequential_offload(modules: list[nn.Module]) -> None:
logger.debug("Removed offload hook from %s", module.__class__.__name__)
def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
hook = registry.get_hook(SequentialOffloadHook._HOOK_NAME) if registry is not None else None
if not isinstance(hook, SequentialOffloadHook):
raise RuntimeError(f"{module.__class__.__name__} has no sequential offload hook")
return hook
@contextmanager
def sequential_offload_component(module: nn.Module) -> Iterator[None]:
"""Activate and release a hooked component called outside ``forward``."""
hook = _get_sequential_offload_hook(module)
try:
hook.pre_forward(module)
yield
except BaseException:
try:
hook._to_cpu(module)
except Exception:
logger.exception("Failed to release %s after component failure", module.__class__.__name__)
raise
else:
hook._to_cpu(module)
class ModelLevelOffloadBackend(OffloadBackend):
"""Model-level (sequential) offloading backend.
@@ -197,23 +235,17 @@ class ModelLevelOffloadBackend(OffloadBackend):
def __init__(self, config: OffloadConfig, device: torch.device):
super().__init__(config, device)
self._offload_modules: list[nn.Module] = [] # Track modules with hooks
self._custom_pipeline: nn.Module | None = None
self._custom_pipeline: SupportsModelCpuOffload | None = None
def enable(self, pipeline: nn.Module) -> None:
if self.enabled:
logger.warning("ModelLevelOffloadBackend already enabled")
return
# Pipelines with a nested transformer (e.g. Cosmos3's reasoner/generator
# pathways) own their own mutual-exclusion swaps and cannot be offloaded
# with the generic encoder<->DiT hooks. Delegate to the custom hook.
# TODO: Cosmos3 is the only implementer today, so we duck-type the
# enable/disable_omni_model_cpu_offload pair via getattr. Once a second
# pipeline needs it, formalize this as a Protocol (like
# SupportsComponentDiscovery) instead of getattr detection.
custom_enable = getattr(pipeline, "enable_omni_model_cpu_offload", None)
if callable(custom_enable):
custom_enable(
# Pipelines with non-forward component entry points own their complete
# mutual-exclusion lifecycle. Delegate through the explicit protocol.
if isinstance(pipeline, SupportsModelCpuOffload):
pipeline.enable_omni_model_cpu_offload(
device=self.device,
pin_memory=self.config.pin_cpu_memory,
use_hsdp=self.config.use_hsdp,
@@ -283,9 +315,7 @@ class ModelLevelOffloadBackend(OffloadBackend):
return
if self._custom_pipeline is not None:
custom_disable = getattr(self._custom_pipeline, "disable_omni_model_cpu_offload", None)
if callable(custom_disable):
custom_disable()
self._custom_pipeline.disable_omni_model_cpu_offload()
self._custom_pipeline = None
self.enabled = False
logger.info("Model-level offloading disabled")