[Refactor] Let diffusion pipelines declare offloadable modules via SupportsModuleOffload (#2427)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick CaoandClaude authored and GitHub committed 2026-04-23 04:23:00 +08:00
1 parent ead87aa521
commit e3b0afbb97
11 files changed
+491 -62

No files matched your search

@@ -12,6 +12,7 @@ from collections.abc import Iterable
from copy import deepcopy
from dataclasses import dataclass
from math import isqrt
from typing import ClassVar
import numpy as np
import torch
@@ -26,6 +27,7 @@ from vllm.transformers_utils.configs.bagel import BagelConfig
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
@@ -148,12 +150,25 @@ class SiglipNaViTWrapper(nn.Module):
return outputs.last_hidden_state.squeeze(0)
class BagelPipeline(nn.Module, DiffusionPipelineProfilerMixin):
class BagelPipeline(nn.Module, SupportsModuleOffload, DiffusionPipelineProfilerMixin):
"""Bagel generation pipeline (MoT) packaged for vllm-omni diffusion engine.
This pipeline is self-contained and uses the ported Bagel core files.
"""
_dit_modules: ClassVar[list[str]] = ["language_model.model"]
_encoder_modules: ClassVar[list[str]] = []
_vae_modules: ClassVar[list[str]] = ["vae"]
_resident_modules: ClassVar[list[str]] = [
"bagel.time_embedder",
"bagel.vae2llm",
"bagel.llm2vae",
"bagel.latent_pos_embed",
"bagel.vit_model",
"bagel.connector",
"bagel.vit_pos_embed",
]
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
super().__init__()
self.od_config = od_config
+21
View File
@@ -58,6 +58,27 @@ class SupportsStepExecution(Protocol):
"""Decode output after denoise loop."""
@runtime_checkable
class SupportsModuleOffload(Protocol):
"""Declares which submodules participate in CPU offload.
All attribute names support dotted paths for nested submodules
(e.g. ``"pipe.transformer"``).
Attributes:
_dit_modules: Denoising submodules (on GPU during diffusion).
_encoder_modules: Encoder submodules (offloaded during diffusion).
_vae_modules: VAE(s) (always on GPU).
_resident_modules: Extra modules pinned on GPU during layerwise
offloading. Optional, defaults to ``[]``.
"""
_dit_modules: ClassVar[list[str]]
_encoder_modules: ClassVar[list[str]]
_vae_modules: ClassVar[list[str]]
_resident_modules: ClassVar[list[str]] = []
def supports_step_execution(pipeline: object) -> bool:
"""Return whether `pipeline` implements :class:`SupportsStepExecution`."""
@@ -9,7 +9,7 @@ import json
import os
from collections.abc import Iterable
from contextlib import nullcontext
from typing import Any
from typing import Any, ClassVar
import numpy as np
import torch
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.lora.request import LoRARequest
@@ -1152,9 +1153,13 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
return loader.load_weights(weights)
class LTX2TwoStagesPipeline(nn.Module):
class LTX2TwoStagesPipeline(nn.Module, SupportsModuleOffload):
"""LTX2TwoStagesPipeline is for two stages image to video generation"""
_dit_modules: ClassVar[list[str]] = ["pipe.transformer"]
_encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"]
_vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"]
def __init__(
self,
*,
@@ -6,7 +6,7 @@ from __future__ import annotations
import copy
import os
from collections.abc import Iterable
from typing import Any
from typing import Any, ClassVar
import numpy as np
import PIL.Image
@@ -26,6 +26,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.lora.request import LoRARequest
@@ -733,11 +734,15 @@ class LTX2ImageToVideoPipeline(LTX2Pipeline):
return DiffusionOutput(output=(video, audio))
class LTX2ImageToVideoTwoStagesPipeline(nn.Module):
class LTX2ImageToVideoTwoStagesPipeline(nn.Module, SupportsModuleOffload):
"""LTXImageToVideoTwoStagesPipeline is for two stages image to video generation"""
support_image_input = True
_dit_modules: ClassVar[list[str]] = ["pipe.transformer"]
_encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"]
_vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"]
def __init__(
self,
*,
@@ -8,7 +8,7 @@ import os
import warnings
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any
from typing import Any, ClassVar
import numpy as np
import PIL.Image
@@ -32,6 +32,7 @@ from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
from vllm_omni.diffusion.models.omnigen2.omnigen2_transformer import (
OmniGen2RotaryPosEmbed,
OmniGen2Transformer2DModel,
@@ -620,7 +621,7 @@ def retrieve_timesteps(
return timesteps, num_inference_steps
class OmniGen2Pipeline(CFGParallelMixin, nn.Module):
class OmniGen2Pipeline(CFGParallelMixin, nn.Module, SupportsModuleOffload):
"""
Pipeline for text-to-image generation using OmniGen2.
@@ -634,6 +635,10 @@ class OmniGen2Pipeline(CFGParallelMixin, nn.Module):
od_config (OmniDiffusionConfig): The OmniDiffusion configuration.
"""
_dit_modules: ClassVar[list[str]] = ["transformer"]
_encoder_modules: ClassVar[list[str]] = ["mllm"]
_vae_modules: ClassVar[list[str]] = ["vae"]
def __init__(
self,
*,