[Refactor] Let diffusion pipelines declare offloadable modules via SupportsModuleOffload (#2427)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
ead87aa521
commit
e3b0afbb97
11 files changed
+491
-62
No files matched your search
@@ -12,6 +12,7 @@ from collections.abc import Iterable
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from math import isqrt
|
||||
from typing import ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -26,6 +27,7 @@ from vllm.transformers_utils.configs.bagel import BagelConfig
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
@@ -148,12 +150,25 @@ class SiglipNaViTWrapper(nn.Module):
|
||||
return outputs.last_hidden_state.squeeze(0)
|
||||
|
||||
|
||||
class BagelPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
class BagelPipeline(nn.Module, SupportsModuleOffload, DiffusionPipelineProfilerMixin):
|
||||
"""Bagel generation pipeline (MoT) packaged for vllm-omni diffusion engine.
|
||||
|
||||
This pipeline is self-contained and uses the ported Bagel core files.
|
||||
"""
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["language_model.model"]
|
||||
_encoder_modules: ClassVar[list[str]] = []
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_resident_modules: ClassVar[list[str]] = [
|
||||
"bagel.time_embedder",
|
||||
"bagel.vae2llm",
|
||||
"bagel.llm2vae",
|
||||
"bagel.latent_pos_embed",
|
||||
"bagel.vit_model",
|
||||
"bagel.connector",
|
||||
"bagel.vit_pos_embed",
|
||||
]
|
||||
|
||||
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.od_config = od_config
|
||||
|
||||
@@ -58,6 +58,27 @@ class SupportsStepExecution(Protocol):
|
||||
"""Decode output after denoise loop."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsModuleOffload(Protocol):
|
||||
"""Declares which submodules participate in CPU offload.
|
||||
|
||||
All attribute names support dotted paths for nested submodules
|
||||
(e.g. ``"pipe.transformer"``).
|
||||
|
||||
Attributes:
|
||||
_dit_modules: Denoising submodules (on GPU during diffusion).
|
||||
_encoder_modules: Encoder submodules (offloaded during diffusion).
|
||||
_vae_modules: VAE(s) (always on GPU).
|
||||
_resident_modules: Extra modules pinned on GPU during layerwise
|
||||
offloading. Optional, defaults to ``[]``.
|
||||
"""
|
||||
|
||||
_dit_modules: ClassVar[list[str]]
|
||||
_encoder_modules: ClassVar[list[str]]
|
||||
_vae_modules: ClassVar[list[str]]
|
||||
_resident_modules: ClassVar[list[str]] = []
|
||||
|
||||
|
||||
def supports_step_execution(pipeline: object) -> bool:
|
||||
"""Return whether `pipeline` implements :class:`SupportsStepExecution`."""
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -34,6 +34,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
|
||||
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
@@ -1152,9 +1153,13 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
class LTX2TwoStagesPipeline(nn.Module):
|
||||
class LTX2TwoStagesPipeline(nn.Module, SupportsModuleOffload):
|
||||
"""LTX2TwoStagesPipeline is for two stages image to video generation"""
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["pipe.transformer"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -26,6 +26,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
|
||||
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
|
||||
@@ -733,11 +734,15 @@ class LTX2ImageToVideoPipeline(LTX2Pipeline):
|
||||
return DiffusionOutput(output=(video, audio))
|
||||
|
||||
|
||||
class LTX2ImageToVideoTwoStagesPipeline(nn.Module):
|
||||
class LTX2ImageToVideoTwoStagesPipeline(nn.Module, SupportsModuleOffload):
|
||||
"""LTXImageToVideoTwoStagesPipeline is for two stages image to video generation"""
|
||||
|
||||
support_image_input = True
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["pipe.transformer"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["pipe.text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["pipe.vae", "pipe.audio_vae"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -8,7 +8,7 @@ import os
|
||||
import warnings
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
@@ -32,6 +32,7 @@ from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportsModuleOffload
|
||||
from vllm_omni.diffusion.models.omnigen2.omnigen2_transformer import (
|
||||
OmniGen2RotaryPosEmbed,
|
||||
OmniGen2Transformer2DModel,
|
||||
@@ -620,7 +621,7 @@ def retrieve_timesteps(
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
|
||||
class OmniGen2Pipeline(CFGParallelMixin, nn.Module):
|
||||
class OmniGen2Pipeline(CFGParallelMixin, nn.Module, SupportsModuleOffload):
|
||||
"""
|
||||
Pipeline for text-to-image generation using OmniGen2.
|
||||
|
||||
@@ -634,6 +635,10 @@ class OmniGen2Pipeline(CFGParallelMixin, nn.Module):
|
||||
od_config (OmniDiffusionConfig): The OmniDiffusion configuration.
|
||||
"""
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["mllm"]
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in new issue
Block a user