[Feat] Add vae-decode-parallel for LTX-2.3 (#4277)
Signed-off-by: mglyn <1203789601@qq.com>
This commit is contained in:
1 parent
f174ee54b0
commit
bf03413c36
4 files changed
+486
-6
No files matched your search
@@ -23,7 +23,7 @@ from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers import AutoencoderKLLTX2Audio, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.pipelines.ltx2 import LTX2TextConnectors
|
||||
from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder
|
||||
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import retrieve_timesteps
|
||||
@@ -36,6 +36,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import DistributedAutoencoderKLLTX2Video
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
@@ -48,6 +49,7 @@ from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_p
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
from .pipeline_ltx2 import (
|
||||
@@ -113,7 +115,13 @@ def get_ltx2_post_process_func(od_config: OmniDiffusionConfig):
|
||||
return post_process_func
|
||||
|
||||
|
||||
class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsComponentDiscovery):
|
||||
class LTX23Pipeline(
|
||||
nn.Module,
|
||||
CFGParallelMixin,
|
||||
ProgressBarMixin,
|
||||
SupportsComponentDiscovery,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
):
|
||||
"""Fully independent LTX-2.3 pipeline.
|
||||
|
||||
Key differences from LTX2Pipeline:
|
||||
@@ -193,7 +201,7 @@ class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsCompo
|
||||
|
||||
# --- VAE, Audio VAE ---
|
||||
self.vae = from_pretrained_with_prefetch(
|
||||
AutoencoderKLLTX2Video.from_pretrained,
|
||||
DistributedAutoencoderKLLTX2Video.from_pretrained,
|
||||
model,
|
||||
subfolder="vae",
|
||||
prefetch_list=ltx2_subfolders,
|
||||
@@ -266,6 +274,10 @@ class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsCompo
|
||||
self._num_timesteps = None
|
||||
self._current_timestep = None
|
||||
|
||||
self.setup_diffusion_pipeline_profiler(
|
||||
enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler
|
||||
)
|
||||
|
||||
def _place_aux_components(self) -> None:
|
||||
parallel_config = getattr(self.od_config, "parallel_config", None)
|
||||
use_managed_placement = bool(
|
||||
@@ -1253,14 +1265,18 @@ class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, SupportsCompo
|
||||
|
||||
latents = latents.to(self.vae.dtype)
|
||||
video = self.vae.decode(latents, timestep_decode, return_dict=False)[0]
|
||||
video = self.video_processor.postprocess_video(video, output_type=output_type)
|
||||
if video.numel() > 0:
|
||||
video = self.video_processor.postprocess_video(video, output_type=output_type)
|
||||
|
||||
audio_latents = audio_latents.to(self.audio_vae.dtype)
|
||||
generated_mel_spectrograms = self.audio_vae.decode(audio_latents, return_dict=False)[0]
|
||||
|
||||
audio = self.vocoder(generated_mel_spectrograms)
|
||||
|
||||
return DiffusionOutput(output=(video, audio))
|
||||
return DiffusionOutput(
|
||||
output=(video, audio),
|
||||
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
Reference in new issue
Block a user