[Model] Add native SANA-Video 2B T2V and I2V support (#5508)

Signed-off-by: leo <92263211+napleon-liu@users.noreply.github.com>
Signed-off-by: leo <92263211+liuyao0322@users.noreply.github.com>
Signed-off-by: liuyao0322 <yaoliu548926@gmail.com>
Co-authored-by: leo <92263211+napleon-liu@users.noreply.github.com>
This commit is contained in:
leoandleo authored and GitHub committed 2026-08-25 09:46:02 +08:00
1 parent 4af0ddc1c2
commit 50b8a150f9
42 files changed
+6512 -18

No files matched your search

@@ -23,7 +23,11 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from torch import nn
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
BasePipelineUtils,
get_pipeline_utils_for_config,
resolve_diffusers_pipeline_class,
)
from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import (
apply_diffusers_quantization_config,
convert_diffusers_quantization_config,
@@ -117,9 +121,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
}
convert_diffusers_quantization_config(load_kwargs)
pipeline_class = self.od_config.diffusers_pipeline_cls
pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None
self._pipeline_utils = get_pipeline_utils(pipeline_class_name)
self._pipeline_utils = get_pipeline_utils_for_config(self.od_config)
self._pipeline_utils.update_load_kwargs(self.od_config, load_kwargs)
component_names = (
self._load_diffusers_component_names(model_id, load_kwargs)
@@ -129,7 +131,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
apply_diffusers_quantization_config(self.od_config, load_kwargs, component_names)
logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}")
self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)
self._pipeline = self._load_pipeline_from_pretrained(model_id, load_kwargs)
self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config)
self._pipeline.to(self.device)
@@ -162,6 +164,23 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
# Attention backend
self._set_attention_backend()
def _load_pipeline_from_pretrained(
self,
model_id: str,
load_kwargs: dict[str, Any],
) -> DiffusionPipeline:
pipeline_class = resolve_diffusers_pipeline_class(self.od_config)
configured_pipeline_class = self.od_config.diffusers_pipeline_cls
# Preserve DiffusionPipeline's normal dynamic class resolution unless
# a model-specific adapter hook explicitly selected another class.
loader_class = (
pipeline_class
if pipeline_class is not None and pipeline_class is not configured_pipeline_class
else DiffusionPipeline
)
logger.info("Loading Diffusers pipeline class %s", loader_class.__name__)
return loader_class.from_pretrained(model_id, **load_kwargs)
# ------------------------------------------------------------------
# Step-wise execution — explicitly rejected
# ------------------------------------------------------------------
@@ -376,6 +395,13 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
if self._accept_call_kwargs is None or key in self._accept_call_kwargs:
kwargs[key] = value
self._pipeline_utils.update_call_kwargs(
req,
sampling,
self._accept_call_kwargs,
kwargs,
)
# Special format fields in sampling params
if output_type := sampling.output_type or self.od_config.output_type:
kwargs["output_type"] = output_type
@@ -1,17 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from typing import Any
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.request import resolve_video_num_frames
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
class BasePipelineUtils:
"""No-op hooks for pipeline-specific diffusers adapter behavior."""
def resolve_pipeline_class(
self,
od_config: OmniDiffusionConfig,
configured_pipeline_class: type[DiffusionPipeline] | None,
) -> type[DiffusionPipeline] | None:
"""Resolve the concrete Diffusers class used to load a checkpoint."""
return configured_pipeline_class
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
pass
@@ -21,6 +31,59 @@ class BasePipelineUtils:
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
pass
def update_call_kwargs(
self,
req: DiffusionRequestBatch,
sampling: OmniDiffusionSamplingParams,
accepted_call_kwargs: set[str] | None,
call_kwargs: dict[str, Any],
) -> None:
pass
class SanaVideoPipelineUtils(BasePipelineUtils):
def resolve_pipeline_class(
self,
od_config: OmniDiffusionConfig,
configured_pipeline_class: type[DiffusionPipeline] | None,
) -> type[DiffusionPipeline] | None:
# Both released SANA-Video checkpoints declare SanaVideoPipeline in
# model_index.json. Honour an explicit Omni I2V class selection by
# loading the same components into Diffusers' I2V pipeline instead.
if od_config.model_class_name == "SanaImageToVideoPipeline":
from diffusers import SanaImageToVideoPipeline
return SanaImageToVideoPipeline
return configured_pipeline_class
def update_call_kwargs(
self,
req: DiffusionRequestBatch,
sampling: OmniDiffusionSamplingParams,
accepted_call_kwargs: set[str] | None,
call_kwargs: dict[str, Any],
) -> None:
# Diffusers SANA-Video calls the shared request's `num_frames`
# parameter `frames`.
if (
accepted_call_kwargs is not None
and "frames" in accepted_call_kwargs
and "num_frames" not in accepted_call_kwargs
):
configured_default = call_kwargs.get("frames", 81)
if configured_default is None:
configured_default = 81
call_kwargs["frames"] = resolve_video_num_frames(
sampling.num_frames,
default_num_frames=configured_default,
is_dummy_run=req.is_dummy_run(),
)
# SANA's startup dummy dimensions can map to a resolution bucket that
# conflicts with the upstream pipeline's divisibility validation.
if req.is_dummy_run() and accepted_call_kwargs is not None and "use_resolution_binning" in accepted_call_kwargs:
call_kwargs["use_resolution_binning"] = False
class WanPipelineUtils(BasePipelineUtils):
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
@@ -49,6 +112,8 @@ class WanPipelineUtils(BasePipelineUtils):
PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
"SanaVideoPipeline": SanaVideoPipelineUtils,
"SanaImageToVideoPipeline": SanaVideoPipelineUtils,
"WanPipeline": WanPipelineUtils,
"WanImageToVideoPipeline": WanPipelineUtils,
"WanVACEPipeline": WanPipelineUtils,
@@ -62,3 +127,20 @@ def get_pipeline_utils(pipeline_class_name: str | None) -> BasePipelineUtils:
return BasePipelineUtils()
pipeline_utils_cls = PIPELINE_UTILS_REGISTRY.get(pipeline_class_name, BasePipelineUtils)
return pipeline_utils_cls()
def get_pipeline_utils_for_config(od_config: OmniDiffusionConfig) -> BasePipelineUtils:
"""Return model-specific adapter hooks for the requested or configured class."""
requested_class_name = od_config.model_class_name
if requested_class_name in PIPELINE_UTILS_REGISTRY:
return get_pipeline_utils(requested_class_name)
configured_pipeline_class = od_config.diffusers_pipeline_cls
configured_class_name = configured_pipeline_class.__name__ if configured_pipeline_class is not None else None
return get_pipeline_utils(configured_class_name)
def resolve_diffusers_pipeline_class(od_config: OmniDiffusionConfig) -> type[DiffusionPipeline] | None:
"""Resolve the concrete Diffusers pipeline used for loading and capability checks."""
pipeline_utils = get_pipeline_utils_for_config(od_config)
return pipeline_utils.resolve_pipeline_class(od_config, od_config.diffusers_pipeline_cls)
@@ -0,0 +1,32 @@
# Copyright 2025 SANA-Video Authors and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .pipeline_output import SanaVideoPipelineOutput
from .pipeline_sana_video import SanaVideoPipeline, get_sana_video_post_process_func
from .pipeline_sana_video_i2v import (
SanaImageToVideoPipeline,
get_sana_video_i2v_post_process_func,
get_sana_video_i2v_pre_process_func,
)
from .transformer_sana_video import SanaVideoTransformer3DModel
__all__ = [
"SanaImageToVideoPipeline",
"SanaVideoPipeline",
"SanaVideoPipelineOutput",
"SanaVideoTransformer3DModel",
"get_sana_video_i2v_post_process_func",
"get_sana_video_i2v_pre_process_func",
"get_sana_video_post_process_func",
]
@@ -0,0 +1,19 @@
from dataclasses import dataclass
import torch
from diffusers.utils import BaseOutput
@dataclass
class SanaVideoPipelineOutput(BaseOutput):
r"""
Output class for Sana-Video pipelines.
Args:
frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]):
List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing
denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape
`(batch_size, num_frames, channels, height, width)`.
"""
frames: torch.Tensor
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,382 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import os
import warnings
from math import sqrt
import PIL.Image
import torch
from diffusers.utils.torch_utils import randn_tensor
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
DistributedAutoencoderKLLTX2Video,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
ASPECT_RATIO_480_BIN,
ASPECT_RATIO_720_BIN,
SanaVideoPipeline,
get_sana_video_default_resolution,
get_sana_video_post_process_func,
resolve_sana_video_sample_size,
retrieve_timesteps,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest, resolve_video_num_frames
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniTextPrompt
SANA_VIDEO_I2V_COMPLEX_HUMAN_INSTRUCTION = [
(
"Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for "
"video generation. Evaluate the level of detail in the user prompt:"
),
(
"- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, motion, and "
"temporal relationships to create vivid and dynamic scenes."
),
("- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating."),
"Here are examples of how to transform or refine prompts:",
(
"- User Prompt: A cat sleeping -> Enhanced: A small, fluffy white cat slowly settling into a curled position, "
"peacefully falling asleep on a warm sunny windowsill, with gentle sunlight filtering through surrounding "
"pots of blooming red flowers."
),
(
"- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street "
"lamps gradually lighting up, a diverse crowd of people in colorful clothing walking past, and a double-decker "
"bus smoothly passing by towering glass skyscrapers."
),
(
"Please generate only the enhanced description for the prompt below and avoid including any additional "
"commentary or evaluations:"
),
"User Prompt: ",
]
def get_sana_video_i2v_post_process_func(od_config: OmniDiffusionConfig):
return get_sana_video_post_process_func(od_config)
def get_sana_video_i2v_pre_process_func(od_config: OmniDiffusionConfig):
default_height, default_width = get_sana_video_default_resolution(resolve_sana_video_sample_size(od_config))
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
prompt = request.prompt
if isinstance(prompt, str):
prompt = OmniTextPrompt(prompt=prompt, multi_modal_data={})
multi_modal_data = prompt.get("multi_modal_data") or {}
raw_image = multi_modal_data.get("image")
if raw_image is None:
raise ValueError('SANA-Video I2V requires `"multi_modal_data": {"image": <PIL image or file path>}`.')
if isinstance(raw_image, str):
if not os.path.isfile(raw_image):
raise ValueError(f"SANA-Video I2V image path does not exist: {raw_image}")
image = PIL.Image.open(raw_image).convert("RGB")
elif isinstance(raw_image, PIL.Image.Image):
image = raw_image.convert("RGB")
else:
raise TypeError(f"Unsupported SANA-Video I2V image type: {type(raw_image).__name__}")
if request.sampling_params.height is None or request.sampling_params.width is None:
max_area = default_height * default_width
aspect_ratio = image.height / image.width
inferred_height = round(sqrt(max_area * aspect_ratio)) // 32 * 32
inferred_width = round(sqrt(max_area / aspect_ratio)) // 32 * 32
if request.sampling_params.height is None:
request.sampling_params.height = inferred_height
if request.sampling_params.width is None:
request.sampling_params.width = inferred_width
image = image.resize(
(request.sampling_params.width, request.sampling_params.height),
PIL.Image.Resampling.LANCZOS,
)
multi_modal_data["image"] = image
prompt["multi_modal_data"] = multi_modal_data
request.prompt = prompt
return request
return pre_process_func
def _retrieve_latents(encoder_output: torch.Tensor) -> torch.Tensor:
if hasattr(encoder_output, "latent_dist"):
return encoder_output.latent_dist.mode()
if hasattr(encoder_output, "latents"):
return encoder_output.latents
raise AttributeError("Could not access latents from the SANA-Video VAE encoder output.")
class SanaImageToVideoPipeline(SanaVideoPipeline, SupportImageInput):
"""Native vLLM-Omni SANA-Video 2B image-to-video pipeline.
The first latent frame is encoded from the input image and remains fixed;
denoising updates only the subsequent video frames.
"""
def _prepare_i2v_latents(
self,
image: torch.Tensor,
batch_size: int,
num_channels_latents: int,
height: int,
width: int,
num_frames: int,
dtype: torch.dtype,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None,
) -> torch.Tensor:
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
shape = (
batch_size,
num_channels_latents,
num_latent_frames,
height // self.vae_scale_factor_spatial,
width // self.vae_scale_factor_spatial,
)
if isinstance(generator, list) and len(generator) != batch_size:
raise ValueError(
f"Generator list length {len(generator)} does not match effective batch size {batch_size}."
)
if latents is None:
latents = randn_tensor(shape, generator=generator, device=self.device, dtype=dtype)
else:
latents = latents.to(device=self.device, dtype=dtype)
image = image.unsqueeze(2).to(device=self.device, dtype=self.vae.dtype)
image_latents = _retrieve_latents(self.vae.encode(image))
image_latents = image_latents.repeat(batch_size, 1, 1, 1, 1)
if isinstance(self.vae, DistributedAutoencoderKLLTX2Video):
latents_mean = self.vae.latents_mean
latents_std = self.vae.latents_std
elif isinstance(self.vae, DistributedAutoencoderKLWan):
latents_mean = torch.tensor(self.vae.config.latents_mean)
latents_std = torch.tensor(self.vae.config.latents_std)
else:
raise TypeError(f"Unsupported SANA-Video I2V VAE type: {type(self.vae).__name__}")
latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(image_latents.device, image_latents.dtype)
latents_std = latents_std.view(1, -1, 1, 1, 1).to(image_latents.device, image_latents.dtype)
image_latents = (image_latents - latents_mean) / latents_std
latents[:, :, :1] = image_latents.to(dtype)
return latents
def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:
if len(req.prompts) != 1:
raise ValueError("SANA-Video I2V currently supports exactly one prompt per request.")
prompt_obj = req.prompts[0]
if isinstance(prompt_obj, str):
raise ValueError("SANA-Video I2V requires an image in `multi_modal_data`.")
prompt = prompt_obj.get("prompt") or ""
negative_prompt = prompt_obj.get("negative_prompt") or ""
image = (prompt_obj.get("multi_modal_data") or {}).get("image")
if not prompt:
raise ValueError("Prompt is required for SANA-Video I2V generation.")
if not isinstance(image, PIL.Image.Image):
raise TypeError("SANA-Video I2V expects a preprocessed PIL image.")
sampling = req.sampling_params
default_height, default_width = get_sana_video_default_resolution(self.transformer.config.sample_size)
height = sampling.height or default_height
width = sampling.width or default_width
frames = resolve_video_num_frames(
sampling.num_frames,
default_num_frames=81,
is_dummy_run=req.is_dummy_run(),
)
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
generator = sampling.generator
if generator is None and sampling.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(sampling.seed)
extra_args = sampling.extra_args or {}
motion_score = extra_args.get("motion_score")
if motion_score is not None:
prompt = f"{prompt} motion score: {int(motion_score)}."
video = self._generate_i2v(
image=image,
prompt=prompt,
negative_prompt=negative_prompt,
height=height,
width=width,
frames=frames,
num_inference_steps=num_steps,
timesteps=sampling.timesteps,
sigmas=sampling.sigmas,
guidance_scale=guidance_scale,
num_videos_per_prompt=sampling.num_outputs_per_prompt or 1,
eta=sampling.eta,
generator=generator,
latents=sampling.latents,
clean_caption=bool(extra_args.get("clean_caption", False)),
use_resolution_binning=bool(extra_args.get("use_resolution_binning", True)),
max_sequence_length=sampling.max_sequence_length or 300,
output_type="latent" if sampling.output_type == "latent" else "raw",
)
return DiffusionOutput(output=video)
@torch.no_grad()
def _generate_i2v(
self,
*,
image: PIL.Image.Image,
prompt: str,
negative_prompt: str,
height: int,
width: int,
frames: int,
num_inference_steps: int,
guidance_scale: float,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None,
use_resolution_binning: bool,
max_sequence_length: int,
timesteps: list[int] | torch.Tensor | None = None,
sigmas: list[float] | None = None,
num_videos_per_prompt: int = 1,
eta: float = 0.0,
clean_caption: bool = False,
output_type: str = "raw",
complex_human_instruction: list[str] = SANA_VIDEO_I2V_COMPLEX_HUMAN_INSTRUCTION,
) -> torch.Tensor:
orig_height, orig_width = height, width
if use_resolution_binning:
if self.transformer.config.sample_size == 30:
aspect_ratio_bin = ASPECT_RATIO_480_BIN
elif self.transformer.config.sample_size == 22:
aspect_ratio_bin = ASPECT_RATIO_720_BIN
else:
raise ValueError(f"Unsupported SANA-Video sample size: {self.transformer.config.sample_size}")
height, width = self.video_processor.classify_height_width_bin(
height,
width,
ratios=aspect_ratio_bin,
)
self.check_inputs(
prompt=prompt,
height=height,
width=width,
negative_prompt=negative_prompt,
)
self._guidance_scale = guidance_scale
self._interrupt = False
prompt_embeds, prompt_mask, negative_embeds, negative_mask = self.encode_prompt(
prompt,
self.do_classifier_free_guidance,
negative_prompt=negative_prompt,
num_videos_per_prompt=num_videos_per_prompt,
device=self.device,
clean_caption=clean_caption,
max_sequence_length=max_sequence_length,
complex_human_instruction=complex_human_instruction,
)
if self.do_classifier_free_guidance:
prompt_embeds = torch.cat([negative_embeds, prompt_embeds])
prompt_mask = torch.cat([negative_mask, prompt_mask])
timesteps, num_inference_steps = retrieve_timesteps(
self.scheduler,
num_inference_steps=num_inference_steps,
device=self.device,
timesteps=timesteps,
sigmas=sigmas,
)
image_tensor = self.video_processor.preprocess(image, height=height, width=width).to(
self.device,
dtype=torch.float32,
)
latent_channels = self.transformer.config.in_channels
latents = self._prepare_i2v_latents(
image_tensor,
batch_size=num_videos_per_prompt,
num_channels_latents=latent_channels,
height=height,
width=width,
num_frames=frames,
dtype=torch.float32,
generator=generator,
latents=latents,
)
patch_t, patch_h, patch_w = self.transformer.config.patch_size
conditioning_mask = latents.new_zeros(
num_videos_per_prompt,
1,
latents.shape[2] // patch_t,
latents.shape[3] // patch_h,
latents.shape[4] // patch_w,
)
conditioning_mask[:, :, 0] = 1
if self.do_classifier_free_guidance:
conditioning_mask = torch.cat([conditioning_mask, conditioning_mask])
self._num_timesteps = len(timesteps)
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
transformer_dtype = self.transformer.dtype
with self.progress_bar(total=num_inference_steps) as progress_bar:
for index, timestep_value in enumerate(timesteps):
latent_input = torch.cat([latents, latents]) if self.do_classifier_free_guidance else latents
timestep = timestep_value.expand(conditioning_mask.shape) * (1 - conditioning_mask)
noise_pred = self.transformer(
latent_input.to(transformer_dtype),
encoder_hidden_states=prompt_embeds.to(transformer_dtype),
encoder_attention_mask=prompt_mask,
timestep=timestep,
return_dict=False,
)[0].float()
if self.do_classifier_free_guidance:
uncond, cond = noise_pred.chunk(2)
noise_pred = uncond + guidance_scale * (cond - uncond)
if self.transformer.config.out_channels // 2 == latent_channels:
noise_pred = noise_pred.chunk(2, dim=1)[0]
denoised = self.scheduler.step(
noise_pred[:, :, 1:],
timestep_value,
latents[:, :, 1:],
**extra_step_kwargs,
return_dict=False,
)[0]
latents = torch.cat([latents[:, :, :1], denoised], dim=2)
if index == len(timesteps) - 1 or (index + 1) % self.scheduler.order == 0:
progress_bar.update()
if output_type == "latent":
return latents
latents = latents.to(self.vae.dtype)
if isinstance(self.vae, DistributedAutoencoderKLLTX2Video):
latents_mean = self.vae.latents_mean
latents_std = self.vae.latents_std
latent_dim = self.vae.config.latent_channels
else:
latents_mean = torch.tensor(self.vae.config.latents_mean)
latents_std = torch.tensor(self.vae.config.latents_std)
latent_dim = self.vae.config.z_dim
latents_mean = latents_mean.view(1, latent_dim, 1, 1, 1).to(latents.device, latents.dtype)
latents_std = latents_std.view(1, latent_dim, 1, 1, 1).to(latents.device, latents.dtype)
latents = latents * latents_std + latents_mean
try:
video = self.vae.decode(latents, return_dict=False)[0]
except torch.OutOfMemoryError as error:
warnings.warn(
f"{error}. Enable VAE tiling for SANA-Video I2V high-resolution decoding.",
stacklevel=2,
)
raise
if use_resolution_binning:
video = self.video_processor.resize_and_crop_tensor(video, orig_width, orig_height)
return video
File diff suppressed because it is too large. Load diff