[Model] Add native SANA-Video 2B T2V and I2V support (#5508)
Signed-off-by: leo <92263211+napleon-liu@users.noreply.github.com> Signed-off-by: leo <92263211+liuyao0322@users.noreply.github.com> Signed-off-by: liuyao0322 <yaoliu548926@gmail.com> Co-authored-by: leo <92263211+napleon-liu@users.noreply.github.com>
This commit is contained in:
1 parent
4af0ddc1c2
commit
50b8a150f9
42 files changed
+6512
-18
No files matched your search
@@ -23,7 +23,11 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
|
||||
BasePipelineUtils,
|
||||
get_pipeline_utils_for_config,
|
||||
resolve_diffusers_pipeline_class,
|
||||
)
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import (
|
||||
apply_diffusers_quantization_config,
|
||||
convert_diffusers_quantization_config,
|
||||
@@ -117,9 +121,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
}
|
||||
convert_diffusers_quantization_config(load_kwargs)
|
||||
|
||||
pipeline_class = self.od_config.diffusers_pipeline_cls
|
||||
pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None
|
||||
self._pipeline_utils = get_pipeline_utils(pipeline_class_name)
|
||||
self._pipeline_utils = get_pipeline_utils_for_config(self.od_config)
|
||||
self._pipeline_utils.update_load_kwargs(self.od_config, load_kwargs)
|
||||
component_names = (
|
||||
self._load_diffusers_component_names(model_id, load_kwargs)
|
||||
@@ -129,7 +131,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
apply_diffusers_quantization_config(self.od_config, load_kwargs, component_names)
|
||||
logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}")
|
||||
|
||||
self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)
|
||||
self._pipeline = self._load_pipeline_from_pretrained(model_id, load_kwargs)
|
||||
self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config)
|
||||
|
||||
self._pipeline.to(self.device)
|
||||
@@ -162,6 +164,23 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
# Attention backend
|
||||
self._set_attention_backend()
|
||||
|
||||
def _load_pipeline_from_pretrained(
|
||||
self,
|
||||
model_id: str,
|
||||
load_kwargs: dict[str, Any],
|
||||
) -> DiffusionPipeline:
|
||||
pipeline_class = resolve_diffusers_pipeline_class(self.od_config)
|
||||
configured_pipeline_class = self.od_config.diffusers_pipeline_cls
|
||||
# Preserve DiffusionPipeline's normal dynamic class resolution unless
|
||||
# a model-specific adapter hook explicitly selected another class.
|
||||
loader_class = (
|
||||
pipeline_class
|
||||
if pipeline_class is not None and pipeline_class is not configured_pipeline_class
|
||||
else DiffusionPipeline
|
||||
)
|
||||
logger.info("Loading Diffusers pipeline class %s", loader_class.__name__)
|
||||
return loader_class.from_pretrained(model_id, **load_kwargs)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step-wise execution — explicitly rejected
|
||||
# ------------------------------------------------------------------
|
||||
@@ -376,6 +395,13 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
if self._accept_call_kwargs is None or key in self._accept_call_kwargs:
|
||||
kwargs[key] = value
|
||||
|
||||
self._pipeline_utils.update_call_kwargs(
|
||||
req,
|
||||
sampling,
|
||||
self._accept_call_kwargs,
|
||||
kwargs,
|
||||
)
|
||||
|
||||
# Special format fields in sampling params
|
||||
if output_type := sampling.output_type or self.od_config.output_type:
|
||||
kwargs["output_type"] = output_type
|
||||
|
||||
@@ -1,17 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from typing import Any
|
||||
|
||||
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.request import resolve_video_num_frames
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
class BasePipelineUtils:
|
||||
"""No-op hooks for pipeline-specific diffusers adapter behavior."""
|
||||
|
||||
def resolve_pipeline_class(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig,
|
||||
configured_pipeline_class: type[DiffusionPipeline] | None,
|
||||
) -> type[DiffusionPipeline] | None:
|
||||
"""Resolve the concrete Diffusers class used to load a checkpoint."""
|
||||
return configured_pipeline_class
|
||||
|
||||
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
@@ -21,6 +31,59 @@ class BasePipelineUtils:
|
||||
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
||||
pass
|
||||
|
||||
def update_call_kwargs(
|
||||
self,
|
||||
req: DiffusionRequestBatch,
|
||||
sampling: OmniDiffusionSamplingParams,
|
||||
accepted_call_kwargs: set[str] | None,
|
||||
call_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class SanaVideoPipelineUtils(BasePipelineUtils):
|
||||
def resolve_pipeline_class(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig,
|
||||
configured_pipeline_class: type[DiffusionPipeline] | None,
|
||||
) -> type[DiffusionPipeline] | None:
|
||||
# Both released SANA-Video checkpoints declare SanaVideoPipeline in
|
||||
# model_index.json. Honour an explicit Omni I2V class selection by
|
||||
# loading the same components into Diffusers' I2V pipeline instead.
|
||||
if od_config.model_class_name == "SanaImageToVideoPipeline":
|
||||
from diffusers import SanaImageToVideoPipeline
|
||||
|
||||
return SanaImageToVideoPipeline
|
||||
return configured_pipeline_class
|
||||
|
||||
def update_call_kwargs(
|
||||
self,
|
||||
req: DiffusionRequestBatch,
|
||||
sampling: OmniDiffusionSamplingParams,
|
||||
accepted_call_kwargs: set[str] | None,
|
||||
call_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
# Diffusers SANA-Video calls the shared request's `num_frames`
|
||||
# parameter `frames`.
|
||||
if (
|
||||
accepted_call_kwargs is not None
|
||||
and "frames" in accepted_call_kwargs
|
||||
and "num_frames" not in accepted_call_kwargs
|
||||
):
|
||||
configured_default = call_kwargs.get("frames", 81)
|
||||
if configured_default is None:
|
||||
configured_default = 81
|
||||
call_kwargs["frames"] = resolve_video_num_frames(
|
||||
sampling.num_frames,
|
||||
default_num_frames=configured_default,
|
||||
is_dummy_run=req.is_dummy_run(),
|
||||
)
|
||||
|
||||
# SANA's startup dummy dimensions can map to a resolution bucket that
|
||||
# conflicts with the upstream pipeline's divisibility validation.
|
||||
if req.is_dummy_run() and accepted_call_kwargs is not None and "use_resolution_binning" in accepted_call_kwargs:
|
||||
call_kwargs["use_resolution_binning"] = False
|
||||
|
||||
|
||||
class WanPipelineUtils(BasePipelineUtils):
|
||||
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
|
||||
@@ -49,6 +112,8 @@ class WanPipelineUtils(BasePipelineUtils):
|
||||
|
||||
|
||||
PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
|
||||
"SanaVideoPipeline": SanaVideoPipelineUtils,
|
||||
"SanaImageToVideoPipeline": SanaVideoPipelineUtils,
|
||||
"WanPipeline": WanPipelineUtils,
|
||||
"WanImageToVideoPipeline": WanPipelineUtils,
|
||||
"WanVACEPipeline": WanPipelineUtils,
|
||||
@@ -62,3 +127,20 @@ def get_pipeline_utils(pipeline_class_name: str | None) -> BasePipelineUtils:
|
||||
return BasePipelineUtils()
|
||||
pipeline_utils_cls = PIPELINE_UTILS_REGISTRY.get(pipeline_class_name, BasePipelineUtils)
|
||||
return pipeline_utils_cls()
|
||||
|
||||
|
||||
def get_pipeline_utils_for_config(od_config: OmniDiffusionConfig) -> BasePipelineUtils:
|
||||
"""Return model-specific adapter hooks for the requested or configured class."""
|
||||
requested_class_name = od_config.model_class_name
|
||||
if requested_class_name in PIPELINE_UTILS_REGISTRY:
|
||||
return get_pipeline_utils(requested_class_name)
|
||||
|
||||
configured_pipeline_class = od_config.diffusers_pipeline_cls
|
||||
configured_class_name = configured_pipeline_class.__name__ if configured_pipeline_class is not None else None
|
||||
return get_pipeline_utils(configured_class_name)
|
||||
|
||||
|
||||
def resolve_diffusers_pipeline_class(od_config: OmniDiffusionConfig) -> type[DiffusionPipeline] | None:
|
||||
"""Resolve the concrete Diffusers pipeline used for loading and capability checks."""
|
||||
pipeline_utils = get_pipeline_utils_for_config(od_config)
|
||||
return pipeline_utils.resolve_pipeline_class(od_config, od_config.diffusers_pipeline_cls)
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright 2025 SANA-Video Authors and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from .pipeline_output import SanaVideoPipelineOutput
|
||||
from .pipeline_sana_video import SanaVideoPipeline, get_sana_video_post_process_func
|
||||
from .pipeline_sana_video_i2v import (
|
||||
SanaImageToVideoPipeline,
|
||||
get_sana_video_i2v_post_process_func,
|
||||
get_sana_video_i2v_pre_process_func,
|
||||
)
|
||||
from .transformer_sana_video import SanaVideoTransformer3DModel
|
||||
|
||||
__all__ = [
|
||||
"SanaImageToVideoPipeline",
|
||||
"SanaVideoPipeline",
|
||||
"SanaVideoPipelineOutput",
|
||||
"SanaVideoTransformer3DModel",
|
||||
"get_sana_video_i2v_post_process_func",
|
||||
"get_sana_video_i2v_pre_process_func",
|
||||
"get_sana_video_post_process_func",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from diffusers.utils import BaseOutput
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideoPipelineOutput(BaseOutput):
|
||||
r"""
|
||||
Output class for Sana-Video pipelines.
|
||||
|
||||
Args:
|
||||
frames (`torch.Tensor`, `np.ndarray`, or list[list[PIL.Image.Image]]):
|
||||
List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing
|
||||
denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape
|
||||
`(batch_size, num_frames, channels, height, width)`.
|
||||
"""
|
||||
|
||||
frames: torch.Tensor
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,382 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import warnings
|
||||
from math import sqrt
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
|
||||
DistributedAutoencoderKLLTX2Video,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
|
||||
ASPECT_RATIO_480_BIN,
|
||||
ASPECT_RATIO_720_BIN,
|
||||
SanaVideoPipeline,
|
||||
get_sana_video_default_resolution,
|
||||
get_sana_video_post_process_func,
|
||||
resolve_sana_video_sample_size,
|
||||
retrieve_timesteps,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest, resolve_video_num_frames
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniTextPrompt
|
||||
|
||||
SANA_VIDEO_I2V_COMPLEX_HUMAN_INSTRUCTION = [
|
||||
(
|
||||
"Given a user prompt, generate an 'Enhanced prompt' that provides detailed visual descriptions suitable for "
|
||||
"video generation. Evaluate the level of detail in the user prompt:"
|
||||
),
|
||||
(
|
||||
"- If the prompt is simple, focus on adding specifics about colors, shapes, sizes, textures, motion, and "
|
||||
"temporal relationships to create vivid and dynamic scenes."
|
||||
),
|
||||
("- If the prompt is already detailed, refine and enhance the existing details slightly without overcomplicating."),
|
||||
"Here are examples of how to transform or refine prompts:",
|
||||
(
|
||||
"- User Prompt: A cat sleeping -> Enhanced: A small, fluffy white cat slowly settling into a curled position, "
|
||||
"peacefully falling asleep on a warm sunny windowsill, with gentle sunlight filtering through surrounding "
|
||||
"pots of blooming red flowers."
|
||||
),
|
||||
(
|
||||
"- User Prompt: A busy city street -> Enhanced: A bustling city street scene at dusk, featuring glowing street "
|
||||
"lamps gradually lighting up, a diverse crowd of people in colorful clothing walking past, and a double-decker "
|
||||
"bus smoothly passing by towering glass skyscrapers."
|
||||
),
|
||||
(
|
||||
"Please generate only the enhanced description for the prompt below and avoid including any additional "
|
||||
"commentary or evaluations:"
|
||||
),
|
||||
"User Prompt: ",
|
||||
]
|
||||
|
||||
|
||||
def get_sana_video_i2v_post_process_func(od_config: OmniDiffusionConfig):
|
||||
return get_sana_video_post_process_func(od_config)
|
||||
|
||||
|
||||
def get_sana_video_i2v_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
default_height, default_width = get_sana_video_default_resolution(resolve_sana_video_sample_size(od_config))
|
||||
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
prompt = request.prompt
|
||||
if isinstance(prompt, str):
|
||||
prompt = OmniTextPrompt(prompt=prompt, multi_modal_data={})
|
||||
|
||||
multi_modal_data = prompt.get("multi_modal_data") or {}
|
||||
raw_image = multi_modal_data.get("image")
|
||||
if raw_image is None:
|
||||
raise ValueError('SANA-Video I2V requires `"multi_modal_data": {"image": <PIL image or file path>}`.')
|
||||
if isinstance(raw_image, str):
|
||||
if not os.path.isfile(raw_image):
|
||||
raise ValueError(f"SANA-Video I2V image path does not exist: {raw_image}")
|
||||
image = PIL.Image.open(raw_image).convert("RGB")
|
||||
elif isinstance(raw_image, PIL.Image.Image):
|
||||
image = raw_image.convert("RGB")
|
||||
else:
|
||||
raise TypeError(f"Unsupported SANA-Video I2V image type: {type(raw_image).__name__}")
|
||||
|
||||
if request.sampling_params.height is None or request.sampling_params.width is None:
|
||||
max_area = default_height * default_width
|
||||
aspect_ratio = image.height / image.width
|
||||
inferred_height = round(sqrt(max_area * aspect_ratio)) // 32 * 32
|
||||
inferred_width = round(sqrt(max_area / aspect_ratio)) // 32 * 32
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = inferred_height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = inferred_width
|
||||
|
||||
image = image.resize(
|
||||
(request.sampling_params.width, request.sampling_params.height),
|
||||
PIL.Image.Resampling.LANCZOS,
|
||||
)
|
||||
multi_modal_data["image"] = image
|
||||
prompt["multi_modal_data"] = multi_modal_data
|
||||
request.prompt = prompt
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
|
||||
|
||||
def _retrieve_latents(encoder_output: torch.Tensor) -> torch.Tensor:
|
||||
if hasattr(encoder_output, "latent_dist"):
|
||||
return encoder_output.latent_dist.mode()
|
||||
if hasattr(encoder_output, "latents"):
|
||||
return encoder_output.latents
|
||||
raise AttributeError("Could not access latents from the SANA-Video VAE encoder output.")
|
||||
|
||||
|
||||
class SanaImageToVideoPipeline(SanaVideoPipeline, SupportImageInput):
|
||||
"""Native vLLM-Omni SANA-Video 2B image-to-video pipeline.
|
||||
|
||||
The first latent frame is encoded from the input image and remains fixed;
|
||||
denoising updates only the subsequent video frames.
|
||||
"""
|
||||
|
||||
def _prepare_i2v_latents(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
batch_size: int,
|
||||
num_channels_latents: int,
|
||||
height: int,
|
||||
width: int,
|
||||
num_frames: int,
|
||||
dtype: torch.dtype,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
shape = (
|
||||
batch_size,
|
||||
num_channels_latents,
|
||||
num_latent_frames,
|
||||
height // self.vae_scale_factor_spatial,
|
||||
width // self.vae_scale_factor_spatial,
|
||||
)
|
||||
if isinstance(generator, list) and len(generator) != batch_size:
|
||||
raise ValueError(
|
||||
f"Generator list length {len(generator)} does not match effective batch size {batch_size}."
|
||||
)
|
||||
if latents is None:
|
||||
latents = randn_tensor(shape, generator=generator, device=self.device, dtype=dtype)
|
||||
else:
|
||||
latents = latents.to(device=self.device, dtype=dtype)
|
||||
|
||||
image = image.unsqueeze(2).to(device=self.device, dtype=self.vae.dtype)
|
||||
image_latents = _retrieve_latents(self.vae.encode(image))
|
||||
image_latents = image_latents.repeat(batch_size, 1, 1, 1, 1)
|
||||
|
||||
if isinstance(self.vae, DistributedAutoencoderKLLTX2Video):
|
||||
latents_mean = self.vae.latents_mean
|
||||
latents_std = self.vae.latents_std
|
||||
elif isinstance(self.vae, DistributedAutoencoderKLWan):
|
||||
latents_mean = torch.tensor(self.vae.config.latents_mean)
|
||||
latents_std = torch.tensor(self.vae.config.latents_std)
|
||||
else:
|
||||
raise TypeError(f"Unsupported SANA-Video I2V VAE type: {type(self.vae).__name__}")
|
||||
|
||||
latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(image_latents.device, image_latents.dtype)
|
||||
latents_std = latents_std.view(1, -1, 1, 1, 1).to(image_latents.device, image_latents.dtype)
|
||||
image_latents = (image_latents - latents_mean) / latents_std
|
||||
latents[:, :, :1] = image_latents.to(dtype)
|
||||
return latents
|
||||
|
||||
def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:
|
||||
if len(req.prompts) != 1:
|
||||
raise ValueError("SANA-Video I2V currently supports exactly one prompt per request.")
|
||||
prompt_obj = req.prompts[0]
|
||||
if isinstance(prompt_obj, str):
|
||||
raise ValueError("SANA-Video I2V requires an image in `multi_modal_data`.")
|
||||
|
||||
prompt = prompt_obj.get("prompt") or ""
|
||||
negative_prompt = prompt_obj.get("negative_prompt") or ""
|
||||
image = (prompt_obj.get("multi_modal_data") or {}).get("image")
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required for SANA-Video I2V generation.")
|
||||
if not isinstance(image, PIL.Image.Image):
|
||||
raise TypeError("SANA-Video I2V expects a preprocessed PIL image.")
|
||||
sampling = req.sampling_params
|
||||
default_height, default_width = get_sana_video_default_resolution(self.transformer.config.sample_size)
|
||||
height = sampling.height or default_height
|
||||
width = sampling.width or default_width
|
||||
frames = resolve_video_num_frames(
|
||||
sampling.num_frames,
|
||||
default_num_frames=81,
|
||||
is_dummy_run=req.is_dummy_run(),
|
||||
)
|
||||
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
|
||||
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
|
||||
generator = sampling.generator
|
||||
if generator is None and sampling.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(sampling.seed)
|
||||
|
||||
extra_args = sampling.extra_args or {}
|
||||
motion_score = extra_args.get("motion_score")
|
||||
if motion_score is not None:
|
||||
prompt = f"{prompt} motion score: {int(motion_score)}."
|
||||
|
||||
video = self._generate_i2v(
|
||||
image=image,
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
height=height,
|
||||
width=width,
|
||||
frames=frames,
|
||||
num_inference_steps=num_steps,
|
||||
timesteps=sampling.timesteps,
|
||||
sigmas=sampling.sigmas,
|
||||
guidance_scale=guidance_scale,
|
||||
num_videos_per_prompt=sampling.num_outputs_per_prompt or 1,
|
||||
eta=sampling.eta,
|
||||
generator=generator,
|
||||
latents=sampling.latents,
|
||||
clean_caption=bool(extra_args.get("clean_caption", False)),
|
||||
use_resolution_binning=bool(extra_args.get("use_resolution_binning", True)),
|
||||
max_sequence_length=sampling.max_sequence_length or 300,
|
||||
output_type="latent" if sampling.output_type == "latent" else "raw",
|
||||
)
|
||||
return DiffusionOutput(output=video)
|
||||
|
||||
@torch.no_grad()
|
||||
def _generate_i2v(
|
||||
self,
|
||||
*,
|
||||
image: PIL.Image.Image,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
height: int,
|
||||
width: int,
|
||||
frames: int,
|
||||
num_inference_steps: int,
|
||||
guidance_scale: float,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None,
|
||||
use_resolution_binning: bool,
|
||||
max_sequence_length: int,
|
||||
timesteps: list[int] | torch.Tensor | None = None,
|
||||
sigmas: list[float] | None = None,
|
||||
num_videos_per_prompt: int = 1,
|
||||
eta: float = 0.0,
|
||||
clean_caption: bool = False,
|
||||
output_type: str = "raw",
|
||||
complex_human_instruction: list[str] = SANA_VIDEO_I2V_COMPLEX_HUMAN_INSTRUCTION,
|
||||
) -> torch.Tensor:
|
||||
orig_height, orig_width = height, width
|
||||
if use_resolution_binning:
|
||||
if self.transformer.config.sample_size == 30:
|
||||
aspect_ratio_bin = ASPECT_RATIO_480_BIN
|
||||
elif self.transformer.config.sample_size == 22:
|
||||
aspect_ratio_bin = ASPECT_RATIO_720_BIN
|
||||
else:
|
||||
raise ValueError(f"Unsupported SANA-Video sample size: {self.transformer.config.sample_size}")
|
||||
height, width = self.video_processor.classify_height_width_bin(
|
||||
height,
|
||||
width,
|
||||
ratios=aspect_ratio_bin,
|
||||
)
|
||||
|
||||
self.check_inputs(
|
||||
prompt=prompt,
|
||||
height=height,
|
||||
width=width,
|
||||
negative_prompt=negative_prompt,
|
||||
)
|
||||
self._guidance_scale = guidance_scale
|
||||
self._interrupt = False
|
||||
|
||||
prompt_embeds, prompt_mask, negative_embeds, negative_mask = self.encode_prompt(
|
||||
prompt,
|
||||
self.do_classifier_free_guidance,
|
||||
negative_prompt=negative_prompt,
|
||||
num_videos_per_prompt=num_videos_per_prompt,
|
||||
device=self.device,
|
||||
clean_caption=clean_caption,
|
||||
max_sequence_length=max_sequence_length,
|
||||
complex_human_instruction=complex_human_instruction,
|
||||
)
|
||||
if self.do_classifier_free_guidance:
|
||||
prompt_embeds = torch.cat([negative_embeds, prompt_embeds])
|
||||
prompt_mask = torch.cat([negative_mask, prompt_mask])
|
||||
|
||||
timesteps, num_inference_steps = retrieve_timesteps(
|
||||
self.scheduler,
|
||||
num_inference_steps=num_inference_steps,
|
||||
device=self.device,
|
||||
timesteps=timesteps,
|
||||
sigmas=sigmas,
|
||||
)
|
||||
image_tensor = self.video_processor.preprocess(image, height=height, width=width).to(
|
||||
self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
latent_channels = self.transformer.config.in_channels
|
||||
latents = self._prepare_i2v_latents(
|
||||
image_tensor,
|
||||
batch_size=num_videos_per_prompt,
|
||||
num_channels_latents=latent_channels,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=frames,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
latents=latents,
|
||||
)
|
||||
|
||||
patch_t, patch_h, patch_w = self.transformer.config.patch_size
|
||||
conditioning_mask = latents.new_zeros(
|
||||
num_videos_per_prompt,
|
||||
1,
|
||||
latents.shape[2] // patch_t,
|
||||
latents.shape[3] // patch_h,
|
||||
latents.shape[4] // patch_w,
|
||||
)
|
||||
conditioning_mask[:, :, 0] = 1
|
||||
if self.do_classifier_free_guidance:
|
||||
conditioning_mask = torch.cat([conditioning_mask, conditioning_mask])
|
||||
|
||||
self._num_timesteps = len(timesteps)
|
||||
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
|
||||
transformer_dtype = self.transformer.dtype
|
||||
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
||||
for index, timestep_value in enumerate(timesteps):
|
||||
latent_input = torch.cat([latents, latents]) if self.do_classifier_free_guidance else latents
|
||||
timestep = timestep_value.expand(conditioning_mask.shape) * (1 - conditioning_mask)
|
||||
noise_pred = self.transformer(
|
||||
latent_input.to(transformer_dtype),
|
||||
encoder_hidden_states=prompt_embeds.to(transformer_dtype),
|
||||
encoder_attention_mask=prompt_mask,
|
||||
timestep=timestep,
|
||||
return_dict=False,
|
||||
)[0].float()
|
||||
if self.do_classifier_free_guidance:
|
||||
uncond, cond = noise_pred.chunk(2)
|
||||
noise_pred = uncond + guidance_scale * (cond - uncond)
|
||||
if self.transformer.config.out_channels // 2 == latent_channels:
|
||||
noise_pred = noise_pred.chunk(2, dim=1)[0]
|
||||
|
||||
denoised = self.scheduler.step(
|
||||
noise_pred[:, :, 1:],
|
||||
timestep_value,
|
||||
latents[:, :, 1:],
|
||||
**extra_step_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
latents = torch.cat([latents[:, :, :1], denoised], dim=2)
|
||||
if index == len(timesteps) - 1 or (index + 1) % self.scheduler.order == 0:
|
||||
progress_bar.update()
|
||||
|
||||
if output_type == "latent":
|
||||
return latents
|
||||
|
||||
latents = latents.to(self.vae.dtype)
|
||||
if isinstance(self.vae, DistributedAutoencoderKLLTX2Video):
|
||||
latents_mean = self.vae.latents_mean
|
||||
latents_std = self.vae.latents_std
|
||||
latent_dim = self.vae.config.latent_channels
|
||||
else:
|
||||
latents_mean = torch.tensor(self.vae.config.latents_mean)
|
||||
latents_std = torch.tensor(self.vae.config.latents_std)
|
||||
latent_dim = self.vae.config.z_dim
|
||||
latents_mean = latents_mean.view(1, latent_dim, 1, 1, 1).to(latents.device, latents.dtype)
|
||||
latents_std = latents_std.view(1, latent_dim, 1, 1, 1).to(latents.device, latents.dtype)
|
||||
latents = latents * latents_std + latents_mean
|
||||
|
||||
try:
|
||||
video = self.vae.decode(latents, return_dict=False)[0]
|
||||
except torch.OutOfMemoryError as error:
|
||||
warnings.warn(
|
||||
f"{error}. Enable VAE tiling for SANA-Video I2V high-resolution decoding.",
|
||||
stacklevel=2,
|
||||
)
|
||||
raise
|
||||
if use_resolution_binning:
|
||||
video = self.video_processor.resize_and_crop_tensor(video, orig_width, orig_height)
|
||||
return video
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user