[Model] Add Cache-DiT and CPU offload support for SANA-Video 2B (#5882)
Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
10 files changed
+558
-26
No files matched your search
@@ -272,6 +272,39 @@ def retrieve_timesteps(
|
||||
return timesteps, num_inference_steps
|
||||
|
||||
|
||||
def _validate_cache_offload_parallelism(od_config: OmniDiffusionConfig) -> None:
|
||||
cache_backend = od_config.cache_backend
|
||||
if cache_backend not in ("none", "cache_dit"):
|
||||
raise NotImplementedError(
|
||||
f"Cache backend {cache_backend!r} is not supported by the native SANA-Video pipeline; "
|
||||
"use 'cache_dit' or 'none'."
|
||||
)
|
||||
if cache_backend == "cache_dit" and od_config.enable_distributed_layerwise_offload:
|
||||
raise NotImplementedError(
|
||||
"SANA-Video does not support Cache-DiT with distributed layerwise offload; "
|
||||
"per-rank cache skips desynchronize the weight AllGather."
|
||||
)
|
||||
|
||||
offload_enabled = (
|
||||
od_config.enable_cpu_offload
|
||||
or od_config.enable_layerwise_offload
|
||||
or od_config.enable_distributed_layerwise_offload
|
||||
)
|
||||
if cache_backend == "none" and not offload_enabled:
|
||||
return
|
||||
|
||||
parallel_config = od_config.parallel_config
|
||||
for name, size in (
|
||||
("tensor_parallel_size", parallel_config.tensor_parallel_size),
|
||||
("cfg_parallel_size", parallel_config.cfg_parallel_size),
|
||||
("sequence_parallel_size", parallel_config.sequence_parallel_size),
|
||||
):
|
||||
if size > 1:
|
||||
raise NotImplementedError(
|
||||
f"SANA-Video cache/offload is currently supported only with TP1, CFG1, and SP1; got {name}={size}."
|
||||
)
|
||||
|
||||
|
||||
class SanaVideoPipeline(
|
||||
nn.Module,
|
||||
CFGParallelMixin,
|
||||
@@ -300,6 +333,7 @@ class SanaVideoPipeline(
|
||||
_encoder_modules = ["text_encoder"]
|
||||
_vae_modules = ["vae"]
|
||||
supports_step_execution = False
|
||||
default_num_inference_steps = 50
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -321,6 +355,7 @@ class SanaVideoPipeline(
|
||||
self.device = get_local_device()
|
||||
self.weights_sources: list[DiffusersPipelineLoader.ComponentSource] = []
|
||||
if od_config is not None:
|
||||
_validate_cache_offload_parallelism(od_config)
|
||||
tokenizer, text_encoder, vae, transformer, scheduler = self._load_components(od_config, prefix)
|
||||
|
||||
if tokenizer is None or text_encoder is None or vae is None or transformer is None or scheduler is None:
|
||||
@@ -368,7 +403,9 @@ class SanaVideoPipeline(
|
||||
model = od_config.model
|
||||
local_files_only = os.path.exists(model)
|
||||
dtype = getattr(od_config, "dtype", torch.bfloat16)
|
||||
device = self.device
|
||||
# The loader loads components under a default-device context that is CPU
|
||||
# when offload is enabled; runtime placement stays with the offloader.
|
||||
component_load_device = torch.get_default_device()
|
||||
# Transformer weights are streamed by DiffusersPipelineLoader below.
|
||||
# Prefetch only components loaded through from_pretrained here.
|
||||
component_subfolders = ["tokenizer", "text_encoder", "vae", "scheduler"]
|
||||
@@ -386,7 +423,7 @@ class SanaVideoPipeline(
|
||||
prefetch_list=component_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(device)
|
||||
).to(component_load_device)
|
||||
|
||||
model_index = _load_json(model, "model_index.json", local_files_only)
|
||||
vae_class_name = model_index.get("vae", [None, "AutoencoderKLWan"])[1]
|
||||
@@ -401,10 +438,13 @@ class SanaVideoPipeline(
|
||||
prefetch_list=component_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=vae_dtype,
|
||||
).to(device)
|
||||
).to(component_load_device)
|
||||
|
||||
transformer_config = _load_json(model, "transformer/config.json", local_files_only)
|
||||
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(dtype=dtype, device=device)
|
||||
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(
|
||||
dtype=dtype,
|
||||
device=component_load_device,
|
||||
)
|
||||
scheduler = DPMSolverMultistepScheduler.from_pretrained(
|
||||
model,
|
||||
subfolder="scheduler",
|
||||
@@ -859,7 +899,11 @@ class SanaVideoPipeline(
|
||||
default_num_frames=81,
|
||||
is_dummy_run=req.is_dummy_run(),
|
||||
)
|
||||
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
|
||||
num_steps = (
|
||||
sampling.num_inference_steps
|
||||
if sampling.num_inference_steps is not None
|
||||
else self.default_num_inference_steps
|
||||
)
|
||||
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
|
||||
generator = sampling.generator
|
||||
if generator is None and sampling.seed is not None:
|
||||
|
||||
@@ -196,7 +196,11 @@ class SanaImageToVideoPipeline(SanaVideoPipeline, SupportImageInput):
|
||||
default_num_frames=81,
|
||||
is_dummy_run=req.is_dummy_run(),
|
||||
)
|
||||
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
|
||||
num_steps = (
|
||||
sampling.num_inference_steps
|
||||
if sampling.num_inference_steps is not None
|
||||
else self.default_num_inference_steps
|
||||
)
|
||||
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
|
||||
generator = sampling.generator
|
||||
if generator is None and sampling.seed is not None:
|
||||
|
||||
@@ -18,6 +18,7 @@ from collections.abc import Iterable
|
||||
from dataclasses import dataclass, fields
|
||||
|
||||
import torch
|
||||
from cache_dit import ForwardPattern
|
||||
from torch import nn
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -30,6 +31,7 @@ from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention as OmniAttention
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
|
||||
|
||||
def validate_sana_video_parallel_config(parallel_config) -> None:
|
||||
@@ -962,6 +964,12 @@ class SanaVideoTransformer3DModel(nn.Module):
|
||||
"""
|
||||
|
||||
_no_split_modules = ["SanaVideoTransformerBlock", "SanaModulatedNorm"]
|
||||
_cache_dit_adapter_config = CacheDiTAdapterConfig(
|
||||
block_forward_patterns={
|
||||
"transformer_blocks": ForwardPattern.Pattern_3,
|
||||
},
|
||||
)
|
||||
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return AutoWeightsLoader(self).load_weights(weights)
|
||||
|
||||
Reference in new issue
Block a user