[Model] Add Cache-DiT and CPU offload support for SANA-Video 2B (#5882)

Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-09-02 23:40:48 +08:00
1 parent 95a9311696
commit 3b6ce98421
10 files changed
+558 -26

No files matched your search

@@ -272,6 +272,39 @@ def retrieve_timesteps(
return timesteps, num_inference_steps
def _validate_cache_offload_parallelism(od_config: OmniDiffusionConfig) -> None:
cache_backend = od_config.cache_backend
if cache_backend not in ("none", "cache_dit"):
raise NotImplementedError(
f"Cache backend {cache_backend!r} is not supported by the native SANA-Video pipeline; "
"use 'cache_dit' or 'none'."
)
if cache_backend == "cache_dit" and od_config.enable_distributed_layerwise_offload:
raise NotImplementedError(
"SANA-Video does not support Cache-DiT with distributed layerwise offload; "
"per-rank cache skips desynchronize the weight AllGather."
)
offload_enabled = (
od_config.enable_cpu_offload
or od_config.enable_layerwise_offload
or od_config.enable_distributed_layerwise_offload
)
if cache_backend == "none" and not offload_enabled:
return
parallel_config = od_config.parallel_config
for name, size in (
("tensor_parallel_size", parallel_config.tensor_parallel_size),
("cfg_parallel_size", parallel_config.cfg_parallel_size),
("sequence_parallel_size", parallel_config.sequence_parallel_size),
):
if size > 1:
raise NotImplementedError(
f"SANA-Video cache/offload is currently supported only with TP1, CFG1, and SP1; got {name}={size}."
)
class SanaVideoPipeline(
nn.Module,
CFGParallelMixin,
@@ -300,6 +333,7 @@ class SanaVideoPipeline(
_encoder_modules = ["text_encoder"]
_vae_modules = ["vae"]
supports_step_execution = False
default_num_inference_steps = 50
def __init__(
self,
@@ -321,6 +355,7 @@ class SanaVideoPipeline(
self.device = get_local_device()
self.weights_sources: list[DiffusersPipelineLoader.ComponentSource] = []
if od_config is not None:
_validate_cache_offload_parallelism(od_config)
tokenizer, text_encoder, vae, transformer, scheduler = self._load_components(od_config, prefix)
if tokenizer is None or text_encoder is None or vae is None or transformer is None or scheduler is None:
@@ -368,7 +403,9 @@ class SanaVideoPipeline(
model = od_config.model
local_files_only = os.path.exists(model)
dtype = getattr(od_config, "dtype", torch.bfloat16)
device = self.device
# The loader loads components under a default-device context that is CPU
# when offload is enabled; runtime placement stays with the offloader.
component_load_device = torch.get_default_device()
# Transformer weights are streamed by DiffusersPipelineLoader below.
# Prefetch only components loaded through from_pretrained here.
component_subfolders = ["tokenizer", "text_encoder", "vae", "scheduler"]
@@ -386,7 +423,7 @@ class SanaVideoPipeline(
prefetch_list=component_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(device)
).to(component_load_device)
model_index = _load_json(model, "model_index.json", local_files_only)
vae_class_name = model_index.get("vae", [None, "AutoencoderKLWan"])[1]
@@ -401,10 +438,13 @@ class SanaVideoPipeline(
prefetch_list=component_subfolders,
local_files_only=local_files_only,
torch_dtype=vae_dtype,
).to(device)
).to(component_load_device)
transformer_config = _load_json(model, "transformer/config.json", local_files_only)
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(dtype=dtype, device=device)
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(
dtype=dtype,
device=component_load_device,
)
scheduler = DPMSolverMultistepScheduler.from_pretrained(
model,
subfolder="scheduler",
@@ -859,7 +899,11 @@ class SanaVideoPipeline(
default_num_frames=81,
is_dummy_run=req.is_dummy_run(),
)
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
num_steps = (
sampling.num_inference_steps
if sampling.num_inference_steps is not None
else self.default_num_inference_steps
)
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
generator = sampling.generator
if generator is None and sampling.seed is not None:
@@ -196,7 +196,11 @@ class SanaImageToVideoPipeline(SanaVideoPipeline, SupportImageInput):
default_num_frames=81,
is_dummy_run=req.is_dummy_run(),
)
num_steps = sampling.num_inference_steps if sampling.num_inference_steps is not None else 50
num_steps = (
sampling.num_inference_steps
if sampling.num_inference_steps is not None
else self.default_num_inference_steps
)
guidance_scale = sampling.guidance_scale if sampling.guidance_scale_provided else 6.0
generator = sampling.generator
if generator is None and sampling.seed is not None:
@@ -18,6 +18,7 @@ from collections.abc import Iterable
from dataclasses import dataclass, fields
import torch
from cache_dit import ForwardPattern
from torch import nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
@@ -30,6 +31,7 @@ from vllm.model_executor.utils import set_weight_attrs
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention as OmniAttention
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
def validate_sana_video_parallel_config(parallel_config) -> None:
@@ -962,6 +964,12 @@ class SanaVideoTransformer3DModel(nn.Module):
"""
_no_split_modules = ["SanaVideoTransformerBlock", "SanaModulatedNorm"]
_cache_dit_adapter_config = CacheDiTAdapterConfig(
block_forward_patterns={
"transformer_blocks": ForwardPattern.Pattern_3,
},
)
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
return AutoWeightsLoader(self).load_weights(weights)