[Model] Add native SANA-Video 2B T2V and I2V support (#5508)

Signed-off-by: leo <92263211+napleon-liu@users.noreply.github.com>
Signed-off-by: leo <92263211+liuyao0322@users.noreply.github.com>
Signed-off-by: liuyao0322 <yaoliu548926@gmail.com>
Co-authored-by: leo <92263211+napleon-liu@users.noreply.github.com>
This commit is contained in:
leoandleo authored and GitHub committed 2026-08-25 09:46:02 +08:00
1 parent 4af0ddc1c2
commit 50b8a150f9
42 files changed
+6512 -18

No files matched your search

@@ -0,0 +1,992 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import nullcontext
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from PIL import Image
from torch import nn
from vllm_omni.diffusion.request import DUMMY_DIFFUSION_REQUEST_ID, OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _DummyComponent:
pass
def _make_request_batch(prompt, request_id="sana-video-test", **sampling_overrides):
sampling = OmniDiffusionSamplingParams(**sampling_overrides)
request = OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id=request_id)
return DiffusionRequestBatch([request])
def test_sana_video_pipeline_import_and_registry():
from vllm_omni.diffusion.models.sana_video import (
SanaImageToVideoPipeline,
SanaVideoPipeline,
SanaVideoTransformer3DModel,
get_sana_video_i2v_post_process_func,
get_sana_video_i2v_pre_process_func,
get_sana_video_post_process_func,
)
from vllm_omni.diffusion.registry import (
_DIFFUSION_MODELS,
_DIFFUSION_POST_PROCESS_FUNCS,
_DIFFUSION_PRE_PROCESS_FUNCS,
)
assert SanaImageToVideoPipeline is not None
assert SanaVideoPipeline is not None
assert SanaVideoTransformer3DModel is not None
assert get_sana_video_i2v_post_process_func is not None
assert get_sana_video_i2v_pre_process_func is not None
assert get_sana_video_post_process_func is not None
assert _DIFFUSION_MODELS["SanaVideoPipeline"] == (
"sana_video",
"pipeline_sana_video",
"SanaVideoPipeline",
)
assert _DIFFUSION_POST_PROCESS_FUNCS["SanaVideoPipeline"] == "get_sana_video_post_process_func"
assert _DIFFUSION_MODELS["SanaImageToVideoPipeline"] == (
"sana_video",
"pipeline_sana_video_i2v",
"SanaImageToVideoPipeline",
)
assert _DIFFUSION_POST_PROCESS_FUNCS["SanaImageToVideoPipeline"] == "get_sana_video_i2v_post_process_func"
assert _DIFFUSION_PRE_PROCESS_FUNCS["SanaImageToVideoPipeline"] == "get_sana_video_i2v_pre_process_func"
def test_component_discovery_declarations():
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
assert SanaVideoPipeline._dit_modules == ["transformer"]
assert SanaVideoPipeline._encoder_modules == ["text_encoder"]
assert SanaVideoPipeline._vae_modules == ["vae"]
assert SanaVideoPipeline.supports_step_execution is False
def test_sana_video_loads_concrete_gemma_tokenizer(monkeypatch):
from vllm_omni.diffusion.models.sana_video import pipeline_sana_video
captured: dict[str, object] = {}
expected = object()
def fake_from_pretrained_with_prefetch(loader, model, **kwargs):
captured.update(loader=loader, model=model, kwargs=kwargs)
return expected
monkeypatch.setattr(
pipeline_sana_video,
"from_pretrained_with_prefetch",
fake_from_pretrained_with_prefetch,
)
result = pipeline_sana_video._load_sana_tokenizer(
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
["tokenizer", "text_encoder"],
local_files_only=False,
)
assert result is expected
assert captured["loader"] == pipeline_sana_video.GemmaTokenizer.from_pretrained
assert captured["model"] == "Efficient-Large-Model/SANA-Video_2B_480p_diffusers"
assert captured["kwargs"] == {
"subfolder": "tokenizer",
"prefetch_list": ["tokenizer", "text_encoder"],
"local_files_only": False,
}
def test_sana_video_declares_extra_body_params():
from vllm_omni.model_extras import get_extra_body_params
assert get_extra_body_params("SanaVideoPipeline") == {
"clean_caption",
"motion_score",
"use_resolution_binning",
}
assert get_extra_body_params("SanaImageToVideoPipeline") == {
"clean_caption",
"motion_score",
"use_resolution_binning",
}
def test_sana_video_i2v_preprocesses_image_and_preserves_aspect_ratio():
from vllm_omni.diffusion.models.sana_video import get_sana_video_i2v_pre_process_func
request = OmniDiffusionRequest(
prompt={"prompt": "a robot walks", "multi_modal_data": {"image": Image.new("RGB", (640, 360))}},
sampling_params=OmniDiffusionSamplingParams(),
request_id="sana-video-i2v-preprocess",
)
processed = get_sana_video_i2v_pre_process_func(SimpleNamespace())(request)
assert processed.sampling_params.height == 448
assert processed.sampling_params.width == 832
assert processed.prompt["multi_modal_data"]["image"].size == (832, 448)
def test_sana_video_i2v_remote_720p_uses_loaded_transformer_config():
from vllm_omni.diffusion.models.sana_video import get_sana_video_i2v_pre_process_func
od_config = SimpleNamespace(
model="Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
tf_model_config={"sample_size": 22},
)
request = OmniDiffusionRequest(
prompt={"prompt": "a robot walks", "multi_modal_data": {"image": Image.new("RGB", (1280, 704))}},
sampling_params=OmniDiffusionSamplingParams(),
request_id="sana-video-i2v-remote-720p",
)
processed = get_sana_video_i2v_pre_process_func(od_config)(request)
assert processed.sampling_params.height == 704
assert processed.sampling_params.width == 1280
assert processed.prompt["multi_modal_data"]["image"].size == (1280, 704)
@pytest.mark.parametrize(
"frames",
[
[[Image.new("RGB", (16, 16)) for _ in range(2)]],
np.zeros((1, 2, 16, 16, 3), dtype=np.float32),
[[np.zeros((16, 16, 3), dtype=np.float32) for _ in range(2)]],
],
)
@pytest.mark.parametrize(
"factory_name",
["get_sana_video_post_process_func", "get_sana_video_i2v_post_process_func"],
)
def test_sana_video_diffusers_preserves_postprocessed_frames(frames, factory_name):
from vllm_omni.diffusion.models import sana_video
post_process = getattr(sana_video, factory_name)(SimpleNamespace(diffusion_load_format="diffusers"))
result = post_process(frames)
assert result["payload"]["video"] is frames
@pytest.mark.parametrize(
"factory_name",
["get_sana_video_post_process_func", "get_sana_video_i2v_post_process_func"],
)
def test_sana_video_diffusers_still_postprocesses_native_tensor(factory_name):
from vllm_omni.diffusion.models import sana_video
decoded_video = torch.zeros(1, 3, 2, 16, 16)
post_process = getattr(sana_video, factory_name)(SimpleNamespace(diffusion_load_format="diffusers"))
result = post_process(decoded_video)
assert isinstance(result["payload"]["video"], np.ndarray)
assert result["payload"]["video"].shape == (1, 2, 16, 16, 3)
@pytest.mark.parametrize(
"factory_name",
["get_sana_video_post_process_func", "get_sana_video_i2v_post_process_func"],
)
def test_sana_video_postprocess_preserves_requested_latents(factory_name):
from vllm_omni.diffusion.models import sana_video
latents = torch.randn(1, 4, 2, 2, 2)
post_process = getattr(sana_video, factory_name)(SimpleNamespace(diffusion_load_format="native"))
result = post_process(latents, sampling_params=SimpleNamespace(output_type="latent"))
assert result is latents
def test_sana_video_720p_model_id_fallback(monkeypatch):
from vllm_omni.diffusion.models.sana_video import pipeline_sana_video
monkeypatch.setattr(pipeline_sana_video, "get_hf_file_to_dict", lambda *_args, **_kwargs: None)
od_config = SimpleNamespace(
model="Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
tf_model_config={},
)
assert pipeline_sana_video.resolve_sana_video_sample_size(od_config) == 22
@pytest.mark.parametrize(
("requested_output_type", "expected_internal_output_type"),
[
(None, "raw"),
("np", "raw"),
("latent", "latent"),
],
)
def test_sana_video_i2v_forward_maps_image_request(requested_output_type, expected_internal_output_type):
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate_i2v(**kwargs):
calls.append(kwargs)
return torch.zeros(kwargs["num_videos_per_prompt"], 3, 9, 192, 320)
pipeline._generate_i2v = fake_generate_i2v
image = Image.new("RGB", (320, 192))
custom_timesteps = torch.tensor([900, 100])
req = _make_request_batch(
{"prompt": "a robot walks", "negative_prompt": "blurry", "multi_modal_data": {"image": image}},
height=192,
width=320,
num_frames=9,
num_inference_steps=2,
timesteps=custom_timesteps,
guidance_scale=4.5,
num_outputs_per_prompt=2,
eta=0.25,
seed=42,
output_type=requested_output_type,
extra_args={"clean_caption": True, "motion_score": 30, "use_resolution_binning": False},
)
output = pipeline.forward(req)
assert output.output.shape == (2, 3, 9, 192, 320)
assert calls[0]["image"] is image
assert calls[0]["prompt"] == "a robot walks motion score: 30."
assert calls[0]["negative_prompt"] == "blurry"
assert calls[0]["frames"] == 9
assert calls[0]["timesteps"] is custom_timesteps
assert calls[0]["sigmas"] is None
assert calls[0]["num_videos_per_prompt"] == 2
assert calls[0]["eta"] == 0.25
assert calls[0]["clean_caption"] is True
assert calls[0]["generator"].initial_seed() == 42
assert calls[0]["output_type"] == expected_internal_output_type
def test_sana_video_i2v_forward_maps_custom_sigmas():
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate_i2v(**kwargs):
calls.append(kwargs)
return torch.zeros(1, 3, 9, 192, 320)
pipeline._generate_i2v = fake_generate_i2v
sigmas = [1.0, 0.5]
output = pipeline.forward(
_make_request_batch(
{
"prompt": "a robot walks",
"multi_modal_data": {"image": Image.new("RGB", (320, 192))},
},
height=192,
width=320,
num_frames=9,
sigmas=sigmas,
generator_device="cpu",
extra_args={"use_resolution_binning": False},
)
)
assert output.output.shape == (1, 3, 9, 192, 320)
assert calls[0]["timesteps"] is None
assert calls[0]["sigmas"] is sigmas
def test_sana_video_i2v_omitted_num_frames_uses_model_default():
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate_i2v(**kwargs):
calls.append(kwargs)
return torch.zeros(1, 3, 81, 2, 2)
pipeline._generate_i2v = fake_generate_i2v
req = _make_request_batch(
{
"prompt": "a robot walks",
"multi_modal_data": {"image": Image.new("RGB", (320, 192))},
},
height=192,
width=320,
num_inference_steps=1,
generator_device="cpu",
extra_args={"use_resolution_binning": False},
)
assert req.sampling_params.num_frames == 1
output = pipeline.forward(req)
assert output.output.shape == (1, 3, 81, 2, 2)
assert calls[0]["frames"] == 81
def test_sana_video_i2v_clean_caption_request_is_explicitly_rejected():
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
pipeline.text_encoder = SimpleNamespace(dtype=torch.float32)
pipeline.tokenizer = None
pipeline.check_inputs = lambda **_kwargs: None
req = _make_request_batch(
{
"prompt": "a robot walks",
"multi_modal_data": {"image": Image.new("RGB", (320, 192))},
},
height=192,
width=320,
num_frames=9,
generator_device="cpu",
extra_args={"clean_caption": True, "use_resolution_binning": False},
)
with pytest.raises(ValueError, match="does not support `clean_caption=True`"):
pipeline.forward(req)
@pytest.mark.parametrize(
("custom_timesteps", "custom_sigmas"),
[
(torch.tensor([4.0, 2.0]), None),
(None, [1.0, 0.5]),
],
)
def test_sana_video_i2v_consumes_batch_schedule_and_step_kwargs(custom_timesteps, custom_sigmas):
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
calls = SimpleNamespace(
encode_prompt=[],
prepare_latents=[],
set_timesteps=[],
step=[],
transformer=[],
)
class StubTransformer:
dtype = torch.float32
config = SimpleNamespace(
sample_size=30,
in_channels=4,
out_channels=4,
patch_size=(1, 1, 1),
)
def __call__(
self,
hidden_states,
*,
encoder_hidden_states,
encoder_attention_mask,
timestep,
return_dict,
):
calls.transformer.append(
(
hidden_states.shape,
encoder_hidden_states.shape,
encoder_attention_mask.shape,
timestep.shape,
return_dict,
)
)
return (torch.zeros_like(hidden_states),)
class StubScheduler:
order = 1
def set_timesteps(
self,
num_inference_steps=None,
*,
device=None,
timesteps=None,
sigmas=None,
):
calls.set_timesteps.append(
{
"num_inference_steps": num_inference_steps,
"timesteps": timesteps,
"sigmas": sigmas,
}
)
schedule_length = len(timesteps if timesteps is not None else sigmas)
self.timesteps = torch.arange(schedule_length, 0, -1, dtype=torch.float32, device=device)
def step(
self,
_noise_pred,
_timestep,
current_latents,
*,
eta,
generator,
return_dict,
):
calls.step.append({"eta": eta, "generator": generator, "return_dict": return_dict})
return (current_latents + 1,)
class VaeMustNotBeUsed:
@property
def dtype(self):
pytest.fail("latent output must return before accessing the VAE")
def decode(self, *_args, **_kwargs):
pytest.fail("latent output must not call VAE decode")
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = StubTransformer()
pipeline.scheduler = StubScheduler()
pipeline.vae = VaeMustNotBeUsed()
pipeline.check_inputs = lambda **_kwargs: None
pipeline.video_processor = SimpleNamespace(
preprocess=lambda _image, height, width: torch.zeros(1, 3, height, width),
)
pipeline.progress_bar = lambda **_kwargs: nullcontext(SimpleNamespace(update=lambda: None))
def capture_encode_prompt(*_args, **kwargs):
calls.encode_prompt.append(kwargs)
batch_size = kwargs["num_videos_per_prompt"]
return (
torch.zeros(batch_size, 1, 1),
torch.ones(batch_size, 1, dtype=torch.bool),
torch.zeros(batch_size, 1, 1),
torch.ones(batch_size, 1, dtype=torch.bool),
)
pipeline.encode_prompt = capture_encode_prompt
initial_latents = torch.zeros(2, 4, 2, 2, 2)
def capture_prepare_latents(*_args, **kwargs):
calls.prepare_latents.append(kwargs)
return initial_latents
pipeline._prepare_i2v_latents = capture_prepare_latents
generator = torch.Generator(device="cpu").manual_seed(42)
output = pipeline._generate_i2v(
image=Image.new("RGB", (2, 2)),
prompt="test",
negative_prompt="",
height=2,
width=2,
frames=2,
num_inference_steps=50,
timesteps=custom_timesteps,
sigmas=custom_sigmas,
guidance_scale=4.5,
num_videos_per_prompt=2,
eta=0.25,
generator=generator,
latents=None,
clean_caption=True,
use_resolution_binning=False,
max_sequence_length=1,
output_type="latent",
)
assert calls.encode_prompt[0]["num_videos_per_prompt"] == 2
assert calls.encode_prompt[0]["clean_caption"] is True
assert calls.prepare_latents[0]["batch_size"] == 2
assert len(calls.set_timesteps) == 1
assert calls.set_timesteps[0]["num_inference_steps"] is None
assert calls.set_timesteps[0]["timesteps"] is custom_timesteps
assert calls.set_timesteps[0]["sigmas"] is custom_sigmas
assert len(calls.step) == 2
assert all(call == {"eta": 0.25, "generator": generator, "return_dict": False} for call in calls.step)
assert all(
call
== (
torch.Size([4, 4, 2, 2, 2]),
torch.Size([4, 1, 1]),
torch.Size([4, 1]),
torch.Size([4, 1, 2, 2, 2]),
False,
)
for call in calls.transformer
)
expected = initial_latents.clone()
expected[:, :, 1:] += 2
torch.testing.assert_close(output, expected)
def test_sana_video_i2v_uses_diffusers_complex_instruction_default():
import inspect
from diffusers import SanaImageToVideoPipeline as DiffusersSanaImageToVideoPipeline
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
reference_default = (
inspect.signature(DiffusersSanaImageToVideoPipeline.__call__).parameters["complex_human_instruction"].default
)
native_default = (
inspect.signature(SanaImageToVideoPipeline._generate_i2v).parameters["complex_human_instruction"].default
)
assert native_default == reference_default
class StopAfterPromptEncodingError(Exception):
pass
pipeline = object.__new__(SanaImageToVideoPipeline)
pipeline.device = torch.device("cpu")
pipeline.check_inputs = lambda **_kwargs: None
captured = {}
def capture_encode_prompt(*_args, **kwargs):
captured.update(kwargs)
raise StopAfterPromptEncodingError
pipeline.encode_prompt = capture_encode_prompt
with pytest.raises(StopAfterPromptEncodingError):
pipeline._generate_i2v(
image=Image.new("RGB", (320, 192)),
prompt="a robot walks",
negative_prompt="blurry",
height=192,
width=320,
frames=9,
num_inference_steps=2,
guidance_scale=4.5,
generator=torch.Generator(device="cpu").manual_seed(42),
latents=None,
use_resolution_binning=False,
max_sequence_length=300,
)
assert captured["complex_human_instruction"] == reference_default
def test_sana_video_t2v_complex_instruction_has_no_mutable_default():
import inspect
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
SANA_VIDEO_COMPLEX_HUMAN_INSTRUCTION,
SanaVideoPipeline,
)
parameter = inspect.signature(SanaVideoPipeline._generate).parameters["complex_human_instruction"]
assert parameter.default is None
assert isinstance(SANA_VIDEO_COMPLEX_HUMAN_INSTRUCTION, tuple)
def test_diffusers_adapter_t2v_omitted_num_frames_uses_sana_default():
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import SanaVideoPipelineUtils
adapter = object.__new__(DiffusersAdapterPipeline)
adapter._accept_call_kwargs = {"prompt", "negative_prompt", "frames", "generator"}
adapter._pipeline_utils = SanaVideoPipelineUtils()
adapter.od_config = SimpleNamespace(diffusers_call_kwargs={}, output_type=None)
req = _make_request_batch(
{"prompt": "a robot walks", "negative_prompt": "blurry"},
seed=42,
generator_device="cpu",
)
assert req.sampling_params.num_frames == 1
kwargs = adapter._build_call_kwargs(req)
assert kwargs["frames"] == 81
assert "num_frames" not in kwargs
assert kwargs["generator"].initial_seed() == 42
def test_diffusers_adapter_i2v_omitted_num_frames_uses_sana_default():
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import SanaVideoPipelineUtils
adapter = object.__new__(DiffusersAdapterPipeline)
adapter._accept_call_kwargs = {"prompt", "negative_prompt", "image", "frames", "generator"}
adapter._pipeline_utils = SanaVideoPipelineUtils()
adapter.od_config = SimpleNamespace(diffusers_call_kwargs={}, output_type=None)
image = Image.new("RGB", (320, 192))
req = _make_request_batch(
{"prompt": "a robot walks", "negative_prompt": "blurry", "multi_modal_data": {"image": image}},
seed=42,
generator_device="cpu",
)
assert req.sampling_params.num_frames == 1
kwargs = adapter._build_call_kwargs(req)
assert kwargs["image"] is image
assert kwargs["prompt"] == "a robot walks"
assert kwargs["negative_prompt"] == "blurry"
assert kwargs["frames"] == 81
assert kwargs["generator"].initial_seed() == 42
@pytest.mark.parametrize(
("sampling_overrides", "expected_frames"),
[
({}, 49),
({"num_frames": 9}, 9),
],
)
def test_diffusers_adapter_sana_resolves_frame_precedence(sampling_overrides, expected_frames):
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import SanaVideoPipelineUtils
adapter = object.__new__(DiffusersAdapterPipeline)
adapter._accept_call_kwargs = {"prompt", "frames", "generator"}
adapter._pipeline_utils = SanaVideoPipelineUtils()
adapter.od_config = SimpleNamespace(diffusers_call_kwargs={"frames": 49}, output_type=None)
kwargs = adapter._build_call_kwargs(
_make_request_batch(
"a robot walks",
seed=42,
generator_device="cpu",
**sampling_overrides,
)
)
assert kwargs["frames"] == expected_frames
@pytest.mark.parametrize("pipeline_class_name", ["SanaVideoPipeline", "SanaImageToVideoPipeline"])
def test_diffusers_adapter_selects_sana_pipeline_utils(pipeline_class_name):
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
SanaVideoPipelineUtils,
get_pipeline_utils,
)
assert isinstance(get_pipeline_utils(pipeline_class_name), SanaVideoPipelineUtils)
def test_diffusers_adapter_resolves_requested_sana_i2v_pipeline():
from diffusers import SanaImageToVideoPipeline, SanaVideoPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
SanaVideoPipelineUtils,
get_pipeline_utils_for_config,
resolve_diffusers_pipeline_class,
)
od_config = SimpleNamespace(
model_class_name="SanaImageToVideoPipeline",
diffusers_pipeline_cls=SanaVideoPipeline,
)
assert isinstance(get_pipeline_utils_for_config(od_config), SanaVideoPipelineUtils)
assert resolve_diffusers_pipeline_class(od_config) is SanaImageToVideoPipeline
def test_diffusers_adapter_pipeline_override_is_sana_specific():
from diffusers import DiffusionPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import resolve_diffusers_pipeline_class
class OtherPipeline(DiffusionPipeline):
pass
od_config = SimpleNamespace(
model_class_name="DiffusersAdapterPipeline",
diffusers_pipeline_cls=OtherPipeline,
)
assert resolve_diffusers_pipeline_class(od_config) is OtherPipeline
def test_diffusers_adapter_loads_resolved_pipeline_class(monkeypatch):
from vllm_omni.diffusion.models.diffusers_adapter import pipeline_diffusers_adapter
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
loaded = []
expected_pipeline = object()
class ResolvedPipeline:
@classmethod
def from_pretrained(cls, model_id, **kwargs):
loaded.append((model_id, kwargs))
return expected_pipeline
monkeypatch.setattr(
pipeline_diffusers_adapter,
"resolve_diffusers_pipeline_class",
lambda _od_config: ResolvedPipeline,
)
adapter = object.__new__(DiffusersAdapterPipeline)
adapter.od_config = SimpleNamespace(diffusers_pipeline_cls=object)
result = adapter._load_pipeline_from_pretrained("sana-checkpoint", {"torch_dtype": torch.bfloat16})
assert result is expected_pipeline
assert loaded == [("sana-checkpoint", {"torch_dtype": torch.bfloat16})]
def test_diffusers_adapter_sana_i2v_override_reports_image_capability():
from diffusers import SanaVideoPipeline
from vllm_omni.diffusion.io_support import supports_multimodal_input
od_config = SimpleNamespace(
diffusion_load_format="diffusers",
model_class_name="SanaImageToVideoPipeline",
diffusers_pipeline_cls=SanaVideoPipeline,
)
assert supports_multimodal_input(od_config) == (True, False)
def test_diffusers_adapter_sana_i2v_warmup_supplies_image():
from diffusers import SanaVideoPipeline
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
engine = object.__new__(DiffusionEngine)
engine.od_config = SimpleNamespace(
diffusion_load_format="diffusers",
model_class_name="SanaImageToVideoPipeline",
diffusers_pipeline_cls=SanaVideoPipeline,
)
captured_requests = []
engine.pre_process_func = lambda request: request
def capture_request(request):
captured_requests.append(request)
return SimpleNamespace(error=None)
engine.add_req_and_wait_for_response = capture_request
engine._dummy_run()
assert len(captured_requests) == 1
prompt = captured_requests[0].prompt
assert isinstance(prompt, dict)
assert isinstance(prompt["multi_modal_data"]["image"], Image.Image)
def test_diffusers_adapter_disables_resolution_binning_for_warmup():
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import DiffusersAdapterPipeline
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import SanaVideoPipelineUtils
adapter = object.__new__(DiffusersAdapterPipeline)
adapter._accept_call_kwargs = {"prompt", "frames", "use_resolution_binning", "generator"}
adapter._pipeline_utils = SanaVideoPipelineUtils()
adapter.od_config = SimpleNamespace(diffusers_call_kwargs={}, output_type=None)
kwargs = adapter._build_call_kwargs(
_make_request_batch(
"dummy run",
request_id=DUMMY_DIFFUSION_REQUEST_ID,
generator_device="cpu",
)
)
assert kwargs["use_resolution_binning"] is False
assert kwargs["frames"] == 1
def test_pipeline_is_torch_module_and_supports_eval():
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
nn.Module.__init__(pipeline)
assert isinstance(pipeline, nn.Module)
assert pipeline.eval() is pipeline
assert pipeline.training is False
@pytest.mark.parametrize(
("vae_scale", "patch_size", "valid_size", "invalid_size", "alignment"),
[
(8, (1, 2, 2), 624, 632, 16),
(32, (1, 1, 1), 704, 712, 32),
],
)
def test_check_inputs_uses_variant_spatial_alignment(vae_scale, patch_size, valid_size, invalid_size, alignment):
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
pipeline.vae_scale_factor_spatial = vae_scale
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(patch_size=patch_size))
pipeline.check_inputs(prompt="test", height=valid_size, width=valid_size)
with pytest.raises(ValueError, match=f"divisible by {alignment}"):
pipeline.check_inputs(prompt="test", height=invalid_size, width=valid_size)
@pytest.mark.parametrize(
("requested_output_type", "expected_internal_output_type"),
[
(None, "raw"),
("np", "raw"),
("latent", "latent"),
],
)
def test_forward_maps_omni_request_to_sana_generation_args(requested_output_type, expected_internal_output_type):
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate(**kwargs):
calls.append(kwargs)
return SimpleNamespace(frames=torch.zeros(1, 3, 9, 192, 320))
pipeline._generate = fake_generate
req = _make_request_batch(
{"prompt": "a robot walks", "negative_prompt": "blurry"},
height=192,
width=320,
num_frames=9,
num_inference_steps=2,
guidance_scale=4.5,
seed=42,
output_type=requested_output_type,
extra_args={"motion_score": 30, "use_resolution_binning": False},
)
output = pipeline.forward(req)
assert output.output.shape == (1, 3, 9, 192, 320)
assert calls[0]["prompt"] == "a robot walks motion score: 30."
assert calls[0]["negative_prompt"] == "blurry"
assert calls[0]["height"] == 192
assert calls[0]["width"] == 320
assert calls[0]["frames"] == 9
assert calls[0]["num_inference_steps"] == 2
assert calls[0]["guidance_scale"] == 4.5
assert calls[0]["use_resolution_binning"] is False
assert calls[0]["generator"].initial_seed() == 42
assert calls[0]["output_type"] == expected_internal_output_type
def test_sana_video_t2v_omitted_num_frames_uses_model_default():
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate(**kwargs):
calls.append(kwargs)
return SimpleNamespace(frames=torch.zeros(1, 3, 81, 2, 2))
pipeline._generate = fake_generate
req = _make_request_batch(
"a robot walks",
height=192,
width=320,
num_inference_steps=1,
generator_device="cpu",
)
assert req.sampling_params.num_frames == 1
output = pipeline.forward(req)
assert output.output.shape == (1, 3, 81, 2, 2)
assert calls[0]["frames"] == 81
def test_sana_video_t2v_dummy_run_keeps_single_frame():
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=30))
calls = []
def fake_generate(**kwargs):
calls.append(kwargs)
return SimpleNamespace(frames=torch.zeros(1, 3, 1, 192, 320))
pipeline._generate = fake_generate
output = pipeline.forward(
_make_request_batch(
"dummy run",
request_id=DUMMY_DIFFUSION_REQUEST_ID,
height=192,
width=320,
num_frames=1,
num_inference_steps=1,
generator_device="cpu",
)
)
assert output.output.shape == (1, 3, 1, 192, 320)
assert calls[0]["frames"] == 1
@pytest.mark.parametrize(
("sample_size", "expected_height", "expected_width"),
[
(30, 480, 832),
(22, 704, 1280),
],
)
def test_t2v_uses_variant_default_resolution(sample_size, expected_height, expected_width):
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
pipeline.transformer = SimpleNamespace(config=SimpleNamespace(sample_size=sample_size))
calls = []
def fake_generate(**kwargs):
calls.append(kwargs)
return SimpleNamespace(frames=torch.zeros(1, 3, 9, expected_height, expected_width))
pipeline._generate = fake_generate
output = pipeline.forward(
_make_request_batch(
"a robot walks",
num_frames=9,
num_inference_steps=1,
guidance_scale=1.0,
seed=42,
generator_device="cpu",
)
)
assert output.output.shape == (1, 3, 9, expected_height, expected_width)
assert calls[0]["height"] == expected_height
assert calls[0]["width"] == expected_width
def test_forward_requires_exactly_one_nonempty_prompt():
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
pipeline = object.__new__(SanaVideoPipeline)
with pytest.raises(ValueError, match="Prompt is required"):
pipeline.forward(_make_request_batch(""))
first = OmniDiffusionRequest(
prompt="first",
sampling_params=OmniDiffusionSamplingParams(),
request_id="first",
)
second = OmniDiffusionRequest(
prompt="second",
sampling_params=OmniDiffusionSamplingParams(),
request_id="second",
)
with pytest.raises(ValueError, match="exactly one prompt"):
pipeline.forward(DiffusionRequestBatch([first, second]))
@@ -0,0 +1,602 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
_TINY_CONFIG = {
"in_channels": 4,
"out_channels": 4,
"num_attention_heads": 2,
"attention_head_dim": 12,
"num_layers": 1,
"num_cross_attention_heads": 2,
"cross_attention_head_dim": 12,
"cross_attention_dim": 24,
"caption_channels": 8,
"mlp_ratio": 2.0,
"patch_size": (1, 2, 2),
"sample_size": 4,
"rope_max_seq_len": 32,
}
_GOLDEN_PREFIX = torch.tensor(
[
-0.1159307063,
0.2003862262,
-0.3754127920,
-0.2403523028,
-1.2376221418,
-0.6513092518,
-0.3358457983,
-0.9894289970,
0.7858567238,
0.5089095831,
-0.5362522602,
-0.3732060790,
]
)
@pytest.mark.parametrize(
("elementwise_affine", "bias", "expected_state_keys"),
[
(False, False, set()),
(False, True, set()),
(True, False, {"weight"}),
(True, True, {"weight", "bias"}),
],
)
@pytest.mark.parametrize(
("input_dtype", "parameter_dtype"),
[
(torch.float16, torch.float16),
(torch.bfloat16, torch.bfloat16),
(torch.float32, torch.float32),
(torch.float64, torch.float64),
(torch.float16, torch.float32),
(torch.bfloat16, torch.float32),
(torch.float32, torch.float16),
(torch.float32, torch.bfloat16),
],
)
def test_sana_rms_norm_matches_diffusers(
elementwise_affine,
bias,
expected_state_keys,
input_dtype,
parameter_dtype,
):
from diffusers.models.normalization import RMSNorm as DiffusersRMSNorm
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaRMSNorm
reference = DiffusersRMSNorm(8, eps=1e-5, elementwise_affine=elementwise_affine, bias=bias).to(
dtype=parameter_dtype
)
actual = SanaRMSNorm(8, eps=1e-5, elementwise_affine=elementwise_affine, bias=bias).to(dtype=parameter_dtype)
assert actual.eps == reference.eps
assert actual.elementwise_affine == reference.elementwise_affine
assert actual.dim == reference.dim
assert set(actual.state_dict()) == expected_state_keys
assert set(actual.state_dict()) == set(reference.state_dict())
if elementwise_affine:
weight = torch.linspace(0.5, 1.5, 8, dtype=parameter_dtype)
actual.weight.data.copy_(weight)
reference.weight.data.copy_(weight)
if bias:
norm_bias = torch.linspace(-0.25, 0.25, 8, dtype=parameter_dtype)
actual.bias.data.copy_(norm_bias)
reference.bias.data.copy_(norm_bias)
else:
assert actual.weight is None
assert actual.bias is None
hidden_states = torch.linspace(-2.0, 2.0, 48, dtype=input_dtype).reshape(2, 3, 8)
expected = reference(hidden_states)
result = actual(hidden_states)
assert result.dtype == expected.dtype
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@pytest.mark.parametrize(
("num_channels", "flip_sin_to_cos", "downscale_freq_shift", "scale"),
[
(256, True, 0, 1),
(7, False, 1, 0.5),
(12, True, 0.5, 2),
],
)
@pytest.mark.parametrize("timestep_dtype", [torch.int64, torch.float32, torch.float64])
def test_sana_timesteps_matches_diffusers(
num_channels,
flip_sin_to_cos,
downscale_freq_shift,
scale,
timestep_dtype,
):
from diffusers.models.embeddings import Timesteps
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaTimesteps
reference = Timesteps(
num_channels=num_channels,
flip_sin_to_cos=flip_sin_to_cos,
downscale_freq_shift=downscale_freq_shift,
scale=scale,
)
actual = SanaTimesteps(
num_channels=num_channels,
flip_sin_to_cos=flip_sin_to_cos,
downscale_freq_shift=downscale_freq_shift,
scale=scale,
)
timesteps = torch.tensor([0, 1, 500, 999], dtype=timestep_dtype)
assert actual.state_dict() == reference.state_dict() == {}
assert actual.num_channels == reference.num_channels
assert actual.flip_sin_to_cos == reference.flip_sin_to_cos
assert actual.downscale_freq_shift == reference.downscale_freq_shift
assert actual.scale == reference.scale
torch.testing.assert_close(actual(timesteps), reference(timesteps), rtol=0, atol=0)
@pytest.mark.parametrize(
("kwargs", "sample_shape", "condition_shape"),
[
({}, (2, 256), None),
(
{
"out_dim": 12,
"post_act_fn": "silu",
"cond_proj_dim": 6,
"sample_proj_bias": False,
},
(2, 256),
(2, 6),
),
],
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_sana_timestep_embedding_matches_diffusers(
kwargs,
sample_shape,
condition_shape,
dtype,
):
from diffusers.models.embeddings import TimestepEmbedding
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaTimestepEmbedding,
)
torch.manual_seed(17)
reference = TimestepEmbedding(256, 24, **kwargs).to(dtype=dtype)
actual = SanaTimestepEmbedding(256, 24, **kwargs).to(dtype=dtype)
assert set(actual.state_dict()) == set(reference.state_dict())
actual.load_state_dict(reference.state_dict())
sample = torch.randn(sample_shape, dtype=dtype)
condition = torch.randn(condition_shape, dtype=dtype) if condition_shape else None
expected = reference(sample, condition)
result = actual(sample, condition)
assert result.shape == expected.shape
assert result.dtype == expected.dtype
torch.testing.assert_close(result, expected, rtol=0, atol=0)
def test_sana_timestep_embedding_rejects_unsupported_activation():
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaTimestepEmbedding,
)
with pytest.raises(ValueError, match="act_fn='silu'"):
SanaTimestepEmbedding(8, 16, act_fn="gelu")
with pytest.raises(ValueError, match="post_act_fn"):
SanaTimestepEmbedding(8, 16, post_act_fn="gelu")
module = SanaTimestepEmbedding(8, 16)
with pytest.raises(ValueError, match="cond_proj_dim"):
module(torch.randn(1, 8), torch.randn(1, 4))
@pytest.mark.parametrize("use_additional_conditions", [False, True])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_sana_ada_layer_norm_single_matches_diffusers(
use_additional_conditions,
dtype,
):
from diffusers.models.normalization import AdaLayerNormSingle
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaAdaLayerNormSingle,
)
torch.manual_seed(19)
reference = AdaLayerNormSingle(
24,
use_additional_conditions=use_additional_conditions,
).to(dtype=dtype)
actual = SanaAdaLayerNormSingle(
24,
use_additional_conditions=use_additional_conditions,
).to(dtype=dtype)
assert set(actual.state_dict()) == set(reference.state_dict())
actual.load_state_dict(reference.state_dict())
timestep = torch.tensor([10, 500])
added_cond_kwargs = None
if use_additional_conditions:
added_cond_kwargs = {
"resolution": torch.tensor([[480, 832], [704, 1280]]),
"aspect_ratio": torch.tensor([[832 / 480], [1280 / 704]]),
}
expected = reference(
timestep,
added_cond_kwargs=added_cond_kwargs,
batch_size=2,
hidden_dtype=dtype,
)
result = actual(
timestep,
added_cond_kwargs=added_cond_kwargs,
batch_size=2,
hidden_dtype=dtype,
)
assert len(result) == len(expected) == 2
for actual_tensor, expected_tensor in zip(result, expected):
assert actual_tensor.shape == expected_tensor.shape
assert actual_tensor.dtype == expected_tensor.dtype
torch.testing.assert_close(actual_tensor, expected_tensor, rtol=0, atol=0)
@pytest.mark.parametrize("act_fn", ["gelu_tanh", "silu", "silu_fp32"])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_sana_pixart_text_projection_matches_diffusers(act_fn, dtype):
from diffusers.models.embeddings import PixArtAlphaTextProjection
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaPixArtAlphaTextProjection,
)
torch.manual_seed(23)
reference = PixArtAlphaTextProjection(
in_features=8,
hidden_size=24,
out_features=12,
act_fn=act_fn,
).to(dtype=dtype)
actual = SanaPixArtAlphaTextProjection(
in_features=8,
hidden_size=24,
out_features=12,
act_fn=act_fn,
).to(dtype=dtype)
assert set(actual.state_dict()) == set(reference.state_dict())
actual.load_state_dict(reference.state_dict())
caption = torch.randn(2, 5, 8, dtype=dtype)
expected = reference(caption)
result = actual(caption)
assert result.shape == expected.shape
assert result.dtype == expected.dtype
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@pytest.mark.parametrize("freqs_dtype", [torch.float32, torch.float64])
@pytest.mark.parametrize(("dim", "max_seq_len", "theta"), [(4, 8, 10000.0), (12, 32, 256.0)])
def test_native_rope_matches_diffusers(dim, max_seq_len, theta, freqs_dtype):
from diffusers.models.embeddings import get_1d_rotary_pos_embed
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
_get_1d_rotary_pos_embed,
)
expected_cos, expected_sin = get_1d_rotary_pos_embed(
dim,
max_seq_len,
theta,
use_real=True,
repeat_interleave_real=True,
freqs_dtype=freqs_dtype,
)
actual_cos, actual_sin = _get_1d_rotary_pos_embed(dim, max_seq_len, theta, freqs_dtype)
torch.testing.assert_close(actual_cos, expected_cos, rtol=0, atol=0)
torch.testing.assert_close(actual_sin, expected_sin, rtol=0, atol=0)
def test_tiny_transformer_matches_diffusers_and_frozen_output():
from diffusers import SanaVideoTransformer3DModel as DiffusersTransformer
from diffusers.configuration_utils import ConfigMixin
from diffusers.models.modeling_utils import ModelMixin
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaLinearAttention,
SanaRMSNorm,
SanaVideoTransformerConfig,
SanaVideoTransformerOutput,
)
torch.manual_seed(7)
reference = DiffusersTransformer(**_TINY_CONFIG).eval()
model = SanaVideoTransformer3DModel(**_TINY_CONFIG).eval()
assert not isinstance(model, ModelMixin)
assert not isinstance(model, ConfigMixin)
assert isinstance(model.config, SanaVideoTransformerConfig)
assert set(model.state_dict()) == set(reference.state_dict())
model.load_state_dict(reference.state_dict())
block = model.transformer_blocks[0]
assert isinstance(block.attn1, SanaLinearAttention)
assert isinstance(block.attn2.attn, Attention)
assert block.attn2.attn.role == "cross"
assert isinstance(block.attn1.norm_q, SanaRMSNorm)
assert isinstance(block.attn1.norm_k, SanaRMSNorm)
assert isinstance(block.attn2.norm_q, SanaRMSNorm)
assert isinstance(block.attn2.norm_k, SanaRMSNorm)
assert isinstance(model.caption_norm, SanaRMSNorm)
torch.manual_seed(11)
hidden_states = torch.randn(1, 4, 3, 4, 4)
encoder_hidden_states = torch.randn(1, 5, 8)
encoder_attention_mask = torch.tensor([[1, 1, 1, 1, 0]])
timestep = torch.tensor([500.0])
with torch.no_grad():
expected = reference(
hidden_states,
encoder_hidden_states,
timestep,
encoder_attention_mask=encoder_attention_mask,
).sample
actual_output = model(
hidden_states,
encoder_hidden_states,
timestep,
encoder_attention_mask=encoder_attention_mask,
)
actual = actual_output.sample
assert isinstance(actual_output, SanaVideoTransformerOutput)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
torch.testing.assert_close(actual.flatten()[: len(_GOLDEN_PREFIX)], _GOLDEN_PREFIX, rtol=1e-5, atol=1e-5)
def test_encoder_attention_mask_uses_bool_sdpa_contract(monkeypatch):
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
torch.manual_seed(37)
model = SanaVideoTransformer3DModel(**_TINY_CONFIG).eval()
captured = {}
omni_attention = model.transformer_blocks[0].attn2.attn
def fake_sdpa_forward(query, key, value, attn_metadata):
captured["query_shape"] = query.shape
captured["key_shape"] = key.shape
captured["mask"] = attn_metadata.attn_mask.detach().clone()
return torch.zeros_like(query)
def unexpected_backend_forward(*args, **kwargs):
raise AssertionError("masked SANA cross-attention must use SDPA")
monkeypatch.setattr(omni_attention.sdpa_fallback, "forward", fake_sdpa_forward)
monkeypatch.setattr(omni_attention, "forward", unexpected_backend_forward)
hidden_states = torch.randn(2, 4, 3, 4, 4)
encoder_hidden_states = torch.randn(2, 5, 8)
input_mask = torch.tensor(
[
[1, 1, 1, 0, 0],
[1, 1, 1, 1, 0],
],
dtype=torch.int64,
)
with torch.no_grad():
output = model(
hidden_states,
encoder_hidden_states,
torch.tensor([100.0, 500.0]),
encoder_attention_mask=input_mask,
).sample
assert output.shape == hidden_states.shape
assert captured["query_shape"][:2] == (2, 12)
assert captured["key_shape"][:2] == (2, 5)
assert captured["query_shape"][1] != captured["key_shape"][1]
assert captured["mask"].shape == (2, 5)
assert captured["mask"].dtype == torch.bool
assert torch.equal(captured["mask"], input_mask.bool())
def test_encoder_attention_mask_rejects_additive_bias():
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
model = SanaVideoTransformer3DModel(**_TINY_CONFIG).eval()
additive_mask = torch.tensor([[0.0, 0.0, 0.0, -10000.0, -10000.0]])
with pytest.raises(TypeError, match="not a floating-point additive attention bias"):
model(
torch.randn(1, 4, 3, 4, 4),
torch.randn(1, 5, 8),
torch.tensor([500.0]),
encoder_attention_mask=additive_mask,
)
def test_linear_attention_requires_rotary_embeddings():
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaLinearAttention
attention = SanaLinearAttention(
dim=24,
num_heads=2,
head_dim=12,
dropout=0.0,
bias=True,
qk_norm="rms_norm_across_heads",
)
with pytest.raises(ValueError, match="requires rotary_emb"):
attention(torch.randn(1, 4, 24))
def test_linear_attention_bfloat16_matches_diffusers_mixed_precision():
from diffusers.models.attention_processor import Attention as DiffusersAttention
from diffusers.models.transformers.transformer_sana_video import SanaLinearAttnProcessor3_0
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaLinearAttention
torch.manual_seed(17)
reference = DiffusersAttention(
query_dim=24,
heads=2,
dim_head=12,
kv_heads=2,
qk_norm="rms_norm_across_heads",
dropout=0.0,
bias=True,
cross_attention_dim=None,
processor=SanaLinearAttnProcessor3_0(),
).to(dtype=torch.bfloat16)
actual = SanaLinearAttention(
dim=24,
num_heads=2,
head_dim=12,
dropout=0.0,
bias=True,
qk_norm="rms_norm_across_heads",
).to(dtype=torch.bfloat16)
actual.load_state_dict(reference.state_dict())
hidden_states = torch.randn(1, 5, 24, dtype=torch.bfloat16)
rotary_emb = (
torch.randn(1, 5, 1, 12, dtype=torch.bfloat16),
torch.randn(1, 5, 1, 12, dtype=torch.bfloat16),
)
with torch.no_grad():
expected = reference(hidden_states, rotary_emb=rotary_emb)
result = actual(hidden_states, rotary_emb=rotary_emb)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
def test_transformer_block_without_cross_attention_keeps_feed_forward_norm():
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaVideoTransformerBlock
block = SanaVideoTransformerBlock(
dim=24,
num_attention_heads=2,
attention_head_dim=12,
num_cross_attention_heads=2,
cross_attention_head_dim=12,
cross_attention_dim=None,
mlp_ratio=2.0,
)
assert block.attn2 is None
assert isinstance(block.norm2, torch.nn.LayerNorm)
def test_guidance_transformer_matches_diffusers_and_tuple_output():
from diffusers import SanaVideoTransformer3DModel as DiffusersTransformer
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import (
SanaTimestepEmbedding,
SanaTimesteps,
)
config = _TINY_CONFIG | {"guidance_embeds": True}
torch.manual_seed(29)
reference = DiffusersTransformer(**config).eval()
model = SanaVideoTransformer3DModel(**config).eval()
assert set(model.state_dict()) == set(reference.state_dict())
model.load_state_dict(reference.state_dict())
assert isinstance(model.time_embed.time_proj, SanaTimesteps)
assert isinstance(model.time_embed.timestep_embedder, SanaTimestepEmbedding)
assert isinstance(model.time_embed.guidance_condition_proj, SanaTimesteps)
assert isinstance(model.time_embed.guidance_embedder, SanaTimestepEmbedding)
torch.manual_seed(31)
hidden_states = torch.randn(1, 4, 3, 4, 4)
encoder_hidden_states = torch.randn(1, 5, 8)
encoder_attention_mask = torch.tensor([[1, 1, 1, 1, 0]])
timestep = torch.tensor([500.0])
guidance = torch.tensor([6.0])
with torch.no_grad():
expected = reference(
hidden_states,
encoder_hidden_states,
timestep,
guidance=guidance,
encoder_attention_mask=encoder_attention_mask,
return_dict=False,
)
actual = model(
hidden_states,
encoder_hidden_states,
timestep,
guidance=guidance,
encoder_attention_mask=encoder_attention_mask,
return_dict=False,
)
assert isinstance(actual, tuple)
assert len(actual) == len(expected) == 1
torch.testing.assert_close(actual[0], expected[0], rtol=0, atol=0)
def test_native_transformer_config_filters_diffusers_metadata():
from vllm_omni.diffusion.models.sana_video.transformer_sana_video import SanaVideoTransformerConfig
config = SanaVideoTransformerConfig.from_dict(
_TINY_CONFIG | {"_class_name": "SanaVideoTransformer3DModel", "_diffusers_version": "0.38.0"}
)
assert config.patch_size == (1, 2, 2)
with pytest.raises(ValueError, match="unsupported_field"):
SanaVideoTransformerConfig.from_dict(_TINY_CONFIG | {"unsupported_field": True})
def test_dual_vae_variant_resolution():
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
DistributedAutoencoderKLLTX2Video,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
DistributedAutoencoderKLWan,
)
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
SanaVideoPipeline,
_resolve_vae_class_and_dtype,
)
assert not issubclass(SanaVideoPipeline, DiffusionPipeline)
assert issubclass(SanaVideoPipeline, ProgressBarMixin)
vae_class, dtype = _resolve_vae_class_and_dtype("AutoencoderKLWan", torch.bfloat16)
assert vae_class is DistributedAutoencoderKLWan
assert dtype is torch.float32
vae_class, dtype = _resolve_vae_class_and_dtype("AutoencoderKLLTX2Video", torch.bfloat16)
assert vae_class is DistributedAutoencoderKLLTX2Video
assert dtype is torch.bfloat16
with pytest.raises(ValueError, match="Unsupported SANA-Video VAE"):
_resolve_vae_class_and_dtype("UnknownVAE", torch.bfloat16)
+63
View File
@@ -0,0 +1,63 @@
# SANA-Video golden outputs
The v2 golden set covers both text-to-video and image-to-video for each
checkpoint variant. Generate references with the Diffusers version pinned by
this repository and the immutable Hub revisions embedded in
`generate_goldens.py`:
```bash
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 480p --task t2v
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 480p --task i2v --input-image /path/to/fixed-input.png
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 720p --task t2v
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 720p --task i2v --input-image /path/to/fixed-input.png
```
Use the same reviewed input image for both I2V variants. The generator
normalizes it to an RGB PNG and includes that exact file and its SHA256 in the
case manifest, so an I2V reference is self-contained.
Each case is written under:
```text
<output-root>/<variant>/<task>/
```
It contains:
- `transformer_case.safetensors`: deterministic transformer inputs and output.
- `pipeline.mp4`: the encoded reference used by online similarity tests.
- `pipeline_reference.safetensors`: final denoised latents and decoded,
pre-MP4 frames quantized to the exact uint8 video input.
- `input.png`: the normalized conditioning image for I2V only.
- `metadata.json`: model, generator, scheduler, input and sampling provenance.
- `manifest.json`: the size and SHA256 of every artifact.
## Publication requirements
A canonical golden must be generated from the pinned Hugging Face model ID and
immutable commit. The generator also requires a clean vLLM-Omni worktree and
records the repository revision plus the generator script hash.
`--model` is available for local development. It hashes every file in the local
checkpoint directory and records optional `--local-source-model` and
`--local-source-revision` claims, but marks the result `publishable: false`.
The golden test rejects such a manifest. A local directory hash proves which
bytes were used; it does not prove that those bytes match a claimed upstream
commit.
After reviewing the metadata, input image and generated videos, upload each
publishable case directory without renaming files to:
```text
s3://vllm-public-assets/omni-assets/sana-video/v2/<variant>/<task>/
```
Set `SANA_VIDEO_GOLDEN_BASE_URL` to the corresponding `v2` HTTP base URL when
running the tests. Uploading is intentionally separate from generation so test
runs cannot overwrite a frozen reference. The storage location must prevent
in-place mutation; publish a new version prefix instead of replacing any v2
file when a model, dependency, input or sampling parameter changes.
@@ -0,0 +1,380 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generate frozen SANA-Video transformer and pipeline reference artifacts."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import time
from pathlib import Path
from typing import Any
import numpy as np
import torch
from diffusers import SanaImageToVideoPipeline, SanaVideoPipeline
from diffusers.utils import export_to_video
from PIL import Image
from safetensors.torch import save_file
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
NEGATIVE_PROMPT = (
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
"jitter, and ghosting effects, creating a disorienting visual experience."
)
SCHEMA_VERSION = 2
SEED = 42
NUM_FRAMES = 81
FPS = 16
NUM_INFERENCE_STEPS = 50
GUIDANCE_SCALE = 6.0
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"height": 480,
"width": 832,
"vae_dtype": torch.float32,
"transformer_case_shape": (1, 16, 3, 8, 8),
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
"height": 704,
"width": 1280,
"vae_dtype": torch.bfloat16,
"transformer_case_shape": (1, 128, 2, 4, 4),
},
}
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _json_sha256(payload: Any) -> str:
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
return hashlib.sha256(encoded).hexdigest()
def _directory_fingerprint(root: Path) -> dict[str, Any]:
"""Fingerprint every local checkpoint file without recording host paths."""
if not root.is_dir():
raise ValueError(f"Local checkpoint must be a directory: {root}")
files = []
digest = hashlib.sha256()
for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()):
relative = path.relative_to(root).as_posix()
file_sha256 = _sha256(path)
size = path.stat().st_size
entry = f"{relative}\0{size}\0{file_sha256}\n".encode()
digest.update(entry)
files.append({"path": relative, "size": size, "sha256": file_sha256})
if not files:
raise ValueError(f"Local checkpoint contains no files: {root}")
return {
"algorithm": "sha256-tree-v1",
"sha256": digest.hexdigest(),
"file_count": len(files),
"total_size": sum(entry["size"] for entry in files),
}
def _repository_provenance() -> dict[str, Any]:
repo_root = Path(__file__).resolve().parents[4]
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
dirty = bool(
subprocess.run(
["git", "status", "--porcelain", "--untracked-files=no"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
)
return {
"repository": "vllm-project/vllm-omni",
"revision": revision,
"dirty": dirty,
"generator_sha256": _sha256(Path(__file__)),
}
def _cpu(tensor: torch.Tensor) -> torch.Tensor:
return tensor.detach().cpu().contiguous()
def _normalize_input_image(source: Path, destination: Path) -> Image.Image:
if not source.is_file():
raise ValueError(f"I2V input image does not exist: {source}")
with Image.open(source) as image:
normalized = image.convert("RGB")
normalized.load()
normalized.save(destination, format="PNG", optimize=False)
return normalized
def _frames_to_uint8(frames: np.ndarray) -> torch.Tensor:
tensor = torch.from_numpy(np.asarray(frames))
if tensor.is_floating_point():
# Match diffusers.utils.export_to_video, which truncates rather than
# rounds when converting float frames to the encoder's uint8 input.
tensor = tensor.clamp(0, 1) * 255
return tensor.to(torch.uint8).contiguous()
def _model_provenance(
config: dict[str, Any],
model: str | None,
local_source_model: str | None,
local_source_revision: str | None,
) -> tuple[dict[str, Any], bool]:
if model is None:
return (
{
"kind": "huggingface_hub",
"model_id": config["model"],
"revision": config["revision"],
},
True,
)
local_path = Path(model).expanduser().resolve()
if bool(local_source_model) != bool(local_source_revision):
raise ValueError("--local-source-model and --local-source-revision must be specified together.")
provenance = {
"kind": "local_checkpoint",
"checkpoint_fingerprint": _directory_fingerprint(local_path),
"claimed_source_model": local_source_model,
"claimed_source_revision": local_source_revision,
"verification": "unverified-local-copy",
}
# A directory hash proves which local bytes were used, but not that those
# bytes match a claimed Hub commit. Such output is useful for development
# comparisons but must never be uploaded as the canonical frozen reference.
return provenance, False
def generate(
variant: str,
task: str,
model: str | None,
input_image: Path | None,
output_root: Path,
local_source_model: str | None = None,
local_source_revision: str | None = None,
) -> Path:
config = dict(VARIANTS[variant])
model_provenance, publishable = _model_provenance(
config,
model,
local_source_model,
local_source_revision,
)
repository_provenance = _repository_provenance()
if publishable and repository_provenance["dirty"]:
raise RuntimeError("Refusing to generate publishable golden assets from a dirty vllm-omni worktree.")
if model is not None:
config["model"] = str(Path(model).expanduser().resolve())
config["revision"] = None
output_dir = output_root / variant / task
output_dir.mkdir(parents=True, exist_ok=True)
normalized_image: Image.Image | None = None
input_path: Path | None = None
if task == "i2v":
if input_image is None:
raise ValueError("--input-image is required for task=i2v.")
input_path = output_dir / "input.png"
normalized_image = _normalize_input_image(input_image, input_path)
elif input_image is not None:
raise ValueError("--input-image is only valid for task=i2v.")
pipeline_class = SanaImageToVideoPipeline if task == "i2v" else SanaVideoPipeline
pipe = pipeline_class.from_pretrained(
config["model"],
revision=config["revision"],
torch_dtype=torch.bfloat16,
)
pipe.text_encoder.to(torch.bfloat16)
pipe.vae.to(config["vae_dtype"])
pipe.to("cuda")
torch.manual_seed(1234)
hidden_states = torch.randn(config["transformer_case_shape"], dtype=torch.bfloat16, device="cuda")
encoder_hidden_states = torch.randn(1, 8, 2304, dtype=torch.bfloat16, device="cuda")
encoder_attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 0]], device="cuda")
timestep = torch.tensor([500.0], device="cuda")
with torch.inference_mode():
transformer_output = pipe.transformer(
hidden_states,
encoder_hidden_states,
timestep,
encoder_attention_mask=encoder_attention_mask,
).sample
transformer_path = output_dir / "transformer_case.safetensors"
save_file(
{
"hidden_states": _cpu(hidden_states),
"encoder_hidden_states": _cpu(encoder_hidden_states),
"encoder_attention_mask": _cpu(encoder_attention_mask),
"timestep": _cpu(timestep),
"output": _cpu(transformer_output),
},
transformer_path,
)
final_latents: dict[str, torch.Tensor] = {}
def capture_final_latents(_pipe, step, _timestep, callback_kwargs):
if step == NUM_INFERENCE_STEPS - 1:
final_latents["latents"] = _cpu(callback_kwargs["latents"])
return callback_kwargs
torch.accelerator.reset_peak_memory_stats()
generator = torch.Generator(device="cuda").manual_seed(SEED)
generation_kwargs = {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": config["height"],
"width": config["width"],
"frames": NUM_FRAMES,
"guidance_scale": GUIDANCE_SCALE,
"num_inference_steps": NUM_INFERENCE_STEPS,
"generator": generator,
"output_type": "np",
"callback_on_step_end": capture_final_latents,
"callback_on_step_end_tensor_inputs": ["latents"],
}
if normalized_image is not None:
generation_kwargs["image"] = normalized_image
torch.accelerator.synchronize()
started = time.perf_counter()
with torch.inference_mode():
frames = pipe(**generation_kwargs).frames[0]
torch.accelerator.synchronize()
generation_seconds = time.perf_counter() - started
if "latents" not in final_latents:
raise RuntimeError("Diffusers did not expose the final denoised latents to the generation callback.")
video_path = output_dir / "pipeline.mp4"
export_to_video(frames, video_path, fps=FPS)
reference_path = output_dir / "pipeline_reference.safetensors"
save_file(
{
"final_latents": final_latents["latents"],
"decoded_frames_uint8": _frames_to_uint8(frames),
},
reference_path,
)
scheduler_config = dict(pipe.scheduler.config)
input_metadata = None
if input_path is not None:
input_metadata = {
"filename": input_path.name,
"sha256": _sha256(input_path),
"size": input_path.stat().st_size,
"format": "png-rgb",
}
metadata = {
"schema_version": SCHEMA_VERSION,
"variant": variant,
"task": task,
"publishable": publishable,
"reference_implementation": {
"library": "diffusers",
"version": __import__("diffusers").__version__,
"pipeline_class": pipeline_class.__name__,
"torch_version": torch.__version__,
},
"model_provenance": model_provenance,
"generator_provenance": repository_provenance,
"scheduler": {
"class": type(pipe.scheduler).__name__,
"config_sha256": _json_sha256(scheduler_config),
"config": scheduler_config,
},
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"seed": SEED,
"generator_device": "cuda",
"dtype": "bfloat16",
"vae_dtype": str(config["vae_dtype"]).removeprefix("torch."),
"height": config["height"],
"width": config["width"],
"num_frames": NUM_FRAMES,
"fps": FPS,
"num_inference_steps": NUM_INFERENCE_STEPS,
"guidance_scale": GUIDANCE_SCALE,
"input_image": input_metadata,
"generation_seconds": generation_seconds,
"peak_reserved_gib": torch.accelerator.max_memory_reserved() / 1024**3,
}
metadata_path = output_dir / "metadata.json"
metadata_path.write_text(json.dumps(metadata, indent=2, default=str) + "\n")
artifact_paths = [transformer_path, video_path, reference_path, metadata_path]
if input_path is not None:
artifact_paths.append(input_path)
manifest = {
"schema_version": SCHEMA_VERSION,
"variant": variant,
"task": task,
"publishable": publishable,
"model_provenance": model_provenance,
"metadata_sha256": _sha256(metadata_path),
"files": {path.name: {"sha256": _sha256(path), "size": path.stat().st_size} for path in artifact_paths},
}
manifest_path = output_dir / "manifest.json"
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n")
return manifest_path
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--variant", choices=VARIANTS, required=True)
parser.add_argument("--task", choices=("t2v", "i2v"), required=True)
parser.add_argument(
"--model",
help=(
"Optional local checkpoint. Local output is fingerprinted and marked development-only; "
"it cannot be published as the canonical golden."
),
)
parser.add_argument("--input-image", type=Path, help="Fixed local RGB input for task=i2v.")
parser.add_argument("--local-source-model", help="Claimed upstream model ID for local checkpoint audit records.")
parser.add_argument("--local-source-revision", help="Claimed upstream commit for local checkpoint audit records.")
parser.add_argument("--output-root", type=Path, default=Path("/tmp/sana-video-goldens-v2"))
args = parser.parse_args()
print(
generate(
args.variant,
args.task,
args.model,
args.input_image,
args.output_root,
args.local_source_model,
args.local_source_revision,
)
)
if __name__ == "__main__":
main()
@@ -0,0 +1,328 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import base64
import hashlib
import json
import os
from pathlib import Path
from typing import Any
import pytest
import requests
import torch
from safetensors.torch import load_file
from torch import nn
from tests.e2e.accuracy.helpers import assert_video_metadata, assert_video_similarity_metrics, probe_video
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
GOLDEN_BASE_URL = os.getenv("SANA_VIDEO_GOLDEN_BASE_URL")
SCHEMA_VERSION = 2
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
NEGATIVE_PROMPT = (
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
"jitter, and ghosting effects, creating a disorienting visual experience."
)
VARIANTS = {
"480p": ("Efficient-Large-Model/SANA-Video_2B_480p_diffusers", 480, 832),
"720p": ("Efficient-Large-Model/SANA-Video_2B_720p_diffusers", 704, 1280),
}
REVISIONS = {
"480p": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"720p": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
}
SSIM_THRESHOLDS = {
("480p", "t2v"): 0.91,
("480p", "i2v"): 0.93,
("720p", "t2v"): 0.93,
("720p", "i2v"): 0.93,
}
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.skipif(
not GOLDEN_BASE_URL,
reason="Set SANA_VIDEO_GOLDEN_BASE_URL after publishing the frozen v2 S3 assets.",
),
]
class _TransformerCheckpoint(nn.Module):
def __init__(self, transformer: nn.Module, model: str, revision: str) -> None:
super().__init__()
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
self.transformer = transformer
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=model,
subfolder="transformer",
revision=revision,
prefix="transformer.",
fall_back_to_pt=True,
)
]
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _json_sha256(payload: Any) -> str:
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
return hashlib.sha256(encoded).hexdigest()
def _assert_hex_digest(value: Any, *, length: int = 64) -> None:
assert isinstance(value, str) and len(value) == length
assert all(character in "0123456789abcdef" for character in value)
def _validate_manifest(payload: Any, variant: str, task: str) -> dict[str, Any]:
assert isinstance(payload, dict)
assert payload["schema_version"] == SCHEMA_VERSION
assert payload["variant"] == variant
assert payload["task"] == task
# Local checkpoint generation is deliberately marked non-publishable. Do
# not permit such output to become the canonical reference by accident.
assert payload["publishable"] is True
assert payload["model_provenance"] == {
"kind": "huggingface_hub",
"model_id": VARIANTS[variant][0],
"revision": REVISIONS[variant],
}
_assert_hex_digest(payload["metadata_sha256"])
expected_files = {
"metadata.json",
"pipeline.mp4",
"pipeline_reference.safetensors",
"transformer_case.safetensors",
}
if task == "i2v":
expected_files.add("input.png")
files = payload["files"]
assert isinstance(files, dict)
assert set(files) == expected_files
for filename, expected in files.items():
# Prevent a compromised manifest from writing outside output_dir.
assert filename == Path(filename).name
assert isinstance(expected, dict) and set(expected) == {"sha256", "size"}
_assert_hex_digest(expected["sha256"])
assert isinstance(expected["size"], int) and expected["size"] > 0
return payload
def _validate_metadata(payload: Any, manifest: dict[str, Any], variant: str, task: str) -> None:
assert isinstance(payload, dict)
_, height, width = VARIANTS[variant]
assert payload["schema_version"] == SCHEMA_VERSION
assert payload["variant"] == variant
assert payload["task"] == task
assert payload["publishable"] is True
assert payload["model_provenance"] == manifest["model_provenance"]
assert payload["prompt"] == PROMPT
assert payload["negative_prompt"] == NEGATIVE_PROMPT
assert payload["seed"] == 42
assert payload["generator_device"] == "cuda"
assert payload["dtype"] == "bfloat16"
assert payload["height"] == height
assert payload["width"] == width
assert payload["num_frames"] == 81
assert payload["fps"] == 16
assert payload["num_inference_steps"] == 50
assert payload["guidance_scale"] == 6.0
implementation = payload["reference_implementation"]
assert implementation["library"] == "diffusers"
assert implementation["pipeline_class"] == ("SanaImageToVideoPipeline" if task == "i2v" else "SanaVideoPipeline")
assert implementation["version"]
assert implementation["torch_version"]
generator = payload["generator_provenance"]
assert generator["repository"] == "vllm-project/vllm-omni"
_assert_hex_digest(generator["revision"], length=40)
_assert_hex_digest(generator["generator_sha256"])
assert generator["dirty"] is False
scheduler = payload["scheduler"]
assert scheduler["class"] == "DPMSolverMultistepScheduler"
_assert_hex_digest(scheduler["config_sha256"])
assert scheduler["config_sha256"] == _json_sha256(scheduler["config"])
input_image = payload["input_image"]
if task == "i2v":
assert input_image["filename"] == "input.png"
_assert_hex_digest(input_image["sha256"])
assert input_image["size"] == manifest["files"]["input.png"]["size"]
assert input_image["sha256"] == manifest["files"]["input.png"]["sha256"]
assert input_image["format"] == "png-rgb"
else:
assert input_image is None
def _download_case(variant: str, task: str, output_dir: Path, filenames: set[str]) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
base_url = f"{GOLDEN_BASE_URL.rstrip('/')}/{variant}/{task}"
response = requests.get(f"{base_url}/manifest.json", timeout=60)
response.raise_for_status()
payload = _validate_manifest(response.json(), variant, task)
filenames = filenames | {"metadata.json"}
if task == "i2v":
filenames.add("input.png")
assert filenames <= payload["files"].keys()
for filename in filenames:
expected = payload["files"][filename]
path = output_dir / filename
response = requests.get(f"{base_url}/{filename}", timeout=300)
response.raise_for_status()
path.write_bytes(response.content)
assert path.stat().st_size == expected["size"]
assert _sha256(path) == expected["sha256"]
assert _sha256(output_dir / "metadata.json") == payload["metadata_sha256"]
metadata = json.loads((output_dir / "metadata.json").read_text())
_validate_metadata(metadata, payload, variant, task)
return payload
@pytest.mark.benchmark
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_transformer_matches_frozen_golden(
variant: str,
_hardware,
accuracy_artifact_root: Path,
) -> None:
del _hardware
from vllm.config import LoadConfig
from vllm.transformers_utils.config import get_hf_file_to_dict
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
model, _, _ = VARIANTS[variant]
revision = REVISIONS[variant]
output_dir = accuracy_artifact_root / "sana_video" / variant / "t2v"
_download_case(variant, "t2v", output_dir, {"transformer_case.safetensors"})
case = load_file(output_dir / "transformer_case.safetensors")
transformer_config = get_hf_file_to_dict("transformer/config.json", model, revision=revision)
assert transformer_config is not None
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(
device="cuda",
dtype=torch.bfloat16,
)
checkpoint = _TransformerCheckpoint(transformer, model, revision)
od_config = OmniDiffusionConfig(
model=model,
model_class_name="SanaVideoPipeline",
dtype=torch.bfloat16,
revision=revision,
)
loader = DiffusersPipelineLoader(LoadConfig(), od_config=od_config)
transformer.load_weights(
(name.removeprefix("transformer."), tensor)
for name, tensor in loader.get_all_weights(checkpoint)
if name.startswith("transformer.")
)
transformer.eval()
with torch.inference_mode():
actual = transformer(
case["hidden_states"].to("cuda"),
case["encoder_hidden_states"].to("cuda"),
case["timestep"].to("cuda"),
encoder_attention_mask=case["encoder_attention_mask"].to("cuda"),
).sample.float()
expected = case["output"].to("cuda").float()
error = actual - expected
max_abs = error.abs().max()
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected)
cosine = torch.nn.functional.cosine_similarity(actual.flatten(), expected.flatten(), dim=0)
# CUDNN_ATTN BF16 reduction order is not bitwise deterministic on
# Blackwell. Bound rare outliers separately while keeping the aggregate
# relative-L2 and cosine checks strict.
max_abs_threshold = 0.2 if variant == "480p" else 0.1
relative_l2_threshold = 0.017 if variant == "480p" else 0.015
assert max_abs.item() <= max_abs_threshold
assert relative_l2.item() <= relative_l2_threshold
assert cosine.item() >= 0.999
SERVER_CASES = [
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--model-class-name",
pipeline_class,
],
),
id=f"{variant}-{task}",
marks=hardware_marks(res={"cuda": "H100"}),
)
for variant, (model, _, _) in VARIANTS.items()
for task, pipeline_class in (
("t2v", "SanaVideoPipeline"),
("i2v", "SanaImageToVideoPipeline"),
)
]
@pytest.mark.benchmark
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_sana_video_pipeline_matches_frozen_golden(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
accuracy_artifact_root: Path,
) -> None:
variant = "720p" if "720p" in omni_server.model else "480p"
task = "i2v" if "SanaImageToVideoPipeline" in omni_server.serve_args else "t2v"
_, height, width = VARIANTS[variant]
output_dir = accuracy_artifact_root / "sana_video" / variant / task
_download_case(variant, task, output_dir, {"pipeline.mp4"})
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": height,
"width": width,
"num_frames": 81,
"fps": 16,
"num_inference_steps": 50,
"guidance_scale": 6.0,
"seed": 42,
},
}
if task == "i2v":
encoded = base64.b64encode((output_dir / "input.png").read_bytes()).decode()
request_config["image_reference"] = f"data:image/png;base64,{encoded}"
result = openai_client.send_video_diffusion_request(request_config)[0]
actual_path = output_dir / "actual.mp4"
actual_path.write_bytes(result.videos[0])
golden_path = output_dir / "pipeline.mp4"
metadata = probe_video(actual_path)
assert_video_metadata(metadata, width=width, height=height, fps=16, frame_count=81)
assert_video_similarity_metrics(
label=f"sana_video_{variant}_{task}",
online_path=actual_path,
offline_path=golden_path,
# The native prompt cross-attention uses vLLM-Omni Attention rather
# than Diffusers SDPA. Frozen stage checks prove exact inputs and
# scheduler state, while the BF16 backend reduction difference grows
# across CFG denoising. Calibrate only the affected 480p T2V case;
# its frozen result is 0.919861 SSIM / 28.7442 dB PSNR.
ssim_threshold=SSIM_THRESHOLDS[(variant, task)],
psnr_threshold=28.0,
)
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"vae_class": "AutoencoderKLWan",
"native_vae_class": "DistributedAutoencoderKLWan",
"dtype": torch.float32,
"latent_channels": 16,
"spatial_scale": 8,
"temporal_scale": 4,
"frames": 5,
"max_abs": 1e-6,
"relative_l2": 1e-6,
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"vae_class": "AutoencoderKLLTX2Video",
"native_vae_class": "DistributedAutoencoderKLLTX2Video",
"dtype": torch.bfloat16,
"latent_channels": 128,
"spatial_scale": 32,
"temporal_scale": 8,
"frames": 9,
"max_abs": 0.04,
"relative_l2": 0.004,
},
}
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_i2v_first_frame_latents_match_diffusers(variant: str, _hardware) -> None:
del _hardware
from diffusers import (
AutoencoderKLLTX2Video,
AutoencoderKLWan,
)
from diffusers import (
SanaImageToVideoPipeline as DiffusersSanaImageToVideoPipeline,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
DistributedAutoencoderKLLTX2Video,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
DistributedAutoencoderKLWan,
)
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
config = VARIANTS[variant]
vae_classes = {
"AutoencoderKLLTX2Video": AutoencoderKLLTX2Video,
"AutoencoderKLWan": AutoencoderKLWan,
}
native_vae_classes = {
"DistributedAutoencoderKLLTX2Video": DistributedAutoencoderKLLTX2Video,
"DistributedAutoencoderKLWan": DistributedAutoencoderKLWan,
}
reference_vae = (
vae_classes[config["vae_class"]]
.from_pretrained(
config["model"],
subfolder="vae",
torch_dtype=config["dtype"],
)
.to("cuda")
)
native_vae = (
native_vae_classes[config["native_vae_class"]]
.from_config(dict(reference_vae.config))
.to(device="cuda", dtype=config["dtype"])
)
native_vae.load_state_dict(reference_vae.state_dict())
reference_vae.eval()
native_vae.eval()
height, width = 192, 320
num_latent_frames = (config["frames"] - 1) // config["temporal_scale"] + 1
image = torch.linspace(
-1,
1,
3 * height * width,
dtype=torch.float32,
device="cuda",
).reshape(1, 3, height, width)
noise = torch.randn(
(
1,
config["latent_channels"],
num_latent_frames,
height // config["spatial_scale"],
width // config["spatial_scale"],
),
generator=torch.Generator(device="cuda").manual_seed(7),
device="cuda",
dtype=torch.float32,
)
reference_pipeline = SimpleNamespace(
vae=reference_vae,
vae_scale_factor_temporal=config["temporal_scale"],
vae_scale_factor_spatial=config["spatial_scale"],
)
native_pipeline = SimpleNamespace(
vae=native_vae,
vae_scale_factor_temporal=config["temporal_scale"],
vae_scale_factor_spatial=config["spatial_scale"],
device=torch.device("cuda"),
)
with torch.inference_mode():
expected = DiffusersSanaImageToVideoPipeline.prepare_latents(
reference_pipeline,
image,
1,
config["latent_channels"],
height,
width,
config["frames"],
torch.float32,
torch.device("cuda"),
None,
noise.clone(),
)
actual = SanaImageToVideoPipeline._prepare_i2v_latents(
native_pipeline,
image,
1,
config["latent_channels"],
height,
width,
config["frames"],
torch.float32,
None,
noise.clone(),
)
# Conditioning replaces only the first latent frame.
torch.testing.assert_close(actual[:, :, 1:], noise[:, :, 1:], rtol=0, atol=0)
error = (actual[:, :, :1] - expected[:, :, :1]).float()
expected_first_frame = expected[:, :, :1].float()
max_abs = error.abs().max()
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected_first_frame)
cosine = torch.nn.functional.cosine_similarity(
actual[:, :, :1].flatten(),
expected[:, :, :1].flatten(),
dim=0,
)
assert max_abs.item() <= config["max_abs"]
assert relative_l2.item() <= config["relative_l2"]
assert cosine.item() >= 0.9999
@@ -0,0 +1,432 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import gc
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import pytest
import torch
import torch.nn.functional as F
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
PROMPT = "A small red panda walks through a bamboo forest."
NEGATIVE_PROMPT = "blur, distortion, abrupt cuts"
SEED = 42
GUIDANCE_SCALE = 6.0
NUM_INFERENCE_STEPS = 2
HEIGHT = 192
WIDTH = 320
NUM_FRAMES = 9
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"model_env": "SANA_VIDEO_480P_MODEL",
"max_abs": 0.2,
"relative_l2": 0.02,
"cosine": 0.999,
"cfg_max_abs": 0.5,
"cfg_relative_l2": 0.05,
"cfg_cosine": 0.999,
"step_max_abs": 0.075,
"step_relative_l2": 0.012,
"step_cosine": 0.9999,
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
"model_env": "SANA_VIDEO_720P_MODEL",
"max_abs": 0.1,
"relative_l2": 0.015,
"cosine": 0.999,
"cfg_max_abs": 0.6,
"cfg_relative_l2": 0.065,
"cfg_cosine": 0.998,
"step_max_abs": 0.08,
"step_relative_l2": 0.014,
"step_cosine": 0.9999,
},
}
@dataclass(frozen=True)
class ErrorMetrics:
shape: tuple[int, ...]
actual_dtype: torch.dtype
expected_dtype: torch.dtype
max_abs: float
mean_abs: float
relative_l2: float
cosine: float
def __str__(self) -> str:
return (
f"shape={self.shape}, actual_dtype={self.actual_dtype}, "
f"expected_dtype={self.expected_dtype}, max_abs={self.max_abs:.8g}, "
f"mean_abs={self.mean_abs:.8g}, relative_l2={self.relative_l2:.8g}, "
f"cosine={self.cosine:.8g}"
)
def _metrics(actual: torch.Tensor, expected: torch.Tensor) -> ErrorMetrics:
assert actual.shape == expected.shape
actual_float = actual.detach().float()
expected_float = expected.detach().float()
error = actual_float - expected_float
expected_norm = torch.linalg.vector_norm(expected_float)
relative_l2 = torch.linalg.vector_norm(error) / expected_norm.clamp_min(torch.finfo(torch.float32).tiny)
if actual_float.numel() == 1:
cosine = torch.tensor(
1.0 if bool(torch.equal(actual_float, expected_float)) else 0.0,
device=actual.device,
)
else:
cosine = F.cosine_similarity(actual_float.flatten(), expected_float.flatten(), dim=0)
return ErrorMetrics(
shape=tuple(actual.shape),
actual_dtype=actual.dtype,
expected_dtype=expected.dtype,
max_abs=error.abs().max().item(),
mean_abs=error.abs().mean().item(),
relative_l2=relative_l2.item(),
cosine=cosine.item(),
)
def _assert_stage(
stage: str,
actual: torch.Tensor,
expected: torch.Tensor,
*,
max_abs: float,
relative_l2: float,
cosine: float,
) -> ErrorMetrics:
result = _metrics(actual, expected)
print(f"\nSANA-Video alignment [{stage}]: {result}")
assert result.actual_dtype == result.expected_dtype, f"{stage}: {result}"
assert result.max_abs <= max_abs, f"{stage}: {result}"
assert result.relative_l2 <= relative_l2, f"{stage}: {result}"
assert result.cosine >= cosine, f"{stage}: {result}"
return result
def _model_source(variant: str) -> tuple[str, str | None]:
config = VARIANTS[variant]
model = os.getenv(config["model_env"], config["model"])
revision = None if Path(model).exists() else config["revision"]
return model, revision
def _native_method_proxy(reference_pipeline: Any):
"""Build the minimum native-pipeline surface needed by stage helpers.
Sharing the tokenizer and text encoder makes this a comparison of pipeline
orchestration rather than a second copy of the same multi-billion-parameter
encoder. Transformer execution still uses separate Diffusers and native
modules below.
"""
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
proxy = object.__new__(SanaVideoPipeline)
torch.nn.Module.__init__(proxy)
proxy.tokenizer = reference_pipeline.tokenizer
proxy.text_encoder = reference_pipeline.text_encoder
proxy.device = torch.device("cuda")
proxy.vae_scale_factor_temporal = reference_pipeline.vae_scale_factor_temporal
proxy.vae_scale_factor_spatial = reference_pipeline.vae_scale_factor_spatial
return proxy
def _cfg_prediction(
prediction: torch.Tensor,
*,
guidance_scale: float,
latent_channels: int,
out_channels: int,
) -> torch.Tensor:
unconditional, conditional = prediction.float().chunk(2)
prediction = unconditional + guidance_scale * (conditional - unconditional)
if out_channels // 2 == latent_channels:
prediction = prediction.chunk(2, dim=1)[0]
return prediction
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_quick_pipeline_stages_match_diffusers(
variant: str,
_hardware,
) -> None:
"""Compare the first denoising step without replacing native attention.
This deliberately stops after the first scheduler step. It keeps the test
useful as a stage-localizing PR/nightly check while the frozen 81-frame,
50-step golden test covers the final latent and decoded-video boundary.
"""
del _hardware
from diffusers import SanaVideoPipeline as DiffusersSanaVideoPipeline
from diffusers.pipelines.sana_video.pipeline_sana_video import (
retrieve_timesteps as diffusers_retrieve_timesteps,
)
from diffusers.schedulers import DPMSolverMultistepScheduler
from vllm_omni.diffusion.models.sana_video import (
SanaVideoPipeline,
SanaVideoTransformer3DModel,
)
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
retrieve_timesteps as native_retrieve_timesteps,
)
model, revision = _model_source(variant)
config = VARIANTS[variant]
load_kwargs: dict[str, Any] = {
"torch_dtype": torch.bfloat16,
# No stage in this test decodes latents. Avoid loading a multi-GB VAE
# that would add memory and runtime without increasing coverage.
"vae": None,
}
if revision is not None:
load_kwargs["revision"] = revision
reference = DiffusersSanaVideoPipeline.from_pretrained(model, **load_kwargs).to("cuda")
assert type(reference) is DiffusersSanaVideoPipeline
assert type(reference.transformer).__module__.startswith("diffusers.")
native_proxy = _native_method_proxy(reference)
assert type(native_proxy) is SanaVideoPipeline
assert type(native_proxy).__module__.startswith("vllm_omni.")
try:
with torch.inference_mode():
reference_prompt = reference.encode_prompt(
PROMPT,
True,
negative_prompt=NEGATIVE_PROMPT,
device=torch.device("cuda"),
clean_caption=False,
max_sequence_length=300,
complex_human_instruction=None,
)
native_prompt = SanaVideoPipeline.encode_prompt(
native_proxy,
PROMPT,
True,
negative_prompt=NEGATIVE_PROMPT,
device=torch.device("cuda"),
clean_caption=False,
max_sequence_length=300,
complex_human_instruction=None,
)
prompt_stage_names = (
"prompt_embeddings",
"prompt_attention_mask",
"negative_prompt_embeddings",
"negative_prompt_attention_mask",
)
for stage, actual, expected in zip(prompt_stage_names, native_prompt, reference_prompt, strict=True):
_assert_stage(
stage,
actual,
expected,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
reference_prompt_embeds, reference_prompt_mask, reference_negative_embeds, reference_negative_mask = (
reference_prompt
)
encoder_hidden_states = torch.cat([reference_negative_embeds, reference_prompt_embeds])
encoder_attention_mask = torch.cat([reference_negative_mask, reference_prompt_mask])
latent_channels = reference.transformer.config.in_channels
reference_latents = reference.prepare_latents(
1,
latent_channels,
HEIGHT,
WIDTH,
NUM_FRAMES,
torch.float32,
torch.device("cuda"),
torch.Generator(device="cuda").manual_seed(SEED),
)
native_latents = SanaVideoPipeline.prepare_latents(
native_proxy,
1,
latent_channels,
HEIGHT,
WIDTH,
NUM_FRAMES,
torch.float32,
torch.device("cuda"),
torch.Generator(device="cuda").manual_seed(SEED),
)
_assert_stage(
"initial_noise_latents",
native_latents,
reference_latents,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
reference_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
native_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
isolated_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
reference_timesteps, reference_num_steps = diffusers_retrieve_timesteps(
reference_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
native_timesteps, native_num_steps = native_retrieve_timesteps(
native_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
isolated_retrieve_timesteps, isolated_num_steps = native_retrieve_timesteps(
isolated_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
assert reference_num_steps == native_num_steps == isolated_num_steps
_assert_stage(
"timesteps",
native_timesteps,
reference_timesteps,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
_assert_stage(
"sigmas",
native_scheduler.sigmas,
reference_scheduler.sigmas,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
torch.testing.assert_close(isolated_retrieve_timesteps, reference_timesteps, rtol=0, atol=0)
native_transformer = SanaVideoTransformer3DModel.from_config(dict(reference.transformer.config)).to(
device="cuda",
dtype=torch.bfloat16,
)
incompatible = native_transformer.load_state_dict(reference.transformer.state_dict(), strict=True)
assert not incompatible.missing_keys
assert not incompatible.unexpected_keys
assert type(native_transformer) is SanaVideoTransformer3DModel
assert type(native_transformer).__module__.startswith("vllm_omni.")
assert type(native_transformer) is not type(reference.transformer)
reference_parameter = next(reference.transformer.parameters())
native_parameter = next(native_transformer.parameters())
assert native_parameter.data_ptr() != reference_parameter.data_ptr()
torch.testing.assert_close(native_parameter, reference_parameter, rtol=0, atol=0)
native_transformer.eval()
reference.transformer.eval()
latent_model_input = torch.cat([reference_latents, reference_latents])
timestep = reference_timesteps[0].expand(latent_model_input.shape[0])
with torch.inference_mode():
reference_raw_prediction = reference.transformer(
latent_model_input.to(dtype=torch.bfloat16),
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
encoder_attention_mask=encoder_attention_mask,
timestep=timestep,
return_dict=False,
)[0]
native_raw_prediction = native_transformer(
latent_model_input.to(dtype=torch.bfloat16),
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
encoder_attention_mask=encoder_attention_mask,
timestep=timestep,
return_dict=False,
)[0]
_assert_stage(
"first_transformer_raw_prediction",
native_raw_prediction,
reference_raw_prediction,
max_abs=config["max_abs"],
relative_l2=config["relative_l2"],
cosine=config["cosine"],
)
reference_cfg_prediction = _cfg_prediction(
reference_raw_prediction,
guidance_scale=GUIDANCE_SCALE,
latent_channels=latent_channels,
out_channels=reference.transformer.config.out_channels,
)
native_cfg_prediction = _cfg_prediction(
native_raw_prediction,
guidance_scale=GUIDANCE_SCALE,
latent_channels=latent_channels,
out_channels=native_transformer.config.out_channels,
)
_assert_stage(
"first_cfg_prediction",
native_cfg_prediction,
reference_cfg_prediction,
max_abs=config["cfg_max_abs"],
relative_l2=config["cfg_relative_l2"],
cosine=config["cfg_cosine"],
)
reference_first_step = reference_scheduler.step(
reference_cfg_prediction,
reference_timesteps[0],
reference_latents,
return_dict=False,
)[0]
# Isolate scheduler orchestration from native-attention error by
# feeding the Diffusers prediction through an independently configured
# scheduler initialized by the native helper.
isolated_first_step = isolated_scheduler.step(
reference_cfg_prediction,
isolated_retrieve_timesteps[0],
reference_latents,
return_dict=False,
)[0]
_assert_stage(
"first_scheduler_step_isolated",
isolated_first_step,
reference_first_step,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
native_first_step = native_scheduler.step(
native_cfg_prediction,
native_timesteps[0],
native_latents,
return_dict=False,
)[0]
_assert_stage(
"first_scheduler_step_end_to_end",
native_first_step,
reference_first_step,
max_abs=config["step_max_abs"],
relative_l2=config["step_relative_l2"],
cosine=config["step_cosine"],
)
finally:
del native_proxy
del reference
gc.collect()
torch.accelerator.empty_cache()
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SANA-Video 2B online smoke tests for the native and Diffusers backends."""
import os
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL_480P = os.environ.get(
"SANA_VIDEO_480P_MODEL",
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
)
MODEL_720P = os.environ.get(
"SANA_VIDEO_720P_MODEL",
"Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
)
PROMPT = "A cat walking on grass toward the camera. motion score: 30."
NEGATIVE_PROMPT = "blurry, low quality, temporal artifacts"
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
def _t2v_backend_cases():
cases = []
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
cases.extend(
[
pytest.param(
OmniServerParams(model=model, server_args=["--model-class-name", "SanaVideoPipeline"]),
id=f"native-{variant}",
marks=SINGLE_CARD_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--diffusion-load-format",
"diffusers",
"--diffusion-attention-backend",
"TORCH_SDPA",
],
),
id=f"diffusers-adapter-{variant}",
marks=SINGLE_CARD_MARKS,
),
]
)
return cases
@pytest.mark.parametrize("omni_server", _t2v_backend_cases(), indirect=True)
def test_sana_video_t2v_backends(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 192,
"width": 320,
"num_frames": 9,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 4.0,
"seed": 42,
},
}
openai_client.send_video_diffusion_request(request_config)
def _i2v_cases():
cases = []
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
cases.extend(
[
pytest.param(
OmniServerParams(model=model, server_args=["--model-class-name", "SanaImageToVideoPipeline"]),
id=f"native-{variant}",
marks=SINGLE_CARD_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--model-class-name",
"SanaImageToVideoPipeline",
"--diffusion-load-format",
"diffusers",
"--diffusion-attention-backend",
"TORCH_SDPA",
],
),
id=f"diffusers-adapter-{variant}",
marks=SINGLE_CARD_MARKS,
),
]
)
return cases
@pytest.mark.parametrize("omni_server", _i2v_cases(), indirect=True)
def test_sana_video_i2v_variants(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 192,
"width": 320,
"num_frames": 9,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 4.0,
"seed": 42,
},
"image_reference": (f"data:image/jpeg;base64,{generate_synthetic_image(320, 192, seed=42)['base64']}"),
}
openai_client.send_video_diffusion_request(request_config)
@@ -46,6 +46,9 @@ def test_text_to_video_builds_canonical_prompt(
@pytest.mark.parametrize(
("model", "model_class_name", "preset_name"),
[
("Efficient-Large-Model/SANA-Video_2B_480p_diffusers", None, "sana_480p"),
("Efficient-Large-Model/SANA-Video_2B_720p_diffusers", None, "sana_720p"),
("/models/custom-checkpoint", "SanaVideoPipeline", "sana_480p"),
("robbyant/lingbot-video-dense-1.3b", None, "lingbot"),
("/models/custom-checkpoint", "LingBotVideoPipeline", "lingbot"),
("Lightricks/LTX-2", "LTX2DistilledPipeline", "ltx2_distilled"),
@@ -1,7 +1,16 @@
from functools import partial
from pathlib import Path
from tests.helpers.tiny_model import build_tiny_from_configs
import torch
from diffusers import (
AutoencoderKLWan,
DPMSolverMultistepScheduler,
SanaVideoPipeline,
SanaVideoTransformer3DModel,
)
from transformers import Gemma2Config, Gemma2Model, GemmaTokenizerFast
from tests.helpers.tiny_model import _get_tiny_model_path, build_tiny_from_configs
TINY_CONFIGS_DIR = Path(__file__).parent / "tiny_configs"
@@ -40,6 +49,72 @@ def tiny_ltx2_builder() -> str:
return build_tiny_from_configs("LTX2Pipeline", "Lightricks/LTX-2", TINY_CONFIGS_DIR / "LTX2Pipeline")
def tiny_sana_video_builder() -> str:
"""Build a tiny 480p SANA-Video model without downloading model weights.
The tokenizer is the only component loaded from the upstream repository.
All modules with weights are initialized locally from intentionally small
configs, and the scheduler is constructed from its weight-free config.
"""
model_id = "Efficient-Large-Model/SANA-Video_2B_480p_diffusers"
model_dir = _get_tiny_model_path("SanaVideoPipeline")
tokenizer = GemmaTokenizerFast.from_pretrained(model_id, subfolder="tokenizer")
scheduler = DPMSolverMultistepScheduler(
algorithm_type="dpmsolver++",
flow_shift=8.0,
prediction_type="flow_prediction",
use_flow_sigmas=True,
)
text_encoder = Gemma2Model(
Gemma2Config(
vocab_size=256000,
hidden_size=32,
intermediate_size=64,
num_hidden_layers=1,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=8,
max_position_embeddings=512,
sliding_window=256,
layer_types=["sliding_attention"],
)
)
transformer = SanaVideoTransformer3DModel(
in_channels=16,
out_channels=16,
num_attention_heads=2,
attention_head_dim=16,
num_layers=1,
num_cross_attention_heads=2,
cross_attention_head_dim=16,
cross_attention_dim=32,
caption_channels=32,
mlp_ratio=2.0,
sample_size=30,
patch_size=(1, 2, 2),
)
vae = AutoencoderKLWan(
base_dim=8,
decoder_base_dim=8,
z_dim=16,
dim_mult=[1, 1, 1, 1],
num_res_blocks=1,
temperal_downsample=[False, True, True],
latents_mean=[0.0] * 16,
latents_std=[1.0] * 16,
)
pipeline = SanaVideoPipeline(
tokenizer=tokenizer,
text_encoder=text_encoder,
vae=vae,
transformer=transformer,
scheduler=scheduler,
)
pipeline.to(dtype=torch.bfloat16).save_pretrained(model_dir)
return model_dir
def _shrink_flux_clip_text_encoder(config: dict) -> dict:
config["num_hidden_layers"] = 2
return config
@@ -49,6 +49,12 @@ DIFFUSION_TEST_SETTINGS = {
[DiffusionAccs.CFG_PARALLEL, DiffusionAccs.TENSOR_PARALLEL, DiffusionAccs.CPU_OFFLOAD],
],
),
"SanaVideoPipeline": DiffusionModelTestOpts(
model="Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
builder=diff_model_builders.tiny_sana_video_builder,
supported_tasks=[DiffusionTasks.TEXT_TO_VIDEO],
check_multi_output=False,
),
"QwenImagePipeline": DiffusionModelTestOpts(
model="Qwen/Qwen-Image",
builder=diff_model_builders.tiny_qwen_image_builder,
@@ -63,6 +63,12 @@ EXCLUDED_MODELS = [
"HunyuanVideo15Pipeline",
"HunyuanVideo15ImageToVideoPipeline",
"LingBotVideoPipeline",
# The common harness does not yet define an image-to-video task or supply
# an input image. SANA I2V is covered by
# tests/diffusion/models/sana_video/test_pipeline_sana_video.py and
# tests/e2e/online_serving/test_sana_video_expansion.py. SANA T2V is exercised through
# the common tiny-model setting in model_settings.py.
"SanaImageToVideoPipeline",
# Requires camera/action assets and persistent AR state; covered by its
# dedicated contract tests and real-checkpoint E2E smoke instead.
"LingBotWorldCausalDMDPipeline",