[Model] Add native SANA-Video 2B T2V and I2V support (#5508)

Signed-off-by: leo <92263211+napleon-liu@users.noreply.github.com>
Signed-off-by: leo <92263211+liuyao0322@users.noreply.github.com>
Signed-off-by: liuyao0322 <yaoliu548926@gmail.com>
Co-authored-by: leo <92263211+napleon-liu@users.noreply.github.com>
This commit is contained in:
leoandleo authored and GitHub committed 2026-08-25 09:46:02 +08:00
1 parent 4af0ddc1c2
commit 50b8a150f9
42 files changed
+6512 -18

No files matched your search

+63
View File
@@ -0,0 +1,63 @@
# SANA-Video golden outputs
The v2 golden set covers both text-to-video and image-to-video for each
checkpoint variant. Generate references with the Diffusers version pinned by
this repository and the immutable Hub revisions embedded in
`generate_goldens.py`:
```bash
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 480p --task t2v
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 480p --task i2v --input-image /path/to/fixed-input.png
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 720p --task t2v
python tests/e2e/accuracy/sana_video/generate_goldens.py \
--variant 720p --task i2v --input-image /path/to/fixed-input.png
```
Use the same reviewed input image for both I2V variants. The generator
normalizes it to an RGB PNG and includes that exact file and its SHA256 in the
case manifest, so an I2V reference is self-contained.
Each case is written under:
```text
<output-root>/<variant>/<task>/
```
It contains:
- `transformer_case.safetensors`: deterministic transformer inputs and output.
- `pipeline.mp4`: the encoded reference used by online similarity tests.
- `pipeline_reference.safetensors`: final denoised latents and decoded,
pre-MP4 frames quantized to the exact uint8 video input.
- `input.png`: the normalized conditioning image for I2V only.
- `metadata.json`: model, generator, scheduler, input and sampling provenance.
- `manifest.json`: the size and SHA256 of every artifact.
## Publication requirements
A canonical golden must be generated from the pinned Hugging Face model ID and
immutable commit. The generator also requires a clean vLLM-Omni worktree and
records the repository revision plus the generator script hash.
`--model` is available for local development. It hashes every file in the local
checkpoint directory and records optional `--local-source-model` and
`--local-source-revision` claims, but marks the result `publishable: false`.
The golden test rejects such a manifest. A local directory hash proves which
bytes were used; it does not prove that those bytes match a claimed upstream
commit.
After reviewing the metadata, input image and generated videos, upload each
publishable case directory without renaming files to:
```text
s3://vllm-public-assets/omni-assets/sana-video/v2/<variant>/<task>/
```
Set `SANA_VIDEO_GOLDEN_BASE_URL` to the corresponding `v2` HTTP base URL when
running the tests. Uploading is intentionally separate from generation so test
runs cannot overwrite a frozen reference. The storage location must prevent
in-place mutation; publish a new version prefix instead of replacing any v2
file when a model, dependency, input or sampling parameter changes.
@@ -0,0 +1,380 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generate frozen SANA-Video transformer and pipeline reference artifacts."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import time
from pathlib import Path
from typing import Any
import numpy as np
import torch
from diffusers import SanaImageToVideoPipeline, SanaVideoPipeline
from diffusers.utils import export_to_video
from PIL import Image
from safetensors.torch import save_file
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
NEGATIVE_PROMPT = (
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
"jitter, and ghosting effects, creating a disorienting visual experience."
)
SCHEMA_VERSION = 2
SEED = 42
NUM_FRAMES = 81
FPS = 16
NUM_INFERENCE_STEPS = 50
GUIDANCE_SCALE = 6.0
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"height": 480,
"width": 832,
"vae_dtype": torch.float32,
"transformer_case_shape": (1, 16, 3, 8, 8),
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
"height": 704,
"width": 1280,
"vae_dtype": torch.bfloat16,
"transformer_case_shape": (1, 128, 2, 4, 4),
},
}
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _json_sha256(payload: Any) -> str:
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
return hashlib.sha256(encoded).hexdigest()
def _directory_fingerprint(root: Path) -> dict[str, Any]:
"""Fingerprint every local checkpoint file without recording host paths."""
if not root.is_dir():
raise ValueError(f"Local checkpoint must be a directory: {root}")
files = []
digest = hashlib.sha256()
for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()):
relative = path.relative_to(root).as_posix()
file_sha256 = _sha256(path)
size = path.stat().st_size
entry = f"{relative}\0{size}\0{file_sha256}\n".encode()
digest.update(entry)
files.append({"path": relative, "size": size, "sha256": file_sha256})
if not files:
raise ValueError(f"Local checkpoint contains no files: {root}")
return {
"algorithm": "sha256-tree-v1",
"sha256": digest.hexdigest(),
"file_count": len(files),
"total_size": sum(entry["size"] for entry in files),
}
def _repository_provenance() -> dict[str, Any]:
repo_root = Path(__file__).resolve().parents[4]
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
dirty = bool(
subprocess.run(
["git", "status", "--porcelain", "--untracked-files=no"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
)
return {
"repository": "vllm-project/vllm-omni",
"revision": revision,
"dirty": dirty,
"generator_sha256": _sha256(Path(__file__)),
}
def _cpu(tensor: torch.Tensor) -> torch.Tensor:
return tensor.detach().cpu().contiguous()
def _normalize_input_image(source: Path, destination: Path) -> Image.Image:
if not source.is_file():
raise ValueError(f"I2V input image does not exist: {source}")
with Image.open(source) as image:
normalized = image.convert("RGB")
normalized.load()
normalized.save(destination, format="PNG", optimize=False)
return normalized
def _frames_to_uint8(frames: np.ndarray) -> torch.Tensor:
tensor = torch.from_numpy(np.asarray(frames))
if tensor.is_floating_point():
# Match diffusers.utils.export_to_video, which truncates rather than
# rounds when converting float frames to the encoder's uint8 input.
tensor = tensor.clamp(0, 1) * 255
return tensor.to(torch.uint8).contiguous()
def _model_provenance(
config: dict[str, Any],
model: str | None,
local_source_model: str | None,
local_source_revision: str | None,
) -> tuple[dict[str, Any], bool]:
if model is None:
return (
{
"kind": "huggingface_hub",
"model_id": config["model"],
"revision": config["revision"],
},
True,
)
local_path = Path(model).expanduser().resolve()
if bool(local_source_model) != bool(local_source_revision):
raise ValueError("--local-source-model and --local-source-revision must be specified together.")
provenance = {
"kind": "local_checkpoint",
"checkpoint_fingerprint": _directory_fingerprint(local_path),
"claimed_source_model": local_source_model,
"claimed_source_revision": local_source_revision,
"verification": "unverified-local-copy",
}
# A directory hash proves which local bytes were used, but not that those
# bytes match a claimed Hub commit. Such output is useful for development
# comparisons but must never be uploaded as the canonical frozen reference.
return provenance, False
def generate(
variant: str,
task: str,
model: str | None,
input_image: Path | None,
output_root: Path,
local_source_model: str | None = None,
local_source_revision: str | None = None,
) -> Path:
config = dict(VARIANTS[variant])
model_provenance, publishable = _model_provenance(
config,
model,
local_source_model,
local_source_revision,
)
repository_provenance = _repository_provenance()
if publishable and repository_provenance["dirty"]:
raise RuntimeError("Refusing to generate publishable golden assets from a dirty vllm-omni worktree.")
if model is not None:
config["model"] = str(Path(model).expanduser().resolve())
config["revision"] = None
output_dir = output_root / variant / task
output_dir.mkdir(parents=True, exist_ok=True)
normalized_image: Image.Image | None = None
input_path: Path | None = None
if task == "i2v":
if input_image is None:
raise ValueError("--input-image is required for task=i2v.")
input_path = output_dir / "input.png"
normalized_image = _normalize_input_image(input_image, input_path)
elif input_image is not None:
raise ValueError("--input-image is only valid for task=i2v.")
pipeline_class = SanaImageToVideoPipeline if task == "i2v" else SanaVideoPipeline
pipe = pipeline_class.from_pretrained(
config["model"],
revision=config["revision"],
torch_dtype=torch.bfloat16,
)
pipe.text_encoder.to(torch.bfloat16)
pipe.vae.to(config["vae_dtype"])
pipe.to("cuda")
torch.manual_seed(1234)
hidden_states = torch.randn(config["transformer_case_shape"], dtype=torch.bfloat16, device="cuda")
encoder_hidden_states = torch.randn(1, 8, 2304, dtype=torch.bfloat16, device="cuda")
encoder_attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 0]], device="cuda")
timestep = torch.tensor([500.0], device="cuda")
with torch.inference_mode():
transformer_output = pipe.transformer(
hidden_states,
encoder_hidden_states,
timestep,
encoder_attention_mask=encoder_attention_mask,
).sample
transformer_path = output_dir / "transformer_case.safetensors"
save_file(
{
"hidden_states": _cpu(hidden_states),
"encoder_hidden_states": _cpu(encoder_hidden_states),
"encoder_attention_mask": _cpu(encoder_attention_mask),
"timestep": _cpu(timestep),
"output": _cpu(transformer_output),
},
transformer_path,
)
final_latents: dict[str, torch.Tensor] = {}
def capture_final_latents(_pipe, step, _timestep, callback_kwargs):
if step == NUM_INFERENCE_STEPS - 1:
final_latents["latents"] = _cpu(callback_kwargs["latents"])
return callback_kwargs
torch.accelerator.reset_peak_memory_stats()
generator = torch.Generator(device="cuda").manual_seed(SEED)
generation_kwargs = {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": config["height"],
"width": config["width"],
"frames": NUM_FRAMES,
"guidance_scale": GUIDANCE_SCALE,
"num_inference_steps": NUM_INFERENCE_STEPS,
"generator": generator,
"output_type": "np",
"callback_on_step_end": capture_final_latents,
"callback_on_step_end_tensor_inputs": ["latents"],
}
if normalized_image is not None:
generation_kwargs["image"] = normalized_image
torch.accelerator.synchronize()
started = time.perf_counter()
with torch.inference_mode():
frames = pipe(**generation_kwargs).frames[0]
torch.accelerator.synchronize()
generation_seconds = time.perf_counter() - started
if "latents" not in final_latents:
raise RuntimeError("Diffusers did not expose the final denoised latents to the generation callback.")
video_path = output_dir / "pipeline.mp4"
export_to_video(frames, video_path, fps=FPS)
reference_path = output_dir / "pipeline_reference.safetensors"
save_file(
{
"final_latents": final_latents["latents"],
"decoded_frames_uint8": _frames_to_uint8(frames),
},
reference_path,
)
scheduler_config = dict(pipe.scheduler.config)
input_metadata = None
if input_path is not None:
input_metadata = {
"filename": input_path.name,
"sha256": _sha256(input_path),
"size": input_path.stat().st_size,
"format": "png-rgb",
}
metadata = {
"schema_version": SCHEMA_VERSION,
"variant": variant,
"task": task,
"publishable": publishable,
"reference_implementation": {
"library": "diffusers",
"version": __import__("diffusers").__version__,
"pipeline_class": pipeline_class.__name__,
"torch_version": torch.__version__,
},
"model_provenance": model_provenance,
"generator_provenance": repository_provenance,
"scheduler": {
"class": type(pipe.scheduler).__name__,
"config_sha256": _json_sha256(scheduler_config),
"config": scheduler_config,
},
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"seed": SEED,
"generator_device": "cuda",
"dtype": "bfloat16",
"vae_dtype": str(config["vae_dtype"]).removeprefix("torch."),
"height": config["height"],
"width": config["width"],
"num_frames": NUM_FRAMES,
"fps": FPS,
"num_inference_steps": NUM_INFERENCE_STEPS,
"guidance_scale": GUIDANCE_SCALE,
"input_image": input_metadata,
"generation_seconds": generation_seconds,
"peak_reserved_gib": torch.accelerator.max_memory_reserved() / 1024**3,
}
metadata_path = output_dir / "metadata.json"
metadata_path.write_text(json.dumps(metadata, indent=2, default=str) + "\n")
artifact_paths = [transformer_path, video_path, reference_path, metadata_path]
if input_path is not None:
artifact_paths.append(input_path)
manifest = {
"schema_version": SCHEMA_VERSION,
"variant": variant,
"task": task,
"publishable": publishable,
"model_provenance": model_provenance,
"metadata_sha256": _sha256(metadata_path),
"files": {path.name: {"sha256": _sha256(path), "size": path.stat().st_size} for path in artifact_paths},
}
manifest_path = output_dir / "manifest.json"
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n")
return manifest_path
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--variant", choices=VARIANTS, required=True)
parser.add_argument("--task", choices=("t2v", "i2v"), required=True)
parser.add_argument(
"--model",
help=(
"Optional local checkpoint. Local output is fingerprinted and marked development-only; "
"it cannot be published as the canonical golden."
),
)
parser.add_argument("--input-image", type=Path, help="Fixed local RGB input for task=i2v.")
parser.add_argument("--local-source-model", help="Claimed upstream model ID for local checkpoint audit records.")
parser.add_argument("--local-source-revision", help="Claimed upstream commit for local checkpoint audit records.")
parser.add_argument("--output-root", type=Path, default=Path("/tmp/sana-video-goldens-v2"))
args = parser.parse_args()
print(
generate(
args.variant,
args.task,
args.model,
args.input_image,
args.output_root,
args.local_source_model,
args.local_source_revision,
)
)
if __name__ == "__main__":
main()
@@ -0,0 +1,328 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import base64
import hashlib
import json
import os
from pathlib import Path
from typing import Any
import pytest
import requests
import torch
from safetensors.torch import load_file
from torch import nn
from tests.e2e.accuracy.helpers import assert_video_metadata, assert_video_similarity_metrics, probe_video
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
GOLDEN_BASE_URL = os.getenv("SANA_VIDEO_GOLDEN_BASE_URL")
SCHEMA_VERSION = 2
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
NEGATIVE_PROMPT = (
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
"jitter, and ghosting effects, creating a disorienting visual experience."
)
VARIANTS = {
"480p": ("Efficient-Large-Model/SANA-Video_2B_480p_diffusers", 480, 832),
"720p": ("Efficient-Large-Model/SANA-Video_2B_720p_diffusers", 704, 1280),
}
REVISIONS = {
"480p": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"720p": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
}
SSIM_THRESHOLDS = {
("480p", "t2v"): 0.91,
("480p", "i2v"): 0.93,
("720p", "t2v"): 0.93,
("720p", "i2v"): 0.93,
}
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.skipif(
not GOLDEN_BASE_URL,
reason="Set SANA_VIDEO_GOLDEN_BASE_URL after publishing the frozen v2 S3 assets.",
),
]
class _TransformerCheckpoint(nn.Module):
def __init__(self, transformer: nn.Module, model: str, revision: str) -> None:
super().__init__()
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
self.transformer = transformer
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=model,
subfolder="transformer",
revision=revision,
prefix="transformer.",
fall_back_to_pt=True,
)
]
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _json_sha256(payload: Any) -> str:
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
return hashlib.sha256(encoded).hexdigest()
def _assert_hex_digest(value: Any, *, length: int = 64) -> None:
assert isinstance(value, str) and len(value) == length
assert all(character in "0123456789abcdef" for character in value)
def _validate_manifest(payload: Any, variant: str, task: str) -> dict[str, Any]:
assert isinstance(payload, dict)
assert payload["schema_version"] == SCHEMA_VERSION
assert payload["variant"] == variant
assert payload["task"] == task
# Local checkpoint generation is deliberately marked non-publishable. Do
# not permit such output to become the canonical reference by accident.
assert payload["publishable"] is True
assert payload["model_provenance"] == {
"kind": "huggingface_hub",
"model_id": VARIANTS[variant][0],
"revision": REVISIONS[variant],
}
_assert_hex_digest(payload["metadata_sha256"])
expected_files = {
"metadata.json",
"pipeline.mp4",
"pipeline_reference.safetensors",
"transformer_case.safetensors",
}
if task == "i2v":
expected_files.add("input.png")
files = payload["files"]
assert isinstance(files, dict)
assert set(files) == expected_files
for filename, expected in files.items():
# Prevent a compromised manifest from writing outside output_dir.
assert filename == Path(filename).name
assert isinstance(expected, dict) and set(expected) == {"sha256", "size"}
_assert_hex_digest(expected["sha256"])
assert isinstance(expected["size"], int) and expected["size"] > 0
return payload
def _validate_metadata(payload: Any, manifest: dict[str, Any], variant: str, task: str) -> None:
assert isinstance(payload, dict)
_, height, width = VARIANTS[variant]
assert payload["schema_version"] == SCHEMA_VERSION
assert payload["variant"] == variant
assert payload["task"] == task
assert payload["publishable"] is True
assert payload["model_provenance"] == manifest["model_provenance"]
assert payload["prompt"] == PROMPT
assert payload["negative_prompt"] == NEGATIVE_PROMPT
assert payload["seed"] == 42
assert payload["generator_device"] == "cuda"
assert payload["dtype"] == "bfloat16"
assert payload["height"] == height
assert payload["width"] == width
assert payload["num_frames"] == 81
assert payload["fps"] == 16
assert payload["num_inference_steps"] == 50
assert payload["guidance_scale"] == 6.0
implementation = payload["reference_implementation"]
assert implementation["library"] == "diffusers"
assert implementation["pipeline_class"] == ("SanaImageToVideoPipeline" if task == "i2v" else "SanaVideoPipeline")
assert implementation["version"]
assert implementation["torch_version"]
generator = payload["generator_provenance"]
assert generator["repository"] == "vllm-project/vllm-omni"
_assert_hex_digest(generator["revision"], length=40)
_assert_hex_digest(generator["generator_sha256"])
assert generator["dirty"] is False
scheduler = payload["scheduler"]
assert scheduler["class"] == "DPMSolverMultistepScheduler"
_assert_hex_digest(scheduler["config_sha256"])
assert scheduler["config_sha256"] == _json_sha256(scheduler["config"])
input_image = payload["input_image"]
if task == "i2v":
assert input_image["filename"] == "input.png"
_assert_hex_digest(input_image["sha256"])
assert input_image["size"] == manifest["files"]["input.png"]["size"]
assert input_image["sha256"] == manifest["files"]["input.png"]["sha256"]
assert input_image["format"] == "png-rgb"
else:
assert input_image is None
def _download_case(variant: str, task: str, output_dir: Path, filenames: set[str]) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
base_url = f"{GOLDEN_BASE_URL.rstrip('/')}/{variant}/{task}"
response = requests.get(f"{base_url}/manifest.json", timeout=60)
response.raise_for_status()
payload = _validate_manifest(response.json(), variant, task)
filenames = filenames | {"metadata.json"}
if task == "i2v":
filenames.add("input.png")
assert filenames <= payload["files"].keys()
for filename in filenames:
expected = payload["files"][filename]
path = output_dir / filename
response = requests.get(f"{base_url}/{filename}", timeout=300)
response.raise_for_status()
path.write_bytes(response.content)
assert path.stat().st_size == expected["size"]
assert _sha256(path) == expected["sha256"]
assert _sha256(output_dir / "metadata.json") == payload["metadata_sha256"]
metadata = json.loads((output_dir / "metadata.json").read_text())
_validate_metadata(metadata, payload, variant, task)
return payload
@pytest.mark.benchmark
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_transformer_matches_frozen_golden(
variant: str,
_hardware,
accuracy_artifact_root: Path,
) -> None:
del _hardware
from vllm.config import LoadConfig
from vllm.transformers_utils.config import get_hf_file_to_dict
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
model, _, _ = VARIANTS[variant]
revision = REVISIONS[variant]
output_dir = accuracy_artifact_root / "sana_video" / variant / "t2v"
_download_case(variant, "t2v", output_dir, {"transformer_case.safetensors"})
case = load_file(output_dir / "transformer_case.safetensors")
transformer_config = get_hf_file_to_dict("transformer/config.json", model, revision=revision)
assert transformer_config is not None
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(
device="cuda",
dtype=torch.bfloat16,
)
checkpoint = _TransformerCheckpoint(transformer, model, revision)
od_config = OmniDiffusionConfig(
model=model,
model_class_name="SanaVideoPipeline",
dtype=torch.bfloat16,
revision=revision,
)
loader = DiffusersPipelineLoader(LoadConfig(), od_config=od_config)
transformer.load_weights(
(name.removeprefix("transformer."), tensor)
for name, tensor in loader.get_all_weights(checkpoint)
if name.startswith("transformer.")
)
transformer.eval()
with torch.inference_mode():
actual = transformer(
case["hidden_states"].to("cuda"),
case["encoder_hidden_states"].to("cuda"),
case["timestep"].to("cuda"),
encoder_attention_mask=case["encoder_attention_mask"].to("cuda"),
).sample.float()
expected = case["output"].to("cuda").float()
error = actual - expected
max_abs = error.abs().max()
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected)
cosine = torch.nn.functional.cosine_similarity(actual.flatten(), expected.flatten(), dim=0)
# CUDNN_ATTN BF16 reduction order is not bitwise deterministic on
# Blackwell. Bound rare outliers separately while keeping the aggregate
# relative-L2 and cosine checks strict.
max_abs_threshold = 0.2 if variant == "480p" else 0.1
relative_l2_threshold = 0.017 if variant == "480p" else 0.015
assert max_abs.item() <= max_abs_threshold
assert relative_l2.item() <= relative_l2_threshold
assert cosine.item() >= 0.999
SERVER_CASES = [
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--model-class-name",
pipeline_class,
],
),
id=f"{variant}-{task}",
marks=hardware_marks(res={"cuda": "H100"}),
)
for variant, (model, _, _) in VARIANTS.items()
for task, pipeline_class in (
("t2v", "SanaVideoPipeline"),
("i2v", "SanaImageToVideoPipeline"),
)
]
@pytest.mark.benchmark
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_sana_video_pipeline_matches_frozen_golden(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
accuracy_artifact_root: Path,
) -> None:
variant = "720p" if "720p" in omni_server.model else "480p"
task = "i2v" if "SanaImageToVideoPipeline" in omni_server.serve_args else "t2v"
_, height, width = VARIANTS[variant]
output_dir = accuracy_artifact_root / "sana_video" / variant / task
_download_case(variant, task, output_dir, {"pipeline.mp4"})
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": height,
"width": width,
"num_frames": 81,
"fps": 16,
"num_inference_steps": 50,
"guidance_scale": 6.0,
"seed": 42,
},
}
if task == "i2v":
encoded = base64.b64encode((output_dir / "input.png").read_bytes()).decode()
request_config["image_reference"] = f"data:image/png;base64,{encoded}"
result = openai_client.send_video_diffusion_request(request_config)[0]
actual_path = output_dir / "actual.mp4"
actual_path.write_bytes(result.videos[0])
golden_path = output_dir / "pipeline.mp4"
metadata = probe_video(actual_path)
assert_video_metadata(metadata, width=width, height=height, fps=16, frame_count=81)
assert_video_similarity_metrics(
label=f"sana_video_{variant}_{task}",
online_path=actual_path,
offline_path=golden_path,
# The native prompt cross-attention uses vLLM-Omni Attention rather
# than Diffusers SDPA. Frozen stage checks prove exact inputs and
# scheduler state, while the BF16 backend reduction difference grows
# across CFG denoising. Calibrate only the affected 480p T2V case;
# its frozen result is 0.919861 SSIM / 28.7442 dB PSNR.
ssim_threshold=SSIM_THRESHOLDS[(variant, task)],
psnr_threshold=28.0,
)
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"vae_class": "AutoencoderKLWan",
"native_vae_class": "DistributedAutoencoderKLWan",
"dtype": torch.float32,
"latent_channels": 16,
"spatial_scale": 8,
"temporal_scale": 4,
"frames": 5,
"max_abs": 1e-6,
"relative_l2": 1e-6,
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"vae_class": "AutoencoderKLLTX2Video",
"native_vae_class": "DistributedAutoencoderKLLTX2Video",
"dtype": torch.bfloat16,
"latent_channels": 128,
"spatial_scale": 32,
"temporal_scale": 8,
"frames": 9,
"max_abs": 0.04,
"relative_l2": 0.004,
},
}
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_i2v_first_frame_latents_match_diffusers(variant: str, _hardware) -> None:
del _hardware
from diffusers import (
AutoencoderKLLTX2Video,
AutoencoderKLWan,
)
from diffusers import (
SanaImageToVideoPipeline as DiffusersSanaImageToVideoPipeline,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
DistributedAutoencoderKLLTX2Video,
)
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
DistributedAutoencoderKLWan,
)
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
config = VARIANTS[variant]
vae_classes = {
"AutoencoderKLLTX2Video": AutoencoderKLLTX2Video,
"AutoencoderKLWan": AutoencoderKLWan,
}
native_vae_classes = {
"DistributedAutoencoderKLLTX2Video": DistributedAutoencoderKLLTX2Video,
"DistributedAutoencoderKLWan": DistributedAutoencoderKLWan,
}
reference_vae = (
vae_classes[config["vae_class"]]
.from_pretrained(
config["model"],
subfolder="vae",
torch_dtype=config["dtype"],
)
.to("cuda")
)
native_vae = (
native_vae_classes[config["native_vae_class"]]
.from_config(dict(reference_vae.config))
.to(device="cuda", dtype=config["dtype"])
)
native_vae.load_state_dict(reference_vae.state_dict())
reference_vae.eval()
native_vae.eval()
height, width = 192, 320
num_latent_frames = (config["frames"] - 1) // config["temporal_scale"] + 1
image = torch.linspace(
-1,
1,
3 * height * width,
dtype=torch.float32,
device="cuda",
).reshape(1, 3, height, width)
noise = torch.randn(
(
1,
config["latent_channels"],
num_latent_frames,
height // config["spatial_scale"],
width // config["spatial_scale"],
),
generator=torch.Generator(device="cuda").manual_seed(7),
device="cuda",
dtype=torch.float32,
)
reference_pipeline = SimpleNamespace(
vae=reference_vae,
vae_scale_factor_temporal=config["temporal_scale"],
vae_scale_factor_spatial=config["spatial_scale"],
)
native_pipeline = SimpleNamespace(
vae=native_vae,
vae_scale_factor_temporal=config["temporal_scale"],
vae_scale_factor_spatial=config["spatial_scale"],
device=torch.device("cuda"),
)
with torch.inference_mode():
expected = DiffusersSanaImageToVideoPipeline.prepare_latents(
reference_pipeline,
image,
1,
config["latent_channels"],
height,
width,
config["frames"],
torch.float32,
torch.device("cuda"),
None,
noise.clone(),
)
actual = SanaImageToVideoPipeline._prepare_i2v_latents(
native_pipeline,
image,
1,
config["latent_channels"],
height,
width,
config["frames"],
torch.float32,
None,
noise.clone(),
)
# Conditioning replaces only the first latent frame.
torch.testing.assert_close(actual[:, :, 1:], noise[:, :, 1:], rtol=0, atol=0)
error = (actual[:, :, :1] - expected[:, :, :1]).float()
expected_first_frame = expected[:, :, :1].float()
max_abs = error.abs().max()
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected_first_frame)
cosine = torch.nn.functional.cosine_similarity(
actual[:, :, :1].flatten(),
expected[:, :, :1].flatten(),
dim=0,
)
assert max_abs.item() <= config["max_abs"]
assert relative_l2.item() <= config["relative_l2"]
assert cosine.item() >= 0.9999
@@ -0,0 +1,432 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import gc
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import pytest
import torch
import torch.nn.functional as F
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
PROMPT = "A small red panda walks through a bamboo forest."
NEGATIVE_PROMPT = "blur, distortion, abrupt cuts"
SEED = 42
GUIDANCE_SCALE = 6.0
NUM_INFERENCE_STEPS = 2
HEIGHT = 192
WIDTH = 320
NUM_FRAMES = 9
VARIANTS = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
"model_env": "SANA_VIDEO_480P_MODEL",
"max_abs": 0.2,
"relative_l2": 0.02,
"cosine": 0.999,
"cfg_max_abs": 0.5,
"cfg_relative_l2": 0.05,
"cfg_cosine": 0.999,
"step_max_abs": 0.075,
"step_relative_l2": 0.012,
"step_cosine": 0.9999,
},
"720p": {
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
"model_env": "SANA_VIDEO_720P_MODEL",
"max_abs": 0.1,
"relative_l2": 0.015,
"cosine": 0.999,
"cfg_max_abs": 0.6,
"cfg_relative_l2": 0.065,
"cfg_cosine": 0.998,
"step_max_abs": 0.08,
"step_relative_l2": 0.014,
"step_cosine": 0.9999,
},
}
@dataclass(frozen=True)
class ErrorMetrics:
shape: tuple[int, ...]
actual_dtype: torch.dtype
expected_dtype: torch.dtype
max_abs: float
mean_abs: float
relative_l2: float
cosine: float
def __str__(self) -> str:
return (
f"shape={self.shape}, actual_dtype={self.actual_dtype}, "
f"expected_dtype={self.expected_dtype}, max_abs={self.max_abs:.8g}, "
f"mean_abs={self.mean_abs:.8g}, relative_l2={self.relative_l2:.8g}, "
f"cosine={self.cosine:.8g}"
)
def _metrics(actual: torch.Tensor, expected: torch.Tensor) -> ErrorMetrics:
assert actual.shape == expected.shape
actual_float = actual.detach().float()
expected_float = expected.detach().float()
error = actual_float - expected_float
expected_norm = torch.linalg.vector_norm(expected_float)
relative_l2 = torch.linalg.vector_norm(error) / expected_norm.clamp_min(torch.finfo(torch.float32).tiny)
if actual_float.numel() == 1:
cosine = torch.tensor(
1.0 if bool(torch.equal(actual_float, expected_float)) else 0.0,
device=actual.device,
)
else:
cosine = F.cosine_similarity(actual_float.flatten(), expected_float.flatten(), dim=0)
return ErrorMetrics(
shape=tuple(actual.shape),
actual_dtype=actual.dtype,
expected_dtype=expected.dtype,
max_abs=error.abs().max().item(),
mean_abs=error.abs().mean().item(),
relative_l2=relative_l2.item(),
cosine=cosine.item(),
)
def _assert_stage(
stage: str,
actual: torch.Tensor,
expected: torch.Tensor,
*,
max_abs: float,
relative_l2: float,
cosine: float,
) -> ErrorMetrics:
result = _metrics(actual, expected)
print(f"\nSANA-Video alignment [{stage}]: {result}")
assert result.actual_dtype == result.expected_dtype, f"{stage}: {result}"
assert result.max_abs <= max_abs, f"{stage}: {result}"
assert result.relative_l2 <= relative_l2, f"{stage}: {result}"
assert result.cosine >= cosine, f"{stage}: {result}"
return result
def _model_source(variant: str) -> tuple[str, str | None]:
config = VARIANTS[variant]
model = os.getenv(config["model_env"], config["model"])
revision = None if Path(model).exists() else config["revision"]
return model, revision
def _native_method_proxy(reference_pipeline: Any):
"""Build the minimum native-pipeline surface needed by stage helpers.
Sharing the tokenizer and text encoder makes this a comparison of pipeline
orchestration rather than a second copy of the same multi-billion-parameter
encoder. Transformer execution still uses separate Diffusers and native
modules below.
"""
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
proxy = object.__new__(SanaVideoPipeline)
torch.nn.Module.__init__(proxy)
proxy.tokenizer = reference_pipeline.tokenizer
proxy.text_encoder = reference_pipeline.text_encoder
proxy.device = torch.device("cuda")
proxy.vae_scale_factor_temporal = reference_pipeline.vae_scale_factor_temporal
proxy.vae_scale_factor_spatial = reference_pipeline.vae_scale_factor_spatial
return proxy
def _cfg_prediction(
prediction: torch.Tensor,
*,
guidance_scale: float,
latent_channels: int,
out_channels: int,
) -> torch.Tensor:
unconditional, conditional = prediction.float().chunk(2)
prediction = unconditional + guidance_scale * (conditional - unconditional)
if out_channels // 2 == latent_channels:
prediction = prediction.chunk(2, dim=1)[0]
return prediction
@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize(
"_hardware",
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
)
def test_sana_video_quick_pipeline_stages_match_diffusers(
variant: str,
_hardware,
) -> None:
"""Compare the first denoising step without replacing native attention.
This deliberately stops after the first scheduler step. It keeps the test
useful as a stage-localizing PR/nightly check while the frozen 81-frame,
50-step golden test covers the final latent and decoded-video boundary.
"""
del _hardware
from diffusers import SanaVideoPipeline as DiffusersSanaVideoPipeline
from diffusers.pipelines.sana_video.pipeline_sana_video import (
retrieve_timesteps as diffusers_retrieve_timesteps,
)
from diffusers.schedulers import DPMSolverMultistepScheduler
from vllm_omni.diffusion.models.sana_video import (
SanaVideoPipeline,
SanaVideoTransformer3DModel,
)
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
retrieve_timesteps as native_retrieve_timesteps,
)
model, revision = _model_source(variant)
config = VARIANTS[variant]
load_kwargs: dict[str, Any] = {
"torch_dtype": torch.bfloat16,
# No stage in this test decodes latents. Avoid loading a multi-GB VAE
# that would add memory and runtime without increasing coverage.
"vae": None,
}
if revision is not None:
load_kwargs["revision"] = revision
reference = DiffusersSanaVideoPipeline.from_pretrained(model, **load_kwargs).to("cuda")
assert type(reference) is DiffusersSanaVideoPipeline
assert type(reference.transformer).__module__.startswith("diffusers.")
native_proxy = _native_method_proxy(reference)
assert type(native_proxy) is SanaVideoPipeline
assert type(native_proxy).__module__.startswith("vllm_omni.")
try:
with torch.inference_mode():
reference_prompt = reference.encode_prompt(
PROMPT,
True,
negative_prompt=NEGATIVE_PROMPT,
device=torch.device("cuda"),
clean_caption=False,
max_sequence_length=300,
complex_human_instruction=None,
)
native_prompt = SanaVideoPipeline.encode_prompt(
native_proxy,
PROMPT,
True,
negative_prompt=NEGATIVE_PROMPT,
device=torch.device("cuda"),
clean_caption=False,
max_sequence_length=300,
complex_human_instruction=None,
)
prompt_stage_names = (
"prompt_embeddings",
"prompt_attention_mask",
"negative_prompt_embeddings",
"negative_prompt_attention_mask",
)
for stage, actual, expected in zip(prompt_stage_names, native_prompt, reference_prompt, strict=True):
_assert_stage(
stage,
actual,
expected,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
reference_prompt_embeds, reference_prompt_mask, reference_negative_embeds, reference_negative_mask = (
reference_prompt
)
encoder_hidden_states = torch.cat([reference_negative_embeds, reference_prompt_embeds])
encoder_attention_mask = torch.cat([reference_negative_mask, reference_prompt_mask])
latent_channels = reference.transformer.config.in_channels
reference_latents = reference.prepare_latents(
1,
latent_channels,
HEIGHT,
WIDTH,
NUM_FRAMES,
torch.float32,
torch.device("cuda"),
torch.Generator(device="cuda").manual_seed(SEED),
)
native_latents = SanaVideoPipeline.prepare_latents(
native_proxy,
1,
latent_channels,
HEIGHT,
WIDTH,
NUM_FRAMES,
torch.float32,
torch.device("cuda"),
torch.Generator(device="cuda").manual_seed(SEED),
)
_assert_stage(
"initial_noise_latents",
native_latents,
reference_latents,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
reference_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
native_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
isolated_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
reference_timesteps, reference_num_steps = diffusers_retrieve_timesteps(
reference_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
native_timesteps, native_num_steps = native_retrieve_timesteps(
native_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
isolated_retrieve_timesteps, isolated_num_steps = native_retrieve_timesteps(
isolated_scheduler,
NUM_INFERENCE_STEPS,
torch.device("cuda"),
)
assert reference_num_steps == native_num_steps == isolated_num_steps
_assert_stage(
"timesteps",
native_timesteps,
reference_timesteps,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
_assert_stage(
"sigmas",
native_scheduler.sigmas,
reference_scheduler.sigmas,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
torch.testing.assert_close(isolated_retrieve_timesteps, reference_timesteps, rtol=0, atol=0)
native_transformer = SanaVideoTransformer3DModel.from_config(dict(reference.transformer.config)).to(
device="cuda",
dtype=torch.bfloat16,
)
incompatible = native_transformer.load_state_dict(reference.transformer.state_dict(), strict=True)
assert not incompatible.missing_keys
assert not incompatible.unexpected_keys
assert type(native_transformer) is SanaVideoTransformer3DModel
assert type(native_transformer).__module__.startswith("vllm_omni.")
assert type(native_transformer) is not type(reference.transformer)
reference_parameter = next(reference.transformer.parameters())
native_parameter = next(native_transformer.parameters())
assert native_parameter.data_ptr() != reference_parameter.data_ptr()
torch.testing.assert_close(native_parameter, reference_parameter, rtol=0, atol=0)
native_transformer.eval()
reference.transformer.eval()
latent_model_input = torch.cat([reference_latents, reference_latents])
timestep = reference_timesteps[0].expand(latent_model_input.shape[0])
with torch.inference_mode():
reference_raw_prediction = reference.transformer(
latent_model_input.to(dtype=torch.bfloat16),
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
encoder_attention_mask=encoder_attention_mask,
timestep=timestep,
return_dict=False,
)[0]
native_raw_prediction = native_transformer(
latent_model_input.to(dtype=torch.bfloat16),
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
encoder_attention_mask=encoder_attention_mask,
timestep=timestep,
return_dict=False,
)[0]
_assert_stage(
"first_transformer_raw_prediction",
native_raw_prediction,
reference_raw_prediction,
max_abs=config["max_abs"],
relative_l2=config["relative_l2"],
cosine=config["cosine"],
)
reference_cfg_prediction = _cfg_prediction(
reference_raw_prediction,
guidance_scale=GUIDANCE_SCALE,
latent_channels=latent_channels,
out_channels=reference.transformer.config.out_channels,
)
native_cfg_prediction = _cfg_prediction(
native_raw_prediction,
guidance_scale=GUIDANCE_SCALE,
latent_channels=latent_channels,
out_channels=native_transformer.config.out_channels,
)
_assert_stage(
"first_cfg_prediction",
native_cfg_prediction,
reference_cfg_prediction,
max_abs=config["cfg_max_abs"],
relative_l2=config["cfg_relative_l2"],
cosine=config["cfg_cosine"],
)
reference_first_step = reference_scheduler.step(
reference_cfg_prediction,
reference_timesteps[0],
reference_latents,
return_dict=False,
)[0]
# Isolate scheduler orchestration from native-attention error by
# feeding the Diffusers prediction through an independently configured
# scheduler initialized by the native helper.
isolated_first_step = isolated_scheduler.step(
reference_cfg_prediction,
isolated_retrieve_timesteps[0],
reference_latents,
return_dict=False,
)[0]
_assert_stage(
"first_scheduler_step_isolated",
isolated_first_step,
reference_first_step,
max_abs=0.0,
relative_l2=0.0,
cosine=0.999999,
)
native_first_step = native_scheduler.step(
native_cfg_prediction,
native_timesteps[0],
native_latents,
return_dict=False,
)[0]
_assert_stage(
"first_scheduler_step_end_to_end",
native_first_step,
reference_first_step,
max_abs=config["step_max_abs"],
relative_l2=config["step_relative_l2"],
cosine=config["step_cosine"],
)
finally:
del native_proxy
del reference
gc.collect()
torch.accelerator.empty_cache()
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SANA-Video 2B online smoke tests for the native and Diffusers backends."""
import os
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL_480P = os.environ.get(
"SANA_VIDEO_480P_MODEL",
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
)
MODEL_720P = os.environ.get(
"SANA_VIDEO_720P_MODEL",
"Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
)
PROMPT = "A cat walking on grass toward the camera. motion score: 30."
NEGATIVE_PROMPT = "blurry, low quality, temporal artifacts"
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
def _t2v_backend_cases():
cases = []
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
cases.extend(
[
pytest.param(
OmniServerParams(model=model, server_args=["--model-class-name", "SanaVideoPipeline"]),
id=f"native-{variant}",
marks=SINGLE_CARD_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--diffusion-load-format",
"diffusers",
"--diffusion-attention-backend",
"TORCH_SDPA",
],
),
id=f"diffusers-adapter-{variant}",
marks=SINGLE_CARD_MARKS,
),
]
)
return cases
@pytest.mark.parametrize("omni_server", _t2v_backend_cases(), indirect=True)
def test_sana_video_t2v_backends(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 192,
"width": 320,
"num_frames": 9,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 4.0,
"seed": 42,
},
}
openai_client.send_video_diffusion_request(request_config)
def _i2v_cases():
cases = []
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
cases.extend(
[
pytest.param(
OmniServerParams(model=model, server_args=["--model-class-name", "SanaImageToVideoPipeline"]),
id=f"native-{variant}",
marks=SINGLE_CARD_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--model-class-name",
"SanaImageToVideoPipeline",
"--diffusion-load-format",
"diffusers",
"--diffusion-attention-backend",
"TORCH_SDPA",
],
),
id=f"diffusers-adapter-{variant}",
marks=SINGLE_CARD_MARKS,
),
]
)
return cases
@pytest.mark.parametrize("omni_server", _i2v_cases(), indirect=True)
def test_sana_video_i2v_variants(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 192,
"width": 320,
"num_frames": 9,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 4.0,
"seed": 42,
},
"image_reference": (f"data:image/jpeg;base64,{generate_synthetic_image(320, 192, seed=42)['base64']}"),
}
openai_client.send_video_diffusion_request(request_config)