[Model] Add native SANA-Video 2B T2V and I2V support (#5508)
Signed-off-by: leo <92263211+napleon-liu@users.noreply.github.com> Signed-off-by: leo <92263211+liuyao0322@users.noreply.github.com> Signed-off-by: liuyao0322 <yaoliu548926@gmail.com> Co-authored-by: leo <92263211+napleon-liu@users.noreply.github.com>
This commit is contained in:
1 parent
4af0ddc1c2
commit
50b8a150f9
42 files changed
+6512
-18
No files matched your search
@@ -0,0 +1,63 @@
|
||||
# SANA-Video golden outputs
|
||||
|
||||
The v2 golden set covers both text-to-video and image-to-video for each
|
||||
checkpoint variant. Generate references with the Diffusers version pinned by
|
||||
this repository and the immutable Hub revisions embedded in
|
||||
`generate_goldens.py`:
|
||||
|
||||
```bash
|
||||
python tests/e2e/accuracy/sana_video/generate_goldens.py \
|
||||
--variant 480p --task t2v
|
||||
python tests/e2e/accuracy/sana_video/generate_goldens.py \
|
||||
--variant 480p --task i2v --input-image /path/to/fixed-input.png
|
||||
python tests/e2e/accuracy/sana_video/generate_goldens.py \
|
||||
--variant 720p --task t2v
|
||||
python tests/e2e/accuracy/sana_video/generate_goldens.py \
|
||||
--variant 720p --task i2v --input-image /path/to/fixed-input.png
|
||||
```
|
||||
|
||||
Use the same reviewed input image for both I2V variants. The generator
|
||||
normalizes it to an RGB PNG and includes that exact file and its SHA256 in the
|
||||
case manifest, so an I2V reference is self-contained.
|
||||
|
||||
Each case is written under:
|
||||
|
||||
```text
|
||||
<output-root>/<variant>/<task>/
|
||||
```
|
||||
|
||||
It contains:
|
||||
|
||||
- `transformer_case.safetensors`: deterministic transformer inputs and output.
|
||||
- `pipeline.mp4`: the encoded reference used by online similarity tests.
|
||||
- `pipeline_reference.safetensors`: final denoised latents and decoded,
|
||||
pre-MP4 frames quantized to the exact uint8 video input.
|
||||
- `input.png`: the normalized conditioning image for I2V only.
|
||||
- `metadata.json`: model, generator, scheduler, input and sampling provenance.
|
||||
- `manifest.json`: the size and SHA256 of every artifact.
|
||||
|
||||
## Publication requirements
|
||||
|
||||
A canonical golden must be generated from the pinned Hugging Face model ID and
|
||||
immutable commit. The generator also requires a clean vLLM-Omni worktree and
|
||||
records the repository revision plus the generator script hash.
|
||||
|
||||
`--model` is available for local development. It hashes every file in the local
|
||||
checkpoint directory and records optional `--local-source-model` and
|
||||
`--local-source-revision` claims, but marks the result `publishable: false`.
|
||||
The golden test rejects such a manifest. A local directory hash proves which
|
||||
bytes were used; it does not prove that those bytes match a claimed upstream
|
||||
commit.
|
||||
|
||||
After reviewing the metadata, input image and generated videos, upload each
|
||||
publishable case directory without renaming files to:
|
||||
|
||||
```text
|
||||
s3://vllm-public-assets/omni-assets/sana-video/v2/<variant>/<task>/
|
||||
```
|
||||
|
||||
Set `SANA_VIDEO_GOLDEN_BASE_URL` to the corresponding `v2` HTTP base URL when
|
||||
running the tests. Uploading is intentionally separate from generation so test
|
||||
runs cannot overwrite a frozen reference. The storage location must prevent
|
||||
in-place mutation; publish a new version prefix instead of replacing any v2
|
||||
file when a model, dependency, input or sampling parameter changes.
|
||||
@@ -0,0 +1,380 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Generate frozen SANA-Video transformer and pipeline reference artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from diffusers import SanaImageToVideoPipeline, SanaVideoPipeline
|
||||
from diffusers.utils import export_to_video
|
||||
from PIL import Image
|
||||
from safetensors.torch import save_file
|
||||
|
||||
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
|
||||
NEGATIVE_PROMPT = (
|
||||
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
|
||||
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
|
||||
"jitter, and ghosting effects, creating a disorienting visual experience."
|
||||
)
|
||||
SCHEMA_VERSION = 2
|
||||
SEED = 42
|
||||
NUM_FRAMES = 81
|
||||
FPS = 16
|
||||
NUM_INFERENCE_STEPS = 50
|
||||
GUIDANCE_SCALE = 6.0
|
||||
VARIANTS = {
|
||||
"480p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
|
||||
"height": 480,
|
||||
"width": 832,
|
||||
"vae_dtype": torch.float32,
|
||||
"transformer_case_shape": (1, 16, 3, 8, 8),
|
||||
},
|
||||
"720p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
|
||||
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
|
||||
"height": 704,
|
||||
"width": 1280,
|
||||
"vae_dtype": torch.bfloat16,
|
||||
"transformer_case_shape": (1, 128, 2, 4, 4),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _json_sha256(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _directory_fingerprint(root: Path) -> dict[str, Any]:
|
||||
"""Fingerprint every local checkpoint file without recording host paths."""
|
||||
if not root.is_dir():
|
||||
raise ValueError(f"Local checkpoint must be a directory: {root}")
|
||||
files = []
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
file_sha256 = _sha256(path)
|
||||
size = path.stat().st_size
|
||||
entry = f"{relative}\0{size}\0{file_sha256}\n".encode()
|
||||
digest.update(entry)
|
||||
files.append({"path": relative, "size": size, "sha256": file_sha256})
|
||||
if not files:
|
||||
raise ValueError(f"Local checkpoint contains no files: {root}")
|
||||
return {
|
||||
"algorithm": "sha256-tree-v1",
|
||||
"sha256": digest.hexdigest(),
|
||||
"file_count": len(files),
|
||||
"total_size": sum(entry["size"] for entry in files),
|
||||
}
|
||||
|
||||
|
||||
def _repository_provenance() -> dict[str, Any]:
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
revision = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
dirty = bool(
|
||||
subprocess.run(
|
||||
["git", "status", "--porcelain", "--untracked-files=no"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
)
|
||||
return {
|
||||
"repository": "vllm-project/vllm-omni",
|
||||
"revision": revision,
|
||||
"dirty": dirty,
|
||||
"generator_sha256": _sha256(Path(__file__)),
|
||||
}
|
||||
|
||||
|
||||
def _cpu(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor.detach().cpu().contiguous()
|
||||
|
||||
|
||||
def _normalize_input_image(source: Path, destination: Path) -> Image.Image:
|
||||
if not source.is_file():
|
||||
raise ValueError(f"I2V input image does not exist: {source}")
|
||||
with Image.open(source) as image:
|
||||
normalized = image.convert("RGB")
|
||||
normalized.load()
|
||||
normalized.save(destination, format="PNG", optimize=False)
|
||||
return normalized
|
||||
|
||||
|
||||
def _frames_to_uint8(frames: np.ndarray) -> torch.Tensor:
|
||||
tensor = torch.from_numpy(np.asarray(frames))
|
||||
if tensor.is_floating_point():
|
||||
# Match diffusers.utils.export_to_video, which truncates rather than
|
||||
# rounds when converting float frames to the encoder's uint8 input.
|
||||
tensor = tensor.clamp(0, 1) * 255
|
||||
return tensor.to(torch.uint8).contiguous()
|
||||
|
||||
|
||||
def _model_provenance(
|
||||
config: dict[str, Any],
|
||||
model: str | None,
|
||||
local_source_model: str | None,
|
||||
local_source_revision: str | None,
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
if model is None:
|
||||
return (
|
||||
{
|
||||
"kind": "huggingface_hub",
|
||||
"model_id": config["model"],
|
||||
"revision": config["revision"],
|
||||
},
|
||||
True,
|
||||
)
|
||||
|
||||
local_path = Path(model).expanduser().resolve()
|
||||
if bool(local_source_model) != bool(local_source_revision):
|
||||
raise ValueError("--local-source-model and --local-source-revision must be specified together.")
|
||||
provenance = {
|
||||
"kind": "local_checkpoint",
|
||||
"checkpoint_fingerprint": _directory_fingerprint(local_path),
|
||||
"claimed_source_model": local_source_model,
|
||||
"claimed_source_revision": local_source_revision,
|
||||
"verification": "unverified-local-copy",
|
||||
}
|
||||
# A directory hash proves which local bytes were used, but not that those
|
||||
# bytes match a claimed Hub commit. Such output is useful for development
|
||||
# comparisons but must never be uploaded as the canonical frozen reference.
|
||||
return provenance, False
|
||||
|
||||
|
||||
def generate(
|
||||
variant: str,
|
||||
task: str,
|
||||
model: str | None,
|
||||
input_image: Path | None,
|
||||
output_root: Path,
|
||||
local_source_model: str | None = None,
|
||||
local_source_revision: str | None = None,
|
||||
) -> Path:
|
||||
config = dict(VARIANTS[variant])
|
||||
model_provenance, publishable = _model_provenance(
|
||||
config,
|
||||
model,
|
||||
local_source_model,
|
||||
local_source_revision,
|
||||
)
|
||||
repository_provenance = _repository_provenance()
|
||||
if publishable and repository_provenance["dirty"]:
|
||||
raise RuntimeError("Refusing to generate publishable golden assets from a dirty vllm-omni worktree.")
|
||||
if model is not None:
|
||||
config["model"] = str(Path(model).expanduser().resolve())
|
||||
config["revision"] = None
|
||||
|
||||
output_dir = output_root / variant / task
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
normalized_image: Image.Image | None = None
|
||||
input_path: Path | None = None
|
||||
if task == "i2v":
|
||||
if input_image is None:
|
||||
raise ValueError("--input-image is required for task=i2v.")
|
||||
input_path = output_dir / "input.png"
|
||||
normalized_image = _normalize_input_image(input_image, input_path)
|
||||
elif input_image is not None:
|
||||
raise ValueError("--input-image is only valid for task=i2v.")
|
||||
|
||||
pipeline_class = SanaImageToVideoPipeline if task == "i2v" else SanaVideoPipeline
|
||||
pipe = pipeline_class.from_pretrained(
|
||||
config["model"],
|
||||
revision=config["revision"],
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
pipe.text_encoder.to(torch.bfloat16)
|
||||
pipe.vae.to(config["vae_dtype"])
|
||||
pipe.to("cuda")
|
||||
|
||||
torch.manual_seed(1234)
|
||||
hidden_states = torch.randn(config["transformer_case_shape"], dtype=torch.bfloat16, device="cuda")
|
||||
encoder_hidden_states = torch.randn(1, 8, 2304, dtype=torch.bfloat16, device="cuda")
|
||||
encoder_attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 0]], device="cuda")
|
||||
timestep = torch.tensor([500.0], device="cuda")
|
||||
with torch.inference_mode():
|
||||
transformer_output = pipe.transformer(
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
timestep,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
).sample
|
||||
|
||||
transformer_path = output_dir / "transformer_case.safetensors"
|
||||
save_file(
|
||||
{
|
||||
"hidden_states": _cpu(hidden_states),
|
||||
"encoder_hidden_states": _cpu(encoder_hidden_states),
|
||||
"encoder_attention_mask": _cpu(encoder_attention_mask),
|
||||
"timestep": _cpu(timestep),
|
||||
"output": _cpu(transformer_output),
|
||||
},
|
||||
transformer_path,
|
||||
)
|
||||
|
||||
final_latents: dict[str, torch.Tensor] = {}
|
||||
|
||||
def capture_final_latents(_pipe, step, _timestep, callback_kwargs):
|
||||
if step == NUM_INFERENCE_STEPS - 1:
|
||||
final_latents["latents"] = _cpu(callback_kwargs["latents"])
|
||||
return callback_kwargs
|
||||
|
||||
torch.accelerator.reset_peak_memory_stats()
|
||||
generator = torch.Generator(device="cuda").manual_seed(SEED)
|
||||
generation_kwargs = {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": config["height"],
|
||||
"width": config["width"],
|
||||
"frames": NUM_FRAMES,
|
||||
"guidance_scale": GUIDANCE_SCALE,
|
||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||
"generator": generator,
|
||||
"output_type": "np",
|
||||
"callback_on_step_end": capture_final_latents,
|
||||
"callback_on_step_end_tensor_inputs": ["latents"],
|
||||
}
|
||||
if normalized_image is not None:
|
||||
generation_kwargs["image"] = normalized_image
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
started = time.perf_counter()
|
||||
with torch.inference_mode():
|
||||
frames = pipe(**generation_kwargs).frames[0]
|
||||
torch.accelerator.synchronize()
|
||||
generation_seconds = time.perf_counter() - started
|
||||
if "latents" not in final_latents:
|
||||
raise RuntimeError("Diffusers did not expose the final denoised latents to the generation callback.")
|
||||
|
||||
video_path = output_dir / "pipeline.mp4"
|
||||
export_to_video(frames, video_path, fps=FPS)
|
||||
reference_path = output_dir / "pipeline_reference.safetensors"
|
||||
save_file(
|
||||
{
|
||||
"final_latents": final_latents["latents"],
|
||||
"decoded_frames_uint8": _frames_to_uint8(frames),
|
||||
},
|
||||
reference_path,
|
||||
)
|
||||
|
||||
scheduler_config = dict(pipe.scheduler.config)
|
||||
input_metadata = None
|
||||
if input_path is not None:
|
||||
input_metadata = {
|
||||
"filename": input_path.name,
|
||||
"sha256": _sha256(input_path),
|
||||
"size": input_path.stat().st_size,
|
||||
"format": "png-rgb",
|
||||
}
|
||||
metadata = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"variant": variant,
|
||||
"task": task,
|
||||
"publishable": publishable,
|
||||
"reference_implementation": {
|
||||
"library": "diffusers",
|
||||
"version": __import__("diffusers").__version__,
|
||||
"pipeline_class": pipeline_class.__name__,
|
||||
"torch_version": torch.__version__,
|
||||
},
|
||||
"model_provenance": model_provenance,
|
||||
"generator_provenance": repository_provenance,
|
||||
"scheduler": {
|
||||
"class": type(pipe.scheduler).__name__,
|
||||
"config_sha256": _json_sha256(scheduler_config),
|
||||
"config": scheduler_config,
|
||||
},
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"seed": SEED,
|
||||
"generator_device": "cuda",
|
||||
"dtype": "bfloat16",
|
||||
"vae_dtype": str(config["vae_dtype"]).removeprefix("torch."),
|
||||
"height": config["height"],
|
||||
"width": config["width"],
|
||||
"num_frames": NUM_FRAMES,
|
||||
"fps": FPS,
|
||||
"num_inference_steps": NUM_INFERENCE_STEPS,
|
||||
"guidance_scale": GUIDANCE_SCALE,
|
||||
"input_image": input_metadata,
|
||||
"generation_seconds": generation_seconds,
|
||||
"peak_reserved_gib": torch.accelerator.max_memory_reserved() / 1024**3,
|
||||
}
|
||||
metadata_path = output_dir / "metadata.json"
|
||||
metadata_path.write_text(json.dumps(metadata, indent=2, default=str) + "\n")
|
||||
|
||||
artifact_paths = [transformer_path, video_path, reference_path, metadata_path]
|
||||
if input_path is not None:
|
||||
artifact_paths.append(input_path)
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"variant": variant,
|
||||
"task": task,
|
||||
"publishable": publishable,
|
||||
"model_provenance": model_provenance,
|
||||
"metadata_sha256": _sha256(metadata_path),
|
||||
"files": {path.name: {"sha256": _sha256(path), "size": path.stat().st_size} for path in artifact_paths},
|
||||
}
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
return manifest_path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--variant", choices=VARIANTS, required=True)
|
||||
parser.add_argument("--task", choices=("t2v", "i2v"), required=True)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
help=(
|
||||
"Optional local checkpoint. Local output is fingerprinted and marked development-only; "
|
||||
"it cannot be published as the canonical golden."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--input-image", type=Path, help="Fixed local RGB input for task=i2v.")
|
||||
parser.add_argument("--local-source-model", help="Claimed upstream model ID for local checkpoint audit records.")
|
||||
parser.add_argument("--local-source-revision", help="Claimed upstream commit for local checkpoint audit records.")
|
||||
parser.add_argument("--output-root", type=Path, default=Path("/tmp/sana-video-goldens-v2"))
|
||||
args = parser.parse_args()
|
||||
print(
|
||||
generate(
|
||||
args.variant,
|
||||
args.task,
|
||||
args.model,
|
||||
args.input_image,
|
||||
args.output_root,
|
||||
args.local_source_model,
|
||||
args.local_source_revision,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,328 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
from safetensors.torch import load_file
|
||||
from torch import nn
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_video_metadata, assert_video_similarity_metrics, probe_video
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
GOLDEN_BASE_URL = os.getenv("SANA_VIDEO_GOLDEN_BASE_URL")
|
||||
SCHEMA_VERSION = 2
|
||||
PROMPT = "A cat walking on the grass, facing the camera. motion score: 30."
|
||||
NEGATIVE_PROMPT = (
|
||||
"A chaotic sequence with misshapen, deformed limbs in heavy motion blur, sudden disappearance, jump cuts, "
|
||||
"jerky movements, rapid shot changes, frames out of sync, inconsistent character shapes, temporal artifacts, "
|
||||
"jitter, and ghosting effects, creating a disorienting visual experience."
|
||||
)
|
||||
VARIANTS = {
|
||||
"480p": ("Efficient-Large-Model/SANA-Video_2B_480p_diffusers", 480, 832),
|
||||
"720p": ("Efficient-Large-Model/SANA-Video_2B_720p_diffusers", 704, 1280),
|
||||
}
|
||||
REVISIONS = {
|
||||
"480p": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
|
||||
"720p": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
|
||||
}
|
||||
SSIM_THRESHOLDS = {
|
||||
("480p", "t2v"): 0.91,
|
||||
("480p", "i2v"): 0.93,
|
||||
("720p", "t2v"): 0.93,
|
||||
("720p", "i2v"): 0.93,
|
||||
}
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.skipif(
|
||||
not GOLDEN_BASE_URL,
|
||||
reason="Set SANA_VIDEO_GOLDEN_BASE_URL after publishing the frozen v2 S3 assets.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _TransformerCheckpoint(nn.Module):
|
||||
def __init__(self, transformer: nn.Module, model: str, revision: str) -> None:
|
||||
super().__init__()
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
|
||||
self.transformer = transformer
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=model,
|
||||
subfolder="transformer",
|
||||
revision=revision,
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _json_sha256(payload: Any) -> str:
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _assert_hex_digest(value: Any, *, length: int = 64) -> None:
|
||||
assert isinstance(value, str) and len(value) == length
|
||||
assert all(character in "0123456789abcdef" for character in value)
|
||||
|
||||
|
||||
def _validate_manifest(payload: Any, variant: str, task: str) -> dict[str, Any]:
|
||||
assert isinstance(payload, dict)
|
||||
assert payload["schema_version"] == SCHEMA_VERSION
|
||||
assert payload["variant"] == variant
|
||||
assert payload["task"] == task
|
||||
# Local checkpoint generation is deliberately marked non-publishable. Do
|
||||
# not permit such output to become the canonical reference by accident.
|
||||
assert payload["publishable"] is True
|
||||
assert payload["model_provenance"] == {
|
||||
"kind": "huggingface_hub",
|
||||
"model_id": VARIANTS[variant][0],
|
||||
"revision": REVISIONS[variant],
|
||||
}
|
||||
_assert_hex_digest(payload["metadata_sha256"])
|
||||
|
||||
expected_files = {
|
||||
"metadata.json",
|
||||
"pipeline.mp4",
|
||||
"pipeline_reference.safetensors",
|
||||
"transformer_case.safetensors",
|
||||
}
|
||||
if task == "i2v":
|
||||
expected_files.add("input.png")
|
||||
files = payload["files"]
|
||||
assert isinstance(files, dict)
|
||||
assert set(files) == expected_files
|
||||
for filename, expected in files.items():
|
||||
# Prevent a compromised manifest from writing outside output_dir.
|
||||
assert filename == Path(filename).name
|
||||
assert isinstance(expected, dict) and set(expected) == {"sha256", "size"}
|
||||
_assert_hex_digest(expected["sha256"])
|
||||
assert isinstance(expected["size"], int) and expected["size"] > 0
|
||||
return payload
|
||||
|
||||
|
||||
def _validate_metadata(payload: Any, manifest: dict[str, Any], variant: str, task: str) -> None:
|
||||
assert isinstance(payload, dict)
|
||||
_, height, width = VARIANTS[variant]
|
||||
assert payload["schema_version"] == SCHEMA_VERSION
|
||||
assert payload["variant"] == variant
|
||||
assert payload["task"] == task
|
||||
assert payload["publishable"] is True
|
||||
assert payload["model_provenance"] == manifest["model_provenance"]
|
||||
assert payload["prompt"] == PROMPT
|
||||
assert payload["negative_prompt"] == NEGATIVE_PROMPT
|
||||
assert payload["seed"] == 42
|
||||
assert payload["generator_device"] == "cuda"
|
||||
assert payload["dtype"] == "bfloat16"
|
||||
assert payload["height"] == height
|
||||
assert payload["width"] == width
|
||||
assert payload["num_frames"] == 81
|
||||
assert payload["fps"] == 16
|
||||
assert payload["num_inference_steps"] == 50
|
||||
assert payload["guidance_scale"] == 6.0
|
||||
|
||||
implementation = payload["reference_implementation"]
|
||||
assert implementation["library"] == "diffusers"
|
||||
assert implementation["pipeline_class"] == ("SanaImageToVideoPipeline" if task == "i2v" else "SanaVideoPipeline")
|
||||
assert implementation["version"]
|
||||
assert implementation["torch_version"]
|
||||
|
||||
generator = payload["generator_provenance"]
|
||||
assert generator["repository"] == "vllm-project/vllm-omni"
|
||||
_assert_hex_digest(generator["revision"], length=40)
|
||||
_assert_hex_digest(generator["generator_sha256"])
|
||||
assert generator["dirty"] is False
|
||||
|
||||
scheduler = payload["scheduler"]
|
||||
assert scheduler["class"] == "DPMSolverMultistepScheduler"
|
||||
_assert_hex_digest(scheduler["config_sha256"])
|
||||
assert scheduler["config_sha256"] == _json_sha256(scheduler["config"])
|
||||
|
||||
input_image = payload["input_image"]
|
||||
if task == "i2v":
|
||||
assert input_image["filename"] == "input.png"
|
||||
_assert_hex_digest(input_image["sha256"])
|
||||
assert input_image["size"] == manifest["files"]["input.png"]["size"]
|
||||
assert input_image["sha256"] == manifest["files"]["input.png"]["sha256"]
|
||||
assert input_image["format"] == "png-rgb"
|
||||
else:
|
||||
assert input_image is None
|
||||
|
||||
|
||||
def _download_case(variant: str, task: str, output_dir: Path, filenames: set[str]) -> dict[str, Any]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
base_url = f"{GOLDEN_BASE_URL.rstrip('/')}/{variant}/{task}"
|
||||
response = requests.get(f"{base_url}/manifest.json", timeout=60)
|
||||
response.raise_for_status()
|
||||
payload = _validate_manifest(response.json(), variant, task)
|
||||
filenames = filenames | {"metadata.json"}
|
||||
if task == "i2v":
|
||||
filenames.add("input.png")
|
||||
assert filenames <= payload["files"].keys()
|
||||
for filename in filenames:
|
||||
expected = payload["files"][filename]
|
||||
path = output_dir / filename
|
||||
response = requests.get(f"{base_url}/{filename}", timeout=300)
|
||||
response.raise_for_status()
|
||||
path.write_bytes(response.content)
|
||||
assert path.stat().st_size == expected["size"]
|
||||
assert _sha256(path) == expected["sha256"]
|
||||
assert _sha256(output_dir / "metadata.json") == payload["metadata_sha256"]
|
||||
metadata = json.loads((output_dir / "metadata.json").read_text())
|
||||
_validate_metadata(metadata, payload, variant, task)
|
||||
return payload
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("variant", VARIANTS)
|
||||
@pytest.mark.parametrize(
|
||||
"_hardware",
|
||||
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
|
||||
)
|
||||
def test_sana_video_transformer_matches_frozen_golden(
|
||||
variant: str,
|
||||
_hardware,
|
||||
accuracy_artifact_root: Path,
|
||||
) -> None:
|
||||
del _hardware
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.sana_video import SanaVideoTransformer3DModel
|
||||
|
||||
model, _, _ = VARIANTS[variant]
|
||||
revision = REVISIONS[variant]
|
||||
output_dir = accuracy_artifact_root / "sana_video" / variant / "t2v"
|
||||
_download_case(variant, "t2v", output_dir, {"transformer_case.safetensors"})
|
||||
case = load_file(output_dir / "transformer_case.safetensors")
|
||||
transformer_config = get_hf_file_to_dict("transformer/config.json", model, revision=revision)
|
||||
assert transformer_config is not None
|
||||
transformer = SanaVideoTransformer3DModel.from_config(transformer_config).to(
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
checkpoint = _TransformerCheckpoint(transformer, model, revision)
|
||||
od_config = OmniDiffusionConfig(
|
||||
model=model,
|
||||
model_class_name="SanaVideoPipeline",
|
||||
dtype=torch.bfloat16,
|
||||
revision=revision,
|
||||
)
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config=od_config)
|
||||
transformer.load_weights(
|
||||
(name.removeprefix("transformer."), tensor)
|
||||
for name, tensor in loader.get_all_weights(checkpoint)
|
||||
if name.startswith("transformer.")
|
||||
)
|
||||
transformer.eval()
|
||||
with torch.inference_mode():
|
||||
actual = transformer(
|
||||
case["hidden_states"].to("cuda"),
|
||||
case["encoder_hidden_states"].to("cuda"),
|
||||
case["timestep"].to("cuda"),
|
||||
encoder_attention_mask=case["encoder_attention_mask"].to("cuda"),
|
||||
).sample.float()
|
||||
expected = case["output"].to("cuda").float()
|
||||
error = actual - expected
|
||||
max_abs = error.abs().max()
|
||||
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected)
|
||||
cosine = torch.nn.functional.cosine_similarity(actual.flatten(), expected.flatten(), dim=0)
|
||||
# CUDNN_ATTN BF16 reduction order is not bitwise deterministic on
|
||||
# Blackwell. Bound rare outliers separately while keeping the aggregate
|
||||
# relative-L2 and cosine checks strict.
|
||||
max_abs_threshold = 0.2 if variant == "480p" else 0.1
|
||||
relative_l2_threshold = 0.017 if variant == "480p" else 0.015
|
||||
assert max_abs.item() <= max_abs_threshold
|
||||
assert relative_l2.item() <= relative_l2_threshold
|
||||
assert cosine.item() >= 0.999
|
||||
|
||||
|
||||
SERVER_CASES = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--model-class-name",
|
||||
pipeline_class,
|
||||
],
|
||||
),
|
||||
id=f"{variant}-{task}",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
)
|
||||
for variant, (model, _, _) in VARIANTS.items()
|
||||
for task, pipeline_class in (
|
||||
("t2v", "SanaVideoPipeline"),
|
||||
("i2v", "SanaImageToVideoPipeline"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
|
||||
def test_sana_video_pipeline_matches_frozen_golden(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
accuracy_artifact_root: Path,
|
||||
) -> None:
|
||||
variant = "720p" if "720p" in omni_server.model else "480p"
|
||||
task = "i2v" if "SanaImageToVideoPipeline" in omni_server.serve_args else "t2v"
|
||||
_, height, width = VARIANTS[variant]
|
||||
output_dir = accuracy_artifact_root / "sana_video" / variant / task
|
||||
_download_case(variant, task, output_dir, {"pipeline.mp4"})
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 6.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
if task == "i2v":
|
||||
encoded = base64.b64encode((output_dir / "input.png").read_bytes()).decode()
|
||||
request_config["image_reference"] = f"data:image/png;base64,{encoded}"
|
||||
result = openai_client.send_video_diffusion_request(request_config)[0]
|
||||
actual_path = output_dir / "actual.mp4"
|
||||
actual_path.write_bytes(result.videos[0])
|
||||
golden_path = output_dir / "pipeline.mp4"
|
||||
|
||||
metadata = probe_video(actual_path)
|
||||
assert_video_metadata(metadata, width=width, height=height, fps=16, frame_count=81)
|
||||
assert_video_similarity_metrics(
|
||||
label=f"sana_video_{variant}_{task}",
|
||||
online_path=actual_path,
|
||||
offline_path=golden_path,
|
||||
# The native prompt cross-attention uses vLLM-Omni Attention rather
|
||||
# than Diffusers SDPA. Frozen stage checks prove exact inputs and
|
||||
# scheduler state, while the BF16 backend reduction difference grows
|
||||
# across CFG denoising. Calibrate only the affected 480p T2V case;
|
||||
# its frozen result is 0.919861 SSIM / 28.7442 dB PSNR.
|
||||
ssim_threshold=SSIM_THRESHOLDS[(variant, task)],
|
||||
psnr_threshold=28.0,
|
||||
)
|
||||
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.benchmark,
|
||||
]
|
||||
|
||||
VARIANTS = {
|
||||
"480p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||
"vae_class": "AutoencoderKLWan",
|
||||
"native_vae_class": "DistributedAutoencoderKLWan",
|
||||
"dtype": torch.float32,
|
||||
"latent_channels": 16,
|
||||
"spatial_scale": 8,
|
||||
"temporal_scale": 4,
|
||||
"frames": 5,
|
||||
"max_abs": 1e-6,
|
||||
"relative_l2": 1e-6,
|
||||
},
|
||||
"720p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
|
||||
"vae_class": "AutoencoderKLLTX2Video",
|
||||
"native_vae_class": "DistributedAutoencoderKLLTX2Video",
|
||||
"dtype": torch.bfloat16,
|
||||
"latent_channels": 128,
|
||||
"spatial_scale": 32,
|
||||
"temporal_scale": 8,
|
||||
"frames": 9,
|
||||
"max_abs": 0.04,
|
||||
"relative_l2": 0.004,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", VARIANTS)
|
||||
@pytest.mark.parametrize(
|
||||
"_hardware",
|
||||
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
|
||||
)
|
||||
def test_sana_video_i2v_first_frame_latents_match_diffusers(variant: str, _hardware) -> None:
|
||||
del _hardware
|
||||
from diffusers import (
|
||||
AutoencoderKLLTX2Video,
|
||||
AutoencoderKLWan,
|
||||
)
|
||||
from diffusers import (
|
||||
SanaImageToVideoPipeline as DiffusersSanaImageToVideoPipeline,
|
||||
)
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
|
||||
DistributedAutoencoderKLLTX2Video,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
|
||||
DistributedAutoencoderKLWan,
|
||||
)
|
||||
from vllm_omni.diffusion.models.sana_video import SanaImageToVideoPipeline
|
||||
|
||||
config = VARIANTS[variant]
|
||||
vae_classes = {
|
||||
"AutoencoderKLLTX2Video": AutoencoderKLLTX2Video,
|
||||
"AutoencoderKLWan": AutoencoderKLWan,
|
||||
}
|
||||
native_vae_classes = {
|
||||
"DistributedAutoencoderKLLTX2Video": DistributedAutoencoderKLLTX2Video,
|
||||
"DistributedAutoencoderKLWan": DistributedAutoencoderKLWan,
|
||||
}
|
||||
reference_vae = (
|
||||
vae_classes[config["vae_class"]]
|
||||
.from_pretrained(
|
||||
config["model"],
|
||||
subfolder="vae",
|
||||
torch_dtype=config["dtype"],
|
||||
)
|
||||
.to("cuda")
|
||||
)
|
||||
native_vae = (
|
||||
native_vae_classes[config["native_vae_class"]]
|
||||
.from_config(dict(reference_vae.config))
|
||||
.to(device="cuda", dtype=config["dtype"])
|
||||
)
|
||||
native_vae.load_state_dict(reference_vae.state_dict())
|
||||
reference_vae.eval()
|
||||
native_vae.eval()
|
||||
|
||||
height, width = 192, 320
|
||||
num_latent_frames = (config["frames"] - 1) // config["temporal_scale"] + 1
|
||||
image = torch.linspace(
|
||||
-1,
|
||||
1,
|
||||
3 * height * width,
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
).reshape(1, 3, height, width)
|
||||
noise = torch.randn(
|
||||
(
|
||||
1,
|
||||
config["latent_channels"],
|
||||
num_latent_frames,
|
||||
height // config["spatial_scale"],
|
||||
width // config["spatial_scale"],
|
||||
),
|
||||
generator=torch.Generator(device="cuda").manual_seed(7),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
reference_pipeline = SimpleNamespace(
|
||||
vae=reference_vae,
|
||||
vae_scale_factor_temporal=config["temporal_scale"],
|
||||
vae_scale_factor_spatial=config["spatial_scale"],
|
||||
)
|
||||
native_pipeline = SimpleNamespace(
|
||||
vae=native_vae,
|
||||
vae_scale_factor_temporal=config["temporal_scale"],
|
||||
vae_scale_factor_spatial=config["spatial_scale"],
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
expected = DiffusersSanaImageToVideoPipeline.prepare_latents(
|
||||
reference_pipeline,
|
||||
image,
|
||||
1,
|
||||
config["latent_channels"],
|
||||
height,
|
||||
width,
|
||||
config["frames"],
|
||||
torch.float32,
|
||||
torch.device("cuda"),
|
||||
None,
|
||||
noise.clone(),
|
||||
)
|
||||
actual = SanaImageToVideoPipeline._prepare_i2v_latents(
|
||||
native_pipeline,
|
||||
image,
|
||||
1,
|
||||
config["latent_channels"],
|
||||
height,
|
||||
width,
|
||||
config["frames"],
|
||||
torch.float32,
|
||||
None,
|
||||
noise.clone(),
|
||||
)
|
||||
|
||||
# Conditioning replaces only the first latent frame.
|
||||
torch.testing.assert_close(actual[:, :, 1:], noise[:, :, 1:], rtol=0, atol=0)
|
||||
error = (actual[:, :, :1] - expected[:, :, :1]).float()
|
||||
expected_first_frame = expected[:, :, :1].float()
|
||||
max_abs = error.abs().max()
|
||||
relative_l2 = torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected_first_frame)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
actual[:, :, :1].flatten(),
|
||||
expected[:, :, :1].flatten(),
|
||||
dim=0,
|
||||
)
|
||||
assert max_abs.item() <= config["max_abs"]
|
||||
assert relative_l2.item() <= config["relative_l2"]
|
||||
assert cosine.item() >= 0.9999
|
||||
@@ -0,0 +1,432 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.benchmark,
|
||||
]
|
||||
|
||||
PROMPT = "A small red panda walks through a bamboo forest."
|
||||
NEGATIVE_PROMPT = "blur, distortion, abrupt cuts"
|
||||
SEED = 42
|
||||
GUIDANCE_SCALE = 6.0
|
||||
NUM_INFERENCE_STEPS = 2
|
||||
HEIGHT = 192
|
||||
WIDTH = 320
|
||||
NUM_FRAMES = 9
|
||||
|
||||
VARIANTS = {
|
||||
"480p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
|
||||
"model_env": "SANA_VIDEO_480P_MODEL",
|
||||
"max_abs": 0.2,
|
||||
"relative_l2": 0.02,
|
||||
"cosine": 0.999,
|
||||
"cfg_max_abs": 0.5,
|
||||
"cfg_relative_l2": 0.05,
|
||||
"cfg_cosine": 0.999,
|
||||
"step_max_abs": 0.075,
|
||||
"step_relative_l2": 0.012,
|
||||
"step_cosine": 0.9999,
|
||||
},
|
||||
"720p": {
|
||||
"model": "Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
|
||||
"revision": "8bda5e623d0f48cd6da3b387b10ca35d15cf1c4e",
|
||||
"model_env": "SANA_VIDEO_720P_MODEL",
|
||||
"max_abs": 0.1,
|
||||
"relative_l2": 0.015,
|
||||
"cosine": 0.999,
|
||||
"cfg_max_abs": 0.6,
|
||||
"cfg_relative_l2": 0.065,
|
||||
"cfg_cosine": 0.998,
|
||||
"step_max_abs": 0.08,
|
||||
"step_relative_l2": 0.014,
|
||||
"step_cosine": 0.9999,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ErrorMetrics:
|
||||
shape: tuple[int, ...]
|
||||
actual_dtype: torch.dtype
|
||||
expected_dtype: torch.dtype
|
||||
max_abs: float
|
||||
mean_abs: float
|
||||
relative_l2: float
|
||||
cosine: float
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"shape={self.shape}, actual_dtype={self.actual_dtype}, "
|
||||
f"expected_dtype={self.expected_dtype}, max_abs={self.max_abs:.8g}, "
|
||||
f"mean_abs={self.mean_abs:.8g}, relative_l2={self.relative_l2:.8g}, "
|
||||
f"cosine={self.cosine:.8g}"
|
||||
)
|
||||
|
||||
|
||||
def _metrics(actual: torch.Tensor, expected: torch.Tensor) -> ErrorMetrics:
|
||||
assert actual.shape == expected.shape
|
||||
actual_float = actual.detach().float()
|
||||
expected_float = expected.detach().float()
|
||||
error = actual_float - expected_float
|
||||
expected_norm = torch.linalg.vector_norm(expected_float)
|
||||
relative_l2 = torch.linalg.vector_norm(error) / expected_norm.clamp_min(torch.finfo(torch.float32).tiny)
|
||||
if actual_float.numel() == 1:
|
||||
cosine = torch.tensor(
|
||||
1.0 if bool(torch.equal(actual_float, expected_float)) else 0.0,
|
||||
device=actual.device,
|
||||
)
|
||||
else:
|
||||
cosine = F.cosine_similarity(actual_float.flatten(), expected_float.flatten(), dim=0)
|
||||
return ErrorMetrics(
|
||||
shape=tuple(actual.shape),
|
||||
actual_dtype=actual.dtype,
|
||||
expected_dtype=expected.dtype,
|
||||
max_abs=error.abs().max().item(),
|
||||
mean_abs=error.abs().mean().item(),
|
||||
relative_l2=relative_l2.item(),
|
||||
cosine=cosine.item(),
|
||||
)
|
||||
|
||||
|
||||
def _assert_stage(
|
||||
stage: str,
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
max_abs: float,
|
||||
relative_l2: float,
|
||||
cosine: float,
|
||||
) -> ErrorMetrics:
|
||||
result = _metrics(actual, expected)
|
||||
print(f"\nSANA-Video alignment [{stage}]: {result}")
|
||||
assert result.actual_dtype == result.expected_dtype, f"{stage}: {result}"
|
||||
assert result.max_abs <= max_abs, f"{stage}: {result}"
|
||||
assert result.relative_l2 <= relative_l2, f"{stage}: {result}"
|
||||
assert result.cosine >= cosine, f"{stage}: {result}"
|
||||
return result
|
||||
|
||||
|
||||
def _model_source(variant: str) -> tuple[str, str | None]:
|
||||
config = VARIANTS[variant]
|
||||
model = os.getenv(config["model_env"], config["model"])
|
||||
revision = None if Path(model).exists() else config["revision"]
|
||||
return model, revision
|
||||
|
||||
|
||||
def _native_method_proxy(reference_pipeline: Any):
|
||||
"""Build the minimum native-pipeline surface needed by stage helpers.
|
||||
|
||||
Sharing the tokenizer and text encoder makes this a comparison of pipeline
|
||||
orchestration rather than a second copy of the same multi-billion-parameter
|
||||
encoder. Transformer execution still uses separate Diffusers and native
|
||||
modules below.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.sana_video import SanaVideoPipeline
|
||||
|
||||
proxy = object.__new__(SanaVideoPipeline)
|
||||
torch.nn.Module.__init__(proxy)
|
||||
proxy.tokenizer = reference_pipeline.tokenizer
|
||||
proxy.text_encoder = reference_pipeline.text_encoder
|
||||
proxy.device = torch.device("cuda")
|
||||
proxy.vae_scale_factor_temporal = reference_pipeline.vae_scale_factor_temporal
|
||||
proxy.vae_scale_factor_spatial = reference_pipeline.vae_scale_factor_spatial
|
||||
return proxy
|
||||
|
||||
|
||||
def _cfg_prediction(
|
||||
prediction: torch.Tensor,
|
||||
*,
|
||||
guidance_scale: float,
|
||||
latent_channels: int,
|
||||
out_channels: int,
|
||||
) -> torch.Tensor:
|
||||
unconditional, conditional = prediction.float().chunk(2)
|
||||
prediction = unconditional + guidance_scale * (conditional - unconditional)
|
||||
if out_channels // 2 == latent_channels:
|
||||
prediction = prediction.chunk(2, dim=1)[0]
|
||||
return prediction
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", VARIANTS)
|
||||
@pytest.mark.parametrize(
|
||||
"_hardware",
|
||||
[pytest.param(None, marks=hardware_marks(res={"cuda": "H100"}))],
|
||||
)
|
||||
def test_sana_video_quick_pipeline_stages_match_diffusers(
|
||||
variant: str,
|
||||
_hardware,
|
||||
) -> None:
|
||||
"""Compare the first denoising step without replacing native attention.
|
||||
|
||||
This deliberately stops after the first scheduler step. It keeps the test
|
||||
useful as a stage-localizing PR/nightly check while the frozen 81-frame,
|
||||
50-step golden test covers the final latent and decoded-video boundary.
|
||||
"""
|
||||
del _hardware
|
||||
from diffusers import SanaVideoPipeline as DiffusersSanaVideoPipeline
|
||||
from diffusers.pipelines.sana_video.pipeline_sana_video import (
|
||||
retrieve_timesteps as diffusers_retrieve_timesteps,
|
||||
)
|
||||
from diffusers.schedulers import DPMSolverMultistepScheduler
|
||||
|
||||
from vllm_omni.diffusion.models.sana_video import (
|
||||
SanaVideoPipeline,
|
||||
SanaVideoTransformer3DModel,
|
||||
)
|
||||
from vllm_omni.diffusion.models.sana_video.pipeline_sana_video import (
|
||||
retrieve_timesteps as native_retrieve_timesteps,
|
||||
)
|
||||
|
||||
model, revision = _model_source(variant)
|
||||
config = VARIANTS[variant]
|
||||
load_kwargs: dict[str, Any] = {
|
||||
"torch_dtype": torch.bfloat16,
|
||||
# No stage in this test decodes latents. Avoid loading a multi-GB VAE
|
||||
# that would add memory and runtime without increasing coverage.
|
||||
"vae": None,
|
||||
}
|
||||
if revision is not None:
|
||||
load_kwargs["revision"] = revision
|
||||
|
||||
reference = DiffusersSanaVideoPipeline.from_pretrained(model, **load_kwargs).to("cuda")
|
||||
assert type(reference) is DiffusersSanaVideoPipeline
|
||||
assert type(reference.transformer).__module__.startswith("diffusers.")
|
||||
native_proxy = _native_method_proxy(reference)
|
||||
assert type(native_proxy) is SanaVideoPipeline
|
||||
assert type(native_proxy).__module__.startswith("vllm_omni.")
|
||||
try:
|
||||
with torch.inference_mode():
|
||||
reference_prompt = reference.encode_prompt(
|
||||
PROMPT,
|
||||
True,
|
||||
negative_prompt=NEGATIVE_PROMPT,
|
||||
device=torch.device("cuda"),
|
||||
clean_caption=False,
|
||||
max_sequence_length=300,
|
||||
complex_human_instruction=None,
|
||||
)
|
||||
native_prompt = SanaVideoPipeline.encode_prompt(
|
||||
native_proxy,
|
||||
PROMPT,
|
||||
True,
|
||||
negative_prompt=NEGATIVE_PROMPT,
|
||||
device=torch.device("cuda"),
|
||||
clean_caption=False,
|
||||
max_sequence_length=300,
|
||||
complex_human_instruction=None,
|
||||
)
|
||||
prompt_stage_names = (
|
||||
"prompt_embeddings",
|
||||
"prompt_attention_mask",
|
||||
"negative_prompt_embeddings",
|
||||
"negative_prompt_attention_mask",
|
||||
)
|
||||
for stage, actual, expected in zip(prompt_stage_names, native_prompt, reference_prompt, strict=True):
|
||||
_assert_stage(
|
||||
stage,
|
||||
actual,
|
||||
expected,
|
||||
max_abs=0.0,
|
||||
relative_l2=0.0,
|
||||
cosine=0.999999,
|
||||
)
|
||||
|
||||
reference_prompt_embeds, reference_prompt_mask, reference_negative_embeds, reference_negative_mask = (
|
||||
reference_prompt
|
||||
)
|
||||
encoder_hidden_states = torch.cat([reference_negative_embeds, reference_prompt_embeds])
|
||||
encoder_attention_mask = torch.cat([reference_negative_mask, reference_prompt_mask])
|
||||
|
||||
latent_channels = reference.transformer.config.in_channels
|
||||
reference_latents = reference.prepare_latents(
|
||||
1,
|
||||
latent_channels,
|
||||
HEIGHT,
|
||||
WIDTH,
|
||||
NUM_FRAMES,
|
||||
torch.float32,
|
||||
torch.device("cuda"),
|
||||
torch.Generator(device="cuda").manual_seed(SEED),
|
||||
)
|
||||
native_latents = SanaVideoPipeline.prepare_latents(
|
||||
native_proxy,
|
||||
1,
|
||||
latent_channels,
|
||||
HEIGHT,
|
||||
WIDTH,
|
||||
NUM_FRAMES,
|
||||
torch.float32,
|
||||
torch.device("cuda"),
|
||||
torch.Generator(device="cuda").manual_seed(SEED),
|
||||
)
|
||||
_assert_stage(
|
||||
"initial_noise_latents",
|
||||
native_latents,
|
||||
reference_latents,
|
||||
max_abs=0.0,
|
||||
relative_l2=0.0,
|
||||
cosine=0.999999,
|
||||
)
|
||||
|
||||
reference_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
|
||||
native_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
|
||||
isolated_scheduler = DPMSolverMultistepScheduler.from_config(reference.scheduler.config)
|
||||
reference_timesteps, reference_num_steps = diffusers_retrieve_timesteps(
|
||||
reference_scheduler,
|
||||
NUM_INFERENCE_STEPS,
|
||||
torch.device("cuda"),
|
||||
)
|
||||
native_timesteps, native_num_steps = native_retrieve_timesteps(
|
||||
native_scheduler,
|
||||
NUM_INFERENCE_STEPS,
|
||||
torch.device("cuda"),
|
||||
)
|
||||
isolated_retrieve_timesteps, isolated_num_steps = native_retrieve_timesteps(
|
||||
isolated_scheduler,
|
||||
NUM_INFERENCE_STEPS,
|
||||
torch.device("cuda"),
|
||||
)
|
||||
assert reference_num_steps == native_num_steps == isolated_num_steps
|
||||
_assert_stage(
|
||||
"timesteps",
|
||||
native_timesteps,
|
||||
reference_timesteps,
|
||||
max_abs=0.0,
|
||||
relative_l2=0.0,
|
||||
cosine=0.999999,
|
||||
)
|
||||
_assert_stage(
|
||||
"sigmas",
|
||||
native_scheduler.sigmas,
|
||||
reference_scheduler.sigmas,
|
||||
max_abs=0.0,
|
||||
relative_l2=0.0,
|
||||
cosine=0.999999,
|
||||
)
|
||||
torch.testing.assert_close(isolated_retrieve_timesteps, reference_timesteps, rtol=0, atol=0)
|
||||
|
||||
native_transformer = SanaVideoTransformer3DModel.from_config(dict(reference.transformer.config)).to(
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
incompatible = native_transformer.load_state_dict(reference.transformer.state_dict(), strict=True)
|
||||
assert not incompatible.missing_keys
|
||||
assert not incompatible.unexpected_keys
|
||||
assert type(native_transformer) is SanaVideoTransformer3DModel
|
||||
assert type(native_transformer).__module__.startswith("vllm_omni.")
|
||||
assert type(native_transformer) is not type(reference.transformer)
|
||||
reference_parameter = next(reference.transformer.parameters())
|
||||
native_parameter = next(native_transformer.parameters())
|
||||
assert native_parameter.data_ptr() != reference_parameter.data_ptr()
|
||||
torch.testing.assert_close(native_parameter, reference_parameter, rtol=0, atol=0)
|
||||
native_transformer.eval()
|
||||
reference.transformer.eval()
|
||||
|
||||
latent_model_input = torch.cat([reference_latents, reference_latents])
|
||||
timestep = reference_timesteps[0].expand(latent_model_input.shape[0])
|
||||
with torch.inference_mode():
|
||||
reference_raw_prediction = reference.transformer(
|
||||
latent_model_input.to(dtype=torch.bfloat16),
|
||||
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
timestep=timestep,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
native_raw_prediction = native_transformer(
|
||||
latent_model_input.to(dtype=torch.bfloat16),
|
||||
encoder_hidden_states=encoder_hidden_states.to(dtype=torch.bfloat16),
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
timestep=timestep,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
_assert_stage(
|
||||
"first_transformer_raw_prediction",
|
||||
native_raw_prediction,
|
||||
reference_raw_prediction,
|
||||
max_abs=config["max_abs"],
|
||||
relative_l2=config["relative_l2"],
|
||||
cosine=config["cosine"],
|
||||
)
|
||||
reference_cfg_prediction = _cfg_prediction(
|
||||
reference_raw_prediction,
|
||||
guidance_scale=GUIDANCE_SCALE,
|
||||
latent_channels=latent_channels,
|
||||
out_channels=reference.transformer.config.out_channels,
|
||||
)
|
||||
native_cfg_prediction = _cfg_prediction(
|
||||
native_raw_prediction,
|
||||
guidance_scale=GUIDANCE_SCALE,
|
||||
latent_channels=latent_channels,
|
||||
out_channels=native_transformer.config.out_channels,
|
||||
)
|
||||
_assert_stage(
|
||||
"first_cfg_prediction",
|
||||
native_cfg_prediction,
|
||||
reference_cfg_prediction,
|
||||
max_abs=config["cfg_max_abs"],
|
||||
relative_l2=config["cfg_relative_l2"],
|
||||
cosine=config["cfg_cosine"],
|
||||
)
|
||||
|
||||
reference_first_step = reference_scheduler.step(
|
||||
reference_cfg_prediction,
|
||||
reference_timesteps[0],
|
||||
reference_latents,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
# Isolate scheduler orchestration from native-attention error by
|
||||
# feeding the Diffusers prediction through an independently configured
|
||||
# scheduler initialized by the native helper.
|
||||
isolated_first_step = isolated_scheduler.step(
|
||||
reference_cfg_prediction,
|
||||
isolated_retrieve_timesteps[0],
|
||||
reference_latents,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
_assert_stage(
|
||||
"first_scheduler_step_isolated",
|
||||
isolated_first_step,
|
||||
reference_first_step,
|
||||
max_abs=0.0,
|
||||
relative_l2=0.0,
|
||||
cosine=0.999999,
|
||||
)
|
||||
|
||||
native_first_step = native_scheduler.step(
|
||||
native_cfg_prediction,
|
||||
native_timesteps[0],
|
||||
native_latents,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
_assert_stage(
|
||||
"first_scheduler_step_end_to_end",
|
||||
native_first_step,
|
||||
reference_first_step,
|
||||
max_abs=config["step_max_abs"],
|
||||
relative_l2=config["step_relative_l2"],
|
||||
cosine=config["step_cosine"],
|
||||
)
|
||||
finally:
|
||||
del native_proxy
|
||||
del reference
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SANA-Video 2B online smoke tests for the native and Diffusers backends."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL_480P = os.environ.get(
|
||||
"SANA_VIDEO_480P_MODEL",
|
||||
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||
)
|
||||
MODEL_720P = os.environ.get(
|
||||
"SANA_VIDEO_720P_MODEL",
|
||||
"Efficient-Large-Model/SANA-Video_2B_720p_diffusers",
|
||||
)
|
||||
PROMPT = "A cat walking on grass toward the camera. motion score: 30."
|
||||
NEGATIVE_PROMPT = "blurry, low quality, temporal artifacts"
|
||||
|
||||
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _t2v_backend_cases():
|
||||
cases = []
|
||||
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
|
||||
cases.extend(
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(model=model, server_args=["--model-class-name", "SanaVideoPipeline"]),
|
||||
id=f"native-{variant}",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--diffusion-load-format",
|
||||
"diffusers",
|
||||
"--diffusion-attention-backend",
|
||||
"TORCH_SDPA",
|
||||
],
|
||||
),
|
||||
id=f"diffusers-adapter-{variant}",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
),
|
||||
]
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _t2v_backend_cases(), indirect=True)
|
||||
def test_sana_video_t2v_backends(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": 192,
|
||||
"width": 320,
|
||||
"num_frames": 9,
|
||||
"fps": 8,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
openai_client.send_video_diffusion_request(request_config)
|
||||
|
||||
|
||||
def _i2v_cases():
|
||||
cases = []
|
||||
for variant, model in (("480p", MODEL_480P), ("720p", MODEL_720P)):
|
||||
cases.extend(
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(model=model, server_args=["--model-class-name", "SanaImageToVideoPipeline"]),
|
||||
id=f"native-{variant}",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--model-class-name",
|
||||
"SanaImageToVideoPipeline",
|
||||
"--diffusion-load-format",
|
||||
"diffusers",
|
||||
"--diffusion-attention-backend",
|
||||
"TORCH_SDPA",
|
||||
],
|
||||
),
|
||||
id=f"diffusers-adapter-{variant}",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
),
|
||||
]
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _i2v_cases(), indirect=True)
|
||||
def test_sana_video_i2v_variants(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": 192,
|
||||
"width": 320,
|
||||
"num_frames": 9,
|
||||
"fps": 8,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
"image_reference": (f"data:image/jpeg;base64,{generate_synthetic_image(320, 192, seed=42)['base64']}"),
|
||||
}
|
||||
openai_client.send_video_diffusion_request(request_config)
|
||||
Reference in new issue
Block a user