[Model] Add SANA-WM support(Stage-1 only) (#4061)

Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Signed-off-by: liuqihao <liuqihao970610@gmail.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
BruceLoveDecimaland汪志鹏 authored and GitHub committed 2026-08-18 23:57:55 -06:00
1 parent 14bd8f8772
commit adf6d0c1ea
21 files changed
+5173 -1

No files matched your search

@@ -0,0 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SANA-WM diffusion model integration."""
from vllm_omni.diffusion.models.sana_wm.config import SanaWmConfig
from vllm_omni.diffusion.models.sana_wm.pipeline_sana_wm import (
SANA_WM_MODEL_ID,
SANA_WM_OUTPUT_HEIGHT,
SANA_WM_OUTPUT_WIDTH,
SanaWmPipeline,
get_sana_wm_pre_process_func,
)
from vllm_omni.diffusion.models.sana_wm.request import normalize_sana_wm_payload
from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import SanaWmTransformer3DModel
__all__ = [
"SANA_WM_MODEL_ID",
"SANA_WM_OUTPUT_HEIGHT",
"SANA_WM_OUTPUT_WIDTH",
"SanaWmConfig",
"SanaWmPipeline",
"SanaWmTransformer3DModel",
"get_sana_wm_pre_process_func",
"normalize_sana_wm_payload",
]
@@ -0,0 +1,356 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Camera-control helpers for SANA-WM.
This module mirrors the public NVlabs/Sana SANA-WM camera preparation path:
WASD/IJKL action rollout or explicit camera-to-world poses are converted into
relative poses, per-latent-frame ray metadata, and per-VAE-chunk Plucker maps.
Model-local projection layers consume these tensors in the Stage-1 DiT.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
SANA_WM_DEFAULT_TRANSLATION_SPEED = 0.05
SANA_WM_DEFAULT_ROTATION_SPEED_DEG = 1.2
SANA_WM_DEFAULT_PITCH_LIMIT_DEG = 85.0
SANA_WM_ALLOWED_ACTION_KEYS = frozenset("wasdijkl")
SANA_WM_DEFAULT_VAE_STRIDE = (8, 32, 32)
@dataclass(frozen=True)
class SanaWmCameraCondition:
poses: Any | None = None
intrinsics: Any | None = None
action: str | None = None
num_frames: int | None = None
height: int = 704
width: int = 1280
translation_speed: float = SANA_WM_DEFAULT_TRANSLATION_SPEED
rotation_speed_deg: float = SANA_WM_DEFAULT_ROTATION_SPEED_DEG
pitch_limit_deg: float = SANA_WM_DEFAULT_PITCH_LIMIT_DEG
def _rot_x(angle_rad: float) -> np.ndarray:
c, s = np.cos(angle_rad), np.sin(angle_rad)
return np.array([[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]], dtype=np.float64)
def _rot_y(angle_rad: float) -> np.ndarray:
c, s = np.cos(angle_rad), np.sin(angle_rad)
return np.array([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]], dtype=np.float64)
def _parse_action_segments(action: str) -> list[tuple[list[str], int]]:
"""Parse an action string into ``(keys, duration)`` pairs without expanding it.
A single segment names an unbounded rollout (``"w-1000000000"`` is valid
syntax), so expanding here would allocate one list entry per frame before
anything checks the frame count. Callers total the durations and validate
against ``num_frames`` first; only :func:`_parse_action_string` expands.
"""
cleaned = "".join(action.replace(",", ",").split())
if not cleaned:
raise ValueError("Sana-WM action string is empty.")
segments: list[tuple[list[str], int]] = []
for segment in cleaned.split(","):
if not segment or "-" not in segment:
raise ValueError(f"Invalid Sana-WM action segment {segment!r}: expected '<keys>-<duration>'.")
keys_part, duration = segment.rsplit("-", 1)
if not duration.isdigit() or int(duration) <= 0:
raise ValueError(f"Sana-WM action segment {segment!r} has a non-positive duration {duration!r}.")
keys_lower = keys_part.lower()
if keys_lower == "none":
keys: list[str] = []
else:
bad = sorted({key for key in keys_lower if key not in SANA_WM_ALLOWED_ACTION_KEYS})
if bad:
allowed = "".join(sorted(SANA_WM_ALLOWED_ACTION_KEYS))
raise ValueError(f"Sana-WM action segment {segment!r} contains unknown keys {bad}; allowed: {allowed}.")
keys = sorted(set(keys_lower))
segments.append((keys, int(duration)))
return segments
def action_rollout_num_frames(action: str) -> int:
"""Frames ``action`` rolls out: the identity start pose plus one per step.
Totals the durations arithmetically, so an oversized request is rejected
before anything allocates per-frame state.
"""
return sum(duration for _, duration in _parse_action_segments(action)) + 1
def _parse_action_string(action: str) -> list[list[str]]:
per_frame: list[list[str]] = []
for keys, duration in _parse_action_segments(action):
per_frame.extend([keys] * duration)
return per_frame
def action_string_to_c2w(
action: str,
*,
translation_speed: float = SANA_WM_DEFAULT_TRANSLATION_SPEED,
rotation_speed_deg: float = SANA_WM_DEFAULT_ROTATION_SPEED_DEG,
pitch_limit_deg: float = SANA_WM_DEFAULT_PITCH_LIMIT_DEG,
) -> np.ndarray:
"""Roll out an OpenCV-convention ``(N + 1, 4, 4)`` c2w trajectory."""
per_frame = _parse_action_string(action)
rotate_rad = math.radians(rotation_speed_deg)
pitch_limit_rad = math.radians(pitch_limit_deg)
current = np.eye(4, dtype=np.float64)
current_pitch = 0.0
poses = [current.copy()]
for keys in per_frame:
held = set(keys)
rotation = current[:3, :3]
translation = current[:3, 3]
pitch_delta = (rotate_rad if "i" in held else 0.0) - (rotate_rad if "k" in held else 0.0)
new_pitch = current_pitch + pitch_delta
if -pitch_limit_rad <= new_pitch <= pitch_limit_rad:
current_pitch = new_pitch
else:
pitch_delta = 0.0
yaw_delta = (rotate_rad if "l" in held else 0.0) - (rotate_rad if "j" in held else 0.0)
new_rotation = _rot_y(yaw_delta) @ rotation @ _rot_x(pitch_delta)
forward = new_rotation[:, 2].copy()
forward[1] = 0.0
right = new_rotation[:, 0].copy()
right[1] = 0.0
forward_norm = float(np.linalg.norm(forward))
right_norm = float(np.linalg.norm(right))
if forward_norm > 0:
forward /= forward_norm + 1e-6
if right_norm > 0:
right /= right_norm + 1e-6
move = np.zeros(3, dtype=np.float64)
if "w" in held:
move += forward * translation_speed
if "s" in held:
move -= forward * translation_speed
if "d" in held:
move += right * translation_speed
if "a" in held:
move -= right * translation_speed
current = np.eye(4, dtype=np.float64)
current[:3, :3] = new_rotation
current[:3, 3] = translation + move
poses.append(current.copy())
return np.stack(poses, axis=0).astype(np.float32)
def get_pose_inverse(transform: torch.Tensor) -> torch.Tensor:
"""Invert homogeneous rigid transforms with shape ``(..., 4, 4)``."""
rotation = transform[..., :3, :3]
translation = transform[..., :3, 3]
rotation_inv = rotation.transpose(-1, -2)
translation_inv = -torch.matmul(rotation_inv, translation.unsqueeze(-1)).squeeze(-1)
output = torch.eye(4, dtype=transform.dtype, device=transform.device).repeat(transform.shape[:-2] + (1, 1))
output[..., :3, :3] = rotation_inv
output[..., :3, 3] = translation_inv
return output
def compute_raymap(
intrinsics: torch.Tensor,
poses: torch.Tensor,
height: int,
width: int,
*,
use_plucker: bool = True,
) -> torch.Tensor:
"""Compute SANA-WM geometry ray maps.
Args:
intrinsics: ``(T, 4)`` tensor in ``[fx, fy, cx, cy]`` order.
poses: ``(T, 4, 4)`` camera-to-world matrices.
height: raymap height in latent pixels.
width: raymap width in latent pixels.
use_plucker: return ``[direction, moment]`` if true, otherwise
``[origin, direction]``.
"""
if intrinsics.ndim != 2 or intrinsics.shape[-1] != 4:
raise ValueError(f"Sana-WM intrinsics must have shape (T, 4), got {tuple(intrinsics.shape)}.")
if poses.ndim != 3 or poses.shape[-2:] != (4, 4):
raise ValueError(f"Sana-WM poses must have shape (T, 4, 4), got {tuple(poses.shape)}.")
if intrinsics.shape[0] != poses.shape[0]:
raise ValueError("Sana-WM intrinsics and poses must have the same temporal length.")
frames = intrinsics.shape[0]
device = intrinsics.device
dtype = intrinsics.dtype
y_grid, x_grid = torch.meshgrid(
torch.arange(height, device=device, dtype=dtype),
torch.arange(width, device=device, dtype=dtype),
indexing="ij",
)
x_grid = x_grid[None].expand(frames, -1, -1)
y_grid = y_grid[None].expand(frames, -1, -1)
fx = intrinsics[:, 0].view(frames, 1, 1)
fy = intrinsics[:, 1].view(frames, 1, 1)
cx = intrinsics[:, 2].view(frames, 1, 1)
cy = intrinsics[:, 3].view(frames, 1, 1)
dirs_cam = torch.stack([(x_grid - cx) / fx, (y_grid - cy) / fy, torch.ones_like(x_grid)], dim=-1)
rotation = poses[:, :3, :3]
translation = poses[:, :3, 3]
dirs_world = torch.einsum("tij,thwj->thwi", rotation, dirs_cam)
dirs_world = dirs_world / torch.norm(dirs_world, dim=-1, keepdim=True).clamp_min(1e-12)
origins = translation.view(frames, 1, 1, 3).expand_as(dirs_world)
if use_plucker:
moments = torch.cross(origins, dirs_world, dim=-1)
return torch.cat([dirs_world, moments], dim=-1)
return torch.cat([origins, dirs_world], dim=-1)
def intrinsics_to_vec4_array(intrinsics: Any, *, num_frames: int, height: int, width: int) -> np.ndarray:
"""Normalize the intrinsics payload to an ``(F, 4)`` ``[fx, fy, cx, cy]`` array.
Accepts either ``None`` (derive from the output resolution) or the
``{fx, fy, cx, cy}`` mapping — the only form the request contract exposes.
"""
if intrinsics is None:
focal = float(max(height, width))
vec = np.array([focal, focal, width / 2.0, height / 2.0], dtype=np.float32)
return np.broadcast_to(vec, (num_frames, 4)).copy()
if isinstance(intrinsics, dict):
vec = np.array(
[intrinsics["fx"], intrinsics["fy"], intrinsics["cx"], intrinsics["cy"]],
dtype=np.float32,
)
return np.broadcast_to(vec, (num_frames, 4)).copy()
raise ValueError("Sana-WM intrinsics must be a {fx, fy, cx, cy} mapping.")
def _pack_camera_conditions(
poses: torch.Tensor,
intrinsics_latent: torch.Tensor,
*,
num_frames: int,
latent_height: int,
latent_width: int,
vae_time_stride: int,
) -> dict[str, torch.Tensor]:
# This yields exactly the latent frame count the VAE produces:
# ``len(arange(0, n, s)) == ceil(n / s) == (n - 1) // s + 1``.
time_indices = torch.arange(0, num_frames, vae_time_stride)
raymap = torch.cat([poses[time_indices].reshape(len(time_indices), -1), intrinsics_latent[time_indices]], dim=-1)
chunks: list[torch.Tensor] = []
spatial_raymap_frames: list[torch.Tensor] = []
chunk_starts = time_indices - (vae_time_stride - 1)
for start, t_idx in zip(chunk_starts, time_indices):
start_idx = max(0, int(start))
end_idx = start_idx + vae_time_stride
chunk_poses = poses[start_idx:end_idx]
chunk_intrinsics = intrinsics_latent[start_idx:end_idx]
if chunk_poses.shape[0] < vae_time_stride:
pad = vae_time_stride - chunk_poses.shape[0]
chunk_poses = torch.cat([chunk_poses, chunk_poses[-1:].repeat(pad, 1, 1)], dim=0)
chunk_intrinsics = torch.cat([chunk_intrinsics, chunk_intrinsics[-1:].repeat(pad, 1)], dim=0)
plucker = compute_raymap(chunk_intrinsics, chunk_poses, latent_height, latent_width, use_plucker=True)
chunks.append(plucker.permute(0, 3, 1, 2).reshape(-1, latent_height, latent_width))
# Per-frame spatial ray-direction map for raymap_embedder: take the
# representative frame in each chunk (the anchor at t_idx) and use only
# the direction channels (last 3 of 6 in the origin+direction layout).
t = int(t_idx)
od = compute_raymap(
intrinsics_latent[t : t + 1],
poses[t : t + 1],
latent_height,
latent_width,
use_plucker=False,
) # [1, H, W, 6]
spatial_raymap_frames.append(od[0, :, :, 3:]) # [H, W, 3] — direction only
chunk_plucker = torch.stack(chunks).permute(1, 0, 2, 3)
# spatial_raymap: [3, F_latent, H, W] — consumed by raymap_embedder
spatial_raymap = torch.stack(spatial_raymap_frames).permute(3, 0, 1, 2)
return {"raymap": raymap, "chunk_plucker": chunk_plucker, "spatial_raymap": spatial_raymap}
def build_plucker_condition(
condition: SanaWmCameraCondition,
*,
vae_stride: tuple[int, int, int] | list[int] = SANA_WM_DEFAULT_VAE_STRIDE,
) -> dict[str, torch.Tensor]:
"""Build native Stage-1 camera tensors from normalized request metadata."""
if bool(condition.action) == (condition.poses is not None):
raise ValueError("Sana-WM camera condition requires exactly one of action or poses.")
if condition.action:
poses_c2w = action_string_to_c2w(
condition.action,
translation_speed=condition.translation_speed,
rotation_speed_deg=condition.rotation_speed_deg,
pitch_limit_deg=condition.pitch_limit_deg,
)
else:
poses_c2w = np.asarray(condition.poses, dtype=np.float32)
if poses_c2w.ndim != 3 or poses_c2w.shape[-2:] != (4, 4):
raise ValueError(f"Sana-WM c2w poses must have shape (F, 4, 4), got {poses_c2w.shape}.")
num_frames = condition.num_frames or int(poses_c2w.shape[0])
if poses_c2w.shape[0] < num_frames:
if condition.action:
num_frames = int(poses_c2w.shape[0])
else:
raise ValueError(f"Sana-WM c2w poses length {poses_c2w.shape[0]} is shorter than num_frames {num_frames}.")
poses_c2w = poses_c2w[:num_frames]
intrinsics = intrinsics_to_vec4_array(
condition.intrinsics,
num_frames=num_frames,
height=condition.height,
width=condition.width,
)
vae_time_stride = int(vae_stride[0])
vae_spatial_stride = int(vae_stride[-1])
latent_height = condition.height // vae_spatial_stride
latent_width = condition.width // vae_spatial_stride
poses = torch.from_numpy(poses_c2w).float()
first_inv = get_pose_inverse(poses[0:1]).squeeze(0)
poses_rel = torch.matmul(first_inv, poses[1:])
poses = torch.cat([torch.eye(4, dtype=poses.dtype).unsqueeze(0), poses_rel], dim=0)
intrinsics_latent = torch.from_numpy(intrinsics).float()
intrinsics_latent[:, [0, 2]] *= latent_width / float(condition.width)
intrinsics_latent[:, [1, 3]] *= latent_height / float(condition.height)
return _pack_camera_conditions(
poses,
intrinsics_latent,
num_frames=num_frames,
latent_height=latent_height,
latent_width=latent_width,
vae_time_stride=vae_time_stride,
)
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Config for the SANA-WM Stage-1 transformer.
Loaded from the standard diffusers ``transformer/config.json`` (a flat dict
keyed by the dataclass field names). The bespoke ``config.yaml`` release format
is handled once, offline, at checkpoint-conversion time — the runtime only ever
sees the converted, diffusers-standard config.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any
def _as_tuple3(value: Any, default: tuple[int, int, int]) -> tuple[int, int, int]:
if value is None:
return default
if isinstance(value, int):
return (value, value, value)
if isinstance(value, (list, tuple)):
if len(value) == 2:
return (1, int(value[0]), int(value[1]))
if len(value) == 3:
return (int(value[0]), int(value[1]), int(value[2]))
return default
# LTX-2 VAE compression ratios (SANA-WM ships ``AutoencoderKLLTX2Video``).
# They live here rather than in the pipeline so the request-normalisation module
# can validate latent geometry without importing the pipeline.
SANA_WM_VAE_SPATIAL_COMPRESSION = 32
SANA_WM_VAE_TEMPORAL_COMPRESSION = 8
@dataclass(frozen=True)
class SanaWmConfig:
"""SANA-WM Stage-1 architecture and runtime config.
Defaults mirror the first public HF release. ``from_json`` (reading
``transformer/config.json``) is the normal construction path.
"""
architecture_name: str | None = "SanaMSVideoCamCtrl_1600M_P1_D20"
num_blocks: int = 20
hidden_size: int = 2240
mlp_ratio: float = 3.0
attn_type: str = "BidirectionalGDNTriton"
softmax_every_n: int = 4
linear_head_dim: int = 112
conv_kernel_size: int = 4
t_kernel_size: int = 3
k_conv_only: bool = True
ffn_type: str = "GLUMBConvTemp"
pos_embed_type: str = "wan_rope"
patch_size: tuple[int, int, int] = (1, 1, 1)
qk_norm: bool = True
cross_norm: bool = True
mixed_precision: str = "bf16"
fp32_attention: bool = True
image_size: int = 720
cam_attn_compress: int = 1
use_chunk_plucker_post_attn: bool = True
chunk_plucker_channels: int = 48
chunk_plucker_post_attn_blocks: int = 20
inference_flow_shift: float = 9.8
scheduler_type: str = "flow_dpm-solver"
chi_prompt: list[str] = field(default_factory=list)
model_max_length: int = 300
@classmethod
def from_dict(cls, data: Mapping[str, Any]) -> SanaWmConfig:
"""Build from a flat mapping of field name -> value.
Unknown keys (e.g. the diffusers ``_class_name`` marker or the
``latent_channels`` / ``prompt_channels`` constructor kwargs) are
ignored. ``patch_size`` is coerced from a JSON list back to a tuple.
"""
field_names = {f.name for f in fields(cls)}
known = {k: v for k, v in data.items() if k in field_names}
if "patch_size" in known:
known["patch_size"] = _as_tuple3(known["patch_size"], cls.patch_size)
return cls(**known)
@classmethod
def from_json(cls, path: str | Path) -> SanaWmConfig:
with open(path, encoding="utf-8") as f:
data = json.load(f)
return cls.from_dict(data if isinstance(data, Mapping) else {})
@@ -0,0 +1,942 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sana-WM pipeline integration.
This module wires the registry-visible surface, release-layout validation, and
the Stage-1 sampling loop. The DiT, Gated-DeltaNet and camera stack are
vLLM-Omni-native layers (``sana_wm_transformer.py`` / ``ucpe.py``); nothing here
calls into NVlabs code at runtime.
"""
from __future__ import annotations
import os
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any, ClassVar
import torch
import torch.nn.functional as F
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery
from vllm_omni.diffusion.models.ltx2.ltx2_latents import (
denormalize_latents,
normalize_latents,
pack_latents,
resolve_video_latent_shape,
unpack_latents,
)
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.models.sana_wm.camera_control import (
SanaWmCameraCondition,
build_plucker_condition,
)
from vllm_omni.diffusion.models.sana_wm.config import (
SANA_WM_VAE_SPATIAL_COMPRESSION,
SANA_WM_VAE_TEMPORAL_COMPRESSION,
SanaWmConfig,
)
from vllm_omni.diffusion.models.sana_wm.request import normalize_sana_wm_payload
from vllm_omni.diffusion.models.sana_wm.sana_wm_transformer import (
SANA_WM_STAGE1_PROMPT_CHANNELS,
SanaWmTransformer3DModel,
)
from vllm_omni.diffusion.models.schedulers import FlowMatchEulerDiscreteScheduler
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import (
DiffusionPipelineProfilerMixin,
)
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
logger = init_logger(__name__)
# The loader expects the standard diffusers layout (transformer/config.json +
# transformer/diffusion_pytorch_model.safetensors + model_index.json), so this
# repo is an offline conversion of the NVlabs release into that layout. It holds
# Stage-1 only — transformer/ + vae/, with model_index.json naming
# SanaWmPipeline — because startup prefetches the whole model repo
# (allow_patterns=["*"]); a repo that also carried the refiner would pull tens of
# GB this path never loads.
SANA_WM_MODEL_ID = "BBBBruce/SANA-WM_bidirectional-stage1-diffusers"
SANA_WM_STAGE1_DIT_FILE = "transformer/diffusion_pytorch_model.safetensors"
SANA_WM_CONFIG_FILE = "transformer/config.json"
SANA_WM_VAE_CONFIG_FILE = "vae/config.json"
SANA_WM_VAE_WEIGHT_FILE = "vae/diffusion_pytorch_model.safetensors"
SANA_WM_STAGE1_TEXT_ENCODER_ID = "google/gemma-2-2b-it"
SANA_WM_STAGE1_TEXT_ENCODER_FALLBACK_ID = "Efficient-Large-Model/gemma-2-2b-it"
SANA_WM_STAGE1_TEXT_ENCODER_ENV = "VLLM_OMNI_SANA_WM_STAGE1_TEXT_ENCODER"
SANA_WM_OUTPUT_HEIGHT = 704
SANA_WM_OUTPUT_WIDTH = 1280
SANA_WM_NUM_TRAIN_TIMESTEPS = 1000
# Maximum supported clip length (20s @16fps) — the native envelope, not a
# request default. Request-side defaults live in
# ``models/sana_wm/request.py``.
SANA_WM_NATIVE_NUM_FRAMES = 321
# Fail-fast latent-token cap for the native Stage-1 path, sized to the native
# 704x1280 output at the maximum clip length so it only rejects genuinely
# oversized requests. Overridable per-request via ``sana_wm_native_max_tokens``.
SANA_WM_NATIVE_MAX_TOKENS = (
((SANA_WM_NATIVE_NUM_FRAMES - 1) // SANA_WM_VAE_TEMPORAL_COMPRESSION + 1)
* (SANA_WM_OUTPUT_HEIGHT // SANA_WM_VAE_SPATIAL_COMPRESSION)
* (SANA_WM_OUTPUT_WIDTH // SANA_WM_VAE_SPATIAL_COMPRESSION)
)
# The NVlabs production inference settings for this release. They belong in the
# model rather than in a deploy YAML: single-stage diffusion models resolve
# through ``create_default_diffusion``, which never reads a YAML's
# ``default_sampling_params``, so a caller that omits either field lands here.
SANA_WM_DEFAULT_NUM_INFERENCE_STEPS = 60
SANA_WM_DEFAULT_GUIDANCE_SCALE = 5.0
SANA_WM_STAGE1_PATTERNS = (
SANA_WM_CONFIG_FILE,
SANA_WM_STAGE1_DIT_FILE,
SANA_WM_VAE_CONFIG_FILE,
SANA_WM_VAE_WEIGHT_FILE,
)
SANA_WM_STAGE1_DIT_SUBFOLDER = "transformer"
SANA_WM_STAGE1_DIT_BASENAME = Path(SANA_WM_STAGE1_DIT_FILE).name
@dataclass(frozen=True)
class SanaWmLocalPaths:
"""Resolved local file paths for a SANA-WM snapshot."""
root: Path
config: Path
stage1_dit: Path
vae_config: Path
vae_weights: Path
@dataclass(frozen=True)
class SanaWmNativeParams:
"""Resolved generation settings for one Stage-1 request.
Built by ``_native_params`` from the request payload and sampling params,
and consumed by ``_run_native_backend`` — this is the production path.
"""
height: int
width: int
num_frames: int
num_inference_steps: int
seed: int
# Defaults to the NVlabs production scale, so a caller that never mentions
# guidance still gets the two-branch CFG forward the model was tuned for.
# An explicit ``guidance_scale <= 1.0`` still selects the single-branch path.
cfg_scale: float = SANA_WM_DEFAULT_GUIDANCE_SCALE
def build_sana_wm_download_patterns() -> tuple[str, ...]:
"""Return the minimal HF allow-patterns needed for SANA-WM."""
return tuple(SANA_WM_STAGE1_PATTERNS)
def resolve_sana_wm_local_paths(snapshot_dir: str | Path) -> SanaWmLocalPaths:
root = Path(snapshot_dir)
return SanaWmLocalPaths(
root=root,
config=root / SANA_WM_CONFIG_FILE,
stage1_dit=root / SANA_WM_STAGE1_DIT_FILE,
vae_config=root / SANA_WM_VAE_CONFIG_FILE,
vae_weights=root / SANA_WM_VAE_WEIGHT_FILE,
)
def validate_sana_wm_local_paths(paths: SanaWmLocalPaths) -> None:
required = [
paths.config,
paths.stage1_dit,
paths.vae_config,
paths.vae_weights,
]
missing = [path for path in required if not path.exists()]
if missing:
joined = ", ".join(str(path) for path in missing)
raise FileNotFoundError(f"Missing SANA-WM checkpoint files: {joined}")
def resolve_or_download_sana_wm_checkpoint(
model: str = SANA_WM_MODEL_ID,
*,
revision: str | None = None,
cache_dir: str | None = None,
) -> SanaWmLocalPaths:
"""Resolve a local SANA-WM tree or download the required HF files."""
root = Path(model)
if root.is_dir():
snapshot_dir = root
else:
snapshot_dir = Path(
download_weights_from_hf_specific(
model,
cache_dir,
list(build_sana_wm_download_patterns()),
revision=revision,
require_all=True,
)
)
paths = resolve_sana_wm_local_paths(snapshot_dir)
validate_sana_wm_local_paths(paths)
return paths
def build_sana_wm_output_envelope(
*,
output: Any,
output_type: str,
metadata: dict[str, Any],
) -> dict[str, Any]:
"""Wrap a Stage-1 result in the canonical output envelope.
``normalize_diffusion_postprocess_output`` splits ``{"payload": ...,
"metadata": ...}`` into the API-facing payload and the metadata groups, so
the model-specific diagnostics ride along under a ``sana_wm`` group instead
of the removed ``DiffusionOutput.custom_output`` field.
"""
payload_key = "latents" if output_type == "latent" else "video"
return {"payload": {payload_key: output}, "metadata": {"sana_wm": metadata}}
def get_sana_wm_pre_process_func(od_config: OmniDiffusionConfig):
del od_config
def pre_process_func(request: Any) -> Any:
from vllm_omni.inputs.data import OmniTextPrompt
sampling_params = getattr(request, "sampling_params", None)
prompt = request.prompt
prompt_mapping = OmniTextPrompt(prompt=prompt) if isinstance(prompt, str) else prompt
prompt_mapping = dict(prompt_mapping)
# The camera block travels as extra_params["sana_wm"], which serving
# merges into extra_args. Offline callers may also put it straight on
# the prompt.
if sampling_params is not None:
extra_args = getattr(sampling_params, "extra_args", None)
if isinstance(extra_args, Mapping):
block = extra_args.get("sana_wm")
if isinstance(block, Mapping):
merged = dict(block)
merged.update(prompt_mapping.get("sana_wm") or {})
prompt_mapping["sana_wm"] = merged
# Geometry always comes from the standard sampling fields.
for field in ("num_frames", "height", "width"):
value = getattr(sampling_params, field, None)
if value is not None and not (field == "num_frames" and value == 1):
prompt_mapping[field] = value
request.prompt = normalize_sana_wm_payload(prompt_mapping)
# Preprocess is the only place the request is normalised, so publish the
# canonical geometry back onto sampling_params for the rest of the stack.
if sampling_params is not None:
payload = request.prompt["additional_information"]["sana_wm"]
sampling_params.num_frames = payload["num_frames"]
sampling_params.height = payload["height"]
sampling_params.width = payload["width"]
return request
return pre_process_func
class SanaWmPipeline(
nn.Module,
CFGParallelMixin,
SupportImageInput,
SupportsComponentDiscovery,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
):
"""Stage-1 SANA-WM image-to-video pipeline."""
support_image_input: ClassVar[bool] = True
color_format: ClassVar[str] = "RGB"
# Skip the startup warmup: the generic dummy request carries a blank image
# with no camera poses and no action string, which SANA-WM's request
# validation always rejects. The first real request does the warming.
dummy_run_num_frames: ClassVar[int] = 0
_dit_modules: ClassVar[list[str]] = ["transformer"]
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
_vae_modules: ClassVar[list[str]] = ["vae"]
_resident_modules: ClassVar[list[str]] = []
def __init__(self, *, od_config: OmniDiffusionConfig | None = None, prefix: str = "") -> None:
super().__init__()
self.od_config = od_config
self.prefix = prefix
self.sana_wm_config = SanaWmConfig()
self.quant_config = getattr(od_config, "quantization_config", None) if od_config is not None else None
self.tokenizer: Any | None = None
self.text_encoder: nn.Module | None = None
self.vae: nn.Module | None = None
self.release_paths: SanaWmLocalPaths | None = None
self.weights_sources = []
if od_config is not None and od_config.model is not None:
# Resolve the checkpoint once at construction (the standard
# startup-time flow, like every other diffusion pipeline) so the
# transformer below is built with the transformer/config.json shape
# rather than defaults.
self.resolve_checkpoint()
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder=SANA_WM_STAGE1_DIT_SUBFOLDER,
revision=od_config.revision,
# The subfolder file carries module-local keys (standard
# diffusers layout); prefix them into the pipeline
# namespace for the loader's strict-coverage check.
prefix="transformer.",
fall_back_to_pt=False,
allow_patterns_overrides=[SANA_WM_STAGE1_DIT_BASENAME],
)
]
# Built eagerly: the loader constructs the pipeline under the target
# device / default-dtype context, and the weight loader then streams
# checkpoint tensors directly into these modules.
self.transformer = SanaWmTransformer3DModel(
config=self.sana_wm_config,
quant_config=self.quant_config,
prefix=f"{prefix}.transformer" if prefix else "transformer",
)
self.device = get_local_device()
if od_config is not None and od_config.model is not None:
self._build_aux_components()
def resolve_checkpoint(self) -> SanaWmLocalPaths:
if self.release_paths is not None:
return self.release_paths
if self.od_config is None or self.od_config.model is None:
raise ValueError("Sana-WM checkpoint resolution requires od_config.model.")
self.release_paths = resolve_or_download_sana_wm_checkpoint(
self.od_config.model,
revision=self.od_config.revision,
)
self.sana_wm_config = SanaWmConfig.from_json(self.release_paths.config)
return self.release_paths
@staticmethod
def _extra_args(sampling_params: Any | None) -> dict[str, Any]:
extra_args = getattr(sampling_params, "extra_args", None) if sampling_params is not None else None
return dict(extra_args or {})
def _native_params(self, payload: dict[str, Any], sampling_params: Any | None) -> SanaWmNativeParams:
extra_args = self._extra_args(sampling_params)
height = int(getattr(sampling_params, "height", None) or payload["height"])
width = int(getattr(sampling_params, "width", None) or payload["width"])
num_frames = int(getattr(sampling_params, "num_frames", None) or payload["num_frames"])
steps = int(
getattr(sampling_params, "num_inference_steps", None)
or extra_args.get("num_inference_steps")
or SANA_WM_DEFAULT_NUM_INFERENCE_STEPS
)
seed = int(getattr(sampling_params, "seed", None) or extra_args.get("seed", 0))
if min(height, width, num_frames, steps) <= 0:
raise ValueError("Sana-WM native height, width, num_frames, and steps must be positive.")
cfg_scale = SANA_WM_DEFAULT_GUIDANCE_SCALE
if sampling_params is not None and getattr(sampling_params, "guidance_scale_provided", False):
cfg_scale = float(getattr(sampling_params, "guidance_scale", 1.0) or 1.0)
return SanaWmNativeParams(
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=steps,
seed=seed,
cfg_scale=cfg_scale,
)
def _native_dtype(self, device: torch.device) -> torch.dtype:
dtype = getattr(self.od_config, "dtype", None) if self.od_config is not None else None
if isinstance(dtype, torch.dtype):
return dtype
return torch.float32 if device.type == "cpu" else torch.bfloat16
def _runtime_device_dtype(self) -> tuple[torch.device, torch.dtype]:
return self.device, self._native_dtype(self.device)
def _build_aux_components(self) -> None:
"""Construct the text encoder and VAE at startup, on CPU.
``ModuleDiscovery`` skips attributes that are ``None``, so components
created lazily on the first request are invisible to sequential
offload, HSDP auxiliary placement, and the shared device/memory
lifecycle. Building them here and deferring placement to the common
logic mirrors LTX-2's ``initialize_pipeline_components``.
"""
dtype = self._native_dtype(self.device)
self._load_stage1_text_encoder(dtype=dtype)
self._load_vae(dtype=dtype)
self._place_aux_components()
def _place_aux_components(self) -> None:
"""Move the discovered auxiliary components onto the local device.
Skipped when offload or HSDP owns placement — those paths expect the
components to still be on CPU when they take over.
"""
parallel_config = getattr(self.od_config, "parallel_config", None)
if (
getattr(self.od_config, "enable_cpu_offload", False)
or getattr(self.od_config, "enable_layerwise_offload", False)
or getattr(parallel_config, "use_hsdp", False)
):
return
modules = ModuleDiscovery.discover(self)
for module in (*modules.encoders, *modules.vaes, *modules.resident_modules):
module.to(self.device)
@staticmethod
def _component_local_files_only(model_id: Any) -> bool:
"""Decide download policy per component, from its own model id.
The Stage-1 release directory is local once ``resolve_checkpoint`` has
run, but the Gemma text encoder lives in a separate Hub repo. Deriving
locality from the release paths forced ``local_files_only=True`` onto
Gemma as well, which forbids the download on a box that has not cached
it yet.
"""
return Path(str(model_id)).expanduser().exists()
def _load_stage1_text_encoder(self, *, dtype: torch.dtype) -> None:
from transformers import AutoModel, AutoTokenizer
model_ids = [os.environ.get(SANA_WM_STAGE1_TEXT_ENCODER_ENV, "").strip()]
model_ids.extend([SANA_WM_STAGE1_TEXT_ENCODER_ID, SANA_WM_STAGE1_TEXT_ENCODER_FALLBACK_ID])
errors: list[str] = []
for model_id in dict.fromkeys(model_id for model_id in model_ids if model_id):
try:
self.tokenizer = AutoTokenizer.from_pretrained(
model_id,
local_files_only=self._component_local_files_only(model_id),
)
text_encoder_model_id = model_id
break
except OSError as exc:
errors.append(f"{model_id}: {exc}")
else:
raise OSError("Could not load SANA-WM Stage-1 text encoder. Tried: " + "; ".join(errors))
if getattr(self.tokenizer, "pad_token", None) is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
self.tokenizer.padding_side = "right"
with torch.device("cpu"):
# AutoModel, not AutoModelForCausalLM: only the final hidden state is
# used, so the LM head would compute a [seq, 256k] logits tensor
# that is thrown away on every prompt encode.
self.text_encoder = AutoModel.from_pretrained(
text_encoder_model_id,
torch_dtype=dtype,
local_files_only=self._component_local_files_only(text_encoder_model_id),
)
def _ensure_stage1_text_encoder(self, *, device: torch.device, dtype: torch.dtype) -> None:
if self.text_encoder is None:
self._load_stage1_text_encoder(dtype=dtype)
self.text_encoder.to(device=device, dtype=dtype)
def _load_vae(self, *, dtype: torch.dtype) -> None:
if self.release_paths is None and self.od_config is not None and self.od_config.model is not None:
self.resolve_checkpoint()
if self.release_paths is None:
raise ValueError("Sana-WM VAE loading requires a resolved checkpoint.")
from diffusers import AutoencoderKLLTX2Video
with torch.device("cpu"):
self.vae = AutoencoderKLLTX2Video.from_pretrained(
str(self.release_paths.root),
subfolder="vae",
torch_dtype=dtype,
local_files_only=self._component_local_files_only(self.release_paths.root),
)
self.vae.use_framewise_encoding = True
self.vae.use_framewise_decoding = True
# NVlabs reads these from its YAML VAE config with 64/96 defaults; the
# diffusers VAE config.json only ships scaling_factor, so the defaults
# are load-bearing (the keys are genuinely absent here).
self.vae.tile_sample_stride_num_frames = int(getattr(self.vae.config, "tile_sample_stride_num_frames", 64))
self.vae.tile_sample_min_num_frames = int(getattr(self.vae.config, "tile_sample_min_num_frames", 96))
def _ensure_vae(self, *, device: torch.device, dtype: torch.dtype) -> None:
if self.vae is None:
self._load_vae(dtype=dtype)
# Tiling is re-asserted here, not in _load_vae: registry.py assigns
# ``model.vae.use_tiling = od_config.vae_use_tiling`` after the pipeline
# is constructed, and that flag defaults to False. Matching NVlabs'
# inference_video_scripts/inference_sana_wm.py, the SANA VAE has to
# tile — a 161-frame decode costs about 9 GiB more without it, and a
# 321-frame request (41 latent frames) decodes as one 3D volume and
# OOMs. Overriding is deliberate, but say so rather than dropping the
# caller's setting silently.
if not self.vae.use_tiling:
logger.warning("Sana-WM forces VAE tiling on; ignoring vae_use_tiling=False (the VAE OOMs without it).")
self.vae.use_tiling = True
self.vae.to(device=device, dtype=dtype)
@staticmethod
def _preprocess_first_frame(
image: Any,
*,
height: int,
width: int,
device: torch.device,
dtype: torch.dtype,
) -> torch.Tensor:
"""Return the first-frame image as ``[1, 3, 1, H, W]`` in ``[-1, 1]``.
``VideoProcessor.preprocess`` covers the PIL / ndarray / tensor cases.
``resample="bicubic"`` keeps the filter PIL's ``Image.resize`` defaulted
to; the processor's own default is lanczos, which would move the
conditioning latent.
"""
import numpy as np
from diffusers.video_processor import VideoProcessor
# The processor normalises to [-1, 1] assuming float inputs in [0, 1].
if isinstance(image, np.ndarray) and image.dtype == np.uint8:
image = image.astype(np.float32) / 255.0
processor = VideoProcessor(vae_scale_factor=SANA_WM_VAE_SPATIAL_COMPRESSION, resample="bicubic")
frame = processor.preprocess(image, height=height, width=width)
return frame.unsqueeze(2).to(device=device, dtype=dtype)
def _vae_normalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
"""LTX-2 VAE per-channel latent normalisation matching NVlabs.
``z_norm = (z_raw - latents_mean) * scaling_factor / latents_std``
The LTX-2 VAE (``AutoencoderKLLTX2Video`` in diffusers) ships with
per-channel ``latents_mean`` and ``latents_std`` tensors that must be
applied alongside ``scaling_factor`` for the round-trip to be
identity. Applying `* scaling_factor` alone gives decoded videos a
systematic G/B colour shift (pred RGB means
~(110, 220, 208) vs ref ~(94, 114, 138)).
Returns the normalised latent in the same dtype as the input.
"""
if self.vae is None:
raise RuntimeError("Sana-WM VAE did not initialize.")
return normalize_latents(
latent,
self.vae.latents_mean,
self.vae.latents_std,
float(self.vae.config.scaling_factor),
)
def _vae_denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
"""Inverse of :meth:`_vae_normalize_latent` — match NVlabs decode."""
if self.vae is None:
raise RuntimeError("Sana-WM VAE did not initialize.")
return denormalize_latents(
latent,
self.vae.latents_mean,
self.vae.latents_std,
float(self.vae.config.scaling_factor),
)
def _vae_encode_first_frame(
self,
image: Any,
*,
height: int,
width: int,
latent_height: int,
latent_width: int,
device: torch.device,
dtype: torch.dtype,
) -> torch.Tensor:
"""Encode the first-frame image to latent space via the LTX-2 VAE.
Returns shape ``[1, 128, 1, latent_height, latent_width]``.
"""
self._ensure_vae(device=device, dtype=dtype)
if self.vae is None:
raise RuntimeError("Sana-WM VAE did not initialize.")
frame = self._preprocess_first_frame(image, height=height, width=width, device=device, dtype=dtype)
with torch.inference_mode():
encoded = self.vae.encode(frame)
first_latent = encoded.latent_dist.mean
first_latent = self._vae_normalize_latent(first_latent.to(dtype=dtype))
# The encoder's own rounding can land a pixel off the latent grid the
# noise tensor was built on, and the two are concatenated below.
if first_latent.shape[-2:] != (latent_height, latent_width):
first_latent = F.interpolate(
first_latent.squeeze(2),
size=(latent_height, latent_width),
mode="bilinear",
align_corners=False,
).unsqueeze(2)
return first_latent # [1, 128, 1, lh, lw]
def _stage1_prompt_text(self, prompt: dict[str, Any]) -> str:
user_prompt = str(prompt.get("prompt") or "")
chi_prompt = "\n".join(part for part in self.sana_wm_config.chi_prompt if part)
if chi_prompt:
return chi_prompt + user_prompt
return user_prompt
def _native_prompt_embeds(
self,
prompt: dict[str, Any],
*,
device: torch.device,
dtype: torch.dtype,
) -> torch.Tensor:
prompt_text = self._stage1_prompt_text(prompt)
self._ensure_stage1_text_encoder(device=device, dtype=dtype)
tokenizer = self.tokenizer
if tokenizer is None or self.text_encoder is None:
raise RuntimeError("Sana-WM Stage-1 text encoder did not initialize.")
chi_prompt = "\n".join(part for part in self.sana_wm_config.chi_prompt if part)
model_max_length = int(self.sana_wm_config.model_max_length)
max_length_all = model_max_length
if chi_prompt:
# NVlabs encodes the chi-prompt-prefixed text with extra room for
# the chi tokens, then keeps BOS plus the final model window.
max_length_all = len(tokenizer.encode(chi_prompt)) + model_max_length - 2
encoded = tokenizer(
[prompt_text],
padding="max_length",
max_length=max_length_all,
truncation=True,
add_special_tokens=True,
return_tensors="pt",
).to(device)
# last_hidden_state is the same tensor the CausalLM wrapper exposed as
# hidden_states[-1] (both are post-final-norm), without retaining every
# intermediate layer.
hidden_states = self.text_encoder(**encoded).last_hidden_state
attention_mask = encoded.attention_mask
if chi_prompt:
select_index = [0] + list(range(-model_max_length + 1, 0))
hidden_states = hidden_states[:, select_index]
attention_mask = attention_mask[:, select_index]
if hidden_states.shape[-1] != SANA_WM_STAGE1_PROMPT_CHANNELS:
raise ValueError(
"Sana-WM Stage-1 Gemma hidden size mismatch: expected "
f"{SANA_WM_STAGE1_PROMPT_CHANNELS}, got {hidden_states.shape[-1]}."
)
# Return RAW Gemma hidden states: the transformer's ``attention_y_norm``
# RMSNorm does the normalisation, so normalising here double-normalises.
prompt_attention_mask = attention_mask.to(device=device, dtype=torch.float32)
# An all-ones mask carries no information. Keep it ``None`` so
# cross-attention takes the mask-free attention path instead of handing
# every block, on every step, a mask the backend must reshape to no
# effect. Evaluated once per prompt encode, so the device sync is paid
# twice per request (cond + uncond), not once per denoising step.
if bool(prompt_attention_mask.all()):
prompt_attention_mask = None
self._last_prompt_attention_mask = prompt_attention_mask
return hidden_states.to(device=device, dtype=dtype)
def predict_noise(self, **kwargs: Any) -> torch.Tensor:
"""Single transformer forward.
Overrides the mixin default, which assumes a tuple-returning transformer
and takes ``result[0]``; SANA-WM's returns the noise prediction directly.
"""
return self.transformer(**kwargs)
def _decode_native_latents(
self,
latents: torch.Tensor,
*,
output_type: str,
device: torch.device,
dtype: torch.dtype,
) -> Any:
if output_type == "latent":
return latents
self._ensure_vae(device=device, dtype=dtype)
if self.vae is None:
raise RuntimeError("Sana-WM VAE did not initialize.")
# NVlabs always passes ``temb=None`` to ``vae.decode`` for this VAE
# (``builder.py::vae_decode -> LTX2VAE_diffusers``); match that.
denorm = self._vae_denormalize_latent(latents.to(getattr(self.vae, "dtype", dtype)))
video = self.vae.decode(denorm, temb=None, return_dict=False)[0]
# Post-process to the requested output_type (np/pil/pt). Let failures
# surface: silently returning the raw decoder tensor would hand the
# caller a wrong-format/range video with no error.
from diffusers.video_processor import VideoProcessor
processor = VideoProcessor(vae_scale_factor=getattr(self.vae, "spatial_compression_ratio", 32))
return processor.postprocess_video(video, output_type=output_type)
def _run_native_backend(
self,
*,
prompt: dict[str, Any],
payload: dict[str, Any],
sampling_params: Any | None,
) -> DiffusionOutput:
params = self._native_params(payload, sampling_params)
latent_frames, latent_height, latent_width = resolve_video_latent_shape(
params.height,
params.width,
params.num_frames,
vae_spatial_compression_ratio=SANA_WM_VAE_SPATIAL_COMPRESSION,
vae_temporal_compression_ratio=SANA_WM_VAE_TEMPORAL_COMPRESSION,
)
token_count = latent_frames * latent_height * latent_width
extra_args = self._extra_args(sampling_params)
max_tokens = int(extra_args.get("sana_wm_native_max_tokens", SANA_WM_NATIVE_MAX_TOKENS))
if token_count > max_tokens:
raise ValueError(
"Sana-WM native latent token count exceeds the configured cap. "
f"Requested latent tokens={token_count}, max={max_tokens}. Request a smaller "
"`height`/`width`/`num_frames`, or raise the cap via "
"`sana_wm_native_max_tokens`."
)
device, dtype = self._runtime_device_dtype()
generator = torch.Generator(device=device)
generator.manual_seed(params.seed)
noise = torch.randn(
(1, 128, latent_frames, latent_height, latent_width),
device=device,
dtype=dtype,
generator=generator,
)
# Shared flow-matching Euler scheduler. NVlabs ``LTXFlowEuler`` drives
# diffusers' ``FlowMatchEulerDiscreteScheduler``; this is vLLM-Omni's
# vendored copy of it, including the ``per_token_timesteps`` step branch.
# The schedule applies the shift twice (once via the shifted ``sigma_min``
# persisted by ``__init__``, once in ``set_timesteps``), which is what
# reproduces the NVlabs timestep table (e.g. shift=9.8, N=3 ->
# [1000, 909.0, 87.7]).
scheduler = FlowMatchEulerDiscreteScheduler(
num_train_timesteps=SANA_WM_NUM_TRAIN_TIMESTEPS,
shift=self.sana_wm_config.inference_flow_shift,
)
scheduler.set_timesteps(params.num_inference_steps, device=device)
timesteps = scheduler.timesteps
# ``normalize_sana_wm_payload`` guarantees the key is present but says
# nothing about its type. Falling back to pure noise on an unusable
# image would silently drop the conditioning and produce a
# plausible-looking but wrong video, so this raises instead.
import numpy as _np
first_frame_image = (prompt.get("multi_modal_data") or {}).get("image")
if not (hasattr(first_frame_image, "convert") or isinstance(first_frame_image, (_np.ndarray, torch.Tensor))):
raise TypeError(
"Sana-WM first-frame image must be a PIL Image, numpy ndarray, or "
f"torch.Tensor; got {type(first_frame_image).__name__}."
)
# Per-frame timestep contract matching NVlabs ``LTXFlowEuler.sample``:
# frame 0 is the clean VAE-encoded conditioning latent and its timestep
# is held at 0 while the sampling loop drives the other frames from
# noise.
first_latent = self._vae_encode_first_frame(
first_frame_image,
height=params.height,
width=params.width,
latent_height=latent_height,
latent_width=latent_width,
device=device,
dtype=dtype,
)
# Place the CLEAN VAE-encoded first frame at frame 0 and rely on
# per-frame timesteps + the `condition_mask` torch.where restore (below)
# to keep it invariant across denoising steps.
latents = torch.cat([first_latent, noise[:, :, 1:]], dim=2)
prompt_embeds = self._native_prompt_embeds(
prompt,
device=device,
dtype=dtype,
)
prompt_attention_mask = self._last_prompt_attention_mask
# Under CFG, encode the negative prompt up front; each denoise step
# then runs a two-branch transformer forward. The negative prompt comes
# from the request (``prompt["negative_prompt"]``, set by
# ``serving_video``); absent, it defaults to the empty string matching
# NVlabs ``GenerationParams.negative_prompt = ""``.
do_cfg = params.cfg_scale > 1.0
if do_cfg:
negative_prompt_obj = {"prompt": str(prompt.get("negative_prompt") or "")}
negative_prompt_embeds = self._native_prompt_embeds(
negative_prompt_obj,
device=device,
dtype=dtype,
)
negative_prompt_attention_mask = self._last_prompt_attention_mask
# Restore the cond mask so later readers of
# `_last_prompt_attention_mask` see the positive branch.
self._last_prompt_attention_mask = prompt_attention_mask
else:
negative_prompt_embeds = None
negative_prompt_attention_mask = None
camera = payload.get("camera") or {}
condition = SanaWmCameraCondition(
poses=camera.get("poses") if isinstance(camera, dict) else None,
intrinsics=payload.get("intrinsics"),
action=payload.get("action"),
num_frames=params.num_frames,
height=params.height,
width=params.width,
translation_speed=float(payload.get("translation_speed", 0.05)),
rotation_speed_deg=float(payload.get("rotation_speed_deg", 1.2)),
)
camera_tensors = build_plucker_condition(condition)
plucker = camera_tensors["chunk_plucker"].to(device=device, dtype=dtype)
raymap = camera_tensors["raymap"].to(device=device, dtype=dtype)
spatial_raymap = camera_tensors.get("spatial_raymap")
if spatial_raymap is not None:
spatial_raymap = spatial_raymap.to(device=device, dtype=dtype)
# Build a per-frame condition mask matching NVlabs ``LTXFlowEuler``:
# frame 0 is the conditioning frame at sigma=0 (preserved after
# each ``scheduler.step``), all other frames carry the current
# sampling sigma. We omit the legacy `add_noise_to_image_conditioning_latents`
# motion-continuity term because the public SANA-WM config sets
# ``condition_frame_info={0: 0.0}`` (image_cond_noise_scale=0).
cam_batch = latents.shape[0]
cam_frames = latents.shape[2]
condition_mask = torch.zeros(cam_batch, 1, cam_frames, 1, 1, device=latents.device, dtype=latents.dtype)
condition_mask[:, :, 0] = 1.0
for timestep in timesteps:
# (B, 1, F) per-frame timestep, frame 0 forced to 0. Stays fp32:
# casting through the bf16 latent dtype would quantise it before
# ``SanaWmTimestepEmbedder``'s sinusoidal embed.
model_timestep = timestep.float().expand(cam_batch, 1, cam_frames).clone()
model_timestep[:, :, 0] = 0.0
positive_kwargs = {
"hidden_states": latents,
"timestep": model_timestep,
"encoder_hidden_states": prompt_embeds,
"encoder_attention_mask": prompt_attention_mask,
"plucker": plucker,
"raymap": raymap,
"spatial_raymap": spatial_raymap,
}
negative_kwargs = (
{
**positive_kwargs,
"encoder_hidden_states": negative_prompt_embeds,
"encoder_attention_mask": negative_prompt_attention_mask,
}
if do_cfg
else None
)
# ``cfg_normalize=False`` keeps the plain NVlabs combine,
# ``uncond + scale * (cond - uncond)``. Under cfg_parallel_size > 1
# the mixin runs one branch per rank and all-gathers instead of
# evaluating both branches serially.
noise_pred = self.predict_noise_maybe_with_cfg(
do_cfg,
params.cfg_scale,
positive_kwargs,
negative_kwargs,
cfg_normalize=False,
)
# The per-frame contract pairs with a per-token flow-matching Euler
# step. Both step branches end at ``prev = sample + dt *
# model_output``, but the scheduler defines dt as
# ``sigma - sigma_next`` per-token against ``sigma_next - sigma``
# standard, so the per-token call takes the sign-flipped prediction
# (matching NVlabs ``LTXFlowEuler.sample``); passing the standard
# sign here would integrate towards more noise.
#
# Broadcast the (B, 1, F) timestep to (B, 1, F, H, W) and flatten
# F*H*W; frame-0 conditioning tokens are already 0. The flatten
# order (F, H, W) matches ``pack_latents``.
pt_t = (
model_timestep.unsqueeze(-1)
.unsqueeze(-1)
.expand(cam_batch, 1, cam_frames, latents.shape[3], latents.shape[4])
.reshape(cam_batch, -1)
)
# The per-token branch consumes/returns the packed (B, N, C) layout
# and keeps its result in fp32; cast back so the latent dtype stays
# stable across steps.
stepped_packed = scheduler.step(
-pack_latents(noise_pred),
timestep,
pack_latents(latents),
per_token_timesteps=pt_t,
return_dict=False,
)[0]
stepped = unpack_latents(
stepped_packed.to(latents.dtype),
cam_frames,
latents.shape[3],
latents.shape[4],
)
# Match NVlabs LTXFlowEuler exactly. This is stricter than a plain
# condition-frame restore: with bf16 latents, the `t=1000`
# comparison rounds `1 - 1e-6` to `1`, so the first full-noise step
# is discarded for generated tokens as well.
tokens_to_denoise_mask = timestep / float(SANA_WM_NUM_TRAIN_TIMESTEPS) - 1e-6 < (1.0 - condition_mask)
latents = torch.where(tokens_to_denoise_mask, stepped, latents)
output_type = getattr(sampling_params, "output_type", None) or "np"
output = self._decode_native_latents(latents, output_type=output_type, device=device, dtype=dtype)
return DiffusionOutput(
output=build_sana_wm_output_envelope(
output=output,
output_type=output_type,
metadata={
"backend": "native_gdn",
"output_space": output_type,
"chi_prompt_applied": bool(self.sana_wm_config.chi_prompt),
"num_frames": params.num_frames,
"height": params.height,
"width": params.width,
"latent_tokens": token_count,
"sampling_steps": params.num_inference_steps,
},
),
)
def forward(self, req: DiffusionRequestBatch, *args: Any, **kwargs: Any) -> DiffusionOutput:
del args, kwargs
if len(req.prompts) != 1:
raise ValueError("Sana-WM native backend currently supports exactly one prompt per request.")
prompt = req.prompts[0]
if isinstance(prompt, str):
raise ValueError("Sana-WM requires a mapping prompt with first-frame image and camera/action metadata.")
# get_sana_wm_pre_process_func is the single canonical entry point: it
# normalises the request and republishes the geometry onto
# sampling_params. normalize_sana_wm_payload is idempotent, so this only
# covers offline callers that bypass the preprocess hook.
prompt = normalize_sana_wm_payload(prompt)
payload = prompt["additional_information"]["sana_wm"]
# The checkpoint is resolved once in __init__; this is a cached no-op
# kept as a guard for pipelines constructed without od_config.model.
if self.od_config is not None and self.od_config.model is not None:
self.resolve_checkpoint()
return self._run_native_backend(
prompt=prompt,
payload=payload,
sampling_params=req.sampling_params,
)
def load_weights(self, weights: Iterable[tuple[str, Any]]) -> set[str]:
# Keys arrive pipeline-prefixed (``transformer.*``, per the component
# source). AutoWeightsLoader streams them to the submodule and returns
# the prefixed names the loader's strict-coverage check compares.
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -0,0 +1,269 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request normalization for Sana-WM image-to-video.
The released SANA-WM checkpoint is first-frame image-to-video. This module keeps
request-side normalization lightweight: it validates the image/camera/action
contract and stores a canonical payload under
``additional_information["sana_wm"]``. Model-internal raymap / Plucker
projection lives with the transformer.
This lives beside the model rather than under ``stage_input_processors``
because nothing loads it as one: the pipeline's ``pre_process_func`` is
registered from ``pipeline_sana_wm``, the same as every other diffusion model,
and no deploy config wires a ``custom_process_input_func`` here. Keeping it in
the stage package forced the pipeline to import *upwards* — the only such
import in ``diffusion/models/`` — which in turn made the VAE compression
constants impossible to share and so duplicated.
"""
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
from typing import Any
from vllm_omni.diffusion.models.sana_wm.camera_control import action_rollout_num_frames
from vllm_omni.diffusion.models.sana_wm.config import (
SANA_WM_VAE_SPATIAL_COMPRESSION,
SANA_WM_VAE_TEMPORAL_COMPRESSION,
)
SANA_WM_CANONICAL_KEY = "sana_wm"
# Request fallbacks used whenever the field is absent, mirroring the deploy YAML
# (161 frames) and the model's native resolution. Both are genuinely reachable:
# a served request that omits num_frames never gets one injected, and callers
# may ask for a different resolution.
SANA_WM_DEFAULT_NUM_FRAMES = 161
SANA_WM_DEFAULT_HEIGHT = 704
SANA_WM_DEFAULT_WIDTH = 1280
SANA_WM_DEFAULT_CAMERA_FORMAT = "c2w_4x4"
SANA_WM_DEFAULT_COORDINATE_SYSTEM = "official"
def _unwrap_single(value: Any) -> Any:
if isinstance(value, list):
return value[0] if value else None
return value
def _as_dict(value: Any, *, name: str) -> dict[str, Any]:
if value is None:
return {}
if not isinstance(value, Mapping):
raise ValueError(f"Sana-WM {name} must be a mapping, got {type(value).__name__}.")
return dict(value)
def _is_sequence(value: Any) -> bool:
return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
def _as_finite_float(value: Any, *, name: str) -> float:
"""Parse a float and reject NaN/inf.
Non-finite geometry does not fail loudly downstream: it propagates through
the raymap into the Plucker maps and comes back as an all-NaN video with a
200 response.
"""
try:
parsed = float(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"Sana-WM {name} must be a number, got {value!r}.") from exc
if not math.isfinite(parsed):
raise ValueError(f"Sana-WM {name} must be finite, got {parsed!r}.")
return parsed
def _validate_numeric_matrix4x4(matrix: Any, *, name: str) -> None:
if not _is_sequence(matrix) or len(matrix) != 4:
raise ValueError(f"Sana-WM {name} must be a 4x4 matrix.")
for row_idx, row in enumerate(matrix):
if not _is_sequence(row) or len(row) != 4:
raise ValueError(f"Sana-WM {name}[{row_idx}] must contain 4 values.")
for col_idx, value in enumerate(row):
_as_finite_float(value, name=f"{name}[{row_idx}][{col_idx}]")
def _validate_camera_poses(poses: Any, *, num_frames: int | None = None) -> None:
if not _is_sequence(poses) or len(poses) == 0:
raise ValueError("Sana-WM camera poses must be a non-empty sequence of 4x4 matrices.")
if num_frames is not None and len(poses) != num_frames:
raise ValueError(f"Sana-WM camera poses length {len(poses)} must equal num_frames {num_frames}.")
for idx, pose in enumerate(poses):
_validate_numeric_matrix4x4(pose, name=f"camera.poses[{idx}]")
def _validate_intrinsics(intrinsics: Any) -> None:
# Only the ``{fx, fy, cx, cy}`` mapping is accepted (the one form the recipe
# and examples use). Camera intrinsics are optional; when omitted the model
# derives them from the output resolution.
if intrinsics is None:
return
if not isinstance(intrinsics, Mapping):
raise ValueError("Sana-WM intrinsics must be a {fx, fy, cx, cy} mapping.")
required = {"fx", "fy", "cx", "cy"}
missing = sorted(required - set(intrinsics))
if missing:
raise ValueError(f"Sana-WM intrinsics mapping is missing keys: {missing}.")
for key in required:
value = _as_finite_float(intrinsics[key], name=f"intrinsics[{key!r}]")
# ``compute_raymap`` divides by fx/fy. cx/cy stay unconstrained beyond
# finiteness — a principal point outside the frame is unusual, not wrong.
if key in ("fx", "fy") and value <= 0:
raise ValueError(f"Sana-WM intrinsics[{key!r}] must be positive, got {value}.")
def _as_positive_int(value: Any, *, name: str) -> int:
try:
parsed = int(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"Sana-WM {name} must be an integer, got {value!r}.") from exc
if parsed <= 0:
raise ValueError(f"Sana-WM {name} must be positive, got {parsed}.")
return parsed
def _as_positive_float(value: Any, *, name: str) -> float:
# Finiteness first: every comparison against NaN is False, so a bare
# ``parsed <= 0`` would accept it.
parsed = _as_finite_float(value, name=name)
if parsed <= 0:
raise ValueError(f"Sana-WM {name} must be positive, got {parsed}.")
return parsed
def _extract_image(mm_data: Mapping[str, Any]) -> Any:
return _unwrap_single(mm_data.get("image"))
def _first_present(*values: Any) -> Any:
for value in values:
if value is not None:
return value
return None
def _validate_latent_geometry(*, num_frames: int, height: int, width: int) -> None:
"""Reject geometry the VAE would silently floor.
The SANA-WM VAE compresses 32x spatially and 8x temporally, so a request
that is not aligned loses pixels/frames instead of failing.
"""
if height % SANA_WM_VAE_SPATIAL_COMPRESSION or width % SANA_WM_VAE_SPATIAL_COMPRESSION:
raise ValueError(
f"Sana-WM height and width must be divisible by {SANA_WM_VAE_SPATIAL_COMPRESSION}, "
f"got {height} and {width}."
)
if (num_frames - 1) % SANA_WM_VAE_TEMPORAL_COMPRESSION:
raise ValueError(
f"Sana-WM num_frames must satisfy (num_frames - 1) % {SANA_WM_VAE_TEMPORAL_COMPRESSION} == 0, "
f"got {num_frames}."
)
def normalize_sana_wm_payload(prompt: Mapping[str, Any]) -> dict[str, Any]:
"""Return a prompt copy with canonical Sana-WM request metadata.
The ``sana_wm`` block is read from the top-level ``sana_wm`` key, falling
back to ``additional_information["sana_wm"]`` so the function is idempotent
when called again on its own output. The first-frame image is read from
``multi_modal_data["image"]``.
"""
if not isinstance(prompt, Mapping):
raise ValueError(f"Sana-WM prompt must be a mapping, got {type(prompt).__name__}.")
result = dict(prompt)
mm_data = _as_dict(result.get("multi_modal_data"), name="multi_modal_data")
image = _extract_image(mm_data)
if image is None:
raise ValueError("Sana-WM requires a first-frame image in `multi_modal_data['image']`.")
additional = _as_dict(result.get("additional_information"), name="additional_information")
raw = _first_present(
result.get(SANA_WM_CANONICAL_KEY),
additional.get(SANA_WM_CANONICAL_KEY),
{},
)
raw = _as_dict(raw, name=SANA_WM_CANONICAL_KEY)
num_frames = _as_positive_int(
_first_present(raw.get("num_frames"), result.get("num_frames"), SANA_WM_DEFAULT_NUM_FRAMES),
name="num_frames",
)
height = _as_positive_int(
_first_present(raw.get("height"), result.get("height"), SANA_WM_DEFAULT_HEIGHT),
name="height",
)
width = _as_positive_int(_first_present(raw.get("width"), result.get("width"), SANA_WM_DEFAULT_WIDTH), name="width")
_validate_latent_geometry(num_frames=num_frames, height=height, width=width)
camera = raw.get("camera")
camera = _as_dict(camera, name="camera") if camera is not None else None
action = raw.get("action")
if action is not None:
if not isinstance(action, str) or not action.strip():
raise ValueError("Sana-WM action must be a non-empty string when provided.")
# Rejected here rather than in the model: a mismatch used to shorten the
# camera tensor silently and then fail deep in the transformer on a
# camera/latent frame mismatch, and an unbounded duration used to be
# expanded one entry per frame before any of that.
rollout_frames = action_rollout_num_frames(action)
if rollout_frames != num_frames:
raise ValueError(
f"Sana-WM action rolls out {rollout_frames} frames but num_frames is {num_frames}. "
"Action segment durations must sum to num_frames - 1."
)
if camera is not None and action is not None:
raise ValueError("Sana-WM accepts exactly one of `camera` or `action`, not both.")
if camera is None and action is None:
raise ValueError("Sana-WM requires either explicit camera poses or an action DSL string.")
canonical_camera = None
if camera is not None:
poses = camera.get("poses")
if poses is None:
raise ValueError("Sana-WM camera payload requires `poses`.")
_validate_camera_poses(poses, num_frames=num_frames)
# Only the defaults are accepted: nothing downstream reads these two
# fields, so honouring another value would mean interpreting w2c poses
# as c2w and silently generating the wrong trajectory.
camera_format = camera.get("format", SANA_WM_DEFAULT_CAMERA_FORMAT)
if camera_format != SANA_WM_DEFAULT_CAMERA_FORMAT:
raise ValueError(
f"Sana-WM only accepts camera format {SANA_WM_DEFAULT_CAMERA_FORMAT!r}; "
f"convert the poses before sending, got {camera_format!r}."
)
coordinate_system = camera.get("coordinate_system", SANA_WM_DEFAULT_COORDINATE_SYSTEM)
if coordinate_system != SANA_WM_DEFAULT_COORDINATE_SYSTEM:
raise ValueError(
f"Sana-WM only accepts coordinate system {SANA_WM_DEFAULT_COORDINATE_SYSTEM!r}, "
f"got {coordinate_system!r}."
)
canonical_camera = {"poses": poses}
intrinsics = raw.get("intrinsics")
_validate_intrinsics(intrinsics)
translation_speed = raw.get("translation_speed")
rotation_speed_deg = raw.get("rotation_speed_deg")
canonical = {
"num_frames": num_frames,
"height": height,
"width": width,
"camera": canonical_camera,
"action": action,
"intrinsics": intrinsics,
}
if translation_speed is not None:
canonical["translation_speed"] = _as_positive_float(translation_speed, name="translation_speed")
if rotation_speed_deg is not None:
canonical["rotation_speed_deg"] = _as_positive_float(rotation_speed_deg, name="rotation_speed_deg")
mm_data["image"] = image
result["multi_modal_data"] = mm_data
additional[SANA_WM_CANONICAL_KEY] = canonical
result["additional_information"] = additional
result.pop(SANA_WM_CANONICAL_KEY, None)
return result
File diff suppressed because it is too large. Load diff
+536
View File
@@ -0,0 +1,536 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SANA-WM UCPE (Unified Camera Pose Embedding) per-block attention transforms.
Ported from NVlabs/Sana ``sana_camctrl_blocks.py`` for the inference-only
SANA-WM bidirectional 1600M release. Scope is intentionally narrow:
* Pinhole camera only (xi=0); the UCM ``xi`` parameter is fixed.
* Inference path only — no training-time camera-branch dropout.
* Online computation only — the precomputed ``cam_pos_embeds`` shortcut
used by NVlabs' fused kernels is omitted because vLLM-Omni builds the
matrices once per transformer forward and shares them across all blocks.
* ``apply_vo=True`` only — the SANA-WM checkpoint always uses the
inverse-output transform.
Public surface:
``prepare_cam_geometry(...)`` -> :class:`SanaWmCamGeometry`, built once per
transformer forward and threaded through every block. It carries both the
``(apply_q, apply_kv, apply_output)`` closures used by the softmax camera
branch and the projection/RoPE tensors used by ``cam_prep_func(...)``, the
NVlabs fused camera-prep contract on the GDN camera branch.
Each closure transforms a tensor of shape ``(B, num_heads, N, D)`` where
``N = T * H * W`` (matching the GDN token layout) and ``D == head_dim``,
returning a tensor of the same shape.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from functools import partial
import torch
import torch.nn.functional as F
# ---------------------------------------------------------------------------
# Pinhole ray-grid construction
# ---------------------------------------------------------------------------
def _make_pixel_grid(
height: int,
width: int,
device: torch.device,
dtype: torch.dtype,
) -> torch.Tensor:
"""Return ``(H, W, 3)`` grid ``(x, y, 1)`` of pixel coordinates."""
xs = torch.arange(width, device=device, dtype=dtype)
ys = torch.arange(height, device=device, dtype=dtype)
y_grid, x_grid = torch.meshgrid(ys, xs, indexing="ij")
return torch.stack((x_grid, y_grid, torch.ones_like(x_grid)), dim=-1)
def _unproject_pinhole(
fx: torch.Tensor,
fy: torch.Tensor,
cx: torch.Tensor,
cy: torch.Tensor,
height: int,
width: int,
) -> torch.Tensor:
"""Unproject a pixel grid into camera-frame direction vectors.
Args:
fx, fy, cx, cy: shape ``(B, T)`` intrinsics in token-grid units — the
caller divides all four by patch size before calling.
height, width: token-grid size.
Returns:
``(B, T, H, W, 3)`` direction tensor (NOT normalised).
"""
B, T = fx.shape
device = fx.device
dtype = fx.dtype
grid = _make_pixel_grid(height, width, device, dtype) # (H, W, 3)
u = grid[..., 0] # (H, W)
v = grid[..., 1]
fx_e = fx.view(B, T, 1, 1)
fy_e = fy.view(B, T, 1, 1)
cx_e = cx.view(B, T, 1, 1)
cy_e = cy.view(B, T, 1, 1)
x = (u - cx_e) / fx_e
y = (v - cy_e) / fy_e
z = torch.ones_like(x)
# Match NVlabs UCM normalisation at xi=0: alpha = 1, gamma = 1/(1+r^2),
# X = gamma*x, Y = gamma*y, Z = gamma. The resulting vector has unit
# norm. Computed compactly via direct stack + normalise.
d_cam = torch.stack((x, y, z), dim=-1) # (B, T, H, W, 3)
d_cam = d_cam / d_cam.norm(dim=-1, keepdim=True).clamp_min(1e-12)
return d_cam
def _world_to_ray_mats(
d_cam: torch.Tensor, # (B, T, H, W, 3) unit rays in camera frame
c2w: torch.Tensor, # (B, T, 4, 4) camera-to-world SE(3)
) -> torch.Tensor:
"""Build per-pixel ``ray <- world`` SE(3) transforms ``(B, T, H, W, 4, 4)``.
The ray frame is anchored at the camera origin with the +Z axis along
the per-pixel viewing direction and a +Y axis defined by the camera-Y
cross product. This matches NVlabs ``world_to_ray_mats``.
"""
B, T, H, W, _ = d_cam.shape
device = d_cam.device
dtype = d_cam.dtype
rot_cam = c2w[..., :3, :3]
trans_cam = c2w[..., :3, 3]
d_world = torch.einsum("btij,bthwj->bthwi", rot_cam, d_cam)
cam_y = rot_cam[..., :, 1].unsqueeze(2).unsqueeze(3).expand(B, T, H, W, 3)
z_ray = F.normalize(d_world, dim=-1, eps=1e-6)
x_ray = F.normalize(torch.cross(cam_y, z_ray, dim=-1), dim=-1, eps=1e-6)
y_ray = F.normalize(torch.cross(z_ray, x_ray, dim=-1), dim=-1, eps=1e-6)
rot_local_to_world = torch.stack((x_ray, y_ray, z_ray), dim=-1) # (B,T,H,W,3,3)
rot_world_to_local = rot_local_to_world.transpose(-1, -2)
t_world = trans_cam.unsqueeze(2).unsqueeze(3).expand(B, T, H, W, 3)
t_w2l = -torch.einsum("bthwij,bthwj->bthwi", rot_world_to_local, t_world)
raymats = torch.zeros(B, T, H, W, 4, 4, device=device, dtype=dtype)
raymats[..., :3, :3] = rot_world_to_local
raymats[..., :3, 3] = t_w2l
raymats[..., 3, 3] = 1.0
# Guard NaNs from degenerate pixels (e.g. nan direction at infinity).
nan_mask = torch.isnan(d_world).any(-1)
if nan_mask.any():
eye = torch.eye(4, device=device, dtype=dtype)
raymats[nan_mask] = eye
return raymats
def _process_camera_conditions(
camera_conditions: torch.Tensor, # (B, T, 20)
hw: tuple[int, int, int],
patch_size: tuple[int, int, int] = (1, 1, 1),
) -> torch.Tensor:
"""Convert ``(B, T, 20)`` SANA-WM camera conditions to ``(B, T, H, W, 4, 4)``
``ray <- world`` transforms.
Layout of the last axis is ``[c2w_flat (16), fx, fy, cx, cy]``.
Intrinsics follow the NVlabs UCPE contract: the packed camera payload is
expressed on the pre-patch latent image grid, while ``HW`` is the
transformer token grid. ``patch_size`` maps those units to the token grid
before unprojection. The released SANA-WM P1 config uses ``(1, 1, 1)``.
Args:
camera_conditions: ``(B, T, 20)`` from ``_pack_camera_conditions``.
hw: ``(T, H, W)`` token-grid shape. Equal to the latent grid only
because the released config patches by ``(1, 1, 1)``.
"""
B, T_cond, last = camera_conditions.shape
if last != 20:
raise ValueError(f"SANA-WM camera_conditions last dim must be 20, got {last}.")
T_latent, H_latent, W_latent = hw
if T_cond != T_latent:
raise ValueError(f"SANA-WM camera_conditions frames {T_cond} != latent frames {T_latent}.")
c2w = camera_conditions[..., :16].reshape(B, T_latent, 4, 4)
patch_t, patch_h, patch_w = patch_size
del patch_t
fx = camera_conditions[..., 16] / float(patch_w)
fy = camera_conditions[..., 17] / float(patch_h)
cx = camera_conditions[..., 18] / float(patch_w)
cy = camera_conditions[..., 19] / float(patch_h)
d_cam = _unproject_pinhole(fx, fy, cx, cy, H_latent, W_latent)
raymats = _world_to_ray_mats(d_cam, c2w)
return raymats
def _invert_se3(transforms: torch.Tensor) -> torch.Tensor:
"""Closed-form inverse of a stack of 4x4 SE(3) matrices."""
if transforms.shape[-2:] != (4, 4):
raise ValueError(f"SE3 inverse expects (..., 4, 4), got {tuple(transforms.shape)}.")
rot_inv = transforms[..., :3, :3].transpose(-1, -2)
out = torch.zeros_like(transforms)
out[..., :3, :3] = rot_inv
out[..., :3, 3] = -torch.einsum("...ij,...j->...i", rot_inv, transforms[..., :3, 3])
out[..., 3, 3] = 1.0
return out
# ---------------------------------------------------------------------------
# Per-token transform primitives
# ---------------------------------------------------------------------------
def _apply_ray_projmat(
feats: torch.Tensor, # (B, num_heads, N, D)
matrix: torch.Tensor, # (B, N, 4, 4)
) -> torch.Tensor:
"""Apply a per-token 4x4 projection to features grouped by 4 channels.
``D`` must be a multiple of 4. Out-of-place; preserves ``feats.shape``.
"""
B, num_heads, N, D = feats.shape
if D % 4 != 0:
raise ValueError(f"UCPE projmat expects D divisible by 4, got D={D}.")
feats_grouped = feats.reshape(B, num_heads, N, D // 4, 4)
# matrix: (B, N, 4, 4); feats_grouped: (B, num_heads, N, K, 4)
out = torch.einsum("bnij,bhnkj->bhnki", matrix, feats_grouped)
return out.reshape(B, num_heads, N, D)
def _apply_complex_rope(
hidden_states: torch.Tensor,
freqs: torch.Tensor,
*,
inverse: bool = False,
) -> torch.Tensor:
"""Apply complex RoPE: treat last-dim pairs as ``(real, imag)`` and multiply
by ``freqs``. Computation runs in fp64 to match the NVlabs reference.
Args:
hidden_states: ``(..., D)`` with ``D % 2 == 0``.
freqs: complex tensor broadcastable to ``(..., D // 2)``.
inverse: when ``True`` apply ``freqs.conj()`` (used for ``apply_o``).
"""
x_real = hidden_states.to(torch.float64).contiguous()
x_complex = torch.view_as_complex(x_real.unflatten(-1, (-1, 2)))
if inverse:
freqs = freqs.conj()
x_out = torch.view_as_real(x_complex * freqs).flatten(-2, -1)
return x_out.to(hidden_states.dtype)
def _apply_block_diagonal(
feats: torch.Tensor,
func_size_pairs: list[tuple[Callable[[torch.Tensor], torch.Tensor], int]],
) -> torch.Tensor:
"""Split ``feats`` along the last axis by ``block_sizes`` and apply one
callable per block, then concatenate. Preserves ``feats.shape``.
"""
funcs, block_sizes = zip(*func_size_pairs)
if feats.shape[-1] != sum(block_sizes):
raise ValueError(f"UCPE block-diagonal block sum {sum(block_sizes)} != feats D {feats.shape[-1]}.")
blocks = torch.split(feats, list(block_sizes), dim=-1)
return torch.cat([func(block) for func, block in zip(funcs, blocks)], dim=-1)
def _slice_rope_for_cam(
rotary_emb: torch.Tensor | None,
rope_dim: int,
) -> torch.Tensor | None:
"""Re-slice WAN-style RoPE frequencies for a smaller ``rope_dim`` using the
same ``(T, H, W)`` split as the main branch. Mirrors NVlabs slicing logic.
The T/H/W band layout is read off the input shape
(``rotary_emb.shape[-1] * 2``) rather than a separately supplied head_dim,
so the split stays correct without the caller passing it. ``cam_head_dim``
still has to equal the main ``head_dim`` — ``SanaWmSelfAttention.__init__``
rejects anything else.
"""
if rotary_emb is None:
return None
# Derive WAN RoPE T/H/W band offsets from the actual input tensor size,
# not from a head_dim argument. This is correct regardless of whether
# cam_head_dim == main_head_dim.
in_head_dim = rotary_emb.shape[-1] * 2
orig_t = in_head_dim // 2 - 2 * (in_head_dim // 6)
orig_h = in_head_dim // 6
new_t = rope_dim // 2 - 2 * (rope_dim // 6)
new_h = rope_dim // 6
new_w = rope_dim // 6
t_part = rotary_emb[..., :new_t]
h_part = rotary_emb[..., orig_t : orig_t + new_h]
w_part = rotary_emb[..., orig_t + orig_h : orig_t + orig_h + new_w]
return torch.cat([t_part, h_part, w_part], dim=-1)
# ---------------------------------------------------------------------------
# Apply-fn preparation
# ---------------------------------------------------------------------------
def _identity(x: torch.Tensor) -> torch.Tensor:
return x
def _prepare_ray_apply_fns(
head_dim: int,
proj: torch.Tensor, # (B, N, 4, 4) ray <- world
proj_t: torch.Tensor, # (B, N, 4, 4) ray <- world transpose
proj_inv: torch.Tensor, # (B, N, 4, 4) world <- ray
rotary_emb: torch.Tensor | None = None,
) -> tuple[Callable, Callable, Callable]:
"""Build ``(apply_q, apply_kv, apply_o)`` block-diagonal callables.
Each callable transforms a ``(B, num_heads, N, head_dim)`` tensor:
the first ``head_dim // 2`` channels go through the per-token 4x4
projection, the remaining ``head_dim // 2`` channels go through complex
RoPE (or identity when ``rotary_emb`` is None).
"""
if rotary_emb is not None:
rope_fwd = partial(_apply_complex_rope, freqs=rotary_emb, inverse=False)
rope_inv = partial(_apply_complex_rope, freqs=rotary_emb, inverse=True)
else:
rope_fwd = _identity
rope_inv = _identity
half = head_dim // 2
transforms_q = [
(partial(_apply_ray_projmat, matrix=proj_t), half),
(rope_fwd, half),
]
transforms_kv = [
(partial(_apply_ray_projmat, matrix=proj_inv), half),
(rope_fwd, half),
]
transforms_o = [
(partial(_apply_ray_projmat, matrix=proj), half),
(rope_inv, half),
]
apply_q = partial(_apply_block_diagonal, func_size_pairs=transforms_q)
apply_kv = partial(_apply_block_diagonal, func_size_pairs=transforms_kv)
apply_o = partial(_apply_block_diagonal, func_size_pairs=transforms_o)
return apply_q, apply_kv, apply_o
@dataclass(frozen=True)
class SanaWmCamGeometry:
"""UCPE geometry shared by every block of one transformer forward.
The ray grid, its SE(3) inverse and the camera RoPE tables depend only on
the camera payload, the latent grid, the patch size and the head dim — all
fixed for the whole forward — yet each block's camera branch used to
rebuild them. At 20 blocks x 60 steps x 2 CFG branches that is ~2400
rebuilds of an 18480-token ray grid and its 4x4 inverses per request.
``apply_q``/``apply_kv``/``apply_output`` serve the softmax camera branch;
``proj_q``/``proj_kv``/``rope_cos``/``rope_sin`` serve the fused
``cam_prep_func`` contract on the GDN branch. Both are derived from the
same projection matrices, so they are built together.
"""
proj_q: torch.Tensor
proj_kv: torch.Tensor
rope_cos: torch.Tensor
rope_sin: torch.Tensor
apply_q: Callable[[torch.Tensor], torch.Tensor]
apply_kv: Callable[[torch.Tensor], torch.Tensor]
apply_output: Callable[[torch.Tensor], torch.Tensor]
def _prepare_ucpe_rope_tables(
rotary_emb_cam: torch.Tensor,
token_count: int,
half_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert complex RoPE freqs to NVlabs' interleaved cos/sin tables.
The output convention is ``out[d] = x[d] * cos[d] + x[d ^ 1] * sin[d]``.
For every complex pair, ``sin[2i] = -imag(freq[i])`` and
``sin[2i + 1] = imag(freq[i])``.
"""
freqs = rotary_emb_cam
while freqs.ndim > 2 and freqs.shape[0] == 1:
freqs = freqs.squeeze(0)
if freqs.ndim != 2:
raise ValueError(f"Sana-WM cam RoPE freqs must be 2D after squeeze, got {tuple(freqs.shape)}.")
if freqs.shape[0] != token_count:
raise ValueError(f"Sana-WM cam RoPE token count {freqs.shape[0]} != {token_count}.")
if freqs.shape[1] * 2 != half_dim:
raise ValueError(f"Sana-WM cam RoPE dim {freqs.shape[1] * 2} != half_dim {half_dim}.")
cos_half = freqs.real.float()
sin_half = freqs.imag.float()
rope_cos = cos_half.repeat_interleave(2, dim=-1).contiguous()
rope_sin = torch.stack((-sin_half, sin_half), dim=-1).reshape(token_count, half_dim).contiguous()
return rope_cos, rope_sin
def prepare_cam_geometry(
*,
camera_conditions: torch.Tensor,
spatial_shape: tuple[int, int, int],
patch_size: tuple[int, int, int],
head_dim: int,
rotary_emb: torch.Tensor | None,
) -> SanaWmCamGeometry:
"""Build the per-forward UCPE geometry: ray projections, RoPE tables, apply fns.
Call once per transformer forward and share the result across every block —
see :class:`SanaWmCamGeometry` for why.
"""
if head_dim % 4 != 0:
raise ValueError(f"UCPE head_dim must be divisible by 4, got {head_dim}.")
if head_dim % 2 != 0 or (head_dim // 2) % 4 != 0:
raise ValueError(f"Sana-WM cam prep head_dim={head_dim} must satisfy D % 2 == 0 and (D/2) % 4 == 0.")
batch = camera_conditions.shape[0]
frames, height, width = spatial_shape
token_count = frames * height * width
half_dim = head_dim // 2
raymats = _process_camera_conditions(camera_conditions, spatial_shape, patch_size=patch_size)
proj = raymats.reshape(batch, token_count, 4, 4).contiguous()
proj_q = proj.transpose(-1, -2).contiguous()
proj_kv = _invert_se3(proj).contiguous()
rotary_emb_cam = _slice_rope_for_cam(rotary_emb, half_dim)
if rotary_emb_cam is None:
rope_cos = torch.ones(token_count, half_dim, device=camera_conditions.device, dtype=torch.float32)
rope_sin = torch.zeros(token_count, half_dim, device=camera_conditions.device, dtype=torch.float32)
else:
rope_cos, rope_sin = _prepare_ucpe_rope_tables(rotary_emb_cam, token_count, half_dim)
apply_q, apply_kv, apply_output = _prepare_ray_apply_fns(head_dim, proj, proj_q, proj_kv, rotary_emb=rotary_emb_cam)
return SanaWmCamGeometry(
proj_q=proj_q,
proj_kv=proj_kv,
rope_cos=rope_cos,
rope_sin=rope_sin,
apply_q=apply_q,
apply_kv=apply_kv,
apply_output=apply_output,
)
def _validate_cam_prep_inputs(
q_raw: torch.Tensor,
k_raw: torch.Tensor,
v_raw: torch.Tensor,
proj_q: torch.Tensor,
proj_kv: torch.Tensor,
rope_cos: torch.Tensor,
rope_sin: torch.Tensor,
) -> int:
"""Validate the fused cam-prep contract and return ``head_dim // 2``."""
if q_raw.shape != k_raw.shape or q_raw.shape != v_raw.shape:
raise ValueError(f"Sana-WM cam prep q/k/v shapes must match, got {q_raw.shape}, {k_raw.shape}, {v_raw.shape}.")
if q_raw.ndim != 4:
raise ValueError(f"Sana-WM cam prep q/k/v must be (B,N,H,D), got {tuple(q_raw.shape)}.")
batch, token_count, _num_heads, head_dim = q_raw.shape
if head_dim % 2 != 0 or (head_dim // 2) % 4 != 0:
raise ValueError(f"Sana-WM cam prep head_dim={head_dim} must satisfy D % 2 == 0 and (D/2) % 4 == 0.")
half_dim = head_dim // 2
if proj_q.shape != (batch, token_count, 4, 4):
raise ValueError(f"Sana-WM cam prep proj_q shape {tuple(proj_q.shape)} != {(batch, token_count, 4, 4)}.")
if proj_kv.shape != (batch, token_count, 4, 4):
raise ValueError(f"Sana-WM cam prep proj_kv shape {tuple(proj_kv.shape)} != {(batch, token_count, 4, 4)}.")
if rope_cos.shape != (token_count, half_dim) or rope_sin.shape != (token_count, half_dim):
raise ValueError(
"Sana-WM cam prep rope table shapes must be "
f"{(token_count, half_dim)}, got {tuple(rope_cos.shape)} and {tuple(rope_sin.shape)}."
)
return half_dim
def _apply_ray_projection(feats: torch.Tensor, matrix: torch.Tensor) -> torch.Tensor:
batch, token_count, num_heads, half_dim = feats.shape
grouped = feats.reshape(batch, token_count, num_heads, half_dim // 4, 4)
out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), grouped)
return out.reshape(batch, token_count, num_heads, half_dim)
def _apply_interleaved_rope(feats: torch.Tensor, rope_cos: torch.Tensor, rope_sin: torch.Tensor) -> torch.Tensor:
half_dim = feats.shape[-1]
pair_index = torch.arange(half_dim, device=feats.device) ^ 1
paired = feats.index_select(-1, pair_index)
return feats * rope_cos.reshape(1, rope_cos.shape[0], 1, half_dim) + paired * rope_sin.reshape(
1, rope_sin.shape[0], 1, half_dim
)
def cam_prep_func(
q_normed: torch.Tensor,
k_normed: torch.Tensor,
v_raw: torch.Tensor,
*,
proj_q: torch.Tensor,
proj_kv: torch.Tensor,
rope_cos: torch.Tensor,
rope_sin: torch.Tensor,
k_scale: float,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Native equivalent of NVlabs ``cam_prep_func``.
The caller is responsible for the q/k RMSNorm, which must run through the
model's norm modules so a tensor-parallel shard still normalises over the
global channel width. This function only applies ReLU, the K scale, the
UCPE ray projection and RoPE.
Args:
q_normed: ``(B, N, H, D)`` camera Q, already RMSNorm'd.
k_normed: ``(B, N, H, D)`` camera K, already RMSNorm'd. K must also
include the temporal short convolution.
v_raw: ``(B, N, H, D)`` raw camera V.
proj_q: ``(B, N, 4, 4)`` Q ray projection matrices.
proj_kv: ``(B, N, 4, 4)`` K/V ray projection matrices.
rope_cos: ``(N, D//2)`` interleaved-pair RoPE cosine table.
rope_sin: ``(N, D//2)`` interleaved-pair RoPE sine table.
k_scale: ``D^-0.5 * spatial_tokens^-0.5``.
Returns:
``q_trans``, ``k_trans``, ``v_trans`` in ``(B, H, D, N)`` layout, and
``inflation_sq`` in ``(B, H, N)``.
"""
half_dim = _validate_cam_prep_inputs(
q_normed,
k_normed,
v_raw,
proj_q,
proj_kv,
rope_cos,
rope_sin,
)
q_norm = torch.relu(q_normed.float())
k_norm = torch.relu(k_normed.float()) * float(k_scale)
value = v_raw.float()
q_half = _apply_ray_projection(q_norm[..., :half_dim], proj_q)
k_half = _apply_ray_projection(k_norm[..., :half_dim], proj_kv)
v_half = _apply_ray_projection(value[..., :half_dim], proj_kv)
q_rope = _apply_interleaved_rope(q_norm[..., half_dim:], rope_cos.float(), rope_sin.float())
k_rope = _apply_interleaved_rope(k_norm[..., half_dim:], rope_cos.float(), rope_sin.float())
v_rope = _apply_interleaved_rope(value[..., half_dim:], rope_cos.float(), rope_sin.float())
k_pre_sq = k_norm.square().sum(dim=-1).permute(0, 2, 1).contiguous()
k_post = torch.cat((k_half, k_rope), dim=-1)
k_post_sq = k_post.square().sum(dim=-1).permute(0, 2, 1).contiguous()
inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12)
def to_bhdn(tensor: torch.Tensor) -> torch.Tensor:
return tensor.permute(0, 2, 3, 1).contiguous().to(v_raw.dtype)
q_out = to_bhdn(torch.cat((q_half, q_rope), dim=-1))
k_out = to_bhdn(k_post)
v_out = to_bhdn(torch.cat((v_half, v_rope), dim=-1))
return q_out, k_out, v_out, inflation_sq
__all__ = [
"SanaWmCamGeometry",
"cam_prep_func",
"prepare_cam_geometry",
]