[Model] Add SANA-WM support(Stage-1 only) (#4061)

Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Signed-off-by: liuqihao <liuqihao970610@gmail.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
BruceLoveDecimaland汪志鹏 authored and GitHub committed 2026-08-18 23:57:55 -06:00
1 parent 14bd8f8772
commit adf6d0c1ea
21 files changed
+5173 -1

No files matched your search

+159
View File
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import os
from typing import Any
import numpy as np
import pytest
pytestmark = [
pytest.mark.advanced_model,
pytest.mark.diffusion,
pytest.mark.gpu,
]
def _make_sana_wm_e2e_image() -> Any:
from PIL import Image
from vllm_omni.diffusion.models.sana_wm import SANA_WM_OUTPUT_HEIGHT, SANA_WM_OUTPUT_WIDTH
return Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160))
def _sana_wm_e2e_tensor_parallel_size() -> int:
return int(
os.environ.get(
"SANA_WM_E2E_TENSOR_PARALLEL_SIZE",
os.environ.get("SANA_WM_E2E_TP", "1"),
)
)
def _coerce_video_array(video: Any) -> np.ndarray:
import torch
if isinstance(video, list):
assert video, "SANA-WM e2e produced an empty video list."
video = video[0]
if isinstance(video, torch.Tensor):
video = video.detach().cpu().float().numpy()
video_array = np.asarray(video)
if video_array.ndim == 5:
assert video_array.shape[0] == 1
video_array = video_array[0]
assert video_array.ndim == 4
return video_array
def _assert_sana_wm_e2e_shape(
video: np.ndarray,
*,
output_type: str,
num_frames: int,
) -> None:
if output_type == "latent":
# Latent output is channel-first: (C, T, H, W) after removing the
# batch dimension. Stage-1 emits LTX-2 latents with 128 channels.
assert video.shape[0] == 128
assert 0 < video.shape[1] <= num_frames
return
assert 0 < video.shape[0] <= num_frames
def _run_sana_wm_e2e(*, output_type: str) -> np.ndarray:
import torch
from vllm_omni.diffusion.models.sana_wm import (
SANA_WM_MODEL_ID,
SANA_WM_OUTPUT_HEIGHT,
SANA_WM_OUTPUT_WIDTH,
)
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
if not torch.cuda.is_available():
pytest.skip("Sana-WM e2e requires CUDA.")
model = os.environ.get("SANA_WM_E2E_MODEL", SANA_WM_MODEL_ID)
model_class_name = os.environ.get("SANA_WM_E2E_MODEL_CLASS", "SanaWmPipeline")
num_frames = int(os.environ.get("SANA_WM_E2E_NUM_FRAMES", "9"))
stage1_steps = int(
os.environ.get(
"SANA_WM_E2E_STAGE1_STEPS",
os.environ.get("SANA_WM_E2E_NUM_INFERENCE_STEPS", "1"),
)
)
action = os.environ.get("SANA_WM_E2E_ACTION", f"w-{max(num_frames - 1, 1)}")
tensor_parallel_size = _sana_wm_e2e_tensor_parallel_size()
image = _make_sana_wm_e2e_image()
omni = Omni(
model=model,
model_class_name=model_class_name,
enforce_eager=True,
tensor_parallel_size=tensor_parallel_size,
)
extra_args: dict[str, Any] = {}
# The native Stage-1 path is latent-token capped; raise it for full-size
# e2e runs (704x1280 exceeds the default 4096-token cap).
native_max_tokens = os.environ.get("SANA_WM_E2E_NATIVE_MAX_TOKENS", "").strip()
if native_max_tokens:
extra_args["sana_wm_native_max_tokens"] = int(native_max_tokens)
# Return the requested output type so the shape assertion matches the
# pipeline output instead of the raw Stage-1 latent default.
output = omni.generate(
{
"prompt": "A slow forward camera move through a quiet city street.",
"multi_modal_data": {"image": image},
"sana_wm": {
"action": action,
"num_frames": num_frames,
"translation_speed": 0.055,
"rotation_speed_deg": 1.2,
# Avoid the optional Pi3X dependency by passing deterministic
# camera intrinsics directly.
"intrinsics": {
"fx": SANA_WM_OUTPUT_WIDTH / 2,
"fy": SANA_WM_OUTPUT_WIDTH / 2,
"cx": SANA_WM_OUTPUT_WIDTH / 2,
"cy": SANA_WM_OUTPUT_HEIGHT / 2,
},
},
},
OmniDiffusionSamplingParams(
height=SANA_WM_OUTPUT_HEIGHT,
width=SANA_WM_OUTPUT_WIDTH,
num_frames=num_frames,
seed=0,
fps=16,
num_inference_steps=stage1_steps,
guidance_scale=1.0,
guidance_scale_provided=True,
output_type=output_type,
extra_args=extra_args,
),
)
request_output = output[0] if isinstance(output, list) else output
assert isinstance(request_output, OmniRequestOutput)
assert request_output.error is None
assert request_output.images
frames = request_output.images[0]
video = _coerce_video_array(frames)
_assert_sana_wm_e2e_shape(video, output_type=output_type, num_frames=num_frames)
return video
def test_sana_wm_native_generates_video() -> None:
output_type = os.environ.get("SANA_WM_E2E_OUTPUT_TYPE", "np")
video = _run_sana_wm_e2e(output_type=output_type)
if output_type == "latent":
assert video.ndim == 4
else:
assert video.shape[-1] in (3, 4)
+106
View File
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving smoke for SANA-WM image-to-video (first-frame I2V via ``/v1/videos``).
Boots the server straight from the model id — the Stage-1 repo's
``model_index.json`` names ``SanaWmPipeline``, so the class resolves on its own
— submits one async ``/v1/videos`` job with a first-frame reference image, and
asserts the job completes and returns video bytes.
From ``tests/``::
pytest -s -v e2e/online_serving/test_sana_wm.py -m "advanced_model and diffusion" --run-level=advanced_model
"""
import base64
import json
import os
from io import BytesIO
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from vllm_omni.diffusion.models.sana_wm import (
SANA_WM_MODEL_ID,
SANA_WM_OUTPUT_HEIGHT,
SANA_WM_OUTPUT_WIDTH,
)
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL = SANA_WM_MODEL_ID
PROMPT = "A slow forward camera move through a quiet city street."
NEGATIVE_PROMPT = "blurry, low quality, distorted, watermark"
# Smoke knobs: a handful of frames / steps so the job finishes quickly.
SMOKE_NUM_FRAMES = 9
SMOKE_NUM_INFERENCE_STEPS = 2
# First-frame I2V needs a camera trajectory; "w-<n>" is the forward-move action DSL.
# Passed via the ``sana_wm`` form field (a JSON string parsed server-side); explicit
# intrinsics avoid the optional Pi3X camera-calibration dependency.
SANA_WM_PARAMS = {
"action": f"w-{SMOKE_NUM_FRAMES - 1}",
"translation_speed": 0.055,
"rotation_speed_deg": 1.2,
"intrinsics": {
"fx": SANA_WM_OUTPUT_WIDTH / 2,
"fy": SANA_WM_OUTPUT_WIDTH / 2,
"cx": SANA_WM_OUTPUT_WIDTH / 2,
"cy": SANA_WM_OUTPUT_HEIGHT / 2,
},
}
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _first_frame_data_url() -> str:
"""A solid-color first frame encoded as a PNG ``data:`` URL for I2V."""
from PIL import Image
image = Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160))
buffer = BytesIO()
image.save(buffer, format="PNG")
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
return f"data:image/png;base64,{encoded}"
def _get_diffusion_feature_cases(model: str):
"""Single default server row."""
return [
pytest.param(
OmniServerParams(model=model),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
def test_image_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
"""Default SANA-WM I2V smoke: async ``/v1/videos`` job completes and returns video bytes."""
request_config = {
"model": omni_server.model,
"image_reference": _first_frame_data_url(),
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
# The camera payload rides inside extra_params; there is no
# top-level sana_wm form field on the video endpoints, so sending
# one is silently dropped and the request fails in the
# preprocessor for missing camera control.
"extra_params": json.dumps({"sana_wm": SANA_WM_PARAMS}),
"height": SANA_WM_OUTPUT_HEIGHT,
"width": SANA_WM_OUTPUT_WIDTH,
"num_frames": SMOKE_NUM_FRAMES,
"fps": 16,
"num_inference_steps": SMOKE_NUM_INFERENCE_STEPS,
"guidance_scale": 5.0,
"seed": 42,
},
}
openai_client.send_video_diffusion_request(request_config)