[Model] Add SANA-WM support(Stage-1 only) (#4061)
Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Signed-off-by: liuqihao <liuqihao970610@gmail.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
1 parent
14bd8f8772
commit
adf6d0c1ea
21 files changed
+5173
-1
No files matched your search
@@ -0,0 +1,159 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.advanced_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.gpu,
|
||||
]
|
||||
|
||||
|
||||
def _make_sana_wm_e2e_image() -> Any:
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.sana_wm import SANA_WM_OUTPUT_HEIGHT, SANA_WM_OUTPUT_WIDTH
|
||||
|
||||
return Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160))
|
||||
|
||||
|
||||
def _sana_wm_e2e_tensor_parallel_size() -> int:
|
||||
return int(
|
||||
os.environ.get(
|
||||
"SANA_WM_E2E_TENSOR_PARALLEL_SIZE",
|
||||
os.environ.get("SANA_WM_E2E_TP", "1"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _coerce_video_array(video: Any) -> np.ndarray:
|
||||
import torch
|
||||
|
||||
if isinstance(video, list):
|
||||
assert video, "SANA-WM e2e produced an empty video list."
|
||||
video = video[0]
|
||||
if isinstance(video, torch.Tensor):
|
||||
video = video.detach().cpu().float().numpy()
|
||||
video_array = np.asarray(video)
|
||||
if video_array.ndim == 5:
|
||||
assert video_array.shape[0] == 1
|
||||
video_array = video_array[0]
|
||||
assert video_array.ndim == 4
|
||||
return video_array
|
||||
|
||||
|
||||
def _assert_sana_wm_e2e_shape(
|
||||
video: np.ndarray,
|
||||
*,
|
||||
output_type: str,
|
||||
num_frames: int,
|
||||
) -> None:
|
||||
if output_type == "latent":
|
||||
# Latent output is channel-first: (C, T, H, W) after removing the
|
||||
# batch dimension. Stage-1 emits LTX-2 latents with 128 channels.
|
||||
assert video.shape[0] == 128
|
||||
assert 0 < video.shape[1] <= num_frames
|
||||
return
|
||||
assert 0 < video.shape[0] <= num_frames
|
||||
|
||||
|
||||
def _run_sana_wm_e2e(*, output_type: str) -> np.ndarray:
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.sana_wm import (
|
||||
SANA_WM_MODEL_ID,
|
||||
SANA_WM_OUTPUT_HEIGHT,
|
||||
SANA_WM_OUTPUT_WIDTH,
|
||||
)
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("Sana-WM e2e requires CUDA.")
|
||||
|
||||
model = os.environ.get("SANA_WM_E2E_MODEL", SANA_WM_MODEL_ID)
|
||||
model_class_name = os.environ.get("SANA_WM_E2E_MODEL_CLASS", "SanaWmPipeline")
|
||||
num_frames = int(os.environ.get("SANA_WM_E2E_NUM_FRAMES", "9"))
|
||||
stage1_steps = int(
|
||||
os.environ.get(
|
||||
"SANA_WM_E2E_STAGE1_STEPS",
|
||||
os.environ.get("SANA_WM_E2E_NUM_INFERENCE_STEPS", "1"),
|
||||
)
|
||||
)
|
||||
action = os.environ.get("SANA_WM_E2E_ACTION", f"w-{max(num_frames - 1, 1)}")
|
||||
tensor_parallel_size = _sana_wm_e2e_tensor_parallel_size()
|
||||
image = _make_sana_wm_e2e_image()
|
||||
omni = Omni(
|
||||
model=model,
|
||||
model_class_name=model_class_name,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
extra_args: dict[str, Any] = {}
|
||||
# The native Stage-1 path is latent-token capped; raise it for full-size
|
||||
# e2e runs (704x1280 exceeds the default 4096-token cap).
|
||||
native_max_tokens = os.environ.get("SANA_WM_E2E_NATIVE_MAX_TOKENS", "").strip()
|
||||
if native_max_tokens:
|
||||
extra_args["sana_wm_native_max_tokens"] = int(native_max_tokens)
|
||||
# Return the requested output type so the shape assertion matches the
|
||||
# pipeline output instead of the raw Stage-1 latent default.
|
||||
|
||||
output = omni.generate(
|
||||
{
|
||||
"prompt": "A slow forward camera move through a quiet city street.",
|
||||
"multi_modal_data": {"image": image},
|
||||
"sana_wm": {
|
||||
"action": action,
|
||||
"num_frames": num_frames,
|
||||
"translation_speed": 0.055,
|
||||
"rotation_speed_deg": 1.2,
|
||||
# Avoid the optional Pi3X dependency by passing deterministic
|
||||
# camera intrinsics directly.
|
||||
"intrinsics": {
|
||||
"fx": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"fy": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"cx": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"cy": SANA_WM_OUTPUT_HEIGHT / 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=SANA_WM_OUTPUT_HEIGHT,
|
||||
width=SANA_WM_OUTPUT_WIDTH,
|
||||
num_frames=num_frames,
|
||||
seed=0,
|
||||
fps=16,
|
||||
num_inference_steps=stage1_steps,
|
||||
guidance_scale=1.0,
|
||||
guidance_scale_provided=True,
|
||||
output_type=output_type,
|
||||
extra_args=extra_args,
|
||||
),
|
||||
)
|
||||
|
||||
request_output = output[0] if isinstance(output, list) else output
|
||||
assert isinstance(request_output, OmniRequestOutput)
|
||||
assert request_output.error is None
|
||||
assert request_output.images
|
||||
|
||||
frames = request_output.images[0]
|
||||
video = _coerce_video_array(frames)
|
||||
_assert_sana_wm_e2e_shape(video, output_type=output_type, num_frames=num_frames)
|
||||
return video
|
||||
|
||||
|
||||
def test_sana_wm_native_generates_video() -> None:
|
||||
output_type = os.environ.get("SANA_WM_E2E_OUTPUT_TYPE", "np")
|
||||
video = _run_sana_wm_e2e(output_type=output_type)
|
||||
if output_type == "latent":
|
||||
assert video.ndim == 4
|
||||
else:
|
||||
assert video.shape[-1] in (3, 4)
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Online serving smoke for SANA-WM image-to-video (first-frame I2V via ``/v1/videos``).
|
||||
|
||||
Boots the server straight from the model id — the Stage-1 repo's
|
||||
``model_index.json`` names ``SanaWmPipeline``, so the class resolves on its own
|
||||
— submits one async ``/v1/videos`` job with a first-frame reference image, and
|
||||
asserts the job completes and returns video bytes.
|
||||
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_sana_wm.py -m "advanced_model and diffusion" --run-level=advanced_model
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
from vllm_omni.diffusion.models.sana_wm import (
|
||||
SANA_WM_MODEL_ID,
|
||||
SANA_WM_OUTPUT_HEIGHT,
|
||||
SANA_WM_OUTPUT_WIDTH,
|
||||
)
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = SANA_WM_MODEL_ID
|
||||
PROMPT = "A slow forward camera move through a quiet city street."
|
||||
NEGATIVE_PROMPT = "blurry, low quality, distorted, watermark"
|
||||
|
||||
# Smoke knobs: a handful of frames / steps so the job finishes quickly.
|
||||
SMOKE_NUM_FRAMES = 9
|
||||
SMOKE_NUM_INFERENCE_STEPS = 2
|
||||
# First-frame I2V needs a camera trajectory; "w-<n>" is the forward-move action DSL.
|
||||
# Passed via the ``sana_wm`` form field (a JSON string parsed server-side); explicit
|
||||
# intrinsics avoid the optional Pi3X camera-calibration dependency.
|
||||
SANA_WM_PARAMS = {
|
||||
"action": f"w-{SMOKE_NUM_FRAMES - 1}",
|
||||
"translation_speed": 0.055,
|
||||
"rotation_speed_deg": 1.2,
|
||||
"intrinsics": {
|
||||
"fx": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"fy": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"cx": SANA_WM_OUTPUT_WIDTH / 2,
|
||||
"cy": SANA_WM_OUTPUT_HEIGHT / 2,
|
||||
},
|
||||
}
|
||||
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _first_frame_data_url() -> str:
|
||||
"""A solid-color first frame encoded as a PNG ``data:`` URL for I2V."""
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (SANA_WM_OUTPUT_WIDTH, SANA_WM_OUTPUT_HEIGHT), (96, 128, 160))
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
return f"data:image/png;base64,{encoded}"
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Single default server row."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(model=model),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
|
||||
def test_image_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
"""Default SANA-WM I2V smoke: async ``/v1/videos`` job completes and returns video bytes."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"image_reference": _first_frame_data_url(),
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
# The camera payload rides inside extra_params; there is no
|
||||
# top-level sana_wm form field on the video endpoints, so sending
|
||||
# one is silently dropped and the request fails in the
|
||||
# preprocessor for missing camera control.
|
||||
"extra_params": json.dumps({"sana_wm": SANA_WM_PARAMS}),
|
||||
"height": SANA_WM_OUTPUT_HEIGHT,
|
||||
"width": SANA_WM_OUTPUT_WIDTH,
|
||||
"num_frames": SMOKE_NUM_FRAMES,
|
||||
"fps": 16,
|
||||
"num_inference_steps": SMOKE_NUM_INFERENCE_STEPS,
|
||||
"guidance_scale": 5.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
openai_client.send_video_diffusion_request(request_config)
|
||||
Reference in new issue
Block a user